b9f80fbdcfcdb5f34b95f9a8d5abc115a617347f
[akaros.git] / kern / drivers / net / bnx2x / bnx2x_cmn.c
1 /* bnx2x_cmn.c: Broadcom Everest network driver.
2  *
3  * Copyright (c) 2007-2013 Broadcom Corporation
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation.
8  *
9  * Maintained by: Ariel Elior <ariel.elior@qlogic.com>
10  * Written by: Eliezer Tamir
11  * Based on code from Michael Chan's bnx2 driver
12  * UDP CSUM errata workaround by Arik Gendelman
13  * Slowpath and fastpath rework by Vladislav Zolotarov
14  * Statistics and Link management by Yitchak Gertner
15  *
16  */
17
18 #include "akaros_compat.h"
19
20 #include "bnx2x_cmn.h"
21 #include "bnx2x_init.h"
22 #include "bnx2x_sp.h"
23
24 static void bnx2x_free_fp_mem_cnic(struct bnx2x *bp);
25 static int bnx2x_alloc_fp_mem_cnic(struct bnx2x *bp);
26 static int bnx2x_alloc_fp_mem(struct bnx2x *bp);
27 static int bnx2x_poll(struct napi_struct *napi, int budget);
28
29 static void bnx2x_add_all_napi_cnic(struct bnx2x *bp)
30 {
31         int i;
32
33         /* Add NAPI objects */
34         for_each_rx_queue_cnic(bp, i) {
35                 netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
36                                bnx2x_poll, NAPI_POLL_WEIGHT);
37                 napi_hash_add(&bnx2x_fp(bp, i, napi));
38         }
39 }
40
41 static void bnx2x_add_all_napi(struct bnx2x *bp)
42 {
43         int i;
44
45         /* Add NAPI objects */
46         for_each_eth_queue(bp, i) {
47                 netif_napi_add(bp->dev, &bnx2x_fp(bp, i, napi),
48                                bnx2x_poll, NAPI_POLL_WEIGHT);
49                 napi_hash_add(&bnx2x_fp(bp, i, napi));
50         }
51 }
52
53 static int bnx2x_calc_num_queues(struct bnx2x *bp)
54 {
55         int nq = bnx2x_num_queues ? : netif_get_num_default_rss_queues();
56
57         /* Reduce memory usage in kdump environment by using only one queue */
58         if (is_kdump_kernel())
59                 nq = 1;
60
61         nq = clamp(nq, 1, BNX2X_MAX_QUEUES(bp));
62         return nq;
63 }
64
65 /**
66  * bnx2x_move_fp - move content of the fastpath structure.
67  *
68  * @bp:         driver handle
69  * @from:       source FP index
70  * @to:         destination FP index
71  *
72  * Makes sure the contents of the bp->fp[to].napi is kept
73  * intact. This is done by first copying the napi struct from
74  * the target to the source, and then mem copying the entire
75  * source onto the target. Update txdata pointers and related
76  * content.
77  */
78 static inline void bnx2x_move_fp(struct bnx2x *bp, int from, int to)
79 {
80         struct bnx2x_fastpath *from_fp = &bp->fp[from];
81         struct bnx2x_fastpath *to_fp = &bp->fp[to];
82         struct bnx2x_sp_objs *from_sp_objs = &bp->sp_objs[from];
83         struct bnx2x_sp_objs *to_sp_objs = &bp->sp_objs[to];
84         struct bnx2x_fp_stats *from_fp_stats = &bp->fp_stats[from];
85         struct bnx2x_fp_stats *to_fp_stats = &bp->fp_stats[to];
86         int old_max_eth_txqs, new_max_eth_txqs;
87         int old_txdata_index = 0, new_txdata_index = 0;
88         struct bnx2x_agg_info *old_tpa_info = to_fp->tpa_info;
89
90         /* Copy the NAPI object as it has been already initialized */
91         from_fp->napi = to_fp->napi;
92
93         /* Move bnx2x_fastpath contents */
94         memcpy(to_fp, from_fp, sizeof(*to_fp));
95         to_fp->index = to;
96
97         /* Retain the tpa_info of the original `to' version as we don't want
98          * 2 FPs to contain the same tpa_info pointer.
99          */
100         to_fp->tpa_info = old_tpa_info;
101
102         /* move sp_objs contents as well, as their indices match fp ones */
103         memcpy(to_sp_objs, from_sp_objs, sizeof(*to_sp_objs));
104
105         /* move fp_stats contents as well, as their indices match fp ones */
106         memcpy(to_fp_stats, from_fp_stats, sizeof(*to_fp_stats));
107
108         /* Update txdata pointers in fp and move txdata content accordingly:
109          * Each fp consumes 'max_cos' txdata structures, so the index should be
110          * decremented by max_cos x delta.
111          */
112
113         old_max_eth_txqs = BNX2X_NUM_ETH_QUEUES(bp) * (bp)->max_cos;
114         new_max_eth_txqs = (BNX2X_NUM_ETH_QUEUES(bp) - from + to) *
115                                 (bp)->max_cos;
116         if (from == FCOE_IDX(bp)) {
117                 old_txdata_index = old_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
118                 new_txdata_index = new_max_eth_txqs + FCOE_TXQ_IDX_OFFSET;
119         }
120
121         memcpy(&bp->bnx2x_txq[new_txdata_index],
122                &bp->bnx2x_txq[old_txdata_index],
123                sizeof(struct bnx2x_fp_txdata));
124         to_fp->txdata_ptr[0] = &bp->bnx2x_txq[new_txdata_index];
125 }
126
127 /**
128  * bnx2x_fill_fw_str - Fill buffer with FW version string.
129  *
130  * @bp:        driver handle
131  * @buf:       character buffer to fill with the fw name
132  * @buf_len:   length of the above buffer
133  *
134  */
135 void bnx2x_fill_fw_str(struct bnx2x *bp, char *buf, size_t buf_len)
136 {
137         if (IS_PF(bp)) {
138                 uint8_t phy_fw_ver[PHY_FW_VER_LEN];
139
140                 phy_fw_ver[0] = '\0';
141                 bnx2x_get_ext_phy_fw_version(&bp->link_params,
142                                              phy_fw_ver, PHY_FW_VER_LEN);
143                 strlcpy(buf, bp->fw_ver, buf_len);
144                 snprintf(buf + strlen(bp->fw_ver), 32 - strlen(bp->fw_ver),
145                          "bc %d.%d.%d%s%s",
146                          (bp->common.bc_ver & 0xff0000) >> 16,
147                          (bp->common.bc_ver & 0xff00) >> 8,
148                          (bp->common.bc_ver & 0xff),
149                          ((phy_fw_ver[0] != '\0') ? " phy " : ""), phy_fw_ver);
150         } else {
151                 bnx2x_vf_fill_fw_str(bp, buf, buf_len);
152         }
153 }
154
155 /**
156  * bnx2x_shrink_eth_fp - guarantees fastpath structures stay intact
157  *
158  * @bp: driver handle
159  * @delta:      number of eth queues which were not allocated
160  */
161 static void bnx2x_shrink_eth_fp(struct bnx2x *bp, int delta)
162 {
163         int i, cos, old_eth_num = BNX2X_NUM_ETH_QUEUES(bp);
164
165         /* Queue pointer cannot be re-set on an fp-basis, as moving pointer
166          * backward along the array could cause memory to be overridden
167          */
168         for (cos = 1; cos < bp->max_cos; cos++) {
169                 for (i = 0; i < old_eth_num - delta; i++) {
170                         struct bnx2x_fastpath *fp = &bp->fp[i];
171                         int new_idx = cos * (old_eth_num - delta) + i;
172
173                         memcpy(&bp->bnx2x_txq[new_idx], fp->txdata_ptr[cos],
174                                sizeof(struct bnx2x_fp_txdata));
175                         fp->txdata_ptr[cos] = &bp->bnx2x_txq[new_idx];
176                 }
177         }
178 }
179
180 int bnx2x_load_count[2][3] = { {0} }; /* per-path: 0-common, 1-port0, 2-port1 */
181
182 /* free skb in the packet ring at pos idx
183  * return idx of last bd freed
184  */
185 static uint16_t bnx2x_free_tx_pkt(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata,
186                              uint16_t idx, unsigned int *pkts_compl,
187                              unsigned int *bytes_compl)
188 {
189         struct sw_tx_bd *tx_buf = &txdata->tx_buf_ring[idx];
190         struct eth_tx_start_bd *tx_start_bd;
191         struct eth_tx_bd *tx_data_bd;
192         struct sk_buff *skb = tx_buf->skb;
193         uint16_t bd_idx = TX_BD(tx_buf->first_bd), new_cons;
194         int nbd;
195         uint16_t split_bd_len = 0;
196
197         /* prefetch skb end pointer to speedup dev_kfree_skb() */
198         prefetch(&skb->end);
199
200         DP(NETIF_MSG_TX_DONE, "fp[%d]: pkt_idx %d  buff @(%p)->skb %p\n",
201            txdata->txq_index, idx, tx_buf, skb);
202
203         tx_start_bd = &txdata->tx_desc_ring[bd_idx].start_bd;
204
205         nbd = le16_to_cpu(tx_start_bd->nbd) - 1;
206 #ifdef BNX2X_STOP_ON_ERROR
207         if ((nbd - 1) > (MAX_SKB_FRAGS + 2)) {
208                 BNX2X_ERR("BAD nbd!\n");
209                 bnx2x_panic();
210         }
211 #endif
212         new_cons = nbd + tx_buf->first_bd;
213
214         /* Get the next bd */
215         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
216
217         /* Skip a parse bd... */
218         --nbd;
219         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
220
221         if (tx_buf->flags & BNX2X_HAS_SECOND_PBD) {
222                 /* Skip second parse bd... */
223                 --nbd;
224                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
225         }
226
227         /* TSO headers+data bds share a common mapping. See bnx2x_tx_split() */
228         if (tx_buf->flags & BNX2X_TSO_SPLIT_BD) {
229                 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
230                 split_bd_len = BD_UNMAP_LEN(tx_data_bd);
231                 --nbd;
232                 bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
233         }
234
235         /* unmap first bd */
236         dma_unmap_single(&bp->pdev->dev, BD_UNMAP_ADDR(tx_start_bd),
237                          BD_UNMAP_LEN(tx_start_bd) + split_bd_len,
238                          DMA_TO_DEVICE);
239
240         /* now free frags */
241         while (nbd > 0) {
242
243                 tx_data_bd = &txdata->tx_desc_ring[bd_idx].reg_bd;
244                 dma_unmap_page(&bp->pdev->dev, BD_UNMAP_ADDR(tx_data_bd),
245                                BD_UNMAP_LEN(tx_data_bd), DMA_TO_DEVICE);
246                 if (--nbd)
247                         bd_idx = TX_BD(NEXT_TX_IDX(bd_idx));
248         }
249
250         /* release skb */
251         WARN_ON(!skb);
252         if (likely(skb)) {
253                 (*pkts_compl)++;
254                 (*bytes_compl) += skb->len;
255         }
256
257         dev_kfree_skb_any(skb);
258         tx_buf->first_bd = 0;
259         tx_buf->skb = NULL;
260
261         return new_cons;
262 }
263
264 int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata)
265 {
266         struct netdev_queue *txq;
267         uint16_t hw_cons, sw_cons, bd_cons = txdata->tx_bd_cons;
268         unsigned int pkts_compl = 0, bytes_compl = 0;
269
270 #ifdef BNX2X_STOP_ON_ERROR
271         if (unlikely(bp->panic))
272                 return -1;
273 #endif
274
275         txq = netdev_get_tx_queue(bp->dev, txdata->txq_index);
276         hw_cons = le16_to_cpu(*txdata->tx_cons_sb);
277         sw_cons = txdata->tx_pkt_cons;
278
279         while (sw_cons != hw_cons) {
280                 uint16_t pkt_cons;
281
282                 pkt_cons = TX_BD(sw_cons);
283
284                 DP(NETIF_MSG_TX_DONE,
285                    "queue[%d]: hw_cons %u  sw_cons %u  pkt_cons %u\n",
286                    txdata->txq_index, hw_cons, sw_cons, pkt_cons);
287
288                 bd_cons = bnx2x_free_tx_pkt(bp, txdata, pkt_cons,
289                                             &pkts_compl, &bytes_compl);
290
291                 sw_cons++;
292         }
293
294         netdev_tx_completed_queue(txq, pkts_compl, bytes_compl);
295
296         txdata->tx_pkt_cons = sw_cons;
297         txdata->tx_bd_cons = bd_cons;
298
299         /* Need to make the tx_bd_cons update visible to start_xmit()
300          * before checking for netif_tx_queue_stopped().  Without the
301          * memory barrier, there is a small possibility that
302          * start_xmit() will miss it and cause the queue to be stopped
303          * forever.
304          * On the other hand we need an rmb() here to ensure the proper
305          * ordering of bit testing in the following
306          * netif_tx_queue_stopped(txq) call.
307          */
308         mb();
309
310         if (unlikely(netif_tx_queue_stopped(txq))) {
311                 /* Taking tx_lock() is needed to prevent re-enabling the queue
312                  * while it's empty. This could have happen if rx_action() gets
313                  * suspended in bnx2x_tx_int() after the condition before
314                  * netif_tx_wake_queue(), while tx_action (bnx2x_start_xmit()):
315                  *
316                  * stops the queue->sees fresh tx_bd_cons->releases the queue->
317                  * sends some packets consuming the whole queue again->
318                  * stops the queue
319                  */
320
321                 __netif_tx_lock(txq, smp_processor_id());
322
323                 if ((netif_tx_queue_stopped(txq)) &&
324                     (bp->state == BNX2X_STATE_OPEN) &&
325                     (bnx2x_tx_avail(bp, txdata) >= MAX_DESC_PER_TX_PKT))
326                         netif_tx_wake_queue(txq);
327
328                 __netif_tx_unlock(txq);
329         }
330         return 0;
331 }
332
333 static inline void bnx2x_update_last_max_sge(struct bnx2x_fastpath *fp,
334                                              uint16_t idx)
335 {
336         uint16_t last_max = fp->last_max_sge;
337
338         if (SUB_S16(idx, last_max) > 0)
339                 fp->last_max_sge = idx;
340 }
341
342 static inline void bnx2x_update_sge_prod(struct bnx2x_fastpath *fp,
343                                          uint16_t sge_len,
344                                          struct eth_end_agg_rx_cqe *cqe)
345 {
346         struct bnx2x *bp = fp->bp;
347         uint16_t last_max, last_elem, first_elem;
348         uint16_t delta = 0;
349         uint16_t i;
350
351         if (!sge_len)
352                 return;
353
354         /* First mark all used pages */
355         for (i = 0; i < sge_len; i++)
356                 BIT_VEC64_CLEAR_BIT(fp->sge_mask,
357                         RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[i])));
358
359         DP(NETIF_MSG_RX_STATUS, "fp_cqe->sgl[%d] = %d\n",
360            sge_len - 1, le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
361
362         /* Here we assume that the last SGE index is the biggest */
363         prefetch((void *)(fp->sge_mask));
364         bnx2x_update_last_max_sge(fp,
365                 le16_to_cpu(cqe->sgl_or_raw_data.sgl[sge_len - 1]));
366
367         last_max = RX_SGE(fp->last_max_sge);
368         last_elem = last_max >> BIT_VEC64_ELEM_SHIFT;
369         first_elem = RX_SGE(fp->rx_sge_prod) >> BIT_VEC64_ELEM_SHIFT;
370
371         /* If ring is not full */
372         if (last_elem + 1 != first_elem)
373                 last_elem++;
374
375         /* Now update the prod */
376         for (i = first_elem; i != last_elem; i = NEXT_SGE_MASK_ELEM(i)) {
377                 if (likely(fp->sge_mask[i]))
378                         break;
379
380                 fp->sge_mask[i] = BIT_VEC64_ELEM_ONE_MASK;
381                 delta += BIT_VEC64_ELEM_SZ;
382         }
383
384         if (delta > 0) {
385                 fp->rx_sge_prod += delta;
386                 /* clear page-end entries */
387                 bnx2x_clear_sge_mask_next_elems(fp);
388         }
389
390         DP(NETIF_MSG_RX_STATUS,
391            "fp->last_max_sge = %d  fp->rx_sge_prod = %d\n",
392            fp->last_max_sge, fp->rx_sge_prod);
393 }
394
395 /* Get Toeplitz hash value in the skb using the value from the
396  * CQE (calculated by HW).
397  */
398 static uint32_t bnx2x_get_rxhash(const struct bnx2x *bp,
399                             const struct eth_fast_path_rx_cqe *cqe,
400                             enum pkt_hash_types *rxhash_type)
401 {
402         /* Get Toeplitz hash from CQE */
403         if ((bp->dev->features & NETIF_F_RXHASH) &&
404             (cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_FLG)) {
405                 enum eth_rss_hash_type htype;
406
407                 htype = cqe->status_flags & ETH_FAST_PATH_RX_CQE_RSS_HASH_TYPE;
408                 *rxhash_type = ((htype == TCP_IPV4_HASH_TYPE) ||
409                                 (htype == TCP_IPV6_HASH_TYPE)) ?
410                                PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3;
411
412                 return le32_to_cpu(cqe->rss_hash_result);
413         }
414         *rxhash_type = PKT_HASH_TYPE_NONE;
415         return 0;
416 }
417
418 static void bnx2x_tpa_start(struct bnx2x_fastpath *fp, uint16_t queue,
419                             uint16_t cons, uint16_t prod,
420                             struct eth_fast_path_rx_cqe *cqe)
421 {
422         struct bnx2x *bp = fp->bp;
423         struct sw_rx_bd *cons_rx_buf = &fp->rx_buf_ring[cons];
424         struct sw_rx_bd *prod_rx_buf = &fp->rx_buf_ring[prod];
425         struct eth_rx_bd *prod_bd = &fp->rx_desc_ring[prod];
426         dma_addr_t mapping;
427         struct bnx2x_agg_info *tpa_info = &fp->tpa_info[queue];
428         struct sw_rx_bd *first_buf = &tpa_info->first_buf;
429
430         /* print error if current state != stop */
431         if (tpa_info->tpa_state != BNX2X_TPA_STOP)
432                 BNX2X_ERR("start of bin not in stop [%d]\n", queue);
433
434         /* Try to map an empty data buffer from the aggregation info  */
435         mapping = dma_map_single(&bp->pdev->dev,
436                                  first_buf->data + NET_SKB_PAD,
437                                  fp->rx_buf_size, DMA_FROM_DEVICE);
438         /*
439          *  ...if it fails - move the skb from the consumer to the producer
440          *  and set the current aggregation state as ERROR to drop it
441          *  when TPA_STOP arrives.
442          */
443
444         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
445                 /* Move the BD from the consumer to the producer */
446                 bnx2x_reuse_rx_data(fp, cons, prod);
447                 tpa_info->tpa_state = BNX2X_TPA_ERROR;
448                 return;
449         }
450
451         /* move empty data from pool to prod */
452         prod_rx_buf->data = first_buf->data;
453         dma_unmap_addr_set(prod_rx_buf, mapping, mapping);
454         /* point prod_bd to new data */
455         prod_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
456         prod_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
457
458         /* move partial skb from cons to pool (don't unmap yet) */
459         *first_buf = *cons_rx_buf;
460
461         /* mark bin state as START */
462         tpa_info->parsing_flags =
463                 le16_to_cpu(cqe->pars_flags.flags);
464         tpa_info->vlan_tag = le16_to_cpu(cqe->vlan_tag);
465         tpa_info->tpa_state = BNX2X_TPA_START;
466         tpa_info->len_on_bd = le16_to_cpu(cqe->len_on_bd);
467         tpa_info->placement_offset = cqe->placement_offset;
468         tpa_info->rxhash = bnx2x_get_rxhash(bp, cqe, &tpa_info->rxhash_type);
469         if (fp->mode == TPA_MODE_GRO) {
470                 uint16_t gro_size = le16_to_cpu(cqe->pkt_len_or_gro_seg_len);
471                 tpa_info->full_page = SGE_PAGES / gro_size * gro_size;
472                 tpa_info->gro_size = gro_size;
473         }
474
475 #ifdef BNX2X_STOP_ON_ERROR
476         fp->tpa_queue_used |= (1 << queue);
477         DP(NETIF_MSG_RX_STATUS, "fp->tpa_queue_used = 0x%llx\n",
478            fp->tpa_queue_used);
479 #endif
480 }
481
482 /* Timestamp option length allowed for TPA aggregation:
483  *
484  *              nop nop kind length echo val
485  */
486 #define TPA_TSTAMP_OPT_LEN      12
487 /**
488  * bnx2x_set_gro_params - compute GRO values
489  *
490  * @skb:                packet skb
491  * @parsing_flags:      parsing flags from the START CQE
492  * @len_on_bd:          total length of the first packet for the
493  *                      aggregation.
494  * @pkt_len:            length of all segments
495  *
496  * Approximate value of the MSS for this aggregation calculated using
497  * the first packet of it.
498  * Compute number of aggregated segments, and gso_type.
499  */
500 static void bnx2x_set_gro_params(struct sk_buff *skb, uint16_t parsing_flags,
501                                  uint16_t len_on_bd, unsigned int pkt_len,
502                                  uint16_t num_of_coalesced_segs)
503 {
504         /* TPA aggregation won't have either IP options or TCP options
505          * other than timestamp or IPv6 extension headers.
506          */
507         uint16_t hdrs_len = ETHERHDRSIZE + sizeof(struct tcphdr);
508
509         if (GET_FLAG(parsing_flags, PARSING_FLAGS_OVER_ETHERNET_PROTOCOL) ==
510             PRS_FLAG_OVERETH_IPV6) {
511                 hdrs_len += sizeof(struct ipv6hdr);
512                 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
513         } else {
514                 hdrs_len += sizeof(struct iphdr);
515                 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
516         }
517
518         /* Check if there was a TCP timestamp, if there is it's will
519          * always be 12 bytes length: nop nop kind length echo val.
520          *
521          * Otherwise FW would close the aggregation.
522          */
523         if (parsing_flags & PARSING_FLAGS_TIME_STAMP_EXIST_FLAG)
524                 hdrs_len += TPA_TSTAMP_OPT_LEN;
525
526         skb_shinfo(skb)->gso_size = len_on_bd - hdrs_len;
527
528         /* tcp_gro_complete() will copy NAPI_GRO_CB(skb)->count
529          * to skb_shinfo(skb)->gso_segs
530          */
531         NAPI_GRO_CB(skb)->count = num_of_coalesced_segs;
532 }
533
534 static int bnx2x_alloc_rx_sge(struct bnx2x *bp, struct bnx2x_fastpath *fp,
535                               uint16_t index, gfp_t gfp_mask)
536 {
537         struct page *page = alloc_pages(gfp_mask, PAGES_PER_SGE_SHIFT);
538         struct sw_rx_page *sw_buf = &fp->rx_page_ring[index];
539         struct eth_rx_sge *sge = &fp->rx_sge_ring[index];
540         dma_addr_t mapping;
541
542         if (unlikely(page == NULL)) {
543                 BNX2X_ERR("Can't alloc sge\n");
544                 return -ENOMEM;
545         }
546
547         mapping = dma_map_page(&bp->pdev->dev, page, 0,
548                                SGE_PAGES, DMA_FROM_DEVICE);
549         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
550                 __free_pages(page, PAGES_PER_SGE_SHIFT);
551                 BNX2X_ERR("Can't map sge\n");
552                 return -ENOMEM;
553         }
554
555         sw_buf->page = page;
556         dma_unmap_addr_set(sw_buf, mapping, mapping);
557
558         sge->addr_hi = cpu_to_le32(U64_HI(mapping));
559         sge->addr_lo = cpu_to_le32(U64_LO(mapping));
560
561         return 0;
562 }
563
564 static int bnx2x_fill_frag_skb(struct bnx2x *bp, struct bnx2x_fastpath *fp,
565                                struct bnx2x_agg_info *tpa_info,
566                                uint16_t pages,
567                                struct sk_buff *skb,
568                                struct eth_end_agg_rx_cqe *cqe,
569                                uint16_t cqe_idx)
570 {
571         struct sw_rx_page *rx_pg, old_rx_pg;
572         uint32_t i, frag_len, frag_size;
573         int err, j, frag_id = 0;
574         uint16_t len_on_bd = tpa_info->len_on_bd;
575         uint16_t full_page = 0, gro_size = 0;
576
577         frag_size = le16_to_cpu(cqe->pkt_len) - len_on_bd;
578
579         if (fp->mode == TPA_MODE_GRO) {
580                 gro_size = tpa_info->gro_size;
581                 full_page = tpa_info->full_page;
582         }
583
584         /* This is needed in order to enable forwarding support */
585         if (frag_size)
586                 bnx2x_set_gro_params(skb, tpa_info->parsing_flags, len_on_bd,
587                                      le16_to_cpu(cqe->pkt_len),
588                                      le16_to_cpu(cqe->num_of_coalesced_segs));
589
590 #ifdef BNX2X_STOP_ON_ERROR
591         if (pages > min_t(uint32_t, 8, MAX_SKB_FRAGS) * SGE_PAGES) {
592                 BNX2X_ERR("SGL length is too long: %d. CQE index is %d\n",
593                           pages, cqe_idx);
594                 BNX2X_ERR("cqe->pkt_len = %d\n", cqe->pkt_len);
595                 bnx2x_panic();
596                 return -EINVAL;
597         }
598 #endif
599
600         /* Run through the SGL and compose the fragmented skb */
601         for (i = 0, j = 0; i < pages; i += PAGES_PER_SGE, j++) {
602                 uint16_t sge_idx = RX_SGE(le16_to_cpu(cqe->sgl_or_raw_data.sgl[j]));
603
604                 /* FW gives the indices of the SGE as if the ring is an array
605                    (meaning that "next" element will consume 2 indices) */
606                 if (fp->mode == TPA_MODE_GRO)
607                         frag_len = min_t(uint32_t, frag_size,
608                                          (uint32_t)full_page);
609                 else /* LRO */
610                         frag_len = min_t(uint32_t, frag_size,
611                                          (uint32_t)SGE_PAGES);
612
613                 rx_pg = &fp->rx_page_ring[sge_idx];
614                 old_rx_pg = *rx_pg;
615
616                 /* If we fail to allocate a substitute page, we simply stop
617                    where we are and drop the whole packet */
618                 err = bnx2x_alloc_rx_sge(bp, fp, sge_idx, GFP_ATOMIC);
619                 if (unlikely(err)) {
620                         bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
621                         return err;
622                 }
623
624                 /* Unmap the page as we're going to pass it to the stack */
625                 dma_unmap_page(&bp->pdev->dev,
626                                dma_unmap_addr(&old_rx_pg, mapping),
627                                SGE_PAGES, DMA_FROM_DEVICE);
628                 /* Add one frag and update the appropriate fields in the skb */
629                 if (fp->mode == TPA_MODE_LRO)
630                         skb_fill_page_desc(skb, j, old_rx_pg.page, 0, frag_len);
631                 else { /* GRO */
632                         int rem;
633                         int offset = 0;
634                         for (rem = frag_len; rem > 0; rem -= gro_size) {
635                                 int len = rem > gro_size ? gro_size : rem;
636                                 skb_fill_page_desc(skb, frag_id++,
637                                                    old_rx_pg.page, offset, len);
638                                 if (offset)
639                                         get_page(old_rx_pg.page);
640                                 offset += len;
641                         }
642                 }
643
644                 skb->data_len += frag_len;
645                 skb->truesize += SGE_PAGES;
646                 skb->len += frag_len;
647
648                 frag_size -= frag_len;
649         }
650
651         return 0;
652 }
653
654 static void bnx2x_frag_free(const struct bnx2x_fastpath *fp, void *data)
655 {
656         if (fp->rx_frag_size)
657                 put_page(virt_to_head_page(data));
658         else
659                 kfree(data);
660 }
661
662 static void *bnx2x_frag_alloc(const struct bnx2x_fastpath *fp, gfp_t gfp_mask)
663 {
664         if (fp->rx_frag_size) {
665                 /* GFP_KERNEL allocations are used only during initialization */
666                 if (unlikely(gfp_mask & __GFP_WAIT))
667                         return (void *)__get_free_page(gfp_mask);
668
669                 return netdev_alloc_frag(fp->rx_frag_size);
670         }
671
672         return kmalloc(fp->rx_buf_size + NET_SKB_PAD, gfp_mask);
673 }
674
675 #ifdef CONFIG_INET
676 static void bnx2x_gro_ip_csum(struct bnx2x *bp, struct sk_buff *skb)
677 {
678         const struct iphdr *iph = ip_hdr(skb);
679         struct tcphdr *th;
680
681         skb_set_transport_header(skb, sizeof(struct iphdr));
682         th = tcp_hdr(skb);
683
684         th->check = ~tcp_v4_check(skb->len - skb_transport_offset(skb),
685                                   iph->saddr, iph->daddr, 0);
686 }
687
688 static void bnx2x_gro_ipv6_csum(struct bnx2x *bp, struct sk_buff *skb)
689 {
690         struct ipv6hdr *iph = ipv6_hdr(skb);
691         struct tcphdr *th;
692
693         skb_set_transport_header(skb, sizeof(struct ipv6hdr));
694         th = tcp_hdr(skb);
695
696         th->check = ~tcp_v6_check(skb->len - skb_transport_offset(skb),
697                                   &iph->saddr, &iph->daddr, 0);
698 }
699
700 static void bnx2x_gro_csum(struct bnx2x *bp, struct sk_buff *skb,
701                             void (*gro_func)(struct bnx2x*, struct sk_buff*))
702 {
703         skb_set_network_header(skb, 0);
704         gro_func(bp, skb);
705         tcp_gro_complete(skb);
706 }
707 #endif
708
709 static void bnx2x_gro_receive(struct bnx2x *bp, struct bnx2x_fastpath *fp,
710                                struct sk_buff *skb)
711 {
712 #ifdef CONFIG_INET
713         if (skb_shinfo(skb)->gso_size) {
714                 switch (be16_to_cpu(skb->protocol)) {
715                 case ETH_P_IP:
716                         bnx2x_gro_csum(bp, skb, bnx2x_gro_ip_csum);
717                         break;
718                 case ETH_P_IPV6:
719                         bnx2x_gro_csum(bp, skb, bnx2x_gro_ipv6_csum);
720                         break;
721                 default:
722                         BNX2X_ERR("Error: FW GRO supports only IPv4/IPv6, not 0x%04x\n",
723                                   be16_to_cpu(skb->protocol));
724                 }
725         }
726 #endif
727         skb_record_rx_queue(skb, fp->rx_queue);
728         napi_gro_receive(&fp->napi, skb);
729 }
730
731 static void bnx2x_tpa_stop(struct bnx2x *bp, struct bnx2x_fastpath *fp,
732                            struct bnx2x_agg_info *tpa_info,
733                            uint16_t pages,
734                            struct eth_end_agg_rx_cqe *cqe,
735                            uint16_t cqe_idx)
736 {
737         struct sw_rx_bd *rx_buf = &tpa_info->first_buf;
738         uint8_t pad = tpa_info->placement_offset;
739         uint16_t len = tpa_info->len_on_bd;
740         struct sk_buff *skb = NULL;
741         uint8_t *new_data, *data = rx_buf->data;
742         uint8_t old_tpa_state = tpa_info->tpa_state;
743
744         tpa_info->tpa_state = BNX2X_TPA_STOP;
745
746         /* If we there was an error during the handling of the TPA_START -
747          * drop this aggregation.
748          */
749         if (old_tpa_state == BNX2X_TPA_ERROR)
750                 goto drop;
751
752         /* Try to allocate the new data */
753         new_data = bnx2x_frag_alloc(fp, GFP_ATOMIC);
754         /* Unmap skb in the pool anyway, as we are going to change
755            pool entry status to BNX2X_TPA_STOP even if new skb allocation
756            fails. */
757         dma_unmap_single(&bp->pdev->dev, dma_unmap_addr(rx_buf, mapping),
758                          fp->rx_buf_size, DMA_FROM_DEVICE);
759         if (likely(new_data))
760                 skb = build_skb(data, fp->rx_frag_size);
761
762         if (likely(skb)) {
763 #ifdef BNX2X_STOP_ON_ERROR
764                 if (pad + len > fp->rx_buf_size) {
765                         BNX2X_ERR("skb_put is about to fail...  pad %d  len %d  rx_buf_size %d\n",
766                                   pad, len, fp->rx_buf_size);
767                         bnx2x_panic();
768                         return;
769                 }
770 #endif
771
772                 skb_reserve(skb, pad + NET_SKB_PAD);
773                 skb_put(skb, len);
774                 skb_set_hash(skb, tpa_info->rxhash, tpa_info->rxhash_type);
775
776                 skb->protocol = eth_type_trans(skb, bp->dev);
777                 skb->ip_summed = CHECKSUM_UNNECESSARY;
778
779                 if (!bnx2x_fill_frag_skb(bp, fp, tpa_info, pages,
780                                          skb, cqe, cqe_idx)) {
781                         if (tpa_info->parsing_flags & PARSING_FLAGS_VLAN)
782                                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tpa_info->vlan_tag);
783                         bnx2x_gro_receive(bp, fp, skb);
784                 } else {
785                         DP(NETIF_MSG_RX_STATUS,
786                            "Failed to allocate new pages - dropping packet!\n");
787                         dev_kfree_skb_any(skb);
788                 }
789
790                 /* put new data in bin */
791                 rx_buf->data = new_data;
792
793                 return;
794         }
795         if (new_data)
796                 bnx2x_frag_free(fp, new_data);
797 drop:
798         /* drop the packet and keep the buffer in the bin */
799         DP(NETIF_MSG_RX_STATUS,
800            "Failed to allocate or map a new skb - dropping packet!\n");
801         bnx2x_fp_stats(bp, fp)->eth_q_stats.rx_skb_alloc_failed++;
802 }
803
804 static int bnx2x_alloc_rx_data(struct bnx2x *bp, struct bnx2x_fastpath *fp,
805                                uint16_t index, gfp_t gfp_mask)
806 {
807         uint8_t *data;
808         struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[index];
809         struct eth_rx_bd *rx_bd = &fp->rx_desc_ring[index];
810         dma_addr_t mapping;
811
812         data = bnx2x_frag_alloc(fp, gfp_mask);
813         if (unlikely(data == NULL))
814                 return -ENOMEM;
815
816         mapping = dma_map_single(&bp->pdev->dev, data + NET_SKB_PAD,
817                                  fp->rx_buf_size,
818                                  DMA_FROM_DEVICE);
819         if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) {
820                 bnx2x_frag_free(fp, data);
821                 BNX2X_ERR("Can't map rx data\n");
822                 return -ENOMEM;
823         }
824
825         rx_buf->data = data;
826         dma_unmap_addr_set(rx_buf, mapping, mapping);
827
828         rx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
829         rx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
830
831         return 0;
832 }
833
834 static
835 void bnx2x_csum_validate(struct sk_buff *skb, union eth_rx_cqe *cqe,
836                                  struct bnx2x_fastpath *fp,
837                                  struct bnx2x_eth_q_stats *qstats)
838 {
839         /* Do nothing if no L4 csum validation was done.
840          * We do not check whether IP csum was validated. For IPv4 we assume
841          * that if the card got as far as validating the L4 csum, it also
842          * validated the IP csum. IPv6 has no IP csum.
843          */
844         if (cqe->fast_path_cqe.status_flags &
845             ETH_FAST_PATH_RX_CQE_L4_XSUM_NO_VALIDATION_FLG)
846                 return;
847
848         /* If L4 validation was done, check if an error was found. */
849
850         if (cqe->fast_path_cqe.type_error_flags &
851             (ETH_FAST_PATH_RX_CQE_IP_BAD_XSUM_FLG |
852              ETH_FAST_PATH_RX_CQE_L4_BAD_XSUM_FLG))
853                 qstats->hw_csum_err++;
854         else
855                 skb->ip_summed = CHECKSUM_UNNECESSARY;
856 }
857
858 static int bnx2x_rx_int(struct bnx2x_fastpath *fp, int budget)
859 {
860         struct bnx2x *bp = fp->bp;
861         uint16_t bd_cons, bd_prod, bd_prod_fw, comp_ring_cons;
862         uint16_t sw_comp_cons, sw_comp_prod;
863         int rx_pkt = 0;
864         union eth_rx_cqe *cqe;
865         struct eth_fast_path_rx_cqe *cqe_fp;
866
867 #ifdef BNX2X_STOP_ON_ERROR
868         if (unlikely(bp->panic))
869                 return 0;
870 #endif
871         if (budget <= 0)
872                 return rx_pkt;
873
874         bd_cons = fp->rx_bd_cons;
875         bd_prod = fp->rx_bd_prod;
876         bd_prod_fw = bd_prod;
877         sw_comp_cons = fp->rx_comp_cons;
878         sw_comp_prod = fp->rx_comp_prod;
879
880         comp_ring_cons = RCQ_BD(sw_comp_cons);
881         cqe = &fp->rx_comp_ring[comp_ring_cons];
882         cqe_fp = &cqe->fast_path_cqe;
883
884         DP(NETIF_MSG_RX_STATUS,
885            "queue[%d]: sw_comp_cons %u\n", fp->index, sw_comp_cons);
886
887         while (BNX2X_IS_CQE_COMPLETED(cqe_fp)) {
888                 struct sw_rx_bd *rx_buf = NULL;
889                 struct sk_buff *skb;
890                 uint8_t cqe_fp_flags;
891                 enum eth_rx_cqe_type cqe_fp_type;
892                 uint16_t len, pad, queue;
893                 uint8_t *data;
894                 uint32_t rxhash;
895                 enum pkt_hash_types rxhash_type;
896
897 #ifdef BNX2X_STOP_ON_ERROR
898                 if (unlikely(bp->panic))
899                         return 0;
900 #endif
901
902                 bd_prod = RX_BD(bd_prod);
903                 bd_cons = RX_BD(bd_cons);
904
905                 /* A rmb() is required to ensure that the CQE is not read
906                  * before it is written by the adapter DMA.  PCI ordering
907                  * rules will make sure the other fields are written before
908                  * the marker at the end of struct eth_fast_path_rx_cqe
909                  * but without rmb() a weakly ordered processor can process
910                  * stale data.  Without the barrier TPA state-machine might
911                  * enter inconsistent state and kernel stack might be
912                  * provided with incorrect packet description - these lead
913                  * to various kernel crashed.
914                  */
915                 rmb();
916
917                 cqe_fp_flags = cqe_fp->type_error_flags;
918                 cqe_fp_type = cqe_fp_flags & ETH_FAST_PATH_RX_CQE_TYPE;
919
920                 DP(NETIF_MSG_RX_STATUS,
921                    "CQE type %x  err %x  status %x  queue %x  vlan %x  len %u\n",
922                    CQE_TYPE(cqe_fp_flags),
923                    cqe_fp_flags, cqe_fp->status_flags,
924                    le32_to_cpu(cqe_fp->rss_hash_result),
925                    le16_to_cpu(cqe_fp->vlan_tag),
926                    le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len));
927
928                 /* is this a slowpath msg? */
929                 if (unlikely(CQE_TYPE_SLOW(cqe_fp_type))) {
930                         bnx2x_sp_event(fp, cqe);
931                         goto next_cqe;
932                 }
933
934                 rx_buf = &fp->rx_buf_ring[bd_cons];
935                 data = rx_buf->data;
936
937                 if (!CQE_TYPE_FAST(cqe_fp_type)) {
938                         struct bnx2x_agg_info *tpa_info;
939                         uint16_t frag_size, pages;
940 #ifdef BNX2X_STOP_ON_ERROR
941                         /* sanity check */
942                         if (fp->disable_tpa &&
943                             (CQE_TYPE_START(cqe_fp_type) ||
944                              CQE_TYPE_STOP(cqe_fp_type)))
945                                 BNX2X_ERR("START/STOP packet while disable_tpa type %x\n",
946                                           CQE_TYPE(cqe_fp_type));
947 #endif
948
949                         if (CQE_TYPE_START(cqe_fp_type)) {
950                                 uint16_t queue = cqe_fp->queue_index;
951                                 DP(NETIF_MSG_RX_STATUS,
952                                    "calling tpa_start on queue %d\n",
953                                    queue);
954
955                                 bnx2x_tpa_start(fp, queue,
956                                                 bd_cons, bd_prod,
957                                                 cqe_fp);
958
959                                 goto next_rx;
960                         }
961                         queue = cqe->end_agg_cqe.queue_index;
962                         tpa_info = &fp->tpa_info[queue];
963                         DP(NETIF_MSG_RX_STATUS,
964                            "calling tpa_stop on queue %d\n",
965                            queue);
966
967                         frag_size = le16_to_cpu(cqe->end_agg_cqe.pkt_len) -
968                                     tpa_info->len_on_bd;
969
970                         if (fp->mode == TPA_MODE_GRO)
971                                 pages = (frag_size + tpa_info->full_page - 1) /
972                                          tpa_info->full_page;
973                         else
974                                 pages = SGE_PAGE_ALIGN(frag_size) >>
975                                         SGE_PAGE_SHIFT;
976
977                         bnx2x_tpa_stop(bp, fp, tpa_info, pages,
978                                        &cqe->end_agg_cqe, comp_ring_cons);
979 #ifdef BNX2X_STOP_ON_ERROR
980                         if (bp->panic)
981                                 return 0;
982 #endif
983
984                         bnx2x_update_sge_prod(fp, pages, &cqe->end_agg_cqe);
985                         goto next_cqe;
986                 }
987                 /* non TPA */
988                 len = le16_to_cpu(cqe_fp->pkt_len_or_gro_seg_len);
989                 pad = cqe_fp->placement_offset;
990                 dma_sync_single_for_cpu(&bp->pdev->dev,
991                                         dma_unmap_addr(rx_buf, mapping),
992                                         pad + RX_COPY_THRESH,
993                                         DMA_FROM_DEVICE);
994                 pad += NET_SKB_PAD;
995                 prefetch(data + pad); /* speedup eth_type_trans() */
996                 /* is this an error packet? */
997                 if (unlikely(cqe_fp_flags & ETH_RX_ERROR_FALGS)) {
998                         DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
999                            "ERROR  flags %x  rx packet %u\n",
1000                            cqe_fp_flags, sw_comp_cons);
1001                         bnx2x_fp_qstats(bp, fp)->rx_err_discard_pkt++;
1002                         goto reuse_rx;
1003                 }
1004
1005                 /* Since we don't have a jumbo ring
1006                  * copy small packets if mtu > 1500
1007                  */
1008                 if ((bp->dev->mtu > ETH_MAX_PACKET_SIZE) &&
1009                     (len <= RX_COPY_THRESH)) {
1010                         skb = napi_alloc_skb(&fp->napi, len);
1011                         if (skb == NULL) {
1012                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1013                                    "ERROR  packet dropped because of alloc failure\n");
1014                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
1015                                 goto reuse_rx;
1016                         }
1017                         memcpy(skb->data, data + pad, len);
1018                         bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
1019                 } else {
1020                         if (likely(bnx2x_alloc_rx_data(bp, fp, bd_prod,
1021                                                        GFP_ATOMIC) == 0)) {
1022                                 dma_unmap_single(&bp->pdev->dev,
1023                                                  dma_unmap_addr(rx_buf, mapping),
1024                                                  fp->rx_buf_size,
1025                                                  DMA_FROM_DEVICE);
1026                                 skb = build_skb(data, fp->rx_frag_size);
1027                                 if (unlikely(!skb)) {
1028                                         bnx2x_frag_free(fp, data);
1029                                         bnx2x_fp_qstats(bp, fp)->
1030                                                         rx_skb_alloc_failed++;
1031                                         goto next_rx;
1032                                 }
1033                                 skb_reserve(skb, pad);
1034                         } else {
1035                                 DP(NETIF_MSG_RX_ERR | NETIF_MSG_RX_STATUS,
1036                                    "ERROR  packet dropped because of alloc failure\n");
1037                                 bnx2x_fp_qstats(bp, fp)->rx_skb_alloc_failed++;
1038 reuse_rx:
1039                                 bnx2x_reuse_rx_data(fp, bd_cons, bd_prod);
1040                                 goto next_rx;
1041                         }
1042                 }
1043
1044                 skb_put(skb, len);
1045                 skb->protocol = eth_type_trans(skb, bp->dev);
1046
1047                 /* Set Toeplitz hash for a none-LRO skb */
1048                 rxhash = bnx2x_get_rxhash(bp, cqe_fp, &rxhash_type);
1049                 skb_set_hash(skb, rxhash, rxhash_type);
1050
1051                 skb_checksum_none_assert(skb);
1052
1053                 if (bp->dev->features & NETIF_F_RXCSUM)
1054                         bnx2x_csum_validate(skb, cqe, fp,
1055                                             bnx2x_fp_qstats(bp, fp));
1056
1057                 skb_record_rx_queue(skb, fp->rx_queue);
1058
1059                 /* Check if this packet was timestamped */
1060                 if (unlikely(cqe->fast_path_cqe.type_error_flags &
1061                              (1 << ETH_FAST_PATH_RX_CQE_PTP_PKT_SHIFT)))
1062                         bnx2x_set_rx_ts(bp, skb);
1063
1064                 if (le16_to_cpu(cqe_fp->pars_flags.flags) &
1065                     PARSING_FLAGS_VLAN)
1066                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
1067                                                le16_to_cpu(cqe_fp->vlan_tag));
1068
1069                 skb_mark_napi_id(skb, &fp->napi);
1070
1071                 if (bnx2x_fp_ll_polling(fp))
1072                         netif_receive_skb(skb);
1073                 else
1074                         napi_gro_receive(&fp->napi, skb);
1075 next_rx:
1076                 rx_buf->data = NULL;
1077
1078                 bd_cons = NEXT_RX_IDX(bd_cons);
1079                 bd_prod = NEXT_RX_IDX(bd_prod);
1080                 bd_prod_fw = NEXT_RX_IDX(bd_prod_fw);
1081                 rx_pkt++;
1082 next_cqe:
1083                 sw_comp_prod = NEXT_RCQ_IDX(sw_comp_prod);
1084                 sw_comp_cons = NEXT_RCQ_IDX(sw_comp_cons);
1085
1086                 /* mark CQE as free */
1087                 BNX2X_SEED_CQE(cqe_fp);
1088
1089                 if (rx_pkt == budget)
1090                         break;
1091
1092                 comp_ring_cons = RCQ_BD(sw_comp_cons);
1093                 cqe = &fp->rx_comp_ring[comp_ring_cons];
1094                 cqe_fp = &cqe->fast_path_cqe;
1095         } /* while */
1096
1097         fp->rx_bd_cons = bd_cons;
1098         fp->rx_bd_prod = bd_prod_fw;
1099         fp->rx_comp_cons = sw_comp_cons;
1100         fp->rx_comp_prod = sw_comp_prod;
1101
1102         /* Update producers */
1103         bnx2x_update_rx_prod(bp, fp, bd_prod_fw, sw_comp_prod,
1104                              fp->rx_sge_prod);
1105
1106         fp->rx_pkt += rx_pkt;
1107         fp->rx_calls++;
1108
1109         return rx_pkt;
1110 }
1111
1112 static irqreturn_t bnx2x_msix_fp_int(int irq, void *fp_cookie)
1113 {
1114         struct bnx2x_fastpath *fp = fp_cookie;
1115         struct bnx2x *bp = fp->bp;
1116         uint8_t cos;
1117
1118         DP(NETIF_MSG_INTR,
1119            "got an MSI-X interrupt on IDX:SB [fp %d fw_sd %d igusb %d]\n",
1120            fp->index, fp->fw_sb_id, fp->igu_sb_id);
1121
1122         bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID, 0, IGU_INT_DISABLE, 0);
1123
1124 #ifdef BNX2X_STOP_ON_ERROR
1125         if (unlikely(bp->panic))
1126                 return IRQ_HANDLED;
1127 #endif
1128
1129         /* Handle Rx and Tx according to MSI-X vector */
1130         for_each_cos_in_tx_queue(fp, cos)
1131                 prefetch(fp->txdata_ptr[cos]->tx_cons_sb);
1132
1133         prefetch(&fp->sb_running_index[SM_RX_ID]);
1134         napi_schedule_irqoff(&bnx2x_fp(bp, fp->index, napi));
1135
1136         return IRQ_HANDLED;
1137 }
1138
1139 /* HW Lock for shared dual port PHYs */
1140 void bnx2x_acquire_phy_lock(struct bnx2x *bp)
1141 {
1142         qlock(&bp->port.phy_mutex);
1143
1144         bnx2x_acquire_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1145 }
1146
1147 void bnx2x_release_phy_lock(struct bnx2x *bp)
1148 {
1149         bnx2x_release_hw_lock(bp, HW_LOCK_RESOURCE_MDIO);
1150
1151         qunlock(&bp->port.phy_mutex);
1152 }
1153
1154 /* calculates MF speed according to current linespeed and MF configuration */
1155 uint16_t bnx2x_get_mf_speed(struct bnx2x *bp)
1156 {
1157         uint16_t line_speed = bp->link_vars.line_speed;
1158         if (IS_MF(bp)) {
1159                 uint16_t maxCfg = bnx2x_extract_max_cfg(bp,
1160                                                    bp->mf_config[BP_VN(bp)]);
1161
1162                 /* Calculate the current MAX line speed limit for the MF
1163                  * devices
1164                  */
1165                 if (IS_MF_SI(bp))
1166                         line_speed = (line_speed * maxCfg) / 100;
1167                 else { /* SD mode */
1168                         uint16_t vn_max_rate = maxCfg * 100;
1169
1170                         if (vn_max_rate < line_speed)
1171                                 line_speed = vn_max_rate;
1172                 }
1173         }
1174
1175         return line_speed;
1176 }
1177
1178 /**
1179  * bnx2x_fill_report_data - fill link report data to report
1180  *
1181  * @bp:         driver handle
1182  * @data:       link state to update
1183  *
1184  * It uses a none-atomic bit operations because is called under the mutex.
1185  */
1186 static void bnx2x_fill_report_data(struct bnx2x *bp,
1187                                    struct bnx2x_link_report_data *data)
1188 {
1189         memset(data, 0, sizeof(*data));
1190
1191         if (IS_PF(bp)) {
1192                 /* Fill the report data: effective line speed */
1193                 data->line_speed = bnx2x_get_mf_speed(bp);
1194
1195                 /* Link is down */
1196                 if (!bp->link_vars.link_up || (bp->flags & MF_FUNC_DIS))
1197                         __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1198                                   &data->link_report_flags);
1199
1200                 if (!BNX2X_NUM_ETH_QUEUES(bp))
1201                         __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1202                                   &data->link_report_flags);
1203
1204                 /* Full DUPLEX */
1205                 if (bp->link_vars.duplex == DUPLEX_FULL)
1206                         __set_bit(BNX2X_LINK_REPORT_FD,
1207                                   &data->link_report_flags);
1208
1209                 /* Rx Flow Control is ON */
1210                 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_RX)
1211                         __set_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1212                                   &data->link_report_flags);
1213
1214                 /* Tx Flow Control is ON */
1215                 if (bp->link_vars.flow_ctrl & BNX2X_FLOW_CTRL_TX)
1216                         __set_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1217                                   &data->link_report_flags);
1218         } else { /* VF */
1219                 *data = bp->vf_link_vars;
1220         }
1221 }
1222
1223 /**
1224  * bnx2x_link_report - report link status to OS.
1225  *
1226  * @bp:         driver handle
1227  *
1228  * Calls the __bnx2x_link_report() under the same locking scheme
1229  * as a link/PHY state managing code to ensure a consistent link
1230  * reporting.
1231  */
1232
1233 void bnx2x_link_report(struct bnx2x *bp)
1234 {
1235         bnx2x_acquire_phy_lock(bp);
1236         __bnx2x_link_report(bp);
1237         bnx2x_release_phy_lock(bp);
1238 }
1239
1240 /**
1241  * __bnx2x_link_report - report link status to OS.
1242  *
1243  * @bp:         driver handle
1244  *
1245  * None atomic implementation.
1246  * Should be called under the phy_lock.
1247  */
1248 void __bnx2x_link_report(struct bnx2x *bp)
1249 {
1250         struct bnx2x_link_report_data cur_data;
1251
1252         /* reread mf_cfg */
1253         if (IS_PF(bp) && !CHIP_IS_E1(bp))
1254                 bnx2x_read_mf_cfg(bp);
1255
1256         /* Read the current link report info */
1257         bnx2x_fill_report_data(bp, &cur_data);
1258
1259         /* Don't report link down or exactly the same link status twice */
1260         if (!memcmp(&cur_data, &bp->last_reported_link, sizeof(cur_data)) ||
1261             (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1262                       &bp->last_reported_link.link_report_flags) &&
1263              test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1264                       &cur_data.link_report_flags)))
1265                 return;
1266
1267         bp->link_cnt++;
1268
1269         /* We are going to report a new link parameters now -
1270          * remember the current data for the next time.
1271          */
1272         memcpy(&bp->last_reported_link, &cur_data, sizeof(cur_data));
1273
1274         /* propagate status to VFs */
1275         if (IS_PF(bp))
1276                 bnx2x_iov_link_update(bp);
1277
1278         if (test_bit(BNX2X_LINK_REPORT_LINK_DOWN,
1279                      &cur_data.link_report_flags)) {
1280                 netif_carrier_off(bp->dev);
1281                 netdev_err(bp->dev, "NIC Link is Down\n");
1282                 return;
1283         } else {
1284                 const char *duplex;
1285                 const char *flow;
1286
1287                 netif_carrier_on(bp->dev);
1288
1289                 if (test_and_clear_bit(BNX2X_LINK_REPORT_FD,
1290                                        &cur_data.link_report_flags))
1291                         duplex = "full";
1292                 else
1293                         duplex = "half";
1294
1295                 /* Handle the FC at the end so that only these flags would be
1296                  * possibly set. This way we may easily check if there is no FC
1297                  * enabled.
1298                  */
1299                 if (cur_data.link_report_flags) {
1300                         if (test_bit(BNX2X_LINK_REPORT_RX_FC_ON,
1301                                      &cur_data.link_report_flags)) {
1302                                 if (test_bit(BNX2X_LINK_REPORT_TX_FC_ON,
1303                                      &cur_data.link_report_flags))
1304                                         flow = "ON - receive & transmit";
1305                                 else
1306                                         flow = "ON - receive";
1307                         } else {
1308                                 flow = "ON - transmit";
1309                         }
1310                 } else {
1311                         flow = "none";
1312                 }
1313                 netdev_info(bp->dev, "NIC Link is Up, %d Mbps %s duplex, Flow control: %s\n",
1314                             cur_data.line_speed, duplex, flow);
1315         }
1316 }
1317
1318 static void bnx2x_set_next_page_sgl(struct bnx2x_fastpath *fp)
1319 {
1320         int i;
1321
1322         for (i = 1; i <= NUM_RX_SGE_PAGES; i++) {
1323                 struct eth_rx_sge *sge;
1324
1325                 sge = &fp->rx_sge_ring[RX_SGE_CNT * i - 2];
1326                 sge->addr_hi =
1327                         cpu_to_le32(U64_HI(fp->rx_sge_mapping +
1328                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1329
1330                 sge->addr_lo =
1331                         cpu_to_le32(U64_LO(fp->rx_sge_mapping +
1332                         BCM_PAGE_SIZE*(i % NUM_RX_SGE_PAGES)));
1333         }
1334 }
1335
1336 static void bnx2x_free_tpa_pool(struct bnx2x *bp,
1337                                 struct bnx2x_fastpath *fp, int last)
1338 {
1339         int i;
1340
1341         for (i = 0; i < last; i++) {
1342                 struct bnx2x_agg_info *tpa_info = &fp->tpa_info[i];
1343                 struct sw_rx_bd *first_buf = &tpa_info->first_buf;
1344                 uint8_t *data = first_buf->data;
1345
1346                 if (data == NULL) {
1347                         DP(NETIF_MSG_IFDOWN, "tpa bin %d empty on free\n", i);
1348                         continue;
1349                 }
1350                 if (tpa_info->tpa_state == BNX2X_TPA_START)
1351                         dma_unmap_single(&bp->pdev->dev,
1352                                          dma_unmap_addr(first_buf, mapping),
1353                                          fp->rx_buf_size, DMA_FROM_DEVICE);
1354                 bnx2x_frag_free(fp, data);
1355                 first_buf->data = NULL;
1356         }
1357 }
1358
1359 void bnx2x_init_rx_rings_cnic(struct bnx2x *bp)
1360 {
1361         int j;
1362
1363         for_each_rx_queue_cnic(bp, j) {
1364                 struct bnx2x_fastpath *fp = &bp->fp[j];
1365
1366                 fp->rx_bd_cons = 0;
1367
1368                 /* Activate BD ring */
1369                 /* Warning!
1370                  * this will generate an interrupt (to the TSTORM)
1371                  * must only be done after chip is initialized
1372                  */
1373                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1374                                      fp->rx_sge_prod);
1375         }
1376 }
1377
1378 void bnx2x_init_rx_rings(struct bnx2x *bp)
1379 {
1380         int func = BP_FUNC(bp);
1381         uint16_t ring_prod;
1382         int i, j;
1383
1384         /* Allocate TPA resources */
1385         for_each_eth_queue(bp, j) {
1386                 struct bnx2x_fastpath *fp = &bp->fp[j];
1387
1388                 DP(NETIF_MSG_IFUP,
1389                    "mtu %d  rx_buf_size %d\n", bp->dev->mtu, fp->rx_buf_size);
1390
1391                 if (!fp->disable_tpa) {
1392                         /* Fill the per-aggregation pool */
1393                         for (i = 0; i < MAX_AGG_QS(bp); i++) {
1394                                 struct bnx2x_agg_info *tpa_info =
1395                                         &fp->tpa_info[i];
1396                                 struct sw_rx_bd *first_buf =
1397                                         &tpa_info->first_buf;
1398
1399                                 first_buf->data =
1400                                         bnx2x_frag_alloc(fp, GFP_KERNEL);
1401                                 if (!first_buf->data) {
1402                                         BNX2X_ERR("Failed to allocate TPA skb pool for queue[%d] - disabling TPA on this queue!\n",
1403                                                   j);
1404                                         bnx2x_free_tpa_pool(bp, fp, i);
1405                                         fp->disable_tpa = 1;
1406                                         break;
1407                                 }
1408                                 dma_unmap_addr_set(first_buf, mapping, 0);
1409                                 tpa_info->tpa_state = BNX2X_TPA_STOP;
1410                         }
1411
1412                         /* "next page" elements initialization */
1413                         bnx2x_set_next_page_sgl(fp);
1414
1415                         /* set SGEs bit mask */
1416                         bnx2x_init_sge_ring_bit_mask(fp);
1417
1418                         /* Allocate SGEs and initialize the ring elements */
1419                         for (i = 0, ring_prod = 0;
1420                              i < MAX_RX_SGE_CNT*NUM_RX_SGE_PAGES; i++) {
1421
1422                                 if (bnx2x_alloc_rx_sge(bp, fp, ring_prod,
1423                                                        GFP_KERNEL) < 0) {
1424                                         BNX2X_ERR("was only able to allocate %d rx sges\n",
1425                                                   i);
1426                                         BNX2X_ERR("disabling TPA for queue[%d]\n",
1427                                                   j);
1428                                         /* Cleanup already allocated elements */
1429                                         bnx2x_free_rx_sge_range(bp, fp,
1430                                                                 ring_prod);
1431                                         bnx2x_free_tpa_pool(bp, fp,
1432                                                             MAX_AGG_QS(bp));
1433                                         fp->disable_tpa = 1;
1434                                         ring_prod = 0;
1435                                         break;
1436                                 }
1437                                 ring_prod = NEXT_SGE_IDX(ring_prod);
1438                         }
1439
1440                         fp->rx_sge_prod = ring_prod;
1441                 }
1442         }
1443
1444         for_each_eth_queue(bp, j) {
1445                 struct bnx2x_fastpath *fp = &bp->fp[j];
1446
1447                 fp->rx_bd_cons = 0;
1448
1449                 /* Activate BD ring */
1450                 /* Warning!
1451                  * this will generate an interrupt (to the TSTORM)
1452                  * must only be done after chip is initialized
1453                  */
1454                 bnx2x_update_rx_prod(bp, fp, fp->rx_bd_prod, fp->rx_comp_prod,
1455                                      fp->rx_sge_prod);
1456
1457                 if (j != 0)
1458                         continue;
1459
1460                 if (CHIP_IS_E1(bp)) {
1461                         REG_WR(bp, BAR_USTRORM_INTMEM +
1462                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func),
1463                                U64_LO(fp->rx_comp_mapping));
1464                         REG_WR(bp, BAR_USTRORM_INTMEM +
1465                                USTORM_MEM_WORKAROUND_ADDRESS_OFFSET(func) + 4,
1466                                U64_HI(fp->rx_comp_mapping));
1467                 }
1468         }
1469 }
1470
1471 static void bnx2x_free_tx_skbs_queue(struct bnx2x_fastpath *fp)
1472 {
1473         uint8_t cos;
1474         struct bnx2x *bp = fp->bp;
1475
1476         for_each_cos_in_tx_queue(fp, cos) {
1477                 struct bnx2x_fp_txdata *txdata = fp->txdata_ptr[cos];
1478                 unsigned pkts_compl = 0, bytes_compl = 0;
1479
1480                 uint16_t sw_prod = txdata->tx_pkt_prod;
1481                 uint16_t sw_cons = txdata->tx_pkt_cons;
1482
1483                 while (sw_cons != sw_prod) {
1484                         bnx2x_free_tx_pkt(bp, txdata, TX_BD(sw_cons),
1485                                           &pkts_compl, &bytes_compl);
1486                         sw_cons++;
1487                 }
1488
1489                 netdev_tx_reset_queue(
1490                         netdev_get_tx_queue(bp->dev,
1491                                             txdata->txq_index));
1492         }
1493 }
1494
1495 static void bnx2x_free_tx_skbs_cnic(struct bnx2x *bp)
1496 {
1497         int i;
1498
1499         for_each_tx_queue_cnic(bp, i) {
1500                 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1501         }
1502 }
1503
1504 static void bnx2x_free_tx_skbs(struct bnx2x *bp)
1505 {
1506         int i;
1507
1508         for_each_eth_queue(bp, i) {
1509                 bnx2x_free_tx_skbs_queue(&bp->fp[i]);
1510         }
1511 }
1512
1513 static void bnx2x_free_rx_bds(struct bnx2x_fastpath *fp)
1514 {
1515         struct bnx2x *bp = fp->bp;
1516         int i;
1517
1518         /* ring wasn't allocated */
1519         if (fp->rx_buf_ring == NULL)
1520                 return;
1521
1522         for (i = 0; i < NUM_RX_BD; i++) {
1523                 struct sw_rx_bd *rx_buf = &fp->rx_buf_ring[i];
1524                 uint8_t *data = rx_buf->data;
1525
1526                 if (data == NULL)
1527                         continue;
1528                 dma_unmap_single(&bp->pdev->dev,
1529                                  dma_unmap_addr(rx_buf, mapping),
1530                                  fp->rx_buf_size, DMA_FROM_DEVICE);
1531
1532                 rx_buf->data = NULL;
1533                 bnx2x_frag_free(fp, data);
1534         }
1535 }
1536
1537 static void bnx2x_free_rx_skbs_cnic(struct bnx2x *bp)
1538 {
1539         int j;
1540
1541         for_each_rx_queue_cnic(bp, j) {
1542                 bnx2x_free_rx_bds(&bp->fp[j]);
1543         }
1544 }
1545
1546 static void bnx2x_free_rx_skbs(struct bnx2x *bp)
1547 {
1548         int j;
1549
1550         for_each_eth_queue(bp, j) {
1551                 struct bnx2x_fastpath *fp = &bp->fp[j];
1552
1553                 bnx2x_free_rx_bds(fp);
1554
1555                 if (!fp->disable_tpa)
1556                         bnx2x_free_tpa_pool(bp, fp, MAX_AGG_QS(bp));
1557         }
1558 }
1559
1560 static void bnx2x_free_skbs_cnic(struct bnx2x *bp)
1561 {
1562         bnx2x_free_tx_skbs_cnic(bp);
1563         bnx2x_free_rx_skbs_cnic(bp);
1564 }
1565
1566 void bnx2x_free_skbs(struct bnx2x *bp)
1567 {
1568         bnx2x_free_tx_skbs(bp);
1569         bnx2x_free_rx_skbs(bp);
1570 }
1571
1572 void bnx2x_update_max_mf_config(struct bnx2x *bp, uint32_t value)
1573 {
1574         /* load old values */
1575         uint32_t mf_cfg = bp->mf_config[BP_VN(bp)];
1576
1577         if (value != bnx2x_extract_max_cfg(bp, mf_cfg)) {
1578                 /* leave all but MAX value */
1579                 mf_cfg &= ~FUNC_MF_CFG_MAX_BW_MASK;
1580
1581                 /* set new MAX value */
1582                 mf_cfg |= (value << FUNC_MF_CFG_MAX_BW_SHIFT)
1583                                 & FUNC_MF_CFG_MAX_BW_MASK;
1584
1585                 bnx2x_fw_command(bp, DRV_MSG_CODE_SET_MF_BW, mf_cfg);
1586         }
1587 }
1588
1589 /**
1590  * bnx2x_free_msix_irqs - free previously requested MSI-X IRQ vectors
1591  *
1592  * @bp:         driver handle
1593  * @nvecs:      number of vectors to be released
1594  */
1595 static void bnx2x_free_msix_irqs(struct bnx2x *bp, int nvecs)
1596 {
1597         int i, offset = 0;
1598
1599         if (nvecs == offset)
1600                 return;
1601
1602         /* VFs don't have a default SB */
1603         if (IS_PF(bp)) {
1604                 free_irq(bp->msix_table[offset].vector, bp->dev);
1605                 DP(NETIF_MSG_IFDOWN, "released sp irq (%d)\n",
1606                    bp->msix_table[offset].vector);
1607                 offset++;
1608         }
1609
1610         if (CNIC_SUPPORT(bp)) {
1611                 if (nvecs == offset)
1612                         return;
1613                 offset++;
1614         }
1615
1616         for_each_eth_queue(bp, i) {
1617                 if (nvecs == offset)
1618                         return;
1619                 DP(NETIF_MSG_IFDOWN, "about to release fp #%d->%d irq\n",
1620                    i, bp->msix_table[offset].vector);
1621
1622                 free_irq(bp->msix_table[offset++].vector, &bp->fp[i]);
1623         }
1624 }
1625
1626 void bnx2x_free_irq(struct bnx2x *bp)
1627 {
1628         if (bp->flags & USING_MSIX_FLAG &&
1629             !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1630                 int nvecs = BNX2X_NUM_ETH_QUEUES(bp) + CNIC_SUPPORT(bp);
1631
1632                 /* vfs don't have a default status block */
1633                 if (IS_PF(bp))
1634                         nvecs++;
1635
1636                 bnx2x_free_msix_irqs(bp, nvecs);
1637         } else {
1638                 free_irq(bp->dev->irq, bp->dev);
1639         }
1640 }
1641
1642 int bnx2x_enable_msix(struct bnx2x *bp)
1643 {
1644         int msix_vec = 0, i, rc;
1645
1646         /* VFs don't have a default status block */
1647         if (IS_PF(bp)) {
1648                 bp->msix_table[msix_vec].entry = msix_vec;
1649                 BNX2X_DEV_INFO("msix_table[0].entry = %d (slowpath)\n",
1650                                bp->msix_table[0].entry);
1651                 msix_vec++;
1652         }
1653
1654         /* Cnic requires an msix vector for itself */
1655         if (CNIC_SUPPORT(bp)) {
1656                 bp->msix_table[msix_vec].entry = msix_vec;
1657                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (CNIC)\n",
1658                                msix_vec, bp->msix_table[msix_vec].entry);
1659                 msix_vec++;
1660         }
1661
1662         /* We need separate vectors for ETH queues only (not FCoE) */
1663         for_each_eth_queue(bp, i) {
1664                 bp->msix_table[msix_vec].entry = msix_vec;
1665                 BNX2X_DEV_INFO("msix_table[%d].entry = %d (fastpath #%u)\n",
1666                                msix_vec, msix_vec, i);
1667                 msix_vec++;
1668         }
1669
1670         DP(BNX2X_MSG_SP, "about to request enable msix with %d vectors\n",
1671            msix_vec);
1672
1673         rc = pci_enable_msix_range(bp->pdev, &bp->msix_table[0],
1674                                    BNX2X_MIN_MSIX_VEC_CNT(bp), msix_vec);
1675         /*
1676          * reconfigure number of tx/rx queues according to available
1677          * MSI-X vectors
1678          */
1679         if (rc == -ENOSPC) {
1680                 /* Get by with single vector */
1681                 rc = pci_enable_msix_range(bp->pdev, &bp->msix_table[0], 1, 1);
1682                 if (rc < 0) {
1683                         BNX2X_DEV_INFO("Single MSI-X is not attainable rc %d\n",
1684                                        rc);
1685                         goto no_msix;
1686                 }
1687
1688                 BNX2X_DEV_INFO("Using single MSI-X vector\n");
1689                 bp->flags |= USING_SINGLE_MSIX_FLAG;
1690
1691                 BNX2X_DEV_INFO("set number of queues to 1\n");
1692                 bp->num_ethernet_queues = 1;
1693                 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1694         } else if (rc < 0) {
1695                 BNX2X_DEV_INFO("MSI-X is not attainable rc %d\n", rc);
1696                 goto no_msix;
1697         } else if (rc < msix_vec) {
1698                 /* how less vectors we will have? */
1699                 int diff = msix_vec - rc;
1700
1701                 BNX2X_DEV_INFO("Trying to use less MSI-X vectors: %d\n", rc);
1702
1703                 /*
1704                  * decrease number of queues by number of unallocated entries
1705                  */
1706                 bp->num_ethernet_queues -= diff;
1707                 bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1708
1709                 BNX2X_DEV_INFO("New queue configuration set: %d\n",
1710                                bp->num_queues);
1711         }
1712
1713         bp->flags |= USING_MSIX_FLAG;
1714
1715         return 0;
1716
1717 no_msix:
1718         /* fall to INTx if not enough memory */
1719         if (rc == -ENOMEM)
1720                 bp->flags |= DISABLE_MSI_FLAG;
1721
1722         return rc;
1723 }
1724
1725 static int bnx2x_req_msix_irqs(struct bnx2x *bp)
1726 {
1727         int i, rc, offset = 0;
1728
1729         /* no default status block for vf */
1730         if (IS_PF(bp)) {
1731                 rc = request_irq(bp->msix_table[offset++].vector,
1732                                  bnx2x_msix_sp_int, 0,
1733                                  bp->dev->name, bp->dev);
1734                 if (rc) {
1735                         BNX2X_ERR("request sp irq failed\n");
1736                         return -EBUSY;
1737                 }
1738         }
1739
1740         if (CNIC_SUPPORT(bp))
1741                 offset++;
1742
1743         for_each_eth_queue(bp, i) {
1744                 struct bnx2x_fastpath *fp = &bp->fp[i];
1745                 snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
1746                          bp->dev->name, i);
1747
1748                 rc = request_irq(bp->msix_table[offset].vector,
1749                                  bnx2x_msix_fp_int, 0, fp->name, fp);
1750                 if (rc) {
1751                         BNX2X_ERR("request fp #%d irq (%d) failed  rc %d\n", i,
1752                               bp->msix_table[offset].vector, rc);
1753                         bnx2x_free_msix_irqs(bp, offset);
1754                         return -EBUSY;
1755                 }
1756
1757                 offset++;
1758         }
1759
1760         i = BNX2X_NUM_ETH_QUEUES(bp);
1761         if (IS_PF(bp)) {
1762                 offset = 1 + CNIC_SUPPORT(bp);
1763                 netdev_info(bp->dev,
1764                             "using MSI-X  IRQs: sp %d  fp[%d] %d ... fp[%d] %d\n",
1765                             bp->msix_table[0].vector,
1766                             0, bp->msix_table[offset].vector,
1767                             i - 1, bp->msix_table[offset + i - 1].vector);
1768         } else {
1769                 offset = CNIC_SUPPORT(bp);
1770                 netdev_info(bp->dev,
1771                             "using MSI-X  IRQs: fp[%d] %d ... fp[%d] %d\n",
1772                             0, bp->msix_table[offset].vector,
1773                             i - 1, bp->msix_table[offset + i - 1].vector);
1774         }
1775         return 0;
1776 }
1777
1778 int bnx2x_enable_msi(struct bnx2x *bp)
1779 {
1780         int rc;
1781
1782         rc = pci_enable_msi(bp->pdev);
1783         if (rc) {
1784                 BNX2X_DEV_INFO("MSI is not attainable\n");
1785                 return -1;
1786         }
1787         bp->flags |= USING_MSI_FLAG;
1788
1789         return 0;
1790 }
1791
1792 static int bnx2x_req_irq(struct bnx2x *bp)
1793 {
1794         unsigned long flags;
1795         unsigned int irq;
1796
1797         if (bp->flags & (USING_MSI_FLAG | USING_MSIX_FLAG))
1798                 flags = 0;
1799         else
1800                 flags = IRQF_SHARED;
1801
1802         if (bp->flags & USING_MSIX_FLAG)
1803                 irq = bp->msix_table[0].vector;
1804         else
1805                 irq = bp->pdev->irq;
1806
1807         return request_irq(irq, bnx2x_interrupt, flags, bp->dev->name, bp->dev);
1808 }
1809
1810 static int bnx2x_setup_irqs(struct bnx2x *bp)
1811 {
1812         int rc = 0;
1813         if (bp->flags & USING_MSIX_FLAG &&
1814             !(bp->flags & USING_SINGLE_MSIX_FLAG)) {
1815                 rc = bnx2x_req_msix_irqs(bp);
1816                 if (rc)
1817                         return rc;
1818         } else {
1819                 rc = bnx2x_req_irq(bp);
1820                 if (rc) {
1821                         BNX2X_ERR("IRQ request failed  rc %d, aborting\n", rc);
1822                         return rc;
1823                 }
1824                 if (bp->flags & USING_MSI_FLAG) {
1825                         bp->dev->irq = bp->pdev->irq;
1826                         netdev_info(bp->dev, "using MSI IRQ %d\n",
1827                                     bp->dev->irq);
1828                 }
1829                 if (bp->flags & USING_MSIX_FLAG) {
1830                         bp->dev->irq = bp->msix_table[0].vector;
1831                         netdev_info(bp->dev, "using MSIX IRQ %d\n",
1832                                     bp->dev->irq);
1833                 }
1834         }
1835
1836         return 0;
1837 }
1838
1839 static void bnx2x_napi_enable_cnic(struct bnx2x *bp)
1840 {
1841         int i;
1842
1843         for_each_rx_queue_cnic(bp, i) {
1844                 bnx2x_fp_init_lock(&bp->fp[i]);
1845                 napi_enable(&bnx2x_fp(bp, i, napi));
1846         }
1847 }
1848
1849 static void bnx2x_napi_enable(struct bnx2x *bp)
1850 {
1851         int i;
1852
1853         for_each_eth_queue(bp, i) {
1854                 bnx2x_fp_init_lock(&bp->fp[i]);
1855                 napi_enable(&bnx2x_fp(bp, i, napi));
1856         }
1857 }
1858
1859 static void bnx2x_napi_disable_cnic(struct bnx2x *bp)
1860 {
1861         int i;
1862
1863         for_each_rx_queue_cnic(bp, i) {
1864                 napi_disable(&bnx2x_fp(bp, i, napi));
1865                 while (!bnx2x_fp_ll_disable(&bp->fp[i]))
1866                         kthread_usleep(1000);
1867         }
1868 }
1869
1870 static void bnx2x_napi_disable(struct bnx2x *bp)
1871 {
1872         int i;
1873
1874         for_each_eth_queue(bp, i) {
1875                 napi_disable(&bnx2x_fp(bp, i, napi));
1876                 while (!bnx2x_fp_ll_disable(&bp->fp[i]))
1877                         kthread_usleep(1000);
1878         }
1879 }
1880
1881 void bnx2x_netif_start(struct bnx2x *bp)
1882 {
1883         if (netif_running(bp->dev)) {
1884                 bnx2x_napi_enable(bp);
1885                 if (CNIC_LOADED(bp))
1886                         bnx2x_napi_enable_cnic(bp);
1887                 bnx2x_int_enable(bp);
1888                 if (bp->state == BNX2X_STATE_OPEN)
1889                         netif_tx_wake_all_queues(bp->dev);
1890         }
1891 }
1892
1893 void bnx2x_netif_stop(struct bnx2x *bp, int disable_hw)
1894 {
1895         bnx2x_int_disable_sync(bp, disable_hw);
1896         bnx2x_napi_disable(bp);
1897         if (CNIC_LOADED(bp))
1898                 bnx2x_napi_disable_cnic(bp);
1899 }
1900
1901 uint16_t bnx2x_select_queue(struct ether *dev, struct sk_buff *skb,
1902                        void *accel_priv, select_queue_fallback_t fallback)
1903 {
1904         struct bnx2x *bp = netdev_priv(dev);
1905
1906         if (CNIC_LOADED(bp) && !NO_FCOE(bp)) {
1907                 struct ethhdr *hdr = (struct ethhdr *)skb->data;
1908                 uint16_t ether_type = ntohs(hdr->h_proto);
1909
1910                 /* Skip VLAN tag if present */
1911                 if (ether_type == ETH_P_8021Q) {
1912                         struct vlan_ethhdr *vhdr =
1913                                 (struct vlan_ethhdr *)skb->data;
1914
1915                         ether_type = ntohs(vhdr->h_vlan_encapsulated_proto);
1916                 }
1917
1918                 /* If ethertype is FCoE or FIP - use FCoE ring */
1919                 if ((ether_type == ETH_P_FCOE) || (ether_type == ETH_P_FIP))
1920                         return bnx2x_fcoe_tx(bp, txq_index);
1921         }
1922
1923         /* select a non-FCoE queue */
1924         return fallback(dev, skb) % BNX2X_NUM_ETH_QUEUES(bp);
1925 }
1926
1927 void bnx2x_set_num_queues(struct bnx2x *bp)
1928 {
1929         /* RSS queues */
1930         bp->num_ethernet_queues = bnx2x_calc_num_queues(bp);
1931
1932         /* override in STORAGE SD modes */
1933         if (IS_MF_STORAGE_ONLY(bp))
1934                 bp->num_ethernet_queues = 1;
1935
1936         /* Add special queues */
1937         bp->num_cnic_queues = CNIC_SUPPORT(bp); /* For FCOE */
1938         bp->num_queues = bp->num_ethernet_queues + bp->num_cnic_queues;
1939
1940         BNX2X_DEV_INFO("set number of queues to %d\n", bp->num_queues);
1941 }
1942
1943 /**
1944  * bnx2x_set_real_num_queues - configure netdev->real_num_[tx,rx]_queues
1945  *
1946  * @bp:         Driver handle
1947  *
1948  * We currently support for at most 16 Tx queues for each CoS thus we will
1949  * allocate a multiple of 16 for ETH L2 rings according to the value of the
1950  * bp->max_cos.
1951  *
1952  * If there is an FCoE L2 queue the appropriate Tx queue will have the next
1953  * index after all ETH L2 indices.
1954  *
1955  * If the actual number of Tx queues (for each CoS) is less than 16 then there
1956  * will be the holes at the end of each group of 16 ETh L2 indices (0..15,
1957  * 16..31,...) with indices that are not coupled with any real Tx queue.
1958  *
1959  * The proper configuration of skb->queue_mapping is handled by
1960  * bnx2x_select_queue() and __skb_tx_hash().
1961  *
1962  * bnx2x_setup_tc() takes care of the proper TC mappings so that __skb_tx_hash()
1963  * will return a proper Tx index if TC is enabled (netdev->num_tc > 0).
1964  */
1965 static int bnx2x_set_real_num_queues(struct bnx2x *bp, int include_cnic)
1966 {
1967         int rc, tx, rx;
1968
1969         tx = BNX2X_NUM_ETH_QUEUES(bp) * bp->max_cos;
1970         rx = BNX2X_NUM_ETH_QUEUES(bp);
1971
1972 /* account for fcoe queue */
1973         if (include_cnic && !NO_FCOE(bp)) {
1974                 rx++;
1975                 tx++;
1976         }
1977
1978         rc = netif_set_real_num_tx_queues(bp->dev, tx);
1979         if (rc) {
1980                 BNX2X_ERR("Failed to set real number of Tx queues: %d\n", rc);
1981                 return rc;
1982         }
1983         rc = netif_set_real_num_rx_queues(bp->dev, rx);
1984         if (rc) {
1985                 BNX2X_ERR("Failed to set real number of Rx queues: %d\n", rc);
1986                 return rc;
1987         }
1988
1989         DP(NETIF_MSG_IFUP, "Setting real num queues to (tx, rx) (%d, %d)\n",
1990                           tx, rx);
1991
1992         return rc;
1993 }
1994
1995 static void bnx2x_set_rx_buf_size(struct bnx2x *bp)
1996 {
1997         int i;
1998
1999         for_each_queue(bp, i) {
2000                 struct bnx2x_fastpath *fp = &bp->fp[i];
2001                 uint32_t mtu;
2002
2003                 /* Always use a mini-jumbo MTU for the FCoE L2 ring */
2004                 if (IS_FCOE_IDX(i))
2005                         /*
2006                          * Although there are no IP frames expected to arrive to
2007                          * this ring we still want to add an
2008                          * IP_HEADER_ALIGNMENT_PADDING to prevent a buffer
2009                          * overrun attack.
2010                          */
2011                         mtu = BNX2X_FCOE_MINI_JUMBO_MTU;
2012                 else
2013                         mtu = bp->dev->mtu;
2014                 fp->rx_buf_size = BNX2X_FW_RX_ALIGN_START +
2015                                   IP_HEADER_ALIGNMENT_PADDING +
2016                                   ETH_OVREHEAD +
2017                                   mtu +
2018                                   BNX2X_FW_RX_ALIGN_END;
2019                 /* Note : rx_buf_size doesn't take into account NET_SKB_PAD */
2020                 if (fp->rx_buf_size + NET_SKB_PAD <= PAGE_SIZE)
2021                         fp->rx_frag_size = fp->rx_buf_size + NET_SKB_PAD;
2022                 else
2023                         fp->rx_frag_size = 0;
2024         }
2025 }
2026
2027 static int bnx2x_init_rss(struct bnx2x *bp)
2028 {
2029         int i;
2030         uint8_t num_eth_queues = BNX2X_NUM_ETH_QUEUES(bp);
2031
2032         /* Prepare the initial contents for the indirection table if RSS is
2033          * enabled
2034          */
2035         for (i = 0; i < sizeof(bp->rss_conf_obj.ind_table); i++)
2036                 bp->rss_conf_obj.ind_table[i] =
2037                         bp->fp->cl_id +
2038                         ethtool_rxfh_indir_default(i, num_eth_queues);
2039
2040         /*
2041          * For 57710 and 57711 SEARCHER configuration (rss_keys) is
2042          * per-port, so if explicit configuration is needed , do it only
2043          * for a PMF.
2044          *
2045          * For 57712 and newer on the other hand it's a per-function
2046          * configuration.
2047          */
2048         return bnx2x_config_rss_eth(bp, bp->port.pmf || !CHIP_IS_E1x(bp));
2049 }
2050
2051 int bnx2x_rss(struct bnx2x *bp, struct bnx2x_rss_config_obj *rss_obj,
2052               bool config_hash, bool enable)
2053 {
2054         struct bnx2x_config_rss_params params = {NULL};
2055
2056         /* Although RSS is meaningless when there is a single HW queue we
2057          * still need it enabled in order to have HW Rx hash generated.
2058          *
2059          * if (!is_eth_multi(bp))
2060          *      bp->multi_mode = ETH_RSS_MODE_DISABLED;
2061          */
2062
2063         params.rss_obj = rss_obj;
2064
2065         __set_bit(RAMROD_COMP_WAIT, &params.ramrod_flags);
2066
2067         if (enable) {
2068                 __set_bit(BNX2X_RSS_MODE_REGULAR, &params.rss_flags);
2069
2070                 /* RSS configuration */
2071                 __set_bit(BNX2X_RSS_IPV4, &params.rss_flags);
2072                 __set_bit(BNX2X_RSS_IPV4_TCP, &params.rss_flags);
2073                 __set_bit(BNX2X_RSS_IPV6, &params.rss_flags);
2074                 __set_bit(BNX2X_RSS_IPV6_TCP, &params.rss_flags);
2075                 if (rss_obj->udp_rss_v4)
2076                         __set_bit(BNX2X_RSS_IPV4_UDP, &params.rss_flags);
2077                 if (rss_obj->udp_rss_v6)
2078                         __set_bit(BNX2X_RSS_IPV6_UDP, &params.rss_flags);
2079
2080                 if (!CHIP_IS_E1x(bp))
2081                         /* valid only for TUNN_MODE_GRE tunnel mode */
2082                         __set_bit(BNX2X_RSS_GRE_INNER_HDRS, &params.rss_flags);
2083         } else {
2084                 __set_bit(BNX2X_RSS_MODE_DISABLED, &params.rss_flags);
2085         }
2086
2087         /* Hash bits */
2088         params.rss_result_mask = MULTI_MASK;
2089
2090         memcpy(params.ind_table, rss_obj->ind_table, sizeof(params.ind_table));
2091
2092         if (config_hash) {
2093                 /* RSS keys */
2094                 netdev_rss_key_fill(params.rss_key, T_ETH_RSS_KEY * 4);
2095                 __set_bit(BNX2X_RSS_SET_SRCH, &params.rss_flags);
2096         }
2097
2098         if (IS_PF(bp))
2099                 return bnx2x_config_rss(bp, &params);
2100         else
2101                 return bnx2x_vfpf_config_rss(bp, &params);
2102 }
2103
2104 static int bnx2x_init_hw(struct bnx2x *bp, uint32_t load_code)
2105 {
2106         struct bnx2x_func_state_params func_params = {NULL};
2107
2108         /* Prepare parameters for function state transitions */
2109         __set_bit(RAMROD_COMP_WAIT, &func_params.ramrod_flags);
2110
2111         func_params.f_obj = &bp->func_obj;
2112         func_params.cmd = BNX2X_F_CMD_HW_INIT;
2113
2114         func_params.params.hw_init.load_phase = load_code;
2115
2116         return bnx2x_func_state_change(bp, &func_params);
2117 }
2118
2119 /*
2120  * Cleans the object that have internal lists without sending
2121  * ramrods. Should be run when interrupts are disabled.
2122  */
2123 void bnx2x_squeeze_objects(struct bnx2x *bp)
2124 {
2125         int rc;
2126         unsigned long ramrod_flags = 0, vlan_mac_flags = 0;
2127         struct bnx2x_mcast_ramrod_params rparam = {NULL};
2128         struct bnx2x_vlan_mac_obj *mac_obj = &bp->sp_objs->mac_obj;
2129
2130         /***************** Cleanup MACs' object first *************************/
2131
2132         /* Wait for completion of requested */
2133         __set_bit(RAMROD_COMP_WAIT, &ramrod_flags);
2134         /* Perform a dry cleanup */
2135         __set_bit(RAMROD_DRV_CLR_ONLY, &ramrod_flags);
2136
2137         /* Clean ETH primary MAC */
2138         __set_bit(BNX2X_ETH_MAC, &vlan_mac_flags);
2139         rc = mac_obj->delete_all(bp, &bp->sp_objs->mac_obj, &vlan_mac_flags,
2140                                  &ramrod_flags);
2141         if (rc != 0)
2142                 BNX2X_ERR("Failed to clean ETH MACs: %d\n", rc);
2143
2144         /* Cleanup UC list */
2145         vlan_mac_flags = 0;
2146         __set_bit(BNX2X_UC_LIST_MAC, &vlan_mac_flags);
2147         rc = mac_obj->delete_all(bp, mac_obj, &vlan_mac_flags,
2148                                  &ramrod_flags);
2149         if (rc != 0)
2150                 BNX2X_ERR("Failed to clean UC list MACs: %d\n", rc);
2151
2152         /***************** Now clean mcast object *****************************/
2153         rparam.mcast_obj = &bp->mcast_obj;
2154         __set_bit(RAMROD_DRV_CLR_ONLY, &rparam.ramrod_flags);
2155
2156         /* Add a DEL command... - Since we're doing a driver cleanup only,
2157          * we take a lock surrounding both the initial send and the CONTs,
2158          * as we don't want a true completion to disrupt us in the middle.
2159          */
2160         netif_addr_lock_bh(bp->dev);
2161         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_DEL);
2162         if (rc < 0)
2163                 BNX2X_ERR("Failed to add a new DEL command to a multi-cast object: %d\n",
2164                           rc);
2165
2166         /* ...and wait until all pending commands are cleared */
2167         rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2168         while (rc != 0) {
2169                 if (rc < 0) {
2170                         BNX2X_ERR("Failed to clean multi-cast object: %d\n",
2171                                   rc);
2172                         netif_addr_unlock_bh(bp->dev);
2173                         return;
2174                 }
2175
2176                 rc = bnx2x_config_mcast(bp, &rparam, BNX2X_MCAST_CMD_CONT);
2177         }
2178         netif_addr_unlock_bh(bp->dev);
2179 }
2180
2181 #ifndef BNX2X_STOP_ON_ERROR
2182 #define LOAD_ERROR_EXIT(bp, label) \
2183         do { \
2184                 (bp)->state = BNX2X_STATE_ERROR; \
2185                 goto label; \
2186         } while (0)
2187
2188 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2189         do { \
2190                 bp->cnic_loaded = false; \
2191                 goto label; \
2192         } while (0)
2193 #else /*BNX2X_STOP_ON_ERROR*/
2194 #define LOAD_ERROR_EXIT(bp, label) \
2195         do { \
2196                 (bp)->state = BNX2X_STATE_ERROR; \
2197                 (bp)->panic = 1; \
2198                 return -EBUSY; \
2199         } while (0)
2200 #define LOAD_ERROR_EXIT_CNIC(bp, label) \
2201         do { \
2202                 bp->cnic_loaded = false; \
2203                 (bp)->panic = 1; \
2204                 return -EBUSY; \
2205         } while (0)
2206 #endif /*BNX2X_STOP_ON_ERROR*/
2207
2208 static void bnx2x_free_fw_stats_mem(struct bnx2x *bp)
2209 {
2210         BNX2X_PCI_FREE(bp->fw_stats, bp->fw_stats_mapping,
2211                        bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2212         return;
2213 }
2214
2215 static int bnx2x_alloc_fw_stats_mem(struct bnx2x *bp)
2216 {
2217         int num_groups, vf_headroom = 0;
2218         int is_fcoe_stats = NO_FCOE(bp) ? 0 : 1;
2219
2220         /* number of queues for statistics is number of eth queues + FCoE */
2221         uint8_t num_queue_stats = BNX2X_NUM_ETH_QUEUES(bp) + is_fcoe_stats;
2222
2223         /* Total number of FW statistics requests =
2224          * 1 for port stats + 1 for PF stats + potential 2 for FCoE (fcoe proper
2225          * and fcoe l2 queue) stats + num of queues (which includes another 1
2226          * for fcoe l2 queue if applicable)
2227          */
2228         bp->fw_stats_num = 2 + is_fcoe_stats + num_queue_stats;
2229
2230         /* vf stats appear in the request list, but their data is allocated by
2231          * the VFs themselves. We don't include them in the bp->fw_stats_num as
2232          * it is used to determine where to place the vf stats queries in the
2233          * request struct
2234          */
2235         if (IS_SRIOV(bp))
2236                 vf_headroom = bnx2x_vf_headroom(bp);
2237
2238         /* Request is built from stats_query_header and an array of
2239          * stats_query_cmd_group each of which contains
2240          * STATS_QUERY_CMD_COUNT rules. The real number or requests is
2241          * configured in the stats_query_header.
2242          */
2243         num_groups =
2244                 (((bp->fw_stats_num + vf_headroom) / STATS_QUERY_CMD_COUNT) +
2245                  (((bp->fw_stats_num + vf_headroom) % STATS_QUERY_CMD_COUNT) ?
2246                  1 : 0));
2247
2248         DP(BNX2X_MSG_SP, "stats fw_stats_num %d, vf headroom %d, num_groups %d\n",
2249            bp->fw_stats_num, vf_headroom, num_groups);
2250         bp->fw_stats_req_sz = sizeof(struct stats_query_header) +
2251                 num_groups * sizeof(struct stats_query_cmd_group);
2252
2253         /* Data for statistics requests + stats_counter
2254          * stats_counter holds per-STORM counters that are incremented
2255          * when STORM has finished with the current request.
2256          * memory for FCoE offloaded statistics are counted anyway,
2257          * even if they will not be sent.
2258          * VF stats are not accounted for here as the data of VF stats is stored
2259          * in memory allocated by the VF, not here.
2260          */
2261         bp->fw_stats_data_sz = sizeof(struct per_port_stats) +
2262                 sizeof(struct per_pf_stats) +
2263                 sizeof(struct fcoe_statistics_params) +
2264                 sizeof(struct per_queue_stats) * num_queue_stats +
2265                 sizeof(struct stats_counter);
2266
2267         bp->fw_stats = BNX2X_PCI_ALLOC(&bp->fw_stats_mapping,
2268                                        bp->fw_stats_data_sz + bp->fw_stats_req_sz);
2269         if (!bp->fw_stats)
2270                 goto alloc_mem_err;
2271
2272         /* Set shortcuts */
2273         bp->fw_stats_req = (struct bnx2x_fw_stats_req *)bp->fw_stats;
2274         bp->fw_stats_req_mapping = bp->fw_stats_mapping;
2275         bp->fw_stats_data = (struct bnx2x_fw_stats_data *)
2276                 ((uint8_t *)bp->fw_stats + bp->fw_stats_req_sz);
2277         bp->fw_stats_data_mapping = bp->fw_stats_mapping +
2278                 bp->fw_stats_req_sz;
2279
2280         DP(BNX2X_MSG_SP, "statistics request base address set to %x %x\n",
2281            U64_HI(bp->fw_stats_req_mapping),
2282            U64_LO(bp->fw_stats_req_mapping));
2283         DP(BNX2X_MSG_SP, "statistics data base address set to %x %x\n",
2284            U64_HI(bp->fw_stats_data_mapping),
2285            U64_LO(bp->fw_stats_data_mapping));
2286         return 0;
2287
2288 alloc_mem_err:
2289         bnx2x_free_fw_stats_mem(bp);
2290         BNX2X_ERR("Can't allocate FW stats memory\n");
2291         return -ENOMEM;
2292 }
2293
2294 /* send load request to mcp and analyze response */
2295 static int bnx2x_nic_load_request(struct bnx2x *bp, uint32_t *load_code)
2296 {
2297         uint32_t param;
2298
2299         /* init fw_seq */
2300         bp->fw_seq =
2301                 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_mb_header) &
2302                  DRV_MSG_SEQ_NUMBER_MASK);
2303         BNX2X_DEV_INFO("fw_seq 0x%08x\n", bp->fw_seq);
2304
2305         /* Get current FW pulse sequence */
2306         bp->fw_drv_pulse_wr_seq =
2307                 (SHMEM_RD(bp, func_mb[BP_FW_MB_IDX(bp)].drv_pulse_mb) &
2308                  DRV_PULSE_SEQ_MASK);
2309         BNX2X_DEV_INFO("drv_pulse 0x%x\n", bp->fw_drv_pulse_wr_seq);
2310
2311         param = DRV_MSG_CODE_LOAD_REQ_WITH_LFA;
2312
2313         if (IS_MF_SD(bp) && bnx2x_port_after_undi(bp))
2314                 param |= DRV_MSG_CODE_LOAD_REQ_FORCE_LFA;
2315
2316         /* load request */
2317         (*load_code) = bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_REQ, param);
2318
2319         /* if mcp fails to respond we must abort */
2320         if (!(*load_code)) {
2321                 BNX2X_ERR("MCP response failure, aborting\n");
2322                 return -EBUSY;
2323         }
2324
2325         /* If mcp refused (e.g. other port is in diagnostic mode) we
2326          * must abort
2327          */
2328         if ((*load_code) == FW_MSG_CODE_DRV_LOAD_REFUSED) {
2329                 BNX2X_ERR("MCP refused load request, aborting\n");
2330                 return -EBUSY;
2331         }
2332         return 0;
2333 }
2334
2335 /* check whether another PF has already loaded FW to chip. In
2336  * virtualized environments a pf from another VM may have already
2337  * initialized the device including loading FW
2338  */
2339 int bnx2x_compare_fw_ver(struct bnx2x *bp, uint32_t load_code,
2340                          bool print_err)
2341 {
2342         /* is another pf loaded on this engine? */
2343         if (load_code != FW_MSG_CODE_DRV_LOAD_COMMON_CHIP &&
2344             load_code != FW_MSG_CODE_DRV_LOAD_COMMON) {
2345                 /* build my FW version dword */
2346                 uint32_t my_fw = (BCM_5710_FW_MAJOR_VERSION) +
2347                         (BCM_5710_FW_MINOR_VERSION << 8) +
2348                         (BCM_5710_FW_REVISION_VERSION << 16) +
2349                         (BCM_5710_FW_ENGINEERING_VERSION << 24);
2350
2351                 /* read loaded FW from chip */
2352                 uint32_t loaded_fw = REG_RD(bp, XSEM_REG_PRAM);
2353
2354                 DP(BNX2X_MSG_SP, "loaded fw %x, my fw %x\n",
2355                    loaded_fw, my_fw);
2356
2357                 /* abort nic load if version mismatch */
2358                 if (my_fw != loaded_fw) {
2359                         if (print_err)
2360                                 BNX2X_ERR("bnx2x with FW %x was already loaded which mismatches my %x FW. Aborting\n",
2361                                           loaded_fw, my_fw);
2362                         else
2363                                 BNX2X_DEV_INFO("bnx2x with FW %x was already loaded which mismatches my %x FW, possibly due to MF UNDI\n",
2364                                                loaded_fw, my_fw);
2365                         return -EBUSY;
2366                 }
2367         }
2368         return 0;
2369 }
2370
2371 /* returns the "mcp load_code" according to global load_count array */
2372 static int bnx2x_nic_load_no_mcp(struct bnx2x *bp, int port)
2373 {
2374         int path = BP_PATH(bp);
2375
2376         DP(NETIF_MSG_IFUP, "NO MCP - load counts[%d]      %d, %d, %d\n",
2377            path, bnx2x_load_count[path][0], bnx2x_load_count[path][1],
2378            bnx2x_load_count[path][2]);
2379         bnx2x_load_count[path][0]++;
2380         bnx2x_load_count[path][1 + port]++;
2381         DP(NETIF_MSG_IFUP, "NO MCP - new load counts[%d]  %d, %d, %d\n",
2382            path, bnx2x_load_count[path][0], bnx2x_load_count[path][1],
2383            bnx2x_load_count[path][2]);
2384         if (bnx2x_load_count[path][0] == 1)
2385                 return FW_MSG_CODE_DRV_LOAD_COMMON;
2386         else if (bnx2x_load_count[path][1 + port] == 1)
2387                 return FW_MSG_CODE_DRV_LOAD_PORT;
2388         else
2389                 return FW_MSG_CODE_DRV_LOAD_FUNCTION;
2390 }
2391
2392 /* mark PMF if applicable */
2393 static void bnx2x_nic_load_pmf(struct bnx2x *bp, uint32_t load_code)
2394 {
2395         if ((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2396             (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP) ||
2397             (load_code == FW_MSG_CODE_DRV_LOAD_PORT)) {
2398                 bp->port.pmf = 1;
2399                 /* We need the barrier to ensure the ordering between the
2400                  * writing to bp->port.pmf here and reading it from the
2401                  * bnx2x_periodic_task().
2402                  */
2403                 mb();
2404         } else {
2405                 bp->port.pmf = 0;
2406         }
2407
2408         DP(NETIF_MSG_LINK, "pmf %d\n", bp->port.pmf);
2409 }
2410
2411 static void bnx2x_nic_load_afex_dcc(struct bnx2x *bp, int load_code)
2412 {
2413         if (((load_code == FW_MSG_CODE_DRV_LOAD_COMMON) ||
2414              (load_code == FW_MSG_CODE_DRV_LOAD_COMMON_CHIP)) &&
2415             (bp->common.shmem2_base)) {
2416                 if (SHMEM2_HAS(bp, dcc_support))
2417                         SHMEM2_WR(bp, dcc_support,
2418                                   (SHMEM_DCC_SUPPORT_DISABLE_ENABLE_PF_TLV |
2419                                    SHMEM_DCC_SUPPORT_BANDWIDTH_ALLOCATION_TLV));
2420                 if (SHMEM2_HAS(bp, afex_driver_support))
2421                         SHMEM2_WR(bp, afex_driver_support,
2422                                   SHMEM_AFEX_SUPPORTED_VERSION_ONE);
2423         }
2424
2425         /* Set AFEX default VLAN tag to an invalid value */
2426         bp->afex_def_vlan_tag = -1;
2427 }
2428
2429 /**
2430  * bnx2x_bz_fp - zero content of the fastpath structure.
2431  *
2432  * @bp:         driver handle
2433  * @index:      fastpath index to be zeroed
2434  *
2435  * Makes sure the contents of the bp->fp[index].napi is kept
2436  * intact.
2437  */
2438 static void bnx2x_bz_fp(struct bnx2x *bp, int index)
2439 {
2440         struct bnx2x_fastpath *fp = &bp->fp[index];
2441         int cos;
2442         struct napi_struct orig_napi = fp->napi;
2443         struct bnx2x_agg_info *orig_tpa_info = fp->tpa_info;
2444
2445         /* bzero bnx2x_fastpath contents */
2446         if (fp->tpa_info)
2447                 memset(fp->tpa_info, 0, ETH_MAX_AGGREGATION_QUEUES_E1H_E2 *
2448                        sizeof(struct bnx2x_agg_info));
2449         memset(fp, 0, sizeof(*fp));
2450
2451         /* Restore the NAPI object as it has been already initialized */
2452         fp->napi = orig_napi;
2453         fp->tpa_info = orig_tpa_info;
2454         fp->bp = bp;
2455         fp->index = index;
2456         if (IS_ETH_FP(fp))
2457                 fp->max_cos = bp->max_cos;
2458         else
2459                 /* Special queues support only one CoS */
2460                 fp->max_cos = 1;
2461
2462         /* Init txdata pointers */
2463         if (IS_FCOE_FP(fp))
2464                 fp->txdata_ptr[0] = &bp->bnx2x_txq[FCOE_TXQ_IDX(bp)];
2465         if (IS_ETH_FP(fp))
2466                 for_each_cos_in_tx_queue(fp, cos)
2467                         fp->txdata_ptr[cos] = &bp->bnx2x_txq[cos *
2468                                 BNX2X_NUM_ETH_QUEUES(bp) + index];
2469
2470         /* set the tpa flag for each queue. The tpa flag determines the queue
2471          * minimal size so it must be set prior to queue memory allocation
2472          */
2473         fp->disable_tpa = !(bp->flags & TPA_ENABLE_FLAG ||
2474                                   (bp->flags & GRO_ENABLE_FLAG &&
2475                                    bnx2x_mtu_allows_gro(bp->dev->mtu)));
2476         if (bp->flags & TPA_ENABLE_FLAG)
2477                 fp->mode = TPA_MODE_LRO;
2478         else if (bp->flags & GRO_ENABLE_FLAG)
2479                 fp->mode = TPA_MODE_GRO;
2480
2481         /* We don't want TPA on an FCoE L2 ring */
2482         if (IS_FCOE_FP(fp))
2483                 fp->disable_tpa = 1;
2484 }
2485
2486 int bnx2x_load_cnic(struct bnx2x *bp)
2487 {
2488         int i, rc, port = BP_PORT(bp);
2489
2490         DP(NETIF_MSG_IFUP, "Starting CNIC-related load\n");
2491
2492         mutex_init(&bp->cnic_mutex);
2493
2494         if (IS_PF(bp)) {
2495                 rc = bnx2x_alloc_mem_cnic(bp);
2496                 if (rc) {
2497                         BNX2X_ERR("Unable to allocate bp memory for cnic\n");
2498                         LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2499                 }
2500         }
2501
2502         rc = bnx2x_alloc_fp_mem_cnic(bp);
2503         if (rc) {
2504                 BNX2X_ERR("Unable to allocate memory for cnic fps\n");
2505                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2506         }
2507
2508         /* Update the number of queues with the cnic queues */
2509         rc = bnx2x_set_real_num_queues(bp, 1);
2510         if (rc) {
2511                 BNX2X_ERR("Unable to set real_num_queues including cnic\n");
2512                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic0);
2513         }
2514
2515         /* Add all CNIC NAPI objects */
2516         bnx2x_add_all_napi_cnic(bp);
2517         DP(NETIF_MSG_IFUP, "cnic napi added\n");
2518         bnx2x_napi_enable_cnic(bp);
2519
2520         rc = bnx2x_init_hw_func_cnic(bp);
2521         if (rc)
2522                 LOAD_ERROR_EXIT_CNIC(bp, load_error_cnic1);
2523
2524         bnx2x_nic_init_cnic(bp);
2525
2526         if (IS_PF(bp)) {
2527                 /* Enable Timer scan */
2528                 REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 1);
2529
2530                 /* setup cnic queues */
2531                 for_each_cnic_queue(bp, i) {
2532                         rc = bnx2x_setup_queue(bp, &bp->fp[i], 0);
2533                         if (rc) {
2534                                 BNX2X_ERR("Queue setup failed\n");
2535                                 LOAD_ERROR_EXIT(bp, load_error_cnic2);
2536                         }
2537                 }
2538         }
2539
2540         /* Initialize Rx filter. */
2541         bnx2x_set_rx_mode_inner(bp);
2542
2543         /* re-read iscsi info */
2544         bnx2x_get_iscsi_info(bp);
2545         bnx2x_setup_cnic_irq_info(bp);
2546         bnx2x_setup_cnic_info(bp);
2547         bp->cnic_loaded = true;
2548         if (bp->state == BNX2X_STATE_OPEN)
2549                 bnx2x_cnic_notify(bp, CNIC_CTL_START_CMD);
2550
2551         DP(NETIF_MSG_IFUP, "Ending successfully CNIC-related load\n");
2552
2553         return 0;
2554
2555 #ifndef BNX2X_STOP_ON_ERROR
2556 load_error_cnic2:
2557         /* Disable Timer scan */
2558         REG_WR(bp, TM_REG_EN_LINEAR0_TIMER + port*4, 0);
2559
2560 load_error_cnic1:
2561         bnx2x_napi_disable_cnic(bp);
2562         /* Update the number of queues without the cnic queues */
2563         if (bnx2x_set_real_num_queues(bp, 0))
2564                 BNX2X_ERR("Unable to set real_num_queues not including cnic\n");
2565 load_error_cnic0:
2566         BNX2X_ERR("CNIC-related load failed\n");
2567         bnx2x_free_fp_mem_cnic(bp);
2568         bnx2x_free_mem_cnic(bp);
2569         return rc;
2570 #endif /* ! BNX2X_STOP_ON_ERROR */
2571 }
2572
2573 /* must be called with rtnl_lock */
2574 int bnx2x_nic_load(struct bnx2x *bp, int load_mode)
2575 {
2576         int port = BP_PORT(bp);
2577         int i, rc = 0, load_code = 0;
2578
2579         DP(NETIF_MSG_IFUP, "Starting NIC load\n");
2580         DP(NETIF_MSG_IFUP,
2581            "CNIC is %s\n", CNIC_ENABLED(bp) ? "enabled" : "disabled");
2582
2583 #ifdef BNX2X_STOP_ON_ERROR
2584         if (unlikely(bp->panic)) {
2585                 BNX2X_ERR("Can't load NIC when there is panic\n");
2586                 return -EPERM;
2587         }
2588 #endif
2589
2590         bp->state = BNX2X_STATE_OPENING_WAIT4_LOAD;
2591
2592         /* zero the structure w/o any lock, before SP handler is initialized */
2593         memset(&bp->last_reported_link, 0, sizeof(bp->last_reported_link));
2594         __set_bit(BNX2X_LINK_REPORT_LINK_DOWN,
2595                 &bp->last_reported_link.link_report_flags);
2596
2597         if (IS_PF(bp))
2598                 /* must be called before memory allocation and HW init */
2599                 bnx2x_ilt_set_info(bp);
2600
2601         /*
2602          * Zero fastpath structures preserving invariants like napi, which are
2603          * allocated only once, fp index, max_cos, bp pointer.
2604          * Also set fp->disable_tpa and txdata_ptr.
2605          */
2606         DP(NETIF_MSG_IFUP, "num queues: %d", bp->num_queues);
2607         for_each_queue(bp, i)
2608                 bnx2x_bz_fp(bp, i);
2609         memset(bp->bnx2x_txq, 0, (BNX2X_MAX_RSS_COUNT(bp) * BNX2X_MULTI_TX_COS +
2610                                   bp->num_cnic_queues) *
2611                                   sizeof(struct bnx2x_fp_txdata));
2612
2613         bp->fcoe_init = false;
2614
2615         /* Set the receive queues buffer size */
2616         bnx2x_set_rx_buf_size(bp);
2617
2618         if (IS_PF(bp)) {
2619                 rc = bnx2x_alloc_mem(bp);
2620                 if (rc) {
2621                         BNX2X_ERR("Unable to allocate bp memory\n");
2622                         return rc;
2623                 }
2624         }
2625
2626         /* need to be done after alloc mem, since it's self adjusting to amount
2627          * of memory available for RSS queues
2628          */
2629         rc = bnx2x_alloc_fp_mem(bp);
2630         if (rc) {
2631                 BNX2X_ERR("Unable to allocate memory for fps\n");
2632                 LOAD_ERROR_EXIT(bp, load_error0);
2633         }
2634
2635         /* Allocated memory for FW statistics  */
2636         if (bnx2x_alloc_fw_stats_mem(bp))
2637                 LOAD_ERROR_EXIT(bp, load_error0);
2638
2639         /* request pf to initialize status blocks */
2640         if (IS_VF(bp)) {
2641                 rc = bnx2x_vfpf_init(bp);
2642                 if (rc)
2643                         LOAD_ERROR_EXIT(bp, load_error0);
2644         }
2645
2646         /* As long as bnx2x_alloc_mem() may possibly update
2647          * bp->num_queues, bnx2x_set_real_num_queues() should always
2648          * come after it. At this stage cnic queues are not counted.
2649          */
2650         rc = bnx2x_set_real_num_queues(bp, 0);
2651         if (rc) {
2652                 BNX2X_ERR("Unable to set real_num_queues\n");
2653                 LOAD_ERROR_EXIT(bp, load_error0);
2654         }
2655
2656         /* configure multi cos mappings in kernel.
2657          * this configuration may be overridden by a multi class queue
2658          * discipline or by a dcbx negotiation result.
2659          */
2660         bnx2x_setup_tc(bp->dev, bp->max_cos);
2661
2662         /* Add all NAPI objects */
2663         bnx2x_add_all_napi(bp);
2664         DP(NETIF_MSG_IFUP, "napi added\n");
2665         bnx2x_napi_enable(bp);
2666
2667         if (IS_PF(bp)) {
2668                 /* set pf load just before approaching the MCP */
2669                 bnx2x_set_pf_load(bp);
2670
2671                 /* if mcp exists send load request and analyze response */
2672                 if (!BP_NOMCP(bp)) {
2673                         /* attempt to load pf */
2674                         rc = bnx2x_nic_load_request(bp, &load_code);
2675                         if (rc)
2676                                 LOAD_ERROR_EXIT(bp, load_error1);
2677
2678                         /* what did mcp say? */
2679                         rc = bnx2x_compare_fw_ver(bp, load_code, true);
2680                         if (rc) {
2681                                 bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2682                                 LOAD_ERROR_EXIT(bp, load_error2);
2683                         }
2684                 } else {
2685                         load_code = bnx2x_nic_load_no_mcp(bp, port);
2686                 }
2687
2688                 /* mark pmf if applicable */
2689                 bnx2x_nic_load_pmf(bp, load_code);
2690
2691                 /* Init Function state controlling object */
2692                 bnx2x__init_func_obj(bp);
2693
2694                 /* Initialize HW */
2695                 rc = bnx2x_init_hw(bp, load_code);
2696                 if (rc) {
2697                         BNX2X_ERR("HW init failed, aborting\n");
2698                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2699                         LOAD_ERROR_EXIT(bp, load_error2);
2700                 }
2701         }
2702
2703         bnx2x_pre_irq_nic_init(bp);
2704
2705         /* Connect to IRQs */
2706         rc = bnx2x_setup_irqs(bp);
2707         if (rc) {
2708                 BNX2X_ERR("setup irqs failed\n");
2709                 if (IS_PF(bp))
2710                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2711                 LOAD_ERROR_EXIT(bp, load_error2);
2712         }
2713
2714         /* Init per-function objects */
2715         if (IS_PF(bp)) {
2716                 /* Setup NIC internals and enable interrupts */
2717                 bnx2x_post_irq_nic_init(bp, load_code);
2718
2719                 bnx2x_init_bp_objs(bp);
2720                 bnx2x_iov_nic_init(bp);
2721
2722                 /* Set AFEX default VLAN tag to an invalid value */
2723                 bp->afex_def_vlan_tag = -1;
2724                 bnx2x_nic_load_afex_dcc(bp, load_code);
2725                 bp->state = BNX2X_STATE_OPENING_WAIT4_PORT;
2726                 rc = bnx2x_func_start(bp);
2727                 if (rc) {
2728                         BNX2X_ERR("Function start failed!\n");
2729                         bnx2x_fw_command(bp, DRV_MSG_CODE_LOAD_DONE, 0);
2730
2731                         LOAD_ERROR_EXIT(bp, load_error3);
2732                 }
2733
2734                 /* Send LOAD_DONE command to MCP */
2735                 if (!BP_NOMCP(bp)) {
2736                         load_code = bnx2x_fw_command(bp,
2737                                                      DRV_MSG_CODE_LOAD_DONE, 0);
2738                         if (!load_code) {
2739                                 BNX2X_ERR("MCP response failure, aborting\n");
2740                                 rc = -EBUSY;
2741                                 LOAD_ERROR_EXIT(bp, load_error3);
2742                         }
2743                 }
2744
2745                 /* initialize FW coalescing state machines in RAM */
2746                 bnx2x_update_coalesce(bp);
2747         }
2748
2749         /* setup the leading queue */
2750         rc = bnx2x_setup_leading(bp);
2751         if (rc) {
2752                 BNX2X_ERR("Setup leading failed!\n");
2753                 LOAD_ERROR_EXIT(bp, load_error3);
2754         }
2755
2756         /* set up the rest of the queues */
2757         for_each_nondefault_eth_queue(bp, i) {
2758                 if (IS_PF(bp))
2759                         rc = bnx2x_setup_queue(bp, &bp->fp[i], false);
2760                 else /* VF */
2761                         rc = bnx2x_vfpf_setup_q(bp, &bp->fp[i], false);
2762                 if (rc) {
2763                         BNX2X_ERR("Queue %d setup failed\n", i);
2764                         LOAD_ERROR_EXIT(bp, load_error3);
2765                 }
2766         }
2767
2768         /* setup rss */
2769         rc = bnx2x_init_rss(bp);
2770         if (rc) {
2771                 BNX2X_ERR("PF RSS init failed\n");
2772                 LOAD_ERROR_EXIT(bp, load_error3);
2773         }
2774
2775         /* Now when Clients are configured we are ready to work */
2776         bp->state = BNX2X_STATE_OPEN;
2777
2778         /* Configure a ucast MAC */
2779         if (IS_PF(bp))
2780                 rc = bnx2x_set_eth_mac(bp, true);
2781         else /* vf */
2782                 rc = bnx2x_vfpf_config_mac(bp, bp->dev->dev_addr, bp->fp->index,
2783                                            true);
2784         if (rc) {
2785                 BNX2X_ERR("Setting Ethernet MAC failed\n");
2786                 LOAD_ERROR_EXIT(bp, load_error3);
2787         }
2788
2789         if (IS_PF(bp) && bp->pending_max) {
2790                 bnx2x_update_max_mf_config(bp, bp->pending_max);
2791                 bp->pending_max = 0;
2792         }
2793
2794         if (bp->port.pmf) {
2795                 rc = bnx2x_initial_phy_init(bp, load_mode);
2796                 if (rc)
2797                         LOAD_ERROR_EXIT(bp, load_error3);
2798         }
2799         bp->link_params.feature_config_flags &= ~FEATURE_CONFIG_BOOT_FROM_SAN;
2800
2801         /* Start fast path */
2802
2803         /* Initialize Rx filter. */
2804         bnx2x_set_rx_mode_inner(bp);
2805
2806         if (bp->flags & PTP_SUPPORTED) {
2807                 bnx2x_init_ptp(bp);
2808                 bnx2x_configure_ptp_filters(bp);
2809         }
2810         /* Start Tx */
2811         switch (load_mode) {
2812         case LOAD_NORMAL:
2813                 /* Tx queue should be only re-enabled */
2814                 netif_tx_wake_all_queues(bp->dev);
2815                 break;
2816
2817         case LOAD_OPEN:
2818                 netif_tx_start_all_queues(bp->dev);
2819                 cmb();
2820                 break;
2821
2822         case LOAD_DIAG:
2823         case LOAD_LOOPBACK_EXT:
2824                 bp->state = BNX2X_STATE_DIAG;
2825                 break;
2826
2827         default:
2828                 break;
2829         }
2830
2831         if (bp->port.pmf)
2832                 bnx2x_update_drv_flags(bp, 1 << DRV_FLAGS_PORT_MASK, 0);
2833         else
2834                 bnx2x__link_status_update(bp);
2835
2836         /* start the timer */
2837         mod_timer(&bp->timer, jiffies + bp->current_interval);
2838
2839         if (CNIC_ENABLED(bp))
2840                 bnx2x_load_cnic(bp);
2841
2842         if (IS_PF(bp))
2843                 bnx2x_schedule_sp_rtnl(bp, BNX2X_SP_RTNL_GET_DRV_VERSION, 0);
2844
2845         if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2846                 /* mark driver is loaded in shmem2 */
2847                 uint32_t val;
2848                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2849                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2850                           val | DRV_FLAGS_CAPABILITIES_LOADED_SUPPORTED |
2851                           DRV_FLAGS_CAPABILITIES_LOADED_L2);
2852         }
2853
2854         /* Wait for all pending SP commands to complete */
2855         if (IS_PF(bp) && !bnx2x_wait_sp_comp(bp, ~0x0UL)) {
2856                 BNX2X_ERR("Timeout waiting for SP elements to complete\n");
2857                 bnx2x_nic_unload(bp, UNLOAD_CLOSE, false);
2858                 return -EBUSY;
2859         }
2860
2861         /* If PMF - send ADMIN DCBX msg to MFW to initiate DCBX FSM */
2862         if (bp->port.pmf && (bp->state != BNX2X_STATE_DIAG))
2863                 bnx2x_dcbx_init(bp, false);
2864
2865         DP(NETIF_MSG_IFUP, "Ending successfully NIC load\n");
2866
2867         return 0;
2868
2869 #ifndef BNX2X_STOP_ON_ERROR
2870 load_error3:
2871         if (IS_PF(bp)) {
2872                 bnx2x_int_disable_sync(bp, 1);
2873
2874                 /* Clean queueable objects */
2875                 bnx2x_squeeze_objects(bp);
2876         }
2877
2878         /* Free SKBs, SGEs, TPA pool and driver internals */
2879         bnx2x_free_skbs(bp);
2880         for_each_rx_queue(bp, i)
2881                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
2882
2883         /* Release IRQs */
2884         bnx2x_free_irq(bp);
2885 load_error2:
2886         if (IS_PF(bp) && !BP_NOMCP(bp)) {
2887                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_REQ_WOL_MCP, 0);
2888                 bnx2x_fw_command(bp, DRV_MSG_CODE_UNLOAD_DONE, 0);
2889         }
2890
2891         bp->port.pmf = 0;
2892 load_error1:
2893         bnx2x_napi_disable(bp);
2894         bnx2x_del_all_napi(bp);
2895
2896         /* clear pf_load status, as it was already set */
2897         if (IS_PF(bp))
2898                 bnx2x_clear_pf_load(bp);
2899 load_error0:
2900         bnx2x_free_fw_stats_mem(bp);
2901         bnx2x_free_fp_mem(bp);
2902         bnx2x_free_mem(bp);
2903
2904         return rc;
2905 #endif /* ! BNX2X_STOP_ON_ERROR */
2906 }
2907
2908 int bnx2x_drain_tx_queues(struct bnx2x *bp)
2909 {
2910         uint8_t rc = 0, cos, i;
2911
2912         /* Wait until tx fastpath tasks complete */
2913         for_each_tx_queue(bp, i) {
2914                 struct bnx2x_fastpath *fp = &bp->fp[i];
2915
2916                 for_each_cos_in_tx_queue(fp, cos)
2917                         rc = bnx2x_clean_tx_queue(bp, fp->txdata_ptr[cos]);
2918                 if (rc)
2919                         return rc;
2920         }
2921         return 0;
2922 }
2923
2924 /* must be called with rtnl_lock */
2925 int bnx2x_nic_unload(struct bnx2x *bp, int unload_mode, bool keep_link)
2926 {
2927         int i;
2928         bool global = false;
2929
2930         DP(NETIF_MSG_IFUP, "Starting NIC unload\n");
2931
2932         /* mark driver is unloaded in shmem2 */
2933         if (IS_PF(bp) && SHMEM2_HAS(bp, drv_capabilities_flag)) {
2934                 uint32_t val;
2935                 val = SHMEM2_RD(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)]);
2936                 SHMEM2_WR(bp, drv_capabilities_flag[BP_FW_MB_IDX(bp)],
2937                           val & ~DRV_FLAGS_CAPABILITIES_LOADED_L2);
2938         }
2939
2940         if (IS_PF(bp) && bp->recovery_state != BNX2X_RECOVERY_DONE &&
2941             (bp->state == BNX2X_STATE_CLOSED ||
2942              bp->state == BNX2X_STATE_ERROR)) {
2943                 /* We can get here if the driver has been unloaded
2944                  * during parity error recovery and is either waiting for a
2945                  * leader to complete or for other functions to unload and
2946                  * then ifdown has been issued. In this case we want to
2947                  * unload and let other functions to complete a recovery
2948                  * process.
2949                  */
2950                 bp->recovery_state = BNX2X_RECOVERY_DONE;
2951                 bp->is_leader = 0;
2952                 bnx2x_release_leader_lock(bp);
2953                 mb();
2954
2955                 DP(NETIF_MSG_IFDOWN, "Releasing a leadership...\n");
2956                 BNX2X_ERR("Can't unload in closed or error state\n");
2957                 return -EINVAL;
2958         }
2959
2960         /* Nothing to do during unload if previous bnx2x_nic_load()
2961          * have not completed successfully - all resources are released.
2962          *
2963          * we can get here only after unsuccessful ndo_* callback, during which
2964          * dev->IFF_UP flag is still on.
2965          */
2966         if (bp->state == BNX2X_STATE_CLOSED || bp->state == BNX2X_STATE_ERROR)
2967                 return 0;
2968
2969         /* It's important to set the bp->state to the value different from
2970          * BNX2X_STATE_OPEN and only then stop the Tx. Otherwise bnx2x_tx_int()
2971          * may restart the Tx from the NAPI context (see bnx2x_tx_int()).
2972          */
2973         bp->state = BNX2X_STATE_CLOSING_WAIT4_HALT;
2974         mb();
2975
2976         /* indicate to VFs that the PF is going down */
2977         bnx2x_iov_channel_down(bp);
2978
2979         if (CNIC_LOADED(bp))
2980                 bnx2x_cnic_notify(bp, CNIC_CTL_STOP_CMD);
2981
2982         /* Stop Tx */
2983         bnx2x_tx_disable(bp);
2984         netdev_reset_tc(bp->dev);
2985
2986         bp->rx_mode = BNX2X_RX_MODE_NONE;
2987
2988         del_timer_sync(&bp->timer);
2989
2990         if (IS_PF(bp)) {
2991                 /* Set ALWAYS_ALIVE bit in shmem */
2992                 bp->fw_drv_pulse_wr_seq |= DRV_PULSE_ALWAYS_ALIVE;
2993                 bnx2x_drv_pulse(bp);
2994                 bnx2x_stats_handle(bp, STATS_EVENT_STOP);
2995                 bnx2x_save_statistics(bp);
2996         }
2997
2998         /* wait till consumers catch up with producers in all queues */
2999         bnx2x_drain_tx_queues(bp);
3000
3001         /* if VF indicate to PF this function is going down (PF will delete sp
3002          * elements and clear initializations
3003          */
3004         if (IS_VF(bp))
3005                 bnx2x_vfpf_close_vf(bp);
3006         else if (unload_mode != UNLOAD_RECOVERY)
3007                 /* if this is a normal/close unload need to clean up chip*/
3008                 bnx2x_chip_cleanup(bp, unload_mode, keep_link);
3009         else {
3010                 /* Send the UNLOAD_REQUEST to the MCP */
3011                 bnx2x_send_unload_req(bp, unload_mode);
3012
3013                 /* Prevent transactions to host from the functions on the
3014                  * engine that doesn't reset global blocks in case of global
3015                  * attention once global blocks are reset and gates are opened
3016                  * (the engine which leader will perform the recovery
3017                  * last).
3018                  */
3019                 if (!CHIP_IS_E1x(bp))
3020                         bnx2x_pf_disable(bp);
3021
3022                 /* Disable HW interrupts, NAPI */
3023                 bnx2x_netif_stop(bp, 1);
3024                 /* Delete all NAPI objects */
3025                 bnx2x_del_all_napi(bp);
3026                 if (CNIC_LOADED(bp))
3027                         bnx2x_del_all_napi_cnic(bp);
3028                 /* Release IRQs */
3029                 bnx2x_free_irq(bp);
3030
3031                 /* Report UNLOAD_DONE to MCP */
3032                 bnx2x_send_unload_done(bp, false);
3033         }
3034
3035         /*
3036          * At this stage no more interrupts will arrive so we may safely clean
3037          * the queueable objects here in case they failed to get cleaned so far.
3038          */
3039         if (IS_PF(bp))
3040                 bnx2x_squeeze_objects(bp);
3041
3042         /* There should be no more pending SP commands at this stage */
3043         bp->sp_state = 0;
3044
3045         bp->port.pmf = 0;
3046
3047         /* clear pending work in rtnl task */
3048         bp->sp_rtnl_state = 0;
3049         mb();
3050
3051         /* Free SKBs, SGEs, TPA pool and driver internals */
3052         bnx2x_free_skbs(bp);
3053         if (CNIC_LOADED(bp))
3054                 bnx2x_free_skbs_cnic(bp);
3055         for_each_rx_queue(bp, i)
3056                 bnx2x_free_rx_sge_range(bp, bp->fp + i, NUM_RX_SGE);
3057
3058         bnx2x_free_fp_mem(bp);
3059         if (CNIC_LOADED(bp))
3060                 bnx2x_free_fp_mem_cnic(bp);
3061
3062         if (IS_PF(bp)) {
3063                 if (CNIC_LOADED(bp))
3064                         bnx2x_free_mem_cnic(bp);
3065         }
3066         bnx2x_free_mem(bp);
3067
3068         bp->state = BNX2X_STATE_CLOSED;
3069         bp->cnic_loaded = false;
3070
3071         /* Clear driver version indication in shmem */
3072         if (IS_PF(bp))
3073                 bnx2x_update_mng_version(bp);
3074
3075         /* Check if there are pending parity attentions. If there are - set
3076          * RECOVERY_IN_PROGRESS.
3077          */
3078         if (IS_PF(bp) && bnx2x_chk_parity_attn(bp, &global, false)) {
3079                 bnx2x_set_reset_in_progress(bp);
3080
3081                 /* Set RESET_IS_GLOBAL if needed */
3082                 if (global)
3083                         bnx2x_set_reset_global(bp);
3084         }
3085
3086         /* The last driver must disable a "close the gate" if there is no
3087          * parity attention or "process kill" pending.
3088          */
3089         if (IS_PF(bp) &&
3090             !bnx2x_clear_pf_load(bp) &&
3091             bnx2x_reset_is_done(bp, BP_PATH(bp)))
3092                 bnx2x_disable_close_the_gate(bp);
3093
3094         DP(NETIF_MSG_IFUP, "Ending NIC unload\n");
3095
3096         return 0;
3097 }
3098
3099 int bnx2x_set_power_state(struct bnx2x *bp, pci_power_t state)
3100 {
3101         uint16_t pmcsr;
3102
3103         /* If there is no power capability, silently succeed */
3104         if (!bp->pdev->pm_cap) {
3105                 BNX2X_DEV_INFO("No power capability. Breaking.\n");
3106                 return 0;
3107         }
3108
3109         pci_read_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL, &pmcsr);
3110
3111         switch (state) {
3112         case PCI_D0:
3113                 pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
3114                                       ((pmcsr & ~PCI_PM_CTRL_STATE_MASK) |
3115                                        PCI_PM_CTRL_PME_STATUS));
3116
3117                 if (pmcsr & PCI_PM_CTRL_STATE_MASK)
3118                         /* delay required during transition out of D3hot */
3119                         kthread_usleep(1000 * 20);
3120                 break;
3121
3122         case PCI_D3hot:
3123                 /* If there are other clients above don't
3124                    shut down the power */
3125                 if (atomic_read(&bp->pdev->enable_cnt) != 1)
3126                         return 0;
3127                 /* Don't shut down the power for emulation and FPGA */
3128                 if (CHIP_REV_IS_SLOW(bp))
3129                         return 0;
3130
3131                 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
3132                 pmcsr |= 3;
3133
3134                 if (bp->wol)
3135                         pmcsr |= PCI_PM_CTRL_PME_ENABLE;
3136
3137                 pci_write_config_word(bp->pdev, bp->pdev->pm_cap + PCI_PM_CTRL,
3138                                       pmcsr);
3139
3140                 /* No more memory access after this point until
3141                 * device is brought back to D0.
3142                 */
3143                 break;
3144
3145         default:
3146                 dev_err(&bp->pdev->dev, "Can't support state = %d\n", state);
3147                 return -EINVAL;
3148         }
3149         return 0;
3150 }
3151
3152 /*
3153  * net_device service functions
3154  */
3155 static int bnx2x_poll(struct napi_struct *napi, int budget)
3156 {
3157         int work_done = 0;
3158         uint8_t cos;
3159         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3160                                                  napi);
3161         struct bnx2x *bp = fp->bp;
3162
3163         while (1) {
3164 #ifdef BNX2X_STOP_ON_ERROR
3165                 if (unlikely(bp->panic)) {
3166                         napi_complete(napi);
3167                         return 0;
3168                 }
3169 #endif
3170                 if (!bnx2x_fp_lock_napi(fp))
3171                         return budget;
3172
3173                 for_each_cos_in_tx_queue(fp, cos)
3174                         if (bnx2x_tx_queue_has_work(fp->txdata_ptr[cos]))
3175                                 bnx2x_tx_int(bp, fp->txdata_ptr[cos]);
3176
3177                 if (bnx2x_has_rx_work(fp)) {
3178                         work_done += bnx2x_rx_int(fp, budget - work_done);
3179
3180                         /* must not complete if we consumed full budget */
3181                         if (work_done >= budget) {
3182                                 bnx2x_fp_unlock_napi(fp);
3183                                 break;
3184                         }
3185                 }
3186
3187                 /* Fall out from the NAPI loop if needed */
3188                 if (!bnx2x_fp_unlock_napi(fp) &&
3189                     !(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3190
3191                         /* No need to update SB for FCoE L2 ring as long as
3192                          * it's connected to the default SB and the SB
3193                          * has been updated when NAPI was scheduled.
3194                          */
3195                         if (IS_FCOE_FP(fp)) {
3196                                 napi_complete(napi);
3197                                 break;
3198                         }
3199                         bnx2x_update_fpsb_idx(fp);
3200                         /* bnx2x_has_rx_work() reads the status block,
3201                          * thus we need to ensure that status block indices
3202                          * have been actually read (bnx2x_update_fpsb_idx)
3203                          * prior to this check (bnx2x_has_rx_work) so that
3204                          * we won't write the "newer" value of the status block
3205                          * to IGU (if there was a DMA right after
3206                          * bnx2x_has_rx_work and if there is no rmb, the memory
3207                          * reading (bnx2x_update_fpsb_idx) may be postponed
3208                          * to right before bnx2x_ack_sb). In this case there
3209                          * will never be another interrupt until there is
3210                          * another update of the status block, while there
3211                          * is still unhandled work.
3212                          */
3213                         rmb();
3214
3215                         if (!(bnx2x_has_rx_work(fp) || bnx2x_has_tx_work(fp))) {
3216                                 napi_complete(napi);
3217                                 /* Re-enable interrupts */
3218                                 DP(NETIF_MSG_RX_STATUS,
3219                                    "Update index to %d\n", fp->fp_hc_idx);
3220                                 bnx2x_ack_sb(bp, fp->igu_sb_id, USTORM_ID,
3221                                              le16_to_cpu(fp->fp_hc_idx),
3222                                              IGU_INT_ENABLE, 1);
3223                                 break;
3224                         }
3225                 }
3226         }
3227
3228         return work_done;
3229 }
3230
3231 #ifdef CONFIG_NET_RX_BUSY_POLL
3232 /* must be called with local_bh_disable()d */
3233 int bnx2x_low_latency_recv(struct napi_struct *napi)
3234 {
3235         struct bnx2x_fastpath *fp = container_of(napi, struct bnx2x_fastpath,
3236                                                  napi);
3237         struct bnx2x *bp = fp->bp;
3238         int found = 0;
3239
3240         if ((bp->state == BNX2X_STATE_CLOSED) ||
3241             (bp->state == BNX2X_STATE_ERROR) ||
3242             (bp->flags & (TPA_ENABLE_FLAG | GRO_ENABLE_FLAG)))
3243                 return LL_FLUSH_FAILED;
3244
3245         if (!bnx2x_fp_lock_poll(fp))
3246                 return LL_FLUSH_BUSY;
3247
3248         if (bnx2x_has_rx_work(fp))
3249                 found = bnx2x_rx_int(fp, 4);
3250
3251         bnx2x_fp_unlock_poll(fp);
3252
3253         return found;
3254 }
3255 #endif
3256
3257 /* we split the first BD into headers and data BDs
3258  * to ease the pain of our fellow microcode engineers
3259  * we use one mapping for both BDs
3260  */
3261 static uint16_t bnx2x_tx_split(struct bnx2x *bp,
3262                           struct bnx2x_fp_txdata *txdata,
3263                           struct sw_tx_bd *tx_buf,
3264                           struct eth_tx_start_bd **tx_bd, uint16_t hlen,
3265                           uint16_t bd_prod)
3266 {
3267         struct eth_tx_start_bd *h_tx_bd = *tx_bd;
3268         struct eth_tx_bd *d_tx_bd;
3269         dma_addr_t mapping;
3270         int old_len = le16_to_cpu(h_tx_bd->nbytes);
3271
3272         /* first fix first BD */
3273         h_tx_bd->nbytes = cpu_to_le16(hlen);
3274
3275         DP(NETIF_MSG_TX_QUEUED, "TSO split header size is %d (%x:%x)\n",
3276            h_tx_bd->nbytes, h_tx_bd->addr_hi, h_tx_bd->addr_lo);
3277
3278         /* now get a new data BD
3279          * (after the pbd) and fill it */
3280         bd_prod = TX_BD(NEXT_TX_IDX(bd_prod));
3281         d_tx_bd = &txdata->tx_desc_ring[bd_prod].reg_bd;
3282
3283         mapping = HILO_U64(le32_to_cpu(h_tx_bd->addr_hi),
3284                            le32_to_cpu(h_tx_bd->addr_lo)) + hlen;
3285
3286         d_tx_bd->addr_hi = cpu_to_le32(U64_HI(mapping));
3287         d_tx_bd->addr_lo = cpu_to_le32(U64_LO(mapping));
3288         d_tx_bd->nbytes = cpu_to_le16(old_len - hlen);
3289
3290         /* this marks the BD as one that has no individual mapping */
3291         tx_buf->flags |= BNX2X_TSO_SPLIT_BD;
3292
3293         DP(NETIF_MSG_TX_QUEUED,
3294            "TSO split data size is %d (%x:%x)\n",
3295            d_tx_bd->nbytes, d_tx_bd->addr_hi, d_tx_bd->addr_lo);
3296
3297         /* update tx_bd */
3298         *tx_bd = (struct eth_tx_start_bd *)d_tx_bd;
3299
3300         return bd_prod;
3301 }
3302
3303 #define bswab32(b32) ((__force __le32) swab32((__force __u32) (b32)))
3304 #define bswab16(b16) ((__force __le16) swab16((__force __u16) (b16)))
3305 static __le16 bnx2x_csum_fix(unsigned char *t_header, uint16_t csum, s8 fix)
3306 {
3307         __sum16 tsum = (__force __sum16) csum;
3308
3309         if (fix > 0)
3310                 tsum = ~csum_fold(csum_sub((__force __wsum) csum,
3311                                   csum_partial(t_header - fix, fix, 0)));
3312
3313         else if (fix < 0)
3314                 tsum = ~csum_fold(csum_add((__force __wsum) csum,
3315                                   csum_partial(t_header, -fix, 0)));
3316
3317         return bswab16(tsum);
3318 }
3319
3320 static uint32_t bnx2x_xmit_type(struct bnx2x *bp, struct sk_buff *skb)
3321 {
3322         uint32_t rc;
3323         __u8 prot = 0;
3324         __be16 protocol;
3325
3326         if (skb->ip_summed != CHECKSUM_PARTIAL)
3327                 return XMIT_PLAIN;
3328
3329         protocol = vlan_get_protocol(skb);
3330         if (protocol == htons(ETH_P_IPV6)) {
3331                 rc = XMIT_CSUM_V6;
3332                 prot = ipv6_hdr(skb)->nexthdr;
3333         } else {
3334                 rc = XMIT_CSUM_V4;
3335                 prot = ip_hdr(skb)->protocol;
3336         }
3337
3338         if (!CHIP_IS_E1x(bp) && skb->encapsulation) {
3339                 if (inner_ip_hdr(skb)->version == 6) {
3340                         rc |= XMIT_CSUM_ENC_V6;
3341                         if (inner_ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
3342                                 rc |= XMIT_CSUM_TCP;
3343                 } else {
3344                         rc |= XMIT_CSUM_ENC_V4;
3345                         if (inner_ip_hdr(skb)->protocol == IPPROTO_TCP)
3346                                 rc |= XMIT_CSUM_TCP;
3347                 }
3348         }
3349         if (prot == IPPROTO_TCP)
3350                 rc |= XMIT_CSUM_TCP;
3351
3352         if (skb_is_gso(skb)) {
3353                 if (skb_is_gso_v6(skb)) {
3354                         rc |= (XMIT_GSO_V6 | XMIT_CSUM_TCP);
3355                         if (rc & XMIT_CSUM_ENC)
3356                                 rc |= XMIT_GSO_ENC_V6;
3357                 } else {
3358                         rc |= (XMIT_GSO_V4 | XMIT_CSUM_TCP);
3359                         if (rc & XMIT_CSUM_ENC)
3360                                 rc |= XMIT_GSO_ENC_V4;
3361                 }
3362         }
3363
3364         return rc;
3365 }
3366
3367 #if (MAX_SKB_FRAGS >= MAX_FETCH_BD - 3)
3368 /* check if packet requires linearization (packet is too fragmented)
3369    no need to check fragmentation if page size > 8K (there will be no
3370    violation to FW restrictions) */
3371 static int bnx2x_pkt_req_lin(struct bnx2x *bp, struct sk_buff *skb,
3372                              uint32_t xmit_type)
3373 {
3374         int to_copy = 0;
3375         int hlen = 0;
3376         int first_bd_sz = 0;
3377
3378         /* 3 = 1 (for linear data BD) + 2 (for PBD and last BD) */
3379         if (skb_shinfo(skb)->nr_frags >= (MAX_FETCH_BD - 3)) {
3380
3381                 if (xmit_type & XMIT_GSO) {
3382                         unsigned short lso_mss = skb_shinfo(skb)->gso_size;
3383                         /* Check if LSO packet needs to be copied:
3384                            3 = 1 (for headers BD) + 2 (for PBD and last BD) */
3385                         int wnd_size = MAX_FETCH_BD - 3;
3386                         /* Number of windows to check */
3387                         int num_wnds = skb_shinfo(skb)->nr_frags - wnd_size;
3388                         int wnd_idx = 0;
3389                         int frag_idx = 0;
3390                         uint32_t wnd_sum = 0;
3391
3392                         /* Headers length */
3393                         hlen = (int)(skb_transport_header(skb) - skb->data) +
3394                                 tcp_hdrlen(skb);
3395
3396                         /* Amount of data (w/o headers) on linear part of SKB*/
3397                         first_bd_sz = skb_headlen(skb) - hlen;
3398
3399                         wnd_sum  = first_bd_sz;
3400
3401                         /* Calculate the first sum - it's special */
3402                         for (frag_idx = 0; frag_idx < wnd_size - 1; frag_idx++)
3403                                 wnd_sum +=
3404                                         skb_frag_size(&skb_shinfo(skb)->frags[frag_idx]);
3405
3406                         /* If there was data on linear skb data - check it */
3407                         if (first_bd_sz > 0) {
3408                                 if (unlikely(wnd_sum < lso_mss)) {
3409                                         to_copy = 1;
3410                                         goto exit_lbl;
3411                                 }
3412
3413                                 wnd_sum -= first_bd_sz;
3414                         }
3415
3416                         /* Others are easier: run through the frag list and
3417                            check all windows */
3418                         for (wnd_idx = 0; wnd_idx <= num_wnds; wnd_idx++) {
3419                                 wnd_sum +=
3420                           skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx + wnd_size - 1]);
3421
3422                                 if (unlikely(wnd_sum < lso_mss)) {
3423                                         to_copy = 1;
3424                                         break;
3425                                 }
3426                                 wnd_sum -=
3427                                         skb_frag_size(&skb_shinfo(skb)->frags[wnd_idx]);
3428                         }
3429                 } else {
3430                         /* in non-LSO too fragmented packet should always
3431                            be linearized */
3432                         to_copy = 1;
3433                 }
3434         }
3435
3436 exit_lbl:
3437         if (unlikely(to_copy))
3438                 DP(NETIF_MSG_TX_QUEUED,
3439                    "Linearization IS REQUIRED for %s packet. num_frags %d  hlen %d  first_bd_sz %d\n",
3440                    (xmit_type & XMIT_GSO) ? "LSO" : "non-LSO",
3441                    skb_shinfo(skb)->nr_frags, hlen, first_bd_sz);
3442
3443         return to_copy;
3444 }
3445 #endif
3446
3447 /**
3448  * bnx2x_set_pbd_gso - update PBD in GSO case.
3449  *
3450  * @skb:        packet skb
3451  * @pbd:        parse BD
3452  * @xmit_type:  xmit flags
3453  */
3454 static void bnx2x_set_pbd_gso(struct sk_buff *skb,
3455                               struct eth_tx_parse_bd_e1x *pbd,
3456                               uint32_t xmit_type)
3457 {
3458         pbd->lso_mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
3459         pbd->tcp_send_seq = bswab32(tcp_hdr(skb)->seq);
3460         pbd->tcp_flags = pbd_tcp_flags(tcp_hdr(skb));
3461
3462         if (xmit_type & XMIT_GSO_V4) {
3463                 pbd->ip_id = bswab16(ip_hdr(skb)->id);
3464                 pbd->tcp_pseudo_csum =
3465                         bswab16(~csum_tcpudp_magic(ip_hdr(skb)->saddr,
3466                                                    ip_hdr(skb)->daddr,
3467                                                    0, IPPROTO_TCP, 0));
3468         } else {
3469                 pbd->tcp_pseudo_csum =
3470                         bswab16(~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3471                                                  &ipv6_hdr(skb)->daddr,
3472                                                  0, IPPROTO_TCP, 0));
3473         }
3474
3475         pbd->global_data |=
3476                 cpu_to_le16(ETH_TX_PARSE_BD_E1X_PSEUDO_CS_WITHOUT_LEN);
3477 }
3478
3479 /**
3480  * bnx2x_set_pbd_csum_enc - update PBD with checksum and return header length
3481  *
3482  * @bp:                 driver handle
3483  * @skb:                packet skb
3484  * @parsing_data:       data to be updated
3485  * @xmit_type:          xmit flags
3486  *
3487  * 57712/578xx related, when skb has encapsulation
3488  */
3489 static uint8_t bnx2x_set_pbd_csum_enc(struct bnx2x *bp, struct sk_buff *skb,
3490                                  uint32_t *parsing_data, uint32_t xmit_type)
3491 {
3492         *parsing_data |=
3493                 ((((uint8_t *)skb_inner_transport_header(skb) - skb->data) >> 1) <<
3494                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3495                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3496
3497         if (xmit_type & XMIT_CSUM_TCP) {
3498                 *parsing_data |= ((inner_tcp_hdrlen(skb) / 4) <<
3499                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3500                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3501
3502                 return skb_inner_transport_header(skb) +
3503                         inner_tcp_hdrlen(skb) - skb->data;
3504         }
3505
3506         /* We support checksum offload for TCP and UDP only.
3507          * No need to pass the UDP header length - it's a constant.
3508          */
3509         return skb_inner_transport_header(skb) +
3510                 sizeof(struct udphdr) - skb->data;
3511 }
3512
3513 /**
3514  * bnx2x_set_pbd_csum_e2 - update PBD with checksum and return header length
3515  *
3516  * @bp:                 driver handle
3517  * @skb:                packet skb
3518  * @parsing_data:       data to be updated
3519  * @xmit_type:          xmit flags
3520  *
3521  * 57712/578xx related
3522  */
3523 static uint8_t bnx2x_set_pbd_csum_e2(struct bnx2x *bp, struct sk_buff *skb,
3524                                 uint32_t *parsing_data, uint32_t xmit_type)
3525 {
3526         *parsing_data |=
3527                 ((((uint8_t *)skb_transport_header(skb) - skb->data) >> 1) <<
3528                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W_SHIFT) &
3529                 ETH_TX_PARSE_BD_E2_L4_HDR_START_OFFSET_W;
3530
3531         if (xmit_type & XMIT_CSUM_TCP) {
3532                 *parsing_data |= ((tcp_hdrlen(skb) / 4) <<
3533                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW_SHIFT) &
3534                         ETH_TX_PARSE_BD_E2_TCP_HDR_LENGTH_DW;
3535
3536                 return skb_transport_header(skb) + tcp_hdrlen(skb) - skb->data;
3537         }
3538         /* We support checksum offload for TCP and UDP only.
3539          * No need to pass the UDP header length - it's a constant.
3540          */
3541         return skb_transport_header(skb) + sizeof(struct udphdr) - skb->data;
3542 }
3543
3544 /* set FW indication according to inner or outer protocols if tunneled */
3545 static void bnx2x_set_sbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3546                                struct eth_tx_start_bd *tx_start_bd,
3547                                uint32_t xmit_type)
3548 {
3549         tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_L4_CSUM;
3550
3551         if (xmit_type & (XMIT_CSUM_ENC_V6 | XMIT_CSUM_V6))
3552                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IPV6;
3553
3554         if (!(xmit_type & XMIT_CSUM_TCP))
3555                 tx_start_bd->bd_flags.as_bitfield |= ETH_TX_BD_FLAGS_IS_UDP;
3556 }
3557
3558 /**
3559  * bnx2x_set_pbd_csum - update PBD with checksum and return header length
3560  *
3561  * @bp:         driver handle
3562  * @skb:        packet skb
3563  * @pbd:        parse BD to be updated
3564  * @xmit_type:  xmit flags
3565  */
3566 static uint8_t bnx2x_set_pbd_csum(struct bnx2x *bp, struct sk_buff *skb,
3567                              struct eth_tx_parse_bd_e1x *pbd,
3568                              uint32_t xmit_type)
3569 {
3570         uint8_t hlen = (skb_network_header(skb) - skb->data) >> 1;
3571
3572         /* for now NS flag is not used in Linux */
3573         pbd->global_data =
3574                 cpu_to_le16(hlen |
3575                             ((skb->protocol == cpu_to_be16(ETH_P_8021Q)) <<
3576                              ETH_TX_PARSE_BD_E1X_LLC_SNAP_EN_SHIFT));
3577
3578         pbd->ip_hlen_w = (skb_transport_header(skb) -
3579                         skb_network_header(skb)) >> 1;
3580
3581         hlen += pbd->ip_hlen_w;
3582
3583         /* We support checksum offload for TCP and UDP only */
3584         if (xmit_type & XMIT_CSUM_TCP)
3585                 hlen += tcp_hdrlen(skb) / 2;
3586         else
3587                 hlen += sizeof(struct udphdr) / 2;
3588
3589         pbd->total_hlen_w = cpu_to_le16(hlen);
3590         hlen = hlen*2;
3591
3592         if (xmit_type & XMIT_CSUM_TCP) {
3593                 pbd->tcp_pseudo_csum = bswab16(tcp_hdr(skb)->check);
3594
3595         } else {
3596                 s8 fix = SKB_CS_OFF(skb); /* signed! */
3597
3598                 DP(NETIF_MSG_TX_QUEUED,
3599                    "hlen %d  fix %d  csum before fix %x\n",
3600                    le16_to_cpu(pbd->total_hlen_w), fix, SKB_CS(skb));
3601
3602                 /* HW bug: fixup the CSUM */
3603                 pbd->tcp_pseudo_csum =
3604                         bnx2x_csum_fix(skb_transport_header(skb),
3605                                        SKB_CS(skb), fix);
3606
3607                 DP(NETIF_MSG_TX_QUEUED, "csum after fix %x\n",
3608                    pbd->tcp_pseudo_csum);
3609         }
3610
3611         return hlen;
3612 }
3613
3614 static void bnx2x_update_pbds_gso_enc(struct sk_buff *skb,
3615                                       struct eth_tx_parse_bd_e2 *pbd_e2,
3616                                       struct eth_tx_parse_2nd_bd *pbd2,
3617                                       uint16_t *global_data,
3618                                       uint32_t xmit_type)
3619 {
3620         uint16_t hlen_w = 0;
3621         uint8_t outerip_off, outerip_len = 0;
3622
3623         /* from outer IP to transport */
3624         hlen_w = (skb_inner_transport_header(skb) -
3625                   skb_network_header(skb)) >> 1;
3626
3627         /* transport len */
3628         hlen_w += inner_tcp_hdrlen(skb) >> 1;
3629
3630         pbd2->fw_ip_hdr_to_payload_w = hlen_w;
3631
3632         /* outer IP header info */
3633         if (xmit_type & XMIT_CSUM_V4) {
3634                 struct iphdr *iph = ip_hdr(skb);
3635                 uint32_t csum = (__force uint32_t)(~iph->check) -
3636                            (__force uint32_t)iph->tot_len -
3637                            (__force uint32_t)iph->frag_off;
3638
3639                 outerip_len = iph->ihl << 1;
3640
3641                 pbd2->fw_ip_csum_wo_len_flags_frag =
3642                         bswab16(csum_fold((__force __wsum)csum));
3643         } else {
3644                 pbd2->fw_ip_hdr_to_payload_w =
3645                         hlen_w - ((sizeof(struct ipv6hdr)) >> 1);
3646                 pbd_e2->data.tunnel_data.flags |=
3647                         ETH_TUNNEL_DATA_IP_HDR_TYPE_OUTER;
3648         }
3649
3650         pbd2->tcp_send_seq = bswab32(inner_tcp_hdr(skb)->seq);