Port over linux 4.1.15 mlx4 kernel bypass driver
[akaros.git] / kern / drivers / net / mlx4u / main.c
1 /*
2  * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved.
3  * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
4  *
5  * This software is available to you under a choice of one of two
6  * licenses.  You may choose to be licensed under the terms of the GNU
7  * General Public License (GPL) Version 2, available from the file
8  * COPYING in the main directory of this source tree, or the
9  * OpenIB.org BSD license below:
10  *
11  *     Redistribution and use in source and binary forms, with or
12  *     without modification, are permitted provided that the following
13  *     conditions are met:
14  *
15  *      - Redistributions of source code must retain the above
16  *        copyright notice, this list of conditions and the following
17  *        disclaimer.
18  *
19  *      - Redistributions in binary form must reproduce the above
20  *        copyright notice, this list of conditions and the following
21  *        disclaimer in the documentation and/or other materials
22  *        provided with the distribution.
23  *
24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31  * SOFTWARE.
32  */
33
34 #if 1   /* AKAROS */
35 #include <linux/rdma/ib_smi.h>
36 #include <linux/rdma/ib_verbs.h>
37 #include <linux/rdma/ib_user_verbs.h>
38 #else   /* AKAROS */
39
40 #include <linux/module.h>
41 #include <linux/init.h>
42 #include <linux/slab.h>
43 #include <linux/errno.h>
44 #include <linux/netdevice.h>
45 #include <linux/inetdevice.h>
46 #include <linux/rtnetlink.h>
47 #include <linux/if_vlan.h>
48 #include <net/ipv6.h>
49 #include <net/addrconf.h>
50
51 #include <rdma/ib_smi.h>
52 #include <rdma/ib_user_verbs.h>
53 #include <rdma/ib_addr.h>
54 #endif  /* AKAROS */
55
56 #include <linux/mlx4/driver.h>
57 #include <linux/mlx4/cmd.h>
58 #include <linux/mlx4/qp.h>
59
60 #include "mlx4_ib.h"
61 #include "user.h"
62
63 #if 1   /* AKAROS */
64
65 #define netdev_master_upper_dev_get(p)  p
66 #define iboe_get_mtu(v)                 IB_MTU_1024     /* TODO */
67
68 /* STUB START */
69 int mlx4_ib_init_sriov(struct mlx4_ib_dev *dev) { return 0; }
70 int mlx4_ib_mad_init(struct mlx4_ib_dev *dev) { return 0; }
71 void mlx4_ib_close_sriov(struct mlx4_ib_dev *dev) {}
72 void mlx4_ib_mad_cleanup(struct mlx4_ib_dev *dev) {}
73 void mlx4_ib_mcg_destroy() { }
74 int mlx4_ib_mcg_init(void) { return 0; }
75 void mlx4_ib_invalidate_all_guid_record(struct mlx4_ib_dev *dev, int port) {}
76 void mlx4_ib_slave_alias_guid_event(struct mlx4_ib_dev *dev, int slave,
77     int port, int slave_init) {}
78 void handle_port_mgmt_change_event(struct work_struct *work){}
79 void mlx4_ib_tunnels_update_work(struct work_struct *work) {}
80 int mlx4_MAD_IFC(struct mlx4_ib_dev *dev, int mad_ifc_flags, int port,
81     struct ib_wc *in_wc, struct ib_grh *in_grh, void *in_mad,
82     void *response_mad) { return 0; }
83 /* STUB END */
84
85 #endif  /* AKAROS */
86
87 #define DRV_NAME        MLX4_IB_DRV_NAME
88 #define DRV_VERSION     "2.2-1"
89 #define DRV_RELDATE     "Feb 2014"
90
91 #define MLX4_IB_FLOW_MAX_PRIO 0xFFF
92 #define MLX4_IB_FLOW_QPN_MASK 0xFFFFFF
93 #define MLX4_IB_CARD_REV_A0   0xA0
94
95 MODULE_AUTHOR("Roland Dreier");
96 MODULE_DESCRIPTION("Mellanox ConnectX HCA InfiniBand driver");
97 MODULE_LICENSE("Dual BSD/GPL");
98 MODULE_VERSION(DRV_VERSION);
99
100 int mlx4_ib_sm_guid_assign = 0;
101 module_param_named(sm_guid_assign, mlx4_ib_sm_guid_assign, int, 0444);
102 MODULE_PARM_DESC(sm_guid_assign, "Enable SM alias_GUID assignment if sm_guid_assign > 0 (Default: 0)");
103
104 static const char mlx4_ib_version[] =
105         DRV_NAME ": Mellanox ConnectX InfiniBand driver v"
106         DRV_VERSION " (" DRV_RELDATE ")\n";
107
108 struct update_gid_work {
109         struct work_struct      work;
110         union ib_gid            gids[128];
111         struct mlx4_ib_dev     *dev;
112         int                     port;
113 };
114
115 static void do_slave_init(struct mlx4_ib_dev *ibdev, int slave, int do_init);
116
117 static struct workqueue_struct *wq;
118
119 static void init_query_mad(struct ib_smp *mad)
120 {
121         mad->base_version  = 1;
122         mad->mgmt_class    = IB_MGMT_CLASS_SUBN_LID_ROUTED;
123         mad->class_version = 1;
124         mad->method        = IB_MGMT_METHOD_GET;
125 }
126
127 static union ib_gid zgid;
128
129 static int check_flow_steering_support(struct mlx4_dev *dev)
130 {
131         int eth_num_ports = 0;
132         int ib_num_ports = 0;
133
134         int dmfs = dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED;
135
136         if (dmfs) {
137                 int i;
138                 mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_ETH)
139                         eth_num_ports++;
140                 mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_IB)
141                         ib_num_ports++;
142                 dmfs &= (!ib_num_ports ||
143                          (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_DMFS_IPOIB)) &&
144                         (!eth_num_ports ||
145                          (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_FS_EN));
146                 if (ib_num_ports && mlx4_is_mfunc(dev)) {
147                         pr_warn("Device managed flow steering is unavailable for IB port in multifunction env.\n");
148                         dmfs = 0;
149                 }
150         }
151         return dmfs;
152 }
153
154 static int num_ib_ports(struct mlx4_dev *dev)
155 {
156         int ib_ports = 0;
157         int i;
158
159         mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_IB)
160                 ib_ports++;
161
162         return ib_ports;
163 }
164
165 static int mlx4_ib_query_device(struct ib_device *ibdev,
166                                 struct ib_device_attr *props)
167 {
168         struct mlx4_ib_dev *dev = to_mdev(ibdev);
169         struct ib_smp *in_mad  = NULL;
170         struct ib_smp *out_mad = NULL;
171         int err = -ENOMEM;
172         int have_ib_ports;
173
174         in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
175         out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
176         if (!in_mad || !out_mad)
177                 goto out;
178
179         init_query_mad(in_mad);
180         in_mad->attr_id = IB_SMP_ATTR_NODE_INFO;
181
182         err = mlx4_MAD_IFC(to_mdev(ibdev), MLX4_MAD_IFC_IGNORE_KEYS,
183                            1, NULL, NULL, in_mad, out_mad);
184         if (err)
185                 goto out;
186
187         memset(props, 0, sizeof *props);
188
189         have_ib_ports = num_ib_ports(dev->dev);
190
191         props->fw_ver = dev->dev->caps.fw_ver;
192         props->device_cap_flags    = IB_DEVICE_CHANGE_PHY_PORT |
193                 IB_DEVICE_PORT_ACTIVE_EVENT             |
194                 IB_DEVICE_SYS_IMAGE_GUID                |
195                 IB_DEVICE_RC_RNR_NAK_GEN                |
196                 IB_DEVICE_BLOCK_MULTICAST_LOOPBACK;
197         if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_BAD_PKEY_CNTR)
198                 props->device_cap_flags |= IB_DEVICE_BAD_PKEY_CNTR;
199         if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_BAD_QKEY_CNTR)
200                 props->device_cap_flags |= IB_DEVICE_BAD_QKEY_CNTR;
201         if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_APM && have_ib_ports)
202                 props->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG;
203         if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_UD_AV_PORT)
204                 props->device_cap_flags |= IB_DEVICE_UD_AV_PORT_ENFORCE;
205         if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_IPOIB_CSUM)
206                 props->device_cap_flags |= IB_DEVICE_UD_IP_CSUM;
207         if (dev->dev->caps.max_gso_sz &&
208             (dev->dev->rev_id != MLX4_IB_CARD_REV_A0) &&
209             (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_BLH))
210                 props->device_cap_flags |= IB_DEVICE_UD_TSO;
211         if (dev->dev->caps.bmme_flags & MLX4_BMME_FLAG_RESERVED_LKEY)
212                 props->device_cap_flags |= IB_DEVICE_LOCAL_DMA_LKEY;
213         if ((dev->dev->caps.bmme_flags & MLX4_BMME_FLAG_LOCAL_INV) &&
214             (dev->dev->caps.bmme_flags & MLX4_BMME_FLAG_REMOTE_INV) &&
215             (dev->dev->caps.bmme_flags & MLX4_BMME_FLAG_FAST_REG_WR))
216                 props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
217         if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC)
218                 props->device_cap_flags |= IB_DEVICE_XRC;
219         if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_MEM_WINDOW)
220                 props->device_cap_flags |= IB_DEVICE_MEM_WINDOW;
221         if (dev->dev->caps.bmme_flags & MLX4_BMME_FLAG_TYPE_2_WIN) {
222                 if (dev->dev->caps.bmme_flags & MLX4_BMME_FLAG_WIN_TYPE_2B)
223                         props->device_cap_flags |= IB_DEVICE_MEM_WINDOW_TYPE_2B;
224                 else
225                         props->device_cap_flags |= IB_DEVICE_MEM_WINDOW_TYPE_2A;
226         if (dev->steering_support ==  MLX4_STEERING_MODE_DEVICE_MANAGED)
227                 props->device_cap_flags |= IB_DEVICE_MANAGED_FLOW_STEERING;
228         }
229
230         props->vendor_id           = be32_to_cpup((__be32 *) (out_mad->data + 36)) &
231                 0xffffff;
232 #if 0   /* AKAROS */
233         props->vendor_part_id      = dev->dev->persist->pdev->device;
234 #endif  /* AKAROS */
235         props->hw_ver              = be32_to_cpup((__be32 *) (out_mad->data + 32));
236         memcpy(&props->sys_image_guid, out_mad->data +  4, 8);
237
238         props->max_mr_size         = ~0ull;
239         props->page_size_cap       = dev->dev->caps.page_size_cap;
240         props->max_qp              = dev->dev->quotas.qp;
241         props->max_qp_wr           = dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE;
242         props->max_sge             = min(dev->dev->caps.max_sq_sg,
243                                          dev->dev->caps.max_rq_sg);
244         props->max_cq              = dev->dev->quotas.cq;
245         props->max_cqe             = dev->dev->caps.max_cqes;
246         props->max_mr              = dev->dev->quotas.mpt;
247         props->max_pd              = dev->dev->caps.num_pds - dev->dev->caps.reserved_pds;
248         props->max_qp_rd_atom      = dev->dev->caps.max_qp_dest_rdma;
249         props->max_qp_init_rd_atom = dev->dev->caps.max_qp_init_rdma;
250         props->max_res_rd_atom     = props->max_qp_rd_atom * props->max_qp;
251         props->max_srq             = dev->dev->quotas.srq;
252         props->max_srq_wr          = dev->dev->caps.max_srq_wqes - 1;
253         props->max_srq_sge         = dev->dev->caps.max_srq_sge;
254         props->max_fast_reg_page_list_len = MLX4_MAX_FAST_REG_PAGES;
255         props->local_ca_ack_delay  = dev->dev->caps.local_ca_ack_delay;
256         props->atomic_cap          = dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_ATOMIC ?
257                 IB_ATOMIC_HCA : IB_ATOMIC_NONE;
258         props->masked_atomic_cap   = props->atomic_cap;
259         props->max_pkeys           = dev->dev->caps.pkey_table_len[1];
260         props->max_mcast_grp       = dev->dev->caps.num_mgms + dev->dev->caps.num_amgms;
261         props->max_mcast_qp_attach = dev->dev->caps.num_qp_per_mgm;
262         props->max_total_mcast_qp_attach = props->max_mcast_qp_attach *
263                                            props->max_mcast_grp;
264         props->max_map_per_fmr = dev->dev->caps.max_fmr_maps;
265
266 out:
267         kfree(in_mad);
268         kfree(out_mad);
269
270         return err;
271 }
272
273 static enum rdma_link_layer
274 mlx4_ib_port_link_layer(struct ib_device *device, u8 port_num)
275 {
276         struct mlx4_dev *dev = to_mdev(device)->dev;
277
278         return dev->caps.port_mask[port_num] == MLX4_PORT_TYPE_IB ?
279                 IB_LINK_LAYER_INFINIBAND : IB_LINK_LAYER_ETHERNET;
280 }
281
282 static int ib_link_query_port(struct ib_device *ibdev, u8 port,
283                               struct ib_port_attr *props, int netw_view)
284 {
285         struct ib_smp *in_mad  = NULL;
286         struct ib_smp *out_mad = NULL;
287         int ext_active_speed;
288         int mad_ifc_flags = MLX4_MAD_IFC_IGNORE_KEYS;
289         int err = -ENOMEM;
290
291         in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
292         out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
293         if (!in_mad || !out_mad)
294                 goto out;
295
296         init_query_mad(in_mad);
297         in_mad->attr_id  = IB_SMP_ATTR_PORT_INFO;
298         in_mad->attr_mod = cpu_to_be32(port);
299
300         if (mlx4_is_mfunc(to_mdev(ibdev)->dev) && netw_view)
301                 mad_ifc_flags |= MLX4_MAD_IFC_NET_VIEW;
302
303         err = mlx4_MAD_IFC(to_mdev(ibdev), mad_ifc_flags, port, NULL, NULL,
304                                 in_mad, out_mad);
305         if (err)
306                 goto out;
307
308
309         props->lid              = be16_to_cpup((__be16 *) (out_mad->data + 16));
310         props->lmc              = out_mad->data[34] & 0x7;
311         props->sm_lid           = be16_to_cpup((__be16 *) (out_mad->data + 18));
312         props->sm_sl            = out_mad->data[36] & 0xf;
313         props->state            = out_mad->data[32] & 0xf;
314         props->phys_state       = out_mad->data[33] >> 4;
315         props->port_cap_flags   = be32_to_cpup((__be32 *) (out_mad->data + 20));
316         if (netw_view)
317                 props->gid_tbl_len = out_mad->data[50];
318         else
319                 props->gid_tbl_len = to_mdev(ibdev)->dev->caps.gid_table_len[port];
320         props->max_msg_sz       = to_mdev(ibdev)->dev->caps.max_msg_sz;
321         props->pkey_tbl_len     = to_mdev(ibdev)->dev->caps.pkey_table_len[port];
322         props->bad_pkey_cntr    = be16_to_cpup((__be16 *) (out_mad->data + 46));
323         props->qkey_viol_cntr   = be16_to_cpup((__be16 *) (out_mad->data + 48));
324         props->active_width     = out_mad->data[31] & 0xf;
325         props->active_speed     = out_mad->data[35] >> 4;
326         props->max_mtu          = out_mad->data[41] & 0xf;
327         props->active_mtu       = out_mad->data[36] >> 4;
328         props->subnet_timeout   = out_mad->data[51] & 0x1f;
329         props->max_vl_num       = out_mad->data[37] >> 4;
330         props->init_type_reply  = out_mad->data[41] >> 4;
331
332         /* Check if extended speeds (EDR/FDR/...) are supported */
333         if (props->port_cap_flags & IB_PORT_EXTENDED_SPEEDS_SUP) {
334                 ext_active_speed = out_mad->data[62] >> 4;
335
336                 switch (ext_active_speed) {
337                 case 1:
338                         props->active_speed = IB_SPEED_FDR;
339                         break;
340                 case 2:
341                         props->active_speed = IB_SPEED_EDR;
342                         break;
343                 }
344         }
345
346         /* If reported active speed is QDR, check if is FDR-10 */
347         if (props->active_speed == IB_SPEED_QDR) {
348                 init_query_mad(in_mad);
349                 in_mad->attr_id = MLX4_ATTR_EXTENDED_PORT_INFO;
350                 in_mad->attr_mod = cpu_to_be32(port);
351
352                 err = mlx4_MAD_IFC(to_mdev(ibdev), mad_ifc_flags, port,
353                                    NULL, NULL, in_mad, out_mad);
354                 if (err)
355                         goto out;
356
357                 /* Checking LinkSpeedActive for FDR-10 */
358                 if (out_mad->data[15] & 0x1)
359                         props->active_speed = IB_SPEED_FDR10;
360         }
361
362         /* Avoid wrong speed value returned by FW if the IB link is down. */
363         if (props->state == IB_PORT_DOWN)
364                  props->active_speed = IB_SPEED_SDR;
365
366 out:
367         kfree(in_mad);
368         kfree(out_mad);
369         return err;
370 }
371
372 static u8 state_to_phys_state(enum ib_port_state state)
373 {
374         return state == IB_PORT_ACTIVE ? 5 : 3;
375 }
376
377 static int eth_link_query_port(struct ib_device *ibdev, u8 port,
378                                struct ib_port_attr *props, int netw_view)
379 {
380
381         struct mlx4_ib_dev *mdev = to_mdev(ibdev);
382         struct mlx4_ib_iboe *iboe = &mdev->iboe;
383         struct net_device *ndev;
384         enum ib_mtu tmp;
385         struct mlx4_cmd_mailbox *mailbox;
386         int err = 0;
387         int is_bonded = mlx4_is_bonded(mdev->dev);
388
389         mailbox = mlx4_alloc_cmd_mailbox(mdev->dev);
390         if (IS_ERR(mailbox))
391                 return PTR_ERR(mailbox);
392
393         err = mlx4_cmd_box(mdev->dev, 0, mailbox->dma, port, 0,
394                            MLX4_CMD_QUERY_PORT, MLX4_CMD_TIME_CLASS_B,
395                            MLX4_CMD_WRAPPED);
396         if (err)
397                 goto out;
398
399         props->active_width     =  (((u8 *)mailbox->buf)[5] == 0x40) ?
400                                                 IB_WIDTH_4X : IB_WIDTH_1X;
401         props->active_speed     = IB_SPEED_QDR;
402         props->port_cap_flags   = IB_PORT_CM_SUP | IB_PORT_IP_BASED_GIDS;
403         props->gid_tbl_len      = mdev->dev->caps.gid_table_len[port];
404         props->max_msg_sz       = mdev->dev->caps.max_msg_sz;
405         props->pkey_tbl_len     = 1;
406         props->max_mtu          = IB_MTU_4096;
407         props->max_vl_num       = 2;
408         props->state            = IB_PORT_DOWN;
409         props->phys_state       = state_to_phys_state(props->state);
410         props->active_mtu       = IB_MTU_256;
411         if (is_bonded)
412                 rtnl_lock(); /* required to get upper dev */
413         spin_lock_bh(&iboe->lock);
414         ndev = iboe->netdevs[port - 1];
415         if (ndev && is_bonded)
416                 ndev = netdev_master_upper_dev_get(ndev);
417         if (!ndev)
418                 goto out_unlock;
419
420         tmp = iboe_get_mtu(ndev->mtu);
421         props->active_mtu = tmp ? min(props->max_mtu, tmp) : IB_MTU_256;
422
423         props->state            = (netif_running(ndev) && netif_carrier_ok(ndev)) ?
424                                         IB_PORT_ACTIVE : IB_PORT_DOWN;
425         props->phys_state       = state_to_phys_state(props->state);
426 out_unlock:
427         spin_unlock_bh(&iboe->lock);
428         if (is_bonded)
429                 rtnl_unlock();
430 out:
431         mlx4_free_cmd_mailbox(mdev->dev, mailbox);
432         return err;
433 }
434
435 int __mlx4_ib_query_port(struct ib_device *ibdev, u8 port,
436                          struct ib_port_attr *props, int netw_view)
437 {
438         int err;
439
440         memset(props, 0, sizeof *props);
441
442         err = mlx4_ib_port_link_layer(ibdev, port) == IB_LINK_LAYER_INFINIBAND ?
443                 ib_link_query_port(ibdev, port, props, netw_view) :
444                                 eth_link_query_port(ibdev, port, props, netw_view);
445
446         return err;
447 }
448
449 static int mlx4_ib_query_port(struct ib_device *ibdev, u8 port,
450                               struct ib_port_attr *props)
451 {
452         /* returns host view */
453         return __mlx4_ib_query_port(ibdev, port, props, 0);
454 }
455
456 int __mlx4_ib_query_gid(struct ib_device *ibdev, u8 port, int index,
457                         union ib_gid *gid, int netw_view)
458 {
459         struct ib_smp *in_mad  = NULL;
460         struct ib_smp *out_mad = NULL;
461         int err = -ENOMEM;
462         struct mlx4_ib_dev *dev = to_mdev(ibdev);
463         int clear = 0;
464         int mad_ifc_flags = MLX4_MAD_IFC_IGNORE_KEYS;
465
466         in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
467         out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
468         if (!in_mad || !out_mad)
469                 goto out;
470
471         init_query_mad(in_mad);
472         in_mad->attr_id  = IB_SMP_ATTR_PORT_INFO;
473         in_mad->attr_mod = cpu_to_be32(port);
474
475         if (mlx4_is_mfunc(dev->dev) && netw_view)
476                 mad_ifc_flags |= MLX4_MAD_IFC_NET_VIEW;
477
478         err = mlx4_MAD_IFC(dev, mad_ifc_flags, port, NULL, NULL, in_mad, out_mad);
479         if (err)
480                 goto out;
481
482         memcpy(gid->raw, out_mad->data + 8, 8);
483
484         if (mlx4_is_mfunc(dev->dev) && !netw_view) {
485                 if (index) {
486                         /* For any index > 0, return the null guid */
487                         err = 0;
488                         clear = 1;
489                         goto out;
490                 }
491         }
492
493         init_query_mad(in_mad);
494         in_mad->attr_id  = IB_SMP_ATTR_GUID_INFO;
495         in_mad->attr_mod = cpu_to_be32(index / 8);
496
497         err = mlx4_MAD_IFC(dev, mad_ifc_flags, port,
498                            NULL, NULL, in_mad, out_mad);
499         if (err)
500                 goto out;
501
502         memcpy(gid->raw + 8, out_mad->data + (index % 8) * 8, 8);
503
504 out:
505         if (clear)
506                 memset(gid->raw + 8, 0, 8);
507         kfree(in_mad);
508         kfree(out_mad);
509         return err;
510 }
511
512 static int iboe_query_gid(struct ib_device *ibdev, u8 port, int index,
513                           union ib_gid *gid)
514 {
515         struct mlx4_ib_dev *dev = to_mdev(ibdev);
516
517         *gid = dev->iboe.gid_table[port - 1][index];
518
519         return 0;
520 }
521
522 static int mlx4_ib_query_gid(struct ib_device *ibdev, u8 port, int index,
523                              union ib_gid *gid)
524 {
525         if (rdma_port_get_link_layer(ibdev, port) == IB_LINK_LAYER_INFINIBAND)
526                 return __mlx4_ib_query_gid(ibdev, port, index, gid, 0);
527         else
528                 return iboe_query_gid(ibdev, port, index, gid);
529 }
530
531 int __mlx4_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
532                          u16 *pkey, int netw_view)
533 {
534         struct ib_smp *in_mad  = NULL;
535         struct ib_smp *out_mad = NULL;
536         int mad_ifc_flags = MLX4_MAD_IFC_IGNORE_KEYS;
537         int err = -ENOMEM;
538
539         in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
540         out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
541         if (!in_mad || !out_mad)
542                 goto out;
543
544         init_query_mad(in_mad);
545         in_mad->attr_id  = IB_SMP_ATTR_PKEY_TABLE;
546         in_mad->attr_mod = cpu_to_be32(index / 32);
547
548         if (mlx4_is_mfunc(to_mdev(ibdev)->dev) && netw_view)
549                 mad_ifc_flags |= MLX4_MAD_IFC_NET_VIEW;
550
551         err = mlx4_MAD_IFC(to_mdev(ibdev), mad_ifc_flags, port, NULL, NULL,
552                            in_mad, out_mad);
553         if (err)
554                 goto out;
555
556         *pkey = be16_to_cpu(((__be16 *) out_mad->data)[index % 32]);
557
558 out:
559         kfree(in_mad);
560         kfree(out_mad);
561         return err;
562 }
563
564 static int mlx4_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
565 {
566         return __mlx4_ib_query_pkey(ibdev, port, index, pkey, 0);
567 }
568
569 static int mlx4_ib_modify_device(struct ib_device *ibdev, int mask,
570                                  struct ib_device_modify *props)
571 {
572         struct mlx4_cmd_mailbox *mailbox;
573         unsigned long flags;
574
575         if (mask & ~IB_DEVICE_MODIFY_NODE_DESC)
576                 return -EOPNOTSUPP;
577
578         if (!(mask & IB_DEVICE_MODIFY_NODE_DESC))
579                 return 0;
580
581         if (mlx4_is_slave(to_mdev(ibdev)->dev))
582                 return -EOPNOTSUPP;
583
584         spin_lock_irqsave(&to_mdev(ibdev)->sm_lock, flags);
585         memcpy(ibdev->node_desc, props->node_desc, 64);
586         spin_unlock_irqrestore(&to_mdev(ibdev)->sm_lock, flags);
587
588         /*
589          * If possible, pass node desc to FW, so it can generate
590          * a 144 trap.  If cmd fails, just ignore.
591          */
592         mailbox = mlx4_alloc_cmd_mailbox(to_mdev(ibdev)->dev);
593         if (IS_ERR(mailbox))
594                 return 0;
595
596         memcpy(mailbox->buf, props->node_desc, 64);
597         mlx4_cmd(to_mdev(ibdev)->dev, mailbox->dma, 1, 0,
598                  MLX4_CMD_SET_NODE, MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE);
599
600         mlx4_free_cmd_mailbox(to_mdev(ibdev)->dev, mailbox);
601
602         return 0;
603 }
604
605 static int mlx4_ib_SET_PORT(struct mlx4_ib_dev *dev, u8 port, int reset_qkey_viols,
606                             u32 cap_mask)
607 {
608         struct mlx4_cmd_mailbox *mailbox;
609         int err;
610
611         mailbox = mlx4_alloc_cmd_mailbox(dev->dev);
612         if (IS_ERR(mailbox))
613                 return PTR_ERR(mailbox);
614
615         if (dev->dev->flags & MLX4_FLAG_OLD_PORT_CMDS) {
616                 *(u8 *) mailbox->buf         = !!reset_qkey_viols << 6;
617                 ((__be32 *) mailbox->buf)[2] = cpu_to_be32(cap_mask);
618         } else {
619                 ((u8 *) mailbox->buf)[3]     = !!reset_qkey_viols;
620                 ((__be32 *) mailbox->buf)[1] = cpu_to_be32(cap_mask);
621         }
622
623         err = mlx4_cmd(dev->dev, mailbox->dma, port, MLX4_SET_PORT_IB_OPCODE,
624                        MLX4_CMD_SET_PORT, MLX4_CMD_TIME_CLASS_B,
625                        MLX4_CMD_WRAPPED);
626
627         mlx4_free_cmd_mailbox(dev->dev, mailbox);
628         return err;
629 }
630
631 static int mlx4_ib_modify_port(struct ib_device *ibdev, u8 port, int mask,
632                                struct ib_port_modify *props)
633 {
634         struct mlx4_ib_dev *mdev = to_mdev(ibdev);
635         u8 is_eth = mdev->dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH;
636         struct ib_port_attr attr;
637         u32 cap_mask;
638         int err;
639
640         /* return OK if this is RoCE. CM calls ib_modify_port() regardless
641          * of whether port link layer is ETH or IB. For ETH ports, qkey
642          * violations and port capabilities are not meaningful.
643          */
644         if (is_eth)
645                 return 0;
646
647         mutex_lock(&mdev->cap_mask_mutex);
648
649         err = mlx4_ib_query_port(ibdev, port, &attr);
650         if (err)
651                 goto out;
652
653         cap_mask = (attr.port_cap_flags | props->set_port_cap_mask) &
654                 ~props->clr_port_cap_mask;
655
656         err = mlx4_ib_SET_PORT(mdev, port,
657                                !!(mask & IB_PORT_RESET_QKEY_CNTR),
658                                cap_mask);
659
660 out:
661         mutex_unlock(&to_mdev(ibdev)->cap_mask_mutex);
662         return err;
663 }
664
665 static struct ib_ucontext *mlx4_ib_alloc_ucontext(struct ib_device *ibdev,
666                                                   struct ib_udata *udata)
667 {
668         struct mlx4_ib_dev *dev = to_mdev(ibdev);
669         struct mlx4_ib_ucontext *context;
670         struct mlx4_ib_alloc_ucontext_resp_v3 resp_v3;
671         struct mlx4_ib_alloc_ucontext_resp resp;
672         int err;
673
674         if (!dev->ib_active)
675                 return ERR_PTR(-EAGAIN);
676
677         if (ibdev->uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION) {
678                 resp_v3.qp_tab_size      = dev->dev->caps.num_qps;
679                 resp_v3.bf_reg_size      = dev->dev->caps.bf_reg_size;
680                 resp_v3.bf_regs_per_page = dev->dev->caps.bf_regs_per_page;
681         } else {
682                 resp.dev_caps         = dev->dev->caps.userspace_caps;
683                 resp.qp_tab_size      = dev->dev->caps.num_qps;
684                 resp.bf_reg_size      = dev->dev->caps.bf_reg_size;
685                 resp.bf_regs_per_page = dev->dev->caps.bf_regs_per_page;
686                 resp.cqe_size         = dev->dev->caps.cqe_size;
687         }
688
689         context = kmalloc(sizeof *context, GFP_KERNEL);
690         if (!context)
691                 return ERR_PTR(-ENOMEM);
692
693         err = mlx4_uar_alloc(to_mdev(ibdev)->dev, &context->uar);
694         if (err) {
695                 kfree(context);
696                 return ERR_PTR(err);
697         }
698
699         INIT_LIST_HEAD(&context->db_page_list);
700         mutex_init(&context->db_page_mutex);
701
702         if (ibdev->uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION)
703                 err = ib_copy_to_udata(udata, &resp_v3, sizeof(resp_v3));
704         else
705                 err = ib_copy_to_udata(udata, &resp, sizeof(resp));
706
707         if (err) {
708                 mlx4_uar_free(to_mdev(ibdev)->dev, &context->uar);
709                 kfree(context);
710                 return ERR_PTR(-EFAULT);
711         }
712
713         return &context->ibucontext;
714 }
715
716 static int mlx4_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
717 {
718         struct mlx4_ib_ucontext *context = to_mucontext(ibcontext);
719
720         mlx4_uar_free(to_mdev(ibcontext->device)->dev, &context->uar);
721         kfree(context);
722
723         return 0;
724 }
725
726 #if 0   /* AKAROS */
727
728 static int mlx4_ib_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
729 {
730         struct mlx4_ib_dev *dev = to_mdev(context->device);
731
732         if (vma->vm_end - vma->vm_start != PAGE_SIZE)
733                 return -EINVAL;
734
735         if (vma->vm_pgoff == 0) {
736                 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
737
738                 if (io_remap_pfn_range(vma, vma->vm_start,
739                                        to_mucontext(context)->uar.pfn,
740                                        PAGE_SIZE, vma->vm_page_prot))
741                         return -EAGAIN;
742         } else if (vma->vm_pgoff == 1 && dev->dev->caps.bf_reg_size != 0) {
743                 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
744
745                 if (io_remap_pfn_range(vma, vma->vm_start,
746                                        to_mucontext(context)->uar.pfn +
747                                        dev->dev->caps.num_uars,
748                                        PAGE_SIZE, vma->vm_page_prot))
749                         return -EAGAIN;
750         } else
751                 return -EINVAL;
752
753         return 0;
754 }
755
756 #else   /* AKAROS */
757
758 static int mlx4_ib_mmap(struct ib_ucontext *context, struct vm_region *vma)
759 {
760         struct mlx4_ib_dev *dev = to_mdev(context->device);
761         int pteprot;
762
763         if (vma->vm_end - vma->vm_base != PAGE_SIZE)
764                 return -EINVAL;
765
766         if (vma->vm_foff == 0) {
767                 pteprot = pgprot_noncached(vma->vm_prot);
768
769                 if (io_remap_pfn_range(vma, vma->vm_start,
770                                        to_mucontext(context)->uar.pfn,
771                                        PAGE_SIZE, pteprot))
772                         return -EAGAIN;
773         } else if ((vma->vm_foff >> PAGE_SHIFT) == 1 && dev->dev->caps.bf_reg_size != 0) {
774                 pteprot = pgprot_writecombine(vma->vm_prot);
775
776                 if (io_remap_pfn_range(vma, vma->vm_start,
777                                        to_mucontext(context)->uar.pfn +
778                                        dev->dev->caps.num_uars,
779                                        PAGE_SIZE, pteprot))
780                         return -EAGAIN;
781         } else
782                 return -EINVAL;
783
784         return 0;
785 }
786
787 #endif  /* AKAROS */
788
789 static struct ib_pd *mlx4_ib_alloc_pd(struct ib_device *ibdev,
790                                       struct ib_ucontext *context,
791                                       struct ib_udata *udata)
792 {
793         struct mlx4_ib_pd *pd;
794         int err;
795
796         pd = kmalloc(sizeof *pd, GFP_KERNEL);
797         if (!pd)
798                 return ERR_PTR(-ENOMEM);
799
800         err = mlx4_pd_alloc(to_mdev(ibdev)->dev, &pd->pdn);
801         if (err) {
802                 kfree(pd);
803                 return ERR_PTR(err);
804         }
805
806         if (context)
807                 if (ib_copy_to_udata(udata, &pd->pdn, sizeof (__u32))) {
808                         mlx4_pd_free(to_mdev(ibdev)->dev, pd->pdn);
809                         kfree(pd);
810                         return ERR_PTR(-EFAULT);
811                 }
812
813         return &pd->ibpd;
814 }
815
816 static int mlx4_ib_dealloc_pd(struct ib_pd *pd)
817 {
818         mlx4_pd_free(to_mdev(pd->device)->dev, to_mpd(pd)->pdn);
819         kfree(pd);
820
821         return 0;
822 }
823
824 static struct ib_xrcd *mlx4_ib_alloc_xrcd(struct ib_device *ibdev,
825                                           struct ib_ucontext *context,
826                                           struct ib_udata *udata)
827 {
828         struct mlx4_ib_xrcd *xrcd;
829         int err;
830
831         if (!(to_mdev(ibdev)->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC))
832                 return ERR_PTR(-ENOSYS);
833
834         xrcd = kmalloc(sizeof *xrcd, GFP_KERNEL);
835         if (!xrcd)
836                 return ERR_PTR(-ENOMEM);
837
838         err = mlx4_xrcd_alloc(to_mdev(ibdev)->dev, &xrcd->xrcdn);
839         if (err)
840                 goto err1;
841
842         xrcd->pd = ib_alloc_pd(ibdev);
843         if (IS_ERR(xrcd->pd)) {
844                 err = PTR_ERR(xrcd->pd);
845                 goto err2;
846         }
847
848         xrcd->cq = ib_create_cq(ibdev, NULL, NULL, xrcd, 1, 0);
849         if (IS_ERR(xrcd->cq)) {
850                 err = PTR_ERR(xrcd->cq);
851                 goto err3;
852         }
853
854         return &xrcd->ibxrcd;
855
856 err3:
857         ib_dealloc_pd(xrcd->pd);
858 err2:
859         mlx4_xrcd_free(to_mdev(ibdev)->dev, xrcd->xrcdn);
860 err1:
861         kfree(xrcd);
862         return ERR_PTR(err);
863 }
864
865 static int mlx4_ib_dealloc_xrcd(struct ib_xrcd *xrcd)
866 {
867         ib_destroy_cq(to_mxrcd(xrcd)->cq);
868         ib_dealloc_pd(to_mxrcd(xrcd)->pd);
869         mlx4_xrcd_free(to_mdev(xrcd->device)->dev, to_mxrcd(xrcd)->xrcdn);
870         kfree(xrcd);
871
872         return 0;
873 }
874
875 static int add_gid_entry(struct ib_qp *ibqp, union ib_gid *gid)
876 {
877         struct mlx4_ib_qp *mqp = to_mqp(ibqp);
878         struct mlx4_ib_dev *mdev = to_mdev(ibqp->device);
879         struct mlx4_ib_gid_entry *ge;
880
881         ge = kzalloc(sizeof *ge, GFP_KERNEL);
882         if (!ge)
883                 return -ENOMEM;
884
885         ge->gid = *gid;
886         if (mlx4_ib_add_mc(mdev, mqp, gid)) {
887                 ge->port = mqp->port;
888                 ge->added = 1;
889         }
890
891         mutex_lock(&mqp->mutex);
892         list_add_tail(&ge->list, &mqp->gid_list);
893         mutex_unlock(&mqp->mutex);
894
895         return 0;
896 }
897
898 int mlx4_ib_add_mc(struct mlx4_ib_dev *mdev, struct mlx4_ib_qp *mqp,
899                    union ib_gid *gid)
900 {
901         struct net_device *ndev;
902         int ret = 0;
903
904         if (!mqp->port)
905                 return 0;
906
907         spin_lock_bh(&mdev->iboe.lock);
908         ndev = mdev->iboe.netdevs[mqp->port - 1];
909         if (ndev)
910                 dev_hold(ndev);
911         spin_unlock_bh(&mdev->iboe.lock);
912
913         if (ndev) {
914                 ret = 1;
915                 dev_put(ndev);
916         }
917
918         return ret;
919 }
920
921 struct mlx4_ib_steering {
922         struct list_head list;
923         struct mlx4_flow_reg_id reg_id;
924         union ib_gid gid;
925 };
926
927 static int parse_flow_attr(struct mlx4_dev *dev,
928                            u32 qp_num,
929                            union ib_flow_spec *ib_spec,
930                            struct _rule_hw *mlx4_spec)
931 {
932         enum mlx4_net_trans_rule_id type;
933
934         switch (ib_spec->type) {
935         case IB_FLOW_SPEC_ETH:
936                 type = MLX4_NET_TRANS_RULE_ID_ETH;
937                 memcpy(mlx4_spec->eth.dst_mac, ib_spec->eth.val.dst_mac,
938                        ETH_ALEN);
939                 memcpy(mlx4_spec->eth.dst_mac_msk, ib_spec->eth.mask.dst_mac,
940                        ETH_ALEN);
941                 mlx4_spec->eth.vlan_tag = ib_spec->eth.val.vlan_tag;
942                 mlx4_spec->eth.vlan_tag_msk = ib_spec->eth.mask.vlan_tag;
943                 break;
944         case IB_FLOW_SPEC_IB:
945                 type = MLX4_NET_TRANS_RULE_ID_IB;
946                 mlx4_spec->ib.l3_qpn =
947                         cpu_to_be32(qp_num);
948                 mlx4_spec->ib.qpn_mask =
949                         cpu_to_be32(MLX4_IB_FLOW_QPN_MASK);
950                 break;
951
952
953         case IB_FLOW_SPEC_IPV4:
954                 type = MLX4_NET_TRANS_RULE_ID_IPV4;
955                 mlx4_spec->ipv4.src_ip = ib_spec->ipv4.val.src_ip;
956                 mlx4_spec->ipv4.src_ip_msk = ib_spec->ipv4.mask.src_ip;
957                 mlx4_spec->ipv4.dst_ip = ib_spec->ipv4.val.dst_ip;
958                 mlx4_spec->ipv4.dst_ip_msk = ib_spec->ipv4.mask.dst_ip;
959                 break;
960
961         case IB_FLOW_SPEC_TCP:
962         case IB_FLOW_SPEC_UDP:
963                 type = ib_spec->type == IB_FLOW_SPEC_TCP ?
964                                         MLX4_NET_TRANS_RULE_ID_TCP :
965                                         MLX4_NET_TRANS_RULE_ID_UDP;
966                 mlx4_spec->tcp_udp.dst_port = ib_spec->tcp_udp.val.dst_port;
967                 mlx4_spec->tcp_udp.dst_port_msk = ib_spec->tcp_udp.mask.dst_port;
968                 mlx4_spec->tcp_udp.src_port = ib_spec->tcp_udp.val.src_port;
969                 mlx4_spec->tcp_udp.src_port_msk = ib_spec->tcp_udp.mask.src_port;
970                 break;
971
972         default:
973                 return -EINVAL;
974         }
975         if (mlx4_map_sw_to_hw_steering_id(dev, type) < 0 ||
976             mlx4_hw_rule_sz(dev, type) < 0)
977                 return -EINVAL;
978         mlx4_spec->id = cpu_to_be16(mlx4_map_sw_to_hw_steering_id(dev, type));
979         mlx4_spec->size = mlx4_hw_rule_sz(dev, type) >> 2;
980         return mlx4_hw_rule_sz(dev, type);
981 }
982
983 struct default_rules {
984         __u32 mandatory_fields[IB_FLOW_SPEC_SUPPORT_LAYERS];
985         __u32 mandatory_not_fields[IB_FLOW_SPEC_SUPPORT_LAYERS];
986         __u32 rules_create_list[IB_FLOW_SPEC_SUPPORT_LAYERS];
987         __u8  link_layer;
988 };
989 static const struct default_rules default_table[] = {
990         {
991                 .mandatory_fields = {IB_FLOW_SPEC_IPV4},
992                 .mandatory_not_fields = {IB_FLOW_SPEC_ETH},
993                 .rules_create_list = {IB_FLOW_SPEC_IB},
994                 .link_layer = IB_LINK_LAYER_INFINIBAND
995         }
996 };
997
998 static int __mlx4_ib_default_rules_match(struct ib_qp *qp,
999                                          struct ib_flow_attr *flow_attr)
1000 {
1001         int i, j, k;
1002         void *ib_flow;
1003         const struct default_rules *pdefault_rules = default_table;
1004         u8 link_layer = rdma_port_get_link_layer(qp->device, flow_attr->port);
1005
1006         for (i = 0; i < ARRAY_SIZE(default_table); i++, pdefault_rules++) {
1007                 __u32 field_types[IB_FLOW_SPEC_SUPPORT_LAYERS];
1008                 memset(&field_types, 0, sizeof(field_types));
1009
1010                 if (link_layer != pdefault_rules->link_layer)
1011                         continue;
1012
1013                 ib_flow = flow_attr + 1;
1014                 /* we assume the specs are sorted */
1015                 for (j = 0, k = 0; k < IB_FLOW_SPEC_SUPPORT_LAYERS &&
1016                      j < flow_attr->num_of_specs; k++) {
1017                         union ib_flow_spec *current_flow =
1018                                 (union ib_flow_spec *)ib_flow;
1019
1020                         /* same layer but different type */
1021                         if (((current_flow->type & IB_FLOW_SPEC_LAYER_MASK) ==
1022                              (pdefault_rules->mandatory_fields[k] &
1023                               IB_FLOW_SPEC_LAYER_MASK)) &&
1024                             (current_flow->type !=
1025                              pdefault_rules->mandatory_fields[k]))
1026                                 goto out;
1027
1028                         /* same layer, try match next one */
1029                         if (current_flow->type ==
1030                             pdefault_rules->mandatory_fields[k]) {
1031                                 j++;
1032                                 ib_flow +=
1033                                         ((union ib_flow_spec *)ib_flow)->size;
1034                         }
1035                 }
1036
1037                 ib_flow = flow_attr + 1;
1038                 for (j = 0; j < flow_attr->num_of_specs;
1039                      j++, ib_flow += ((union ib_flow_spec *)ib_flow)->size)
1040                         for (k = 0; k < IB_FLOW_SPEC_SUPPORT_LAYERS; k++)
1041                                 /* same layer and same type */
1042                                 if (((union ib_flow_spec *)ib_flow)->type ==
1043                                     pdefault_rules->mandatory_not_fields[k])
1044                                         goto out;
1045
1046                 return i;
1047         }
1048 out:
1049         return -1;
1050 }
1051
1052 static int __mlx4_ib_create_default_rules(
1053                 struct mlx4_ib_dev *mdev,
1054                 struct ib_qp *qp,
1055                 const struct default_rules *pdefault_rules,
1056                 struct _rule_hw *mlx4_spec) {
1057         int size = 0;
1058         int i;
1059
1060         for (i = 0; i < ARRAY_SIZE(pdefault_rules->rules_create_list); i++) {
1061                 int ret;
1062                 union ib_flow_spec ib_spec;
1063                 switch (pdefault_rules->rules_create_list[i]) {
1064                 case 0:
1065                         /* no rule */
1066                         continue;
1067                 case IB_FLOW_SPEC_IB:
1068                         ib_spec.type = IB_FLOW_SPEC_IB;
1069                         ib_spec.size = sizeof(struct ib_flow_spec_ib);
1070
1071                         break;
1072                 default:
1073                         /* invalid rule */
1074                         return -EINVAL;
1075                 }
1076                 /* We must put empty rule, qpn is being ignored */
1077                 ret = parse_flow_attr(mdev->dev, 0, &ib_spec,
1078                                       mlx4_spec);
1079                 if (ret < 0) {
1080                         pr_info("invalid parsing\n");
1081                         return -EINVAL;
1082                 }
1083
1084                 mlx4_spec = (void *)mlx4_spec + ret;
1085                 size += ret;
1086         }
1087         return size;
1088 }
1089
1090 static int __mlx4_ib_create_flow(struct ib_qp *qp, struct ib_flow_attr *flow_attr,
1091                           int domain,
1092                           enum mlx4_net_trans_promisc_mode flow_type,
1093                           u64 *reg_id)
1094 {
1095         int ret, i;
1096         int size = 0;
1097         void *ib_flow;
1098         struct mlx4_ib_dev *mdev = to_mdev(qp->device);
1099         struct mlx4_cmd_mailbox *mailbox;
1100         struct mlx4_net_trans_rule_hw_ctrl *ctrl;
1101         int default_flow;
1102
1103         static const u16 __mlx4_domain[] = {
1104                 [IB_FLOW_DOMAIN_USER] = MLX4_DOMAIN_UVERBS,
1105                 [IB_FLOW_DOMAIN_ETHTOOL] = MLX4_DOMAIN_ETHTOOL,
1106                 [IB_FLOW_DOMAIN_RFS] = MLX4_DOMAIN_RFS,
1107                 [IB_FLOW_DOMAIN_NIC] = MLX4_DOMAIN_NIC,
1108         };
1109
1110         if (flow_attr->priority > MLX4_IB_FLOW_MAX_PRIO) {
1111                 pr_err("Invalid priority value %d\n", flow_attr->priority);
1112                 return -EINVAL;
1113         }
1114
1115         if (domain >= IB_FLOW_DOMAIN_NUM) {
1116                 pr_err("Invalid domain value %d\n", domain);
1117                 return -EINVAL;
1118         }
1119
1120         if (mlx4_map_sw_to_hw_steering_mode(mdev->dev, flow_type) < 0)
1121                 return -EINVAL;
1122
1123         mailbox = mlx4_alloc_cmd_mailbox(mdev->dev);
1124         if (IS_ERR(mailbox))
1125                 return PTR_ERR(mailbox);
1126         ctrl = mailbox->buf;
1127
1128         ctrl->prio = cpu_to_be16(__mlx4_domain[domain] |
1129                                  flow_attr->priority);
1130         ctrl->type = mlx4_map_sw_to_hw_steering_mode(mdev->dev, flow_type);
1131         ctrl->port = flow_attr->port;
1132         ctrl->qpn = cpu_to_be32(qp->qp_num);
1133
1134         ib_flow = flow_attr + 1;
1135         size += sizeof(struct mlx4_net_trans_rule_hw_ctrl);
1136         /* Add default flows */
1137         default_flow = __mlx4_ib_default_rules_match(qp, flow_attr);
1138         if (default_flow >= 0) {
1139                 ret = __mlx4_ib_create_default_rules(
1140                                 mdev, qp, default_table + default_flow,
1141                                 mailbox->buf + size);
1142                 if (ret < 0) {
1143                         mlx4_free_cmd_mailbox(mdev->dev, mailbox);
1144                         return -EINVAL;
1145                 }
1146                 size += ret;
1147         }
1148         for (i = 0; i < flow_attr->num_of_specs; i++) {
1149                 ret = parse_flow_attr(mdev->dev, qp->qp_num, ib_flow,
1150                                       mailbox->buf + size);
1151                 if (ret < 0) {
1152                         mlx4_free_cmd_mailbox(mdev->dev, mailbox);
1153                         return -EINVAL;
1154                 }
1155                 ib_flow += ((union ib_flow_spec *) ib_flow)->size;
1156                 size += ret;
1157         }
1158
1159         ret = mlx4_cmd_imm(mdev->dev, mailbox->dma, reg_id, size >> 2, 0,
1160                            MLX4_QP_FLOW_STEERING_ATTACH, MLX4_CMD_TIME_CLASS_A,
1161                            MLX4_CMD_NATIVE);
1162         if (ret == -ENOMEM)
1163                 pr_err("mcg table is full. Fail to register network rule.\n");
1164         else if (ret == -ENXIO)
1165                 pr_err("Device managed flow steering is disabled. Fail to register network rule.\n");
1166         else if (ret)
1167                 pr_err("Invalid argumant. Fail to register network rule.\n");
1168
1169         mlx4_free_cmd_mailbox(mdev->dev, mailbox);
1170         return ret;
1171 }
1172
1173 static int __mlx4_ib_destroy_flow(struct mlx4_dev *dev, u64 reg_id)
1174 {
1175         int err;
1176         err = mlx4_cmd(dev, reg_id, 0, 0,
1177                        MLX4_QP_FLOW_STEERING_DETACH, MLX4_CMD_TIME_CLASS_A,
1178                        MLX4_CMD_NATIVE);
1179         if (err)
1180                 pr_err("Fail to detach network rule. registration id = 0x%llx\n",
1181                        reg_id);
1182         return err;
1183 }
1184
1185 static int mlx4_ib_tunnel_steer_add(struct ib_qp *qp, struct ib_flow_attr *flow_attr,
1186                                     u64 *reg_id)
1187 {
1188         void *ib_flow;
1189         union ib_flow_spec *ib_spec;
1190         struct mlx4_dev *dev = to_mdev(qp->device)->dev;
1191         int err = 0;
1192
1193         if (dev->caps.tunnel_offload_mode != MLX4_TUNNEL_OFFLOAD_MODE_VXLAN ||
1194             dev->caps.dmfs_high_steer_mode == MLX4_STEERING_DMFS_A0_STATIC)
1195                 return 0; /* do nothing */
1196
1197         ib_flow = flow_attr + 1;
1198         ib_spec = (union ib_flow_spec *)ib_flow;
1199
1200         if (ib_spec->type !=  IB_FLOW_SPEC_ETH || flow_attr->num_of_specs != 1)
1201                 return 0; /* do nothing */
1202
1203         err = mlx4_tunnel_steer_add(to_mdev(qp->device)->dev, ib_spec->eth.val.dst_mac,
1204                                     flow_attr->port, qp->qp_num,
1205                                     MLX4_DOMAIN_UVERBS | (flow_attr->priority & 0xff),
1206                                     reg_id);
1207         return err;
1208 }
1209
1210 static struct ib_flow *mlx4_ib_create_flow(struct ib_qp *qp,
1211                                     struct ib_flow_attr *flow_attr,
1212                                     int domain)
1213 {
1214         int err = 0, i = 0, j = 0;
1215         struct mlx4_ib_flow *mflow;
1216         enum mlx4_net_trans_promisc_mode type[2];
1217         struct mlx4_dev *dev = (to_mdev(qp->device))->dev;
1218         int is_bonded = mlx4_is_bonded(dev);
1219
1220         memset(type, 0, sizeof(type));
1221
1222         mflow = kzalloc(sizeof(*mflow), GFP_KERNEL);
1223         if (!mflow) {
1224                 err = -ENOMEM;
1225                 goto err_free;
1226         }
1227
1228         switch (flow_attr->type) {
1229         case IB_FLOW_ATTR_NORMAL:
1230                 type[0] = MLX4_FS_REGULAR;
1231                 break;
1232
1233         case IB_FLOW_ATTR_ALL_DEFAULT:
1234                 type[0] = MLX4_FS_ALL_DEFAULT;
1235                 break;
1236
1237         case IB_FLOW_ATTR_MC_DEFAULT:
1238                 type[0] = MLX4_FS_MC_DEFAULT;
1239                 break;
1240
1241         case IB_FLOW_ATTR_SNIFFER:
1242                 type[0] = MLX4_FS_UC_SNIFFER;
1243                 type[1] = MLX4_FS_MC_SNIFFER;
1244                 break;
1245
1246         default:
1247                 err = -EINVAL;
1248                 goto err_free;
1249         }
1250
1251         while (i < ARRAY_SIZE(type) && type[i]) {
1252                 err = __mlx4_ib_create_flow(qp, flow_attr, domain, type[i],
1253                                             &mflow->reg_id[i].id);
1254                 if (err)
1255                         goto err_create_flow;
1256                 i++;
1257                 if (is_bonded) {
1258                         /* Application always sees one port so the mirror rule
1259                          * must be on port #2
1260                          */
1261                         flow_attr->port = 2;
1262                         err = __mlx4_ib_create_flow(qp, flow_attr,
1263                                                     domain, type[j],
1264                                                     &mflow->reg_id[j].mirror);
1265                         flow_attr->port = 1;
1266                         if (err)
1267                                 goto err_create_flow;
1268                         j++;
1269                 }
1270
1271         }
1272
1273         if (i < ARRAY_SIZE(type) && flow_attr->type == IB_FLOW_ATTR_NORMAL) {
1274                 err = mlx4_ib_tunnel_steer_add(qp, flow_attr,
1275                                                &mflow->reg_id[i].id);
1276                 if (err)
1277                         goto err_create_flow;
1278                 i++;
1279                 if (is_bonded) {
1280                         flow_attr->port = 2;
1281                         err = mlx4_ib_tunnel_steer_add(qp, flow_attr,
1282                                                        &mflow->reg_id[j].mirror);
1283                         flow_attr->port = 1;
1284                         if (err)
1285                                 goto err_create_flow;
1286                         j++;
1287                 }
1288                 /* function to create mirror rule */
1289         }
1290
1291         return &mflow->ibflow;
1292
1293 err_create_flow:
1294         while (i) {
1295                 (void)__mlx4_ib_destroy_flow(to_mdev(qp->device)->dev,
1296                                              mflow->reg_id[i].id);
1297                 i--;
1298         }
1299
1300         while (j) {
1301                 (void)__mlx4_ib_destroy_flow(to_mdev(qp->device)->dev,
1302                                              mflow->reg_id[j].mirror);
1303                 j--;
1304         }
1305 err_free:
1306         kfree(mflow);
1307         return ERR_PTR(err);
1308 }
1309
1310 static int mlx4_ib_destroy_flow(struct ib_flow *flow_id)
1311 {
1312         int err, ret = 0;
1313         int i = 0;
1314         struct mlx4_ib_dev *mdev = to_mdev(flow_id->qp->device);
1315         struct mlx4_ib_flow *mflow = to_mflow(flow_id);
1316
1317         while (i < ARRAY_SIZE(mflow->reg_id) && mflow->reg_id[i].id) {
1318                 err = __mlx4_ib_destroy_flow(mdev->dev, mflow->reg_id[i].id);
1319                 if (err)
1320                         ret = err;
1321                 if (mflow->reg_id[i].mirror) {
1322                         err = __mlx4_ib_destroy_flow(mdev->dev,
1323                                                      mflow->reg_id[i].mirror);
1324                         if (err)
1325                                 ret = err;
1326                 }
1327                 i++;
1328         }
1329
1330         kfree(mflow);
1331         return ret;
1332 }
1333
1334 static int mlx4_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
1335 {
1336         int err;
1337         struct mlx4_ib_dev *mdev = to_mdev(ibqp->device);
1338         struct mlx4_dev *dev = mdev->dev;
1339         struct mlx4_ib_qp *mqp = to_mqp(ibqp);
1340         struct mlx4_ib_steering *ib_steering = NULL;
1341         enum mlx4_protocol prot = MLX4_PROT_IB_IPV6;
1342         struct mlx4_flow_reg_id reg_id;
1343
1344         if (mdev->dev->caps.steering_mode ==
1345             MLX4_STEERING_MODE_DEVICE_MANAGED) {
1346                 ib_steering = kmalloc(sizeof(*ib_steering), GFP_KERNEL);
1347                 if (!ib_steering)
1348                         return -ENOMEM;
1349         }
1350
1351         err = mlx4_multicast_attach(mdev->dev, &mqp->mqp, gid->raw, mqp->port,
1352                                     !!(mqp->flags &
1353                                        MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK),
1354                                     prot, &reg_id.id);
1355         if (err) {
1356                 pr_err("multicast attach op failed, err %d\n", err);
1357                 goto err_malloc;
1358         }
1359
1360         reg_id.mirror = 0;
1361         if (mlx4_is_bonded(dev)) {
1362                 err = mlx4_multicast_attach(mdev->dev, &mqp->mqp, gid->raw,
1363                                             (mqp->port == 1) ? 2 : 1,
1364                                             !!(mqp->flags &
1365                                             MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK),
1366                                             prot, &reg_id.mirror);
1367                 if (err)
1368                         goto err_add;
1369         }
1370
1371         err = add_gid_entry(ibqp, gid);
1372         if (err)
1373                 goto err_add;
1374
1375         if (ib_steering) {
1376                 memcpy(ib_steering->gid.raw, gid->raw, 16);
1377                 ib_steering->reg_id = reg_id;
1378                 mutex_lock(&mqp->mutex);
1379                 list_add(&ib_steering->list, &mqp->steering_rules);
1380                 mutex_unlock(&mqp->mutex);
1381         }
1382         return 0;
1383
1384 err_add:
1385         mlx4_multicast_detach(mdev->dev, &mqp->mqp, gid->raw,
1386                               prot, reg_id.id);
1387         if (reg_id.mirror)
1388                 mlx4_multicast_detach(mdev->dev, &mqp->mqp, gid->raw,
1389                                       prot, reg_id.mirror);
1390 err_malloc:
1391         kfree(ib_steering);
1392
1393         return err;
1394 }
1395
1396 static struct mlx4_ib_gid_entry *find_gid_entry(struct mlx4_ib_qp *qp, u8 *raw)
1397 {
1398         struct mlx4_ib_gid_entry *ge;
1399         struct mlx4_ib_gid_entry *tmp;
1400         struct mlx4_ib_gid_entry *ret = NULL;
1401
1402         list_for_each_entry_safe(ge, tmp, &qp->gid_list, list) {
1403                 if (!memcmp(raw, ge->gid.raw, 16)) {
1404                         ret = ge;
1405                         break;
1406                 }
1407         }
1408
1409         return ret;
1410 }
1411
1412 static int mlx4_ib_mcg_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
1413 {
1414         int err;
1415         struct mlx4_ib_dev *mdev = to_mdev(ibqp->device);
1416         struct mlx4_dev *dev = mdev->dev;
1417         struct mlx4_ib_qp *mqp = to_mqp(ibqp);
1418         struct net_device *ndev;
1419         struct mlx4_ib_gid_entry *ge;
1420         struct mlx4_flow_reg_id reg_id = {0, 0};
1421         enum mlx4_protocol prot =  MLX4_PROT_IB_IPV6;
1422
1423         if (mdev->dev->caps.steering_mode ==
1424             MLX4_STEERING_MODE_DEVICE_MANAGED) {
1425                 struct mlx4_ib_steering *ib_steering;
1426
1427                 mutex_lock(&mqp->mutex);
1428                 list_for_each_entry(ib_steering, &mqp->steering_rules, list) {
1429                         if (!memcmp(ib_steering->gid.raw, gid->raw, 16)) {
1430                                 list_del(&ib_steering->list);
1431                                 break;
1432                         }
1433                 }
1434                 mutex_unlock(&mqp->mutex);
1435                 if (&ib_steering->list == &mqp->steering_rules) {
1436                         pr_err("Couldn't find reg_id for mgid. Steering rule is left attached\n");
1437                         return -EINVAL;
1438                 }
1439                 reg_id = ib_steering->reg_id;
1440                 kfree(ib_steering);
1441         }
1442
1443         err = mlx4_multicast_detach(mdev->dev, &mqp->mqp, gid->raw,
1444                                     prot, reg_id.id);
1445         if (err)
1446                 return err;
1447
1448         if (mlx4_is_bonded(dev)) {
1449                 err = mlx4_multicast_detach(mdev->dev, &mqp->mqp, gid->raw,
1450                                             prot, reg_id.mirror);
1451                 if (err)
1452                         return err;
1453         }
1454
1455         mutex_lock(&mqp->mutex);
1456         ge = find_gid_entry(mqp, gid->raw);
1457         if (ge) {
1458                 spin_lock_bh(&mdev->iboe.lock);
1459                 ndev = ge->added ? mdev->iboe.netdevs[ge->port - 1] : NULL;
1460                 if (ndev)
1461                         dev_hold(ndev);
1462                 spin_unlock_bh(&mdev->iboe.lock);
1463                 if (ndev)
1464                         dev_put(ndev);
1465                 list_del(&ge->list);
1466                 kfree(ge);
1467         } else
1468                 pr_warn("could not find mgid entry\n");
1469
1470         mutex_unlock(&mqp->mutex);
1471
1472         return 0;
1473 }
1474
1475 static int init_node_data(struct mlx4_ib_dev *dev)
1476 {
1477         struct ib_smp *in_mad  = NULL;
1478         struct ib_smp *out_mad = NULL;
1479         int mad_ifc_flags = MLX4_MAD_IFC_IGNORE_KEYS;
1480         int err = -ENOMEM;
1481
1482         in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
1483         out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
1484         if (!in_mad || !out_mad)
1485                 goto out;
1486
1487         init_query_mad(in_mad);
1488         in_mad->attr_id = IB_SMP_ATTR_NODE_DESC;
1489         if (mlx4_is_master(dev->dev))
1490                 mad_ifc_flags |= MLX4_MAD_IFC_NET_VIEW;
1491
1492         err = mlx4_MAD_IFC(dev, mad_ifc_flags, 1, NULL, NULL, in_mad, out_mad);
1493         if (err)
1494                 goto out;
1495
1496         memcpy(dev->ib_dev.node_desc, out_mad->data, 64);
1497
1498         in_mad->attr_id = IB_SMP_ATTR_NODE_INFO;
1499
1500         err = mlx4_MAD_IFC(dev, mad_ifc_flags, 1, NULL, NULL, in_mad, out_mad);
1501         if (err)
1502                 goto out;
1503
1504         dev->dev->rev_id = be32_to_cpup((__be32 *) (out_mad->data + 32));
1505         memcpy(&dev->ib_dev.node_guid, out_mad->data + 12, 8);
1506
1507 out:
1508         kfree(in_mad);
1509         kfree(out_mad);
1510         return err;
1511 }
1512
1513 #if 0   /* AKAROS */
1514
1515 static ssize_t show_hca(struct device *device, struct device_attribute *attr,
1516                         char *buf)
1517 {
1518         struct mlx4_ib_dev *dev =
1519                 container_of(device, struct mlx4_ib_dev, ib_dev.dev);
1520         return sprintf(buf, "MT%d\n", dev->dev->persist->pdev->device);
1521 }
1522
1523 static ssize_t show_fw_ver(struct device *device, struct device_attribute *attr,
1524                            char *buf)
1525 {
1526         struct mlx4_ib_dev *dev =
1527                 container_of(device, struct mlx4_ib_dev, ib_dev.dev);
1528         return sprintf(buf, "%d.%d.%d\n", (int) (dev->dev->caps.fw_ver >> 32),
1529                        (int) (dev->dev->caps.fw_ver >> 16) & 0xffff,
1530                        (int) dev->dev->caps.fw_ver & 0xffff);
1531 }
1532
1533 static ssize_t show_rev(struct device *device, struct device_attribute *attr,
1534                         char *buf)
1535 {
1536         struct mlx4_ib_dev *dev =
1537                 container_of(device, struct mlx4_ib_dev, ib_dev.dev);
1538         return sprintf(buf, "%x\n", dev->dev->rev_id);
1539 }
1540
1541 static ssize_t show_board(struct device *device, struct device_attribute *attr,
1542                           char *buf)
1543 {
1544         struct mlx4_ib_dev *dev =
1545                 container_of(device, struct mlx4_ib_dev, ib_dev.dev);
1546         return sprintf(buf, "%.*s\n", MLX4_BOARD_ID_LEN,
1547                        dev->dev->board_id);
1548 }
1549
1550 static DEVICE_ATTR(hw_rev,   S_IRUGO, show_rev,    NULL);
1551 static DEVICE_ATTR(fw_ver,   S_IRUGO, show_fw_ver, NULL);
1552 static DEVICE_ATTR(hca_type, S_IRUGO, show_hca,    NULL);
1553 static DEVICE_ATTR(board_id, S_IRUGO, show_board,  NULL);
1554
1555 static struct device_attribute *mlx4_class_attributes[] = {
1556         &dev_attr_hw_rev,
1557         &dev_attr_fw_ver,
1558         &dev_attr_hca_type,
1559         &dev_attr_board_id
1560 };
1561
1562 #endif  /* AKAROS */
1563
1564 static void mlx4_addrconf_ifid_eui48(u8 *eui, u16 vlan_id,
1565                                      struct net_device *dev)
1566 {
1567         memcpy(eui, dev->dev_addr, 3);
1568         memcpy(eui + 5, dev->dev_addr + 3, 3);
1569         if (vlan_id < 0x1000) {
1570                 eui[3] = vlan_id >> 8;
1571                 eui[4] = vlan_id & 0xff;
1572         } else {
1573                 eui[3] = 0xff;
1574                 eui[4] = 0xfe;
1575         }
1576         eui[0] ^= 2;
1577 }
1578
1579 static void update_gids_task(struct work_struct *work)
1580 {
1581         struct update_gid_work *gw = container_of(work, struct update_gid_work, work);
1582         struct mlx4_cmd_mailbox *mailbox;
1583         union ib_gid *gids;
1584         int err;
1585         struct mlx4_dev *dev = gw->dev->dev;
1586         int is_bonded = mlx4_is_bonded(dev);
1587
1588         if (!gw->dev->ib_active)
1589                 return;
1590
1591         mailbox = mlx4_alloc_cmd_mailbox(dev);
1592         if (IS_ERR(mailbox)) {
1593                 pr_warn("update gid table failed %ld\n", PTR_ERR(mailbox));
1594                 return;
1595         }
1596
1597         gids = mailbox->buf;
1598         memcpy(gids, gw->gids, sizeof gw->gids);
1599
1600         err = mlx4_cmd(dev, mailbox->dma, MLX4_SET_PORT_GID_TABLE << 8 | gw->port,
1601                        MLX4_SET_PORT_ETH_OPCODE, MLX4_CMD_SET_PORT,
1602                        MLX4_CMD_TIME_CLASS_B, MLX4_CMD_WRAPPED);
1603         if (err)
1604                 pr_warn("set port command failed\n");
1605         else
1606                 if ((gw->port == 1) || !is_bonded)
1607                         mlx4_ib_dispatch_event(gw->dev,
1608                                                is_bonded ? 1 : gw->port,
1609                                                IB_EVENT_GID_CHANGE);
1610
1611         mlx4_free_cmd_mailbox(dev, mailbox);
1612         kfree(gw);
1613 }
1614
1615 static void reset_gids_task(struct work_struct *work)
1616 {
1617         struct update_gid_work *gw =
1618                         container_of(work, struct update_gid_work, work);
1619         struct mlx4_cmd_mailbox *mailbox;
1620         union ib_gid *gids;
1621         int err;
1622         struct mlx4_dev *dev = gw->dev->dev;
1623
1624         if (!gw->dev->ib_active)
1625                 return;
1626
1627         mailbox = mlx4_alloc_cmd_mailbox(dev);
1628         if (IS_ERR(mailbox)) {
1629                 pr_warn("reset gid table failed\n");
1630                 goto free;
1631         }
1632
1633         gids = mailbox->buf;
1634         memcpy(gids, gw->gids, sizeof(gw->gids));
1635
1636         if (mlx4_ib_port_link_layer(&gw->dev->ib_dev, gw->port) ==
1637                                     IB_LINK_LAYER_ETHERNET) {
1638                 err = mlx4_cmd(dev, mailbox->dma,
1639                                MLX4_SET_PORT_GID_TABLE << 8 | gw->port,
1640                                MLX4_SET_PORT_ETH_OPCODE, MLX4_CMD_SET_PORT,
1641                                MLX4_CMD_TIME_CLASS_B,
1642                                MLX4_CMD_WRAPPED);
1643                 if (err)
1644                         pr_warn("set port %d command failed\n", gw->port);
1645         }
1646
1647         mlx4_free_cmd_mailbox(dev, mailbox);
1648 free:
1649         kfree(gw);
1650 }
1651
1652 static int update_gid_table(struct mlx4_ib_dev *dev, int port,
1653                             union ib_gid *gid, int clear,
1654                             int default_gid)
1655 {
1656         struct update_gid_work *work;
1657         int i;
1658         int need_update = 0;
1659         int free = -1;
1660         int found = -1;
1661         int max_gids;
1662
1663         if (default_gid) {
1664                 free = 0;
1665         } else {
1666                 max_gids = dev->dev->caps.gid_table_len[port];
1667                 for (i = 1; i < max_gids; ++i) {
1668                         if (!memcmp(&dev->iboe.gid_table[port - 1][i], gid,
1669                                     sizeof(*gid)))
1670                                 found = i;
1671
1672                         if (clear) {
1673                                 if (found >= 0) {
1674                                         need_update = 1;
1675                                         dev->iboe.gid_table[port - 1][found] =
1676                                                 zgid;
1677                                         break;
1678                                 }
1679                         } else {
1680                                 if (found >= 0)
1681                                         break;
1682
1683                                 if (free < 0 &&
1684                                     !memcmp(&dev->iboe.gid_table[port - 1][i],
1685                                             &zgid, sizeof(*gid)))
1686                                         free = i;
1687                         }
1688                 }
1689         }
1690
1691         if (found == -1 && !clear && free >= 0) {
1692                 dev->iboe.gid_table[port - 1][free] = *gid;
1693                 need_update = 1;
1694         }
1695
1696         if (!need_update)
1697                 return 0;
1698
1699         work = kzalloc(sizeof(*work), GFP_ATOMIC);
1700         if (!work)
1701                 return -ENOMEM;
1702
1703         memcpy(work->gids, dev->iboe.gid_table[port - 1], sizeof(work->gids));
1704         INIT_WORK(&work->work, update_gids_task);
1705         work->port = port;
1706         work->dev = dev;
1707         queue_work(wq, &work->work);
1708
1709         return 0;
1710 }
1711
1712 static void mlx4_make_default_gid(struct  net_device *dev, union ib_gid *gid)
1713 {
1714         gid->global.subnet_prefix = cpu_to_be64(0xfe80000000000000LL);
1715         mlx4_addrconf_ifid_eui48(&gid->raw[8], 0xffff, dev);
1716 }
1717
1718
1719 static int reset_gid_table(struct mlx4_ib_dev *dev, u8 port)
1720 {
1721         struct update_gid_work *work;
1722
1723         work = kzalloc(sizeof(*work), GFP_ATOMIC);
1724         if (!work)
1725                 return -ENOMEM;
1726
1727         memset(dev->iboe.gid_table[port - 1], 0, sizeof(work->gids));
1728         memset(work->gids, 0, sizeof(work->gids));
1729 #if 1   /* AKAROS */
1730         return 0;
1731 #endif  /* AKAROS */
1732         INIT_WORK(&work->work, reset_gids_task);
1733         work->dev = dev;
1734         work->port = port;
1735         queue_work(wq, &work->work);
1736         return 0;
1737 }
1738
1739 #if 0   /* AKAROS */
1740
1741 static int mlx4_ib_addr_event(int event, struct net_device *event_netdev,
1742                               struct mlx4_ib_dev *ibdev, union ib_gid *gid)
1743 {
1744         struct mlx4_ib_iboe *iboe;
1745         int port = 0;
1746         struct net_device *real_dev = rdma_vlan_dev_real_dev(event_netdev) ?
1747                                 rdma_vlan_dev_real_dev(event_netdev) :
1748                                 event_netdev;
1749         union ib_gid default_gid;
1750
1751         mlx4_make_default_gid(real_dev, &default_gid);
1752
1753         if (!memcmp(gid, &default_gid, sizeof(*gid)))
1754                 return 0;
1755
1756         if (event != NETDEV_DOWN && event != NETDEV_UP)
1757                 return 0;
1758
1759         if ((real_dev != event_netdev) &&
1760             (event == NETDEV_DOWN) &&
1761             rdma_link_local_addr((struct in6_addr *)gid))
1762                 return 0;
1763
1764         iboe = &ibdev->iboe;
1765         spin_lock_bh(&iboe->lock);
1766
1767         for (port = 1; port <= ibdev->dev->caps.num_ports; ++port)
1768                 if ((netif_is_bond_master(real_dev) &&
1769                      (real_dev == iboe->masters[port - 1])) ||
1770                      (!netif_is_bond_master(real_dev) &&
1771                      (real_dev == iboe->netdevs[port - 1])))
1772                         update_gid_table(ibdev, port, gid,
1773                                          event == NETDEV_DOWN, 0);
1774
1775         spin_unlock_bh(&iboe->lock);
1776         return 0;
1777
1778 }
1779
1780 static u8 mlx4_ib_get_dev_port(struct net_device *dev,
1781                                struct mlx4_ib_dev *ibdev)
1782 {
1783         u8 port = 0;
1784         struct mlx4_ib_iboe *iboe;
1785         struct net_device *real_dev = rdma_vlan_dev_real_dev(dev) ?
1786                                 rdma_vlan_dev_real_dev(dev) : dev;
1787
1788         iboe = &ibdev->iboe;
1789
1790         for (port = 1; port <= ibdev->dev->caps.num_ports; ++port)
1791                 if ((netif_is_bond_master(real_dev) &&
1792                      (real_dev == iboe->masters[port - 1])) ||
1793                      (!netif_is_bond_master(real_dev) &&
1794                      (real_dev == iboe->netdevs[port - 1])))
1795                         break;
1796
1797         if ((port == 0) || (port > ibdev->dev->caps.num_ports))
1798                 return 0;
1799         else
1800                 return port;
1801 }
1802
1803 static int mlx4_ib_inet_event(struct notifier_block *this, unsigned long event,
1804                                 void *ptr)
1805 {
1806         struct mlx4_ib_dev *ibdev;
1807         struct in_ifaddr *ifa = ptr;
1808         union ib_gid gid;
1809         struct net_device *event_netdev = ifa->ifa_dev->dev;
1810
1811         ipv6_addr_set_v4mapped(ifa->ifa_address, (struct in6_addr *)&gid);
1812
1813         ibdev = container_of(this, struct mlx4_ib_dev, iboe.nb_inet);
1814
1815         mlx4_ib_addr_event(event, event_netdev, ibdev, &gid);
1816         return NOTIFY_DONE;
1817 }
1818
1819 #if IS_ENABLED(CONFIG_IPV6)
1820 static int mlx4_ib_inet6_event(struct notifier_block *this, unsigned long event,
1821                                 void *ptr)
1822 {
1823         struct mlx4_ib_dev *ibdev;
1824         struct inet6_ifaddr *ifa = ptr;
1825         union  ib_gid *gid = (union ib_gid *)&ifa->addr;
1826         struct net_device *event_netdev = ifa->idev->dev;
1827
1828         ibdev = container_of(this, struct mlx4_ib_dev, iboe.nb_inet6);
1829
1830         mlx4_ib_addr_event(event, event_netdev, ibdev, gid);
1831         return NOTIFY_DONE;
1832 }
1833 #endif
1834
1835 #define MLX4_IB_INVALID_MAC     ((u64)-1)
1836 static void mlx4_ib_update_qps(struct mlx4_ib_dev *ibdev,
1837                                struct net_device *dev,
1838                                int port)
1839 {
1840         u64 new_smac = 0;
1841         u64 release_mac = MLX4_IB_INVALID_MAC;
1842         struct mlx4_ib_qp *qp;
1843
1844         read_lock(&dev_base_lock);
1845         new_smac = mlx4_mac_to_u64(dev->dev_addr);
1846         read_unlock(&dev_base_lock);
1847
1848         atomic64_set(&ibdev->iboe.mac[port - 1], new_smac);
1849
1850         /* no need for update QP1 and mac registration in non-SRIOV */
1851         if (!mlx4_is_mfunc(ibdev->dev))
1852                 return;
1853
1854         mutex_lock(&ibdev->qp1_proxy_lock[port - 1]);
1855         qp = ibdev->qp1_proxy[port - 1];
1856         if (qp) {
1857                 int new_smac_index;
1858                 u64 old_smac;
1859                 struct mlx4_update_qp_params update_params;
1860
1861                 mutex_lock(&qp->mutex);
1862                 old_smac = qp->pri.smac;
1863                 if (new_smac == old_smac)
1864                         goto unlock;
1865
1866                 new_smac_index = mlx4_register_mac(ibdev->dev, port, new_smac);
1867
1868                 if (new_smac_index < 0)
1869                         goto unlock;
1870
1871                 update_params.smac_index = new_smac_index;
1872                 if (mlx4_update_qp(ibdev->dev, qp->mqp.qpn, MLX4_UPDATE_QP_SMAC,
1873                                    &update_params)) {
1874                         release_mac = new_smac;
1875                         goto unlock;
1876                 }
1877                 /* if old port was zero, no mac was yet registered for this QP */
1878                 if (qp->pri.smac_port)
1879                         release_mac = old_smac;
1880                 qp->pri.smac = new_smac;
1881                 qp->pri.smac_port = port;
1882                 qp->pri.smac_index = new_smac_index;
1883         }
1884
1885 unlock:
1886         if (release_mac != MLX4_IB_INVALID_MAC)
1887                 mlx4_unregister_mac(ibdev->dev, port, release_mac);
1888         if (qp)
1889                 mutex_unlock(&qp->mutex);
1890         mutex_unlock(&ibdev->qp1_proxy_lock[port - 1]);
1891 }
1892
1893 static void mlx4_ib_get_dev_addr(struct net_device *dev,
1894                                  struct mlx4_ib_dev *ibdev, u8 port)
1895 {
1896         struct in_device *in_dev;
1897 #if IS_ENABLED(CONFIG_IPV6)
1898         struct inet6_dev *in6_dev;
1899         union ib_gid  *pgid;
1900         struct inet6_ifaddr *ifp;
1901         union ib_gid default_gid;
1902 #endif
1903         union ib_gid gid;
1904
1905
1906         if ((port == 0) || (port > ibdev->dev->caps.num_ports))
1907                 return;
1908
1909         /* IPv4 gids */
1910         in_dev = in_dev_get(dev);
1911         if (in_dev) {
1912                 for_ifa(in_dev) {
1913                         /*ifa->ifa_address;*/
1914                         ipv6_addr_set_v4mapped(ifa->ifa_address,
1915                                                (struct in6_addr *)&gid);
1916                         update_gid_table(ibdev, port, &gid, 0, 0);
1917                 }
1918                 endfor_ifa(in_dev);
1919                 in_dev_put(in_dev);
1920         }
1921 #if IS_ENABLED(CONFIG_IPV6)
1922         mlx4_make_default_gid(dev, &default_gid);
1923         /* IPv6 gids */
1924         in6_dev = in6_dev_get(dev);
1925         if (in6_dev) {
1926                 read_lock_bh(&in6_dev->lock);
1927                 list_for_each_entry(ifp, &in6_dev->addr_list, if_list) {
1928                         pgid = (union ib_gid *)&ifp->addr;
1929                         if (!memcmp(pgid, &default_gid, sizeof(*pgid)))
1930                                 continue;
1931                         update_gid_table(ibdev, port, pgid, 0, 0);
1932                 }
1933                 read_unlock_bh(&in6_dev->lock);
1934                 in6_dev_put(in6_dev);
1935         }
1936 #endif
1937 }
1938
1939 #endif  /* AKAROS */
1940
1941 static void mlx4_ib_set_default_gid(struct mlx4_ib_dev *ibdev,
1942                                  struct  net_device *dev, u8 port)
1943 {
1944         union ib_gid gid;
1945         mlx4_make_default_gid(dev, &gid);
1946         update_gid_table(ibdev, port, &gid, 0, 1);
1947 }
1948
1949 static int mlx4_ib_init_gid_table(struct mlx4_ib_dev *ibdev)
1950 {
1951         struct  net_device *dev;
1952         struct mlx4_ib_iboe *iboe = &ibdev->iboe;
1953         int i;
1954         int err = 0;
1955
1956         for (i = 1; i <= ibdev->num_ports; ++i) {
1957                 if (rdma_port_get_link_layer(&ibdev->ib_dev, i) ==
1958                     IB_LINK_LAYER_ETHERNET) {
1959                         err = reset_gid_table(ibdev, i);
1960                         if (err)
1961                                 goto out;
1962                 }
1963         }
1964
1965 #if 0   /* AKAROS */
1966         read_lock(&dev_base_lock);
1967         spin_lock_bh(&iboe->lock);
1968
1969         for_each_netdev(&init_net, dev) {
1970                 u8 port = mlx4_ib_get_dev_port(dev, ibdev);
1971                 /* port will be non-zero only for ETH ports */
1972                 if (port) {
1973                         mlx4_ib_set_default_gid(ibdev, dev, port);
1974                         mlx4_ib_get_dev_addr(dev, ibdev, port);
1975                 }
1976         }
1977
1978         spin_unlock_bh(&iboe->lock);
1979         read_unlock(&dev_base_lock);
1980 #endif  /* AKAROS */
1981 out:
1982         return err;
1983 }
1984
1985 #if 0   /* AKAROS */
1986
1987 static void mlx4_ib_scan_netdevs(struct mlx4_ib_dev *ibdev,
1988                                  struct net_device *dev,
1989                                  unsigned long event)
1990
1991 {
1992         struct mlx4_ib_iboe *iboe;
1993         int update_qps_port = -1;
1994         int port;
1995
1996         iboe = &ibdev->iboe;
1997
1998         spin_lock_bh(&iboe->lock);
1999         mlx4_foreach_ib_transport_port(port, ibdev->dev) {
2000                 enum ib_port_state      port_state = IB_PORT_NOP;
2001                 struct net_device *old_master = iboe->masters[port - 1];
2002                 struct net_device *curr_netdev;
2003                 struct net_device *curr_master;
2004
2005                 iboe->netdevs[port - 1] =
2006                         mlx4_get_protocol_dev(ibdev->dev, MLX4_PROT_ETH, port);
2007                 if (iboe->netdevs[port - 1])
2008                         mlx4_ib_set_default_gid(ibdev,
2009                                                 iboe->netdevs[port - 1], port);
2010                 curr_netdev = iboe->netdevs[port - 1];
2011
2012                 if (iboe->netdevs[port - 1] &&
2013                     netif_is_bond_slave(iboe->netdevs[port - 1])) {
2014                         iboe->masters[port - 1] = netdev_master_upper_dev_get(
2015                                 iboe->netdevs[port - 1]);
2016                 } else {
2017                         iboe->masters[port - 1] = NULL;
2018                 }
2019                 curr_master = iboe->masters[port - 1];
2020
2021                 if (dev == iboe->netdevs[port - 1] &&
2022                     (event == NETDEV_CHANGEADDR || event == NETDEV_REGISTER ||
2023                      event == NETDEV_UP || event == NETDEV_CHANGE))
2024                         update_qps_port = port;
2025
2026                 if (curr_netdev) {
2027                         port_state = (netif_running(curr_netdev) && netif_carrier_ok(curr_netdev)) ?
2028                                                 IB_PORT_ACTIVE : IB_PORT_DOWN;
2029                         mlx4_ib_set_default_gid(ibdev, curr_netdev, port);
2030                         if (curr_master) {
2031                                 /* if using bonding/team and a slave port is down, we
2032                                  * don't want the bond IP based gids in the table since
2033                                  * flows that select port by gid may get the down port.
2034                                 */
2035                                 if (port_state == IB_PORT_DOWN &&
2036                                     !mlx4_is_bonded(ibdev->dev)) {
2037                                         reset_gid_table(ibdev, port);
2038                                         mlx4_ib_set_default_gid(ibdev,
2039                                                                 curr_netdev,
2040                                                                 port);
2041                                 } else {
2042                                         /* gids from the upper dev (bond/team)
2043                                          * should appear in port's gid table
2044                                         */
2045                                         mlx4_ib_get_dev_addr(curr_master,
2046                                                              ibdev, port);
2047                                 }
2048                         }
2049                         /* if bonding is used it is possible that we add it to
2050                          * masters only after IP address is assigned to the
2051                          * net bonding interface.
2052                         */
2053                         if (curr_master && (old_master != curr_master)) {
2054                                 reset_gid_table(ibdev, port);
2055                                 mlx4_ib_set_default_gid(ibdev,
2056                                                         curr_netdev, port);
2057                                 mlx4_ib_get_dev_addr(curr_master, ibdev, port);
2058                         }
2059
2060                         if (!curr_master && (old_master != curr_master)) {
2061                                 reset_gid_table(ibdev, port);
2062                                 mlx4_ib_set_default_gid(ibdev,
2063                                                         curr_netdev, port);
2064                                 mlx4_ib_get_dev_addr(curr_netdev, ibdev, port);
2065                         }
2066                 } else {
2067                         reset_gid_table(ibdev, port);
2068                 }
2069         }
2070
2071         spin_unlock_bh(&iboe->lock);
2072
2073         if (update_qps_port > 0)
2074                 mlx4_ib_update_qps(ibdev, dev, update_qps_port);
2075 }
2076
2077 static int mlx4_ib_netdev_event(struct notifier_block *this,
2078                                 unsigned long event, void *ptr)
2079 {
2080         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
2081         struct mlx4_ib_dev *ibdev;
2082
2083         if (!net_eq(dev_net(dev), &init_net))
2084                 return NOTIFY_DONE;
2085
2086         ibdev = container_of(this, struct mlx4_ib_dev, iboe.nb);
2087         mlx4_ib_scan_netdevs(ibdev, dev, event);
2088
2089         return NOTIFY_DONE;
2090 }
2091
2092 #endif  /* AKAROS */
2093
2094 static void init_pkeys(struct mlx4_ib_dev *ibdev)
2095 {
2096         int port;
2097         int slave;
2098         int i;
2099
2100         if (mlx4_is_master(ibdev->dev)) {
2101                 for (slave = 0; slave <= ibdev->dev->persist->num_vfs;
2102                      ++slave) {
2103                         for (port = 1; port <= ibdev->dev->caps.num_ports; ++port) {
2104                                 for (i = 0;
2105                                      i < ibdev->dev->phys_caps.pkey_phys_table_len[port];
2106                                      ++i) {
2107                                         ibdev->pkeys.virt2phys_pkey[slave][port - 1][i] =
2108                                         /* master has the identity virt2phys pkey mapping */
2109                                                 (slave == mlx4_master_func_num(ibdev->dev) || !i) ? i :
2110                                                         ibdev->dev->phys_caps.pkey_phys_table_len[port] - 1;
2111                                         mlx4_sync_pkey_table(ibdev->dev, slave, port, i,
2112                                                              ibdev->pkeys.virt2phys_pkey[slave][port - 1][i]);
2113                                 }
2114                         }
2115                 }
2116                 /* initialize pkey cache */
2117                 for (port = 1; port <= ibdev->dev->caps.num_ports; ++port) {
2118                         for (i = 0;
2119                              i < ibdev->dev->phys_caps.pkey_phys_table_len[port];
2120                              ++i)
2121                                 ibdev->pkeys.phys_pkey_cache[port-1][i] =
2122                                         (i) ? 0 : 0xFFFF;
2123                 }
2124         }
2125 }
2126
2127 static void mlx4_ib_alloc_eqs(struct mlx4_dev *dev, struct mlx4_ib_dev *ibdev)
2128 {
2129         char name[80];
2130         int eq_per_port = 0;
2131         int added_eqs = 0;
2132         int total_eqs = 0;
2133         int i, j, eq;
2134
2135         /* Legacy mode or comp_pool is not large enough */
2136         if (dev->caps.comp_pool == 0 ||
2137             dev->caps.num_ports > dev->caps.comp_pool)
2138                 return;
2139
2140         eq_per_port = dev->caps.comp_pool / dev->caps.num_ports;
2141
2142         /* Init eq table */
2143         added_eqs = 0;
2144         mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_IB)
2145                 added_eqs += eq_per_port;
2146
2147         total_eqs = dev->caps.num_comp_vectors + added_eqs;
2148
2149         ibdev->eq_table = kzalloc(total_eqs * sizeof(int), GFP_KERNEL);
2150         if (!ibdev->eq_table)
2151                 return;
2152
2153         ibdev->eq_added = added_eqs;
2154
2155         eq = 0;
2156         mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_IB) {
2157                 for (j = 0; j < eq_per_port; j++) {
2158 #if 0   /* AKAROS */
2159                         snprintf(name, sizeof(name), "mlx4-ib-%d-%d@%s",
2160                                  i, j, dev->persist->pdev->bus->name);
2161 #else   /* AKAROS */
2162                         snprintf(name, sizeof(name), "mlx4-ib-%d-%d@%s",
2163                                  i, j, dev->persist->pdev->bus);
2164 #endif  /* AKAROS */
2165                         /* Set IRQ for specific name (per ring) */
2166                         if (mlx4_assign_eq(dev, name, NULL,
2167                                            &ibdev->eq_table[eq])) {
2168                                 /* Use legacy (same as mlx4_en driver) */
2169                                 pr_warn("Can't allocate EQ %d; reverting to legacy\n", eq);
2170                                 ibdev->eq_table[eq] =
2171                                         (eq % dev->caps.num_comp_vectors);
2172                         }
2173                         eq++;
2174                 }
2175         }
2176
2177         /* Fill the reset of the vector with legacy EQ */
2178         for (i = 0, eq = added_eqs; i < dev->caps.num_comp_vectors; i++)
2179                 ibdev->eq_table[eq++] = i;
2180
2181         /* Advertise the new number of EQs to clients */
2182         ibdev->ib_dev.num_comp_vectors = total_eqs;
2183 }
2184
2185 static void mlx4_ib_free_eqs(struct mlx4_dev *dev, struct mlx4_ib_dev *ibdev)
2186 {
2187         int i;
2188
2189         /* no additional eqs were added */
2190         if (!ibdev->eq_table)
2191                 return;
2192
2193         /* Reset the advertised EQ number */
2194         ibdev->ib_dev.num_comp_vectors = dev->caps.num_comp_vectors;
2195
2196         /* Free only the added eqs */
2197         for (i = 0; i < ibdev->eq_added; i++) {
2198                 /* Don't free legacy eqs if used */
2199                 if (ibdev->eq_table[i] <= dev->caps.num_comp_vectors)
2200                         continue;
2201                 mlx4_release_eq(dev, ibdev->eq_table[i]);
2202         }
2203
2204         kfree(ibdev->eq_table);
2205 }
2206
2207 static void *mlx4_ib_add(struct mlx4_dev *dev)
2208 {
2209         struct mlx4_ib_dev *ibdev;
2210         int num_ports = 0;
2211         int i, j;
2212         int err;
2213         struct mlx4_ib_iboe *iboe;
2214         int ib_num_ports = 0;
2215         int num_req_counters;
2216
2217         pr_info_once("%s", mlx4_ib_version);
2218
2219         num_ports = 0;
2220         mlx4_foreach_ib_transport_port(i, dev)
2221                 num_ports++;
2222
2223         /* No point in registering a device with no ports... */
2224         if (num_ports == 0)
2225                 return NULL;
2226
2227         ibdev = (struct mlx4_ib_dev *) ib_alloc_device(sizeof *ibdev);
2228         if (!ibdev) {
2229                 dev_err(&dev->persist->pdev->dev,
2230                         "Device struct alloc failed\n");
2231                 return NULL;
2232         }
2233
2234         iboe = &ibdev->iboe;
2235
2236         if (mlx4_pd_alloc(dev, &ibdev->priv_pdn))
2237                 goto err_dealloc;
2238
2239         if (mlx4_uar_alloc(dev, &ibdev->priv_uar))
2240                 goto err_pd;
2241
2242         ibdev->uar_map = ioremap((phys_addr_t) ibdev->priv_uar.pfn << PAGE_SHIFT,
2243                                  PAGE_SIZE);
2244         if (!ibdev->uar_map)
2245                 goto err_uar;
2246         MLX4_INIT_DOORBELL_LOCK(&ibdev->uar_lock);
2247
2248         ibdev->dev = dev;
2249         ibdev->bond_next_port   = 0;
2250
2251         strlcpy(ibdev->ib_dev.name, "mlx4_%d", IB_DEVICE_NAME_MAX);
2252         ibdev->ib_dev.owner             = THIS_MODULE;
2253         ibdev->ib_dev.node_type         = RDMA_NODE_IB_CA;
2254         ibdev->ib_dev.local_dma_lkey    = dev->caps.reserved_lkey;
2255         ibdev->num_ports                = num_ports;
2256         ibdev->ib_dev.phys_port_cnt     = mlx4_is_bonded(dev) ?
2257                                                 1 : ibdev->num_ports;
2258         ibdev->ib_dev.num_comp_vectors  = dev->caps.num_comp_vectors;
2259 #if 0   /* AKAROS */
2260         ibdev->ib_dev.dma_device        = &dev->persist->pdev->dev;
2261 #endif  /* AKAROS */
2262
2263         if (dev->caps.userspace_caps)
2264                 ibdev->ib_dev.uverbs_abi_ver = MLX4_IB_UVERBS_ABI_VERSION;
2265         else
2266                 ibdev->ib_dev.uverbs_abi_ver = MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION;
2267
2268 #if 0   /* AKAROS */
2269         ibdev->ib_dev.uverbs_cmd_mask   =
2270                 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT)         |
2271                 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE)        |
2272                 (1ull << IB_USER_VERBS_CMD_QUERY_PORT)          |
2273                 (1ull << IB_USER_VERBS_CMD_ALLOC_PD)            |
2274                 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD)          |
2275                 (1ull << IB_USER_VERBS_CMD_REG_MR)              |
2276                 (1ull << IB_USER_VERBS_CMD_REREG_MR)            |
2277                 (1ull << IB_USER_VERBS_CMD_DEREG_MR)            |
2278                 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
2279                 (1ull << IB_USER_VERBS_CMD_CREATE_CQ)           |
2280                 (1ull << IB_USER_VERBS_CMD_RESIZE_CQ)           |
2281                 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ)          |
2282                 (1ull << IB_USER_VERBS_CMD_CREATE_QP)           |
2283                 (1ull << IB_USER_VERBS_CMD_MODIFY_QP)           |
2284                 (1ull << IB_USER_VERBS_CMD_QUERY_QP)            |
2285                 (1ull << IB_USER_VERBS_CMD_DESTROY_QP)          |
2286                 (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST)        |
2287                 (1ull << IB_USER_VERBS_CMD_DETACH_MCAST)        |
2288                 (1ull << IB_USER_VERBS_CMD_CREATE_SRQ)          |
2289                 (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ)          |
2290                 (1ull << IB_USER_VERBS_CMD_QUERY_SRQ)           |
2291                 (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ)         |
2292                 (1ull << IB_USER_VERBS_CMD_CREATE_XSRQ)         |
2293                 (1ull << IB_USER_VERBS_CMD_OPEN_QP);
2294 #else   /* AKAROS */
2295         ibdev->ib_dev.uverbs_cmd_mask   =
2296                 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT)         |
2297                 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE)        |
2298                 (1ull << IB_USER_VERBS_CMD_QUERY_PORT)          |
2299                 (1ull << IB_USER_VERBS_CMD_ALLOC_PD)            |
2300                 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD)          |
2301                 (1ull << IB_USER_VERBS_CMD_REG_MR)              |
2302                 (1ull << IB_USER_VERBS_CMD_DEREG_MR)            |
2303                 (1ull << IB_USER_VERBS_CMD_CREATE_CQ)           |
2304                 (1ull << IB_USER_VERBS_CMD_RESIZE_CQ)           |
2305                 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ)          |
2306                 (1ull << IB_USER_VERBS_CMD_CREATE_QP)           |
2307                 (1ull << IB_USER_VERBS_CMD_MODIFY_QP)           |
2308                 (1ull << IB_USER_VERBS_CMD_QUERY_QP)            |
2309                 (1ull << IB_USER_VERBS_CMD_DESTROY_QP)          |
2310                 (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST)        |
2311                 (1ull << IB_USER_VERBS_CMD_DETACH_MCAST);
2312 #endif  /* AKAROS */
2313
2314         ibdev->ib_dev.query_device      = mlx4_ib_query_device;
2315         ibdev->ib_dev.query_port        = mlx4_ib_query_port;
2316         ibdev->ib_dev.get_link_layer    = mlx4_ib_port_link_layer;
2317         ibdev->ib_dev.query_gid         = mlx4_ib_query_gid;
2318         ibdev->ib_dev.query_pkey        = mlx4_ib_query_pkey;
2319         ibdev->ib_dev.modify_device     = mlx4_ib_modify_device;
2320         ibdev->ib_dev.modify_port       = mlx4_ib_modify_port;
2321         ibdev->ib_dev.alloc_ucontext    = mlx4_ib_alloc_ucontext;
2322         ibdev->ib_dev.dealloc_ucontext  = mlx4_ib_dealloc_ucontext;
2323         ibdev->ib_dev.mmap              = mlx4_ib_mmap;
2324         ibdev->ib_dev.alloc_pd          = mlx4_ib_alloc_pd;
2325         ibdev->ib_dev.dealloc_pd        = mlx4_ib_dealloc_pd;
2326         ibdev->ib_dev.create_ah         = mlx4_ib_create_ah;
2327 #if 0   /* AKAROS */
2328         ibdev->ib_dev.query_ah          = mlx4_ib_query_ah;
2329 #endif  /* AKAROS */
2330         ibdev->ib_dev.destroy_ah        = mlx4_ib_destroy_ah;
2331 #if 0   /* AKAROS */
2332         ibdev->ib_dev.create_srq        = mlx4_ib_create_srq;
2333         ibdev->ib_dev.modify_srq        = mlx4_ib_modify_srq;
2334         ibdev->ib_dev.query_srq         = mlx4_ib_query_srq;
2335         ibdev->ib_dev.destroy_srq       = mlx4_ib_destroy_srq;
2336         ibdev->ib_dev.post_srq_recv     = mlx4_ib_post_srq_recv;
2337 #endif  /* AKAROS */
2338         ibdev->ib_dev.create_qp         = mlx4_ib_create_qp;
2339         ibdev->ib_dev.modify_qp         = mlx4_ib_modify_qp;
2340         ibdev->ib_dev.query_qp          = mlx4_ib_query_qp;
2341         ibdev->ib_dev.destroy_qp        = mlx4_ib_destroy_qp;
2342         ibdev->ib_dev.post_send         = mlx4_ib_post_send;
2343         ibdev->ib_dev.post_recv         = mlx4_ib_post_recv;
2344         ibdev->ib_dev.create_cq         = mlx4_ib_create_cq;
2345         ibdev->ib_dev.modify_cq         = mlx4_ib_modify_cq;
2346         ibdev->ib_dev.resize_cq         = mlx4_ib_resize_cq;
2347         ibdev->ib_dev.destroy_cq        = mlx4_ib_destroy_cq;
2348         ibdev->ib_dev.poll_cq           = mlx4_ib_poll_cq;
2349         ibdev->ib_dev.req_notify_cq     = mlx4_ib_arm_cq;
2350         ibdev->ib_dev.get_dma_mr        = mlx4_ib_get_dma_mr;
2351         ibdev->ib_dev.reg_user_mr       = mlx4_ib_reg_user_mr;
2352 #if 0   /* AKAROS */
2353         ibdev->ib_dev.rereg_user_mr     = mlx4_ib_rereg_user_mr;
2354 #endif  /* AKAROS */
2355         ibdev->ib_dev.dereg_mr          = mlx4_ib_dereg_mr;
2356 #if 0   /* AKAROS */
2357         ibdev->ib_dev.alloc_fast_reg_mr = mlx4_ib_alloc_fast_reg_mr;
2358         ibdev->ib_dev.alloc_fast_reg_page_list = mlx4_ib_alloc_fast_reg_page_list;
2359         ibdev->ib_dev.free_fast_reg_page_list  = mlx4_ib_free_fast_reg_page_list;
2360 #endif  /* AKAROS */
2361         ibdev->ib_dev.attach_mcast      = mlx4_ib_mcg_attach;
2362         ibdev->ib_dev.detach_mcast      = mlx4_ib_mcg_detach;
2363 #if 0   /* AKAROS */
2364         ibdev->ib_dev.process_mad       = mlx4_ib_process_mad;
2365 #endif  /* AKAROS */
2366
2367         if (!mlx4_is_slave(ibdev->dev)) {
2368                 ibdev->ib_dev.alloc_fmr         = mlx4_ib_fmr_alloc;
2369                 ibdev->ib_dev.map_phys_fmr      = mlx4_ib_map_phys_fmr;
2370                 ibdev->ib_dev.unmap_fmr         = mlx4_ib_unmap_fmr;
2371                 ibdev->ib_dev.dealloc_fmr       = mlx4_ib_fmr_dealloc;
2372         }
2373
2374         if (dev->caps.flags & MLX4_DEV_CAP_FLAG_MEM_WINDOW ||
2375             dev->caps.bmme_flags & MLX4_BMME_FLAG_TYPE_2_WIN) {
2376                 ibdev->ib_dev.alloc_mw = mlx4_ib_alloc_mw;
2377                 ibdev->ib_dev.bind_mw = mlx4_ib_bind_mw;
2378                 ibdev->ib_dev.dealloc_mw = mlx4_ib_dealloc_mw;
2379
2380                 ibdev->ib_dev.uverbs_cmd_mask |=
2381                         (1ull << IB_USER_VERBS_CMD_ALLOC_MW) |
2382                         (1ull << IB_USER_VERBS_CMD_DEALLOC_MW);
2383         }
2384
2385         if (dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC) {
2386                 ibdev->ib_dev.alloc_xrcd = mlx4_ib_alloc_xrcd;
2387                 ibdev->ib_dev.dealloc_xrcd = mlx4_ib_dealloc_xrcd;
2388                 ibdev->ib_dev.uverbs_cmd_mask |=
2389                         (1ull << IB_USER_VERBS_CMD_OPEN_XRCD) |
2390                         (1ull << IB_USER_VERBS_CMD_CLOSE_XRCD);
2391         }
2392
2393         if (check_flow_steering_support(dev)) {
2394                 ibdev->steering_support = MLX4_STEERING_MODE_DEVICE_MANAGED;
2395                 ibdev->ib_dev.create_flow       = mlx4_ib_create_flow;
2396                 ibdev->ib_dev.destroy_flow      = mlx4_ib_destroy_flow;
2397
2398                 ibdev->ib_dev.uverbs_ex_cmd_mask        |=
2399                         (1ull << IB_USER_VERBS_EX_CMD_CREATE_FLOW) |
2400                         (1ull << IB_USER_VERBS_EX_CMD_DESTROY_FLOW);
2401         }
2402
2403         mlx4_ib_alloc_eqs(dev, ibdev);
2404
2405         spin_lock_init(&iboe->lock);
2406
2407         if (init_node_data(ibdev))
2408                 goto err_map;
2409
2410         num_req_counters = mlx4_is_bonded(dev) ? 1 : ibdev->num_ports;
2411         for (i = 0; i < num_req_counters; ++i) {
2412                 mutex_init(&ibdev->qp1_proxy_lock[i]);
2413                 if (mlx4_ib_port_link_layer(&ibdev->ib_dev, i + 1) ==
2414                                                 IB_LINK_LAYER_ETHERNET) {
2415                         err = mlx4_counter_alloc(ibdev->dev, &ibdev->counters[i]);
2416                         if (err)
2417                                 ibdev->counters[i] = -1;
2418                 } else {
2419                         ibdev->counters[i] = -1;
2420                 }
2421         }
2422         if (mlx4_is_bonded(dev))
2423                 for (i = 1; i < ibdev->num_ports ; ++i)
2424                         ibdev->counters[i] = ibdev->counters[0];
2425
2426
2427         mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_IB)
2428                 ib_num_ports++;
2429
2430         spin_lock_init(&ibdev->sm_lock);
2431         mutex_init(&ibdev->cap_mask_mutex);
2432         INIT_LIST_HEAD(&ibdev->qp_list);
2433         spin_lock_init(&ibdev->reset_flow_resource_lock);
2434
2435         if (ibdev->steering_support == MLX4_STEERING_MODE_DEVICE_MANAGED &&
2436             ib_num_ports) {
2437                 ibdev->steer_qpn_count = MLX4_IB_UC_MAX_NUM_QPS;
2438                 err = mlx4_qp_reserve_range(dev, ibdev->steer_qpn_count,
2439                                             MLX4_IB_UC_STEER_QPN_ALIGN,
2440                                             &ibdev->steer_qpn_base, 0);
2441                 if (err)
2442                         goto err_counter;
2443
2444                 ibdev->ib_uc_qpns_bitmap =
2445                         kmalloc(BITS_TO_LONGS(ibdev->steer_qpn_count) *
2446                                 sizeof(long),
2447                                 GFP_KERNEL);
2448                 if (!ibdev->ib_uc_qpns_bitmap) {
2449                         dev_err(&dev->persist->pdev->dev,
2450                                 "bit map alloc failed\n");
2451                         goto err_steer_qp_release;
2452                 }
2453
2454                 bitmap_zero(ibdev->ib_uc_qpns_bitmap, ibdev->steer_qpn_count);
2455
2456                 err = mlx4_FLOW_STEERING_IB_UC_QP_RANGE(
2457                                 dev, ibdev->steer_qpn_base,
2458                                 ibdev->steer_qpn_base +
2459                                 ibdev->steer_qpn_count - 1);
2460                 if (err)
2461                         goto err_steer_free_bitmap;
2462         }
2463
2464         for (j = 1; j <= ibdev->dev->caps.num_ports; j++)
2465                 atomic64_set(&iboe->mac[j - 1], ibdev->dev->caps.def_mac[j]);
2466
2467         if (ib_register_device(&ibdev->ib_dev, NULL))
2468                 goto err_steer_free_bitmap;
2469
2470         if (mlx4_ib_mad_init(ibdev))
2471                 goto err_reg;
2472
2473         if (mlx4_ib_init_sriov(ibdev))
2474                 goto err_mad;
2475
2476         if (dev->caps.flags & MLX4_DEV_CAP_FLAG_IBOE) {
2477 #if 0   /* AKAROS */
2478                 if (!iboe->nb.notifier_call) {
2479                         iboe->nb.notifier_call = mlx4_ib_netdev_event;
2480                         err = register_netdevice_notifier(&iboe->nb);
2481                         if (err) {
2482                                 iboe->nb.notifier_call = NULL;
2483                                 goto err_notif;
2484                         }
2485                 }
2486                 if (!iboe->nb_inet.notifier_call) {
2487                         iboe->nb_inet.notifier_call = mlx4_ib_inet_event;
2488                         err = register_inetaddr_notifier(&iboe->nb_inet);
2489                         if (err) {
2490                                 iboe->nb_inet.notifier_call = NULL;
2491                                 goto err_notif;
2492                         }
2493                 }
2494 #if IS_ENABLED(CONFIG_IPV6)
2495                 if (!iboe->nb_inet6.notifier_call) {
2496                         iboe->nb_inet6.notifier_call = mlx4_ib_inet6_event;
2497                         err = register_inet6addr_notifier(&iboe->nb_inet6);
2498                         if (err) {
2499                                 iboe->nb_inet6.notifier_call = NULL;
2500                                 goto err_notif;
2501                         }
2502                 }
2503 #endif
2504 #endif  /* AKAROS */
2505                 if (mlx4_ib_init_gid_table(ibdev))
2506                         goto err_notif;
2507         }
2508
2509 #if 0   /* AKAROS */
2510         for (j = 0; j < ARRAY_SIZE(mlx4_class_attributes); ++j) {
2511                 if (device_create_file(&ibdev->ib_dev.dev,
2512                                        mlx4_class_attributes[j]))
2513                         goto err_notif;
2514         }
2515 #endif  /* AKAROS */
2516
2517         ibdev->ib_active = true;
2518
2519         if (mlx4_is_mfunc(ibdev->dev))
2520                 init_pkeys(ibdev);
2521
2522         /* create paravirt contexts for any VFs which are active */
2523         if (mlx4_is_master(ibdev->dev)) {
2524                 for (j = 0; j < MLX4_MFUNC_MAX; j++) {
2525                         if (j == mlx4_master_func_num(ibdev->dev))
2526                                 continue;
2527                         if (mlx4_is_slave_active(ibdev->dev, j))
2528                                 do_slave_init(ibdev, j, 1);
2529                 }
2530         }
2531         return ibdev;
2532
2533 err_notif:
2534 #if 0   /* AKAROS */
2535         if (ibdev->iboe.nb.notifier_call) {
2536                 if (unregister_netdevice_notifier(&ibdev->iboe.nb))
2537                         pr_warn("failure unregistering notifier\n");
2538                 ibdev->iboe.nb.notifier_call = NULL;
2539         }
2540         if (ibdev->iboe.nb_inet.notifier_call) {
2541                 if (unregister_inetaddr_notifier(&ibdev->iboe.nb_inet))
2542                         pr_warn("failure unregistering notifier\n");
2543                 ibdev->iboe.nb_inet.notifier_call = NULL;
2544         }
2545 #if IS_ENABLED(CONFIG_IPV6)
2546         if (ibdev->iboe.nb_inet6.notifier_call) {
2547                 if (unregister_inet6addr_notifier(&ibdev->iboe.nb_inet6))
2548                         pr_warn("failure unregistering notifier\n");
2549                 ibdev->iboe.nb_inet6.notifier_call = NULL;
2550         }
2551 #endif  /* AKAROS */
2552 #endif
2553         flush_workqueue(wq);
2554
2555         mlx4_ib_close_sriov(ibdev);
2556
2557 err_mad:
2558         mlx4_ib_mad_cleanup(ibdev);
2559
2560 err_reg:
2561         ib_unregister_device(&ibdev->ib_dev);
2562
2563 err_steer_free_bitmap:
2564         kfree(ibdev->ib_uc_qpns_bitmap);
2565
2566 err_steer_qp_release:
2567         if (ibdev->steering_support == MLX4_STEERING_MODE_DEVICE_MANAGED)
2568                 mlx4_qp_release_range(dev, ibdev->steer_qpn_base,
2569                                       ibdev->steer_qpn_count);
2570 err_counter:
2571         for (; i; --i)
2572                 if (ibdev->counters[i - 1] != -1)
2573                         mlx4_counter_free(ibdev->dev, ibdev->counters[i - 1]);
2574
2575 err_map:
2576         iounmap(ibdev->uar_map);
2577
2578 err_uar:
2579         mlx4_uar_free(dev, &ibdev->priv_uar);
2580
2581 err_pd:
2582         mlx4_pd_free(dev, ibdev->priv_pdn);
2583
2584 err_dealloc:
2585         ib_dealloc_device(&ibdev->ib_dev);
2586
2587         return NULL;
2588 }
2589
2590 int mlx4_ib_steer_qp_alloc(struct mlx4_ib_dev *dev, int count, int *qpn)
2591 {
2592         int offset;
2593
2594         WARN_ON(!dev->ib_uc_qpns_bitmap);
2595
2596         offset = bitmap_find_free_region(dev->ib_uc_qpns_bitmap,
2597                                          dev->steer_qpn_count,
2598                                          get_count_order(count));
2599         if (offset < 0)
2600                 return offset;
2601
2602         *qpn = dev->steer_qpn_base + offset;
2603         return 0;
2604 }
2605
2606 void mlx4_ib_steer_qp_free(struct mlx4_ib_dev *dev, u32 qpn, int count)
2607 {
2608         if (!qpn ||
2609             dev->steering_support != MLX4_STEERING_MODE_DEVICE_MANAGED)
2610                 return;
2611
2612         BUG_ON(qpn < dev->steer_qpn_base);
2613
2614         bitmap_release_region(dev->ib_uc_qpns_bitmap,
2615                               qpn - dev->steer_qpn_base,
2616                               get_count_order(count));
2617 }
2618
2619 int mlx4_ib_steer_qp_reg(struct mlx4_ib_dev *mdev, struct mlx4_ib_qp *mqp,
2620                          int is_attach)
2621 {
2622         int err;
2623         size_t flow_size;
2624         struct ib_flow_attr *flow = NULL;
2625         struct ib_flow_spec_ib *ib_spec;
2626
2627         if (is_attach) {
2628                 flow_size = sizeof(struct ib_flow_attr) +
2629                             sizeof(struct ib_flow_spec_ib);
2630                 flow = kzalloc(flow_size, GFP_KERNEL);
2631                 if (!flow)
2632                         return -ENOMEM;
2633                 flow->port = mqp->port;
2634                 flow->num_of_specs = 1;
2635                 flow->size = flow_size;
2636                 ib_spec = (struct ib_flow_spec_ib *)(flow + 1);
2637                 ib_spec->type = IB_FLOW_SPEC_IB;
2638                 ib_spec->size = sizeof(struct ib_flow_spec_ib);
2639                 /* Add an empty rule for IB L2 */
2640                 memset(&ib_spec->mask, 0, sizeof(ib_spec->mask));
2641
2642                 err = __mlx4_ib_create_flow(&mqp->ibqp, flow,
2643                                             IB_FLOW_DOMAIN_NIC,
2644                                             MLX4_FS_REGULAR,
2645                                             &mqp->reg_id);
2646         } else {
2647                 err = __mlx4_ib_destroy_flow(mdev->dev, mqp->reg_id);
2648         }
2649         kfree(flow);
2650         return err;
2651 }
2652
2653 static void mlx4_ib_remove(struct mlx4_dev *dev, void *ibdev_ptr)
2654 {
2655         struct mlx4_ib_dev *ibdev = ibdev_ptr;
2656         int p;
2657
2658         ibdev->ib_active = false;
2659         flush_workqueue(wq);
2660
2661         mlx4_ib_close_sriov(ibdev);
2662         mlx4_ib_mad_cleanup(ibdev);
2663         ib_unregister_device(&ibdev->ib_dev);
2664 #if 0   /* AKAROS */
2665         if (ibdev->iboe.nb.notifier_call) {
2666                 if (unregister_netdevice_notifier(&ibdev->iboe.nb))
2667                         pr_warn("failure unregistering notifier\n");
2668                 ibdev->iboe.nb.notifier_call = NULL;
2669         }
2670 #endif  /* AKAROS */
2671
2672         if (ibdev->steering_support == MLX4_STEERING_MODE_DEVICE_MANAGED) {
2673                 mlx4_qp_release_range(dev, ibdev->steer_qpn_base,
2674                                       ibdev->steer_qpn_count);
2675                 kfree(ibdev->ib_uc_qpns_bitmap);
2676         }
2677
2678 #if 0   /* AKAROS */
2679         if (ibdev->iboe.nb_inet.notifier_call) {
2680                 if (unregister_inetaddr_notifier(&ibdev->iboe.nb_inet))
2681                         pr_warn("failure unregistering notifier\n");
2682                 ibdev->iboe.nb_inet.notifier_call = NULL;
2683         }
2684 #if IS_ENABLED(CONFIG_IPV6)
2685         if (ibdev->iboe.nb_inet6.notifier_call) {
2686                 if (unregister_inet6addr_notifier(&ibdev->iboe.nb_inet6))
2687                         pr_warn("failure unregistering notifier\n");
2688                 ibdev->iboe.nb_inet6.notifier_call = NULL;
2689         }
2690 #endif
2691 #endif  /* AKAROS */
2692
2693         iounmap(ibdev->uar_map);
2694         for (p = 0; p < ibdev->num_ports; ++p)
2695                 if (ibdev->counters[p] != -1)
2696                         mlx4_counter_free(ibdev->dev, ibdev->counters[p]);
2697         mlx4_foreach_port(p, dev, MLX4_PORT_TYPE_IB)
2698                 mlx4_CLOSE_PORT(dev, p);
2699
2700         mlx4_ib_free_eqs(dev, ibdev);
2701
2702         mlx4_uar_free(dev, &ibdev->priv_uar);
2703         mlx4_pd_free(dev, ibdev->priv_pdn);
2704         ib_dealloc_device(&ibdev->ib_dev);
2705 }
2706
2707 static void do_slave_init(struct mlx4_ib_dev *ibdev, int slave, int do_init)
2708 {
2709         struct mlx4_ib_demux_work **dm = NULL;
2710         struct mlx4_dev *dev = ibdev->dev;
2711         int i;
2712         unsigned long flags;
2713         struct mlx4_active_ports actv_ports;
2714         unsigned int ports;
2715         unsigned int first_port;
2716
2717         if (!mlx4_is_master(dev))
2718                 return;
2719
2720         actv_ports = mlx4_get_active_ports(dev, slave);
2721         ports = bitmap_weight(actv_ports.ports, dev->caps.num_ports);
2722         first_port = find_first_bit(actv_ports.ports, dev->caps.num_ports);
2723
2724         dm = kcalloc(ports, sizeof(*dm), GFP_ATOMIC);
2725         if (!dm) {
2726                 pr_err("failed to allocate memory for tunneling qp update\n");
2727                 goto out;
2728         }
2729
2730         for (i = 0; i < ports; i++) {
2731                 dm[i] = kmalloc(sizeof (struct mlx4_ib_demux_work), GFP_ATOMIC);
2732                 if (!dm[i]) {
2733                         pr_err("failed to allocate memory for tunneling qp update work struct\n");
2734                         for (i = 0; i < dev->caps.num_ports; i++) {
2735                                 if (dm[i])
2736                                         kfree(dm[i]);
2737                         }
2738                         goto out;
2739                 }
2740         }
2741         /* initialize or tear down tunnel QPs for the slave */
2742         for (i = 0; i < ports; i++) {
2743                 INIT_WORK(&dm[i]->work, mlx4_ib_tunnels_update_work);
2744                 dm[i]->port = first_port + i + 1;
2745                 dm[i]->slave = slave;
2746                 dm[i]->do_init = do_init;
2747                 dm[i]->dev = ibdev;
2748                 spin_lock_irqsave(&ibdev->sriov.going_down_lock, flags);
2749                 if (!ibdev->sriov.is_going_down)
2750                         queue_work(ibdev->sriov.demux[i].ud_wq, &dm[i]->work);
2751                 spin_unlock_irqrestore(&ibdev->sriov.going_down_lock, flags);
2752         }
2753 out:
2754         kfree(dm);
2755         return;
2756 }
2757
2758 static void mlx4_ib_handle_catas_error(struct mlx4_ib_dev *ibdev)
2759 {
2760         struct mlx4_ib_qp *mqp;
2761         unsigned long flags_qp;
2762         unsigned long flags_cq;
2763         struct mlx4_ib_cq *send_mcq, *recv_mcq;
2764         struct list_head    cq_notify_list;
2765         struct mlx4_cq *mcq;
2766         unsigned long flags;
2767
2768         pr_warn("mlx4_ib_handle_catas_error was started\n");
2769         INIT_LIST_HEAD(&cq_notify_list);
2770
2771         /* Go over qp list reside on that ibdev, sync with create/destroy qp.*/
2772         spin_lock_irqsave(&ibdev->reset_flow_resource_lock, flags);
2773
2774         list_for_each_entry(mqp, &ibdev->qp_list, qps_list) {
2775                 spin_lock_irqsave(&mqp->sq.lock, flags_qp);
2776                 if (mqp->sq.tail != mqp->sq.head) {
2777                         send_mcq = to_mcq(mqp->ibqp.send_cq);
2778                         spin_lock_irqsave(&send_mcq->lock, flags_cq);
2779                         if (send_mcq->mcq.comp &&
2780                             mqp->ibqp.send_cq->comp_handler) {
2781                                 if (!send_mcq->mcq.reset_notify_added) {
2782                                         send_mcq->mcq.reset_notify_added = 1;
2783                                         list_add_tail(&send_mcq->mcq.reset_notify,
2784                                                       &cq_notify_list);
2785                                 }
2786                         }
2787                         spin_unlock_irqrestore(&send_mcq->lock, flags_cq);
2788                 }
2789                 spin_unlock_irqrestore(&mqp->sq.lock, flags_qp);
2790                 /* Now, handle the QP's receive queue */
2791                 spin_lock_irqsave(&mqp->rq.lock, flags_qp);
2792                 /* no handling is needed for SRQ */
2793                 if (!mqp->ibqp.srq) {
2794                         if (mqp->rq.tail != mqp->rq.head) {
2795                                 recv_mcq = to_mcq(mqp->ibqp.recv_cq);
2796                                 spin_lock_irqsave(&recv_mcq->lock, flags_cq);
2797                                 if (recv_mcq->mcq.comp &&
2798                                     mqp->ibqp.recv_cq->comp_handler) {
2799                                         if (!recv_mcq->mcq.reset_notify_added) {
2800                                                 recv_mcq->mcq.reset_notify_added = 1;
2801                                                 list_add_tail(&recv_mcq->mcq.reset_notify,
2802                                                               &cq_notify_list);
2803                                         }
2804                                 }
2805                                 spin_unlock_irqrestore(&recv_mcq->lock,
2806                                                        flags_cq);
2807                         }
2808                 }
2809                 spin_unlock_irqrestore(&mqp->rq.lock, flags_qp);
2810         }
2811
2812         list_for_each_entry(mcq, &cq_notify_list, reset_notify) {
2813                 mcq->comp(mcq);
2814         }
2815         spin_unlock_irqrestore(&ibdev->reset_flow_resource_lock, flags);
2816         pr_warn("mlx4_ib_handle_catas_error ended\n");
2817 }
2818
2819 static void handle_bonded_port_state_event(struct work_struct *work)
2820 {
2821         struct ib_event_work *ew =
2822                 container_of(work, struct ib_event_work, work);
2823         struct mlx4_ib_dev *ibdev = ew->ib_dev;
2824         enum ib_port_state bonded_port_state = IB_PORT_NOP;
2825         int i;
2826         struct ib_event ibev;
2827
2828         kfree(ew);
2829         spin_lock_bh(&ibdev->iboe.lock);
2830         for (i = 0; i < MLX4_MAX_PORTS; ++i) {
2831                 struct net_device *curr_netdev = ibdev->iboe.netdevs[i];
2832                 enum ib_port_state curr_port_state;
2833
2834                 if (!curr_netdev)
2835                         continue;
2836
2837                 curr_port_state =
2838                         (netif_running(curr_netdev) &&
2839                          netif_carrier_ok(curr_netdev)) ?
2840                         IB_PORT_ACTIVE : IB_PORT_DOWN;
2841
2842                 bonded_port_state = (bonded_port_state != IB_PORT_ACTIVE) ?
2843                         curr_port_state : IB_PORT_ACTIVE;
2844         }
2845         spin_unlock_bh(&ibdev->iboe.lock);
2846
2847         ibev.device = &ibdev->ib_dev;
2848         ibev.element.port_num = 1;
2849         ibev.event = (bonded_port_state == IB_PORT_ACTIVE) ?
2850                 IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
2851
2852         ib_dispatch_event(&ibev);
2853 }
2854
2855 static void mlx4_ib_event(struct mlx4_dev *dev, void *ibdev_ptr,
2856                           enum mlx4_dev_event event, unsigned long param)
2857 {
2858         struct ib_event ibev;
2859         struct mlx4_ib_dev *ibdev = to_mdev((struct ib_device *) ibdev_ptr);
2860         struct mlx4_eqe *eqe = NULL;
2861         struct ib_event_work *ew;
2862         int p = 0;
2863
2864         if (mlx4_is_bonded(dev) &&
2865             ((event == MLX4_DEV_EVENT_PORT_UP) ||
2866             (event == MLX4_DEV_EVENT_PORT_DOWN))) {
2867                 ew = kmalloc(sizeof(*ew), GFP_ATOMIC);
2868                 if (!ew)
2869                         return;
2870                 INIT_WORK(&ew->work, handle_bonded_port_state_event);
2871                 ew->ib_dev = ibdev;
2872                 queue_work(wq, &ew->work);
2873                 return;
2874         }
2875
2876         if (event == MLX4_DEV_EVENT_PORT_MGMT_CHANGE)
2877                 eqe = (struct mlx4_eqe *)param;
2878         else
2879                 p = (int) param;
2880
2881         switch (event) {
2882         case MLX4_DEV_EVENT_PORT_UP:
2883                 if (p > ibdev->num_ports)
2884                         return;
2885                 if (mlx4_is_master(dev) &&
2886                     rdma_port_get_link_layer(&ibdev->ib_dev, p) ==
2887                         IB_LINK_LAYER_INFINIBAND) {
2888                         mlx4_ib_invalidate_all_guid_record(ibdev, p);
2889                 }
2890                 ibev.event = IB_EVENT_PORT_ACTIVE;
2891                 break;
2892
2893         case MLX4_DEV_EVENT_PORT_DOWN:
2894                 if (p > ibdev->num_ports)
2895                         return;
2896                 ibev.event = IB_EVENT_PORT_ERR;
2897                 break;
2898
2899         case MLX4_DEV_EVENT_CATASTROPHIC_ERROR:
2900                 ibdev->ib_active = false;
2901                 ibev.event = IB_EVENT_DEVICE_FATAL;
2902                 mlx4_ib_handle_catas_error(ibdev);
2903                 break;
2904
2905         case MLX4_DEV_EVENT_PORT_MGMT_CHANGE:
2906                 ew = kmalloc(sizeof *ew, GFP_ATOMIC);
2907                 if (!ew) {
2908                         pr_err("failed to allocate memory for events work\n");
2909                         break;
2910                 }
2911
2912                 INIT_WORK(&ew->work, handle_port_mgmt_change_event);
2913                 memcpy(&ew->ib_eqe, eqe, sizeof *eqe);
2914                 ew->ib_dev = ibdev;
2915                 /* need to queue only for port owner, which uses GEN_EQE */
2916                 if (mlx4_is_master(dev))
2917                         queue_work(wq, &ew->work);
2918                 else
2919                         handle_port_mgmt_change_event(&ew->work);
2920                 return;
2921
2922         case MLX4_DEV_EVENT_SLAVE_INIT:
2923                 /* here, p is the slave id */
2924                 do_slave_init(ibdev, p, 1);
2925                 if (mlx4_is_master(dev)) {
2926                         int i;
2927
2928                         for (i = 1; i <= ibdev->num_ports; i++) {
2929                                 if (rdma_port_get_link_layer(&ibdev->ib_dev, i)
2930                                         == IB_LINK_LAYER_INFINIBAND)
2931                                         mlx4_ib_slave_alias_guid_event(ibdev,
2932                                                                        p, i,
2933                                                                        1);
2934                         }
2935                 }
2936                 return;
2937
2938         case MLX4_DEV_EVENT_SLAVE_SHUTDOWN:
2939                 if (mlx4_is_master(dev)) {
2940                         int i;
2941
2942                         for (i = 1; i <= ibdev->num_ports; i++) {
2943                                 if (rdma_port_get_link_layer(&ibdev->ib_dev, i)
2944                                         == IB_LINK_LAYER_INFINIBAND)
2945                                         mlx4_ib_slave_alias_guid_event(ibdev,
2946                                                                        p, i,
2947                                                                        0);
2948                         }
2949                 }
2950                 /* here, p is the slave id */
2951                 do_slave_init(ibdev, p, 0);
2952                 return;
2953
2954         default:
2955                 return;
2956         }
2957
2958         ibev.device           = ibdev_ptr;
2959         ibev.element.port_num = mlx4_is_bonded(ibdev->dev) ? 1 : (u8)p;
2960
2961         ib_dispatch_event(&ibev);
2962 }
2963
2964 static struct mlx4_interface mlx4_ib_interface = {
2965         .add            = mlx4_ib_add,
2966         .remove         = mlx4_ib_remove,
2967         .event          = mlx4_ib_event,
2968         .protocol       = MLX4_PROT_IB_IPV6,
2969         .flags          = MLX4_INTFF_BONDING
2970 };
2971
2972 #if 0   /* AKAROS */
2973 static int __init mlx4_ib_init(void)
2974 #else   /* AKAROS */
2975 int __init mlx4_ib_init(void)
2976 #endif  /* AKAROS */
2977 {
2978         int err;
2979
2980         wq = create_singlethread_workqueue("mlx4_ib");
2981         if (!wq)
2982                 return -ENOMEM;
2983
2984         err = mlx4_ib_mcg_init();
2985         if (err)
2986                 goto clean_wq;
2987
2988         err = mlx4_register_interface(&mlx4_ib_interface);
2989         if (err)
2990                 goto clean_mcg;
2991
2992         return 0;
2993
2994 clean_mcg:
2995         mlx4_ib_mcg_destroy();
2996
2997 clean_wq:
2998         destroy_workqueue(wq);
2999         return err;
3000 }
3001
3002 static void __exit mlx4_ib_cleanup(void)
3003 {
3004         mlx4_unregister_interface(&mlx4_ib_interface);
3005         mlx4_ib_mcg_destroy();
3006         destroy_workqueue(wq);
3007 }
3008
3009 module_init(mlx4_ib_init);
3010 module_exit(mlx4_ib_cleanup);