mlx4: /dev/ -> /dev_vfs/
[akaros.git] / kern / drivers / net / mlx4 / qp.c
1 /*
2  * Copyright (c) 2004 Topspin Communications.  All rights reserved.
3  * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved.
4  * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies. All rights reserved.
5  * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
6  *
7  * This software is available to you under a choice of one of two
8  * licenses.  You may choose to be licensed under the terms of the GNU
9  * General Public License (GPL) Version 2, available from the file
10  * COPYING in the main directory of this source tree, or the
11  * OpenIB.org BSD license below:
12  *
13  *     Redistribution and use in source and binary forms, with or
14  *     without modification, are permitted provided that the following
15  *     conditions are met:
16  *
17  *      - Redistributions of source code must retain the above
18  *        copyright notice, this list of conditions and the following
19  *        disclaimer.
20  *
21  *      - Redistributions in binary form must reproduce the above
22  *        copyright notice, this list of conditions and the following
23  *        disclaimer in the documentation and/or other materials
24  *        provided with the distribution.
25  *
26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33  * SOFTWARE.
34  */
35
36 #include <linux_compat.h>
37 #include <linux/mlx4/cmd.h>
38 #include <linux/mlx4/qp.h>
39 #include "mlx4.h"
40 #include "icm.h"
41
42 /* QP to support BF should have bits 6,7 cleared */
43 #define MLX4_BF_QP_SKIP_MASK    0xc0
44 #define MLX4_MAX_BF_QP_RANGE    0x40
45
46 void mlx4_qp_event(struct mlx4_dev *dev, uint32_t qpn, int event_type)
47 {
48         struct mlx4_qp_table *qp_table = &mlx4_priv(dev)->qp_table;
49         struct mlx4_qp *qp;
50
51         spin_lock(&qp_table->lock);
52
53         qp = __mlx4_qp_lookup(dev, qpn);
54         if (qp)
55                 atomic_inc(&qp->refcount);
56
57         spin_unlock(&qp_table->lock);
58
59         if (!qp) {
60                 mlx4_dbg(dev, "Async event for none existent QP %08x\n", qpn);
61                 return;
62         }
63
64         qp->event(qp, event_type);
65
66         if (atomic_sub_and_test(&qp->refcount, 1))
67                 complete(&qp->free);
68 }
69
70 /* used for INIT/CLOSE port logic */
71 static int is_master_qp0(struct mlx4_dev *dev, struct mlx4_qp *qp, int *real_qp0, int *proxy_qp0)
72 {
73         /* this procedure is called after we already know we are on the master */
74         /* qp0 is either the proxy qp0, or the real qp0 */
75         uint32_t pf_proxy_offset = dev->phys_caps.base_proxy_sqpn + 8 * mlx4_master_func_num(dev);
76         *proxy_qp0 = qp->qpn >= pf_proxy_offset && qp->qpn <= pf_proxy_offset + 1;
77
78         *real_qp0 = qp->qpn >= dev->phys_caps.base_sqpn &&
79                 qp->qpn <= dev->phys_caps.base_sqpn + 1;
80
81         return *real_qp0 || *proxy_qp0;
82 }
83
84 static int __mlx4_qp_modify(struct mlx4_dev *dev, struct mlx4_mtt *mtt,
85                      enum mlx4_qp_state cur_state, enum mlx4_qp_state new_state,
86                      struct mlx4_qp_context *context,
87                      enum mlx4_qp_optpar optpar,
88                      int sqd_event, struct mlx4_qp *qp, int native)
89 {
90         static const uint16_t op[MLX4_QP_NUM_STATE][MLX4_QP_NUM_STATE] = {
91                 [MLX4_QP_STATE_RST] = {
92                         [MLX4_QP_STATE_RST]     = MLX4_CMD_2RST_QP,
93                         [MLX4_QP_STATE_ERR]     = MLX4_CMD_2ERR_QP,
94                         [MLX4_QP_STATE_INIT]    = MLX4_CMD_RST2INIT_QP,
95                 },
96                 [MLX4_QP_STATE_INIT]  = {
97                         [MLX4_QP_STATE_RST]     = MLX4_CMD_2RST_QP,
98                         [MLX4_QP_STATE_ERR]     = MLX4_CMD_2ERR_QP,
99                         [MLX4_QP_STATE_INIT]    = MLX4_CMD_INIT2INIT_QP,
100                         [MLX4_QP_STATE_RTR]     = MLX4_CMD_INIT2RTR_QP,
101                 },
102                 [MLX4_QP_STATE_RTR]   = {
103                         [MLX4_QP_STATE_RST]     = MLX4_CMD_2RST_QP,
104                         [MLX4_QP_STATE_ERR]     = MLX4_CMD_2ERR_QP,
105                         [MLX4_QP_STATE_RTS]     = MLX4_CMD_RTR2RTS_QP,
106                 },
107                 [MLX4_QP_STATE_RTS]   = {
108                         [MLX4_QP_STATE_RST]     = MLX4_CMD_2RST_QP,
109                         [MLX4_QP_STATE_ERR]     = MLX4_CMD_2ERR_QP,
110                         [MLX4_QP_STATE_RTS]     = MLX4_CMD_RTS2RTS_QP,
111                         [MLX4_QP_STATE_SQD]     = MLX4_CMD_RTS2SQD_QP,
112                 },
113                 [MLX4_QP_STATE_SQD] = {
114                         [MLX4_QP_STATE_RST]     = MLX4_CMD_2RST_QP,
115                         [MLX4_QP_STATE_ERR]     = MLX4_CMD_2ERR_QP,
116                         [MLX4_QP_STATE_RTS]     = MLX4_CMD_SQD2RTS_QP,
117                         [MLX4_QP_STATE_SQD]     = MLX4_CMD_SQD2SQD_QP,
118                 },
119                 [MLX4_QP_STATE_SQER] = {
120                         [MLX4_QP_STATE_RST]     = MLX4_CMD_2RST_QP,
121                         [MLX4_QP_STATE_ERR]     = MLX4_CMD_2ERR_QP,
122                         [MLX4_QP_STATE_RTS]     = MLX4_CMD_SQERR2RTS_QP,
123                 },
124                 [MLX4_QP_STATE_ERR] = {
125                         [MLX4_QP_STATE_RST]     = MLX4_CMD_2RST_QP,
126                         [MLX4_QP_STATE_ERR]     = MLX4_CMD_2ERR_QP,
127                 }
128         };
129
130         struct mlx4_priv *priv = mlx4_priv(dev);
131         struct mlx4_cmd_mailbox *mailbox;
132         int ret = 0;
133         int real_qp0 = 0;
134         int proxy_qp0 = 0;
135         uint8_t port;
136
137         if (cur_state >= MLX4_QP_NUM_STATE || new_state >= MLX4_QP_NUM_STATE ||
138             !op[cur_state][new_state])
139                 return -EINVAL;
140
141         if (op[cur_state][new_state] == MLX4_CMD_2RST_QP) {
142                 ret = mlx4_cmd(dev, 0, qp->qpn, 2,
143                         MLX4_CMD_2RST_QP, MLX4_CMD_TIME_CLASS_A, native);
144                 if (mlx4_is_master(dev) && cur_state != MLX4_QP_STATE_ERR &&
145                     cur_state != MLX4_QP_STATE_RST &&
146                     is_master_qp0(dev, qp, &real_qp0, &proxy_qp0)) {
147                         port = (qp->qpn & 1) + 1;
148                         if (proxy_qp0)
149                                 priv->mfunc.master.qp0_state[port].proxy_qp0_active = 0;
150                         else
151                                 priv->mfunc.master.qp0_state[port].qp0_active = 0;
152                 }
153                 return ret;
154         }
155
156         mailbox = mlx4_alloc_cmd_mailbox(dev);
157         if (IS_ERR(mailbox))
158                 return PTR_ERR(mailbox);
159
160         if (cur_state == MLX4_QP_STATE_RST && new_state == MLX4_QP_STATE_INIT) {
161                 uint64_t mtt_addr = mlx4_mtt_addr(dev, mtt);
162                 context->mtt_base_addr_h = mtt_addr >> 32;
163                 context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff);
164                 context->log_page_size   = mtt->page_shift - MLX4_ICM_PAGE_SHIFT;
165         }
166
167         *(__be32 *) mailbox->buf = cpu_to_be32(optpar);
168         memcpy(mailbox->buf + 8, context, sizeof *context);
169
170         ((struct mlx4_qp_context *) (mailbox->buf + 8))->local_qpn =
171                 cpu_to_be32(qp->qpn);
172
173         ret = mlx4_cmd(dev, mailbox->dma,
174                        qp->qpn | (!!sqd_event << 31),
175                        new_state == MLX4_QP_STATE_RST ? 2 : 0,
176                        op[cur_state][new_state], MLX4_CMD_TIME_CLASS_C, native);
177
178         if (mlx4_is_master(dev) && is_master_qp0(dev, qp, &real_qp0, &proxy_qp0)) {
179                 port = (qp->qpn & 1) + 1;
180                 if (cur_state != MLX4_QP_STATE_ERR &&
181                     cur_state != MLX4_QP_STATE_RST &&
182                     new_state == MLX4_QP_STATE_ERR) {
183                         if (proxy_qp0)
184                                 priv->mfunc.master.qp0_state[port].proxy_qp0_active = 0;
185                         else
186                                 priv->mfunc.master.qp0_state[port].qp0_active = 0;
187                 } else if (new_state == MLX4_QP_STATE_RTR) {
188                         if (proxy_qp0)
189                                 priv->mfunc.master.qp0_state[port].proxy_qp0_active = 1;
190                         else
191                                 priv->mfunc.master.qp0_state[port].qp0_active = 1;
192                 }
193         }
194
195         mlx4_free_cmd_mailbox(dev, mailbox);
196         return ret;
197 }
198
199 int mlx4_qp_modify(struct mlx4_dev *dev, struct mlx4_mtt *mtt,
200                    enum mlx4_qp_state cur_state, enum mlx4_qp_state new_state,
201                    struct mlx4_qp_context *context,
202                    enum mlx4_qp_optpar optpar,
203                    int sqd_event, struct mlx4_qp *qp)
204 {
205         return __mlx4_qp_modify(dev, mtt, cur_state, new_state, context,
206                                 optpar, sqd_event, qp, 0);
207 }
208 EXPORT_SYMBOL_GPL(mlx4_qp_modify);
209
210 int __mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align,
211                             int *base, uint8_t flags)
212 {
213         uint32_t uid;
214         int bf_qp = !!(flags & (uint8_t)MLX4_RESERVE_ETH_BF_QP);
215
216         struct mlx4_priv *priv = mlx4_priv(dev);
217         struct mlx4_qp_table *qp_table = &priv->qp_table;
218
219         if (cnt > MLX4_MAX_BF_QP_RANGE && bf_qp)
220                 return -ENOMEM;
221
222         uid = MLX4_QP_TABLE_ZONE_GENERAL;
223         if (flags & (uint8_t)MLX4_RESERVE_A0_QP) {
224                 if (bf_qp)
225                         uid = MLX4_QP_TABLE_ZONE_RAW_ETH;
226                 else
227                         uid = MLX4_QP_TABLE_ZONE_RSS;
228         }
229
230         *base = mlx4_zone_alloc_entries(qp_table->zones, uid, cnt, align,
231                                         bf_qp ? MLX4_BF_QP_SKIP_MASK : 0, NULL);
232         if (*base == -1)
233                 return -ENOMEM;
234
235         return 0;
236 }
237
238 int mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align,
239                           int *base, uint8_t flags)
240 {
241         uint64_t in_param = 0;
242         uint64_t out_param;
243         int err;
244
245         /* Turn off all unsupported QP allocation flags */
246         flags &= dev->caps.alloc_res_qp_mask;
247
248         if (mlx4_is_mfunc(dev)) {
249                 panic("Disabled");
250 #if 0 // AKAROS_PORT
251                 set_param_l(&in_param,
252                             (((uint32_t)flags) << 24) | (uint32_t)cnt);
253                 set_param_h(&in_param, align);
254                 err = mlx4_cmd_imm(dev, in_param, &out_param,
255                                    RES_QP, RES_OP_RESERVE,
256                                    MLX4_CMD_ALLOC_RES,
257                                    MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
258                 if (err)
259                         return err;
260
261                 *base = get_param_l(&out_param);
262                 return 0;
263 #endif
264         }
265         return __mlx4_qp_reserve_range(dev, cnt, align, base, flags);
266 }
267 EXPORT_SYMBOL_GPL(mlx4_qp_reserve_range);
268
269 void __mlx4_qp_release_range(struct mlx4_dev *dev, int base_qpn, int cnt)
270 {
271         struct mlx4_priv *priv = mlx4_priv(dev);
272         struct mlx4_qp_table *qp_table = &priv->qp_table;
273
274         if (mlx4_is_qp_reserved(dev, (uint32_t) base_qpn))
275                 return;
276         mlx4_zone_free_entries_unique(qp_table->zones, base_qpn, cnt);
277 }
278
279 void mlx4_qp_release_range(struct mlx4_dev *dev, int base_qpn, int cnt)
280 {
281         uint64_t in_param = 0;
282         int err;
283
284         if (mlx4_is_mfunc(dev)) {
285                 set_param_l(&in_param, base_qpn);
286                 set_param_h(&in_param, cnt);
287                 err = mlx4_cmd(dev, in_param, RES_QP, RES_OP_RESERVE,
288                                MLX4_CMD_FREE_RES,
289                                MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
290                 if (err) {
291                         mlx4_warn(dev, "Failed to release qp range base:%d cnt:%d\n",
292                                   base_qpn, cnt);
293                 }
294         } else
295                  __mlx4_qp_release_range(dev, base_qpn, cnt);
296 }
297 EXPORT_SYMBOL_GPL(mlx4_qp_release_range);
298
299 int __mlx4_qp_alloc_icm(struct mlx4_dev *dev, int qpn, gfp_t gfp)
300 {
301         struct mlx4_priv *priv = mlx4_priv(dev);
302         struct mlx4_qp_table *qp_table = &priv->qp_table;
303         int err;
304
305         err = mlx4_table_get(dev, &qp_table->qp_table, qpn, gfp);
306         if (err)
307                 goto err_out;
308
309         err = mlx4_table_get(dev, &qp_table->auxc_table, qpn, gfp);
310         if (err)
311                 goto err_put_qp;
312
313         err = mlx4_table_get(dev, &qp_table->altc_table, qpn, gfp);
314         if (err)
315                 goto err_put_auxc;
316
317         err = mlx4_table_get(dev, &qp_table->rdmarc_table, qpn, gfp);
318         if (err)
319                 goto err_put_altc;
320
321         err = mlx4_table_get(dev, &qp_table->cmpt_table, qpn, gfp);
322         if (err)
323                 goto err_put_rdmarc;
324
325         return 0;
326
327 err_put_rdmarc:
328         mlx4_table_put(dev, &qp_table->rdmarc_table, qpn);
329
330 err_put_altc:
331         mlx4_table_put(dev, &qp_table->altc_table, qpn);
332
333 err_put_auxc:
334         mlx4_table_put(dev, &qp_table->auxc_table, qpn);
335
336 err_put_qp:
337         mlx4_table_put(dev, &qp_table->qp_table, qpn);
338
339 err_out:
340         return err;
341 }
342
343 static int mlx4_qp_alloc_icm(struct mlx4_dev *dev, int qpn, gfp_t gfp)
344 {
345         uint64_t param = 0;
346
347         if (mlx4_is_mfunc(dev)) {
348                 set_param_l(&param, qpn);
349                 return mlx4_cmd_imm(dev, param, &param, RES_QP, RES_OP_MAP_ICM,
350                                     MLX4_CMD_ALLOC_RES, MLX4_CMD_TIME_CLASS_A,
351                                     MLX4_CMD_WRAPPED);
352         }
353         return __mlx4_qp_alloc_icm(dev, qpn, gfp);
354 }
355
356 void __mlx4_qp_free_icm(struct mlx4_dev *dev, int qpn)
357 {
358         struct mlx4_priv *priv = mlx4_priv(dev);
359         struct mlx4_qp_table *qp_table = &priv->qp_table;
360
361         mlx4_table_put(dev, &qp_table->cmpt_table, qpn);
362         mlx4_table_put(dev, &qp_table->rdmarc_table, qpn);
363         mlx4_table_put(dev, &qp_table->altc_table, qpn);
364         mlx4_table_put(dev, &qp_table->auxc_table, qpn);
365         mlx4_table_put(dev, &qp_table->qp_table, qpn);
366 }
367
368 static void mlx4_qp_free_icm(struct mlx4_dev *dev, int qpn)
369 {
370         uint64_t in_param = 0;
371
372         if (mlx4_is_mfunc(dev)) {
373                 set_param_l(&in_param, qpn);
374                 if (mlx4_cmd(dev, in_param, RES_QP, RES_OP_MAP_ICM,
375                              MLX4_CMD_FREE_RES, MLX4_CMD_TIME_CLASS_A,
376                              MLX4_CMD_WRAPPED))
377                         mlx4_warn(dev, "Failed to free icm of qp:%d\n", qpn);
378         } else
379                 __mlx4_qp_free_icm(dev, qpn);
380 }
381
382 int mlx4_qp_alloc(struct mlx4_dev *dev, int qpn, struct mlx4_qp *qp, gfp_t gfp)
383 {
384         struct mlx4_priv *priv = mlx4_priv(dev);
385         struct mlx4_qp_table *qp_table = &priv->qp_table;
386         int err;
387
388         if (!qpn)
389                 return -EINVAL;
390
391         qp->qpn = qpn;
392
393         err = mlx4_qp_alloc_icm(dev, qpn, gfp);
394         if (err)
395                 return err;
396
397         spin_lock_irqsave(&qp_table->lock);
398         err = radix_tree_insert(&dev->qp_table_tree, qp->qpn &
399                                 (dev->caps.num_qps - 1), qp);
400         spin_unlock_irqsave(&qp_table->lock);
401         if (err)
402                 goto err_icm;
403
404         atomic_set(&qp->refcount, 1);
405         init_completion(&qp->free);
406
407         return 0;
408
409 err_icm:
410         mlx4_qp_free_icm(dev, qpn);
411         return err;
412 }
413
414 EXPORT_SYMBOL_GPL(mlx4_qp_alloc);
415
416 int mlx4_update_qp(struct mlx4_dev *dev, uint32_t qpn,
417                    enum mlx4_update_qp_attr attr,
418                    struct mlx4_update_qp_params *params)
419 {
420         struct mlx4_cmd_mailbox *mailbox;
421         struct mlx4_update_qp_context *cmd;
422         uint64_t pri_addr_path_mask = 0;
423         uint64_t qp_mask = 0;
424         int err = 0;
425
426         mailbox = mlx4_alloc_cmd_mailbox(dev);
427         if (IS_ERR(mailbox))
428                 return PTR_ERR(mailbox);
429
430         cmd = (struct mlx4_update_qp_context *)mailbox->buf;
431
432         if (!attr || (attr & ~MLX4_UPDATE_QP_SUPPORTED_ATTRS))
433                 return -EINVAL;
434
435         if (attr & MLX4_UPDATE_QP_SMAC) {
436                 pri_addr_path_mask |= 1ULL << MLX4_UPD_QP_PATH_MASK_MAC_INDEX;
437                 cmd->qp_context.pri_path.grh_mylmc = params->smac_index;
438         }
439
440         if (attr & MLX4_UPDATE_QP_VSD) {
441                 qp_mask |= 1ULL << MLX4_UPD_QP_MASK_VSD;
442                 if (params->flags & MLX4_UPDATE_QP_PARAMS_FLAGS_VSD_ENABLE)
443                         cmd->qp_context.param3 |= cpu_to_be32(MLX4_STRIP_VLAN);
444         }
445
446         if (attr & MLX4_UPDATE_QP_RATE_LIMIT) {
447                 qp_mask |= 1ULL << MLX4_UPD_QP_MASK_RATE_LIMIT;
448                 cmd->qp_context.rate_limit_params = cpu_to_be16((params->rate_unit << 14) | params->rate_val);
449         }
450
451         if (attr & MLX4_UPDATE_QP_QOS_VPORT) {
452                 qp_mask |= 1ULL << MLX4_UPD_QP_MASK_QOS_VPP;
453                 cmd->qp_context.qos_vport = params->qos_vport;
454         }
455
456         cmd->primary_addr_path_mask = cpu_to_be64(pri_addr_path_mask);
457         cmd->qp_mask = cpu_to_be64(qp_mask);
458
459         err = mlx4_cmd(dev, mailbox->dma, qpn & 0xffffff, 0,
460                        MLX4_CMD_UPDATE_QP, MLX4_CMD_TIME_CLASS_A,
461                        MLX4_CMD_NATIVE);
462
463         mlx4_free_cmd_mailbox(dev, mailbox);
464         return err;
465 }
466 EXPORT_SYMBOL_GPL(mlx4_update_qp);
467
468 void mlx4_qp_remove(struct mlx4_dev *dev, struct mlx4_qp *qp)
469 {
470         struct mlx4_qp_table *qp_table = &mlx4_priv(dev)->qp_table;
471         unsigned long flags;
472
473         spin_lock_irqsave(&qp_table->lock);
474         radix_tree_delete(&dev->qp_table_tree, qp->qpn & (dev->caps.num_qps - 1));
475         spin_unlock_irqsave(&qp_table->lock);
476 }
477 EXPORT_SYMBOL_GPL(mlx4_qp_remove);
478
479 void mlx4_qp_free(struct mlx4_dev *dev, struct mlx4_qp *qp)
480 {
481         if (atomic_sub_and_test(&qp->refcount, 1))
482                 complete(&qp->free);
483         wait_for_completion(&qp->free);
484
485         mlx4_qp_free_icm(dev, qp->qpn);
486 }
487 EXPORT_SYMBOL_GPL(mlx4_qp_free);
488
489 static int mlx4_CONF_SPECIAL_QP(struct mlx4_dev *dev, uint32_t base_qpn)
490 {
491         return mlx4_cmd(dev, 0, base_qpn, 0, MLX4_CMD_CONF_SPECIAL_QP,
492                         MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE);
493 }
494
495 #define MLX4_QP_TABLE_RSS_ETH_PRIORITY 2
496 #define MLX4_QP_TABLE_RAW_ETH_PRIORITY 1
497 #define MLX4_QP_TABLE_RAW_ETH_SIZE     256
498
499 static int mlx4_create_zones(struct mlx4_dev *dev,
500                              uint32_t reserved_bottom_general,
501                              uint32_t reserved_top_general,
502                              uint32_t reserved_bottom_rss,
503                              uint32_t start_offset_rss,
504                              uint32_t max_table_offset)
505 {
506         struct mlx4_qp_table *qp_table = &mlx4_priv(dev)->qp_table;
507         struct mlx4_bitmap (*bitmap)[MLX4_QP_TABLE_ZONE_NUM] = NULL;
508         int bitmap_initialized = 0;
509         uint32_t last_offset;
510         int k;
511         int err;
512
513         qp_table->zones = mlx4_zone_allocator_create(MLX4_ZONE_ALLOC_FLAGS_NO_OVERLAP);
514
515         if (NULL == qp_table->zones)
516                 return -ENOMEM;
517
518         bitmap = kmalloc(sizeof(*bitmap), MEM_WAIT);
519
520         if (NULL == bitmap) {
521                 err = -ENOMEM;
522                 goto free_zone;
523         }
524
525         err = mlx4_bitmap_init(*bitmap + MLX4_QP_TABLE_ZONE_GENERAL, dev->caps.num_qps,
526                                (1 << 23) - 1, reserved_bottom_general,
527                                reserved_top_general);
528
529         if (err)
530                 goto free_bitmap;
531
532         ++bitmap_initialized;
533
534         err = mlx4_zone_add_one(qp_table->zones, *bitmap + MLX4_QP_TABLE_ZONE_GENERAL,
535                                 MLX4_ZONE_FALLBACK_TO_HIGHER_PRIO |
536                                 MLX4_ZONE_USE_RR, 0,
537                                 0, qp_table->zones_uids + MLX4_QP_TABLE_ZONE_GENERAL);
538
539         if (err)
540                 goto free_bitmap;
541
542         err = mlx4_bitmap_init(*bitmap + MLX4_QP_TABLE_ZONE_RSS,
543                                reserved_bottom_rss,
544                                reserved_bottom_rss - 1,
545                                dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW],
546                                reserved_bottom_rss - start_offset_rss);
547
548         if (err)
549                 goto free_bitmap;
550
551         ++bitmap_initialized;
552
553         err = mlx4_zone_add_one(qp_table->zones, *bitmap + MLX4_QP_TABLE_ZONE_RSS,
554                                 MLX4_ZONE_ALLOW_ALLOC_FROM_LOWER_PRIO |
555                                 MLX4_ZONE_ALLOW_ALLOC_FROM_EQ_PRIO |
556                                 MLX4_ZONE_USE_RR, MLX4_QP_TABLE_RSS_ETH_PRIORITY,
557                                 0, qp_table->zones_uids + MLX4_QP_TABLE_ZONE_RSS);
558
559         if (err)
560                 goto free_bitmap;
561
562         last_offset = dev->caps.reserved_qps_cnt[MLX4_QP_REGION_FW];
563         /*  We have a single zone for the A0 steering QPs area of the FW. This area
564          *  needs to be split into subareas. One set of subareas is for RSS QPs
565          *  (in which qp number bits 6 and/or 7 are set); the other set of subareas
566          *  is for RAW_ETH QPs, which require that both bits 6 and 7 are zero.
567          *  Currently, the values returned by the FW (A0 steering area starting qp number
568          *  and A0 steering area size) are such that there are only two subareas -- one
569          *  for RSS and one for RAW_ETH.
570          */
571         for (k = MLX4_QP_TABLE_ZONE_RSS + 1; k < sizeof(*bitmap)/sizeof((*bitmap)[0]);
572              k++) {
573                 int size;
574                 uint32_t offset = start_offset_rss;
575                 uint32_t bf_mask;
576                 uint32_t requested_size;
577
578                 /* Assuming MLX4_BF_QP_SKIP_MASK is consecutive ones, this calculates
579                  * a mask of all LSB bits set until (and not including) the first
580                  * set bit of  MLX4_BF_QP_SKIP_MASK. For example, if MLX4_BF_QP_SKIP_MASK
581                  * is 0xc0, bf_mask will be 0x3f.
582                  */
583                 bf_mask = (MLX4_BF_QP_SKIP_MASK & ~(MLX4_BF_QP_SKIP_MASK - 1)) - 1;
584                 requested_size = MIN((uint32_t)MLX4_QP_TABLE_RAW_ETH_SIZE,
585                                      bf_mask + 1);
586
587                 if (((last_offset & MLX4_BF_QP_SKIP_MASK) &&
588                      ((int)(max_table_offset - last_offset)) >=
589                      ROUNDUPPWR2(MLX4_BF_QP_SKIP_MASK)) ||
590                     (!(last_offset & MLX4_BF_QP_SKIP_MASK) &&
591                      !((last_offset + requested_size - 1) &
592                        MLX4_BF_QP_SKIP_MASK)))
593                         size = requested_size;
594                 else {
595                         uint32_t candidate_offset =
596                                 (last_offset | MLX4_BF_QP_SKIP_MASK | bf_mask) + 1;
597
598                         if (last_offset & MLX4_BF_QP_SKIP_MASK)
599                                 last_offset = candidate_offset;
600
601                         /* From this point, the BF bits are 0 */
602
603                         if (last_offset > max_table_offset) {
604                                 /* need to skip */
605                                 size = -1;
606                         } else {
607                                 size = min3(max_table_offset - last_offset,
608                                             bf_mask - (last_offset & bf_mask),
609                                             requested_size);
610                                 if (size < requested_size) {
611                                         int candidate_size;
612
613                                         candidate_size = min3(
614                                                 max_table_offset - candidate_offset,
615                                                 bf_mask - (last_offset & bf_mask),
616                                                 requested_size);
617
618                                         /*  We will not take this path if last_offset was
619                                          *  already set above to candidate_offset
620                                          */
621                                         if (candidate_size > size) {
622                                                 last_offset = candidate_offset;
623                                                 size = candidate_size;
624                                         }
625                                 }
626                         }
627                 }
628
629                 if (size > 0) {
630                         /* mlx4_bitmap_alloc_range will find a contiguous range of "size"
631                          * QPs in which both bits 6 and 7 are zero, because we pass it the
632                          * MLX4_BF_SKIP_MASK).
633                          */
634                         offset = mlx4_bitmap_alloc_range(
635                                         *bitmap + MLX4_QP_TABLE_ZONE_RSS,
636                                         size, 1,
637                                         MLX4_BF_QP_SKIP_MASK);
638
639                         if (offset == (uint32_t)-1) {
640                                 err = -ENOMEM;
641                                 break;
642                         }
643
644                         last_offset = offset + size;
645
646                         err = mlx4_bitmap_init(*bitmap + k, ROUNDUPPWR2(size),
647                                                ROUNDUPPWR2(size) - 1, 0,
648                                                ROUNDUPPWR2(size) - size);
649                 } else {
650                         /* Add an empty bitmap, we'll allocate from different zones (since
651                          * at least one is reserved)
652                          */
653                         err = mlx4_bitmap_init(*bitmap + k, 1,
654                                                MLX4_QP_TABLE_RAW_ETH_SIZE - 1, 0,
655                                                0);
656                         mlx4_bitmap_alloc_range(*bitmap + k, 1, 1, 0);
657                 }
658
659                 if (err)
660                         break;
661
662                 ++bitmap_initialized;
663
664                 err = mlx4_zone_add_one(qp_table->zones, *bitmap + k,
665                                         MLX4_ZONE_ALLOW_ALLOC_FROM_LOWER_PRIO |
666                                         MLX4_ZONE_ALLOW_ALLOC_FROM_EQ_PRIO |
667                                         MLX4_ZONE_USE_RR, MLX4_QP_TABLE_RAW_ETH_PRIORITY,
668                                         offset, qp_table->zones_uids + k);
669
670                 if (err)
671                         break;
672         }
673
674         if (err)
675                 goto free_bitmap;
676
677         qp_table->bitmap_gen = *bitmap;
678
679         return err;
680
681 free_bitmap:
682         for (k = 0; k < bitmap_initialized; k++)
683                 mlx4_bitmap_cleanup(*bitmap + k);
684         kfree(bitmap);
685 free_zone:
686         mlx4_zone_allocator_destroy(qp_table->zones);
687         return err;
688 }
689
690 static void mlx4_cleanup_qp_zones(struct mlx4_dev *dev)
691 {
692         struct mlx4_qp_table *qp_table = &mlx4_priv(dev)->qp_table;
693
694         if (qp_table->zones) {
695                 int i;
696
697                 for (i = 0;
698                      i < sizeof(qp_table->zones_uids)/sizeof(qp_table->zones_uids[0]);
699                      i++) {
700                         struct mlx4_bitmap *bitmap =
701                                 mlx4_zone_get_bitmap(qp_table->zones,
702                                                      qp_table->zones_uids[i]);
703
704                         mlx4_zone_remove_one(qp_table->zones, qp_table->zones_uids[i]);
705                         if (NULL == bitmap)
706                                 continue;
707
708                         mlx4_bitmap_cleanup(bitmap);
709                 }
710                 mlx4_zone_allocator_destroy(qp_table->zones);
711                 kfree(qp_table->bitmap_gen);
712                 qp_table->bitmap_gen = NULL;
713                 qp_table->zones = NULL;
714         }
715 }
716
717 int mlx4_init_qp_table(struct mlx4_dev *dev)
718 {
719         struct mlx4_qp_table *qp_table = &mlx4_priv(dev)->qp_table;
720         int err;
721         int reserved_from_top = 0;
722         int reserved_from_bot;
723         int k;
724         int fixed_reserved_from_bot_rv = 0;
725         int bottom_reserved_for_rss_bitmap;
726         uint32_t max_table_offset = dev->caps.dmfs_high_rate_qpn_base +
727                         dev->caps.dmfs_high_rate_qpn_range;
728
729         spinlock_init_irqsave(&qp_table->lock);
730         INIT_RADIX_TREE(&dev->qp_table_tree, 0);
731         if (mlx4_is_slave(dev))
732                 return 0;
733
734         /* We reserve 2 extra QPs per port for the special QPs.  The
735          * block of special QPs must be aligned to a multiple of 8, so
736          * round up.
737          *
738          * We also reserve the MSB of the 24-bit QP number to indicate
739          * that a QP is an XRC QP.
740          */
741         for (k = 0; k <= MLX4_QP_REGION_BOTTOM; k++)
742                 fixed_reserved_from_bot_rv += dev->caps.reserved_qps_cnt[k];
743
744         if (fixed_reserved_from_bot_rv < max_table_offset)
745                 fixed_reserved_from_bot_rv = max_table_offset;
746
747         /* We reserve at least 1 extra for bitmaps that we don't have enough space for*/
748         bottom_reserved_for_rss_bitmap =
749                 ROUNDUPPWR2(fixed_reserved_from_bot_rv + 1);
750         dev->phys_caps.base_sqpn = ALIGN(bottom_reserved_for_rss_bitmap, 8);
751
752         {
753                 int sort[MLX4_NUM_QP_REGION];
754                 int i, j, tmp;
755                 int last_base = dev->caps.num_qps;
756
757                 for (i = 1; i < MLX4_NUM_QP_REGION; ++i)
758                         sort[i] = i;
759
760                 for (i = MLX4_NUM_QP_REGION; i > MLX4_QP_REGION_BOTTOM; --i) {
761                         for (j = MLX4_QP_REGION_BOTTOM + 2; j < i; ++j) {
762                                 if (dev->caps.reserved_qps_cnt[sort[j]] >
763                                     dev->caps.reserved_qps_cnt[sort[j - 1]]) {
764                                         tmp             = sort[j];
765                                         sort[j]         = sort[j - 1];
766                                         sort[j - 1]     = tmp;
767                                 }
768                         }
769                 }
770
771                 for (i = MLX4_QP_REGION_BOTTOM + 1; i < MLX4_NUM_QP_REGION; ++i) {
772                         last_base -= dev->caps.reserved_qps_cnt[sort[i]];
773                         dev->caps.reserved_qps_base[sort[i]] = last_base;
774                         reserved_from_top +=
775                                 dev->caps.reserved_qps_cnt[sort[i]];
776                 }
777         }
778
779        /* Reserve 8 real SQPs in both native and SRIOV modes.
780         * In addition, in SRIOV mode, reserve 8 proxy SQPs per function
781         * (for all PFs and VFs), and 8 corresponding tunnel QPs.
782         * Each proxy SQP works opposite its own tunnel QP.
783         *
784         * The QPs are arranged as follows:
785         * a. 8 real SQPs
786         * b. All the proxy SQPs (8 per function)
787         * c. All the tunnel QPs (8 per function)
788         */
789         reserved_from_bot = mlx4_num_reserved_sqps(dev);
790         if (reserved_from_bot + reserved_from_top > dev->caps.num_qps) {
791                 mlx4_err(dev, "Number of reserved QPs is higher than number of QPs\n");
792                 return -EINVAL;
793         }
794
795         err = mlx4_create_zones(dev, reserved_from_bot, reserved_from_bot,
796                                 bottom_reserved_for_rss_bitmap,
797                                 fixed_reserved_from_bot_rv,
798                                 max_table_offset);
799
800         if (err)
801                 return err;
802
803         if (mlx4_is_mfunc(dev)) {
804                 /* for PPF use */
805                 dev->phys_caps.base_proxy_sqpn = dev->phys_caps.base_sqpn + 8;
806                 dev->phys_caps.base_tunnel_sqpn = dev->phys_caps.base_sqpn + 8 + 8 * MLX4_MFUNC_MAX;
807
808                 /* In mfunc, calculate proxy and tunnel qp offsets for the PF here,
809                  * since the PF does not call mlx4_slave_caps */
810                 dev->caps.qp0_tunnel = kzmalloc((dev->caps.num_ports) * (sizeof(uint32_t)),
811                                                 MEM_WAIT);
812                 dev->caps.qp0_proxy = kzmalloc((dev->caps.num_ports) * (sizeof(uint32_t)),
813                                                MEM_WAIT);
814                 dev->caps.qp1_tunnel = kzmalloc((dev->caps.num_ports) * (sizeof(uint32_t)),
815                                                 MEM_WAIT);
816                 dev->caps.qp1_proxy = kzmalloc((dev->caps.num_ports) * (sizeof(uint32_t)),
817                                                MEM_WAIT);
818
819                 if (!dev->caps.qp0_tunnel || !dev->caps.qp0_proxy ||
820                     !dev->caps.qp1_tunnel || !dev->caps.qp1_proxy) {
821                         err = -ENOMEM;
822                         goto err_mem;
823                 }
824
825                 for (k = 0; k < dev->caps.num_ports; k++) {
826                         dev->caps.qp0_proxy[k] = dev->phys_caps.base_proxy_sqpn +
827                                 8 * mlx4_master_func_num(dev) + k;
828                         dev->caps.qp0_tunnel[k] = dev->caps.qp0_proxy[k] + 8 * MLX4_MFUNC_MAX;
829                         dev->caps.qp1_proxy[k] = dev->phys_caps.base_proxy_sqpn +
830                                 8 * mlx4_master_func_num(dev) + MLX4_MAX_PORTS + k;
831                         dev->caps.qp1_tunnel[k] = dev->caps.qp1_proxy[k] + 8 * MLX4_MFUNC_MAX;
832                 }
833         }
834
835
836         err = mlx4_CONF_SPECIAL_QP(dev, dev->phys_caps.base_sqpn);
837         if (err)
838                 goto err_mem;
839
840         return err;
841
842 err_mem:
843         kfree(dev->caps.qp0_tunnel);
844         kfree(dev->caps.qp0_proxy);
845         kfree(dev->caps.qp1_tunnel);
846         kfree(dev->caps.qp1_proxy);
847         dev->caps.qp0_tunnel = dev->caps.qp0_proxy =
848                 dev->caps.qp1_tunnel = dev->caps.qp1_proxy = NULL;
849         mlx4_cleanup_qp_zones(dev);
850         return err;
851 }
852
853 void mlx4_cleanup_qp_table(struct mlx4_dev *dev)
854 {
855         if (mlx4_is_slave(dev))
856                 return;
857
858         mlx4_CONF_SPECIAL_QP(dev, 0);
859
860         mlx4_cleanup_qp_zones(dev);
861 }
862
863 int mlx4_qp_query(struct mlx4_dev *dev, struct mlx4_qp *qp,
864                   struct mlx4_qp_context *context)
865 {
866         struct mlx4_cmd_mailbox *mailbox;
867         int err;
868
869         mailbox = mlx4_alloc_cmd_mailbox(dev);
870         if (IS_ERR(mailbox))
871                 return PTR_ERR(mailbox);
872
873         err = mlx4_cmd_box(dev, 0, mailbox->dma, qp->qpn, 0,
874                            MLX4_CMD_QUERY_QP, MLX4_CMD_TIME_CLASS_A,
875                            MLX4_CMD_WRAPPED);
876         if (!err)
877                 memcpy(context, mailbox->buf + 8, sizeof *context);
878
879         mlx4_free_cmd_mailbox(dev, mailbox);
880         return err;
881 }
882 EXPORT_SYMBOL_GPL(mlx4_qp_query);
883
884 int mlx4_qp_to_ready(struct mlx4_dev *dev, struct mlx4_mtt *mtt,
885                      struct mlx4_qp_context *context,
886                      struct mlx4_qp *qp, enum mlx4_qp_state *qp_state)
887 {
888         int err;
889         int i;
890         enum mlx4_qp_state states[] = {
891                 MLX4_QP_STATE_RST,
892                 MLX4_QP_STATE_INIT,
893                 MLX4_QP_STATE_RTR,
894                 MLX4_QP_STATE_RTS
895         };
896
897         for (i = 0; i < ARRAY_SIZE(states) - 1; i++) {
898                 context->flags &= cpu_to_be32(~(0xf << 28));
899                 context->flags |= cpu_to_be32(states[i + 1] << 28);
900                 if (states[i + 1] != MLX4_QP_STATE_RTR)
901                         context->params2 &= ~MLX4_QP_BIT_FPP;
902                 err = mlx4_qp_modify(dev, mtt, states[i], states[i + 1],
903                                      context, 0, 0, qp);
904                 if (err) {
905                         mlx4_err(dev, "Failed to bring QP to state: %d with error: %d\n",
906                                  states[i + 1], err);
907                         return err;
908                 }
909
910                 *qp_state = states[i + 1];
911         }
912
913         return 0;
914 }
915 EXPORT_SYMBOL_GPL(mlx4_qp_to_ready);