strace: Remove the ability to write to Qstrace
[akaros.git] / kern / drivers / net / udrvr / verbs.c
1 /*
2  * Copyright (c) 2004 Mellanox Technologies Ltd.  All rights reserved.
3  * Copyright (c) 2004 Infinicon Corporation.  All rights reserved.
4  * Copyright (c) 2004 Intel Corporation.  All rights reserved.
5  * Copyright (c) 2004 Topspin Corporation.  All rights reserved.
6  * Copyright (c) 2004 Voltaire Corporation.  All rights reserved.
7  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
8  * Copyright (c) 2005, 2006 Cisco Systems.  All rights reserved.
9  *
10  * This software is available to you under a choice of one of two
11  * licenses.  You may choose to be licensed under the terms of the GNU
12  * General Public License (GPL) Version 2, available from the file
13  * COPYING in the main directory of this source tree, or the
14  * OpenIB.org BSD license below:
15  *
16  *     Redistribution and use in source and binary forms, with or
17  *     without modification, are permitted provided that the following
18  *     conditions are met:
19  *
20  *      - Redistributions of source code must retain the above
21  *        copyright notice, this list of conditions and the following
22  *        disclaimer.
23  *
24  *      - Redistributions in binary form must reproduce the above
25  *        copyright notice, this list of conditions and the following
26  *        disclaimer in the documentation and/or other materials
27  *        provided with the distribution.
28  *
29  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
30  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
31  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
32  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
33  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
34  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
35  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36  * SOFTWARE.
37  */
38
39 #if 0   /* AKAROS */
40 #include <linux/errno.h>
41 #include <linux/err.h>
42 #include <linux/export.h>
43 #include <linux/string.h>
44 #include <linux/slab.h>
45
46 #include <rdma/ib_verbs.h>
47 #include <rdma/ib_cache.h>
48 #include <rdma/ib_addr.h>
49
50 #include "core_priv.h"
51 #else   /* AKAROS */
52 #include <linux/rdma/ib_verbs.h>
53 #endif  /* AKAROS */
54
55 __attribute_const__ int ib_rate_to_mult(enum ib_rate rate)
56 {
57         switch (rate) {
58         case IB_RATE_2_5_GBPS: return  1;
59         case IB_RATE_5_GBPS:   return  2;
60         case IB_RATE_10_GBPS:  return  4;
61         case IB_RATE_20_GBPS:  return  8;
62         case IB_RATE_30_GBPS:  return 12;
63         case IB_RATE_40_GBPS:  return 16;
64         case IB_RATE_60_GBPS:  return 24;
65         case IB_RATE_80_GBPS:  return 32;
66         case IB_RATE_120_GBPS: return 48;
67         default:               return -1;
68         }
69 }
70 EXPORT_SYMBOL(ib_rate_to_mult);
71
72 __attribute_const__ enum ib_rate mult_to_ib_rate(int mult)
73 {
74         switch (mult) {
75         case 1:  return IB_RATE_2_5_GBPS;
76         case 2:  return IB_RATE_5_GBPS;
77         case 4:  return IB_RATE_10_GBPS;
78         case 8:  return IB_RATE_20_GBPS;
79         case 12: return IB_RATE_30_GBPS;
80         case 16: return IB_RATE_40_GBPS;
81         case 24: return IB_RATE_60_GBPS;
82         case 32: return IB_RATE_80_GBPS;
83         case 48: return IB_RATE_120_GBPS;
84         default: return IB_RATE_PORT_CURRENT;
85         }
86 }
87 EXPORT_SYMBOL(mult_to_ib_rate);
88
89 __attribute_const__ int ib_rate_to_mbps(enum ib_rate rate)
90 {
91         switch (rate) {
92         case IB_RATE_2_5_GBPS: return 2500;
93         case IB_RATE_5_GBPS:   return 5000;
94         case IB_RATE_10_GBPS:  return 10000;
95         case IB_RATE_20_GBPS:  return 20000;
96         case IB_RATE_30_GBPS:  return 30000;
97         case IB_RATE_40_GBPS:  return 40000;
98         case IB_RATE_60_GBPS:  return 60000;
99         case IB_RATE_80_GBPS:  return 80000;
100         case IB_RATE_120_GBPS: return 120000;
101         case IB_RATE_14_GBPS:  return 14062;
102         case IB_RATE_56_GBPS:  return 56250;
103         case IB_RATE_112_GBPS: return 112500;
104         case IB_RATE_168_GBPS: return 168750;
105         case IB_RATE_25_GBPS:  return 25781;
106         case IB_RATE_100_GBPS: return 103125;
107         case IB_RATE_200_GBPS: return 206250;
108         case IB_RATE_300_GBPS: return 309375;
109         default:               return -1;
110         }
111 }
112 EXPORT_SYMBOL(ib_rate_to_mbps);
113
114 __attribute_const__ enum rdma_transport_type
115 rdma_node_get_transport(enum rdma_node_type node_type)
116 {
117         switch (node_type) {
118         case RDMA_NODE_IB_CA:
119         case RDMA_NODE_IB_SWITCH:
120         case RDMA_NODE_IB_ROUTER:
121                 return RDMA_TRANSPORT_IB;
122         case RDMA_NODE_RNIC:
123                 return RDMA_TRANSPORT_IWARP;
124         case RDMA_NODE_USNIC:
125                 return RDMA_TRANSPORT_USNIC;
126         case RDMA_NODE_USNIC_UDP:
127                 return RDMA_TRANSPORT_USNIC_UDP;
128         default:
129                 BUG();
130                 return 0;
131         }
132 }
133 EXPORT_SYMBOL(rdma_node_get_transport);
134
135 enum rdma_link_layer rdma_port_get_link_layer(struct ib_device *device, u8 port_num)
136 {
137         if (device->get_link_layer)
138                 return device->get_link_layer(device, port_num);
139
140         switch (rdma_node_get_transport(device->node_type)) {
141         case RDMA_TRANSPORT_IB:
142                 return IB_LINK_LAYER_INFINIBAND;
143         case RDMA_TRANSPORT_IWARP:
144         case RDMA_TRANSPORT_USNIC:
145         case RDMA_TRANSPORT_USNIC_UDP:
146                 return IB_LINK_LAYER_ETHERNET;
147         default:
148                 return IB_LINK_LAYER_UNSPECIFIED;
149         }
150 }
151 EXPORT_SYMBOL(rdma_port_get_link_layer);
152
153 /* Protection domains */
154
155 struct ib_pd *ib_alloc_pd(struct ib_device *device)
156 {
157         struct ib_pd *pd;
158
159         pd = device->alloc_pd(device, NULL, NULL);
160
161         if (!IS_ERR(pd)) {
162                 pd->device  = device;
163                 pd->uobject = NULL;
164                 atomic_set(&pd->usecnt, 0);
165         }
166
167         return pd;
168 }
169 EXPORT_SYMBOL(ib_alloc_pd);
170
171 int ib_dealloc_pd(struct ib_pd *pd)
172 {
173         if (atomic_read(&pd->usecnt))
174                 return -EBUSY;
175
176         return pd->device->dealloc_pd(pd);
177 }
178 EXPORT_SYMBOL(ib_dealloc_pd);
179
180 /* Address handles */
181
182 struct ib_ah *ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr)
183 {
184         struct ib_ah *ah;
185
186         ah = pd->device->create_ah(pd, ah_attr);
187
188         if (!IS_ERR(ah)) {
189                 ah->device  = pd->device;
190                 ah->pd      = pd;
191                 ah->uobject = NULL;
192                 atomic_inc(&pd->usecnt);
193         }
194
195         return ah;
196 }
197 EXPORT_SYMBOL(ib_create_ah);
198
199 int ib_init_ah_from_wc(struct ib_device *device, u8 port_num, struct ib_wc *wc,
200                        struct ib_grh *grh, struct ib_ah_attr *ah_attr)
201 {
202 #if 0   /* AKAROS */
203         u32 flow_class;
204         u16 gid_index;
205         int ret;
206         int is_eth = (rdma_port_get_link_layer(device, port_num) ==
207                         IB_LINK_LAYER_ETHERNET);
208
209         memset(ah_attr, 0, sizeof *ah_attr);
210         if (is_eth) {
211                 if (!(wc->wc_flags & IB_WC_GRH))
212                         return -EPROTOTYPE;
213
214                 if (wc->wc_flags & IB_WC_WITH_SMAC &&
215                     wc->wc_flags & IB_WC_WITH_VLAN) {
216                         memcpy(ah_attr->dmac, wc->smac, ETH_ALEN);
217                         ah_attr->vlan_id = wc->vlan_id;
218                 } else {
219                         ret = rdma_addr_find_dmac_by_grh(&grh->dgid, &grh->sgid,
220                                         ah_attr->dmac, &ah_attr->vlan_id);
221                         if (ret)
222                                 return ret;
223                 }
224         } else {
225                 ah_attr->vlan_id = 0xffff;
226         }
227
228         ah_attr->dlid = wc->slid;
229         ah_attr->sl = wc->sl;
230         ah_attr->src_path_bits = wc->dlid_path_bits;
231         ah_attr->port_num = port_num;
232
233         if (wc->wc_flags & IB_WC_GRH) {
234                 ah_attr->ah_flags = IB_AH_GRH;
235                 ah_attr->grh.dgid = grh->sgid;
236
237                 ret = ib_find_cached_gid(device, &grh->dgid, &port_num,
238                                          &gid_index);
239                 if (ret)
240                         return ret;
241
242                 ah_attr->grh.sgid_index = (u8) gid_index;
243                 flow_class = be32_to_cpu(grh->version_tclass_flow);
244                 ah_attr->grh.flow_label = flow_class & 0xFFFFF;
245                 ah_attr->grh.hop_limit = 0xFF;
246                 ah_attr->grh.traffic_class = (flow_class >> 20) & 0xFF;
247         }
248 #else   /* AKAROS */
249         BUG_ON(1);
250 #endif  /* AKAROS */
251         return 0;
252 }
253 EXPORT_SYMBOL(ib_init_ah_from_wc);
254
255 struct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, struct ib_wc *wc,
256                                    struct ib_grh *grh, u8 port_num)
257 {
258         struct ib_ah_attr ah_attr;
259         int ret;
260
261         ret = ib_init_ah_from_wc(pd->device, port_num, wc, grh, &ah_attr);
262         if (ret)
263                 return ERR_PTR(ret);
264
265         return ib_create_ah(pd, &ah_attr);
266 }
267 EXPORT_SYMBOL(ib_create_ah_from_wc);
268
269 int ib_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
270 {
271         return ah->device->modify_ah ?
272                 ah->device->modify_ah(ah, ah_attr) :
273                 -ENOSYS;
274 }
275 EXPORT_SYMBOL(ib_modify_ah);
276
277 int ib_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
278 {
279         return ah->device->query_ah ?
280                 ah->device->query_ah(ah, ah_attr) :
281                 -ENOSYS;
282 }
283 EXPORT_SYMBOL(ib_query_ah);
284
285 int ib_destroy_ah(struct ib_ah *ah)
286 {
287         struct ib_pd *pd;
288         int ret;
289
290         pd = ah->pd;
291         ret = ah->device->destroy_ah(ah);
292         if (!ret)
293                 atomic_dec(&pd->usecnt);
294
295         return ret;
296 }
297 EXPORT_SYMBOL(ib_destroy_ah);
298
299 /* Shared receive queues */
300
301 struct ib_srq *ib_create_srq(struct ib_pd *pd,
302                              struct ib_srq_init_attr *srq_init_attr)
303 {
304         struct ib_srq *srq;
305
306         if (!pd->device->create_srq)
307                 return ERR_PTR(-ENOSYS);
308
309         srq = pd->device->create_srq(pd, srq_init_attr, NULL);
310
311         if (!IS_ERR(srq)) {
312                 srq->device        = pd->device;
313                 srq->pd            = pd;
314                 srq->uobject       = NULL;
315                 srq->event_handler = srq_init_attr->event_handler;
316                 srq->srq_context   = srq_init_attr->srq_context;
317                 srq->srq_type      = srq_init_attr->srq_type;
318                 if (srq->srq_type == IB_SRQT_XRC) {
319                         srq->ext.xrc.xrcd = srq_init_attr->ext.xrc.xrcd;
320                         srq->ext.xrc.cq   = srq_init_attr->ext.xrc.cq;
321                         atomic_inc(&srq->ext.xrc.xrcd->usecnt);
322                         atomic_inc(&srq->ext.xrc.cq->usecnt);
323                 }
324                 atomic_inc(&pd->usecnt);
325                 atomic_set(&srq->usecnt, 0);
326         }
327
328         return srq;
329 }
330 EXPORT_SYMBOL(ib_create_srq);
331
332 int ib_modify_srq(struct ib_srq *srq,
333                   struct ib_srq_attr *srq_attr,
334                   enum ib_srq_attr_mask srq_attr_mask)
335 {
336         return srq->device->modify_srq ?
337                 srq->device->modify_srq(srq, srq_attr, srq_attr_mask, NULL) :
338                 -ENOSYS;
339 }
340 EXPORT_SYMBOL(ib_modify_srq);
341
342 int ib_query_srq(struct ib_srq *srq,
343                  struct ib_srq_attr *srq_attr)
344 {
345         return srq->device->query_srq ?
346                 srq->device->query_srq(srq, srq_attr) : -ENOSYS;
347 }
348 EXPORT_SYMBOL(ib_query_srq);
349
350 int ib_destroy_srq(struct ib_srq *srq)
351 {
352         struct ib_pd *pd;
353         enum ib_srq_type srq_type;
354         struct ib_xrcd *uninitialized_var(xrcd);
355         struct ib_cq *uninitialized_var(cq);
356         int ret;
357
358         if (atomic_read(&srq->usecnt))
359                 return -EBUSY;
360
361         pd = srq->pd;
362         srq_type = srq->srq_type;
363         if (srq_type == IB_SRQT_XRC) {
364                 xrcd = srq->ext.xrc.xrcd;
365                 cq = srq->ext.xrc.cq;
366         }
367
368         ret = srq->device->destroy_srq(srq);
369         if (!ret) {
370                 atomic_dec(&pd->usecnt);
371                 if (srq_type == IB_SRQT_XRC) {
372                         atomic_dec(&xrcd->usecnt);
373                         atomic_dec(&cq->usecnt);
374                 }
375         }
376
377         return ret;
378 }
379 EXPORT_SYMBOL(ib_destroy_srq);
380
381 /* Queue pairs */
382
383 static void __ib_shared_qp_event_handler(struct ib_event *event, void *context)
384 {
385         struct ib_qp *qp = context;
386         unsigned long flags;
387
388         spin_lock_irqsave(&qp->device->event_handler_lock, flags);
389         list_for_each_entry(event->element.qp, &qp->open_list, open_list)
390                 if (event->element.qp->event_handler)
391                         event->element.qp->event_handler(event, event->element.qp->qp_context);
392         spin_unlock_irqrestore(&qp->device->event_handler_lock, flags);
393 }
394
395 static void __ib_insert_xrcd_qp(struct ib_xrcd *xrcd, struct ib_qp *qp)
396 {
397         mutex_lock(&xrcd->tgt_qp_mutex);
398         list_add(&qp->xrcd_list, &xrcd->tgt_qp_list);
399         mutex_unlock(&xrcd->tgt_qp_mutex);
400 }
401
402 static struct ib_qp *__ib_open_qp(struct ib_qp *real_qp,
403                                   void (*event_handler)(struct ib_event *, void *),
404                                   void *qp_context)
405 {
406         struct ib_qp *qp;
407         unsigned long flags;
408
409         qp = kzalloc(sizeof *qp, GFP_KERNEL);
410         if (!qp)
411                 return ERR_PTR(-ENOMEM);
412
413         qp->real_qp = real_qp;
414         atomic_inc(&real_qp->usecnt);
415         qp->device = real_qp->device;
416         qp->event_handler = event_handler;
417         qp->qp_context = qp_context;
418         qp->qp_num = real_qp->qp_num;
419         qp->qp_type = real_qp->qp_type;
420
421         spin_lock_irqsave(&real_qp->device->event_handler_lock, flags);
422         list_add(&qp->open_list, &real_qp->open_list);
423         spin_unlock_irqrestore(&real_qp->device->event_handler_lock, flags);
424
425         return qp;
426 }
427
428 struct ib_qp *ib_open_qp(struct ib_xrcd *xrcd,
429                          struct ib_qp_open_attr *qp_open_attr)
430 {
431         struct ib_qp *qp, *real_qp;
432
433         if (qp_open_attr->qp_type != IB_QPT_XRC_TGT)
434                 return ERR_PTR(-EINVAL);
435
436         qp = ERR_PTR(-EINVAL);
437         mutex_lock(&xrcd->tgt_qp_mutex);
438         list_for_each_entry(real_qp, &xrcd->tgt_qp_list, xrcd_list) {
439                 if (real_qp->qp_num == qp_open_attr->qp_num) {
440                         qp = __ib_open_qp(real_qp, qp_open_attr->event_handler,
441                                           qp_open_attr->qp_context);
442                         break;
443                 }
444         }
445         mutex_unlock(&xrcd->tgt_qp_mutex);
446         return qp;
447 }
448 EXPORT_SYMBOL(ib_open_qp);
449
450 struct ib_qp *ib_create_qp(struct ib_pd *pd,
451                            struct ib_qp_init_attr *qp_init_attr)
452 {
453         struct ib_qp *qp, *real_qp;
454         struct ib_device *device;
455
456         device = pd ? pd->device : qp_init_attr->xrcd->device;
457         qp = device->create_qp(pd, qp_init_attr, NULL);
458
459         if (!IS_ERR(qp)) {
460                 qp->device     = device;
461                 qp->real_qp    = qp;
462                 qp->uobject    = NULL;
463                 qp->qp_type    = qp_init_attr->qp_type;
464
465                 atomic_set(&qp->usecnt, 0);
466                 if (qp_init_attr->qp_type == IB_QPT_XRC_TGT) {
467                         qp->event_handler = __ib_shared_qp_event_handler;
468                         qp->qp_context = qp;
469                         qp->pd = NULL;
470                         qp->send_cq = qp->recv_cq = NULL;
471                         qp->srq = NULL;
472                         qp->xrcd = qp_init_attr->xrcd;
473                         atomic_inc(&qp_init_attr->xrcd->usecnt);
474                         INIT_LIST_HEAD(&qp->open_list);
475
476                         real_qp = qp;
477                         qp = __ib_open_qp(real_qp, qp_init_attr->event_handler,
478                                           qp_init_attr->qp_context);
479                         if (!IS_ERR(qp))
480                                 __ib_insert_xrcd_qp(qp_init_attr->xrcd, real_qp);
481                         else
482                                 real_qp->device->destroy_qp(real_qp);
483                 } else {
484                         qp->event_handler = qp_init_attr->event_handler;
485                         qp->qp_context = qp_init_attr->qp_context;
486                         if (qp_init_attr->qp_type == IB_QPT_XRC_INI) {
487                                 qp->recv_cq = NULL;
488                                 qp->srq = NULL;
489                         } else {
490                                 qp->recv_cq = qp_init_attr->recv_cq;
491                                 atomic_inc(&qp_init_attr->recv_cq->usecnt);
492                                 qp->srq = qp_init_attr->srq;
493                                 if (qp->srq)
494                                         atomic_inc(&qp_init_attr->srq->usecnt);
495                         }
496
497                         qp->pd      = pd;
498                         qp->send_cq = qp_init_attr->send_cq;
499                         qp->xrcd    = NULL;
500
501                         atomic_inc(&pd->usecnt);
502                         atomic_inc(&qp_init_attr->send_cq->usecnt);
503                 }
504         }
505
506         return qp;
507 }
508 EXPORT_SYMBOL(ib_create_qp);
509
510 static const struct {
511         int                     valid;
512         enum ib_qp_attr_mask    req_param[IB_QPT_MAX];
513         enum ib_qp_attr_mask    req_param_add_eth[IB_QPT_MAX];
514         enum ib_qp_attr_mask    opt_param[IB_QPT_MAX];
515         enum ib_qp_attr_mask    opt_param_add_eth[IB_QPT_MAX];
516 } qp_state_table[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = {
517         [IB_QPS_RESET] = {
518                 [IB_QPS_RESET] = { .valid = 1 },
519                 [IB_QPS_INIT]  = {
520                         .valid = 1,
521                         .req_param = {
522                                 [IB_QPT_UD]  = (IB_QP_PKEY_INDEX                |
523                                                 IB_QP_PORT                      |
524                                                 IB_QP_QKEY),
525                                 [IB_QPT_RAW_PACKET] = IB_QP_PORT,
526                                 [IB_QPT_UC]  = (IB_QP_PKEY_INDEX                |
527                                                 IB_QP_PORT                      |
528                                                 IB_QP_ACCESS_FLAGS),
529                                 [IB_QPT_RC]  = (IB_QP_PKEY_INDEX                |
530                                                 IB_QP_PORT                      |
531                                                 IB_QP_ACCESS_FLAGS),
532                                 [IB_QPT_XRC_INI] = (IB_QP_PKEY_INDEX            |
533                                                 IB_QP_PORT                      |
534                                                 IB_QP_ACCESS_FLAGS),
535                                 [IB_QPT_XRC_TGT] = (IB_QP_PKEY_INDEX            |
536                                                 IB_QP_PORT                      |
537                                                 IB_QP_ACCESS_FLAGS),
538                                 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX                |
539                                                 IB_QP_QKEY),
540                                 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX                |
541                                                 IB_QP_QKEY),
542                         }
543                 },
544         },
545         [IB_QPS_INIT]  = {
546                 [IB_QPS_RESET] = { .valid = 1 },
547                 [IB_QPS_ERR] =   { .valid = 1 },
548                 [IB_QPS_INIT]  = {
549                         .valid = 1,
550                         .opt_param = {
551                                 [IB_QPT_UD]  = (IB_QP_PKEY_INDEX                |
552                                                 IB_QP_PORT                      |
553                                                 IB_QP_QKEY),
554                                 [IB_QPT_UC]  = (IB_QP_PKEY_INDEX                |
555                                                 IB_QP_PORT                      |
556                                                 IB_QP_ACCESS_FLAGS),
557                                 [IB_QPT_RC]  = (IB_QP_PKEY_INDEX                |
558                                                 IB_QP_PORT                      |
559                                                 IB_QP_ACCESS_FLAGS),
560                                 [IB_QPT_XRC_INI] = (IB_QP_PKEY_INDEX            |
561                                                 IB_QP_PORT                      |
562                                                 IB_QP_ACCESS_FLAGS),
563                                 [IB_QPT_XRC_TGT] = (IB_QP_PKEY_INDEX            |
564                                                 IB_QP_PORT                      |
565                                                 IB_QP_ACCESS_FLAGS),
566                                 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX                |
567                                                 IB_QP_QKEY),
568                                 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX                |
569                                                 IB_QP_QKEY),
570                         }
571                 },
572                 [IB_QPS_RTR]   = {
573                         .valid = 1,
574                         .req_param = {
575                                 [IB_QPT_UC]  = (IB_QP_AV                        |
576                                                 IB_QP_PATH_MTU                  |
577                                                 IB_QP_DEST_QPN                  |
578                                                 IB_QP_RQ_PSN),
579                                 [IB_QPT_RC]  = (IB_QP_AV                        |
580                                                 IB_QP_PATH_MTU                  |
581                                                 IB_QP_DEST_QPN                  |
582                                                 IB_QP_RQ_PSN                    |
583                                                 IB_QP_MAX_DEST_RD_ATOMIC        |
584                                                 IB_QP_MIN_RNR_TIMER),
585                                 [IB_QPT_XRC_INI] = (IB_QP_AV                    |
586                                                 IB_QP_PATH_MTU                  |
587                                                 IB_QP_DEST_QPN                  |
588                                                 IB_QP_RQ_PSN),
589                                 [IB_QPT_XRC_TGT] = (IB_QP_AV                    |
590                                                 IB_QP_PATH_MTU                  |
591                                                 IB_QP_DEST_QPN                  |
592                                                 IB_QP_RQ_PSN                    |
593                                                 IB_QP_MAX_DEST_RD_ATOMIC        |
594                                                 IB_QP_MIN_RNR_TIMER),
595                         },
596                         .req_param_add_eth = {
597                                 [IB_QPT_RC]  = (IB_QP_SMAC),
598                                 [IB_QPT_UC]  = (IB_QP_SMAC),
599                                 [IB_QPT_XRC_INI]  = (IB_QP_SMAC),
600                                 [IB_QPT_XRC_TGT]  = (IB_QP_SMAC)
601                         },
602                         .opt_param = {
603                                  [IB_QPT_UD]  = (IB_QP_PKEY_INDEX               |
604                                                  IB_QP_QKEY),
605                                  [IB_QPT_UC]  = (IB_QP_ALT_PATH                 |
606                                                  IB_QP_ACCESS_FLAGS             |
607                                                  IB_QP_PKEY_INDEX),
608                                  [IB_QPT_RC]  = (IB_QP_ALT_PATH                 |
609                                                  IB_QP_ACCESS_FLAGS             |
610                                                  IB_QP_PKEY_INDEX),
611                                  [IB_QPT_XRC_INI] = (IB_QP_ALT_PATH             |
612                                                  IB_QP_ACCESS_FLAGS             |
613                                                  IB_QP_PKEY_INDEX),
614                                  [IB_QPT_XRC_TGT] = (IB_QP_ALT_PATH             |
615                                                  IB_QP_ACCESS_FLAGS             |
616                                                  IB_QP_PKEY_INDEX),
617                                  [IB_QPT_SMI] = (IB_QP_PKEY_INDEX               |
618                                                  IB_QP_QKEY),
619                                  [IB_QPT_GSI] = (IB_QP_PKEY_INDEX               |
620                                                  IB_QP_QKEY),
621                          },
622                         .opt_param_add_eth = {
623                                 [IB_QPT_RC]  = (IB_QP_ALT_SMAC                  |
624                                                 IB_QP_VID                       |
625                                                 IB_QP_ALT_VID),
626                                 [IB_QPT_UC]  = (IB_QP_ALT_SMAC                  |
627                                                 IB_QP_VID                       |
628                                                 IB_QP_ALT_VID),
629                                 [IB_QPT_XRC_INI]  = (IB_QP_ALT_SMAC                     |
630                                                 IB_QP_VID                       |
631                                                 IB_QP_ALT_VID),
632                                 [IB_QPT_XRC_TGT]  = (IB_QP_ALT_SMAC                     |
633                                                 IB_QP_VID                       |
634                                                 IB_QP_ALT_VID)
635                         }
636                 }
637         },
638         [IB_QPS_RTR]   = {
639                 [IB_QPS_RESET] = { .valid = 1 },
640                 [IB_QPS_ERR] =   { .valid = 1 },
641                 [IB_QPS_RTS]   = {
642                         .valid = 1,
643                         .req_param = {
644                                 [IB_QPT_UD]  = IB_QP_SQ_PSN,
645                                 [IB_QPT_UC]  = IB_QP_SQ_PSN,
646                                 [IB_QPT_RC]  = (IB_QP_TIMEOUT                   |
647                                                 IB_QP_RETRY_CNT                 |
648                                                 IB_QP_RNR_RETRY                 |
649                                                 IB_QP_SQ_PSN                    |
650                                                 IB_QP_MAX_QP_RD_ATOMIC),
651                                 [IB_QPT_XRC_INI] = (IB_QP_TIMEOUT               |
652                                                 IB_QP_RETRY_CNT                 |
653                                                 IB_QP_RNR_RETRY                 |
654                                                 IB_QP_SQ_PSN                    |
655                                                 IB_QP_MAX_QP_RD_ATOMIC),
656                                 [IB_QPT_XRC_TGT] = (IB_QP_TIMEOUT               |
657                                                 IB_QP_SQ_PSN),
658                                 [IB_QPT_SMI] = IB_QP_SQ_PSN,
659                                 [IB_QPT_GSI] = IB_QP_SQ_PSN,
660                         },
661                         .opt_param = {
662                                  [IB_QPT_UD]  = (IB_QP_CUR_STATE                |
663                                                  IB_QP_QKEY),
664                                  [IB_QPT_UC]  = (IB_QP_CUR_STATE                |
665                                                  IB_QP_ALT_PATH                 |
666                                                  IB_QP_ACCESS_FLAGS             |
667                                                  IB_QP_PATH_MIG_STATE),
668                                  [IB_QPT_RC]  = (IB_QP_CUR_STATE                |
669                                                  IB_QP_ALT_PATH                 |
670                                                  IB_QP_ACCESS_FLAGS             |
671                                                  IB_QP_MIN_RNR_TIMER            |
672                                                  IB_QP_PATH_MIG_STATE),
673                                  [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE            |
674                                                  IB_QP_ALT_PATH                 |
675                                                  IB_QP_ACCESS_FLAGS             |
676                                                  IB_QP_PATH_MIG_STATE),
677                                  [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE            |
678                                                  IB_QP_ALT_PATH                 |
679                                                  IB_QP_ACCESS_FLAGS             |
680                                                  IB_QP_MIN_RNR_TIMER            |
681                                                  IB_QP_PATH_MIG_STATE),
682                                  [IB_QPT_SMI] = (IB_QP_CUR_STATE                |
683                                                  IB_QP_QKEY),
684                                  [IB_QPT_GSI] = (IB_QP_CUR_STATE                |
685                                                  IB_QP_QKEY),
686                          }
687                 }
688         },
689         [IB_QPS_RTS]   = {
690                 [IB_QPS_RESET] = { .valid = 1 },
691                 [IB_QPS_ERR] =   { .valid = 1 },
692                 [IB_QPS_RTS]   = {
693                         .valid = 1,
694                         .opt_param = {
695                                 [IB_QPT_UD]  = (IB_QP_CUR_STATE                 |
696                                                 IB_QP_QKEY),
697                                 [IB_QPT_UC]  = (IB_QP_CUR_STATE                 |
698                                                 IB_QP_ACCESS_FLAGS              |
699                                                 IB_QP_ALT_PATH                  |
700                                                 IB_QP_PATH_MIG_STATE),
701                                 [IB_QPT_RC]  = (IB_QP_CUR_STATE                 |
702                                                 IB_QP_ACCESS_FLAGS              |
703                                                 IB_QP_ALT_PATH                  |
704                                                 IB_QP_PATH_MIG_STATE            |
705                                                 IB_QP_MIN_RNR_TIMER),
706                                 [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE             |
707                                                 IB_QP_ACCESS_FLAGS              |
708                                                 IB_QP_ALT_PATH                  |
709                                                 IB_QP_PATH_MIG_STATE),
710                                 [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE             |
711                                                 IB_QP_ACCESS_FLAGS              |
712                                                 IB_QP_ALT_PATH                  |
713                                                 IB_QP_PATH_MIG_STATE            |
714                                                 IB_QP_MIN_RNR_TIMER),
715                                 [IB_QPT_SMI] = (IB_QP_CUR_STATE                 |
716                                                 IB_QP_QKEY),
717                                 [IB_QPT_GSI] = (IB_QP_CUR_STATE                 |
718                                                 IB_QP_QKEY),
719                         }
720                 },
721                 [IB_QPS_SQD]   = {
722                         .valid = 1,
723                         .opt_param = {
724                                 [IB_QPT_UD]  = IB_QP_EN_SQD_ASYNC_NOTIFY,
725                                 [IB_QPT_UC]  = IB_QP_EN_SQD_ASYNC_NOTIFY,
726                                 [IB_QPT_RC]  = IB_QP_EN_SQD_ASYNC_NOTIFY,
727                                 [IB_QPT_XRC_INI] = IB_QP_EN_SQD_ASYNC_NOTIFY,
728                                 [IB_QPT_XRC_TGT] = IB_QP_EN_SQD_ASYNC_NOTIFY, /* ??? */
729                                 [IB_QPT_SMI] = IB_QP_EN_SQD_ASYNC_NOTIFY,
730                                 [IB_QPT_GSI] = IB_QP_EN_SQD_ASYNC_NOTIFY
731                         }
732                 },
733         },
734         [IB_QPS_SQD]   = {
735                 [IB_QPS_RESET] = { .valid = 1 },
736                 [IB_QPS_ERR] =   { .valid = 1 },
737                 [IB_QPS_RTS]   = {
738                         .valid = 1,
739                         .opt_param = {
740                                 [IB_QPT_UD]  = (IB_QP_CUR_STATE                 |
741                                                 IB_QP_QKEY),
742                                 [IB_QPT_UC]  = (IB_QP_CUR_STATE                 |
743                                                 IB_QP_ALT_PATH                  |
744                                                 IB_QP_ACCESS_FLAGS              |
745                                                 IB_QP_PATH_MIG_STATE),
746                                 [IB_QPT_RC]  = (IB_QP_CUR_STATE                 |
747                                                 IB_QP_ALT_PATH                  |
748                                                 IB_QP_ACCESS_FLAGS              |
749                                                 IB_QP_MIN_RNR_TIMER             |
750                                                 IB_QP_PATH_MIG_STATE),
751                                 [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE             |
752                                                 IB_QP_ALT_PATH                  |
753                                                 IB_QP_ACCESS_FLAGS              |
754                                                 IB_QP_PATH_MIG_STATE),
755                                 [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE             |
756                                                 IB_QP_ALT_PATH                  |
757                                                 IB_QP_ACCESS_FLAGS              |
758                                                 IB_QP_MIN_RNR_TIMER             |
759                                                 IB_QP_PATH_MIG_STATE),
760                                 [IB_QPT_SMI] = (IB_QP_CUR_STATE                 |
761                                                 IB_QP_QKEY),
762                                 [IB_QPT_GSI] = (IB_QP_CUR_STATE                 |
763                                                 IB_QP_QKEY),
764                         }
765                 },
766                 [IB_QPS_SQD]   = {
767                         .valid = 1,
768                         .opt_param = {
769                                 [IB_QPT_UD]  = (IB_QP_PKEY_INDEX                |
770                                                 IB_QP_QKEY),
771                                 [IB_QPT_UC]  = (IB_QP_AV                        |
772                                                 IB_QP_ALT_PATH                  |
773                                                 IB_QP_ACCESS_FLAGS              |
774                                                 IB_QP_PKEY_INDEX                |
775                                                 IB_QP_PATH_MIG_STATE),
776                                 [IB_QPT_RC]  = (IB_QP_PORT                      |
777                                                 IB_QP_AV                        |
778                                                 IB_QP_TIMEOUT                   |
779                                                 IB_QP_RETRY_CNT                 |
780                                                 IB_QP_RNR_RETRY                 |
781                                                 IB_QP_MAX_QP_RD_ATOMIC          |
782                                                 IB_QP_MAX_DEST_RD_ATOMIC        |
783                                                 IB_QP_ALT_PATH                  |
784                                                 IB_QP_ACCESS_FLAGS              |
785                                                 IB_QP_PKEY_INDEX                |
786                                                 IB_QP_MIN_RNR_TIMER             |
787                                                 IB_QP_PATH_MIG_STATE),
788                                 [IB_QPT_XRC_INI] = (IB_QP_PORT                  |
789                                                 IB_QP_AV                        |
790                                                 IB_QP_TIMEOUT                   |
791                                                 IB_QP_RETRY_CNT                 |
792                                                 IB_QP_RNR_RETRY                 |
793                                                 IB_QP_MAX_QP_RD_ATOMIC          |
794                                                 IB_QP_ALT_PATH                  |
795                                                 IB_QP_ACCESS_FLAGS              |
796                                                 IB_QP_PKEY_INDEX                |
797                                                 IB_QP_PATH_MIG_STATE),
798                                 [IB_QPT_XRC_TGT] = (IB_QP_PORT                  |
799                                                 IB_QP_AV                        |
800                                                 IB_QP_TIMEOUT                   |
801                                                 IB_QP_MAX_DEST_RD_ATOMIC        |
802                                                 IB_QP_ALT_PATH                  |
803                                                 IB_QP_ACCESS_FLAGS              |
804                                                 IB_QP_PKEY_INDEX                |
805                                                 IB_QP_MIN_RNR_TIMER             |
806                                                 IB_QP_PATH_MIG_STATE),
807                                 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX                |
808                                                 IB_QP_QKEY),
809                                 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX                |
810                                                 IB_QP_QKEY),
811                         }
812                 }
813         },
814         [IB_QPS_SQE]   = {
815                 [IB_QPS_RESET] = { .valid = 1 },
816                 [IB_QPS_ERR] =   { .valid = 1 },
817                 [IB_QPS_RTS]   = {
818                         .valid = 1,
819                         .opt_param = {
820                                 [IB_QPT_UD]  = (IB_QP_CUR_STATE                 |
821                                                 IB_QP_QKEY),
822                                 [IB_QPT_UC]  = (IB_QP_CUR_STATE                 |
823                                                 IB_QP_ACCESS_FLAGS),
824                                 [IB_QPT_SMI] = (IB_QP_CUR_STATE                 |
825                                                 IB_QP_QKEY),
826                                 [IB_QPT_GSI] = (IB_QP_CUR_STATE                 |
827                                                 IB_QP_QKEY),
828                         }
829                 }
830         },
831         [IB_QPS_ERR] = {
832                 [IB_QPS_RESET] = { .valid = 1 },
833                 [IB_QPS_ERR] =   { .valid = 1 }
834         }
835 };
836
837 int ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state,
838                        enum ib_qp_type type, enum ib_qp_attr_mask mask,
839                        enum rdma_link_layer ll)
840 {
841         enum ib_qp_attr_mask req_param, opt_param;
842
843         if (cur_state  < 0 || cur_state  > IB_QPS_ERR ||
844             next_state < 0 || next_state > IB_QPS_ERR)
845                 return 0;
846
847         if (mask & IB_QP_CUR_STATE  &&
848             cur_state != IB_QPS_RTR && cur_state != IB_QPS_RTS &&
849             cur_state != IB_QPS_SQD && cur_state != IB_QPS_SQE)
850                 return 0;
851
852         if (!qp_state_table[cur_state][next_state].valid)
853                 return 0;
854
855         req_param = qp_state_table[cur_state][next_state].req_param[type];
856         opt_param = qp_state_table[cur_state][next_state].opt_param[type];
857
858         if (ll == IB_LINK_LAYER_ETHERNET) {
859                 req_param |= qp_state_table[cur_state][next_state].
860                         req_param_add_eth[type];
861                 opt_param |= qp_state_table[cur_state][next_state].
862                         opt_param_add_eth[type];
863         }
864
865         if ((mask & req_param) != req_param)
866                 return 0;
867
868         if (mask & ~(req_param | opt_param | IB_QP_STATE))
869                 return 0;
870
871         return 1;
872 }
873 EXPORT_SYMBOL(ib_modify_qp_is_ok);
874
875 int ib_resolve_eth_l2_attrs(struct ib_qp *qp,
876                             struct ib_qp_attr *qp_attr, int *qp_attr_mask)
877 {
878         int           ret = 0;
879 #if 0   /* AKAROS */
880         union ib_gid  sgid;
881
882         if ((*qp_attr_mask & IB_QP_AV)  &&
883             (rdma_port_get_link_layer(qp->device, qp_attr->ah_attr.port_num) == IB_LINK_LAYER_ETHERNET)) {
884                 ret = ib_query_gid(qp->device, qp_attr->ah_attr.port_num,
885                                    qp_attr->ah_attr.grh.sgid_index, &sgid);
886                 if (ret)
887                         goto out;
888                 if (rdma_link_local_addr((struct in6_addr *)qp_attr->ah_attr.grh.dgid.raw)) {
889                         rdma_get_ll_mac((struct in6_addr *)qp_attr->ah_attr.grh.dgid.raw, qp_attr->ah_attr.dmac);
890                         rdma_get_ll_mac((struct in6_addr *)sgid.raw, qp_attr->smac);
891                         if (!(*qp_attr_mask & IB_QP_VID))
892                                 qp_attr->vlan_id = rdma_get_vlan_id(&sgid);
893                 } else {
894                         ret = rdma_addr_find_dmac_by_grh(&sgid, &qp_attr->ah_attr.grh.dgid,
895                                         qp_attr->ah_attr.dmac, &qp_attr->vlan_id);
896                         if (ret)
897                                 goto out;
898                         ret = rdma_addr_find_smac_by_sgid(&sgid, qp_attr->smac, NULL);
899                         if (ret)
900                                 goto out;
901                 }
902                 *qp_attr_mask |= IB_QP_SMAC;
903                 if (qp_attr->vlan_id < 0xFFFF)
904                         *qp_attr_mask |= IB_QP_VID;
905         }
906 out:
907 #else   /* AKAROS */
908         BUG_ON(1);
909 #endif  /* AKAROS */
910         return ret;
911 }
912 EXPORT_SYMBOL(ib_resolve_eth_l2_attrs);
913
914
915 int ib_modify_qp(struct ib_qp *qp,
916                  struct ib_qp_attr *qp_attr,
917                  int qp_attr_mask)
918 {
919         int ret;
920
921         ret = ib_resolve_eth_l2_attrs(qp, qp_attr, &qp_attr_mask);
922         if (ret)
923                 return ret;
924
925         return qp->device->modify_qp(qp->real_qp, qp_attr, qp_attr_mask, NULL);
926 }
927 EXPORT_SYMBOL(ib_modify_qp);
928
929 int ib_query_qp(struct ib_qp *qp,
930                 struct ib_qp_attr *qp_attr,
931                 int qp_attr_mask,
932                 struct ib_qp_init_attr *qp_init_attr)
933 {
934         return qp->device->query_qp ?
935                 qp->device->query_qp(qp->real_qp, qp_attr, qp_attr_mask, qp_init_attr) :
936                 -ENOSYS;
937 }
938 EXPORT_SYMBOL(ib_query_qp);
939
940 int ib_close_qp(struct ib_qp *qp)
941 {
942         struct ib_qp *real_qp;
943         unsigned long flags;
944
945         real_qp = qp->real_qp;
946         if (real_qp == qp)
947                 return -EINVAL;
948
949         spin_lock_irqsave(&real_qp->device->event_handler_lock, flags);
950         list_del(&qp->open_list);
951         spin_unlock_irqrestore(&real_qp->device->event_handler_lock, flags);
952
953         atomic_dec(&real_qp->usecnt);
954         kfree(qp);
955
956         return 0;
957 }
958 EXPORT_SYMBOL(ib_close_qp);
959
960 static int __ib_destroy_shared_qp(struct ib_qp *qp)
961 {
962         struct ib_xrcd *xrcd;
963         struct ib_qp *real_qp;
964         int ret;
965
966         real_qp = qp->real_qp;
967         xrcd = real_qp->xrcd;
968
969         mutex_lock(&xrcd->tgt_qp_mutex);
970         ib_close_qp(qp);
971         if (atomic_read(&real_qp->usecnt) == 0)
972                 list_del(&real_qp->xrcd_list);
973         else
974                 real_qp = NULL;
975         mutex_unlock(&xrcd->tgt_qp_mutex);
976
977         if (real_qp) {
978                 ret = ib_destroy_qp(real_qp);
979                 if (!ret)
980                         atomic_dec(&xrcd->usecnt);
981                 else
982                         __ib_insert_xrcd_qp(xrcd, real_qp);
983         }
984
985         return 0;
986 }
987
988 int ib_destroy_qp(struct ib_qp *qp)
989 {
990         struct ib_pd *pd;
991         struct ib_cq *scq, *rcq;
992         struct ib_srq *srq;
993         int ret;
994
995         if (atomic_read(&qp->usecnt))
996                 return -EBUSY;
997
998         if (qp->real_qp != qp)
999                 return __ib_destroy_shared_qp(qp);
1000
1001         pd   = qp->pd;
1002         scq  = qp->send_cq;
1003         rcq  = qp->recv_cq;
1004         srq  = qp->srq;
1005
1006         ret = qp->device->destroy_qp(qp);
1007         if (!ret) {
1008                 if (pd)
1009                         atomic_dec(&pd->usecnt);
1010                 if (scq)
1011                         atomic_dec(&scq->usecnt);
1012                 if (rcq)
1013                         atomic_dec(&rcq->usecnt);
1014                 if (srq)
1015                         atomic_dec(&srq->usecnt);
1016         }
1017
1018         return ret;
1019 }
1020 EXPORT_SYMBOL(ib_destroy_qp);
1021
1022 /* Completion queues */
1023
1024 struct ib_cq *ib_create_cq(struct ib_device *device,
1025                            ib_comp_handler comp_handler,
1026                            void (*event_handler)(struct ib_event *, void *),
1027                            void *cq_context, int cqe, int comp_vector)
1028 {
1029         struct ib_cq *cq;
1030
1031         cq = device->create_cq(device, cqe, comp_vector, NULL, NULL);
1032
1033         if (!IS_ERR(cq)) {
1034                 cq->device        = device;
1035                 cq->uobject       = NULL;
1036                 cq->comp_handler  = comp_handler;
1037                 cq->event_handler = event_handler;
1038                 cq->cq_context    = cq_context;
1039                 atomic_set(&cq->usecnt, 0);
1040         }
1041
1042         return cq;
1043 }
1044 EXPORT_SYMBOL(ib_create_cq);
1045
1046 int ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
1047 {
1048         return cq->device->modify_cq ?
1049                 cq->device->modify_cq(cq, cq_count, cq_period) : -ENOSYS;
1050 }
1051 EXPORT_SYMBOL(ib_modify_cq);
1052
1053 int ib_destroy_cq(struct ib_cq *cq)
1054 {
1055         if (atomic_read(&cq->usecnt))
1056                 return -EBUSY;
1057
1058         return cq->device->destroy_cq(cq);
1059 }
1060 EXPORT_SYMBOL(ib_destroy_cq);
1061
1062 int ib_resize_cq(struct ib_cq *cq, int cqe)
1063 {
1064         return cq->device->resize_cq ?
1065                 cq->device->resize_cq(cq, cqe, NULL) : -ENOSYS;
1066 }
1067 EXPORT_SYMBOL(ib_resize_cq);
1068
1069 /* Memory regions */
1070
1071 struct ib_mr *ib_get_dma_mr(struct ib_pd *pd, int mr_access_flags)
1072 {
1073         struct ib_mr *mr;
1074         int err;
1075
1076         err = ib_check_mr_access(mr_access_flags);
1077         if (err)
1078                 return ERR_PTR(err);
1079
1080         mr = pd->device->get_dma_mr(pd, mr_access_flags);
1081
1082         if (!IS_ERR(mr)) {
1083                 mr->device  = pd->device;
1084                 mr->pd      = pd;
1085                 mr->uobject = NULL;
1086                 atomic_inc(&pd->usecnt);
1087                 atomic_set(&mr->usecnt, 0);
1088         }
1089
1090         return mr;
1091 }
1092 EXPORT_SYMBOL(ib_get_dma_mr);
1093
1094 struct ib_mr *ib_reg_phys_mr(struct ib_pd *pd,
1095                              struct ib_phys_buf *phys_buf_array,
1096                              int num_phys_buf,
1097                              int mr_access_flags,
1098                              u64 *iova_start)
1099 {
1100         struct ib_mr *mr;
1101         int err;
1102
1103         err = ib_check_mr_access(mr_access_flags);
1104         if (err)
1105                 return ERR_PTR(err);
1106
1107         if (!pd->device->reg_phys_mr)
1108                 return ERR_PTR(-ENOSYS);
1109
1110         mr = pd->device->reg_phys_mr(pd, phys_buf_array, num_phys_buf,
1111                                      mr_access_flags, iova_start);
1112
1113         if (!IS_ERR(mr)) {
1114                 mr->device  = pd->device;
1115                 mr->pd      = pd;
1116                 mr->uobject = NULL;
1117                 atomic_inc(&pd->usecnt);
1118                 atomic_set(&mr->usecnt, 0);
1119         }
1120
1121         return mr;
1122 }
1123 EXPORT_SYMBOL(ib_reg_phys_mr);
1124
1125 int ib_rereg_phys_mr(struct ib_mr *mr,
1126                      int mr_rereg_mask,
1127                      struct ib_pd *pd,
1128                      struct ib_phys_buf *phys_buf_array,
1129                      int num_phys_buf,
1130                      int mr_access_flags,
1131                      u64 *iova_start)
1132 {
1133         struct ib_pd *old_pd;
1134         int ret;
1135
1136         ret = ib_check_mr_access(mr_access_flags);
1137         if (ret)
1138                 return ret;
1139
1140         if (!mr->device->rereg_phys_mr)
1141                 return -ENOSYS;
1142
1143         if (atomic_read(&mr->usecnt))
1144                 return -EBUSY;
1145
1146         old_pd = mr->pd;
1147
1148         ret = mr->device->rereg_phys_mr(mr, mr_rereg_mask, pd,
1149                                         phys_buf_array, num_phys_buf,
1150                                         mr_access_flags, iova_start);
1151
1152         if (!ret && (mr_rereg_mask & IB_MR_REREG_PD)) {
1153                 atomic_dec(&old_pd->usecnt);
1154                 atomic_inc(&pd->usecnt);
1155         }
1156
1157         return ret;
1158 }
1159 EXPORT_SYMBOL(ib_rereg_phys_mr);
1160
1161 int ib_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr)
1162 {
1163         return mr->device->query_mr ?
1164                 mr->device->query_mr(mr, mr_attr) : -ENOSYS;
1165 }
1166 EXPORT_SYMBOL(ib_query_mr);
1167
1168 int ib_dereg_mr(struct ib_mr *mr)
1169 {
1170         struct ib_pd *pd;
1171         int ret;
1172
1173         if (atomic_read(&mr->usecnt))
1174                 return -EBUSY;
1175
1176         pd = mr->pd;
1177         ret = mr->device->dereg_mr(mr);
1178         if (!ret)
1179                 atomic_dec(&pd->usecnt);
1180
1181         return ret;
1182 }
1183 EXPORT_SYMBOL(ib_dereg_mr);
1184
1185 struct ib_mr *ib_create_mr(struct ib_pd *pd,
1186                            struct ib_mr_init_attr *mr_init_attr)
1187 {
1188         struct ib_mr *mr;
1189
1190         if (!pd->device->create_mr)
1191                 return ERR_PTR(-ENOSYS);
1192
1193         mr = pd->device->create_mr(pd, mr_init_attr);
1194
1195         if (!IS_ERR(mr)) {
1196                 mr->device  = pd->device;
1197                 mr->pd      = pd;
1198                 mr->uobject = NULL;
1199                 atomic_inc(&pd->usecnt);
1200                 atomic_set(&mr->usecnt, 0);
1201         }
1202
1203         return mr;
1204 }
1205 EXPORT_SYMBOL(ib_create_mr);
1206
1207 int ib_destroy_mr(struct ib_mr *mr)
1208 {
1209         struct ib_pd *pd;
1210         int ret;
1211
1212         if (atomic_read(&mr->usecnt))
1213                 return -EBUSY;
1214
1215         pd = mr->pd;
1216         ret = mr->device->destroy_mr(mr);
1217         if (!ret)
1218                 atomic_dec(&pd->usecnt);
1219
1220         return ret;
1221 }
1222 EXPORT_SYMBOL(ib_destroy_mr);
1223
1224 struct ib_mr *ib_alloc_fast_reg_mr(struct ib_pd *pd, int max_page_list_len)
1225 {
1226         struct ib_mr *mr;
1227
1228         if (!pd->device->alloc_fast_reg_mr)
1229                 return ERR_PTR(-ENOSYS);
1230
1231         mr = pd->device->alloc_fast_reg_mr(pd, max_page_list_len);
1232
1233         if (!IS_ERR(mr)) {
1234                 mr->device  = pd->device;
1235                 mr->pd      = pd;
1236                 mr->uobject = NULL;
1237                 atomic_inc(&pd->usecnt);
1238                 atomic_set(&mr->usecnt, 0);
1239         }
1240
1241         return mr;
1242 }
1243 EXPORT_SYMBOL(ib_alloc_fast_reg_mr);
1244
1245 struct ib_fast_reg_page_list *ib_alloc_fast_reg_page_list(struct ib_device *device,
1246                                                           int max_page_list_len)
1247 {
1248         struct ib_fast_reg_page_list *page_list;
1249
1250         if (!device->alloc_fast_reg_page_list)
1251                 return ERR_PTR(-ENOSYS);
1252
1253         page_list = device->alloc_fast_reg_page_list(device, max_page_list_len);
1254
1255         if (!IS_ERR(page_list)) {
1256                 page_list->device = device;
1257                 page_list->max_page_list_len = max_page_list_len;
1258         }
1259
1260         return page_list;
1261 }
1262 EXPORT_SYMBOL(ib_alloc_fast_reg_page_list);
1263
1264 void ib_free_fast_reg_page_list(struct ib_fast_reg_page_list *page_list)
1265 {
1266         page_list->device->free_fast_reg_page_list(page_list);
1267 }
1268 EXPORT_SYMBOL(ib_free_fast_reg_page_list);
1269
1270 /* Memory windows */
1271
1272 struct ib_mw *ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type)
1273 {
1274         struct ib_mw *mw;
1275
1276         if (!pd->device->alloc_mw)
1277                 return ERR_PTR(-ENOSYS);
1278
1279         mw = pd->device->alloc_mw(pd, type);
1280         if (!IS_ERR(mw)) {
1281                 mw->device  = pd->device;
1282                 mw->pd      = pd;
1283                 mw->uobject = NULL;
1284                 mw->type    = type;
1285                 atomic_inc(&pd->usecnt);
1286         }
1287
1288         return mw;
1289 }
1290 EXPORT_SYMBOL(ib_alloc_mw);
1291
1292 int ib_dealloc_mw(struct ib_mw *mw)
1293 {
1294         struct ib_pd *pd;
1295         int ret;
1296
1297         pd = mw->pd;
1298         ret = mw->device->dealloc_mw(mw);
1299         if (!ret)
1300                 atomic_dec(&pd->usecnt);
1301
1302         return ret;
1303 }
1304 EXPORT_SYMBOL(ib_dealloc_mw);
1305
1306 /* "Fast" memory regions */
1307
1308 struct ib_fmr *ib_alloc_fmr(struct ib_pd *pd,
1309                             int mr_access_flags,
1310                             struct ib_fmr_attr *fmr_attr)
1311 {
1312         struct ib_fmr *fmr;
1313
1314         if (!pd->device->alloc_fmr)
1315                 return ERR_PTR(-ENOSYS);
1316
1317         fmr = pd->device->alloc_fmr(pd, mr_access_flags, fmr_attr);
1318         if (!IS_ERR(fmr)) {
1319                 fmr->device = pd->device;
1320                 fmr->pd     = pd;
1321                 atomic_inc(&pd->usecnt);
1322         }
1323
1324         return fmr;
1325 }
1326 EXPORT_SYMBOL(ib_alloc_fmr);
1327
1328 int ib_unmap_fmr(struct list_head *fmr_list)
1329 {
1330         struct ib_fmr *fmr;
1331
1332         if (list_empty(fmr_list))
1333                 return 0;
1334
1335         fmr = list_entry(fmr_list->next, struct ib_fmr, list);
1336         return fmr->device->unmap_fmr(fmr_list);
1337 }
1338 EXPORT_SYMBOL(ib_unmap_fmr);
1339
1340 int ib_dealloc_fmr(struct ib_fmr *fmr)
1341 {
1342         struct ib_pd *pd;
1343         int ret;
1344
1345         pd = fmr->pd;
1346         ret = fmr->device->dealloc_fmr(fmr);
1347         if (!ret)
1348                 atomic_dec(&pd->usecnt);
1349
1350         return ret;
1351 }
1352 EXPORT_SYMBOL(ib_dealloc_fmr);
1353
1354 /* Multicast groups */
1355
1356 int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
1357 {
1358         int ret;
1359
1360         if (!qp->device->attach_mcast)
1361                 return -ENOSYS;
1362         if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD)
1363                 return -EINVAL;
1364
1365         ret = qp->device->attach_mcast(qp, gid, lid);
1366         if (!ret)
1367                 atomic_inc(&qp->usecnt);
1368         return ret;
1369 }
1370 EXPORT_SYMBOL(ib_attach_mcast);
1371
1372 int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
1373 {
1374         int ret;
1375
1376         if (!qp->device->detach_mcast)
1377                 return -ENOSYS;
1378         if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD)
1379                 return -EINVAL;
1380
1381         ret = qp->device->detach_mcast(qp, gid, lid);
1382         if (!ret)
1383                 atomic_dec(&qp->usecnt);
1384         return ret;
1385 }
1386 EXPORT_SYMBOL(ib_detach_mcast);
1387
1388 struct ib_xrcd *ib_alloc_xrcd(struct ib_device *device)
1389 {
1390         struct ib_xrcd *xrcd;
1391
1392         if (!device->alloc_xrcd)
1393                 return ERR_PTR(-ENOSYS);
1394
1395         xrcd = device->alloc_xrcd(device, NULL, NULL);
1396         if (!IS_ERR(xrcd)) {
1397                 xrcd->device = device;
1398                 xrcd->inode = NULL;
1399                 atomic_set(&xrcd->usecnt, 0);
1400                 mutex_init(&xrcd->tgt_qp_mutex);
1401                 INIT_LIST_HEAD(&xrcd->tgt_qp_list);
1402         }
1403
1404         return xrcd;
1405 }
1406 EXPORT_SYMBOL(ib_alloc_xrcd);
1407
1408 int ib_dealloc_xrcd(struct ib_xrcd *xrcd)
1409 {
1410         struct ib_qp *qp;
1411         int ret;
1412
1413         if (atomic_read(&xrcd->usecnt))
1414                 return -EBUSY;
1415
1416         while (!list_empty(&xrcd->tgt_qp_list)) {
1417                 qp = list_entry(xrcd->tgt_qp_list.next, struct ib_qp, xrcd_list);
1418                 ret = ib_destroy_qp(qp);
1419                 if (ret)
1420                         return ret;
1421         }
1422
1423         return xrcd->device->dealloc_xrcd(xrcd);
1424 }
1425 EXPORT_SYMBOL(ib_dealloc_xrcd);
1426
1427 struct ib_flow *ib_create_flow(struct ib_qp *qp,
1428                                struct ib_flow_attr *flow_attr,
1429                                int domain)
1430 {
1431         struct ib_flow *flow_id;
1432         if (!qp->device->create_flow)
1433                 return ERR_PTR(-ENOSYS);
1434
1435         flow_id = qp->device->create_flow(qp, flow_attr, domain);
1436         if (!IS_ERR(flow_id))
1437                 atomic_inc(&qp->usecnt);
1438         return flow_id;
1439 }
1440 EXPORT_SYMBOL(ib_create_flow);
1441
1442 int ib_destroy_flow(struct ib_flow *flow_id)
1443 {
1444         int err;
1445         struct ib_qp *qp = flow_id->qp;
1446
1447         err = qp->device->destroy_flow(flow_id);
1448         if (!err)
1449                 atomic_dec(&qp->usecnt);
1450         return err;
1451 }
1452 EXPORT_SYMBOL(ib_destroy_flow);
1453
1454 int ib_check_mr_status(struct ib_mr *mr, u32 check_mask,
1455                        struct ib_mr_status *mr_status)
1456 {
1457         return mr->device->check_mr_status ?
1458                 mr->device->check_mr_status(mr, check_mask, mr_status) : -ENOSYS;
1459 }
1460 EXPORT_SYMBOL(ib_check_mr_status);