mlx4: /dev/ -> /dev_vfs/
[akaros.git] / kern / drivers / net / mlx4 / eq.c
1 /*
2  * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies. All rights reserved.
3  * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc. All rights reserved.
4  *
5  * This software is available to you under a choice of one of two
6  * licenses.  You may choose to be licensed under the terms of the GNU
7  * General Public License (GPL) Version 2, available from the file
8  * COPYING in the main directory of this source tree, or the
9  * OpenIB.org BSD license below:
10  *
11  *     Redistribution and use in source and binary forms, with or
12  *     without modification, are permitted provided that the following
13  *     conditions are met:
14  *
15  *      - Redistributions of source code must retain the above
16  *        copyright notice, this list of conditions and the following
17  *        disclaimer.
18  *
19  *      - Redistributions in binary form must reproduce the above
20  *        copyright notice, this list of conditions and the following
21  *        disclaimer in the documentation and/or other materials
22  *        provided with the distribution.
23  *
24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31  * SOFTWARE.
32  */
33
34 #include <linux_compat.h>
35 #include <linux/mlx4/cmd.h>
36 #include "mlx4.h"
37 #include "fw.h"
38
39 enum {
40         MLX4_IRQNAME_SIZE       = 32
41 };
42
43 enum {
44         MLX4_NUM_ASYNC_EQE      = 0x100,
45         MLX4_NUM_SPARE_EQE      = 0x80,
46         MLX4_EQ_ENTRY_SIZE      = 0x20
47 };
48
49 #define MLX4_EQ_STATUS_OK          ( 0 << 28)
50 #define MLX4_EQ_STATUS_WRITE_FAIL  (10 << 28)
51 #define MLX4_EQ_OWNER_SW           ( 0 << 24)
52 #define MLX4_EQ_OWNER_HW           ( 1 << 24)
53 #define MLX4_EQ_FLAG_EC            ( 1 << 18)
54 #define MLX4_EQ_FLAG_OI            ( 1 << 17)
55 #define MLX4_EQ_STATE_ARMED        ( 9 <<  8)
56 #define MLX4_EQ_STATE_FIRED        (10 <<  8)
57 #define MLX4_EQ_STATE_ALWAYS_ARMED (11 <<  8)
58
59 #define MLX4_ASYNC_EVENT_MASK ((1ull << MLX4_EVENT_TYPE_PATH_MIG)           | \
60                                (1ull << MLX4_EVENT_TYPE_COMM_EST)           | \
61                                (1ull << MLX4_EVENT_TYPE_SQ_DRAINED)         | \
62                                (1ull << MLX4_EVENT_TYPE_CQ_ERROR)           | \
63                                (1ull << MLX4_EVENT_TYPE_WQ_CATAS_ERROR)     | \
64                                (1ull << MLX4_EVENT_TYPE_EEC_CATAS_ERROR)    | \
65                                (1ull << MLX4_EVENT_TYPE_PATH_MIG_FAILED)    | \
66                                (1ull << MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR) | \
67                                (1ull << MLX4_EVENT_TYPE_WQ_ACCESS_ERROR)    | \
68                                (1ull << MLX4_EVENT_TYPE_PORT_CHANGE)        | \
69                                (1ull << MLX4_EVENT_TYPE_ECC_DETECT)         | \
70                                (1ull << MLX4_EVENT_TYPE_SRQ_CATAS_ERROR)    | \
71                                (1ull << MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE)    | \
72                                (1ull << MLX4_EVENT_TYPE_SRQ_LIMIT)          | \
73                                (1ull << MLX4_EVENT_TYPE_CMD)                | \
74                                (1ull << MLX4_EVENT_TYPE_OP_REQUIRED)        | \
75                                (1ull << MLX4_EVENT_TYPE_COMM_CHANNEL)       | \
76                                (1ull << MLX4_EVENT_TYPE_FLR_EVENT)          | \
77                                (1ull << MLX4_EVENT_TYPE_FATAL_WARNING))
78
79 static uint64_t get_async_ev_mask(struct mlx4_dev *dev)
80 {
81         uint64_t async_ev_mask = MLX4_ASYNC_EVENT_MASK;
82         if (dev->caps.flags & MLX4_DEV_CAP_FLAG_PORT_MNG_CHG_EV)
83                 async_ev_mask |= (1ull << MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT);
84         if (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RECOVERABLE_ERROR_EVENT)
85                 async_ev_mask |= (1ull << MLX4_EVENT_TYPE_RECOVERABLE_ERROR_EVENT);
86
87         return async_ev_mask;
88 }
89
90 static void eq_set_ci(struct mlx4_eq *eq, int req_not)
91 {
92         __raw_write32((__force uint32_t) cpu_to_be32((eq->cons_index & 0xffffff) |
93                                                     req_not << 31),
94                      eq->doorbell);
95         /* We still want ordering, just not swabbing, so add a barrier */
96         mb();
97 }
98
99 static struct mlx4_eqe *get_eqe(struct mlx4_eq *eq, uint32_t entry,
100                                 uint8_t eqe_factor,
101                                 uint8_t eqe_size)
102 {
103         /* (entry & (eq->nent - 1)) gives us a cyclic array */
104         unsigned long offset = (entry & (eq->nent - 1)) * eqe_size;
105         /* CX3 is capable of extending the EQE from 32 to 64 bytes with
106          * strides of 64B,128B and 256B.
107          * When 64B EQE is used, the first (in the lower addresses)
108          * 32 bytes in the 64 byte EQE are reserved and the next 32 bytes
109          * contain the legacy EQE information.
110          * In all other cases, the first 32B contains the legacy EQE info.
111          */
112         return eq->page_list[offset / PAGE_SIZE].buf + (offset + (eqe_factor ? MLX4_EQ_ENTRY_SIZE : 0)) % PAGE_SIZE;
113 }
114
115 static struct mlx4_eqe *next_eqe_sw(struct mlx4_eq *eq, uint8_t eqe_factor,
116                                     uint8_t size)
117 {
118         struct mlx4_eqe *eqe = get_eqe(eq, eq->cons_index, eqe_factor, size);
119         return !!(eqe->owner & 0x80) ^ !!(eq->cons_index & eq->nent) ? NULL : eqe;
120 }
121
122 static struct mlx4_eqe *next_slave_event_eqe(struct mlx4_slave_event_eq *slave_eq)
123 {
124         struct mlx4_eqe *eqe =
125                 &slave_eq->event_eqe[slave_eq->cons & (SLAVE_EVENT_EQ_SIZE - 1)];
126         return (!!(eqe->owner & 0x80) ^
127                 !!(slave_eq->cons & SLAVE_EVENT_EQ_SIZE)) ?
128                 eqe : NULL;
129 }
130
131 void mlx4_gen_slave_eqe(struct work_struct *work)
132 {
133         struct mlx4_mfunc_master_ctx *master =
134                 container_of(work, struct mlx4_mfunc_master_ctx,
135                              slave_event_work);
136         struct mlx4_mfunc *mfunc =
137                 container_of(master, struct mlx4_mfunc, master);
138         struct mlx4_priv *priv = container_of(mfunc, struct mlx4_priv, mfunc);
139         struct mlx4_dev *dev = &priv->dev;
140         struct mlx4_slave_event_eq *slave_eq = &mfunc->master.slave_eq;
141         struct mlx4_eqe *eqe;
142         uint8_t slave;
143         int i;
144
145         for (eqe = next_slave_event_eqe(slave_eq); eqe;
146               eqe = next_slave_event_eqe(slave_eq)) {
147                 slave = eqe->slave_id;
148
149                 /* All active slaves need to receive the event */
150                 if (slave == ALL_SLAVES) {
151                         for (i = 0; i <= dev->persist->num_vfs; i++) {
152                                 if (mlx4_GEN_EQE(dev, i, eqe))
153                                         mlx4_warn(dev, "Failed to generate event for slave %d\n",
154                                                   i);
155                         }
156                 } else {
157                         if (mlx4_GEN_EQE(dev, slave, eqe))
158                                 mlx4_warn(dev, "Failed to generate event for slave %d\n",
159                                           slave);
160                 }
161                 ++slave_eq->cons;
162         }
163 }
164
165
166 static void slave_event(struct mlx4_dev *dev, uint8_t slave,
167                         struct mlx4_eqe *eqe)
168 {
169         struct mlx4_priv *priv = mlx4_priv(dev);
170         struct mlx4_slave_event_eq *slave_eq = &priv->mfunc.master.slave_eq;
171         struct mlx4_eqe *s_eqe;
172         unsigned long flags;
173
174         spin_lock_irqsave(&slave_eq->event_lock);
175         s_eqe = &slave_eq->event_eqe[slave_eq->prod & (SLAVE_EVENT_EQ_SIZE - 1)];
176         if ((!!(s_eqe->owner & 0x80)) ^
177             (!!(slave_eq->prod & SLAVE_EVENT_EQ_SIZE))) {
178                 mlx4_warn(dev, "Master failed to generate an EQE for slave: %d. No free EQE on slave events queue\n",
179                           slave);
180                 spin_unlock_irqsave(&slave_eq->event_lock);
181                 return;
182         }
183
184         memcpy(s_eqe, eqe, dev->caps.eqe_size - 1);
185         s_eqe->slave_id = slave;
186         /* ensure all information is written before setting the ownersip bit */
187         bus_wmb();
188         s_eqe->owner = !!(slave_eq->prod & SLAVE_EVENT_EQ_SIZE) ? 0x0 : 0x80;
189         ++slave_eq->prod;
190
191         queue_work(priv->mfunc.master.comm_wq,
192                    &priv->mfunc.master.slave_event_work);
193         spin_unlock_irqsave(&slave_eq->event_lock);
194 }
195
196 static void mlx4_slave_event(struct mlx4_dev *dev, int slave,
197                              struct mlx4_eqe *eqe)
198 {
199         struct mlx4_priv *priv = mlx4_priv(dev);
200
201         if (slave < 0 || slave > dev->persist->num_vfs ||
202             slave == dev->caps.function ||
203             !priv->mfunc.master.slave_state[slave].active)
204                 return;
205
206         slave_event(dev, slave, eqe);
207 }
208
209 int mlx4_gen_pkey_eqe(struct mlx4_dev *dev, int slave, uint8_t port)
210 {
211         struct mlx4_eqe eqe;
212
213         struct mlx4_priv *priv = mlx4_priv(dev);
214         struct mlx4_slave_state *s_slave = &priv->mfunc.master.slave_state[slave];
215
216         if (!s_slave->active)
217                 return 0;
218
219         memset(&eqe, 0, sizeof eqe);
220
221         eqe.type = MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT;
222         eqe.subtype = MLX4_DEV_PMC_SUBTYPE_PKEY_TABLE;
223         eqe.event.port_mgmt_change.port = port;
224
225         return mlx4_GEN_EQE(dev, slave, &eqe);
226 }
227 EXPORT_SYMBOL(mlx4_gen_pkey_eqe);
228
229 int mlx4_gen_guid_change_eqe(struct mlx4_dev *dev, int slave, uint8_t port)
230 {
231         struct mlx4_eqe eqe;
232
233         /*don't send if we don't have the that slave */
234         if (dev->persist->num_vfs < slave)
235                 return 0;
236         memset(&eqe, 0, sizeof eqe);
237
238         eqe.type = MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT;
239         eqe.subtype = MLX4_DEV_PMC_SUBTYPE_GUID_INFO;
240         eqe.event.port_mgmt_change.port = port;
241
242         return mlx4_GEN_EQE(dev, slave, &eqe);
243 }
244 EXPORT_SYMBOL(mlx4_gen_guid_change_eqe);
245
246 int mlx4_gen_port_state_change_eqe(struct mlx4_dev *dev, int slave,
247                                    uint8_t port,
248                                    uint8_t port_subtype_change)
249 {
250         struct mlx4_eqe eqe;
251
252         /*don't send if we don't have the that slave */
253         if (dev->persist->num_vfs < slave)
254                 return 0;
255         memset(&eqe, 0, sizeof eqe);
256
257         eqe.type = MLX4_EVENT_TYPE_PORT_CHANGE;
258         eqe.subtype = port_subtype_change;
259         eqe.event.port_change.port = cpu_to_be32(port << 28);
260
261         mlx4_dbg(dev, "%s: sending: %d to slave: %d on port: %d\n", __func__,
262                  port_subtype_change, slave, port);
263         return mlx4_GEN_EQE(dev, slave, &eqe);
264 }
265 EXPORT_SYMBOL(mlx4_gen_port_state_change_eqe);
266
267 enum slave_port_state mlx4_get_slave_port_state(struct mlx4_dev *dev, int slave,
268                                                 uint8_t port)
269 {
270         struct mlx4_priv *priv = mlx4_priv(dev);
271         struct mlx4_slave_state *s_state = priv->mfunc.master.slave_state;
272         struct mlx4_active_ports actv_ports = mlx4_get_active_ports(dev, slave);
273
274         if (slave >= dev->num_slaves || port > dev->caps.num_ports ||
275             port <= 0 || !test_bit(port - 1, actv_ports.ports)) {
276                 pr_err("%s: Error: asking for slave:%d, port:%d\n",
277                        __func__, slave, port);
278                 return SLAVE_PORT_DOWN;
279         }
280         return s_state[slave].port_state[port];
281 }
282 EXPORT_SYMBOL(mlx4_get_slave_port_state);
283
284 static int mlx4_set_slave_port_state(struct mlx4_dev *dev, int slave,
285                                      uint8_t port,
286                                      enum slave_port_state state)
287 {
288         struct mlx4_priv *priv = mlx4_priv(dev);
289         struct mlx4_slave_state *s_state = priv->mfunc.master.slave_state;
290         struct mlx4_active_ports actv_ports = mlx4_get_active_ports(dev, slave);
291
292         if (slave >= dev->num_slaves || port > dev->caps.num_ports ||
293             port <= 0 || !test_bit(port - 1, actv_ports.ports)) {
294                 pr_err("%s: Error: asking for slave:%d, port:%d\n",
295                        __func__, slave, port);
296                 return -1;
297         }
298         s_state[slave].port_state[port] = state;
299
300         return 0;
301 }
302
303 static void set_all_slave_state(struct mlx4_dev *dev, uint8_t port, int event)
304 {
305         int i;
306         enum slave_port_gen_event gen_event;
307         struct mlx4_slaves_pport slaves_pport = mlx4_phys_to_slaves_pport(dev,
308                                                                           port);
309
310         for (i = 0; i < dev->persist->num_vfs + 1; i++)
311                 if (test_bit(i, slaves_pport.slaves))
312                         set_and_calc_slave_port_state(dev, i, port,
313                                                       event, &gen_event);
314 }
315 /**************************************************************************
316         The function get as input the new event to that port,
317         and according to the prev state change the slave's port state.
318         The events are:
319                 MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN,
320                 MLX4_PORT_STATE_DEV_EVENT_PORT_UP
321                 MLX4_PORT_STATE_IB_EVENT_GID_VALID
322                 MLX4_PORT_STATE_IB_EVENT_GID_INVALID
323 ***************************************************************************/
324 int set_and_calc_slave_port_state(struct mlx4_dev *dev, int slave,
325                                   uint8_t port, int event,
326                                   enum slave_port_gen_event *gen_event)
327 {
328         struct mlx4_priv *priv = mlx4_priv(dev);
329         struct mlx4_slave_state *ctx = NULL;
330         unsigned long flags;
331         int ret = -1;
332         struct mlx4_active_ports actv_ports = mlx4_get_active_ports(dev, slave);
333         enum slave_port_state cur_state =
334                 mlx4_get_slave_port_state(dev, slave, port);
335
336         *gen_event = SLAVE_PORT_GEN_EVENT_NONE;
337
338         if (slave >= dev->num_slaves || port > dev->caps.num_ports ||
339             port <= 0 || !test_bit(port - 1, actv_ports.ports)) {
340                 pr_err("%s: Error: asking for slave:%d, port:%d\n",
341                        __func__, slave, port);
342                 return ret;
343         }
344
345         ctx = &priv->mfunc.master.slave_state[slave];
346         spin_lock_irqsave(&ctx->lock);
347
348         switch (cur_state) {
349         case SLAVE_PORT_DOWN:
350                 if (MLX4_PORT_STATE_DEV_EVENT_PORT_UP == event)
351                         mlx4_set_slave_port_state(dev, slave, port,
352                                                   SLAVE_PENDING_UP);
353                 break;
354         case SLAVE_PENDING_UP:
355                 if (MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN == event)
356                         mlx4_set_slave_port_state(dev, slave, port,
357                                                   SLAVE_PORT_DOWN);
358                 else if (MLX4_PORT_STATE_IB_PORT_STATE_EVENT_GID_VALID == event) {
359                         mlx4_set_slave_port_state(dev, slave, port,
360                                                   SLAVE_PORT_UP);
361                         *gen_event = SLAVE_PORT_GEN_EVENT_UP;
362                 }
363                 break;
364         case SLAVE_PORT_UP:
365                 if (MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN == event) {
366                         mlx4_set_slave_port_state(dev, slave, port,
367                                                   SLAVE_PORT_DOWN);
368                         *gen_event = SLAVE_PORT_GEN_EVENT_DOWN;
369                 } else if (MLX4_PORT_STATE_IB_EVENT_GID_INVALID ==
370                                 event) {
371                         mlx4_set_slave_port_state(dev, slave, port,
372                                                   SLAVE_PENDING_UP);
373                         *gen_event = SLAVE_PORT_GEN_EVENT_DOWN;
374                 }
375                 break;
376         default:
377                 pr_err("%s: BUG!!! UNKNOWN state: slave:%d, port:%d\n",
378                        __func__, slave, port);
379                 goto out;
380         }
381         ret = mlx4_get_slave_port_state(dev, slave, port);
382
383 out:
384         spin_unlock_irqsave(&ctx->lock);
385         return ret;
386 }
387
388 EXPORT_SYMBOL(set_and_calc_slave_port_state);
389
390 int mlx4_gen_slaves_port_mgt_ev(struct mlx4_dev *dev, uint8_t port, int attr)
391 {
392         struct mlx4_eqe eqe;
393
394         memset(&eqe, 0, sizeof eqe);
395
396         eqe.type = MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT;
397         eqe.subtype = MLX4_DEV_PMC_SUBTYPE_PORT_INFO;
398         eqe.event.port_mgmt_change.port = port;
399         eqe.event.port_mgmt_change.params.port_info.changed_attr =
400                 cpu_to_be32((uint32_t) attr);
401
402         slave_event(dev, ALL_SLAVES, &eqe);
403         return 0;
404 }
405 EXPORT_SYMBOL(mlx4_gen_slaves_port_mgt_ev);
406
407 void mlx4_master_handle_slave_flr(struct work_struct *work)
408 {
409         struct mlx4_mfunc_master_ctx *master =
410                 container_of(work, struct mlx4_mfunc_master_ctx,
411                              slave_flr_event_work);
412         struct mlx4_mfunc *mfunc =
413                 container_of(master, struct mlx4_mfunc, master);
414         struct mlx4_priv *priv =
415                 container_of(mfunc, struct mlx4_priv, mfunc);
416         struct mlx4_dev *dev = &priv->dev;
417         struct mlx4_slave_state *slave_state = priv->mfunc.master.slave_state;
418         int i;
419         int err;
420         unsigned long flags;
421
422         mlx4_dbg(dev, "mlx4_handle_slave_flr\n");
423
424         for (i = 0 ; i < dev->num_slaves; i++) {
425
426                 if (MLX4_COMM_CMD_FLR == slave_state[i].last_cmd) {
427                         mlx4_dbg(dev, "mlx4_handle_slave_flr: clean slave: %d\n",
428                                  i);
429                         /* In case of 'Reset flow' FLR can be generated for
430                          * a slave before mlx4_load_one is done.
431                          * make sure interface is up before trying to delete
432                          * slave resources which weren't allocated yet.
433                          */
434                         if (dev->persist->interface_state &
435                             MLX4_INTERFACE_STATE_UP)
436                                 mlx4_delete_all_resources_for_slave(dev, i);
437                         /*return the slave to running mode*/
438                         spin_lock_irqsave(&priv->mfunc.master.slave_state_lock);
439                         slave_state[i].last_cmd = MLX4_COMM_CMD_RESET;
440                         slave_state[i].is_slave_going_down = 0;
441                         spin_unlock_irqsave(&priv->mfunc.master.slave_state_lock);
442                         /*notify the FW:*/
443                         err = mlx4_cmd(dev, 0, i, 0, MLX4_CMD_INFORM_FLR_DONE,
444                                        MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
445                         if (err)
446                                 mlx4_warn(dev, "Failed to notify FW on FLR done (slave:%d)\n",
447                                           i);
448                 }
449         }
450 }
451
452 static int mlx4_eq_int(struct mlx4_dev *dev, struct mlx4_eq *eq)
453 {
454         struct mlx4_priv *priv = mlx4_priv(dev);
455         struct mlx4_eqe *eqe;
456         int cqn = -1;
457         int eqes_found = 0;
458         int set_ci = 0;
459         int port;
460         int slave = 0;
461         int ret;
462         uint32_t flr_slave;
463         uint8_t update_slave_state;
464         int i;
465         enum slave_port_gen_event gen_event;
466         unsigned long flags;
467         struct mlx4_vport_state *s_info;
468         int eqe_size = dev->caps.eqe_size;
469
470         while ((eqe = next_eqe_sw(eq, dev->caps.eqe_factor, eqe_size))) {
471                 /*
472                  * Make sure we read EQ entry contents after we've
473                  * checked the ownership bit.
474                  */
475                 bus_rmb();
476
477                 switch (eqe->type) {
478                 case MLX4_EVENT_TYPE_COMP:
479                         cqn = be32_to_cpu(eqe->event.comp.cqn) & 0xffffff;
480                         mlx4_cq_completion(dev, cqn);
481                         break;
482
483                 case MLX4_EVENT_TYPE_PATH_MIG:
484                 case MLX4_EVENT_TYPE_COMM_EST:
485                 case MLX4_EVENT_TYPE_SQ_DRAINED:
486                 case MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE:
487                 case MLX4_EVENT_TYPE_WQ_CATAS_ERROR:
488                 case MLX4_EVENT_TYPE_PATH_MIG_FAILED:
489                 case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
490                 case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR:
491                         mlx4_dbg(dev, "event %d arrived\n", eqe->type);
492                         if (mlx4_is_master(dev)) {
493                                 /* forward only to slave owning the QP */
494                                 ret = mlx4_get_slave_from_resource_id(dev,
495                                                 RES_QP,
496                                                 be32_to_cpu(eqe->event.qp.qpn)
497                                                 & 0xffffff, &slave);
498                                 if (ret && ret != -ENOENT) {
499                                         mlx4_dbg(dev, "QP event %02x(%02x) on EQ %d at index %u: could not get slave id (%d)\n",
500                                                  eqe->type, eqe->subtype,
501                                                  eq->eqn, eq->cons_index, ret);
502                                         break;
503                                 }
504
505                                 if (!ret && slave != dev->caps.function) {
506                                         mlx4_slave_event(dev, slave, eqe);
507                                         break;
508                                 }
509
510                         }
511                         mlx4_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) &
512                                       0xffffff, eqe->type);
513                         break;
514
515                 case MLX4_EVENT_TYPE_SRQ_LIMIT:
516                         mlx4_dbg(dev, "%s: MLX4_EVENT_TYPE_SRQ_LIMIT\n",
517                                  __func__);
518                 case MLX4_EVENT_TYPE_SRQ_CATAS_ERROR:
519                         if (mlx4_is_master(dev)) {
520                                 /* forward only to slave owning the SRQ */
521                                 ret = mlx4_get_slave_from_resource_id(dev,
522                                                 RES_SRQ,
523                                                 be32_to_cpu(eqe->event.srq.srqn)
524                                                 & 0xffffff,
525                                                 &slave);
526                                 if (ret && ret != -ENOENT) {
527                                         mlx4_warn(dev, "SRQ event %02x(%02x) on EQ %d at index %u: could not get slave id (%d)\n",
528                                                   eqe->type, eqe->subtype,
529                                                   eq->eqn, eq->cons_index, ret);
530                                         break;
531                                 }
532                                 mlx4_warn(dev, "%s: slave:%d, srq_no:0x%x, event: %02x(%02x)\n",
533                                           __func__, slave,
534                                           be32_to_cpu(eqe->event.srq.srqn),
535                                           eqe->type, eqe->subtype);
536
537                                 if (!ret && slave != dev->caps.function) {
538                                         mlx4_warn(dev, "%s: sending event %02x(%02x) to slave:%d\n",
539                                                   __func__, eqe->type,
540                                                   eqe->subtype, slave);
541                                         mlx4_slave_event(dev, slave, eqe);
542                                         break;
543                                 }
544                         }
545                         mlx4_srq_event(dev, be32_to_cpu(eqe->event.srq.srqn) &
546                                        0xffffff, eqe->type);
547                         break;
548
549                 case MLX4_EVENT_TYPE_CMD:
550                         mlx4_cmd_event(dev,
551                                        be16_to_cpu(eqe->event.cmd.token),
552                                        eqe->event.cmd.status,
553                                        be64_to_cpu(eqe->event.cmd.out_param));
554                         break;
555
556                 case MLX4_EVENT_TYPE_PORT_CHANGE: {
557                         struct mlx4_slaves_pport slaves_port;
558                         port = be32_to_cpu(eqe->event.port_change.port) >> 28;
559                         slaves_port = mlx4_phys_to_slaves_pport(dev, port);
560                         if (eqe->subtype == MLX4_PORT_CHANGE_SUBTYPE_DOWN) {
561                                 mlx4_dispatch_event(dev, MLX4_DEV_EVENT_PORT_DOWN,
562                                                     port);
563                                 mlx4_priv(dev)->sense.do_sense_port[port] = 1;
564                                 if (!mlx4_is_master(dev))
565                                         break;
566                                 for (i = 0; i < dev->persist->num_vfs + 1;
567                                      i++) {
568                                         if (!test_bit(i, slaves_port.slaves))
569                                                 continue;
570                                         if (dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH) {
571                                                 if (i == mlx4_master_func_num(dev))
572                                                         continue;
573                                                 mlx4_dbg(dev, "%s: Sending MLX4_PORT_CHANGE_SUBTYPE_DOWN to slave: %d, port:%d\n",
574                                                          __func__, i, port);
575                                                 panic("Disabled");
576 #if 0 // AKAROS_PORT
577                                                 s_info = &priv->mfunc.master.vf_oper[slave].vport[port].state;
578                                                 if (IFLA_VF_LINK_STATE_AUTO == s_info->link_state) {
579                                                         eqe->event.port_change.port =
580                                                                 cpu_to_be32(
581                                                                 (be32_to_cpu(eqe->event.port_change.port) & 0xFFFFFFF)
582                                                                 | (mlx4_phys_to_slave_port(dev, i, port) << 28));
583                                                         mlx4_slave_event(dev, i, eqe);
584                                                 }
585 #endif
586                                         } else {  /* IB port */
587                                                 set_and_calc_slave_port_state(dev, i, port,
588                                                                               MLX4_PORT_STATE_DEV_EVENT_PORT_DOWN,
589                                                                               &gen_event);
590                                                 /*we can be in pending state, then do not send port_down event*/
591                                                 if (SLAVE_PORT_GEN_EVENT_DOWN ==  gen_event) {
592                                                         if (i == mlx4_master_func_num(dev))
593                                                                 continue;
594                                                         mlx4_slave_event(dev, i, eqe);
595                                                 }
596                                         }
597                                 }
598                         } else {
599                                 mlx4_dispatch_event(dev, MLX4_DEV_EVENT_PORT_UP, port);
600
601                                 mlx4_priv(dev)->sense.do_sense_port[port] = 0;
602
603                                 if (!mlx4_is_master(dev))
604                                         break;
605                                 if (dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH)
606                                         for (i = 0;
607                                              i < dev->persist->num_vfs + 1;
608                                              i++) {
609                                                 if (!test_bit(i, slaves_port.slaves))
610                                                         continue;
611                                                 if (i == mlx4_master_func_num(dev))
612                                                         continue;
613                                                 panic("Disabled");
614 #if 0 // AKAROS_PORT
615                                                 s_info = &priv->mfunc.master.vf_oper[slave].vport[port].state;
616                                                 if (IFLA_VF_LINK_STATE_AUTO == s_info->link_state) {
617                                                         eqe->event.port_change.port =
618                                                                 cpu_to_be32(
619                                                                 (be32_to_cpu(eqe->event.port_change.port) & 0xFFFFFFF)
620                                                                 | (mlx4_phys_to_slave_port(dev, i, port) << 28));
621                                                         mlx4_slave_event(dev, i, eqe);
622                                                 }
623 #endif
624                                         }
625                                 else /* IB port */
626                                         /* port-up event will be sent to a slave when the
627                                          * slave's alias-guid is set. This is done in alias_GUID.c
628                                          */
629                                         set_all_slave_state(dev, port, MLX4_DEV_EVENT_PORT_UP);
630                         }
631                         break;
632                 }
633
634                 case MLX4_EVENT_TYPE_CQ_ERROR:
635                         mlx4_warn(dev, "CQ %s on CQN %06x\n",
636                                   eqe->event.cq_err.syndrome == 1 ?
637                                   "overrun" : "access violation",
638                                   be32_to_cpu(eqe->event.cq_err.cqn) & 0xffffff);
639                         if (mlx4_is_master(dev)) {
640                                 ret = mlx4_get_slave_from_resource_id(dev,
641                                         RES_CQ,
642                                         be32_to_cpu(eqe->event.cq_err.cqn)
643                                         & 0xffffff, &slave);
644                                 if (ret && ret != -ENOENT) {
645                                         mlx4_dbg(dev, "CQ event %02x(%02x) on EQ %d at index %u: could not get slave id (%d)\n",
646                                                  eqe->type, eqe->subtype,
647                                                  eq->eqn, eq->cons_index, ret);
648                                         break;
649                                 }
650
651                                 if (!ret && slave != dev->caps.function) {
652                                         mlx4_slave_event(dev, slave, eqe);
653                                         break;
654                                 }
655                         }
656                         mlx4_cq_event(dev,
657                                       be32_to_cpu(eqe->event.cq_err.cqn)
658                                       & 0xffffff,
659                                       eqe->type);
660                         break;
661
662                 case MLX4_EVENT_TYPE_EQ_OVERFLOW:
663                         mlx4_warn(dev, "EQ overrun on EQN %d\n", eq->eqn);
664                         break;
665
666                 case MLX4_EVENT_TYPE_OP_REQUIRED:
667                         atomic_inc(&priv->opreq_count);
668                         /* FW commands can't be executed from interrupt context
669                          * working in deferred task
670                          */
671                         queue_work(mlx4_wq, &priv->opreq_task);
672                         break;
673
674                 case MLX4_EVENT_TYPE_COMM_CHANNEL:
675                         if (!mlx4_is_master(dev)) {
676                                 mlx4_warn(dev, "Received comm channel event for non master device\n");
677                                 break;
678                         }
679                         memcpy(&priv->mfunc.master.comm_arm_bit_vector,
680                                eqe->event.comm_channel_arm.bit_vec,
681                                sizeof eqe->event.comm_channel_arm.bit_vec);
682                         queue_work(priv->mfunc.master.comm_wq,
683                                    &priv->mfunc.master.comm_work);
684                         break;
685
686                 case MLX4_EVENT_TYPE_FLR_EVENT:
687                         flr_slave = be32_to_cpu(eqe->event.flr_event.slave_id);
688                         if (!mlx4_is_master(dev)) {
689                                 mlx4_warn(dev, "Non-master function received FLR event\n");
690                                 break;
691                         }
692
693                         mlx4_dbg(dev, "FLR event for slave: %d\n", flr_slave);
694
695                         if (flr_slave >= dev->num_slaves) {
696                                 mlx4_warn(dev,
697                                           "Got FLR for unknown function: %d\n",
698                                           flr_slave);
699                                 update_slave_state = 0;
700                         } else
701                                 update_slave_state = 1;
702
703                         spin_lock_irqsave(&priv->mfunc.master.slave_state_lock);
704                         if (update_slave_state) {
705                                 priv->mfunc.master.slave_state[flr_slave].active = false;
706                                 priv->mfunc.master.slave_state[flr_slave].last_cmd = MLX4_COMM_CMD_FLR;
707                                 priv->mfunc.master.slave_state[flr_slave].is_slave_going_down = 1;
708                         }
709                         spin_unlock_irqsave(&priv->mfunc.master.slave_state_lock);
710                         mlx4_dispatch_event(dev, MLX4_DEV_EVENT_SLAVE_SHUTDOWN,
711                                             flr_slave);
712                         queue_work(priv->mfunc.master.comm_wq,
713                                    &priv->mfunc.master.slave_flr_event_work);
714                         break;
715
716                 case MLX4_EVENT_TYPE_FATAL_WARNING:
717                         if (eqe->subtype == MLX4_FATAL_WARNING_SUBTYPE_WARMING) {
718                                 if (mlx4_is_master(dev))
719                                         for (i = 0; i < dev->num_slaves; i++) {
720                                                 mlx4_dbg(dev, "%s: Sending MLX4_FATAL_WARNING_SUBTYPE_WARMING to slave: %d\n",
721                                                          __func__, i);
722                                                 if (i == dev->caps.function)
723                                                         continue;
724                                                 mlx4_slave_event(dev, i, eqe);
725                                         }
726                                 mlx4_err(dev, "Temperature Threshold was reached! Threshold: %d celsius degrees; Current Temperature: %d\n",
727                                          be16_to_cpu(eqe->event.warming.warning_threshold),
728                                          be16_to_cpu(eqe->event.warming.current_temperature));
729                         } else
730                                 mlx4_warn(dev, "Unhandled event FATAL WARNING (%02x), subtype %02x on EQ %d at index %u. owner=%x, nent=0x%x, slave=%x, ownership=%s\n",
731                                           eqe->type, eqe->subtype, eq->eqn,
732                                           eq->cons_index, eqe->owner, eq->nent,
733                                           eqe->slave_id,
734                                           !!(eqe->owner & 0x80) ^
735                                           !!(eq->cons_index & eq->nent) ? "HW" : "SW");
736
737                         break;
738
739                 case MLX4_EVENT_TYPE_PORT_MNG_CHG_EVENT:
740                         mlx4_dispatch_event(dev, MLX4_DEV_EVENT_PORT_MGMT_CHANGE,
741                                             (unsigned long) eqe);
742                         break;
743
744                 case MLX4_EVENT_TYPE_RECOVERABLE_ERROR_EVENT:
745                         switch (eqe->subtype) {
746                         case MLX4_RECOVERABLE_ERROR_EVENT_SUBTYPE_BAD_CABLE:
747                                 mlx4_warn(dev, "Bad cable detected on port %u\n",
748                                           eqe->event.bad_cable.port);
749                                 break;
750                         case MLX4_RECOVERABLE_ERROR_EVENT_SUBTYPE_UNSUPPORTED_CABLE:
751                                 mlx4_warn(dev, "Unsupported cable detected\n");
752                                 break;
753                         default:
754                                 mlx4_dbg(dev,
755                                          "Unhandled recoverable error event detected: %02x(%02x) on EQ %d at index %u. owner=%x, nent=0x%x, ownership=%s\n",
756                                          eqe->type, eqe->subtype, eq->eqn,
757                                          eq->cons_index, eqe->owner, eq->nent,
758                                          !!(eqe->owner & 0x80) ^
759                                          !!(eq->cons_index & eq->nent) ? "HW" : "SW");
760                                 break;
761                         }
762                         break;
763
764                 case MLX4_EVENT_TYPE_EEC_CATAS_ERROR:
765                 case MLX4_EVENT_TYPE_ECC_DETECT:
766                 default:
767                         mlx4_warn(dev, "Unhandled event %02x(%02x) on EQ %d at index %u. owner=%x, nent=0x%x, slave=%x, ownership=%s\n",
768                                   eqe->type, eqe->subtype, eq->eqn,
769                                   eq->cons_index, eqe->owner, eq->nent,
770                                   eqe->slave_id,
771                                   !!(eqe->owner & 0x80) ^
772                                   !!(eq->cons_index & eq->nent) ? "HW" : "SW");
773                         break;
774                 };
775
776                 ++eq->cons_index;
777                 eqes_found = 1;
778                 ++set_ci;
779
780                 /*
781                  * The HCA will think the queue has overflowed if we
782                  * don't tell it we've been processing events.  We
783                  * create our EQs with MLX4_NUM_SPARE_EQE extra
784                  * entries, so we must update our consumer index at
785                  * least that often.
786                  */
787                 if (unlikely(set_ci >= MLX4_NUM_SPARE_EQE)) {
788                         eq_set_ci(eq, 0);
789                         set_ci = 0;
790                 }
791         }
792
793         eq_set_ci(eq, 1);
794
795         /* cqn is 24bit wide but is initialized such that its higher bits
796          * are ones too. Thus, if we got any event, cqn's high bits should be off
797          * and we need to schedule the tasklet.
798          */
799         if (!(cqn & ~0xffffff))
800                 tasklet_schedule(&eq->tasklet_ctx.task);
801
802         return eqes_found;
803 }
804
805 #if 0 // AKAROS_PORT
806 static void mlx4_interrupt(struct hw_trapframe *hw_tf, void *dev_ptr)
807 {
808         struct mlx4_dev *dev = dev_ptr;
809         struct mlx4_priv *priv = mlx4_priv(dev);
810         int work = 0;
811         int i;
812
813         write32(priv->eq_table.clr_mask, priv->eq_table.clr_int);
814
815         for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i)
816                 work |= mlx4_eq_int(dev, &priv->eq_table.eq[i]);
817
818         return IRQ_RETVAL(work);
819 }
820
821 static void mlx4_msi_x_interrupt(struct hw_trapframe *hw_tf, void *eq_ptr)
822 {
823         struct mlx4_eq  *eq  = eq_ptr;
824         struct mlx4_dev *dev = eq->dev;
825
826         mlx4_eq_int(dev, eq);
827
828         /* MSI-X vectors always belong to us */
829         return;
830 }
831 #endif
832
833 static void mlx4_msi_x_interrupt_akaros(struct hw_trapframe *hw_tf, void *data)
834 {
835         struct mlx4_eq  *eq = data;
836         struct mlx4_dev *dev = eq ? eq->dev : 0;
837
838         if (!eq || !dev)
839                 panic("!eq || !dev");
840         mlx4_eq_int(dev, eq);
841 }
842
843 int mlx4_MAP_EQ_wrapper(struct mlx4_dev *dev, int slave,
844                         struct mlx4_vhcr *vhcr,
845                         struct mlx4_cmd_mailbox *inbox,
846                         struct mlx4_cmd_mailbox *outbox,
847                         struct mlx4_cmd_info *cmd)
848 {
849         struct mlx4_priv *priv = mlx4_priv(dev);
850         struct mlx4_slave_event_eq_info *event_eq =
851                 priv->mfunc.master.slave_state[slave].event_eq;
852         uint32_t in_modifier = vhcr->in_modifier;
853         uint32_t eqn = in_modifier & 0x3FF;
854         uint64_t in_param =  vhcr->in_param;
855         int err = 0;
856         int i;
857
858         if (slave == dev->caps.function)
859                 err = mlx4_cmd(dev, in_param, (in_modifier & 0x80000000) | eqn,
860                                0, MLX4_CMD_MAP_EQ, MLX4_CMD_TIME_CLASS_B,
861                                MLX4_CMD_NATIVE);
862         if (!err)
863                 for (i = 0; i < MLX4_EVENT_TYPES_NUM; ++i)
864                         if (in_param & (1LL << i))
865                                 event_eq[i].eqn = in_modifier >> 31 ? -1 : eqn;
866
867         return err;
868 }
869
870 static int mlx4_MAP_EQ(struct mlx4_dev *dev, uint64_t event_mask, int unmap,
871                         int eq_num)
872 {
873         return mlx4_cmd(dev, event_mask, (unmap << 31) | eq_num,
874                         0, MLX4_CMD_MAP_EQ, MLX4_CMD_TIME_CLASS_B,
875                         MLX4_CMD_WRAPPED);
876 }
877
878 static int mlx4_SW2HW_EQ(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox,
879                          int eq_num)
880 {
881         return mlx4_cmd(dev, mailbox->dma, eq_num, 0,
882                         MLX4_CMD_SW2HW_EQ, MLX4_CMD_TIME_CLASS_A,
883                         MLX4_CMD_WRAPPED);
884 }
885
886 static int mlx4_HW2SW_EQ(struct mlx4_dev *dev,  int eq_num)
887 {
888         return mlx4_cmd(dev, 0, eq_num, 1, MLX4_CMD_HW2SW_EQ,
889                         MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED);
890 }
891
892 static int mlx4_num_eq_uar(struct mlx4_dev *dev)
893 {
894         /*
895          * Each UAR holds 4 EQ doorbells.  To figure out how many UARs
896          * we need to map, take the difference of highest index and
897          * the lowest index we'll use and add 1.
898          */
899         return (dev->caps.num_comp_vectors + 1 + dev->caps.reserved_eqs +
900                  dev->caps.comp_pool)/4 - dev->caps.reserved_eqs/4 + 1;
901 }
902
903 static void __iomem *mlx4_get_eq_uar(struct mlx4_dev *dev, struct mlx4_eq *eq)
904 {
905         struct mlx4_priv *priv = mlx4_priv(dev);
906         int index;
907
908         index = eq->eqn / 4 - dev->caps.reserved_eqs / 4;
909
910         if (!priv->eq_table.uar_map[index]) {
911                 priv->eq_table.uar_map[index] =
912                         ioremap(pci_resource_start(dev->persist->pdev, 2) +
913                                 ((eq->eqn / 4) << PAGE_SHIFT),
914                                 PAGE_SIZE);
915                 if (!priv->eq_table.uar_map[index]) {
916                         mlx4_err(dev, "Couldn't map EQ doorbell for EQN 0x%06x\n",
917                                  eq->eqn);
918                         return NULL;
919                 }
920         }
921
922         return priv->eq_table.uar_map[index] + 0x800 + 8 * (eq->eqn % 4);
923 }
924
925 static void mlx4_unmap_uar(struct mlx4_dev *dev)
926 {
927         struct mlx4_priv *priv = mlx4_priv(dev);
928         int i;
929
930         for (i = 0; i < mlx4_num_eq_uar(dev); ++i)
931                 if (priv->eq_table.uar_map[i]) {
932                         iounmap(priv->eq_table.uar_map[i]);
933                         priv->eq_table.uar_map[i] = NULL;
934                 }
935 }
936
937 static int mlx4_create_eq(struct mlx4_dev *dev, int nent,
938                           uint8_t intr, struct mlx4_eq *eq)
939 {
940         struct mlx4_priv *priv = mlx4_priv(dev);
941         struct mlx4_cmd_mailbox *mailbox;
942         struct mlx4_eq_context *eq_context;
943         int npages;
944         uint64_t *dma_list = NULL;
945         dma_addr_t t;
946         uint64_t mtt_addr;
947         int err = -ENOMEM;
948         int i;
949
950         eq->dev   = dev;
951         eq->nent  = ROUNDUPPWR2(MAX(nent, 2));
952         /* CX3 is capable of extending the CQE/EQE from 32 to 64 bytes, with
953          * strides of 64B,128B and 256B.
954          */
955         npages = PAGE_ALIGN(eq->nent * dev->caps.eqe_size) / PAGE_SIZE;
956
957         eq->page_list = kmalloc(npages * sizeof *eq->page_list,
958                                 MEM_WAIT);
959         if (!eq->page_list)
960                 goto err_out;
961
962         for (i = 0; i < npages; ++i)
963                 eq->page_list[i].buf = NULL;
964
965         dma_list = kmalloc(npages * sizeof *dma_list, MEM_WAIT);
966         if (!dma_list)
967                 goto err_out_free;
968
969         mailbox = mlx4_alloc_cmd_mailbox(dev);
970         if (IS_ERR(mailbox))
971                 goto err_out_free;
972         eq_context = mailbox->buf;
973
974         for (i = 0; i < npages; ++i) {
975                 eq->page_list[i].buf = dma_alloc_coherent(&dev->persist->
976                                                           pdev->dev,
977                                                           PAGE_SIZE, &t,
978                                                           MEM_WAIT);
979                 if (!eq->page_list[i].buf)
980                         goto err_out_free_pages;
981
982                 dma_list[i] = t;
983                 eq->page_list[i].map = t;
984
985                 memset(eq->page_list[i].buf, 0, PAGE_SIZE);
986         }
987
988         eq->eqn = mlx4_bitmap_alloc(&priv->eq_table.bitmap);
989         if (eq->eqn == -1)
990                 goto err_out_free_pages;
991
992         eq->doorbell = mlx4_get_eq_uar(dev, eq);
993         if (!eq->doorbell) {
994                 err = -ENOMEM;
995                 goto err_out_free_eq;
996         }
997
998         err = mlx4_mtt_init(dev, npages, PAGE_SHIFT, &eq->mtt);
999         if (err)
1000                 goto err_out_free_eq;
1001
1002         err = mlx4_write_mtt(dev, &eq->mtt, 0, npages, dma_list);
1003         if (err)
1004                 goto err_out_free_mtt;
1005
1006         eq_context->flags         = cpu_to_be32(MLX4_EQ_STATUS_OK   |
1007                                                 MLX4_EQ_STATE_ARMED);
1008         eq_context->log_eq_size   = LOG2_UP(eq->nent);
1009         eq_context->intr          = intr;
1010         eq_context->log_page_size = PAGE_SHIFT - MLX4_ICM_PAGE_SHIFT;
1011
1012         mtt_addr = mlx4_mtt_addr(dev, &eq->mtt);
1013         eq_context->mtt_base_addr_h = mtt_addr >> 32;
1014         eq_context->mtt_base_addr_l = cpu_to_be32(mtt_addr & 0xffffffff);
1015
1016         err = mlx4_SW2HW_EQ(dev, mailbox, eq->eqn);
1017         if (err) {
1018                 mlx4_warn(dev, "SW2HW_EQ failed (%d)\n", err);
1019                 goto err_out_free_mtt;
1020         }
1021
1022         kfree(dma_list);
1023         mlx4_free_cmd_mailbox(dev, mailbox);
1024
1025         eq->cons_index = 0;
1026
1027         INIT_LIST_HEAD(&eq->tasklet_ctx.list);
1028         INIT_LIST_HEAD(&eq->tasklet_ctx.process_list);
1029         spinlock_init_irqsave(&eq->tasklet_ctx.lock);
1030         tasklet_init(&eq->tasklet_ctx.task, mlx4_cq_tasklet_cb,
1031                      (unsigned long)&eq->tasklet_ctx);
1032
1033         return err;
1034
1035 err_out_free_mtt:
1036         mlx4_mtt_cleanup(dev, &eq->mtt);
1037
1038 err_out_free_eq:
1039         mlx4_bitmap_free(&priv->eq_table.bitmap, eq->eqn, MLX4_USE_RR);
1040
1041 err_out_free_pages:
1042         for (i = 0; i < npages; ++i)
1043                 if (eq->page_list[i].buf)
1044                         dma_free_coherent(&dev->persist->pdev->dev, PAGE_SIZE,
1045                                           eq->page_list[i].buf,
1046                                           eq->page_list[i].map);
1047
1048         mlx4_free_cmd_mailbox(dev, mailbox);
1049
1050 err_out_free:
1051         kfree(eq->page_list);
1052         kfree(dma_list);
1053
1054 err_out:
1055         return err;
1056 }
1057
1058 static void mlx4_free_eq(struct mlx4_dev *dev,
1059                          struct mlx4_eq *eq)
1060 {
1061         struct mlx4_priv *priv = mlx4_priv(dev);
1062         int err;
1063         int i;
1064         /* CX3 is capable of extending the CQE/EQE from 32 to 64 bytes, with
1065          * strides of 64B,128B and 256B
1066          */
1067         int npages = PAGE_ALIGN(dev->caps.eqe_size  * eq->nent) / PAGE_SIZE;
1068
1069         err = mlx4_HW2SW_EQ(dev, eq->eqn);
1070         if (err)
1071                 mlx4_warn(dev, "HW2SW_EQ failed (%d)\n", err);
1072
1073         synchronize_irq(eq->irq);
1074         tasklet_disable(&eq->tasklet_ctx.task);
1075
1076         mlx4_mtt_cleanup(dev, &eq->mtt);
1077         for (i = 0; i < npages; ++i)
1078                 dma_free_coherent(&dev->persist->pdev->dev, PAGE_SIZE,
1079                                   eq->page_list[i].buf,
1080                                   eq->page_list[i].map);
1081
1082         kfree(eq->page_list);
1083         mlx4_bitmap_free(&priv->eq_table.bitmap, eq->eqn, MLX4_USE_RR);
1084 }
1085
1086 static void mlx4_free_irqs(struct mlx4_dev *dev)
1087 {
1088         panic("Not implemented");
1089 #if 0 // AKAROS_PORT
1090         struct mlx4_eq_table *eq_table = &mlx4_priv(dev)->eq_table;
1091         struct mlx4_priv *priv = mlx4_priv(dev);
1092         int     i, vec;
1093
1094         if (eq_table->have_irq)
1095                 free_irq(dev->persist->pdev->irq, dev);
1096
1097         for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i)
1098                 if (eq_table->eq[i].have_irq) {
1099                         free_irq(eq_table->eq[i].irq, eq_table->eq + i);
1100                         eq_table->eq[i].have_irq = 0;
1101                 }
1102
1103         for (i = 0; i < dev->caps.comp_pool; i++) {
1104                 /*
1105                  * Freeing the assigned irq's
1106                  * all bits should be 0, but we need to validate
1107                  */
1108                 if (priv->msix_ctl.pool_bm & 1ULL << i) {
1109                         /* NO need protecting*/
1110                         vec = dev->caps.num_comp_vectors + 1 + i;
1111                         free_irq(priv->eq_table.eq[vec].irq,
1112                                  &priv->eq_table.eq[vec]);
1113                 }
1114         }
1115
1116
1117         kfree(eq_table->irq_names);
1118 #endif
1119 }
1120
1121 static int mlx4_map_clr_int(struct mlx4_dev *dev)
1122 {
1123         struct mlx4_priv *priv = mlx4_priv(dev);
1124
1125         priv->clr_base = ioremap(pci_resource_start(dev->persist->pdev,
1126                                  priv->fw.clr_int_bar) +
1127                                  priv->fw.clr_int_base, MLX4_CLR_INT_SIZE);
1128         if (!priv->clr_base) {
1129                 mlx4_err(dev, "Couldn't map interrupt clear register, aborting\n");
1130                 return -ENOMEM;
1131         }
1132
1133         return 0;
1134 }
1135
1136 static void mlx4_unmap_clr_int(struct mlx4_dev *dev)
1137 {
1138         struct mlx4_priv *priv = mlx4_priv(dev);
1139
1140         iounmap(priv->clr_base);
1141 }
1142
1143 int mlx4_alloc_eq_table(struct mlx4_dev *dev)
1144 {
1145         struct mlx4_priv *priv = mlx4_priv(dev);
1146
1147         priv->eq_table.eq = kzmalloc((dev->caps.num_eqs - dev->caps.reserved_eqs) * (sizeof *priv->eq_table.eq),
1148                                      MEM_WAIT);
1149         if (!priv->eq_table.eq)
1150                 return -ENOMEM;
1151
1152         return 0;
1153 }
1154
1155 void mlx4_free_eq_table(struct mlx4_dev *dev)
1156 {
1157         kfree(mlx4_priv(dev)->eq_table.eq);
1158 }
1159
1160 int mlx4_init_eq_table(struct mlx4_dev *dev)
1161 {
1162         struct mlx4_priv *priv = mlx4_priv(dev);
1163         int err;
1164         int i;
1165
1166         priv->eq_table.uar_map = kzmalloc((mlx4_num_eq_uar(dev)) * (sizeof *priv->eq_table.uar_map),
1167                                           MEM_WAIT);
1168         if (!priv->eq_table.uar_map) {
1169                 err = -ENOMEM;
1170                 goto err_out_free;
1171         }
1172
1173         err = mlx4_bitmap_init(&priv->eq_table.bitmap,
1174                                ROUNDUPPWR2(dev->caps.num_eqs),
1175                                dev->caps.num_eqs - 1,
1176                                dev->caps.reserved_eqs,
1177                                ROUNDUPPWR2(dev->caps.num_eqs) -
1178                                dev->caps.num_eqs);
1179         if (err)
1180                 goto err_out_free;
1181
1182         for (i = 0; i < mlx4_num_eq_uar(dev); ++i)
1183                 priv->eq_table.uar_map[i] = NULL;
1184
1185         if (!mlx4_is_slave(dev)) {
1186                 err = mlx4_map_clr_int(dev);
1187                 if (err)
1188                         goto err_out_bitmap;
1189
1190                 priv->eq_table.clr_mask =
1191                         swab32(1 << (priv->eq_table.inta_pin & 31));
1192                 priv->eq_table.clr_int  = priv->clr_base +
1193                         (priv->eq_table.inta_pin < 32 ? 4 : 0);
1194         }
1195
1196         priv->eq_table.irq_names =
1197                 kmalloc(MLX4_IRQNAME_SIZE * (dev->caps.num_comp_vectors + 1 +
1198                                              dev->caps.comp_pool),
1199                         MEM_WAIT);
1200         if (!priv->eq_table.irq_names) {
1201                 err = -ENOMEM;
1202                 goto err_out_bitmap;
1203         }
1204
1205         for (i = 0; i < dev->caps.num_comp_vectors; ++i) {
1206                 err = mlx4_create_eq(dev, dev->caps.num_cqs -
1207                                           dev->caps.reserved_cqs +
1208                                           MLX4_NUM_SPARE_EQE,
1209                                      (dev->flags & MLX4_FLAG_MSI_X) ? i : 0,
1210                                      &priv->eq_table.eq[i]);
1211                 if (err) {
1212                         --i;
1213                         goto err_out_unmap;
1214                 }
1215         }
1216
1217         err = mlx4_create_eq(dev, MLX4_NUM_ASYNC_EQE + MLX4_NUM_SPARE_EQE,
1218                              (dev->flags & MLX4_FLAG_MSI_X) ? dev->caps.num_comp_vectors : 0,
1219                              &priv->eq_table.eq[dev->caps.num_comp_vectors]);
1220         if (err)
1221                 goto err_out_comp;
1222
1223         /*if additional completion vectors poolsize is 0 this loop will not run*/
1224         for (i = dev->caps.num_comp_vectors + 1;
1225               i < dev->caps.num_comp_vectors + dev->caps.comp_pool + 1; ++i) {
1226
1227                 err = mlx4_create_eq(dev, dev->caps.num_cqs -
1228                                           dev->caps.reserved_cqs +
1229                                           MLX4_NUM_SPARE_EQE,
1230                                      (dev->flags & MLX4_FLAG_MSI_X) ? i : 0,
1231                                      &priv->eq_table.eq[i]);
1232                 if (err) {
1233                         --i;
1234                         goto err_out_unmap;
1235                 }
1236         }
1237
1238
1239         if (dev->flags & MLX4_FLAG_MSI_X) {
1240                 const char *eq_name;
1241
1242                 for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i) {
1243                         if (i < dev->caps.num_comp_vectors) {
1244                                 snprintf(priv->eq_table.irq_names +
1245                                          i * MLX4_IRQNAME_SIZE,
1246                                          MLX4_IRQNAME_SIZE,
1247                                          "mlx4-comp-%d@pci:%s", i,
1248                                          pci_name(dev->persist->pdev));
1249                         } else {
1250                                 snprintf(priv->eq_table.irq_names +
1251                                          i * MLX4_IRQNAME_SIZE,
1252                                          MLX4_IRQNAME_SIZE,
1253                                          "mlx4-async@pci:%s",
1254                                          pci_name(dev->persist->pdev));
1255                         }
1256
1257                         eq_name = priv->eq_table.irq_names +
1258                                   i * MLX4_IRQNAME_SIZE;
1259 #if 0 // AKAROS_PORT
1260                         err = register_irq(priv->eq_table.eq[i].irq,
1261                                            mlx4_msi_x_interrupt,
1262                                            priv->eq_table.eq + i,
1263                                            pci_to_tbdf(PCIDEV));
1264 #else
1265                         err = register_irq(priv->eq_table.eq[i].irq,
1266                                            mlx4_msi_x_interrupt_akaros,
1267                                            priv->eq_table.eq + i,
1268                                            pci_to_tbdf(dev->persist->pdev));
1269 #endif
1270                         if (err)
1271                                 goto err_out_async;
1272
1273                         priv->eq_table.eq[i].have_irq = 1;
1274                 }
1275         } else {
1276                 panic("Disabled");
1277 #if 0 // AKAROS_PORT
1278                 snprintf(priv->eq_table.irq_names,
1279                          MLX4_IRQNAME_SIZE,
1280                          DRV_NAME "@pci:%s",
1281                          pci_name(dev->persist->pdev));
1282                 err = register_irq(dev->persist->pdev->irq, mlx4_interrupt,
1283                                    dev, pci_to_tbdf(PCIDEV));
1284                 if (err)
1285                         goto err_out_async;
1286
1287                 priv->eq_table.have_irq = 1;
1288 #endif
1289         }
1290
1291         err = mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 0,
1292                           priv->eq_table.eq[dev->caps.num_comp_vectors].eqn);
1293         if (err)
1294                 mlx4_warn(dev, "MAP_EQ for async EQ %d failed (%d)\n",
1295                            priv->eq_table.eq[dev->caps.num_comp_vectors].eqn, err);
1296
1297         for (i = 0; i < dev->caps.num_comp_vectors + 1; ++i)
1298                 eq_set_ci(&priv->eq_table.eq[i], 1);
1299
1300         return 0;
1301
1302 err_out_async:
1303         mlx4_free_eq(dev, &priv->eq_table.eq[dev->caps.num_comp_vectors]);
1304
1305 err_out_comp:
1306         i = dev->caps.num_comp_vectors - 1;
1307
1308 err_out_unmap:
1309         while (i >= 0) {
1310                 mlx4_free_eq(dev, &priv->eq_table.eq[i]);
1311                 --i;
1312         }
1313         if (!mlx4_is_slave(dev))
1314                 mlx4_unmap_clr_int(dev);
1315         mlx4_free_irqs(dev);
1316
1317 err_out_bitmap:
1318         mlx4_unmap_uar(dev);
1319         mlx4_bitmap_cleanup(&priv->eq_table.bitmap);
1320
1321 err_out_free:
1322         kfree(priv->eq_table.uar_map);
1323
1324         return err;
1325 }
1326
1327 void mlx4_cleanup_eq_table(struct mlx4_dev *dev)
1328 {
1329         struct mlx4_priv *priv = mlx4_priv(dev);
1330         int i;
1331
1332         mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 1,
1333                     priv->eq_table.eq[dev->caps.num_comp_vectors].eqn);
1334
1335         mlx4_free_irqs(dev);
1336
1337         for (i = 0; i < dev->caps.num_comp_vectors + dev->caps.comp_pool + 1; ++i)
1338                 mlx4_free_eq(dev, &priv->eq_table.eq[i]);
1339
1340         if (!mlx4_is_slave(dev))
1341                 mlx4_unmap_clr_int(dev);
1342
1343         mlx4_unmap_uar(dev);
1344         mlx4_bitmap_cleanup(&priv->eq_table.bitmap);
1345
1346         kfree(priv->eq_table.uar_map);
1347 }
1348
1349 /* A test that verifies that we can accept interrupts on all
1350  * the irq vectors of the device.
1351  * Interrupts are checked using the NOP command.
1352  */
1353 int mlx4_test_interrupts(struct mlx4_dev *dev)
1354 {
1355         struct mlx4_priv *priv = mlx4_priv(dev);
1356         int i;
1357         int err;
1358
1359         err = mlx4_NOP(dev);
1360         /* When not in MSI_X, there is only one irq to check */
1361         if (!(dev->flags & MLX4_FLAG_MSI_X) || mlx4_is_slave(dev))
1362                 return err;
1363
1364         /* A loop over all completion vectors, for each vector we will check
1365          * whether it works by mapping command completions to that vector
1366          * and performing a NOP command
1367          */
1368         for(i = 0; !err && (i < dev->caps.num_comp_vectors); ++i) {
1369                 /* Temporary use polling for command completions */
1370                 mlx4_cmd_use_polling(dev);
1371
1372                 /* Map the new eq to handle all asynchronous events */
1373                 err = mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 0,
1374                                   priv->eq_table.eq[i].eqn);
1375                 if (err) {
1376                         mlx4_warn(dev, "Failed mapping eq for interrupt test\n");
1377                         mlx4_cmd_use_events(dev);
1378                         break;
1379                 }
1380
1381                 /* Go back to using events */
1382                 mlx4_cmd_use_events(dev);
1383                 err = mlx4_NOP(dev);
1384         }
1385
1386         /* Return to default */
1387         mlx4_MAP_EQ(dev, get_async_ev_mask(dev), 0,
1388                     priv->eq_table.eq[dev->caps.num_comp_vectors].eqn);
1389         return err;
1390 }
1391 EXPORT_SYMBOL(mlx4_test_interrupts);
1392
1393 int mlx4_assign_eq(struct mlx4_dev *dev, char *name, struct cpu_rmap *rmap,
1394                    int *vector)
1395 {
1396         panic("Disabled");
1397 #if 0 // AKAROS_PORT
1398
1399         struct mlx4_priv *priv = mlx4_priv(dev);
1400         int vec = 0, err = 0, i;
1401
1402         qlock(&priv->msix_ctl.pool_lock);
1403         for (i = 0; !vec && i < dev->caps.comp_pool; i++) {
1404                 if (~priv->msix_ctl.pool_bm & 1ULL << i) {
1405                         priv->msix_ctl.pool_bm |= 1ULL << i;
1406                         vec = dev->caps.num_comp_vectors + 1 + i;
1407                         snprintf(priv->eq_table.irq_names +
1408                                         vec * MLX4_IRQNAME_SIZE,
1409                                         MLX4_IRQNAME_SIZE, "%s", name);
1410 #ifdef CONFIG_RFS_ACCEL
1411                         if (rmap) {
1412                                 err = irq_cpu_rmap_add(rmap,
1413                                                        priv->eq_table.eq[vec].irq);
1414                                 if (err)
1415                                         mlx4_warn(dev, "Failed adding irq rmap\n");
1416                         }
1417 #endif
1418                         err = register_irq(priv->eq_table.eq[vec].irq,
1419                                            mlx4_msi_x_interrupt,
1420                                            priv->eq_table.eq + vec,
1421                                            pci_to_tbdf(PCIDEV));
1422                         if (err) {
1423                                 /*zero out bit by fliping it*/
1424                                 priv->msix_ctl.pool_bm ^= 1 << i;
1425                                 vec = 0;
1426                                 continue;
1427                                 /*we dont want to break here*/
1428                         }
1429
1430                         eq_set_ci(&priv->eq_table.eq[vec], 1);
1431                 }
1432         }
1433         qunlock(&priv->msix_ctl.pool_lock);
1434
1435         if (vec) {
1436                 *vector = vec;
1437         } else {
1438                 *vector = 0;
1439                 err = (i == dev->caps.comp_pool) ? -ENOSPC : err;
1440         }
1441         return err;
1442 #endif
1443 }
1444 EXPORT_SYMBOL(mlx4_assign_eq);
1445
1446 int mlx4_eq_get_irq(struct mlx4_dev *dev, int vec)
1447 {
1448         panic("Disabled");
1449 #if 0 // AKAROS_PORT
1450         struct mlx4_priv *priv = mlx4_priv(dev);
1451
1452         return priv->eq_table.eq[vec].irq;
1453 #endif
1454 }
1455 EXPORT_SYMBOL(mlx4_eq_get_irq);
1456
1457 void mlx4_release_eq(struct mlx4_dev *dev, int vec)
1458 {
1459         panic("Disabled");
1460 #if 0 // AKAROS_PORT
1461         struct mlx4_priv *priv = mlx4_priv(dev);
1462         /*bm index*/
1463         int i = vec - dev->caps.num_comp_vectors - 1;
1464
1465         if (likely(i >= 0)) {
1466                 /*sanity check , making sure were not trying to free irq's
1467                   Belonging to a legacy EQ*/
1468                 qlock(&priv->msix_ctl.pool_lock);
1469                 if (priv->msix_ctl.pool_bm & 1ULL << i) {
1470                         free_irq(priv->eq_table.eq[vec].irq,
1471                                  &priv->eq_table.eq[vec]);
1472                         priv->msix_ctl.pool_bm &= ~(1ULL << i);
1473                 }
1474                 qunlock(&priv->msix_ctl.pool_lock);
1475         }
1476 #endif
1477 }
1478 EXPORT_SYMBOL(mlx4_release_eq);
1479