ceph/src/msg/async/rdma/ib_dbg.h

   1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
   2 // vim: ts=8 sw=2 smarttab
   3
   4 #ifndef __IB_DBG_H__
   5 #define __IB_DBG_H__
   6
   7 #define dout_subsys ceph_subsys_ms
   8 #undef dout_prefix
   9 #define dout_prefix *_dout << "ib_dbg "
  10
  11 #define ibdbg_ldout \
  12   if (g_ceph_context && g_ceph_context->_log) \
  13     ldout(g_ceph_context,1) << f << ":" << l << " ib command: " << __func__
  14
  15 static inline struct ibv_srq *_ibv_create_srq(const char *f, int l,
  16                                               struct ibv_pd *pd,
  17                                 struct ibv_srq_init_attr *srq_init_attr)
  18 {
  19   auto ret = ibv_create_srq(pd, srq_init_attr);
  20
  21   ibdbg_ldout << " pd: " << pd <<
  22     " | srq: " << ret << dendl;
  23
  24   return ret;
  25 }
  26 #define ibv_create_srq(...)           _ibv_create_srq(__func__, __LINE__, __VA_ARGS__)
  27
  28 static inline void _ibv_ack_cq_events(const char *f, int l,
  29                                       struct ibv_cq *cq, unsigned int nevents)
  30 {
  31   ibdbg_ldout << " cq: " << cq << " nevents: " << nevents << dendl;
  32
  33   ibv_ack_cq_events(cq, nevents);
  34 }
  35 #define ibv_ack_cq_events(...)        _ibv_ack_cq_events(__func__, __LINE__, __VA_ARGS__)
  36
  37 static inline struct ibv_pd *_ibv_alloc_pd(const char *f, int l,
  38                                           struct ibv_context *context)
  39 {
  40   auto ret = ibv_alloc_pd(context);
  41
  42
  43   ibdbg_ldout << " ctxt: " << context <<
  44     " | pd: " << ret << dendl;
  45
  46   return ret;
  47 }
  48 #define ibv_alloc_pd(...)             _ibv_alloc_pd(__func__, __LINE__, __VA_ARGS__)
  49
  50 static inline int _ibv_close_device(const char *f, int l,
  51                                    struct ibv_context *context)
  52 {
  53   auto ret = ibv_close_device(context);
  54
  55   ibdbg_ldout << " ctxt: " << context <<
  56     " | ret: " << ret << dendl;
  57
  58   return ret;
  59 }
  60 #define ibv_close_device(...)         _ibv_close_device(__func__, __LINE__, __VA_ARGS__)
  61
  62 static inline struct ibv_comp_channel *_ibv_create_comp_channel(const char *f, int l,
  63                                                                struct ibv_context *context)
  64 {
  65   auto ret = ibv_create_comp_channel(context);
  66
  67   ibdbg_ldout << " ctxt: " << context <<
  68     " | channel: " << ret << " fd: " << ( ret ? ret->fd : -1 ) << dendl;
  69
  70   return ret;
  71 }
  72 #define ibv_create_comp_channel(...)  _ibv_create_comp_channel(__func__, __LINE__, __VA_ARGS__)
  73
  74 static inline struct ibv_cq *_ibv_create_cq(const char *f, int l,
  75                                            struct ibv_context *context, int cqe,
  76                                            void *cq_context,
  77                                            struct ibv_comp_channel *channel,
  78                                            int comp_vector)
  79 {
  80   auto ret = ibv_create_cq(context, cqe, cq_context, channel, comp_vector);
  81
  82   ibdbg_ldout << " context: " << context << " channel: " << channel <<
  83     " | cq: " << ret << dendl;
  84
  85   return ret;
  86 }
  87 #define ibv_create_cq(...)            _ibv_create_cq(__func__, __LINE__, __VA_ARGS__)
  88
  89 static inline struct ibv_qp *_ibv_create_qp(const char *f, int l,
  90                                             struct ibv_pd *pd,
  91                                             struct ibv_qp_init_attr *qp_init_attr)
  92 {
  93   auto ret = ibv_create_qp(pd, qp_init_attr);
  94
  95   ibdbg_ldout << " pd: " << pd <<
  96     " send_cq: " << qp_init_attr->send_cq <<
  97     " recv_cq: " << qp_init_attr->recv_cq <<
  98     " srq: " << qp_init_attr->srq <<
  99     " | qp: " << ret << dendl;
 100
 101   return ret;
 102 }
 103 #define ibv_create_qp(...)            _ibv_create_qp(__func__, __LINE__, __VA_ARGS__)
 104
 105 static inline int _ibv_dealloc_pd(const char *f, int l,
 106                                   struct ibv_pd *pd)
 107 {
 108   auto ret = ibv_dealloc_pd(pd);
 109
 110   ibdbg_ldout << " pd: " << pd <<
 111     " | ret: " << ret << dendl;
 112
 113   return ret;
 114 }
 115 #define ibv_dealloc_pd(...)           _ibv_dealloc_pd(__func__, __LINE__, __VA_ARGS__)
 116
 117 static inline int _ibv_destroy_comp_channel(const char *f, int l,
 118                                             struct ibv_comp_channel *channel)
 119 {
 120   auto ret = ibv_destroy_comp_channel(channel);
 121
 122   ibdbg_ldout << " channel: " << channel <<
 123     " | ret: " << ret << dendl;
 124
 125   return ret;
 126 }
 127 #define ibv_destroy_comp_channel(...) _ibv_destroy_comp_channel(__func__, __LINE__, __VA_ARGS__)
 128
 129 static inline int _ibv_destroy_cq(const char *f, int l,
 130                                   struct ibv_cq *cq)
 131 {
 132   auto ret = ibv_destroy_cq(cq);
 133
 134   if (ret) {
 135     ibdbg_ldout << " cq: " << cq <<
 136       " | ret: " << cpp_strerror(ret) << dendl;
 137   } else {
 138     ibdbg_ldout << " cq: " << cq <<
 139       " | ret: " << ret << dendl;
 140   }
 141
 142   return ret;
 143 }
 144 #define ibv_destroy_cq(...)           _ibv_destroy_cq(__func__, __LINE__, __VA_ARGS__)
 145
 146 static inline int _ibv_destroy_qp(const char *f, int l,
 147                                   struct ibv_qp *qp)
 148 {
 149   auto ret = ibv_destroy_qp(qp);
 150
 151   ibdbg_ldout << " qp: " << qp <<
 152     " | ret: " << ret << dendl;
 153
 154   return ret;
 155 }
 156 #define ibv_destroy_qp(...)           _ibv_destroy_qp(__func__, __LINE__, __VA_ARGS__)
 157
 158 static inline int _ibv_destroy_srq(const char *f, int l,
 159                                    struct ibv_srq *srq)
 160 {
 161   auto ret = ibv_destroy_srq(srq);
 162
 163   ibdbg_ldout << " srq: " << srq <<
 164     " | ret: " << ret << dendl;
 165
 166   return ret;
 167 }
 168 #define ibv_destroy_srq(...)          _ibv_destroy_srq(__func__, __LINE__, __VA_ARGS__)
 169
 170 //#define ibv_exp_query_gid_attr(...)   _ibv_exp_query_gid_attr(__func__, __LINE__, __VA_ARGS__)
 171
 172 static inline int _ibv_fork_init(const char *f, int l)
 173 {
 174   auto ret = ibv_fork_init();
 175
 176   ibdbg_ldout <<
 177     " | ret: " << ret << dendl;
 178
 179   return ret;
 180 }
 181 #define ibv_fork_init()            _ibv_fork_init(__func__, __LINE__)
 182
 183 static inline void _ibv_free_device_list(const char *f, int l,
 184                                          struct ibv_device **list)
 185 {
 186   ibdbg_ldout << " list: " << list << dendl;
 187
 188   ibv_free_device_list(list);
 189 }
 190 #define ibv_free_device_list(...)     _ibv_free_device_list(__func__, __LINE__, __VA_ARGS__)
 191
 192 static inline int _ibv_get_cq_event(const char *f, int l,
 193                                     struct ibv_comp_channel *channel,
 194                                     struct ibv_cq **cq, void **cq_context)
 195 {
 196   auto ret = ibv_get_cq_event(channel, cq, cq_context);
 197
 198   ibdbg_ldout << " channel: " << channel << " cq: " << cq << " cq_context:" << cq_context <<
 199     " | ret: " << ret << dendl;
 200
 201   return ret;
 202 }
 203 #define ibv_get_cq_event(...)         _ibv_get_cq_event(__func__, __LINE__, __VA_ARGS__)
 204
 205 static inline struct ibv_device **_ibv_get_device_list(const char *f, int l,
 206                                                        int *num_devices)
 207 {
 208   auto ret = ibv_get_device_list(num_devices);
 209
 210   ibdbg_ldout <<
 211     " | num: " << *num_devices << " list: " << ret << dendl;
 212
 213   return ret;
 214 }
 215 #define ibv_get_device_list(...)      _ibv_get_device_list(__func__, __LINE__, __VA_ARGS__)
 216
 217 static inline const char *_ibv_get_device_name(const char *f, int l,
 218                                                struct ibv_device *device)
 219 {
 220   auto ret = ibv_get_device_name(device);
 221
 222   ibdbg_ldout << " device: " << device <<
 223     " | name: " << ret << dendl;
 224
 225   return ret;
 226 }
 227 #define ibv_get_device_name(...)      _ibv_get_device_name(__func__, __LINE__, __VA_ARGS__)
 228
 229 static inline int _ibv_modify_qp(const char *f, int l,
 230                                  struct ibv_qp *qp, struct ibv_qp_attr *attr,
 231                                  int attr_mask)
 232 {
 233   auto ret = ibv_modify_qp(qp, attr, attr_mask);
 234
 235   ibdbg_ldout << " qp: " << qp <<
 236     " | ret: " << ret << dendl;
 237
 238   return ret;
 239 }
 240 #define ibv_modify_qp(...)            _ibv_modify_qp(__func__, __LINE__, __VA_ARGS__)
 241
 242 static inline struct ibv_context *_ibv_open_device(const char *f, int l,
 243                                                    struct ibv_device *device)
 244 {
 245   auto ret = ibv_open_device(device);
 246
 247   ibdbg_ldout << " device: " << device <<
 248     " | ret: " << ret << dendl;
 249
 250   return ret;
 251 }
 252 #define ibv_open_device(...)          _ibv_open_device(__func__, __LINE__, __VA_ARGS__)
 253
 254 static inline int _ibv_poll_cq(const char *f, int l,
 255                                struct ibv_cq *cq, int num_entries, struct ibv_wc *wc)
 256 {
 257   auto ret = ibv_poll_cq(cq, num_entries, wc);
 258   static utime_t last_print = ceph_clock_now();
 259   static int suppressed_counter = 0;
 260
 261   if ((ceph_clock_now() - last_print).to_msec() < 100) {
 262     suppressed_counter++;
 263     return ret;
 264   }
 265
 266   if (suppressed_counter > 0) {
 267     ibdbg_ldout << " cq: " << cq << " num_entries: " << num_entries <<
 268       " | ret: " << ret <<
 269       " {suppressed " << suppressed_counter << " prints}" << dendl;
 270     suppressed_counter = 0;
 271   } else {
 272     ibdbg_ldout << " cq: " << cq << " num_entries: " << num_entries <<
 273       " | ret: " << ret << dendl;
 274   }
 275
 276   last_print = ceph_clock_now();
 277
 278   return ret;
 279 }
 280 #define ibv_poll_cq(...)              _ibv_poll_cq(__func__, __LINE__, __VA_ARGS__)
 281
 282 static inline int _ibv_post_send(const char *f, int l,
 283                                  struct ibv_qp *qp, struct ibv_send_wr *wr,
 284                                  struct ibv_send_wr **bad_wr)
 285 {
 286   auto ret = ibv_post_send(qp, wr, bad_wr);
 287
 288   ibdbg_ldout << " qp: " << qp <<
 289     " | ret: " << ret << dendl;
 290
 291   return ret;
 292 }
 293 #define ibv_post_send(...)            _ibv_post_send(__func__, __LINE__, __VA_ARGS__)
 294
 295 static inline int _ibv_post_srq_recv(const char *f, int l,
 296                                      struct ibv_srq *srq,
 297                                      struct ibv_recv_wr *recv_wr,
 298                                      struct ibv_recv_wr **bad_recv_wr)
 299 {
 300   auto ret = ibv_post_srq_recv(srq, recv_wr, bad_recv_wr);
 301
 302   static struct ibv_srq *last_srq = nullptr;
 303   if (last_srq != srq) {
 304     ibdbg_ldout << " srq: " << srq <<
 305       " | ret: " << ret << dendl;
 306     last_srq = srq;
 307   }
 308
 309   return ret;
 310 }
 311 #define ibv_post_srq_recv(...)        _ibv_post_srq_recv(__func__, __LINE__, __VA_ARGS__)
 312
 313 static inline int _ibv_query_device(const char *f, int l,
 314                                     struct ibv_context *context,
 315                                     struct ibv_device_attr *device_attr)
 316 {
 317   auto ret = ibv_query_device(context, device_attr);
 318
 319   ibdbg_ldout << " context: " << context <<
 320     " | ret: " << ret << dendl;
 321
 322   return ret;
 323 }
 324 #define ibv_query_device(...)         _ibv_query_device(__func__, __LINE__, __VA_ARGS__)
 325
 326 static inline int _ibv_query_gid(const char *f, int l,
 327                                  struct ibv_context *context, int port_num,
 328                                  int index, union ibv_gid *gid)
 329 {
 330   auto ret = ibv_query_gid(context, port_num, index, gid);
 331
 332   ibdbg_ldout << " context: " << context << " port_num: " << port_num << " index: " << index <<
 333     " | ret: " << ret << dendl;
 334
 335   return ret;
 336 }
 337 #define ibv_query_gid(...)            _ibv_query_gid(__func__, __LINE__, __VA_ARGS__)
 338
 339 static inline int ____ibv_query_port(const char *f, int l,
 340                                      struct ibv_context *context,
 341                                      int port_num,
 342                                      struct ibv_port_attr *port_attr)
 343 {
 344   auto ret = ___ibv_query_port(context, port_num, port_attr);
 345
 346   ibdbg_ldout << " context: " << context << " port_num: " << port_num <<
 347     " | ret: " << ret << dendl;
 348
 349   return ret;
 350 }
 351 #define ___ibv_query_port(...)        ____ibv_query_port(__func__, __LINE__, __VA_ARGS__)
 352
 353 static inline int _ibv_req_notify_cq(const char *f, int l,
 354                                      struct ibv_cq *cq, int solicited_only)
 355 {
 356   auto ret = ibv_req_notify_cq(cq, solicited_only);
 357
 358   ibdbg_ldout << " cq: " << cq << " solicited_only: " << solicited_only <<
 359     " | ret: " << ret << dendl;
 360
 361   return ret;
 362 }
 363 #define ibv_req_notify_cq(...)        _ibv_req_notify_cq(__func__, __LINE__, __VA_ARGS__)
 364
 365 static inline int _ibv_get_async_event(const char *f, int l,
 366                                        struct ibv_context *context,
 367                                        struct ibv_async_event *event)
 368 {
 369   auto ret = ibv_get_async_event(context, event);
 370
 371   ibdbg_ldout << " context: " << context <<
 372     " | event: " << event << " ret: " << ret << dendl;
 373
 374   return ret;
 375 }
 376 #define ibv_get_async_event(...)        _ibv_get_async_event(__func__, __LINE__, __VA_ARGS__)
 377
 378 static inline void _ibv_ack_async_event(const char *f, int l,
 379                                         struct ibv_async_event *event)
 380 {
 381   ibdbg_ldout << " event: " << event << dendl;
 382
 383   ibv_ack_async_event(event);
 384 }
 385 #define ibv_ack_async_event(...)        _ibv_ack_async_event(__func__, __LINE__, __VA_ARGS__)
 386
 387 #endif // __IB_DBG_H__