1IBV_WR API(3)           Libibverbs Programmer’s Manual           IBV_WR API(3)
2
3
4

NAME

6       ibv_wr_abort, ibv_wr_complete, ibv_wr_start - Manage regions allowed to
7       post work
8
9       ibv_wr_atomic_cmp_swp, ibv_wr_atomic_fetch_add - Post remote atomic op‐
10       eration work requests
11
12       ibv_wr_bind_mw,  ibv_wr_local_inv  - Post work requests for memory win‐
13       dows
14
15       ibv_wr_rdma_read, ibv_wr_rdma_write, ibv_wr_rdma_write_imm - Post  RDMA
16       work requests
17
18       ibv_wr_send, ibv_wr_send_imm, ibv_wr_send_inv - Post send work requests
19
20       ibv_wr_send_tso - Post segmentation offload work requests
21
22       ibv_wr_set_inline_data, ibv_wr_set_inline_data_list - Attach inline da‐
23       ta to the last work request
24
25       ibv_wr_set_sge, ibv_wr_set_sge_list - Attach data to the last work  re‐
26       quest
27
28       ibv_wr_set_ud_addr - Attach UD addressing info to the last work request
29
30       ibv_wr_set_xrc_srqn - Attach an XRC SRQN to the last work request
31

SYNOPSIS

33              #include <infiniband/verbs.h>
34
35              void ibv_wr_abort(struct ibv_qp_ex *qp);
36              int ibv_wr_complete(struct ibv_qp_ex *qp);
37              void ibv_wr_start(struct ibv_qp_ex *qp);
38
39              void ibv_wr_atomic_cmp_swp(struct ibv_qp_ex *qp, uint32_t rkey,
40                                         uint64_t remote_addr, uint64_t compare,
41                                         uint64_t swap);
42              void ibv_wr_atomic_fetch_add(struct ibv_qp_ex *qp, uint32_t rkey,
43                                           uint64_t remote_addr, uint64_t add);
44
45              void ibv_wr_bind_mw(struct ibv_qp_ex *qp, struct ibv_mw *mw, uint32_t rkey,
46                                  const struct ibv_mw_bind_info *bind_info);
47              void ibv_wr_local_inv(struct ibv_qp_ex *qp, uint32_t invalidate_rkey);
48
49              void ibv_wr_rdma_read(struct ibv_qp_ex *qp, uint32_t rkey,
50                                    uint64_t remote_addr);
51              void ibv_wr_rdma_write(struct ibv_qp_ex *qp, uint32_t rkey,
52                                     uint64_t remote_addr);
53              void ibv_wr_rdma_write_imm(struct ibv_qp_ex *qp, uint32_t rkey,
54                                         uint64_t remote_addr, __be32 imm_data);
55
56              void ibv_wr_send(struct ibv_qp_ex *qp);
57              void ibv_wr_send_imm(struct ibv_qp_ex *qp, __be32 imm_data);
58              void ibv_wr_send_inv(struct ibv_qp_ex *qp, uint32_t invalidate_rkey);
59              void ibv_wr_send_tso(struct ibv_qp_ex *qp, void *hdr, uint16_t hdr_sz,
60                                   uint16_t mss);
61
62              void ibv_wr_set_inline_data(struct ibv_qp_ex *qp, void *addr, size_t length);
63              void ibv_wr_set_inline_data_list(struct ibv_qp_ex *qp, size_t num_buf,
64                                               const struct ibv_data_buf *buf_list);
65              void ibv_wr_set_sge(struct ibv_qp_ex *qp, uint32_t lkey, uint64_t addr,
66                                  uint32_t length);
67              void ibv_wr_set_sge_list(struct ibv_qp_ex *qp, size_t num_sge,
68                                       const struct ibv_sge *sg_list);
69
70              void ibv_wr_set_ud_addr(struct ibv_qp_ex *qp, struct ibv_ah *ah,
71                                      uint32_t remote_qpn, uint32_t remote_qkey);
72              void ibv_wr_set_xrc_srqn(struct ibv_qp_ex *qp, uint32_t remote_srqn);
73

DESCRIPTION

75       The  verbs work request API (ibv_wr_*) allows efficient posting of work
76       to a send queue using  function  calls  instead  of  the  struct  based
77       ibv_post_send()  scheme.   This  approach  is  designed to minimize CPU
78       branching and locking during the posting process.
79
80       This API is intended to be used to access additional functionality  be‐
81       yond what is provided by ibv_post_send().
82
83       WRs  batches of ibv_post_send() and this API WRs batches can interleave
84       together just if they are not posted within the critical region of each
85       other.   (A  critical  region  in this API formed by ibv_wr_start() and
86       ibv_wr_complete()/ibv_wr_abort())
87

USAGE

89       To use these APIs the QP must be created using ibv_create_qp_ex() which
90       allows  setting  the IBV_QP_INIT_ATTR_SEND_OPS_FLAGS in comp_mask.  The
91       send_ops_flags should be set to the OR of the work request  types  that
92       will be posted to the QP.
93
94       If the QP does not support all the requested work request types then QP
95       creation will fail.
96
97       Posting work requests to the QP is  done  within  the  critical  region
98       formed by ibv_wr_start() and ibv_wr_complete()/ibv_wr_abort() (see CON‐
99       CURRENCY below).
100
101       Each work request is created by calling a WR builder function (see  the
102       table column WR builder below) to start creating the work request, fol‐
103       lowed by allowed/required setter functions described below.
104
105       The WR builder and setter combination can be called multiple  times  to
106       efficiently  post  multiple  work requests within a single critical re‐
107       gion.
108
109       Each WR builder will use the wr_id member of struct  ibv_qp_ex  to  set
110       the  value to be returned in the completion.  Some operations will also
111       use the wr_flags member  to  influence  operation  (see  Flags  below).
112       These values should be set before invoking the WR builder function.
113
114       For example a simple send could be formed as follows:
115
116              qpx->wr_id = 1;
117              ibv_wr_send(qpx);
118              ibv_wr_set_sge(qpx, lkey, &data, sizeof(data));
119
120       The section WORK REQUESTS describes the various WR builders and setters
121       in details.
122
123       Posting   work   is   completed   by   calling   ibv_wr_complete()   or
124       ibv_wr_abort().   No  work  is  executed to the queue until ibv_wr_com‐
125       plete() returns success.  ibv_wr_abort() will discard all work prepared
126       since ibv_wr_start().
127

WORK REQUESTS

129       Many of the operations match the opcodes available for ibv_post_send().
130       Each operation has a WR builder function, a list  of  allowed  setters,
131       and  a  flag bit to request the operation with send_ops_flags in struct
132       ibv_qp_init_attr_ex (see the EXAMPLE below).
133
134       Operation           WR builder          QP Type Support‐   setters
135                                               ed
136       ────────────────────────────────────────────────────────────────────
137       ATOM‐               ibv_wr_atom‐        RC, XRC_SEND       DATA, QP
138       IC_CMP_AND_SWP      ic_cmp_swp()
139       ATOM‐               ibv_wr_atom‐        RC, XRC_SEND       DATA, QP
140       IC_FETCH_AND_ADD    ic_fetch_add()
141       BIND_MW             ibv_wr_bind_mw()    UC, RC, XRC_SEND   NONE
142       LOCAL_INV           ibv_wr_lo‐          UC, RC, XRC_SEND   NONE
143                           cal_inv()
144       RDMA_READ           ibv_wr_rd‐          RC, XRC_SEND       DATA, QP
145                           ma_read()
146       RDMA_WRITE          ibv_wr_rd‐          UC, RC, XRC_SEND   DATA, QP
147                           ma_write()
148       RD‐                 ibv_wr_rd‐          UC, RC, XRC_SEND   DATA, QP
149       MA_WRITE_WITH_IMM   ma_write_imm()
150       SEND                ibv_wr_send()       UD,    UC,   RC,   DATA, QP
151                                               XRC_SEND,
152                                               RAW_PACKET
153       SEND_WITH_IMM       ibv_wr_send_imm()   UD, UC, RC,  SRC   DATA, QP
154                                               SEND
155       SEND_WITH_INV       ibv_wr_send_inv()   UC, RC, XRC_SEND   DATA, QP
156       TSO                 ibv_wr_send_tso()   UD, RAW_PACKET     DATA, QP
157
158   Atomic operations
159       Atomic  operations  are  only atomic so long as all writes to memory go
160       only through the same RDMA hardware.  It is not atomic with writes per‐
161       formed by the CPU, or by other RDMA hardware in the system.
162
163       ibv_wr_atomic_cmp_swp()
164              If  the  remote 64 bit memory location specified by rkey and re‐
165              mote_addr equals compare then set it to swap.
166
167       ibv_wr_atomic_fetch_add()
168              Add add to the 64 bit memory location  specified  rkey  and  re‐
169              mote_addr.
170
171   Memory Windows
172       Memory window type 2 operations (See man page for ibv_alloc_mw).
173
174       ibv_wr_bind_mw()
175              Bind  a  MW  type  2 specified by mw, set a new rkey and set its
176              properties by bind_info.
177
178       ibv_wr_local_inv()
179              Invalidate a MW type 2 which is associated with rkey.
180
181   RDMA
182       ibv_wr_rdma_read()
183              Read from the remote memory  location  specified  rkey  and  re‐
184              mote_addr.   The number of bytes to read, and the local location
185              to store the data, is determined by the DATA buffers  set  after
186              this call.
187
188       ibv_wr_rdma_write(), ibv_wr_rdma_write_imm()
189              Write  to  the  remote  memory  location  specified rkey and re‐
190              mote_addr.  The number of bytes to read, and the local  location
191              to  get  the  data,  is determined by the DATA buffers set after
192              this call.
193
194              The _imm version causes the remote side to get a IBV_WC_RECV_RD‐
195              MA_WITH_IMM containing the 32 bits of immediate data.
196
197   Message Send
198       ibv_wr_send(), ibv_wr_send_imm()
199              Send  a message.  The number of bytes to send, and the local lo‐
200              cation to get the data, is determined by the  DATA  buffers  set
201              after this call.
202
203              The _imm version causes the remote side to get a IBV_WC_RECV_RD‐
204              MA_WITH_IMM containing the 32 bits of immediate data.
205
206       ibv_wr_send_inv()
207              The data transfer is the same as for ibv_wr_send(), however  the
208              remote  side will invalidate the MR specified by invalidate_rkey
209              before delivering a completion.
210
211       ibv_wr_send_tso()
212              Produce multiple SEND messages using TCP  Segmentation  Offload.
213              The  SGE  points  to a TCP Stream buffer which will be segmented
214              into MSS size SENDs.  The hdr includes the entire network  head‐
215              ers  up  to  and including the TCP header and is prefixed before
216              each segment.
217
218   QP Specific setters
219       Certain QP types require each post to be accompanied by additional set‐
220       ters, these setters are mandatory for any operation listing a QP setter
221       in the above table.
222
223       UD QPs ibv_wr_set_ud_addr() must be called to set the  destination  ad‐
224              dress of the work.
225
226       XRC_SEND QPs
227              ibv_wr_set_xrc_srqn() must be called to set the destination SRQN
228              field.
229
230   DATA transfer setters
231       For work that requires to transfer data one of  the  following  setters
232       should be called once after the WR builder:
233
234       ibv_wr_set_sge()
235              Transfer  data  to/from  a single buffer given by the lkey, addr
236              and length.  This is equivalent to ibv_wr_set_sge_list() with  a
237              single element.
238
239       ibv_wr_set_sge_list()
240              Transfer  data to/from a list of buffers, logically concatenated
241              together.  Each buffer is specified by an element in an array of
242              struct ibv_sge.
243
244       Inline setters will copy the send data during the setter and allows the
245       caller to immediately re-use the buffer.  This behavior is identical to
246       the  IBV_SEND_INLINE  flag.   Generally this copy is done in a way that
247       optimizes SEND  latency  and  is  suitable  for  small  messages.   The
248       provider will limit the amount of data it can support in a single oper‐
249       ation.  This limit is requested in the max_inline_data member of struct
250       ibv_qp_init_attr.  Valid only for SEND and RDMA_WRITE.
251
252       ibv_wr_set_inline_data()
253              Copy  send  data  from  a  single  buffer  given by the addr and
254              length.  This  is  equivalent  to  ibv_wr_set_inline_data_list()
255              with a single element.
256
257       ibv_wr_set_inline_data_list()
258              Copy  send  data  from a list of buffers, logically concatenated
259              together.  Each buffer is specified by an element in an array of
260              struct ibv_inl_data.
261
262   Flags
263       A  bit mask of flags may be specified in wr_flags to control the behav‐
264       ior of the work request.
265
266       IBV_SEND_FENCE
267              Do not start this work request until prior work has completed.
268
269       IBV_SEND_IP_CSUM
270              Offload the IPv4 and TCP/UDP checksum calculation
271
272       IBV_SEND_SIGNALED
273              A completion will be generated in the completion queue  for  the
274              operation.
275
276       IBV_SEND_SOLICITED
277              Set the solicited bit in the RDMA packet.  This informs the oth‐
278              er side to generate a completion event upon receiving  the  RDMA
279              operation.
280

CONCURRENCY

282       The  provider  will  provide  locking to ensure that ibv_wr_start() and
283       ibv_wr_complete()/abort() form a per-QP critical section where no other
284       threads can enter.
285
286       If  an  ibv_td  is  provided during QP creation then no locking will be
287       performed and it is up to the caller to ensure that only one thread can
288       be within the critical region at a time.
289

RETURN VALUE

291       Applications  should  use  this API in a way that does not create fail‐
292       ures.  The individual APIs do not return a failure indication to  avoid
293       branching.
294
295       If  a  failure is detected during operation, for instance due to an in‐
296       valid argument, then ibv_wr_complete() will return failure and the  en‐
297       tire posting will be aborted.
298

EXAMPLE

300              /* create RC QP type and specify the required send opcodes */
301              qp_init_attr_ex.qp_type = IBV_QPT_RC;
302              qp_init_attr_ex.comp_mask |= IBV_QP_INIT_ATTR_SEND_OPS_FLAGS;
303              qp_init_attr_ex.send_ops_flags |= IBV_QP_EX_WITH_RDMA_WRITE;
304              qp_init_attr_ex.send_ops_flags |= IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM;
305
306              ibv_qp *qp = ibv_create_qp_ex(ctx, qp_init_attr_ex);
307              ibv_qp_ex *qpx = ibv_qp_to_qp_ex(qp);
308
309              ibv_wr_start(qpx);
310
311              /* create 1st WRITE WR entry */
312              qpx->wr_id = my_wr_id_1;
313              ibv_wr_rdma_write(qpx, rkey, remote_addr_1);
314              ibv_wr_set_sge(qpx, lkey, local_addr_1, length_1);
315
316              /* create 2nd WRITE_WITH_IMM WR entry */
317              qpx->wr_id = my_wr_id_2;
318              qpx->wr_flags = IBV_SEND_SIGNALED;
319              ibv_wr_rdma_write_imm(qpx, rkey, remote_addr_2, htonl(0x1234));
320              ibv_set_wr_sge(qpx, lkey, local_addr_2, length_2);
321
322              /* Begin processing WRs */
323              ret = ibv_wr_complete(qpx);
324

SEE ALSO

326       ibv_post_send(3), ibv_create_qp_ex(3).
327

AUTHOR

329       Jason Gunthorpe <jgg@mellanox.com> Guy Levi <guyle@mellanox.com>
330
331
332
333libibverbs                        2018-11-27                     IBV_WR API(3)
Impressum