1fi_cq(3)                     Libfabric v1.12.0rc1                     fi_cq(3)
2
3
4

NAME

6       fi_cq - Completion queue operations
7
8       fi_cq_open / fi_close
9              Open/close a completion queue
10
11       fi_control
12              Control CQ operation or attributes.
13
14       fi_cq_read / fi_cq_readfrom / fi_cq_readerr
15              Read a completion from a completion queue
16
17       fi_cq_sread / fi_cq_sreadfrom
18              A  synchronous (blocking) read that waits until a specified con‐
19              dition has been met before reading a completion from  a  comple‐
20              tion queue.
21
22       fi_cq_signal
23              Unblock any thread waiting in fi_cq_sread or fi_cq_sreadfrom.
24
25       fi_cq_strerror
26              Converts  provider  specific  error information into a printable
27              string
28

SYNOPSIS

30              #include <rdma/fi_domain.h>
31
32              int fi_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
33                  struct fid_cq **cq, void *context);
34
35              int fi_close(struct fid *cq);
36
37              int fi_control(struct fid *cq, int command, void *arg);
38
39              ssize_t fi_cq_read(struct fid_cq *cq, void *buf, size_t count);
40
41              ssize_t fi_cq_readfrom(struct fid_cq *cq, void *buf, size_t count,
42                  fi_addr_t *src_addr);
43
44              ssize_t fi_cq_readerr(struct fid_cq *cq, struct fi_cq_err_entry *buf,
45                  uint64_t flags);
46
47              ssize_t fi_cq_sread(struct fid_cq *cq, void *buf, size_t count,
48                  const void *cond, int timeout);
49
50              ssize_t fi_cq_sreadfrom(struct fid_cq *cq, void *buf, size_t count,
51                  fi_addr_t *src_addr, const void *cond, int timeout);
52
53              int fi_cq_signal(struct fid_cq *cq);
54
55              const char * fi_cq_strerror(struct fid_cq *cq, int prov_errno,
56                    const void *err_data, char *buf, size_t len);
57

ARGUMENTS

59       domain Open resource domain
60
61       cq     Completion queue
62
63       attr   Completion queue attributes
64
65       context
66              User specified context associated with the completion queue.
67
68       buf    For read calls, the data buffer to write completions into.   For
69              write  calls,  a completion to insert into the completion queue.
70              For fi_cq_strerror, an optional buffer that  receives  printable
71              error information.
72
73       count  Number of CQ entries.
74
75       len    Length of data buffer
76
77       src_addr
78              Source address of a completed receive operation
79
80       flags  Additional flags to apply to the operation
81
82       command
83              Command of control operation to perform on CQ.
84
85       arg    Optional control argument
86
87       cond   Condition that must be met before a completion is generated
88
89       timeout
90              Time  in milliseconds to wait.  A negative value indicates infi‐
91              nite timeout.
92
93       prov_errno
94              Provider specific error value
95
96       err_data
97              Provider specific error data related to a completion
98

DESCRIPTION

100       Completion queues are used to report events associated with data trans‐
101       fers.   They are associated with message sends and receives, RMA, atom‐
102       ic, tagged messages, and triggered events.  Reported events are usually
103       associated with a fabric endpoint, but may also refer to memory regions
104       used as the target of an RMA or atomic operation.
105
106   fi_cq_open
107       fi_cq_open allocates a new completion queue.  Unlike event queues, com‐
108       pletion  queues  are  associated with a resource domain and may be off‐
109       loaded entirely in provider hardware.
110
111       The properties and behavior  of  a  completion  queue  are  defined  by
112       struct fi_cq_attr.
113
114              struct fi_cq_attr {
115                  size_t               size;      /* # entries for CQ */
116                  uint64_t             flags;     /* operation flags */
117                  enum fi_cq_format    format;    /* completion format */
118                  enum fi_wait_obj     wait_obj;  /* requested wait object */
119                  int                  signaling_vector; /* interrupt affinity */
120                  enum fi_cq_wait_cond wait_cond; /* wait condition format */
121                  struct fid_wait     *wait_set;  /* optional wait set */
122              };
123
124       size   Specifies  the minimum size of a completion queue.  A value of 0
125              indicates that the provider may choose a default value.
126
127       flags  Flags that control the configuration of the CQ.
128
129       - FI_AFFINITY
130              Indicates that the signaling_vector field (see below) is valid.
131
132       format Completion queues allow the application to select the amount  of
133              detail  that it must store and report.  The format attribute al‐
134              lows the application to select one of  several  completion  for‐
135              mats,  indicating  the structure of the data that the completion
136              queue should return when read.  Supported formats and the struc‐
137              tures  that correspond to each are listed below.  The meaning of
138              the CQ entry fields are defined in the  Completion  Fields  sec‐
139              tion.
140
141       - FI_CQ_FORMAT_UNSPEC
142              If  an  unspecified  format is requested, then the CQ will use a
143              provider selected default format.
144
145       - FI_CQ_FORMAT_CONTEXT
146              Provides only user specified context that  was  associated  with
147              the completion.
148
149              struct fi_cq_entry {
150                  void     *op_context; /* operation context */
151              };
152              · .RS 2
153
154       FI_CQ_FORMAT_MSG
155              Provides  minimal data for processing completions, with expanded
156              support for reporting information about received messages.
157
158              struct fi_cq_msg_entry {
159                  void     *op_context; /* operation context */
160                  uint64_t flags;       /* completion flags */
161                  size_t   len;         /* size of received data */
162              };
163              · .RS 2
164
165       FI_CQ_FORMAT_DATA
166              Provides data associated with a  completion.   Includes  support
167              for  received  message length, remote CQ data, and multi-receive
168              buffers.
169
170              struct fi_cq_data_entry {
171                  void     *op_context; /* operation context */
172                  uint64_t flags;       /* completion flags */
173                  size_t   len;         /* size of received data */
174                  void     *buf;        /* receive data buffer */
175                  uint64_t data;        /* completion data */
176              };
177              · .RS 2
178
179       FI_CQ_FORMAT_TAGGED
180              Expands completion data to include support for the  tagged  mes‐
181              sage interfaces.
182
183              struct fi_cq_tagged_entry {
184                  void     *op_context; /* operation context */
185                  uint64_t flags;       /* completion flags */
186                  size_t   len;         /* size of received data */
187                  void     *buf;        /* receive data buffer */
188                  uint64_t data;        /* completion data */
189                  uint64_t tag;         /* received tag */
190              };
191
192       wait_obj
193              CQ's  may  be  associated with a specific wait object.  Wait ob‐
194              jects allow applications to block until the wait object is  sig‐
195              naled,  indicating  that  a  completion is available to be read.
196              Users may use fi_control to retrieve the underlying wait  object
197              associated  with a CQ, in order to use it in other system calls.
198              The following values may be used to specify the type of wait ob‐
199              ject   associated   with  a  CQ:  FI_WAIT_NONE,  FI_WAIT_UNSPEC,
200              FI_WAIT_SET, FI_WAIT_FD, FI_WAIT_MUTEX_COND, and  FI_WAIT_YIELD.
201              The default is FI_WAIT_NONE.
202
203       - FI_WAIT_NONE
204              Used to indicate that the user will not block (wait) for comple‐
205              tions on the CQ.  When FI_WAIT_NONE is specified,  the  applica‐
206              tion may not call fi_cq_sread or fi_cq_sreadfrom.
207
208       - FI_WAIT_UNSPEC
209              Specifies  that  the  user will only wait on the CQ using fabric
210              interface calls, such as  fi_cq_sread  or  fi_cq_sreadfrom.   In
211              this case, the underlying provider may select the most appropri‐
212              ate or highest performing wait object available, including  cus‐
213              tom  wait  mechanisms.   Applications that select FI_WAIT_UNSPEC
214              are not guaranteed to retrieve the underlying wait object.
215
216       - FI_WAIT_SET
217              Indicates that the completion queue should use a wait set object
218              to  wait for completions.  If specified, the wait_set field must
219              reference an existing wait set object.
220
221       - FI_WAIT_FD
222              Indicates that the CQ should use a file descriptor as  its  wait
223              mechanism.   A file descriptor wait object must be usable in se‐
224              lect, poll, and epoll routines.  However, a provider may  signal
225              an  FD  wait object by marking it as readable, writable, or with
226              an error.
227
228       - FI_WAIT_MUTEX_COND
229              Specifies that the CQ should use a pthread mutex and cond  vari‐
230              able as a wait object.
231
232       - FI_WAIT_YIELD
233              Indicates  that  the  CQ will wait without a wait object but in‐
234              stead yield on every wait.   Allows  usage  of  fi_cq_sread  and
235              fi_cq_sreadfrom through a spin.
236
237       signaling_vector
238              If  the  FI_AFFINITY flag is set, this indicates the logical cpu
239              number (0..max cpu - 1) that interrupts associated with  the  CQ
240              should  target.   This  field should be treated as a hint to the
241              provider and may be ignored if the provider does not support in‐
242              terrupt affinity.
243
244       wait_cond
245              By  default,  when  a completion is inserted into a CQ that sup‐
246              ports blocking reads (fi_cq_sread/fi_cq_sreadfrom),  the  corre‐
247              sponding wait object is signaled.  Users may specify a condition
248              that must first be met before the wait is satisfied.  This field
249              indicates  how  the  provider  should  interpret the cond field,
250              which describes the condition needed to signal the wait object.
251
252       A wait condition should be treated as an optimization.   Providers  are
253       not required to meet the requirements of the condition before signaling
254       the wait object.  Applications should not rely on the condition  neces‐
255       sarily being true when a blocking read call returns.
256
257       If  wait_cond  is set to FI_CQ_COND_NONE, then no additional conditions
258       are applied to the signaling of the CQ wait object, and  the  insertion
259       of  any new entry will trigger the wait condition.  If wait_cond is set
260       to FI_CQ_COND_THRESHOLD, then the cond field is interpreted as a size_t
261       threshold  value.   The  threshold indicates the number of entries that
262       are to be queued before at the CQ before the wait is satisfied.
263
264       This field is ignored if wait_obj is set to FI_WAIT_NONE.
265
266       wait_set
267              If wait_obj is FI_WAIT_SET, this field references a wait  object
268              to  which  the completion queue should attach.  When an event is
269              inserted into the completion queue, the corresponding  wait  set
270              will  be  signaled if all necessary conditions are met.  The use
271              of a wait_set enables an optimized method of waiting for  events
272              across  multiple event and completion queues.  This field is ig‐
273              nored if wait_obj is not FI_WAIT_SET.
274
275   fi_close
276       The fi_close call releases all resources associated with  a  completion
277       queue.   Any  completions  which remain on the CQ when it is closed are
278       lost.
279
280       When closing the CQ, there must be no opened endpoints,  transmit  con‐
281       texts,  or  receive  contexts associated with the CQ.  If resources are
282       still associated with the CQ when attempting to close,  the  call  will
283       return -FI_EBUSY.
284
285   fi_control
286       The  fi_control  call is used to access provider or implementation spe‐
287       cific details of the completion queue.  Access to the CQ should be  se‐
288       rialized  across  all calls when fi_control is invoked, as it may redi‐
289       rect the implementation of CQ operations.  The following  control  com‐
290       mands are usable with a CQ.
291
292       FI_GETWAIT (void **)
293              This  command allows the user to retrieve the low-level wait ob‐
294              ject associated with the CQ.  The format of the  wait-object  is
295              specified  during  CQ  creation, through the CQ attributes.  The
296              fi_control arg parameter should be an address where a pointer to
297              the returned wait object will be written.  See fi_eq.3 for addi‐
298              tion details using fi_control with FI_GETWAIT.
299
300   fi_cq_read
301       The fi_cq_read operation performs a non-blocking read of completion da‐
302       ta from the CQ.  The format of the completion event is determined using
303       the fi_cq_format option that was specified  when  the  CQ  was  opened.
304       Multiple  completions may be retrieved from a CQ in a single call.  The
305       maximum number of entries to return is limited to the  specified  count
306       parameter, with the number of entries successfully read from the CQ re‐
307       turned by the call.  (See return values section below.)
308
309       CQs are optimized to report operations which have completed successful‐
310       ly.  Operations which fail are reported 'out of band'.  Such operations
311       are retrieved using the fi_cq_readerr function.  When an operation that
312       has completed with an unexpected error is encountered, it is placed in‐
313       to a temporary error queue.  Attempting to read from a CQ while an item
314       is  in the error queue results in fi_cq_read failing with a return code
315       of -FI_EAVAIL.  Applications may use this return code to determine when
316       to call fi_cq_readerr.
317
318   fi_cq_readfrom
319       The  fi_cq_readfrom  call behaves identical to fi_cq_read, with the ex‐
320       ception that it allows the CQ to return source address  information  to
321       the  user for any received data.  Source address data is only available
322       for  those  endpoints  configured  with   FI_SOURCE   capability.    If
323       fi_cq_readfrom is called on an endpoint for which source addressing da‐
324       ta is not available, the source address  will  be  set  to  FI_ADDR_NO‐
325       TAVAIL.   The  number of input src_addr entries must be the same as the
326       count parameter.
327
328       Returned source addressing data is converted from  the  native  address
329       used  by  the underlying fabric into an fi_addr_t, which may be used in
330       transmit operations.  Under most circumstances, returning fi_addr_t re‐
331       quires  that the source address already have been inserted into the ad‐
332       dress vector associated with the receiving endpoint.  This is true  for
333       address   vectors  of  type  FI_AV_TABLE.   In  select  providers  when
334       FI_AV_MAP is used, source addresses may  be  converted  algorithmically
335       into  a  usable  fi_addr_t, even though the source address has not been
336       inserted into the address vector.  This is permitted by the API, as  it
337       allows the provider to avoid address look-up as part of receive message
338       processing.  In no case do providers insert addresses into an AV  sepa‐
339       rate from an application calling fi_av_insert or similar call.
340
341       For  endpoints  allocated  using  the  FI_SOURCE_ERR capability, if the
342       source address cannot  be  converted  into  a  valid  fi_addr_t  value,
343       fi_cq_readfrom  will  return -FI_EAVAIL, even if the data were received
344       successfully.  The completion will then be reported through fi_cq_read‐
345       err with error code -FI_EADDRNOTAVAIL.  See fi_cq_readerr for details.
346
347       If FI_SOURCE is specified without FI_SOURCE_ERR, source addresses which
348       cannot be mapped to a usable fi_addr_t will be reported as  FI_ADDR_NO‐
349       TAVAIL.
350
351   fi_cq_sread / fi_cq_sreadfrom
352       The  fi_cq_sread  and fi_cq_sreadfrom calls are the blocking equivalent
353       operations to fi_cq_read and fi_cq_readfrom.  Their behavior is similar
354       to  the  non-blocking calls, with the exception that the calls will not
355       return until either a completion has been read from the CQ or an  error
356       or timeout occurs.
357
358       Threads blocking in this function will return to the caller if they are
359       signaled by some external source.  This is true even if the timeout has
360       not occurred or was specified as infinite.
361
362       It  is  invalid  for applications to call these functions if the CQ has
363       been configured with a wait object of FI_WAIT_NONE or FI_WAIT_SET.
364
365   fi_cq_readerr
366       The read error function, fi_cq_readerr, retrieves information regarding
367       any  asynchronous  operation which has completed with an unexpected er‐
368       ror.  fi_cq_readerr  is  a  non-blocking  call,  returning  immediately
369       whether an error completion was found or not.
370
371       Error  information is reported to the user through struct fi_cq_err_en‐
372       try.  The format of this structure is defined below.
373
374              struct fi_cq_err_entry {
375                  void     *op_context; /* operation context */
376                  uint64_t flags;       /* completion flags */
377                  size_t   len;         /* size of received data */
378                  void     *buf;        /* receive data buffer */
379                  uint64_t data;        /* completion data */
380                  uint64_t tag;         /* message tag */
381                  size_t   olen;        /* overflow length */
382                  int      err;         /* positive error code */
383                  int      prov_errno;  /* provider error code */
384                  void    *err_data;    /*  error data */
385                  size_t   err_data_size; /* size of err_data */
386              };
387
388       The general reason for the error is provided  through  the  err  field.
389       Provider  specific  error information may also be available through the
390       prov_errno and err_data fields.  Users may call fi_cq_strerror to  con‐
391       vert  provider  specific  error information into a printable string for
392       debugging purposes.  See field details below for  more  information  on
393       the use of err_data and err_data_size.
394
395       Note that error completions are generated for all operations, including
396       those for which a completion was not requested (e.g.   an  endpoint  is
397       configured  with  FI_SELECTIVE_COMPLETION, but the request did not have
398       the FI_COMPLETION flag set).  In such cases, providers will  return  as
399       much information as made available by the underlying software and hard‐
400       ware about the failure, other fields will be set to NULL  or  0.   This
401       includes  the op_context value, which may not have been provided or was
402       ignored on input as part of the transfer.
403
404       Notable completion error codes are given below.
405
406       FI_EADDRNOTAVAIL
407              This error code is used by CQs configured with FI_SOURCE_ERR  to
408              report  completions  for which a usable fi_addr_t source address
409              could not be found.  An error code of FI_EADDRNOTAVAIL indicates
410              that  the data transfer was successfully received and processed,
411              with the fi_cq_err_entry fields containing information about the
412              completion.   The  err_data  field will be set to the source ad‐
413              dress data.  The source address will be in the  same  format  as
414              specified  through  the fi_info addr_format field for the opened
415              domain.  This may be passed directly into an  fi_av_insert  call
416              to add the source address to the address vector.
417
418   fi_cq_signal
419       The fi_cq_signal call will unblock any thread waiting in fi_cq_sread or
420       fi_cq_sreadfrom.  This may be used to wake-up a thread that is  blocked
421       waiting  to read a completion operation.  The fi_cq_signal operation is
422       only available if the CQ was configured with a wait object.
423

COMPLETION FIELDS

425       The CQ entry data structures share many of the same fields.  The  mean‐
426       ings of these fields are the same for all CQ entry structure formats.
427
428       op_context
429              The operation context is the application specified context value
430              that was provided with an asynchronous operation.   The  op_con‐
431              text field is valid for all completions that are associated with
432              an asynchronous operation.
433
434       For completion events that are not associated with a posted  operation,
435       this field will be set to NULL.  This includes completions generated at
436       the target in response to RMA  write  operations  that  carry  CQ  data
437       (FI_REMOTE_WRITE | FI_REMOTE_CQ_DATA flags set), when the FI_RX_CQ_DATA
438       mode bit is not required.
439
440       flags  This specifies flags associated with  the  completed  operation.
441              The  Completion  Flags  section  below  lists valid flag values.
442              Flags are set for all relevant completions.
443
444       len    This len field only  applies  to  completed  receive  operations
445              (e.g.   fi_recv,  fi_trecv, etc.).  It indicates the size of re‐
446              ceived message data -- i.e.  how many data bytes were placed in‐
447              to   the   associated   receive   buffer   by   a  corresponding
448              fi_send/fi_tsend/et al call.  If an endpoint has been configured
449              with  the  FI_MSG_PREFIX mode, the len also reflects the size of
450              the prefix buffer.
451
452       buf    The buf field is only valid for  completed  receive  operations,
453              and  only  applies  when  the receive buffer was posted with the
454              FI_MULTI_RECV flag.  In this case, buf points  to  the  starting
455              location where the receive data was placed.
456
457       data   The data field is only valid if the FI_REMOTE_CQ_DATA completion
458              flag is set, and only applies to receive completions.  If FI_RE‐
459              MOTE_CQ_DATA is set, this field will contain the completion data
460              provided by the peer as part of  their  transmit  request.   The
461              completion data will be given in host byte order.
462
463       tag    A  tag  applies  only  to received messages that occur using the
464              tagged interfaces.  This field contains the tag that was includ‐
465              ed  with the received message.  The tag will be in host byte or‐
466              der.
467
468       olen   The olen field applies to received messages.  It is used to  in‐
469              dicate  that a received message has overrun the available buffer
470              space and has been truncated.  The olen specifies the amount  of
471              data  that did not fit into the available receive buffer and was
472              discarded.
473
474       err    This err code is a positive fabric errno associated with a  com‐
475              pletion.   The err value indicates the general reason for an er‐
476              ror, if one occurred.  See fi_errno.3 for a list of possible er‐
477              ror codes.
478
479       prov_errno
480              On  an  error,  prov_errno may contain a provider specific error
481              code.  The use of this field and its meaning is provider specif‐
482              ic.   It  is  intended  to  be  used  as  a  debugging aid.  See
483              fi_cq_strerror for additional details on converting  this  error
484              value into a human readable string.
485
486       err_data
487              The  err_data field is used to return provider specific informa‐
488              tion, if available, about the error.  On input, err_data  should
489              reference  a  data buffer of size err_data_size.  On output, the
490              provider will fill in this buffer with any provider specific da‐
491              ta which may help identify the cause of the error.  The contents
492              of the err_data field and its meaning is provider specific.   It
493              is  intended  to be used as a debugging aid.  See fi_cq_strerror
494              for additional details on converting this error data into a  hu‐
495              man  readable  string.   See the compatibility note below on how
496              this field is used for older libfabric releases.
497
498       err_data_size
499              On input, err_data_size indicates the size of the err_data  buf‐
500              fer  in bytes.  On output, err_data_size will be set to the num‐
501              ber of bytes copied to the err_data buffer.  The err_data infor‐
502              mation  is typically used with fi_cq_strerror to provide details
503              about the type of error that occurred.
504
505       For compatibility purposes, the behavior of the  err_data  and  err_da‐
506       ta_size  fields  is may be modified from that listed above.  If err_da‐
507       ta_size is 0 on input, or the fabric was opened  with  release  <  1.5,
508       then  any  buffer  referenced by err_data will be ignored on input.  In
509       this situation, on output err_data will be set to a data  buffer  owned
510       by  the provider.  The contents of the buffer will remain valid until a
511       subsequent read call against the CQ.  Applications must  serialize  ac‐
512       cess  to the CQ when processing errors to ensure that the buffer refer‐
513       enced by err_data does not change.
514

COMPLETION FLAGS

516       Completion flags provide additional details regarding the completed op‐
517       eration.  The following completion flags are defined.
518
519       FI_SEND
520              Indicates  that  the  completion was for a send operation.  This
521              flag may be combined with an FI_MSG or FI_TAGGED flag.
522
523       FI_RECV
524              Indicates that the completion was for a receive operation.  This
525              flag may be combined with an FI_MSG or FI_TAGGED flag.
526
527       FI_RMA Indicates  that  an  RMA  operation completed.  This flag may be
528              combined with an FI_READ, FI_WRITE,  FI_REMOTE_READ,  or  FI_RE‐
529              MOTE_WRITE flag.
530
531       FI_ATOMIC
532              Indicates  that an atomic operation completed.  This flag may be
533              combined with an FI_READ, FI_WRITE,  FI_REMOTE_READ,  or  FI_RE‐
534              MOTE_WRITE flag.
535
536       FI_MSG Indicates  that  a message-based operation completed.  This flag
537              may be combined with an FI_SEND or FI_RECV flag.
538
539       FI_TAGGED
540              Indicates that a tagged message operation completed.  This  flag
541              may be combined with an FI_SEND or FI_RECV flag.
542
543       FI_MULTICAST
544              Indicates  that  a multicast operation completed.  This flag may
545              be combined with FI_MSG and relevant flags.  This flag  is  only
546              guaranteed to be valid for received messages if the endpoint has
547              been configured with FI_SOURCE.
548
549       FI_READ
550              Indicates that a locally initiated RMA or atomic read  operation
551              has  completed.   This  flag  may  be combined with an FI_RMA or
552              FI_ATOMIC flag.
553
554       FI_WRITE
555              Indicates that a locally initiated RMA or atomic write operation
556              has  completed.   This  flag  may  be combined with an FI_RMA or
557              FI_ATOMIC flag.
558
559       FI_REMOTE_READ
560              Indicates that a remotely initiated RMA or atomic read operation
561              has  completed.   This  flag  may  be combined with an FI_RMA or
562              FI_ATOMIC flag.
563
564       FI_REMOTE_WRITE
565              Indicates that a remotely initiated RMA or atomic  write  opera‐
566              tion has completed.  This flag may be combined with an FI_RMA or
567              FI_ATOMIC flag.
568
569       FI_REMOTE_CQ_DATA
570              This indicates that remote CQ data is available as part  of  the
571              completion.
572
573       FI_MULTI_RECV
574              This  flag  applies to receive buffers that were posted with the
575              FI_MULTI_RECV flag set.  This completion flag indicates that the
576              original  receive  buffer  referenced by the completion has been
577              consumed and was released by the provider.   Providers  may  set
578              this  flag  on the last message that is received into the multi-
579              recv buffer, or may generate a separate  completion  that  indi‐
580              cates that the buffer has been released.
581
582       Applications  can  distinguish between these two cases by examining the
583       completion entry flags field.  If additional flags,  such  as  FI_RECV,
584       are set, the completion is associated with a received message.  In this
585       case, the buf field will reference the location where the received mes‐
586       sage  was  placed into the multi-recv buffer.  Other fields in the com‐
587       pletion entry will be determined based on  the  received  message.   If
588       other flag bits are zero, the provider is reporting that the multi-recv
589       buffer has been released, and the completion entry  is  not  associated
590       with a received message.
591
592       FI_MORE
593              See  the  'Buffered  Receives' section in fi_msg(3) for more de‐
594              tails.  This flag is associated with receive completions on end‐
595              points  that  have  FI_BUFFERED_RECV  mode enabled.  When set to
596              one, it indicates that the buffer referenced by  the  completion
597              is limited by the FI_OPT_BUFFERED_LIMIT threshold, and addition‐
598              al message data must be retrieved by the  application  using  an
599              FI_CLAIM operation.
600
601       FI_CLAIM
602              See  the  'Buffered  Receives' section in fi_msg(3) for more de‐
603              tails.  This flag is set on completions associated with  receive
604              operations  that  claim  buffered  receive data.  Note that this
605              flag   only   applies   to   endpoints   configured   with   the
606              FI_BUFFERED_RECV mode bit.
607

COMPLETION EVENT SEMANTICS

609       Libfabric  defines several completion 'levels', identified using opera‐
610       tional flags.  Each flag indicates the soonest that a completion  event
611       may be generated by a provider, and the assumptions that an application
612       may make upon processing a completion.  The operational flags  are  de‐
613       fined  below,  along  with an example of how a provider might implement
614       the semantic.  Note that only meeting the semantic is required  of  the
615       provider  and not the implementation.  Providers may implement stronger
616       completion semantics than necessary for a given operation, but only the
617       behavior defined by the completion level is guaranteed.
618
619       To  help  understand  the  conceptual differences in completion levels,
620       consider mailing a letter.  Placing the letter into the  local  mailbox
621       for  pick-up is similar to 'inject complete'.  Having the letter picked
622       up and dropped off at the destination mailbox is equivalent to  'trans‐
623       mit  complete'.  The 'delivery complete' semantic is a stronger guaran‐
624       tee, with a person at the destination signing for the letter.  However,
625       the  person  who  signed for the letter is not necessarily the intended
626       recipient.  The 'match complete' option is  similar  to  delivery  com‐
627       plete, but requires the intended recipient to sign for the letter.
628
629       The 'commit complete' level has different semantics than the previously
630       mentioned levels.  Commit complete would be closer to the letter arriv‐
631       ing at the destination and being placed into a fire proof safe.
632
633       The  operational  flags for the described completion levels are defined
634       below.
635
636       FI_INJECT_COMPLETE
637              Indicates that a completion should be generated when the  source
638              buffer(s)  may be reused.  A completion guarantees that the buf‐
639              fers will not be read from again and the application may reclaim
640              them.  No other guarantees are made with respect to the state of
641              the operation.
642
643       Example: A provider may generate this completion  event  after  copying
644       the  source  buffer  into a network buffer, either in host memory or on
645       the NIC.  An inject completion does not indicate that the data has been
646       transmitted  onto  the network, and a local error could occur after the
647       completion event has been generated that could prevent  it  from  being
648       transmitted.
649
650       Inject  complete  allows  for  the  fastest  completion reporting (and,
651       hence, buffer reuse), but provides the weakest guarantees against  net‐
652       work errors.
653
654       Note:  This flag is used to control when a completion entry is inserted
655       into a completion queue.  It does not apply to operations that  do  not
656       generate a completion queue entry, such as the fi_inject operation, and
657       is not subject to the inject_size message limit restriction.
658
659       FI_TRANSMIT_COMPLETE
660              Indicates that a completion should be generated when the  trans‐
661              mit operation has completed relative to the local provider.  The
662              exact behavior is dependent on the endpoint type.
663
664       For reliable endpoints:
665
666       Indicates that a completion should be generated when the operation  has
667       been  delivered to the peer endpoint.  A completion guarantees that the
668       operation is no longer dependent on the fabric or local resources.  The
669       state of the operation at the peer endpoint is not defined.
670
671       Example: A provider may generate a transmit complete event upon receiv‐
672       ing an ack from the peer endpoint.  The state of  the  message  at  the
673       peer  is  unknown and may be buffered in the target NIC at the time the
674       ack has been generated.
675
676       For unreliable endpoints:
677
678       Indicates that a completion should be generated when the operation  has
679       been  delivered to the fabric.  A completion guarantees that the opera‐
680       tion is no longer dependent on local resources.  The state of the oper‐
681       ation within the fabric is not defined.
682
683       FI_DELIVERY_COMPLETE
684              Indicates that a completion should not be generated until an op‐
685              eration has been processed by the  destination  endpoint(s).   A
686              completion guarantees that the result of the operation is avail‐
687              able; however, additional steps may need to be taken at the des‐
688              tination  to  retrieve the results.  For example, an application
689              may need to provide a receive buffers in order to retrieve  mes‐
690              sages that were buffered by the provider.
691
692       Delivery  complete indicates that the message has been processed by the
693       peer.  If an application buffer was ready to receive the results of the
694       message when it arrived, then delivery complete indicates that the data
695       was placed into the application's buffer.
696
697       This completion mode applies only to reliable  endpoints.   For  opera‐
698       tions  that  return  data  to  the initiator, such as RMA read or atom‐
699       ic-fetch, the source endpoint is also  considered  a  destination  end‐
700       point.  This is the default completion mode for such operations.
701
702       FI_MATCH_COMPLETE
703              Indicates  that  a completion should be generated only after the
704              operation has been matched with an application specified buffer.
705              Operations  using  this completion semantic are dependent on the
706              application at the target claiming the message or results.  As a
707              result, match complete may involve additional provider level ac‐
708              knowledgements or lengthy delays.  However, this completion mod‐
709              el  enables  peer  applications  to synchronize their execution.
710              Many providers may not support this semantic.
711
712       FI_COMMIT_COMPLETE
713              Indicates that a completion should not be generated (locally  or
714              at  the  peer)  until  the result of an operation have been made
715              persistent.  A completion guarantees that  the  result  is  both
716              available and durable, in the case of power failure.
717
718       This  completion mode applies only to operations that target persistent
719       memory regions over reliable endpoints.  This completion mode is exper‐
720       imental.
721
722       FI_FENCE
723              This  is not a completion level, but plays a role in the comple‐
724              tion ordering between operations that would not normally be  or‐
725              dered.   An  operation that is marked with the FI_FENCE flag and
726              all operations posted after the fenced  operation  are  deferred
727              until  all  previous operations targeting the same peer endpoint
728              have completed.  Additionally, the completion of the fenced  op‐
729              eration  indicates  that prior operations have met the same com‐
730              pletion level as the fenced operation.  For example, if an oper‐
731              ation  is  posted  as  FI_DELIVERY_COMPLETE | FI_FENCE, then its
732              completion indicates prior operations have met the semantic  re‐
733              quired for FI_DELIVERY_COMPLETE.  This is true even if the prior
734              operation was posted with a  lower  completion  level,  such  as
735              FI_TRANSMIT_COMPLETE or FI_INJECT_COMPLETE.
736
737       Note  that  a completion generated for an operation posted prior to the
738       fenced operation only guarantees that the  completion  level  that  was
739       originally  requested has been met.  It is the completion of the fenced
740       operation that guarantees that the additional semantics have been met.
741

NOTES

743       A completion queue must be bound to at least one enabled  endpoint  be‐
744       fore  any  operation  such  as fi_cq_read, fi_cq_readfrom, fi_cq_sread,
745       fi_cq_sreadfrom etc.  can be called on it.
746
747       Completion flags may be suppressed if the FI_NOTIFY_FLAGS_ONLY mode bit
748       has been set.  When enabled, only the following flags are guaranteed to
749       be set in completion data  when  they  are  valid:  FI_REMOTE_READ  and
750       FI_REMOTE_WRITE (when FI_RMA_EVENT capability bit has been set), FI_RE‐
751       MOTE_CQ_DATA, and FI_MULTI_RECV.
752
753       If a completion queue has been overrun,  it  will  be  placed  into  an
754       'overrun'  state.   Read  operations will continue to return any valid,
755       non-corrupted completions, if available.  After all  valid  completions
756       have  been  retrieved, any attempt to read the CQ will result in it re‐
757       turning an FI_EOVERRUN error event.  Overrun completion queues are con‐
758       sidered fatal and may not be used to report additional completions once
759       the overrun occurs.
760

RETURN VALUES

762       fi_cq_open / fi_cq_signal
763              Returns 0 on success.  On error, a negative value  corresponding
764              to fabric errno is returned.
765
766       fi_cq_read  / fi_cq_readfrom / fi_cq_readerr fi_cq_sread / fi_cq_sread‐
767       from : On success, returns the number of  completion  events  retrieved
768       from the completion queue.  On error, a negative value corresponding to
769       fabric errno is returned.  If no completions are  available  to  return
770       from the CQ, -FI_EAGAIN will be returned.
771
772       fi_cq_sread / fi_cq_sreadfrom
773              On  success,  returns  the number of completion events retrieved
774              from the completion queue.  On error, a  negative  value  corre‐
775              sponding to fabric errno is returned.  If the timeout expires or
776              the calling thread is signaled and no data is  available  to  be
777              read from the completion queue, -FI_EAGAIN is returned.
778
779       fi_cq_strerror
780              Returns  a  character string interpretation of the provider spe‐
781              cific error returned with a completion.
782
783       Fabric errno values are defined in rdma/fi_errno.h.
784

SEE ALSO

786       fi_getinfo(3),  fi_endpoint(3),  fi_domain(3),  fi_eq(3),   fi_cntr(3),
787       fi_poll(3)
788

AUTHORS

790       OpenFabrics.
791
792
793
794Libfabric Programmer's Manual     2019-12-13                          fi_cq(3)
Impressum