1fi_cq(3)                       Libfabric v1.17.0                      fi_cq(3)
2
3
4

NAME

6       fi_cq - Completion queue operations
7
8       fi_cq_open / fi_close
9              Open/close a completion queue
10
11       fi_control
12              Control CQ operation or attributes.
13
14       fi_cq_read / fi_cq_readfrom / fi_cq_readerr
15              Read a completion from a completion queue
16
17       fi_cq_sread / fi_cq_sreadfrom
18              A  synchronous (blocking) read that waits until a specified con‐
19              dition has been met before reading a completion from  a  comple‐
20              tion queue.
21
22       fi_cq_signal
23              Unblock any thread waiting in fi_cq_sread or fi_cq_sreadfrom.
24
25       fi_cq_strerror
26              Converts  provider  specific  error information into a printable
27              string
28

SYNOPSIS

30              #include <rdma/fi_domain.h>
31
32              int fi_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
33                  struct fid_cq **cq, void *context);
34
35              int fi_close(struct fid *cq);
36
37              int fi_control(struct fid *cq, int command, void *arg);
38
39              ssize_t fi_cq_read(struct fid_cq *cq, void *buf, size_t count);
40
41              ssize_t fi_cq_readfrom(struct fid_cq *cq, void *buf, size_t count,
42                  fi_addr_t *src_addr);
43
44              ssize_t fi_cq_readerr(struct fid_cq *cq, struct fi_cq_err_entry *buf,
45                  uint64_t flags);
46
47              ssize_t fi_cq_sread(struct fid_cq *cq, void *buf, size_t count,
48                  const void *cond, int timeout);
49
50              ssize_t fi_cq_sreadfrom(struct fid_cq *cq, void *buf, size_t count,
51                  fi_addr_t *src_addr, const void *cond, int timeout);
52
53              int fi_cq_signal(struct fid_cq *cq);
54
55              const char * fi_cq_strerror(struct fid_cq *cq, int prov_errno,
56                    const void *err_data, char *buf, size_t len);
57

ARGUMENTS

59       domain Open resource domain
60
61       cq     Completion queue
62
63       attr   Completion queue attributes
64
65       context
66              User specified context associated with the completion queue.
67
68       buf    For read calls, the data buffer to write completions into.   For
69              write  calls,  a completion to insert into the completion queue.
70              For fi_cq_strerror, an optional buffer that  receives  printable
71              error information.
72
73       count  Number of CQ entries.
74
75       len    Length of data buffer
76
77       src_addr
78              Source address of a completed receive operation
79
80       flags  Additional flags to apply to the operation
81
82       command
83              Command of control operation to perform on CQ.
84
85       arg    Optional control argument
86
87       cond   Condition that must be met before a completion is generated
88
89       timeout
90              Time  in milliseconds to wait.  A negative value indicates infi‐
91              nite timeout.
92
93       prov_errno
94              Provider specific error value
95
96       err_data
97              Provider specific error data related to a completion
98

DESCRIPTION

100       Completion queues are used to report events associated with data trans‐
101       fers.   They are associated with message sends and receives, RMA, atom‐
102       ic, tagged messages, and triggered events.  Reported events are usually
103       associated with a fabric endpoint, but may also refer to memory regions
104       used as the target of an RMA or atomic operation.
105
106   fi_cq_open
107       fi_cq_open allocates a new completion queue.  Unlike event queues, com‐
108       pletion  queues  are  associated with a resource domain and may be off‐
109       loaded entirely in provider hardware.
110
111       The properties and behavior of a completion queue are defined by struct
112       fi_cq_attr.
113
114              struct fi_cq_attr {
115                  size_t               size;      /* # entries for CQ */
116                  uint64_t             flags;     /* operation flags */
117                  enum fi_cq_format    format;    /* completion format */
118                  enum fi_wait_obj     wait_obj;  /* requested wait object */
119                  int                  signaling_vector; /* interrupt affinity */
120                  enum fi_cq_wait_cond wait_cond; /* wait condition format */
121                  struct fid_wait     *wait_set;  /* optional wait set */
122              };
123
124       size   Specifies  the minimum size of a completion queue.  A value of 0
125              indicates that the provider may choose a default value.
126
127       flags  Flags that control the configuration of the CQ.
128
129       - FI_AFFINITY
130              Indicates that the signaling_vector field (see below) is valid.
131
132       format Completion queues allow the application to select the amount  of
133              detail  that it must store and report.  The format attribute al‐
134              lows the application to select one of  several  completion  for‐
135              mats,  indicating  the structure of the data that the completion
136              queue should return when read.  Supported formats and the struc‐
137              tures  that correspond to each are listed below.  The meaning of
138              the CQ entry fields are defined in the  Completion  Fields  sec‐
139              tion.
140
141       - FI_CQ_FORMAT_UNSPEC
142              If  an  unspecified  format is requested, then the CQ will use a
143              provider selected default format.
144
145       - FI_CQ_FORMAT_CONTEXT
146              Provides only user specified context that  was  associated  with
147              the completion.
148
149              struct fi_cq_entry {
150                  void     *op_context; /* operation context */
151              };
152              • .RS 2
153
154       FI_CQ_FORMAT_MSG
155              Provides  minimal data for processing completions, with expanded
156              support for reporting information about received messages.
157
158              struct fi_cq_msg_entry {
159                  void     *op_context; /* operation context */
160                  uint64_t flags;       /* completion flags */
161                  size_t   len;         /* size of received data */
162              };
163              • .RS 2
164
165       FI_CQ_FORMAT_DATA
166              Provides data associated with a  completion.   Includes  support
167              for  received  message length, remote CQ data, and multi-receive
168              buffers.
169
170              struct fi_cq_data_entry {
171                  void     *op_context; /* operation context */
172                  uint64_t flags;       /* completion flags */
173                  size_t   len;         /* size of received data */
174                  void     *buf;        /* receive data buffer */
175                  uint64_t data;        /* completion data */
176              };
177              • .RS 2
178
179       FI_CQ_FORMAT_TAGGED
180              Expands completion data to include support for the  tagged  mes‐
181              sage interfaces.
182
183              struct fi_cq_tagged_entry {
184                  void     *op_context; /* operation context */
185                  uint64_t flags;       /* completion flags */
186                  size_t   len;         /* size of received data */
187                  void     *buf;        /* receive data buffer */
188                  uint64_t data;        /* completion data */
189                  uint64_t tag;         /* received tag */
190              };
191
192       wait_obj
193              CQ’s  may  be  associated with a specific wait object.  Wait ob‐
194              jects allow applications to block until the wait object is  sig‐
195              naled,  indicating  that  a  completion is available to be read.
196              Users may use fi_control to retrieve the underlying wait  object
197              associated  with a CQ, in order to use it in other system calls.
198              The following values may be used to specify the type of wait ob‐
199              ject   associated   with  a  CQ:  FI_WAIT_NONE,  FI_WAIT_UNSPEC,
200              FI_WAIT_SET, FI_WAIT_FD, FI_WAIT_MUTEX_COND, and  FI_WAIT_YIELD.
201              The default is FI_WAIT_NONE.
202
203       - FI_WAIT_NONE
204              Used to indicate that the user will not block (wait) for comple‐
205              tions on the CQ.  When FI_WAIT_NONE is specified,  the  applica‐
206              tion may not call fi_cq_sread or fi_cq_sreadfrom.
207
208       - FI_WAIT_UNSPEC
209              Specifies  that  the  user will only wait on the CQ using fabric
210              interface calls, such as  fi_cq_sread  or  fi_cq_sreadfrom.   In
211              this case, the underlying provider may select the most appropri‐
212              ate or highest performing wait object available, including  cus‐
213              tom  wait  mechanisms.   Applications that select FI_WAIT_UNSPEC
214              are not guaranteed to retrieve the underlying wait object.
215
216       - FI_WAIT_SET
217              Indicates that the completion queue should use a wait set object
218              to  wait for completions.  If specified, the wait_set field must
219              reference an existing wait set object.
220
221       - FI_WAIT_FD
222              Indicates that the CQ should use a file descriptor as  its  wait
223              mechanism.   A file descriptor wait object must be usable in se‐
224              lect, poll, and epoll routines.  However, a provider may  signal
225              an  FD  wait object by marking it as readable, writable, or with
226              an error.
227
228       - FI_WAIT_MUTEX_COND
229              Specifies that the CQ should use a pthread mutex and cond  vari‐
230              able as a wait object.
231
232       - FI_WAIT_YIELD
233              Indicates  that  the  CQ will wait without a wait object but in‐
234              stead yield on every wait.   Allows  usage  of  fi_cq_sread  and
235              fi_cq_sreadfrom through a spin.
236
237       signaling_vector
238              If  the  FI_AFFINITY flag is set, this indicates the logical cpu
239              number (0..max cpu - 1) that interrupts associated with  the  CQ
240              should  target.   This  field should be treated as a hint to the
241              provider and may be ignored if the provider does not support in‐
242              terrupt affinity.
243
244       wait_cond
245              By  default,  when  a completion is inserted into a CQ that sup‐
246              ports blocking reads (fi_cq_sread/fi_cq_sreadfrom),  the  corre‐
247              sponding wait object is signaled.  Users may specify a condition
248              that must first be met before the wait is satisfied.  This field
249              indicates  how  the  provider  should  interpret the cond field,
250              which describes the condition needed to signal the wait object.
251
252       A wait condition should be treated as an optimization.   Providers  are
253       not required to meet the requirements of the condition before signaling
254       the wait object.  Applications should not rely on the condition  neces‐
255       sarily being true when a blocking read call returns.
256
257       If  wait_cond  is set to FI_CQ_COND_NONE, then no additional conditions
258       are applied to the signaling of the CQ wait object, and  the  insertion
259       of  any new entry will trigger the wait condition.  If wait_cond is set
260       to FI_CQ_COND_THRESHOLD, then the cond field is interpreted as a size_t
261       threshold  value.   The  threshold indicates the number of entries that
262       are to be queued before at the CQ before the wait is satisfied.
263
264       This field is ignored if wait_obj is set to FI_WAIT_NONE.
265
266       wait_set
267              If wait_obj is FI_WAIT_SET, this field references a wait  object
268              to  which  the completion queue should attach.  When an event is
269              inserted into the completion queue, the corresponding  wait  set
270              will  be  signaled if all necessary conditions are met.  The use
271              of a wait_set enables an optimized method of waiting for  events
272              across  multiple event and completion queues.  This field is ig‐
273              nored if wait_obj is not FI_WAIT_SET.
274
275   fi_close
276       The fi_close call releases all resources associated with  a  completion
277       queue.   Any  completions  which remain on the CQ when it is closed are
278       lost.
279
280       When closing the CQ, there must be no opened endpoints,  transmit  con‐
281       texts,  or  receive  contexts associated with the CQ.  If resources are
282       still associated with the CQ when attempting to close,  the  call  will
283       return -FI_EBUSY.
284
285   fi_control
286       The  fi_control  call is used to access provider or implementation spe‐
287       cific details of the completion queue.  Access to the CQ should be  se‐
288       rialized  across  all calls when fi_control is invoked, as it may redi‐
289       rect the implementation of CQ operations.  The following  control  com‐
290       mands are usable with a CQ.
291
292       FI_GETWAIT (void **)
293              This  command allows the user to retrieve the low-level wait ob‐
294              ject associated with the CQ.  The format of the  wait-object  is
295              specified  during  CQ  creation, through the CQ attributes.  The
296              fi_control arg parameter should be an address where a pointer to
297              the returned wait object will be written.  See fi_eq.3 for addi‐
298              tion details using fi_control with FI_GETWAIT.
299
300   fi_cq_read
301       The fi_cq_read operation performs a non-blocking read of completion da‐
302       ta from the CQ.  The format of the completion event is determined using
303       the fi_cq_format option that was specified  when  the  CQ  was  opened.
304       Multiple  completions may be retrieved from a CQ in a single call.  The
305       maximum number of entries to return is limited to the  specified  count
306       parameter, with the number of entries successfully read from the CQ re‐
307       turned by the call.  (See return values section below.) A  count  value
308       of  0 may be used to drive progress on associated endpoints when manual
309       progress is enabled.
310
311       CQs are optimized to report operations which have completed successful‐
312       ly.  Operations which fail are reported `out of band'.  Such operations
313       are retrieved using the fi_cq_readerr function.  When an operation that
314       has completed with an unexpected error is encountered, it is placed in‐
315       to a temporary error queue.  Attempting to read from a CQ while an item
316       is  in the error queue results in fi_cq_read failing with a return code
317       of -FI_EAVAIL.  Applications may use this return code to determine when
318       to call fi_cq_readerr.
319
320   fi_cq_readfrom
321       The  fi_cq_readfrom  call behaves identical to fi_cq_read, with the ex‐
322       ception that it allows the CQ to return source address  information  to
323       the  user for any received data.  Source address data is only available
324       for  those  endpoints  configured  with   FI_SOURCE   capability.    If
325       fi_cq_readfrom is called on an endpoint for which source addressing da‐
326       ta is not available, the source address  will  be  set  to  FI_ADDR_NO‐
327       TAVAIL.   The  number of input src_addr entries must be the same as the
328       count parameter.
329
330       Returned source addressing data is converted from  the  native  address
331       used  by  the underlying fabric into an fi_addr_t, which may be used in
332       transmit operations.  Under most circumstances, returning fi_addr_t re‐
333       quires  that the source address already have been inserted into the ad‐
334       dress vector associated with the receiving endpoint.  This is true  for
335       address   vectors  of  type  FI_AV_TABLE.   In  select  providers  when
336       FI_AV_MAP is used, source addresses may  be  converted  algorithmically
337       into  a  usable  fi_addr_t, even though the source address has not been
338       inserted into the address vector.  This is permitted by the API, as  it
339       allows the provider to avoid address look-up as part of receive message
340       processing.  In no case do providers insert addresses into an AV  sepa‐
341       rate from an application calling fi_av_insert or similar call.
342
343       For  endpoints  allocated  using  the  FI_SOURCE_ERR capability, if the
344       source address cannot  be  converted  into  a  valid  fi_addr_t  value,
345       fi_cq_readfrom  will  return -FI_EAVAIL, even if the data were received
346       successfully.  The completion will then be reported through fi_cq_read‐
347       err with error code -FI_EADDRNOTAVAIL.  See fi_cq_readerr for details.
348
349       If FI_SOURCE is specified without FI_SOURCE_ERR, source addresses which
350       cannot be mapped to a usable fi_addr_t will be reported as  FI_ADDR_NO‐
351       TAVAIL.
352
353   fi_cq_sread / fi_cq_sreadfrom
354       The  fi_cq_sread  and fi_cq_sreadfrom calls are the blocking equivalent
355       operations to fi_cq_read and fi_cq_readfrom.  Their behavior is similar
356       to  the  non-blocking calls, with the exception that the calls will not
357       return until either a completion has been read from the CQ or an  error
358       or timeout occurs.
359
360       Threads blocking in this function will return to the caller if they are
361       signaled by some external source.  This is true even if the timeout has
362       not occurred or was specified as infinite.
363
364       It  is  invalid  for applications to call these functions if the CQ has
365       been configured with a wait object of FI_WAIT_NONE or FI_WAIT_SET.
366
367   fi_cq_readerr
368       The read error function, fi_cq_readerr, retrieves information regarding
369       any  asynchronous  operation which has completed with an unexpected er‐
370       ror.  fi_cq_readerr  is  a  non-blocking  call,  returning  immediately
371       whether an error completion was found or not.
372
373       Error  information is reported to the user through struct fi_cq_err_en‐
374       try.  The format of this structure is defined below.
375
376              struct fi_cq_err_entry {
377                  void     *op_context; /* operation context */
378                  uint64_t flags;       /* completion flags */
379                  size_t   len;         /* size of received data */
380                  void     *buf;        /* receive data buffer */
381                  uint64_t data;        /* completion data */
382                  uint64_t tag;         /* message tag */
383                  size_t   olen;        /* overflow length */
384                  int      err;         /* positive error code */
385                  int      prov_errno;  /* provider error code */
386                  void    *err_data;    /*  error data */
387                  size_t   err_data_size; /* size of err_data */
388              };
389
390       The general reason for the error is provided  through  the  err  field.
391       Provider  specific  error information may also be available through the
392       prov_errno and err_data fields.  Users may call fi_cq_strerror to  con‐
393       vert  provider  specific  error information into a printable string for
394       debugging purposes.  See field details below for  more  information  on
395       the use of err_data and err_data_size.
396
397       Note that error completions are generated for all operations, including
398       those for which a completion was not  requested  (e.g. an  endpoint  is
399       configured  with  FI_SELECTIVE_COMPLETION, but the request did not have
400       the FI_COMPLETION flag set).  In such cases, providers will  return  as
401       much information as made available by the underlying software and hard‐
402       ware about the failure, other fields will be set to NULL  or  0.   This
403       includes  the op_context value, which may not have been provided or was
404       ignored on input as part of the transfer.
405
406       Notable completion error codes are given below.
407
408       FI_EADDRNOTAVAIL
409              This error code is used by CQs configured with FI_SOURCE_ERR  to
410              report  completions  for which a usable fi_addr_t source address
411              could not be found.  An error code of FI_EADDRNOTAVAIL indicates
412              that  the data transfer was successfully received and processed,
413              with the fi_cq_err_entry fields containing information about the
414              completion.   The  err_data  field will be set to the source ad‐
415              dress data.  The source address will be in the  same  format  as
416              specified  through  the fi_info addr_format field for the opened
417              domain.  This may be passed directly into an  fi_av_insert  call
418              to add the source address to the address vector.
419
420   fi_cq_signal
421       The fi_cq_signal call will unblock any thread waiting in fi_cq_sread or
422       fi_cq_sreadfrom.  This may be used to wake-up a thread that is  blocked
423       waiting  to read a completion operation.  The fi_cq_signal operation is
424       only available if the CQ was configured with a wait object.
425

COMPLETION FIELDS

427       The CQ entry data structures share many of the same fields.  The  mean‐
428       ings of these fields are the same for all CQ entry structure formats.
429
430       op_context
431              The operation context is the application specified context value
432              that was provided with an asynchronous operation.   The  op_con‐
433              text field is valid for all completions that are associated with
434              an asynchronous operation.
435
436       For completion events that are not associated with a posted  operation,
437       this field will be set to NULL.  This includes completions generated at
438       the target in response to RMA  write  operations  that  carry  CQ  data
439       (FI_REMOTE_WRITE | FI_REMOTE_CQ_DATA flags set), when the FI_RX_CQ_DATA
440       mode bit is not required.
441
442       flags  This specifies flags associated with  the  completed  operation.
443              The  Completion  Flags  section  below  lists valid flag values.
444              Flags are set for all relevant completions.
445
446       len    This len field only  applies  to  completed  receive  operations
447              (e.g. fi_recv,  fi_trecv,  etc.).   It indicates the size of re‐
448              ceived message data – i.e. how many data bytes were placed  into
449              the    associated    receive    buffer    by   a   corresponding
450              fi_send/fi_tsend/et al call.  If an endpoint has been configured
451              with  the  FI_MSG_PREFIX mode, the len also reflects the size of
452              the prefix buffer.
453
454       buf    The buf field is only valid for  completed  receive  operations,
455              and  only  applies  when  the receive buffer was posted with the
456              FI_MULTI_RECV flag.  In this case, buf points  to  the  starting
457              location where the receive data was placed.
458
459       data   The data field is only valid if the FI_REMOTE_CQ_DATA completion
460              flag is set, and only applies to receive completions.  If FI_RE‐
461              MOTE_CQ_DATA is set, this field will contain the completion data
462              provided by the peer as part of  their  transmit  request.   The
463              completion data will be given in host byte order.
464
465       tag    A  tag  applies  only  to received messages that occur using the
466              tagged interfaces.  This field contains the tag that was includ‐
467              ed  with the received message.  The tag will be in host byte or‐
468              der.
469
470       olen   The olen field applies to received messages.  It is used to  in‐
471              dicate  that a received message has overrun the available buffer
472              space and has been truncated.  The olen specifies the amount  of
473              data  that did not fit into the available receive buffer and was
474              discarded.
475
476       err    This err code is a positive fabric errno associated with a  com‐
477              pletion.   The err value indicates the general reason for an er‐
478              ror, if one occurred.  See fi_errno.3 for a list of possible er‐
479              ror codes.
480
481       prov_errno
482              On  an  error,  prov_errno may contain a provider specific error
483              code.  The use of this field and its meaning is provider specif‐
484              ic.   It  is  intended  to  be  used  as  a  debugging aid.  See
485              fi_cq_strerror for additional details on converting  this  error
486              value into a human readable string.
487
488       err_data
489              The  err_data field is used to return provider specific informa‐
490              tion, if available, about the error.  On input, err_data  should
491              reference  a  data buffer of size err_data_size.  On output, the
492              provider will fill in this buffer with any provider specific da‐
493              ta which may help identify the cause of the error.  The contents
494              of the err_data field and its meaning is provider specific.   It
495              is  intended  to be used as a debugging aid.  See fi_cq_strerror
496              for additional details on converting this error data into a  hu‐
497              man  readable  string.   See the compatibility note below on how
498              this field is used for older libfabric releases.
499
500       err_data_size
501              On input, err_data_size indicates the size of the err_data  buf‐
502              fer  in bytes.  On output, err_data_size will be set to the num‐
503              ber of bytes copied to the err_data buffer.  The err_data infor‐
504              mation  is typically used with fi_cq_strerror to provide details
505              about the type of error that occurred.
506
507       For compatibility purposes, the behavior of the  err_data  and  err_da‐
508       ta_size  fields  is may be modified from that listed above.  If err_da‐
509       ta_size is 0 on input, or the fabric was opened  with  release  <  1.5,
510       then  any  buffer  referenced by err_data will be ignored on input.  In
511       this situation, on output err_data will be set to a data  buffer  owned
512       by  the provider.  The contents of the buffer will remain valid until a
513       subsequent read call against the CQ.  Applications must  serialize  ac‐
514       cess  to the CQ when processing errors to ensure that the buffer refer‐
515       enced by err_data does not change.
516

COMPLETION FLAGS

518       Completion flags provide additional details regarding the completed op‐
519       eration.  The following completion flags are defined.
520
521       FI_SEND
522              Indicates  that  the  completion was for a send operation.  This
523              flag may be combined with an FI_MSG or FI_TAGGED flag.
524
525       FI_RECV
526              Indicates that the completion was for a receive operation.  This
527              flag may be combined with an FI_MSG or FI_TAGGED flag.
528
529       FI_RMA Indicates  that  an  RMA  operation completed.  This flag may be
530              combined with an FI_READ, FI_WRITE,  FI_REMOTE_READ,  or  FI_RE‐
531              MOTE_WRITE flag.
532
533       FI_ATOMIC
534              Indicates  that an atomic operation completed.  This flag may be
535              combined with an FI_READ, FI_WRITE,  FI_REMOTE_READ,  or  FI_RE‐
536              MOTE_WRITE flag.
537
538       FI_MSG Indicates  that  a message-based operation completed.  This flag
539              may be combined with an FI_SEND or FI_RECV flag.
540
541       FI_TAGGED
542              Indicates that a tagged message operation completed.  This  flag
543              may be combined with an FI_SEND or FI_RECV flag.
544
545       FI_MULTICAST
546              Indicates  that  a multicast operation completed.  This flag may
547              be combined with FI_MSG and relevant flags.  This flag  is  only
548              guaranteed to be valid for received messages if the endpoint has
549              been configured with FI_SOURCE.
550
551       FI_READ
552              Indicates that a locally initiated RMA or atomic read  operation
553              has  completed.   This  flag  may  be combined with an FI_RMA or
554              FI_ATOMIC flag.
555
556       FI_WRITE
557              Indicates that a locally initiated RMA or atomic write operation
558              has  completed.   This  flag  may  be combined with an FI_RMA or
559              FI_ATOMIC flag.
560
561       FI_REMOTE_READ
562              Indicates that a remotely initiated RMA or atomic read operation
563              has  completed.   This  flag  may  be combined with an FI_RMA or
564              FI_ATOMIC flag.
565
566       FI_REMOTE_WRITE
567              Indicates that a remotely initiated RMA or atomic  write  opera‐
568              tion has completed.  This flag may be combined with an FI_RMA or
569              FI_ATOMIC flag.
570
571       FI_REMOTE_CQ_DATA
572              This indicates that remote CQ data is available as part  of  the
573              completion.
574
575       FI_MULTI_RECV
576              This  flag  applies to receive buffers that were posted with the
577              FI_MULTI_RECV flag set.  This completion flag indicates that the
578              original  receive  buffer  referenced by the completion has been
579              consumed and was released by the provider.   Providers  may  set
580              this  flag  on the last message that is received into the multi-
581              recv buffer, or may generate a separate  completion  that  indi‐
582              cates that the buffer has been released.
583
584       Applications  can  distinguish between these two cases by examining the
585       completion entry flags field.  If additional flags,  such  as  FI_RECV,
586       are set, the completion is associated with a received message.  In this
587       case, the buf field will reference the location where the received mes‐
588       sage  was  placed into the multi-recv buffer.  Other fields in the com‐
589       pletion entry will be determined based on  the  received  message.   If
590       other flag bits are zero, the provider is reporting that the multi-recv
591       buffer has been released, and the completion entry  is  not  associated
592       with a received message.
593
594       FI_MORE
595              See  the  `Buffered  Receives' section in fi_msg(3) for more de‐
596              tails.  This flag is associated with receive completions on end‐
597              points  that  have  FI_BUFFERED_RECV  mode enabled.  When set to
598              one, it indicates that the buffer referenced by  the  completion
599              is limited by the FI_OPT_BUFFERED_LIMIT threshold, and addition‐
600              al message data must be retrieved by the  application  using  an
601              FI_CLAIM operation.
602
603       FI_CLAIM
604              See  the  `Buffered  Receives' section in fi_msg(3) for more de‐
605              tails.  This flag is set on completions associated with  receive
606              operations  that  claim  buffered  receive data.  Note that this
607              flag   only   applies   to   endpoints   configured   with   the
608              FI_BUFFERED_RECV mode bit.
609

COMPLETION EVENT SEMANTICS

611       Libfabric  defines several completion `levels', identified using opera‐
612       tional flags.  Each flag indicates the soonest that a completion  event
613       may be generated by a provider, and the assumptions that an application
614       may make upon processing a completion.  The operational flags  are  de‐
615       fined  below,  along  with an example of how a provider might implement
616       the semantic.  Note that only meeting the semantic is required  of  the
617       provider  and not the implementation.  Providers may implement stronger
618       completion semantics than necessary for a given operation, but only the
619       behavior defined by the completion level is guaranteed.
620
621       To  help  understand  the  conceptual differences in completion levels,
622       consider mailing a letter.  Placing the letter into the  local  mailbox
623       for  pick-up is similar to `inject complete'.  Having the letter picked
624       up and dropped off at the destination mailbox is equivalent to  `trans‐
625       mit  complete'.  The `delivery complete' semantic is a stronger guaran‐
626       tee, with a person at the destination signing for the letter.  However,
627       the  person  who  signed for the letter is not necessarily the intended
628       recipient.  The `match complete' option is  similar  to  delivery  com‐
629       plete, but requires the intended recipient to sign for the letter.
630
631       The `commit complete' level has different semantics than the previously
632       mentioned levels.  Commit complete would be closer to the letter arriv‐
633       ing at the destination and being placed into a fire proof safe.
634
635       The  operational  flags for the described completion levels are defined
636       below.
637
638       FI_INJECT_COMPLETE
639              Indicates that a completion should be generated when the  source
640              buffer(s)  may be reused.  A completion guarantees that the buf‐
641              fers will not be read from again and the application may reclaim
642              them.  No other guarantees are made with respect to the state of
643              the operation.
644
645       Example: A provider may generate this completion  event  after  copying
646       the  source  buffer  into a network buffer, either in host memory or on
647       the NIC.  An inject completion does not indicate that the data has been
648       transmitted  onto  the network, and a local error could occur after the
649       completion event has been generated that could prevent  it  from  being
650       transmitted.
651
652       Inject  complete  allows  for  the  fastest  completion reporting (and,
653       hence, buffer reuse), but provides the weakest guarantees against  net‐
654       work errors.
655
656       Note:  This flag is used to control when a completion entry is inserted
657       into a completion queue.  It does not apply to operations that  do  not
658       generate a completion queue entry, such as the fi_inject operation, and
659       is not subject to the inject_size message limit restriction.
660
661       FI_TRANSMIT_COMPLETE
662              Indicates that a completion should be generated when the  trans‐
663              mit operation has completed relative to the local provider.  The
664              exact behavior is dependent on the endpoint type.
665
666       For reliable endpoints:
667
668       Indicates that a completion should be generated when the operation  has
669       been  delivered to the peer endpoint.  A completion guarantees that the
670       operation is no longer dependent on the fabric or local resources.  The
671       state of the operation at the peer endpoint is not defined.
672
673       Example: A provider may generate a transmit complete event upon receiv‐
674       ing an ack from the peer endpoint.  The state of  the  message  at  the
675       peer  is  unknown and may be buffered in the target NIC at the time the
676       ack has been generated.
677
678       For unreliable endpoints:
679
680       Indicates that a completion should be generated when the operation  has
681       been  delivered to the fabric.  A completion guarantees that the opera‐
682       tion is no longer dependent on local resources.  The state of the oper‐
683       ation within the fabric is not defined.
684
685       FI_DELIVERY_COMPLETE
686              Indicates that a completion should not be generated until an op‐
687              eration has been processed by the  destination  endpoint(s).   A
688              completion guarantees that the result of the operation is avail‐
689              able; however, additional steps may need to be taken at the des‐
690              tination  to  retrieve the results.  For example, an application
691              may need to provide a receive buffers in order to retrieve  mes‐
692              sages that were buffered by the provider.
693
694       Delivery  complete indicates that the message has been processed by the
695       peer.  If an application buffer was ready to receive the results of the
696       message when it arrived, then delivery complete indicates that the data
697       was placed into the application’s buffer.
698
699       This completion mode applies only to reliable  endpoints.   For  opera‐
700       tions  that  return  data to the initiator, such as RMA read or atomic-
701       fetch, the source endpoint is also considered a  destination  endpoint.
702       This is the default completion mode for such operations.
703
704       FI_MATCH_COMPLETE
705              Indicates  that  a completion should be generated only after the
706              operation has been matched with an application specified buffer.
707              Operations  using  this completion semantic are dependent on the
708              application at the target claiming the message or results.  As a
709              result, match complete may involve additional provider level ac‐
710              knowledgements or lengthy delays.  However, this completion mod‐
711              el  enables  peer  applications  to synchronize their execution.
712              Many providers may not support this semantic.
713
714       FI_COMMIT_COMPLETE
715              Indicates that a completion should not be generated (locally  or
716              at  the  peer)  until  the result of an operation have been made
717              persistent.  A completion guarantees that  the  result  is  both
718              available and durable, in the case of power failure.
719
720       This  completion mode applies only to operations that target persistent
721       memory regions over reliable endpoints.  This completion mode is exper‐
722       imental.
723
724       FI_FENCE
725              This  is not a completion level, but plays a role in the comple‐
726              tion ordering between operations that would not normally be  or‐
727              dered.   An  operation that is marked with the FI_FENCE flag and
728              all operations posted after the fenced  operation  are  deferred
729              until  all  previous operations targeting the same peer endpoint
730              have completed.  Additionally, the completion of the fenced  op‐
731              eration  indicates  that prior operations have met the same com‐
732              pletion level as the fenced operation.  For example, if an oper‐
733              ation  is  posted  as  FI_DELIVERY_COMPLETE | FI_FENCE, then its
734              completion indicates prior operations have met the semantic  re‐
735              quired for FI_DELIVERY_COMPLETE.  This is true even if the prior
736              operation was posted with a  lower  completion  level,  such  as
737              FI_TRANSMIT_COMPLETE or FI_INJECT_COMPLETE.
738
739       Note  that  a completion generated for an operation posted prior to the
740       fenced operation only guarantees that the  completion  level  that  was
741       originally  requested has been met.  It is the completion of the fenced
742       operation that guarantees that the additional semantics have been met.
743
744       The above completion semantics are defined with respect to the  initia‐
745       tor  of the operation.  The different semantics are useful for describ‐
746       ing when the initiator may re-use a data buffer,  and  guarantees  what
747       state  a  transfer  must  reach  prior to a completion being generated.
748       This allows applications to determine  appropriate  error  handling  in
749       case of communication failures.
750

TARGET COMPLETION SEMANTICS

752       The completion semantic at the target is used to determine when data at
753       the target is visible to the peer  application.   Visibility  indicates
754       that  a  memory  read to the same address that was the target of a data
755       transfer will return the results of the  transfer.   The  target  of  a
756       transfer can be identified by the initiator, as may be the case for RMA
757       and atomic operations, or determined by the target, for example by pro‐
758       viding a matching receive buffer.  Global visibility indicates that the
759       results are available regardless of where the memory  read  originates.
760       For  example, the read could come from a process running on a host CPU,
761       it may be accessed by subsequent data transfer over the fabric, or read
762       from a peer device such as a GPU.
763
764       In terms of completion semantics, visibility usually indicates that the
765       transfer meets the FI_DELIVERY_COMPLETE requirements from the  perspec‐
766       tive  of the target.  The target completion semantic may be, but is not
767       necessarily, linked with the completion semantic specified by the  ini‐
768       tiator of the transfer.
769
770       Often,  target  processes  do not explicitly state a desired completion
771       semantic and instead rely on the default semantic.  The default  behav‐
772       ior is based on several factors, including:
773
774       • whether a completion even is generated at the target
775
776       • the type of transfer involved (e.g. msg vs RMA)
777
778       • endpoint data and message ordering guarantees
779
780       • properties of the targeted memory buffer
781
782       • the initiator’s specified completion semantic
783
784       Broadly,  target  completion  semantics are grouped based on whether or
785       not the transfer generates a completion event at the target.  This  in‐
786       cludes  writing a CQ entry or updating a completion counter.  In common
787       use cases, transfers that use a message interface (FI_MSG or FI_TAGGED)
788       typically  generate target events, while transfers involving an RMA in‐
789       terface (FI_RMA or FI_ATOMIC) often do not.  There  are  exceptions  to
790       both  these cases, depending on endpoint to CQ and counter bindings and
791       operational flags.  For example, RMA writes that carry remote  CQ  data
792       will generate a completion event at the target, and are frequently used
793       to convey visibility to the target application.  The general guidelines
794       for  target  side semantics are described below, followed by exceptions
795       that modify that behavior.
796
797       By default, completions generated  at  the  target  indicate  that  the
798       transferred  data  is  immediately available to be read from the target
799       buffer.  That is, the target sees FI_DELIVERY_COMPLETE (or better)  se‐
800       mantics, even if the initiator requested lower semantics.  For applica‐
801       tions using only data buffers allocated from host memory, this is often
802       sufficient.
803
804       For  operations  that do not generate a completion event at the target,
805       the visibility of the data at the target may need to be inferred  based
806       on subsequent operations that do generate target completions.  Absent a
807       target completion, when a completion of an operation is written at  the
808       initiator,  the  visibility  semantic  of  the  operation at the target
809       aligns with the initiator completion semantic.  For instance, if an RMA
810       operation  completes  at  the initiator as either FI_INJECT_COMPLETE or
811       FI_TRANSMIT_COMPLETE, the data visibility at the target is not  guaran‐
812       teed.
813
814       One  or  more  of  the  following  mechanisms can be used by the target
815       process to guarantee that the results of a data transfer that  did  not
816       generate  a  completion at the target is now visible.  This list is not
817       inclusive of all options, but defines common uses.  In the descriptions
818       below,  the first transfer does not result in a completion event at the
819       target, but is eventually followed by a transfer which does.
820
821       • If the endpoint guarantees message ordering  between  two  transfers,
822         the target completion of a second transfer will indicate that the da‐
823         ta from the first transfer is available.  For example,  if  the  end‐
824         point  supports  send after write ordering (FI_ORDER_SAW), then a re‐
825         ceive completion corresponding to the send  will  indicate  that  the
826         write  data  is available.  This holds independent of the initiator’s
827         completion semantic for either the write or send.  When  ordering  is
828         guaranteed, the second transfer can be queued with the provider imme‐
829         diately after queuing the first.
830
831       • If the endpoint does not guarantee message  ordering,  the  initiator
832         must  take  additional  steps to ensure visibility.  If initiator re‐
833         quests FI_DELIVERY_COMPLETE semantics for the  first  operation,  the
834         initiator  can  wait for the operation to complete locally.  Once the
835         completion has been read, the target completion of a second  transfer
836         will indicate that the first transfer’s data is visible.
837
838       • Alternatively, if message ordering is not guaranteed by the endpoint,
839         the initiator can use the FI_FENCE and FI_DELIVERY_COMPLETE flags  on
840         the  second  data  transfer  to force the first transfers to meet the
841         FI_DELIVERY_COMPLETE semantics.  If the second transfer  generates  a
842         completion  at  the target, that will indicate that the data is visi‐
843         ble.  Otherwise, a target  completion  for  any  transfer  after  the
844         fenced operation will indicate that the data is visible.
845
846       The above semantics apply for transfers targeting traditional host mem‐
847       ory buffers.  However, the  behavior  may  differ  when  device  memory
848       and/or  persistent  memory  is involved (FI_HMEM and FI_PMEM capability
849       bits).  When heterogenous memory is involved, the concept of memory do‐
850       mains  come into play.  Memory domains identify the physical separation
851       of memory, which may or may not be accessible through the same  virtual
852       address space.  See the fi_mr(3) man page for further details on memory
853       domains.
854
855       Completion ordering and  data  visibility  are  only  well-defined  for
856       transfers  that target the same memory domain.  Applications need to be
857       aware of ordering and visibility differences when transfers target dif‐
858       ferent memory domains.  Additionally, applications also need to be con‐
859       cerned with the memory domain that completions themselves  are  written
860       and  if  it  differs from the memory domain targeted by a transfer.  In
861       some situations, either the provider or application may  need  to  call
862       device  specific  APIs  to synchronize or flush device memory caches in
863       order to achieve the desired data visibility.
864
865       When heterogenous memory is in use, the default target  completion  se‐
866       mantic  for transfers that generate a completion at the target is still
867       FI_DELIVERY_COMPLETE, however, applications should be aware that  there
868       may  be  a negative impact on overall performance for providers to meet
869       this requirement.
870
871       For example, a target process may be using a GPU to accelerate computa‐
872       tions.   A memory region mapping to memory on the GPU may be exposed to
873       peers as either an RMA target or posted locally as  a  receive  buffer.
874       In  this  case,  the application is concerned with two memory domains –
875       system and GPU memory.  Completions are written to system memory.
876
877       Continuing the example, a peer process sends a  tagged  message.   That
878       message  is matched with the receive buffer located in GPU memory.  The
879       NIC copies the data from the network into the receive buffer and writes
880       an entry into the completion queue.  Note that both memory domains were
881       accessed as part of this transfer.  The message data  was  directed  to
882       the  GPU memory, but the completion went to host memory.  Because sepa‐
883       rate memory domains may not be synchronized with each other, it is pos‐
884       sible  for  the host CPU to see and process the completion entry before
885       the transfer to the GPU memory is visible to either  the  host  GPU  or
886       even  software  running  on  the  GPU.   From  the  perspective  of the
887       provider, visibility of the completion does not imply visibility of da‐
888       ta written to the GPU’s memory domain.
889
890       The  default  completion semantic at the target application for message
891       operations is FI_DELIVERY_COMPLETE.  An anticipated provider  implemen‐
892       tation  in  this  situation is for the provider software running on the
893       host CPU to intercept the CQ entry, detect that the data landed in het‐
894       erogenous  memory,  and perform the necessary device synchronization or
895       flush operation before reporting the completion up to the  application.
896       This  ensures that the data is visible to CPU and GPU software prior to
897       the application processing the completion.
898
899       In addition to the cost of provider software  intercepting  completions
900       and  checking  if  a transfer targeted heterogenous memory, device syn‐
901       chronization itself may impact performance.  As a result,  applications
902       can  request  a  lower completion semantic when posting receives.  That
903       indicates to the provider that the application will be responsible  for
904       handling  any  device  specific  flush operations that might be needed.
905       See fi_msg(3) FLAGS.
906
907       For data transfers that do not generate a  completion  at  the  target,
908       such  as RMA or atomics, it is the responsibility of the application to
909       ensure that all target buffers meet the necessary  visibility  require‐
910       ments  of  the  application.  The previously mentioned bulleted methods
911       for notifying the target that the data is visible  may  not  be  suffi‐
912       cient,  as  the  provider software at the target could lack the context
913       needed to ensure visibility.  This implies  that  the  application  may
914       need to call device synchronization/flush APIs directly.
915
916       For  example,  a peer application could perform several RMA writes that
917       target GPU memory buffers.  If the provider offloads RMA operations in‐
918       to  the  NIC,  the provider software at the target will be unaware that
919       the RMA operations have occurred.  If the peer sends a message  to  the
920       target application that indicates that the RMA operations are done, the
921       application must ensure that the RMA data is visible to the host CPU or
922       GPU prior to executing code that accesses the data.  The target comple‐
923       tion of having received the sent message is  not  sufficient,  even  if
924       send-after-write ordering is supported.
925
926       Most  target  heterogenous memory completion semantics map to FI_TRANS‐
927       MIT_COMPLETE or FI_DELIVERY_COMPLETE.  Persistent memory (FI_PMEM capa‐
928       bility),  however,  is  often  used  with FI_COMMIT_COMPLETE semantics.
929       Heterogenous completion concepts still apply.
930
931       For transfers flagged by the initiator with FI_COMMIT_COMPLETE, a  com‐
932       pletion  at  the  target  indicates  that  the  results are visible and
933       durable.  For transfers targeting persistent memory, but using  a  dif‐
934       ferent completion semantic at the initiator, the visibility at the tar‐
935       get is similar to that described above.  Durability is only  associated
936       with transfers marked with FI_COMMIT_COMPLETE.
937
938       For transfers targeting persistent memory that request FI_DELIVERY_COM‐
939       PLETE, then a completion, at either the initiator or target,  indicates
940       that the data is visible.  Visibility at the target can be conveyed us‐
941       ing one of the above describe mechanism – generating a  target  comple‐
942       tion,  sending  a  message  from the initiator, etc.  Similarly, if the
943       initiator requested FI_TRANSMIT_COMPLETE,  then  additional  steps  are
944       needed  to  ensure visibility at the target.  For example, the transfer
945       can generate a completion at the target, which would indicate visibili‐
946       ty,  but  not  durability.   The initiator can also follow the transfer
947       with another operation that forces visibility, such as  using  FI_FENCE
948       in conjunction with FI_DELIVERY_COMPLETE.
949

NOTES

951       A  completion  queue must be bound to at least one enabled endpoint be‐
952       fore any operation such  as  fi_cq_read,  fi_cq_readfrom,  fi_cq_sread,
953       fi_cq_sreadfrom etc.  can be called on it.
954
955       Completion flags may be suppressed if the FI_NOTIFY_FLAGS_ONLY mode bit
956       has been set.  When enabled, only the following flags are guaranteed to
957       be  set  in  completion  data  when  they are valid: FI_REMOTE_READ and
958       FI_REMOTE_WRITE (when FI_RMA_EVENT capability bit has been set), FI_RE‐
959       MOTE_CQ_DATA, and FI_MULTI_RECV.
960
961       If  a  completion  queue  has  been  overrun, it will be placed into an
962       `overrun' state.  Read operations will continue to  return  any  valid,
963       non-corrupted  completions,  if available.  After all valid completions
964       have been retrieved, any attempt to read the CQ will result in  it  re‐
965       turning an FI_EOVERRUN error event.  Overrun completion queues are con‐
966       sidered fatal and may not be used to report additional completions once
967       the overrun occurs.
968

RETURN VALUES

970   fi_cq_open / fi_cq_signal
971       : Returns 0 on success.  On error, returns a negative fabric errno.
972
973   fi_cq_read / fi_cq_readfrom
974       : On success, returns the number of completions retrieved from the com‐
975       pletion queue.  On error, returns a negative fabric errno,  with  these
976       two  errors  explicitly  identified: If no completions are available to
977       read from the CQ, returns -FI_EAGAIN.  If the topmost completion is for
978       a failed transfer (an error entry), returns -FI_EAVAIL.
979
980   fi_cq_sread / fi_cq_sreadfrom
981       : On success, returns the number of completions retrieved from the com‐
982       pletion queue.  On error, returns a negative fabric errno,  with  these
983       two errors explicitly identified: If the timeout expires or the calling
984       thread is signaled and no data is available to be read from the comple‐
985       tion  queue,  returns  -FI_EAGAIN.   If the topmost completion is for a
986       failed transfer (an error entry), returns -FI_EAVAIL.
987
988   fi_cq_readerr
989       : On success, returns the positive value 1 (number of error entries re‐
990       turned).   On  error,  returns a negative fabric errno, with this error
991       explicitly identified: If no error completions are  available  to  read
992       from the CQ, returns -FI_EAGAIN.
993
994   fi_cq_strerror
995       :  Returns  a  character string interpretation of the provider specific
996       error returned with a completion.
997
998       Fabric errno values are defined in rdma/fi_errno.h.
999

SEE ALSO

1001       fi_getinfo(3),  fi_endpoint(3),  fi_domain(3),  fi_eq(3),   fi_cntr(3),
1002       fi_poll(3)
1003

AUTHORS

1005       OpenFabrics.
1006
1007
1008
1009Libfabric Programmer’s Manual     2022-12-11                          fi_cq(3)
Impressum