1fi_atomic(3)                   Libfabric v1.6.1                   fi_atomic(3)
2
3
4

NAME

6       fi_atomic - Remote atomic functions
7
8       fi_atomic / fi_atomicv / fi_atomicmsg / fi_inject_atomic : Initiates an
9       atomic operation to remote memory
10
11       fi_fetch_atomic / fi_fetch_atomicv / fi_fetch_atomicmsg : Initiates  an
12       atomic operation to remote memory, retrieving the initial value.
13
14       fi_compare_atomic  / fi_compare_atomicv / fi_compare_atomicmsg : Initi‐
15       ates an atomic compare-operation to remote memory, retrieving the  ini‐
16       tial value.
17
18       fi_atomicvalid   /   fi_fetch_atomicvalid  /  fi_compare_atomicvalid  /
19       fi_query_atomic : Indicates if a provider supports  a  specific  atomic
20       operation
21

SYNOPSIS

23              #include <rdma/fi_atomic.h>
24
25              ssize_t fi_atomic(struct fid_ep *ep, const void *buf,
26                  size_t count, void *desc, fi_addr_t dest_addr,
27                  uint64_t addr, uint64_t key,
28                  enum fi_datatype datatype, enum fi_op op, void *context);
29
30              ssize_t fi_atomicv(struct fid_ep *ep, const struct fi_ioc *iov,
31                  void **desc, size_t count, fi_addr_t dest_addr,
32                  uint64_t addr, uint64_t key,
33                  enum fi_datatype datatype, enum fi_op op, void *context);
34
35              ssize_t fi_atomicmsg(struct fid_ep *ep, const struct fi_msg_atomic *msg,
36                  uint64_t flags);
37
38              ssize_t fi_inject_atomic(struct fid_ep *ep, const void *buf,
39                  size_t count, fi_addr_t dest_addr,
40                  uint64_t addr, uint64_t key,
41                  enum fi_datatype datatype, enum fi_op op);
42
43              ssize_t fi_fetch_atomic(struct fid_ep *ep, const void *buf,
44                  size_t count, void *desc, void *result, void *result_desc,
45                  fi_addr_t dest_addr, uint64_t addr, uint64_t key,
46                  enum fi_datatype datatype, enum fi_op op, void *context);
47
48              ssize_t fi_fetch_atomicv(struct fid_ep *ep, const struct fi_ioc *iov,
49                  void **desc, size_t count, struct fi_ioc *resultv,
50                  void **result_desc, size_t result_count, fi_addr_t dest_addr,
51                  uint64_t addr, uint64_t key, enum fi_datatype datatype,
52                  enum fi_op op, void *context);
53
54              ssize_t fi_fetch_atomicmsg(struct fid_ep *ep,
55                  const struct fi_msg_atomic *msg, struct fi_ioc *resultv,
56                  void **result_desc, size_t result_count, uint64_t flags);
57
58              ssize_t fi_compare_atomic(struct fid_ep *ep, const void *buf,
59                  size_t count, void *desc, const void *compare,
60                  void *compare_desc, void *result, void *result_desc,
61                  fi_addr_t dest_addr, uint64_t addr, uint64_t key,
62                  enum fi_datatype datatype, enum fi_op op, void *context);
63
64              size_t fi_compare_atomicv(struct fid_ep *ep, const struct fi_ioc *iov,
65                     void **desc, size_t count, const struct fi_ioc *comparev,
66                     void **compare_desc, size_t compare_count, struct fi_ioc *resultv,
67                     void **result_desc, size_t result_count, fi_addr_t dest_addr,
68                     uint64_t addr, uint64_t key, enum fi_datatype datatype,
69                     enum fi_op op, void *context);
70
71              ssize_t fi_compare_atomicmsg(struct fid_ep *ep,
72                  const struct fi_msg_atomic *msg, const struct fi_ioc *comparev,
73                  void **compare_desc, size_t compare_count,
74                  struct fi_ioc *resultv, void **result_desc, size_t result_count,
75                  uint64_t flags);
76
77              int fi_atomicvalid(struct fid_ep *ep, enum fi_datatype datatype,
78                  enum fi_op op, size_t *count);
79
80              int fi_fetch_atomicvalid(struct fid_ep *ep, enum fi_datatype datatype,
81                  enum fi_op op, size_t *count);
82
83              int fi_compare_atomicvalid(struct fid_ep *ep, enum fi_datatype datatype,
84                  enum fi_op op, size_t *count);
85
86              int fi_query_atomic(struct fid_domain *domain,
87                  enum fi_datatype datatype, enum fi_op op,
88                  struct fi_atomic_attr *attr, uint64_t flags);
89

ARGUMENTS

91       ep : Fabric endpoint on which to initiate atomic operation.
92
93       buf  :  Local data buffer that specifies first operand of atomic opera‐
94       tion
95
96       iov / comparev / resultv : Vectored data buffer(s).
97
98       count / compare_count / result_count : Count of vectored data  entries.
99       The  number of elements referenced, where each element is the indicated
100       datatype.
101
102       addr : Address of remote memory to access.
103
104       key : Protection key associated with the remote memory.
105
106       datatype : Datatype associated with atomic operands
107
108       op : Atomic operation to perform
109
110       compare : Local compare buffer, containing comparison data.
111
112       result : Local data buffer to store initial value of remote buffer
113
114       desc / compare_desc / result_desc : Data descriptor associated with the
115       local  data  buffer,  local  compare  buffer,  and local result buffer,
116       respectively.
117
118       dest_addr : Destination address for connectionless  atomic  operations.
119       Ignored for connected endpoints.
120
121       msg : Message descriptor for atomic operations
122
123       flags : Additional flags to apply for the atomic operation
124
125       context : User specified pointer to associate with the operation.
126

DESCRIPTION

128       Atomic  transfers  are  used  to read and update data located in remote
129       memory regions in an atomic fashion.  Conceptually, they are similar to
130       local  atomic  operations  of a similar nature (e.g.  atomic increment,
131       compare and swap, etc.).  Updates to remote data involve one of several
132       operations  on  the  data, and act on specific types of data, as listed
133       below.  As such, atomic transfers have knowledge of the format  of  the
134       data  being  accessed.   A single atomic function may operate across an
135       array of data applying an atomic operation to each entry, but the atom‐
136       icity of an operation is limited to a single datatype or entry.
137
138   Atomic Data Types
139       Atomic  functions  may  operate on one of the following identified data
140       types.  A given atomic function may support any  datatype,  subject  to
141       provider implementation constraints.
142
143       FI_INT8 : Signed 8-bit integer.
144
145       FI_UINT8 : Unsigned 8-bit integer.
146
147       FI_INT16 : Signed 16-bit integer.
148
149       FI_UINT16 : Unsigned 16-bit integer.
150
151       FI_INT32 : Signed 32-bit integer.
152
153       FI_UINT32 : Unsigned 32-bit integer.
154
155       FI_INT64 : Signed 64-bit integer.
156
157       FI_UINT64 : Unsigned 64-bit integer.
158
159       FI_FLOAT : A single-precision floating point value (IEEE 754).
160
161       FI_DOUBLE : A double-precision floating point value (IEEE 754).
162
163       FI_FLOAT_COMPLEX  :  An ordered pair of single-precision floating point
164       values (IEEE 754), with the first value representing the  real  portion
165       of a complex number and the second representing the imaginary portion.
166
167       FI_DOUBLE_COMPLEX  : An ordered pair of double-precision floating point
168       values (IEEE 754), with the first value representing the  real  portion
169       of a complex number and the second representing the imaginary portion.
170
171       FI_LONG_DOUBLE : A double-extended precision floating point value (IEEE
172       754).  Note that the size of a long double and number of bits used  for
173       precision  is compiler, platform, and/or provider specific.  Developers
174       that use long double should ensure that libfabric is built using a long
175       double  format that is compatible with their application, and that for‐
176       mat is supported by the provider.  The mechanism used for this  valida‐
177       tion is currently beyond the scope of the libfabric API.
178
179       FI_LONG_DOUBLE_COMPLEX  :  An ordered pair of double-extended precision
180       floating point values (IEEE 754), with the first value representing the
181       real portion of a complex number and the second representing the imagi‐
182       nary portion.
183
184   Atomic Operations
185       The following atomic operations are defined.  An atomic operation often
186       acts  against  a  target  value  in the remote memory buffer and source
187       value provided with the atomic function.  It may also carry source data
188       to  replace the target value in compare and swap operations.  A concep‐
189       tual description of each operation is provided.
190
191       FI_MIN : Minimum
192
193              if (buf[i] < addr[i])
194                  addr[i] = buf[i]
195
196       FI_MAX : Maximum
197
198              if (buf[i] > addr[i])
199                  addr[i] = buf[i]
200
201       FI_SUM : Sum
202
203              addr[i] = addr[i] + buf[i]
204
205       FI_PROD : Product
206
207              addr[i] = addr[i] * buf[i]
208
209       FI_LOR : Logical OR
210
211              addr[i] = (addr[i] || buf[i])
212
213       FI_LAND : Logical AND
214
215              addr[i] = (addr[i] && buf[i])
216
217       FI_BOR : Bitwise OR
218
219              addr[i] = addr[i] | buf[i]
220
221       FI_BAND : Bitwise AND
222
223              addr[i] = addr[i] & buf[i]
224
225       FI_LXOR : Logical exclusive-OR (XOR)
226
227              addr[i] = ((addr[i] && !buf[i]) || (!addr[i] && buf[i]))
228
229       FI_BXOR : Bitwise exclusive-OR (XOR)
230
231              addr[i] = addr[i] ^ buf[i]
232
233       FI_ATOMIC_READ : Read data atomically
234
235              result[i] = addr[i]
236
237       FI_ATOMIC_WRITE : Write data atomically
238
239              addr[i] = buf[i]
240
241       FI_CSWAP : Compare values and if equal swap with data
242
243              if (compare[i] == addr[i])
244                  addr[i] = buf[i]
245
246       FI_CSWAP_NE : Compare values and if not equal swap with data
247
248              if (compare[i] != addr[i])
249                  addr[i] = buf[i]
250
251       FI_CSWAP_LE : Compare values and if less than or equal swap with data
252
253              if (compare[i] <= addr[i])
254                  addr[i] = buf[i]
255
256       FI_CSWAP_LT : Compare values and if less than swap with data
257
258              if (compare[i] < addr[i])
259                  addr[i] = buf[i]
260
261       FI_CSWAP_GE : Compare values and if greater than  or  equal  swap  with
262       data
263
264              if (compare[i] >= addr[i])
265                  addr[i] = buf[i]
266
267       FI_CSWAP_GT : Compare values and if greater than swap with data
268
269              if (compare[i] > addr[i])
270                  addr[i] = buf[i]
271
272       FI_MSWAP : Swap masked bits with data
273
274              addr[i] = (buf[i] & compare[i]) | (addr[i] & ~compare[i])
275
276   Base Atomic Functions
277       The base atomic functions -- fi_atomic, fi_atomicv, fi_atomicmsg -- are
278       used to transmit data to a remote  node,  where  the  specified  atomic
279       operation  is  performed against the target data.  The result of a base
280       atomic function is stored at the remote memory region.  The  main  dif‐
281       ference  between atomic functions are the number and type of parameters
282       that they accept as input.  Otherwise, they perform  the  same  general
283       function.
284
285       The  call  fi_atomic transfers the data contained in the user-specified
286       data buffer to a remote node.  For unconnected endpoints, the  destina‐
287       tion endpoint is specified through the dest_addr parameter.  Unless the
288       endpoint has been configured differently, the data buffer  passed  into
289       fi_atomic  must  not  be touched by the application until the fi_atomic
290       call completes asynchronously.  The target  buffer  of  a  base  atomic
291       operation  must  allow for remote read an/or write access, as appropri‐
292       ate.
293
294       The  fi_atomicv  call  adds  support  for  a  scatter-gather  list   to
295       fi_atomic.  The fi_atomicv transfers the set of data buffers referenced
296       by the ioc parameter to the remote node for processing.
297
298       The fi_inject_atomic call is an optimized version  of  fi_atomic.   The
299       fi_inject_atomic  function  behaves  as  if the FI_INJECT transfer flag
300       were set, and FI_COMPLETION were not.  That  is,  the  data  buffer  is
301       available    for    reuse    immediately   on   returning   from   from
302       fi_inject_atomic, and no completion event will be  generated  for  this
303       atomic.   The  completion event will be suppressed even if the endpoint
304       has not been configured with FI_SELECTIVE_COMPLETION.   See  the  flags
305       discussion below for more details.  The requested message size that can
306       be used with fi_inject_atomic is limited by inject_size.
307
308       The fi_atomicmsg call supports atomic functions over both connected and
309       unconnected endpoints, with the ability to control the atomic operation
310       per call through the use of flags.  The fi_atomicmsg function  takes  a
311       struct fi_msg_atomic as input.
312
313              struct fi_msg_atomic {
314                  const struct fi_ioc *msg_iov; /* local scatter-gather array */
315                  void                **desc;   /* local access descriptors */
316                  size_t              iov_count;/* # elements in ioc */
317                  const void          *addr;    /* optional endpoint address */
318                  const struct fi_rma_ioc *rma_iov; /* remote SGL */
319                  size_t              rma_iov_count;/* # elements in remote SGL */
320                  enum fi_datatype    datatype; /* operand datatype */
321                  enum fi_op          op;       /* atomic operation */
322                  void                *context; /* user-defined context */
323                  uint64_t            data;     /* optional data */
324              };
325
326              struct fi_ioc {
327                  void        *addr;    /* local address */
328                  size_t      count;    /* # target operands */
329              };
330
331              struct fi_rma_ioc {
332                  uint64_t    addr;     /* target address */
333                  size_t      count;    /* # target operands */
334                  uint64_t    key;      /* access key */
335              };
336
337       The  following  list  of  atomic operations are usable with base atomic
338       operations: FI_MIN, FI_MAX, FI_SUM, FI_PROD, FI_LOR,  FI_LAND,  FI_BOR,
339       FI_BAND, FI_LXOR, FI_BXOR, and FI_ATOMIC_WRITE.
340
341   Fetch-Atomic Functions
342       The  fetch  atomic  functions -- fi_fetch_atomic, fi_fetch_atomicv, and
343       fi_fetch atomicmsg -- behave similar  to  the  equivalent  base  atomic
344       function.   The  difference between the fetch and base atomic calls are
345       the fetch atomic routines return the initial value that was  stored  at
346       the  target  to the user.  The initial value is read into the user pro‐
347       vided result buffer.  The target buffer of fetch-atomic operations must
348       be enabled for remote read access.
349
350       The  following  list  of atomic operations are usable with fetch atomic
351       operations: FI_MIN, FI_MAX, FI_SUM, FI_PROD, FI_LOR,  FI_LAND,  FI_BOR,
352       FI_BAND, FI_LXOR, FI_BXOR, FI_ATOMIC_READ, and FI_ATOMIC_WRITE.
353
354       For   FI_ATOMIC_READ   operations,  the  source  buffer  operand  (e.g.
355       fi_fetch_atomic buf parameter) is ignored and may be NULL.  The results
356       are written into the result buffer.
357
358   Compare-Atomic Functions
359       The  compare atomic functions -- fi_compare_atomic, fi_compare_atomicv,
360       and fi_compare atomicmsg -- are used for operations that  require  com‐
361       paring  the target data against a value before performing a swap opera‐
362       tion.  The compare atomic  functions  support:  FI_CSWAP,  FI_CSWAP_NE,
363       FI_CSWAP_LE, FI_CSWAP_LT, FI_CSWAP_GE, FI_CSWAP_GT, and FI_MSWAP.
364
365   Atomic Valid Functions
366       The atomic valid functions -- fi_atomicvalid, fi_fetch_atomicvalid, and
367       fi_compare_atomicvalid --indicate which operations the  local  provider
368       supports.  Needed operations not supported by the provider must be emu‐
369       lated by the application.  Each valid call  corresponds  to  a  set  of
370       atomic  functions.  fi_atomicvalid checks whether a provider supports a
371       specific base atomic operation for  a  given  datatype  and  operation.
372       fi_fetch_atomicvalid  indicates  if  a  provider  supports  a  specific
373       fetch-atomic operation for a given datatype and operation.  And fi_com‐
374       pare_atomicvalid  checks  if  a  provider  supports  a  specified  com‐
375       pare-atomic operation for a given datatype and operation.
376
377       If an operation is supported, an atomic valid call will return 0, along
378       with  a  count  of  atomic  data units that a single function call will
379       operate on.
380
381   Query Atomic Attributes
382       The fi_query_atomic call acts as an  enhanced  atomic  valid  operation
383       (see  the atomic valid function definitions above).  It is provided, in
384       part, for future extensibility.   The  query  operation  reports  which
385       atomic  operations are supported by the domain, for suitably configured
386       endpoints.
387
388       The behavior of fi_query_atomic is adjusted based on the flags  parame‐
389       ter.   If  flags  is 0, then the operation reports the supported atomic
390       attributes for base atomic operations, similar  to  fi_atomicvalid  for
391       endpoints.   If  flags  has  the FI_FETCH_ATOMIC bit set, the operation
392       behaves similar  to  fi_fetch_atomicvalid.   Similarly,  the  flag  bit
393       FI_COMPARE_ATOMIC  results  in  query acting as fi_compare_atomicvalid.
394       The FI_FETCH_ATOMIC and FI_COMPARE_ATOMIC bits may not both be set.
395
396       If the FI_TAGGED bit is set, the provider will indicate if it  supports
397       atomic  operations to tagged receive buffers.  The FI_TAGGED bit may be
398       used by itself, or in conjunction with the FI_FETCH_ATOMIC and  FI_COM‐
399       PARE_ATOMIC flags.
400
401       The output of fi_query_atomic is struct fi_atomic_attr:
402
403              struct fi_atomic_attr {
404                  size_t count;
405                  size_t size;
406              };
407
408       The  count  attribute  field  is as defined for the atomic valid calls.
409       The size field indicates the size in bytes of the atomic datatype.
410
411   Completions
412       Completed atomic operations are reported to the  user  through  one  or
413       more event collectors associated with the endpoint.  Users provide con‐
414       text which are associated with each operation, and is returned  to  the
415       user  as  part of the event completion.  See fi_cq for completion event
416       details.
417
418       Updates to the target buffer of an atomic operation are visible to pro‐
419       cesses  running on the target system either after a completion has been
420       generated, or after the completion of an operation initiated after  the
421       atomic  call  with a fencing operation occurring in between.  For exam‐
422       ple, the target process may be notified by the initiator sending a mes‐
423       sage after the atomic call completes, or sending a fenced message imme‐
424       diately after initiating the atomic operation.
425

FLAGS

427       The fi_atomicmsg, fi_fetch_atomicmsg,  and  fi_compare_atomicmsg  calls
428       allow  the  user  to  specify  flags  which can change the default data
429       transfer operation.  Flags specified  with  atomic  message  operations
430       override  most  flags  previously  configured with the endpoint, except
431       where noted (see fi_control).  The following list of flags  are  usable
432       with atomic message calls.
433
434       FI_COMPLETION  :  Indicates that a completion entry should be generated
435       for the specified operation.  The endpoint must be bound to  a  comple‐
436       tion  queue with FI_SELECTIVE_COMPLETION that corresponds to the speci‐
437       fied operation, or this flag is ignored.
438
439       FI_MORE : Indicates that the user has  additional  requests  that  will
440       immediately be posted after the current call returns.  Use of this flag
441       may improve performance by enabling the provider to optimize its access
442       to the fabric hardware.
443
444       FI_INJECT : Indicates that the outbound non-const data buffers (buf and
445       compare parameters) should be returned to user  immediately  after  the
446       call  returns,  even  if the operation is handled asynchronously.  This
447       may require that the underlying provider implementation copy  the  data
448       into a local buffer and transfer out of that buffer.  The use of output
449       result buffers are not affected by this flag.  This flag  can  only  be
450       used with messages smaller than inject_size.
451
452       FI_FENCE  :  Applies to transmits.  Indicates that the requested opera‐
453       tion, also known as the fenced  operation,  and  any  operation  posted
454       after  the  fenced operation will be deferred until all previous opera‐
455       tions targeting the same  peer  endpoint  have  completed.   Operations
456       posted  after  the  fencing  will see and/or replace the results of any
457       operations initiated prior to the fenced operation.
458
459       The ordering of operations starting at the posting of the fenced opera‐
460       tion  (inclusive)  to  the  posting  of  a  subsequent fenced operation
461       (exclusive) is controlled by the endpoint's ordering semantics.
462
463       FI_TAGGED : Specifies that the target of  the  atomic  operation  is  a
464       tagged  receive  buffer instead of an RMA buffer.  When a tagged buffer
465       is the target memory region, the addr parameter is used  as  a  0-based
466       byte  offset  into the tagged buffer, with the key parameter specifying
467       the tag.
468

RETURN VALUE

470       Returns 0 on success.  On error, a negative value corresponding to fab‐
471       ric   errno   is   returned.    Fabric  errno  values  are  defined  in
472       rdma/fi_errno.h.
473

ERRORS

475       -FI_EAGAIN : See fi_msg(3)  for  a  detailed  description  of  handling
476       FI_EAGAIN.
477
478       -FI_EOPNOTSUPP  :  The  requested  atomic operation is not supported on
479       this endpoint.
480
481       -FI_EMSGSIZE : The number of atomic  operations  in  a  single  request
482       exceeds that supported by the underlying provider.
483

NOTES

485       Atomic  operations  operate  on  an  array of values of a specific data
486       type.  Atomicity is only guaranteed for each data type  operation,  not
487       across  the  entire array.  The following pseudo-code demonstrates this
488       operation for 64-bit unsigned  atomic  write.   ATOMIC_WRITE_U64  is  a
489       platform  dependent  macro that atomically writes 8 bytes to an aligned
490       memory location.
491
492              fi_atomic(ep, buf, count, NULL, dest_addr, addr, key,
493                    FI_UINT64, FI_ATOMIC_WRITE, context)
494              {
495                  for (i = 1; i < count; i ++)
496                      ATOMIC_WRITE_U64(((uint64_t *) addr)[i],
497                               ((uint64_t *) buf)[i]);
498              }
499
500       The number of array elements to operate on is specified through a count
501       parameter.  This must be between 1 and the maximum returned through the
502       relevant valid operation, inclusive.  The requested operation and  data
503       type must also be valid for the given provider.
504

SEE ALSO

506       fi_getinfo(3), fi_endpoint(3), fi_domain(3), fi_cq(3), fi_rma(3)
507

AUTHORS

509       OpenFabrics.
510
511
512
513Libfabric Programmer's Manual     2017-09-25                      fi_atomic(3)
Impressum