1MLX5DV_DR API(3) mlx5 Programmer’s Manual MLX5DV_DR API(3)
2
6 mlx5dv_dr_domain_create, mlx5dv_dr_domain_sync, mlx5dv_dr_domain_de‐
7 stroy, mlx5dv_dr_domain_set_reclaim_device_memory, mlx5dv_dr_domain_al‐
8 low_duplicate_rules - Manage flow domains
9 mlx5dv_dr_table_create, mlx5dv_dr_table_destroy - Manage flow tables
11 mlx5dv_dr_matcher_create, mlx5dv_dr_matcher_destroy, mlx5dv_dr_match‐
13 er_set_layout - Manage flow matchers
14 mlx5dv_dr_rule_create, mlx5dv_dr_rule_destroy - Manage flow rules
16 mlx5dv_dr_action_create_drop - Create drop action
18 mlx5dv_dr_action_create_default_miss - Create default miss action
20 mlx5dv_dr_action_create_tag - Create tag actions
22 mlx5dv_dr_action_create_dest_ibv_qp - Create packet destination QP ac‐
24 tion
25 mlx5dv_dr_action_create_dest_table - Create packet destination dr table
27 action
28 mlx5dv_dr_action_create_dest_vport - Create packet destination vport
30 action
31 mlx5dv_dr_action_create_dest_ib_port - Create packet destination IB
33 port action
34 mlx5dv_dr_action_create_dest_devx_tir - Create packet destination TIR
36 action
37 mlx5dv_dr_action_create_dest_array - Create destination array action
39 mlx5dv_dr_action_create_packet_reformat - Create packet reformat ac‐
41 tions
42 mlx5dv_dr_action_create_modify_header - Create modify header actions
44 mlx5dv_dr_action_create_flow_counter - Create devx flow counter actions
46 mlx5dv_dr_action_create_aso, mlx5dv_dr_action_modify_aso - Create and
48 modify ASO actions
49 mlx5dv_dr_action_create_flow_meter, mlx5dv_dr_action_modify_flow_meter
51 - Create and modify meter action
52 mlx5dv_dr_action_create_flow_sampler - Create flow sampler action
54 mlx5dv_dr_action_create_pop_vlan - Create pop vlan action
56 mlx5dv_dr_action_create_push_vlan- Create push vlan action
58 mlx5dv_dr_action_destroy - Destroy actions
60 mlx5dv_dr_aso_other_domain_link, mlx5dv_dr_aso_other_domain_unlink -
62 link/unlink ASO devx object to work with different domains
63
65 #include <infiniband/mlx5dv.h>
66 struct mlx5dv_dr_domain *mlx5dv_dr_domain_create(
68 struct ibv_context *ctx,
69 enum mlx5dv_dr_domain_type type);
70 int mlx5dv_dr_domain_sync(
72 struct mlx5dv_dr_domain *domain,
73 uint32_t flags);
74 int mlx5dv_dr_domain_destroy(struct mlx5dv_dr_domain *domain);
76 void mlx5dv_dr_domain_set_reclaim_device_memory(
78 struct mlx5dv_dr_domain *dmn,
79 bool enable);
80 void mlx5dv_dr_domain_allow_duplicate_rules(struct mlx5dv_dr_domain *dmn, bool allow);
82 struct mlx5dv_dr_table *mlx5dv_dr_table_create(
84 struct mlx5dv_dr_domain *domain,
85 uint32_t level);
86 int mlx5dv_dr_table_destroy(struct mlx5dv_dr_table *table);
88 struct mlx5dv_dr_matcher *mlx5dv_dr_matcher_create(
90 struct mlx5dv_dr_table *table,
91 uint16_t priority,
92 uint8_t match_criteria_enable,
93 struct mlx5dv_flow_match_parameters *mask);
94 int mlx5dv_dr_matcher_destroy(struct mlx5dv_dr_matcher *matcher);
96 int mlx5dv_dr_matcher_set_layout(struct mlx5dv_dr_matcher *matcher, struct mlx5dv_dr_matcher_layout *matcher_layout);
99 struct mlx5dv_dr_rule *mlx5dv_dr_rule_create(
101 struct mlx5dv_dr_matcher *matcher,
102 struct mlx5dv_flow_match_parameters *value,
103 size_t num_actions,
104 struct mlx5dv_dr_action *actions[]);
105 void mlx5dv_dr_rule_destroy(struct mlx5dv_dr_rule *rule);
107 struct mlx5dv_dr_action *mlx5dv_dr_action_create_drop(void);
109 struct mlx5dv_dr_action *mlx5dv_dr_action_create_default_miss(void);
111 struct mlx5dv_dr_action *mlx5dv_dr_action_create_tag(
113 uint32_t tag_value);
114 struct mlx5dv_dr_action *mlx5dv_dr_action_create_dest_ibv_qp(
116 struct ibv_qp *ibqp);
117 struct mlx5dv_dr_action *mlx5dv_dr_action_create_dest_table(
119 struct mlx5dv_dr_table *table);
120 struct mlx5dv_dr_action *mlx5dv_dr_action_create_dest_vport(
122 struct mlx5dv_dr_domain *domain,
123 uint32_t vport);
124 struct mlx5dv_dr_action *mlx5dv_dr_action_create_dest_ib_port(
126 struct mlx5dv_dr_domain *domain,
127 uint32_t ib_port);
128 struct mlx5dv_dr_action *mlx5dv_dr_action_create_dest_devx_tir(
130 struct mlx5dv_devx_obj *devx_obj);
131 struct mlx5dv_dr_action *mlx5dv_dr_action_create_packet_reformat(
133 struct mlx5dv_dr_domain *domain,
134 uint32_t flags,
135 enum mlx5dv_flow_action_packet_reformat_type reformat_type,
136 size_t data_sz, void *data);
137 struct mlx5dv_dr_action *mlx5dv_dr_action_create_modify_header(
139 struct mlx5dv_dr_domain *domain,
140 uint32_t flags,
141 size_t actions_sz,
142 __be64 actions[]);
143 struct mlx5dv_dr_action *mlx5dv_dr_action_create_flow_counter(
145 struct mlx5dv_devx_obj *devx_obj,
146 uint32_t offset);
147 struct mlx5dv_dr_action *
149 mlx5dv_dr_action_create_aso(struct mlx5dv_dr_domain *domain,
150 struct mlx5dv_devx_obj *devx_obj,
151 uint32_t offset,
152 uint32_t flags,
153 uint8_t return_reg_c);
154 int mlx5dv_dr_action_modify_aso(struct mlx5dv_dr_action *action,
156 uint32_t offset,
157 uint32_t flags,
158 uint8_t return_reg_c);
159 struct mlx5dv_dr_action *
161 mlx5dv_dr_action_create_flow_meter(struct mlx5dv_dr_flow_meter_attr *attr);
162 int mlx5dv_dr_action_modify_flow_meter(struct mlx5dv_dr_action *action,
164 struct mlx5dv_dr_flow_meter_attr *attr,
165 __be64 modify_field_select);
166 struct mlx5dv_dr_action *
168 mlx5dv_dr_action_create_flow_sampler(struct mlx5dv_dr_flow_sampler_attr *attr);
169 struct mlx5dv_dr_action *
171 mlx5dv_dr_action_create_dest_array(struct mlx5dv_dr_domain *domain,
172 size_t num_dest,
173 struct mlx5dv_dr_action_dest_attr *dests[]);
174 struct mlx5dv_dr_action *mlx5dv_dr_action_create_pop_vlan(void);
176 struct mlx5dv_dr_action *mlx5dv_dr_action_create_push_vlan(
178 struct mlx5dv_dr_domain *dmn,
179 __be32 vlan_hdr)
180 int mlx5dv_dr_action_destroy(struct mlx5dv_dr_action *action);
182 int mlx5dv_dr_aso_other_domain_link(struct mlx5dv_devx_obj *devx_obj,
184 struct mlx5dv_dr_domain *peer_dmn,
185 struct mlx5dv_dr_domain *dmn,
186 uint32_t flags,
187 uint8_t return_reg_c);
188 int mlx5dv_dr_aso_other_domain_unlink(struct mlx5dv_devx_obj *devx_obj,
190 struct mlx5dv_dr_domain *dmn);
191
193 The Direct Rule API (mlx5dv_dr_*) allows complete access by verbs ap‐
194 plication to the device`s packet steering functionality.
195 Steering flow rules are the combination of attributes with a match pat‐
197 tern and a list of actions. Rules can have several distinct actions
198 (such as counting, encapsulating, decapsulating before redirecting
199 packets to a particular queue or port, etc.). In order to manage the
200 rule execution order for the packet processing matching by HW, multiple
201 flow tables in an ordered chain and multiple flow matchers sorted by
202 priorities are defined.
203 Domain
205 mlx5dv_dr_domain_create() creates a DR domain object to be used with
206 mlx5dv_dr_table_create() and mlx5dv_dr_action_create_*().
207 A domain should be destroyed by calling mlx5dv_dr_domain_destroy() once
209 all depended resources are released.
210 The device support the following domains types:
212 MLX5DV_DR_DOMAIN_TYPE_NIC_RX Manage ethernet packets received on the
214 NIC. Packets in this domain can be dropped, dispatched to QP`s, modi‐
215 fied or redirected to additional tables inside the domain. Default be‐
216 havior: Drop packet.
217 MLX5DV_DR_DOMAIN_TYPE_NIC_TX Manage ethernet packets transmit on the
219 NIC. Packets in this domain can be dropped, modified or redirected to
220 additional tables inside the domain. Default behavior: Forward packet
221 to NIC vport (to eSwitch or wire).
222 MLX5DV_DR_DOMAIN_TYPE_FDB Manage ethernet packets in the eSwitch For‐
224 warding Data Base for packets received from wire or from any other
225 vport. Packets in this domain can be dropped, dispatched to vport,
226 modified or redirected to additional tables inside the domain. Default
227 behavior: Forward packet to eSwitch manager vport.
228 mlx5dv_dr_domain_sync() is used in order to flush the rule submission
230 queue. By default, rules in a domain are updated in HW asynchronously.
231 flags should be a set of type enum mlx5dv_dr_domain_sync_flags:
232 MLX5DV_DR_DOMAIN_SYNC_FLAGS_SW: block until completion of all software
234 queued tasks.
235 MLX5DV_DR_DOMAIN_SYNC_FLAGS_HW: clear the steering HW cache to enforce
237 next packet hits the latest rules, in addition to the SW SYNC handling.
238 MLX5DV_DR_DOMAIN_SYNC_FLAGS_MEM: sync device memory to free cached mem‐
240 ory.
241 mlx5dv_dr_domain_set_reclaim_device_memory() is used to enable the re‐
243 claiming of device memory back to the system when not in use, by de‐
244 fault this feature is disabled.
245 mlx5dv_dr_domain_allow_duplicate_rules() is used to allow or prevent
247 insertion of rules matching on same fields(duplicates) on non root ta‐
248 bles, by default this feature is allowed.
249 Table
251 mlx5dv_dr_table_create() creates a DR table in the domain, at the ap‐
252 propriate level, and can be used with mlx5dv_dr_matcher_create() and
253 mlx5dv_dr_action_create_dest_table(). All packets start traversing the
254 steering domain tree at table level [22mzero (0). Using rule and action,
255 packets can by redirected to other tables in the domain.
256 A table should be destroyed by calling mlx5dv_dr_table_destroy() once
258 all depended resources are released.
259 Matcher
261 mlx5dv_dr_matcher_create() create a matcher object in table, at sorted
262 priority (lower value is check first). A matcher can hold multiple
263 rules, all with identical mask of type struct mlx5dv_flow_match_parame‐
264 ters which represents the exact attributes to be compared by HW steer‐
265 ing. The match_criteria_enable and mask are defined in a device spec
266 format. Only the fields that where masked in the matcher should be
267 filled by the rule in mlx5dv_dr_rule_create().
268 A matcher should be destroyed by calling mlx5dv_dr_matcher_destroy()
270 once all depended resources are released.
271 mlx5dv_dr_matcher_set_layout() is used to set specific layout parame‐
273 ters of a matcher, on some conditions setting some attributes might not
274 be supported, in such cases ENOTSUP will be returned. flags should be
275 a set of type enum mlx5dv_dr_matcher_layout_flags:
276 MLX5DV_DR_MATCHER_LAYOUT_RESIZABLE: The matcher can resize its scale
278 and resources according to the rules that are inserted or removed.
279 MLX5DV_DR_MATCHER_LAYOUT_NUM_RULE: Indicates a hint from the applica‐
281 tion about the number of the rules the matcher is expected to handle.
282 This allows preallocation of matcher resources for faster rule updates
283 when using with non-resizable layout mode.
284 Actions
286 A set of action create API are defined by mlx5dv_dr_action_create_*().
287 All action are created as struct mlx5dv_dr_action. An action should be
288 destroyed by calling mlx5dv_dr_action_destroy() once all depended rules
289 are destroyed.
290 When an action handle is reused for multiple rules, the same action
292 will be executed. e.g.: action `count' will count multiple flows rules
293 on the same HW flow counter context. action `drop' will drop packets
294 of different rule from any matcher.
295 Action: Drop mlx5dv_dr_action_create_drop create a terminating action
297 which drops packets. Can not be mixed with Destination actions.
298 Action: Default miss mlx5dv_dr_action_create_default_miss create a ter‐
300 minating action which will execute the default behavior based on the
301 domain type.
302 Action: Tag mlx5dv_dr_action_create_tag creates a non-terminating ac‐
304 tion which tags packets with tag_value. The tag_value is available in
305 the CQE of the packet received. Valid only on domain type NIC_RX.
306 Action: Destination mlx5dv_dr_action_create_dest_ibv_qp creates a ter‐
308 minating action delivering the packet to a QP, defined by ibqp. Valid
309 only on domain type NIC_RX. mlx5dv_dr_action_create_dest_table creates
310 a forwarding action to another flow table, defined by table. The des‐
311 tination table must be from the same domain with a level higher than
312 zero. mlx5dv_dr_action_create_dest_vport creates a forwarding action
313 to a vport on the same domain. Valid only on domain type FDB.
314 mlx5dv_dr_action_create_dest_ib_port creates a forwarding action to a
315 ib_port on the same domain. The valid range of ports is a based on the
316 capability phys_port_cnt_ex provided by ibq_query_device_ex and it is
317 possible to query the ports details using mlx5dv_query_port. Action is
318 supported only on domain type FDB. mlx5dv_dr_action_create_dest_de‐
319 vx_tir creates a terminating action delivering the packet to a TIR, de‐
320 fined by devx_obj. Valid only on domain type NIC_RX.
321 Action: Array mlx5dv_dr_action_create_dest_array creates an action
323 which replicates a packet to multiple destinations. num_dest defines
324 the number of replication destinations. Each dests destination array
325 entry can be of different type. Use type MLX5DV_DR_ACTION_DEST for di‐
326 rect forwarding to an action destination. Use type MLX5DV_DR_AC‐
327 TION_DEST_REFORMAT when reformat action should be performed on the
328 packet before it is forwarding to the destination action.
329 Action: Packet Reformat mlx5dv_dr_action_create_packet_reformat create
331 a packet reformat context and action in the domain. The reformat_type,
332 data_sz and data are defined in man mlx5dv_create_flow_action_pack‐
333 et_reformat.
334 Action: Modify Header mlx5dv_dr_action_create_modify_header create a
336 modify header context and action in the domain. The actions_sz and ac‐
337 tions are defined in man mlx5dv_create_flow_action_modify_header.
338 Action: Flow Count mlx5dv_dr_action_create_flow_counter creates a flow
340 counter action from a DEVX flow counter object, based on devx_obj and
341 specific counter index from offset in the counter bulk.
342 Action: ASO mlx5dv_dr_action_create_aso receives a domain pointer and
344 creates an ASO action from the DEVX ASO object, based on devx_obj. Use
345 offset to select the specific ASO object in the devx_obj bulk. DR
346 rules using this action can optionally update the ASO object value ac‐
347 cording to flags to choose the specific wanted behavior of this object.
348 After a packet hits the rule with the ASO object the value of the ASO
349 object will be copied into the chosen return_reg_c which can be used
350 for match in following DR rules.
351 mlx5dv_dr_action_modify_aso modifies ASO action action with new values
353 for offset, return_reg_c and flags. Only new DR rules using this ac‐
354 tion will use the modified values. Existing DR rules do not change the
355 HW action values stored.
356 flags can be set to one of the types of mlx5dv_dr_ac‐
358 tion_aso_first_hit_flags or mlx5dv_dr_action_aso_flow_meter_flags or
359 mlx5dv_dr_action_aso_ct_flags: MLX5DV_DR_AC‐
360 TION_ASO_FIRST_HIT_FLAGS_SET: is used to set the ASO first hit object
361 context, else the context is only copied to the return_reg_c.
362 MLX5DV_DR_ACTION_FLAGS_ASO_FLOW_METER_RED: is used to indicate to up‐
363 date the initial color in ASO flow meter object value to red.
364 MLX5DV_DR_ACTION_FLAGS_ASO_FLOW_METER_YELLOW: is used to indicate to
365 update the initial color in ASO flow meter object value to yellow.
366 MLX5DV_DR_ACTION_FLAGS_ASO_FLOW_METER_GREEN: is used to indicate to up‐
367 date the initial color in ASO flow meter object value to green.
368 MLX5DV_DR_ACTION_FLAGS_ASO_FLOW_METER_UNDEFINED: is used to indicate to
369 update the initial color in ASO flow meter object value to undefined.
370 MLX5DV_DR_ACTION_FLAGS_ASO_CT_DIRECTION_INITIATOR: is used to indicate
371 the TCP connection direction the SYN packet was sent on. MLX5DV_DR_AC‐
372 TION_FLAGS_ASO_CT_DIRECTION_RESPONDER: is used to indicate the TCP con‐
373 nection direction the SYN-ACK packet was sent on.
374 Action: Meter mlx5dv_dr_action_create_flow_meter creates a meter action
376 based on the flow meter parameters. The paramertes are according to
377 the device specification. mlx5dv_dr_action_modify_flow_meter modifies
378 existing flow meter action based on modify_field_select. modi‐
379 fy_field_select is according to the device specification.
380 Action: Sampler mlx5dv_dr_action_create_flow_sampler creates a sampler
382 action, allowing us to duplicate and sample a portion of traffic.
383 Packets steered to the sampler action will be sampled with an approxi‐
384 mate probability of 1/sample_ratio provided in attr, and sample_actions
385 provided in attr will be executed over them. All original packets will
386 be steered to default_next_table in attr. A modify header format
387 SET_ACTION data can be provided in action of attr, which can be execut‐
388 ed on packets before going to default flow table. On some devices,
389 this is required to set register value.
390 Action Flags: action flags can be set to one of the types of enum
392 mlx5dv_dr_action_flags:
393 Action: Pop Vlan mlx5dv_dr_action_create_pop_vlan creates a pop vlan
395 action which removes VLAN tags from packets layer 2.
396 Action: Push Vlan mlx5dv_dr_action_create_push_vlan creates a push vlan
398 action which adds VLAN tags to packets layer 2.
399 MLX5DV_DR_ACTION_FLAGS_ROOT_LEVEL: is used to indicate the action is
401 targeted for flow table in level=0 (ROOT) of the specific domain.
402 Rule
404 mlx5dv_dr_rule_create() creates a HW steering rule entry in matcher.
405 The value of type struct mlx5dv_flow_match_parameters holds the exact
406 attribute values of the steering rule to be matched, in a device spec
407 format. Only the fields that where masked in the matcher should be
408 filled. HW will perform the set of num_actions from the action array
409 of type struct mlx5dv_dr_action, once a packet matches the exact value
410 of the rule (referred to as a `hit').
411 mlx5dv_dr_rule_destroy() destroys the rule.
413 Other
415 mlx5dv_dr_aso_other_domain_link() links the ASO devx object, devx_obj
416 to a domain dmn, this will allow to create a rule with ASO action using
417 the given object on the linked domain dmn. peer_dmn is the domain that
418 the ASO devx object was created on. dmn is the domain that ASO devx
419 object will be linked to. flags choose the specific wanted behavior of
420 this object according to the flags, same as for ASO action creation
421 flags. regc_index After a packet hits the rule with the ASO object the
422 value of the ASO object will be copied into the regc register indicated
423 by this param, and then we can use the value for matching in the fol‐
424 lowing DR rules.
425 mlx5dv_dr_aso_other_domain_unlink() will unlink the devx_obj from the
427 linked dmn. dmn is the domain that ASO devx object is linked to.
428
430 The create API calls will return a pointer to the relevant object: ta‐
431 ble, matcher, action, rule. on failure, NULL will be returned and er‐
432 rno will be set.
433 The destroy API calls will returns 0 on success, or the value of errno
435 on failure (which indicates the failure reason).
436
438 Application can verify is a feature is supported by trail and error.
439 No capabilities are exposed, as the combination of all the options ex‐
440 posed are way to large to define.
441 Tables are size less by definition. They are expected to grow and
443 shrink to accommodate for all rules, according to driver capabilities.
444 Once reaching a limit, an error is returned.
445 Matchers in same priority, in the same table, will have undefined or‐
447 dered.
448 A rule with identical value pattern to another rule on a given matcher
450 are rejected.
451 IP version in matcher mask and rule should be equal and set to 4, 6 or
453 0. # SEE ALSO
454 mlx5dv_open_device(3), mlx5dv_create_flow_action_packet_reformat(3),
456 mlx5dv_create_flow_action_modify_header(3).
457
459 Alex Rosenbaum <alexr@mellanox.com> Alex Vesker <valex@mellanox.com>
460mlx5 2019-03-28 MLX5DV_DR API(3)