1MCE::Candy(3)         User Contributed Perl Documentation        MCE::Candy(3)
2
3
4

NAME

6       MCE::Candy - Sugar methods and output iterators
7

VERSION

9       This document describes MCE::Candy version 1.866
10

DESCRIPTION

12       This module provides a collection of sugar methods and helpful output
13       iterators for preserving output order.
14

"FOR" SUGAR METHODS

16       The sugar methods described below were created prior to the 1.5 release
17       which added MCE Models. This module is loaded automatically upon
18       calling a "for" method.
19
20   $mce->forchunk ( $input_data [, { options } ], sub { ... } )
21       Forchunk, foreach, and forseq are sugar methods in MCE. Workers are
22       spawned automatically, the code block is executed in parallel, and
23       shutdown is called. Do not call these methods if workers must persist
24       afterwards.
25
26       Specifying options is optional. Valid options are the same as for the
27       process method.
28
29        ## Declare a MCE instance.
30
31        my $mce = MCE->new(
32           max_workers => $max_workers,
33           chunk_size  => 20
34        );
35
36        ## Arguments inside the code block are the same as passed to user_func.
37
38        $mce->forchunk(\@input_array, sub {
39           my ($mce, $chunk_ref, $chunk_id) = @_;
40           foreach ( @{ $chunk_ref } ) {
41              MCE->print("$chunk_id: $_\n");
42           }
43        });
44
45        ## Input hash, current API available since 1.828.
46
47        $mce->forchunk(\%input_hash, sub {
48           my ($mce, $chunk_ref, $chunk_id) = @_;
49           for my $key ( keys %{ $chunk_ref } ) {
50              MCE->print("$chunk_id: [ $key ] ", $chunk_ref->{$key}, "\n");
51           }
52        });
53
54        ## Passing chunk_size as an option.
55
56        $mce->forchunk(\@input_array, { chunk_size => 30 }, sub { ... });
57        $mce->forchunk(\%input_hash, { chunk_size => 30 }, sub { ... });
58
59   $mce->foreach ( $input_data [, { options } ], sub { ... } )
60       Foreach implies chunk_size => 1 and cannot be overwritten. Thus,
61       looping is not necessary inside the block. Unlike forchunk above, a
62       hash reference as input data isn't allowed.
63
64        my $mce = MCE->new(
65           max_workers => $max_workers
66        );
67
68        $mce->foreach(\@input_data, sub {
69           my ($mce, $chunk_ref, $chunk_id) = @_;
70           my $row = $chunk_ref->[0];
71           MCE->print("$chunk_id: $row\n");
72        });
73
74   $mce->forseq ( $sequence_spec [, { options } ], sub { ... } )
75       Sequence may be defined using an array or hash reference.
76
77        my $mce = MCE->new(
78           max_workers => 3
79        );
80
81        $mce->forseq([ 20, 40 ], sub {
82           my ($mce, $n, $chunk_id) = @_;
83           my $result = `ping 192.168.1.${n}`;
84           ...
85        });
86
87        $mce->forseq({ begin => 15, end => 10, step => -1 }, sub {
88           my ($mce, $n, $chunk_id) = @_;
89           print $n, " from ", MCE->wid, "\n";
90        });
91
92       The $n_seq variable points to an array_ref of sequences. Chunk size
93       defaults to 1 when not specified.
94
95        $mce->forseq([ 20, 80 ], { chunk_size => 10 }, sub {
96           my ($mce, $n_seq, $chunk_id) = @_;
97           for my $n ( @{ $n_seq } ) {
98              my $result = `ping 192.168.1.${n}`;
99              ...
100           }
101        });
102

OUTPUT ITERATORS WITH INPUT

104       This module includes 2 output iterators which are useful for preserving
105       output order while gathering data. These cover the 2 general use cases.
106       The chunk_id value must be the first argument to gather. Gather must
107       also not be called more than once inside the block.
108
109   gather => MCE::Candy::out_iter_array( \@array )
110       The example utilizes the Core API with chunking disabled. Basically,
111       setting chunk_size to 1.
112
113        use MCE;
114        use MCE::Candy;
115
116        my @results;
117
118        my $mce = MCE->new(
119           chunk_size => 1, max_workers => 4,
120           gather => MCE::Candy::out_iter_array(\@results),
121           user_func => sub {
122              my ($mce, $chunk_ref, $chunk_id) = @_;
123              $mce->gather($chunk_id, $chunk_ref->[0] * 2);
124           }
125        );
126
127        $mce->process([ 100 .. 109 ]);
128
129        print "@results", "\n";
130
131        -- Output
132
133        200 202 204 206 208 210 212 214 216 218
134
135       Chunking may be desired for thousands or more items. In other words,
136       wanting to reduce the overhead placed on IPC.
137
138        use MCE;
139        use MCE::Candy;
140
141        my @results;
142
143        my $mce = MCE->new(
144           chunk_size => 100, max_workers => 4,
145           gather => MCE::Candy::out_iter_array(\@results),
146           user_func => sub {
147              my ($mce, $chunk_ref, $chunk_id) = @_;
148              my @output;
149              foreach my $item (@{ $chunk_ref }) {
150                 push @output, $item * 2;
151              }
152              $mce->gather($chunk_id, @output);
153           }
154        );
155
156        $mce->process([ 100_000 .. 200_000 - 1 ]);
157
158        print scalar @results, "\n";
159
160        -- Output
161
162        100000
163
164   gather => MCE::Candy::out_iter_fh( $fh )
165       Let's change things a bit and use MCE::Flow for the next 2 examples.
166       Chunking is not desired for the first example.
167
168        use MCE::Flow;
169        use MCE::Candy;
170
171        open my $fh, '>', '/tmp/foo.txt';
172
173        mce_flow {
174           chunk_size => 1, max_workers => 4,
175           gather => MCE::Candy::out_iter_fh($fh)
176        },
177        sub {
178           my ($mce, $chunk_ref, $chunk_id) = @_;
179           $mce->gather($chunk_id, $chunk_ref->[0] * 2, "\n");
180
181        }, (100 .. 109);
182
183        close $fh;
184
185        -- Output sent to '/tmp/foo.txt'
186
187        200
188        202
189        204
190        206
191        208
192        210
193        212
194        214
195        216
196        218
197
198   gather => MCE::Candy::out_iter_fh( $io )
199       Same thing, an "IO::*" object that can "print" is supported since MCE
200       1.845.
201
202        use IO::All;
203        use MCE::Flow;
204        use MCE::Candy;
205
206        my $io = io('/tmp/foo.txt');  # i.e. $io->can('print')
207
208        mce_flow {
209           chunk_size => 1, max_workers => 4,
210           gather => MCE::Candy::out_iter_fh($io)
211        },
212        sub {
213           my ($mce, $chunk_ref, $chunk_id) = @_;
214           $mce->gather($chunk_id, $chunk_ref->[0] * 2, "\n");
215
216        }, (100 .. 109);
217
218        $io->close;
219
220        -- Output sent to '/tmp/foo.txt'
221
222        200
223        202
224        204
225        206
226        208
227        210
228        212
229        214
230        216
231        218
232
233       Chunking is desired for the next example due to processing many
234       thousands.
235
236        use MCE::Flow;
237        use MCE::Candy;
238
239        open my $fh, '>', '/tmp/foo.txt';
240
241        mce_flow {
242           chunk_size => 100, max_workers => 4,
243           gather => MCE::Candy::out_iter_fh( $fh )
244        },
245        sub {
246           my ($mce, $chunk_ref, $chunk_id) = @_;
247           my @output;
248           foreach my $item (@{ $chunk_ref }) {
249              push @output, ($item * 2) . "\n";
250           }
251           $mce->gather($chunk_id, @output);
252
253        }, (100_000 .. 200_000 - 1);
254
255        close $fh;
256
257        print -s '/tmp/foo.txt', "\n";
258
259        -- Output
260
261        700000
262

OUTPUT ITERATORS WITHOUT INPUT

264       Input data is not a requirement for using the output iterators included
265       in this module. The 'chunk_id' value is set uniquely and the same as
266       'wid' when not processing input data.
267
268   gather => MCE::Candy::out_iter_array( \@array )
269        use MCE::Flow;
270        use MCE::Candy;
271
272        my @results;
273
274        mce_flow {
275           max_workers => 'auto', ## Note that 'auto' is never greater than 8
276           gather => MCE::Candy::out_iter_array(\@results)
277        },
278        sub {
279           my ($mce) = @_;        ## This line is not necessary
280                                  ## Calling via module okay; e.g: MCE->method
281           ## Do work
282           ## Sending a complex data structure is allowed
283
284           ## Output will become orderly by iterator
285           $mce->gather( $mce->chunk_id, {
286              wid => $mce->wid, result => $mce->wid * 2
287           });
288        };
289
290        foreach my $href (@results) {
291           print $href->{wid} .": ". $href->{result} ."\n";
292        }
293
294        -- Output
295
296        1: 2
297        2: 4
298        3: 6
299        4: 8
300        5: 10
301        6: 12
302        7: 14
303        8: 16
304
305   gather => MCE::Candy::out_iter_fh( $fh )
306        use MCE::Flow;
307        use MCE::Candy;
308
309        open my $fh, '>', '/tmp/out.txt';
310
311        mce_flow {
312           max_workers => 'auto', ## See get_ncpu in <MCE::Util|MCE::Util>
313           gather => MCE::Candy::out_iter_fh($fh)
314        },
315        sub {
316           my $output = "# Worker ID: " . MCE->wid . "\n";
317
318           ## Append results to $output string
319           $output .= (MCE->wid * 2) . "\n\n";
320
321           ## Output will become orderly by iterator
322           MCE->gather( MCE->wid, $output );
323        };
324
325        close $fh;
326
327        -- Output
328
329        # Worker ID: 1
330        2
331
332        # Worker ID: 2
333        4
334
335        # Worker ID: 3
336        6
337
338        # Worker ID: 4
339        8
340
341        # Worker ID: 5
342        10
343
344        # Worker ID: 6
345        12
346
347        # Worker ID: 7
348        14
349
350        # Worker ID: 8
351        16
352

INDEX

354       MCE, MCE::Core
355

AUTHOR

357       Mario E. Roy, <marioeroy AT gmail DOT com>
358
359
360
361perl v5.30.1                      2020-02-09                     MCE::Candy(3)