1MCE::Candy(3)         User Contributed Perl Documentation        MCE::Candy(3)
2
3
4

NAME

6       MCE::Candy - Sugar methods and output iterators
7

VERSION

9       This document describes MCE::Candy version 1.838
10

DESCRIPTION

12       This module provides a collection of sugar methods and helpful output
13       iterators for preserving output order.
14

"FOR" SUGAR METHODS

16       The sugar methods described below were created prior to the 1.5 release
17       which added MCE Models. This module is loaded automatically upon
18       calling a "for" method.
19
20   $mce->forchunk ( $input_data [, { options } ], sub { ... } )
21       Forchunk, foreach, and forseq are sugar methods in MCE. Workers are
22       spawned automatically, the code block is executed in parallel, and
23       shutdown is called. Do not call these methods if workers must persist
24       afterwards.
25
26       Specifying options is optional. Valid options are the same as for the
27       process method.
28
29        ## Declare a MCE instance.
30
31        my $mce = MCE->new(
32           max_workers => $max_workers,
33           chunk_size  => 20
34        );
35
36        ## Arguments inside the code block are the same as passed to user_func.
37
38        $mce->forchunk(\@input_array, sub {
39           my ($mce, $chunk_ref, $chunk_id) = @_;
40           foreach ( @{ $chunk_ref } ) {
41              MCE->print("$chunk_id: $_\n");
42           }
43        });
44
45        ## Input hash, current API available since 1.828.
46
47        $mce->forchunk(\%input_hash, sub {
48           my ($mce, $chunk_ref, $chunk_id) = @_;
49           for my $key ( keys %{ $chunk_ref } ) {
50              MCE->print("$chunk_id: [ $key ] ", $chunk_ref->{$key}, "\n");
51           }
52        });
53
54        ## Passing chunk_size as an option.
55
56        $mce->forchunk(\@input_array, { chunk_size => 30 }, sub { ... });
57        $mce->forchunk(\%input_hash, { chunk_size => 30 }, sub { ... });
58
59   $mce->foreach ( $input_data [, { options } ], sub { ... } )
60       Foreach implies chunk_size => 1 and cannot be overwritten. Thus,
61       looping is not necessary inside the block. Unlike forchunk above, a
62       hash reference as input data isn't allowed.
63
64        my $mce = MCE->new(
65           max_workers => $max_workers
66        );
67
68        $mce->foreach(\@input_data, sub {
69           my ($mce, $chunk_ref, $chunk_id) = @_;
70           my $row = $chunk_ref->[0];
71           MCE->print("$chunk_id: $row\n");
72        });
73
74   $mce->forseq ( $sequence_spec [, { options } ], sub { ... } )
75       Sequence may be defined using an array or hash reference.
76
77        my $mce = MCE->new(
78           max_workers => 3
79        );
80
81        $mce->forseq([ 20, 40 ], sub {
82           my ($mce, $n, $chunk_id) = @_;
83           my $result = `ping 192.168.1.${n}`;
84           ...
85        });
86
87        $mce->forseq({ begin => 15, end => 10, step => -1 }, sub {
88           my ($mce, $n, $chunk_id) = @_;
89           print $n, " from ", MCE->wid, "\n";
90        });
91
92       The $n_seq variable points to an array_ref of sequences. Chunk size
93       defaults to 1 when not specified.
94
95        $mce->forseq([ 20, 80 ], { chunk_size => 10 }, sub {
96           my ($mce, $n_seq, $chunk_id) = @_;
97           for my $n ( @{ $n_seq } ) {
98              my $result = `ping 192.168.1.${n}`;
99              ...
100           }
101        });
102

OUTPUT ITERATORS WITH INPUT

104       This module includes 2 output iterators which are useful for preserving
105       output order while gathering data. These cover the 2 general use cases.
106       The chunk_id value must be the first argument to gather. Gather must
107       also not be called more than once inside the block.
108
109   gather => MCE::Candy::out_iter_array( \@array )
110       The example utilizes the Core API with chunking disabled. Basically,
111       setting chunk_size to 1.
112
113        use MCE;
114        use MCE::Candy;
115
116        my @results;
117
118        my $mce = MCE->new(
119           chunk_size => 1, max_workers => 4,
120           gather => MCE::Candy::out_iter_array(\@results),
121           user_func => sub {
122              my ($mce, $chunk_ref, $chunk_id) = @_;
123              $mce->gather($chunk_id, $chunk_ref->[0] * 2);
124           }
125        );
126
127        $mce->process([ 100 .. 109 ]);
128
129        print "@results", "\n";
130
131        -- Output
132
133        200 202 204 206 208 210 212 214 216 218
134
135       Chunking may be desired for thousands or more items. In other words,
136       wanting to reduce the overhead placed on IPC.
137
138        use MCE;
139        use MCE::Candy;
140
141        my @results;
142
143        my $mce = MCE->new(
144           chunk_size => 100, max_workers => 4,
145           gather => MCE::Candy::out_iter_array(\@results),
146           user_func => sub {
147              my ($mce, $chunk_ref, $chunk_id) = @_;
148              my @output;
149              foreach my $item (@{ $chunk_ref }) {
150                 push @output, $item * 2;
151              }
152              $mce->gather($chunk_id, @output);
153           }
154        );
155
156        $mce->process([ 100_000 .. 200_000 - 1 ]);
157
158        print scalar @results, "\n";
159
160        -- Output
161
162        100000
163
164   gather => MCE::Candy::out_iter_fh( $fh )
165       Let's change things a bit and use MCE::Flow for the next 2 examples.
166       Chunking is not desired for the first example.
167
168        use MCE::Flow;
169        use MCE::Candy;
170
171        open my $fh, '>', '/tmp/foo.txt';
172
173        mce_flow {
174           chunk_size => 1, max_workers => 4,
175           gather => MCE::Candy::out_iter_fh($fh)
176        },
177        sub {
178           my ($mce, $chunk_ref, $chunk_id) = @_;
179           $mce->gather($chunk_id, $chunk_ref->[0] * 2, "\n");
180
181        }, (100 .. 109);
182
183        close $fh;
184
185        -- Output sent to '/tmp/foo.txt'
186
187        200
188        202
189        204
190        206
191        208
192        210
193        212
194        214
195        216
196        218
197
198       Chunking is desired for the next example due to processing many
199       thousands.
200
201        use MCE::Flow;
202        use MCE::Candy;
203
204        open my $fh, '>', '/tmp/foo.txt';
205
206        mce_flow {
207           chunk_size => 100, max_workers => 4,
208           gather => MCE::Candy::out_iter_fh( $fh )
209        },
210        sub {
211           my ($mce, $chunk_ref, $chunk_id) = @_;
212           my @output;
213           foreach my $item (@{ $chunk_ref }) {
214              push @output, ($item * 2) . "\n";
215           }
216           $mce->gather($chunk_id, @output);
217
218        }, (100_000 .. 200_000 - 1);
219
220        close $fh;
221
222        print -s '/tmp/foo.txt', "\n";
223
224        -- Output
225
226        700000
227

OUTPUT ITERATORS WITHOUT INPUT

229       Input data is not a requirement for using the output iterators included
230       in this module. The 'chunk_id' value is set uniquely and the same as
231       'wid' when not processing input data.
232
233   gather => MCE::Candy::out_iter_array( \@array )
234        use MCE::Flow;
235        use MCE::Candy;
236
237        my @results;
238
239        mce_flow {
240           max_workers => 'auto', ## Note that 'auto' is never higher than 8
241           gather => MCE::Candy::out_iter_array(\@results)
242        },
243        sub {
244           my ($mce) = @_;        ## This line is not necessary
245                                  ## Calling via module okay; e.g: MCE->method
246           ## Do work
247           ## Sending a complex data structure is allowed
248
249           ## Output will become orderly by iterator
250           $mce->gather( $mce->chunk_id, {
251              wid => $mce->wid, result => $mce->wid * 2
252           });
253        };
254
255        foreach my $href (@results) {
256           print $href->{wid} .": ". $href->{result} ."\n";
257        }
258
259        -- Output
260
261        1: 2
262        2: 4
263        3: 6
264        4: 8
265        5: 10
266        6: 12
267        7: 14
268        8: 16
269
270   gather => MCE::Candy::out_iter_fh( $fh )
271        use MCE::Flow;
272        use MCE::Candy;
273
274        open my $fh, '>', '/tmp/out.txt';
275
276        mce_flow {
277           max_workers => 'auto', ## See get_ncpu in <MCE::Util|MCE::Util>
278           gather => MCE::Candy::out_iter_fh($fh)
279        },
280        sub {
281           my $output = "# Worker ID: " . MCE->wid . "\n";
282
283           ## Append results to $output string
284           $output .= (MCE->wid * 2) . "\n\n";
285
286           ## Output will become orderly by iterator
287           MCE->gather( MCE->wid, $output );
288        };
289
290        close $fh;
291
292        -- Output
293
294        # Worker ID: 1
295        2
296
297        # Worker ID: 2
298        4
299
300        # Worker ID: 3
301        6
302
303        # Worker ID: 4
304        8
305
306        # Worker ID: 5
307        10
308
309        # Worker ID: 6
310        12
311
312        # Worker ID: 7
313        14
314
315        # Worker ID: 8
316        16
317

INDEX

319       MCE, MCE::Core
320

AUTHOR

322       Mario E. Roy, <marioeroy AT gmail DOT com>
323
324
325
326perl v5.28.1                      2019-01-23                     MCE::Candy(3)