MCE::Shared(3pm)

1MCE::Shared(3)        User Contributed Perl Documentation       MCE::Shared(3)
2
3
4

NAME

6       MCE::Shared - MCE extension for sharing data supporting threads and
7       processes
8

VERSION

10       This document describes MCE::Shared version 1.862
11

SYNOPSIS

13        # OO construction.
14
15        use MCE::Shared;
16
17        my $ar = MCE::Shared->array( @list );
18        my $ca = MCE::Shared->cache( max_keys => 500, max_age => 60 );
19        my $cv = MCE::Shared->condvar( 0 );
20        my $fh = MCE::Shared->handle( '>>', \*STDOUT ) or die "$!";
21        my $ha = MCE::Shared->hash( @pairs );
22        my $oh = MCE::Shared->ordhash( @pairs );
23        my $db = MCE::Shared->minidb();
24        my $qu = MCE::Shared->queue( await => 1, fast => 0 );
25        my $va = MCE::Shared->scalar( $value );
26        my $se = MCE::Shared->sequence( $begin, $end, $step, $fmt );
27        my $ob = MCE::Shared->share( $blessed_object );
28
29        # Mutex locking is supported for all shared objects since 1.841.
30        # Previously only shared C<condvar>s allowed locking.
31
32        $ar->lock;
33        $ar->unlock;
34        ...
35        $ob->lock;
36        $ob->unlock;
37
38        # The Perl-like mce_open function is available since 1.002.
39
40        mce_open my $fh, ">>", "/foo/bar.log" or die "open error: $!";
41
42        # Tie construction. The module API option is available since 1.825.
43
44        use v5.10;
45        use MCE::Flow;
46        use MCE::Shared;
47
48        my %args  = ( max_keys => 500, max_age => 60 );
49        my @pairs = ( foo => 'bar', woo => 'baz' );
50        my @list  = ( 'a' .. 'z' );
51
52        tie my $va1, 'MCE::Shared', { module => 'MCE::Shared::Scalar' }, 0;
53        tie my @ar1, 'MCE::Shared', { module => 'MCE::Shared::Array' }, @list;
54        tie my %ca1, 'MCE::Shared', { module => 'MCE::Shared::Cache' }, %args;
55        tie my %ha1, 'MCE::Shared', { module => 'MCE::Shared::Hash' }, @pairs;
56        tie my %oh1, 'MCE::Shared', { module => 'MCE::Shared::Ordhash' }, @pairs;
57        tie my %oh2, 'MCE::Shared', { module => 'Hash::Ordered' }, @pairs;
58        tie my %oh3, 'MCE::Shared', { module => 'Tie::IxHash' }, @pairs;
59        tie my $cy1, 'MCE::Shared', { module => 'Tie::Cycle' }, [ 1 .. 8 ];
60        tie my $va2, 'MCE::Shared', { module => 'Tie::StdScalar' }, 'hello';
61        tie my @ar3, 'MCE::Shared', { module => 'Tie::StdArray' }, @list;
62        tie my %ha2, 'MCE::Shared', { module => 'Tie::StdHash' }, @pairs;
63        tie my %ha3, 'MCE::Shared', { module => 'Tie::ExtraHash' }, @pairs;
64
65        tie my $cnt, 'MCE::Shared', 0; # default MCE::Shared::Scalar
66        tie my @foo, 'MCE::Shared';    # default MCE::Shared::Array
67        tie my %bar, 'MCE::Shared';    # default MCE::Shared::Hash
68
69        tie my @ary, 'MCE::Shared', qw( a list of values );
70        tie my %ha,  'MCE::Shared', key1 => 'val1', key2 => 'val2';
71        tie my %ca,  'MCE::Shared', { max_keys => 500, max_age => 60 };
72        tie my %oh,  'MCE::Shared', { ordered => 1 }, key1 => 'value';
73
74        # Mutex locking is supported for all shared objects since 1.841.
75
76        tied($va1)->lock;
77        tied($va1)->unlock;
78        ...
79        tied(%bar)->lock;
80        tied(%bar)->unlock;
81
82        # Demonstration.
83
84        my $mutex = MCE::Mutex->new;
85
86        mce_flow {
87           max_workers => 4
88        },
89        sub {
90           my ( $mce ) = @_;
91           my ( $pid, $wid ) = ( MCE->pid, MCE->wid );
92
93           # Locking is necessary when multiple workers update the same
94           # element. The reason is that it may involve 2 trips to the
95           # shared-manager process: fetch and store in this case.
96
97           $mutex->enter( sub { $cnt += 1 } );
98
99           # Otherwise, locking is optional for unique elements.
100
101           $foo[ $wid - 1 ] = $pid;
102           $bar{ $pid }     = $wid;
103
104           # From 1.841 onwards, all shared objects include mutex locking
105           # to not need to construct MCE::Mutex separately.
106
107           tied($va1)->lock;
108           $va1 += 1;
109           tied($va1)->unlock;
110
111           return;
112        };
113
114        say "scalar : $cnt";
115        say "scalar : $va1";
116        say " array : $_" for (@foo);
117        say "  hash : $_ => $bar{$_}" for (sort keys %bar);
118
119        __END__
120
121        # Output
122
123        scalar : 4
124        scalar : 4
125         array : 37847
126         array : 37848
127         array : 37849
128         array : 37850
129          hash : 37847 => 1
130          hash : 37848 => 2
131          hash : 37849 => 3
132          hash : 37850 => 4
133

DESCRIPTION

135       This module provides data sharing capabilities for MCE supporting
136       threads and processes. MCE::Hobo provides threads-like parallelization
137       for running code asynchronously.
138

EXTRA FUNCTIONALITY

140       MCE::Shared enables extra functionality on systems with IO::FDPass
141       installed.  Without it, MCE::Shared is unable to send file descriptors
142       to the shared-manager process. The use applies to Condvar, Queue, and
143       Handle (mce_open). IO::FDpass isn't used for anything else.
144
145        use MCE::Shared;
146
147        # One may want to start the shared-manager early.
148
149        MCE::Shared->start();
150
151        # Typically, the shared-manager is started automatically when
152        # constructing a shared object.
153
154        my $ca = MCE::Shared->cache( max_keys => 500 );
155
156        # IO::FDPass is necessary for constructing a shared condvar or queue
157        # while the manager is running in order to send file descriptors
158        # associated with the object.
159
160        # Workers block using a socket handle for ->wait and ->timedwait.
161
162        my $cv = MCE::Shared->condvar();
163
164        # Workers block using a socket handle for ->dequeue and ->await.
165
166        my $q1 = MCE::Shared->queue();
167        my $q2 = MCE::Shared->queue( await => 1 );
168
169       For platforms where IO::FDPass isn't possible (e.g. Cygwin), construct
170       "condvar" and "queue" before other classes. The shared-manager process
171       will be delayed until sharing other classes (e.g. Array, Hash) or
172       starting explicitly.
173
174        use MCE::Shared;
175
176        my $has_IO_FDPass = $INC{'IO/FDPass.pm'} ? 1 : 0;
177
178        my $cv  = MCE::Shared->condvar( 0 );
179        my $que = MCE::Shared->queue( fast => 1 );
180
181        MCE::Shared->start() unless $has_IO_FDPass;
182
183        my $ha = MCE::Shared->hash();  # started implicitly
184
185       Note that MCE starts the shared-manager, prior to spawning workers, if
186       not yet started. Ditto for MCE::Hobo.
187
188       Regarding mce_open, "IO::FDPass" is needed for constructing a shared-
189       handle from a non-shared handle not yet available inside the shared-
190       manager process.  The workaround is to have the non-shared handle made
191       before the shared-manager is started. Passing a file by reference is
192       fine for the three STD* handles.
193
194        # The shared-manager knows of \*STDIN, \*STDOUT, \*STDERR.
195
196        mce_open my $shared_in,  "<",  \*STDIN;   # ok
197        mce_open my $shared_out, ">>", \*STDOUT;  # ok
198        mce_open my $shared_err, ">>", \*STDERR;  # ok
199        mce_open my $shared_fh1, "<",  "/path/to/sequence.fasta";  # ok
200        mce_open my $shared_fh2, ">>", "/path/to/results.log";     # ok
201
202        mce_open my $shared_fh, ">>", \*NON_SHARED_FH;  # requires IO::FDPass
203
204       The IO::FDPass module is known to work reliably on most platforms.
205       Install 1.1 or later to rid of limitations described above.
206
207        perl -MIO::FDPass -le "print 'Cheers! Perl has IO::FDPass.'"
208

DATA SHARING

210       MCE::Shared->array MCE::Shared::Array
211       MCE::Shared->cache MCE::Shared::Cache
212       MCE::Shared->condvar MCE::Shared::Condvar
213       MCE::Shared->handle MCE::Shared::Handle
214       MCE::Shared->hash MCE::Shared::Hash
215       MCE::Shared->minidb MCE::Shared::Minidb
216       MCE::Shared->ordhash MCE::Shared::Ordhash
217       MCE::Shared->queue MCE::Shared::Queue
218       MCE::Shared->scalar MCE::Shared::Scalar
219       MCE::Shared->sequence MCE::Shared::Sequence
220
221       Below, synopsis for sharing classes included with MCE::Shared.
222
223        use MCE::Shared;
224
225        # short form
226
227        $ar = MCE::Shared->array( @list );
228        $ca = MCE::Shared->cache( max_keys => 500, max_age => 60 );
229        $cv = MCE::Shared->condvar( 0 );
230        $fh = MCE::Shared->handle( ">>", \*STDOUT ); # see mce_open below
231        $ha = MCE::Shared->hash( @pairs );
232        $db = MCE::Shared->minidb();
233        $oh = MCE::Shared->ordhash( @pairs );
234        $qu = MCE::Shared->queue( await => 1, fast => 0 );
235        $va = MCE::Shared->scalar( $value );
236        $se = MCE::Shared->sequence( $begin, $end, $step, $fmt );
237
238        mce_open my $fh, ">>", \*STDOUT or die "open error: $!";
239
240        # long form
241
242        $ar = MCE::Shared->share( { module => 'MCE::Shared::Array'    }, ... );
243        $ca = MCE::Shared->share( { module => 'MCE::Shared::Cache'    }, ... );
244        $cv = MCE::Shared->share( { module => 'MCE::Shared::Condvar'  }, ... );
245        $fh = MCE::Shared->share( { module => 'MCE::Shared::Handle'   }, ... );
246        $ha = MCE::Shared->share( { module => 'MCE::Shared::Hash'     }, ... );
247        $db = MCE::Shared->share( { module => 'MCE::Shared::Minidb'   }, ... );
248        $oh = MCE::Shared->share( { module => 'MCE::Shared::Ordhash'  }, ... );
249        $qu = MCE::Shared->share( { module => 'MCE::Shared::Queue'    }, ... );
250        $va = MCE::Shared->share( { module => 'MCE::Shared::Scalar'   }, ... );
251        $se = MCE::Shared->share( { module => 'MCE::Shared::Sequence' }, ... );
252
253       The restriction for sharing classes not included with MCE::Shared is
254       that the object must not have file-handles nor code-blocks.
255
256        $oh = MCE::Shared->share( { module => 'Hash::Ordered' }, ... );
257
258       open ( filehandle, expr )
259       open ( filehandle, mode, expr )
260       open ( filehandle, mode, reference )
261
262       In version 1.002 and later, constructs a new object by opening the file
263       whose filename is given by "expr", and associates it with "filehandle".
264       When omitting error checking at the application level, MCE::Shared
265       emits a message and stop if open fails.
266
267       See MCE::Shared::Handle for chunk IO demonstrations.
268
269        {
270          use MCE::Shared::Handle;
271
272          # "non-shared" or "local construction" for use by a single process
273          MCE::Shared::Handle->open( my $fh, "<", "file.log" ) or die "$!";
274          MCE::Shared::Handle::open  my $fh, "<", "file.log"   or die "$!";
275
276          # mce_open is an alias for MCE::Shared::Handle::open
277          mce_open my $fh, "<", "file.log" or die "$!";
278        }
279
280        {
281          use MCE::Shared;
282
283          # construction for "sharing" with other threads and processes
284          MCE::Shared->open( my $fh, "<", "file.log" ) or die "$!";
285          MCE::Shared::open  my $fh, "<", "file.log"   or die "$!";
286
287          # mce_open is an alias for MCE::Shared::open
288          mce_open my $fh, "<", "file.log" or die "$!";
289        }
290
291       mce_open ( filehandle, expr )
292       mce_open ( filehandle, mode, expr )
293       mce_open ( filehandle, mode, reference )
294
295       Native Perl-like syntax to open a shared-file for reading:
296
297        use MCE::Shared;
298
299        # mce_open is exported by MCE::Shared or MCE::Shared::Handle.
300        # It creates a shared file handle with MCE::Shared present
301        # or a non-shared handle otherwise.
302
303        mce_open my $fh, "< input.txt"     or die "open error: $!";
304        mce_open my $fh, "<", "input.txt"  or die "open error: $!";
305        mce_open my $fh, "<", \*STDIN      or die "open error: $!";
306
307       and for writing:
308
309        mce_open my $fh, "> output.txt"    or die "open error: $!";
310        mce_open my $fh, ">", "output.txt" or die "open error: $!";
311        mce_open my $fh, ">", \*STDOUT     or die "open error: $!";
312
313       num_sequence
314
315       "num_sequence" is an alias for "sequence".
316

DEEPLY SHARING

318       The following is a demonstration for a shared tied-hash variable.
319       Before venturing into the actual code, notice the dump function making
320       a call to "export" explicitly for objects of type
321       "MCE::Shared::Object". This is necessary in order to retrieve the data
322       from the shared-manager process.
323
324       The "export" method is described later under the Common API section.
325
326        use MCE::Shared;
327
328        sub _dump {
329           require Data::Dumper unless $INC{'Data/Dumper.pm'};
330           no warnings 'once';
331
332           local $Data::Dumper::Varname  = 'VAR';
333           local $Data::Dumper::Deepcopy = 1;
334           local $Data::Dumper::Indent   = 1;
335           local $Data::Dumper::Purity   = 1;
336           local $Data::Dumper::Sortkeys = 0;
337           local $Data::Dumper::Terse    = 0;
338
339           ( ref $_[0] eq 'MCE::Shared::Object' )
340              ? print Data::Dumper::Dumper( $_[0]->export ) . "\n"
341              : print Data::Dumper::Dumper( $_[0] ) . "\n";
342        }
343
344        tie my %abc, 'MCE::Shared';
345
346        my @parents = qw( a b c );
347        my @children = qw( 1 2 3 4 );
348
349        for my $parent ( @parents ) {
350           for my $child ( @children ) {
351              $abc{ $parent }{ $child } = 1;
352           }
353        }
354
355        _dump( tied( %abc ) );
356
357        __END__
358
359        # Output
360
361        $VAR1 = bless( {
362          'c' => bless( {
363            '1' => '1',
364            '4' => '1',
365            '3' => '1',
366            '2' => '1'
367          }, 'MCE::Shared::Hash' ),
368          'a' => bless( {
369            '1' => '1',
370            '4' => '1',
371            '3' => '1',
372            '2' => '1'
373          }, 'MCE::Shared::Hash' ),
374          'b' => bless( {
375            '1' => '1',
376            '4' => '1',
377            '3' => '1',
378            '2' => '1'
379          }, 'MCE::Shared::Hash' )
380        }, 'MCE::Shared::Hash' );
381
382       Dereferencing provides hash-like behavior for "hash" and "ordhash".
383       Array-like behavior is allowed for "array", not shown below.
384
385        use MCE::Shared;
386        use Data::Dumper;
387
388        my $abc = MCE::Shared->hash;
389
390        my @parents = qw( a b c );
391        my @children = qw( 1 2 3 4 );
392
393        for my $parent ( @parents ) {
394           for my $child ( @children ) {
395              $abc->{ $parent }{ $child } = 1;
396           }
397        }
398
399        print Dumper( $abc->export({ unbless => 1 }) ), "\n";
400
401       Each level in a deeply structure requires a separate trip to the
402       shared-manager process. The included "MCE::Shared::Minidb" module
403       provides optimized methods for working with hash of hashes "HoH" and
404       hash of arrays "HoA".
405
406        use MCE::Shared;
407        use Data::Dumper;
408
409        my $abc = MCE::Shared->minidb;
410
411        my @parents = qw( a b c );
412        my @children = qw( 1 2 3 4 );
413
414        for my $parent ( @parents ) {
415           for my $child ( @children ) {
416              $abc->hset($parent, $child, 1);
417           }
418        }
419
420        print Dumper( $abc->export ), "\n";
421
422       For further reading, see MCE::Shared::Minidb.
423

OBJECT SHARING

425       MCE::Shared->share
426
427       This class method transfers the blessed-object to the shared-manager
428       process and returns a "MCE::Shared::Object" containing the "SHARED_ID".
429       Starting with the 1.827 release, the "module" option sends parameters
430       to the shared-manager, where the object is then constructed. This is
431       useful for classes involving XS code or a file handle.
432
433        use MCE::Shared;
434
435        {
436          use Math::BigFloat try => 'GMP';
437          use Math::BigInt   try => 'GMP';
438
439          my $bf  = MCE::Shared->share({ module => 'Math::BigFloat' }, 0);
440          my $bi  = MCE::Shared->share({ module => 'Math::BigInt'   }, 0);
441          my $y   = 1e9;
442
443          $bf->badd($y);  # addition (add $y to shared BigFloat object)
444          $bi->badd($y);  # addition (add $y to shared BigInt object)
445        }
446
447        {
448          use Bio::Seq;
449          use Bio::SeqIO;
450
451          my $seq_io = MCE::Shared->share({ module => 'Bio::SeqIO' },
452             -file    => ">/path/to/fasta/file.fa",
453             -format  => 'Fasta',
454             -verbose => -1,
455          );
456
457          my $seq_obj = Bio::Seq->new(
458             -display_id => "name", -desc => "desc", -seq => "seq",
459             -alphabet   => "dna"
460          );
461
462          $seq_io->write_seq($seq_obj);  # write to shared SeqIO handle
463        }
464
465        {
466          my $oh1 = MCE::Shared->share({ module => 'MCE::Shared::Ordhash' });
467          my $oh2 = MCE::Shared->ordhash();  # same thing
468
469          $oh1->assign( @pairs );
470          $oh2->assign( @pairs );
471        }
472
473        {
474          my ($ho_shared, $ho_nonshared);
475
476          $ho_shared = MCE::Shared->share({ module => 'Hash::Ordered' });
477          $ho_shared->push( @pairs );
478
479          $ho_nonshared = $ho_shared->export();   # back to non-shared
480          $ho_nonshared = $ho_shared->destroy();  # including shared destruction
481        }
482
483       The following provides long and short forms for constructing a shared
484       array, hash, or scalar object.
485
486        use MCE::Shared;
487
488        my $a1 = MCE::Shared->share( { module => 'MCE::Shared::Array' }, @list );
489        my $a2 = MCE::Shared->share( [ @list ] );
490        my $a3 = MCE::Shared->array( @list );
491
492        my $h1 = MCE::Shared->share( { module => 'MCE::Shared::Hash' }, @pairs );
493        my $h2 = MCE::Shared->share( { @pairs } );
494        my $h3 = MCE::Shared->hash( @pairs );
495
496        my $s1 = MCE::Shared->share( { module => 'MCE::Shared::Scalar' }, 20 );
497        my $s2 = MCE::Shared->share( \do{ my $o = 20 } );
498        my $s3 = MCE::Shared->scalar( 20 );
499
500       When the "module" option is given, one may optionally specify the
501       constructor function via the "new" option. This is necessary for the
502       CDB_File module, which provides two different objects. One is created
503       by new (default), and accessed by insert and finish. The other is
504       created by TIEHASH, and accessed by FETCH.
505
506        use MCE::Hobo;
507        use MCE::Shared;
508
509        # populate CDB file
510        my $cdb = MCE::Shared->share({ module => 'CDB_File' }, 't.cdb', "t.cdb.$$")
511           or die "$!\n";
512
513        $cdb->insert( $_ => $_ ) for ('aa'..'zz');
514        $cdb->finish;
515
516        # use CDB file
517        my $cdb1 = tie my %hash, 'MCE::Shared', { module => 'CDB_File' }, 't.cdb';
518
519        # same thing, without involving TIE and extra hash variable
520        my $cdb2 = MCE::Shared->share(
521           { module => 'CDB_File', new => 'TIEHASH' }, 't.cdb'
522        );
523
524        print $hash{'aa'}, "\n";
525        print $cdb1->FETCH('bb'), "\n";
526        print $cdb2->FETCH('cc'), "\n";
527
528        # rewind may be omitted on first use for parallel iteration
529        $cdb2->rewind;
530
531        for ( 1 .. 3 ) {
532           mce_async {
533              while ( my ($k,$v) = $cdb2->next ) {
534                 print "[$$] $k => $v\n";
535              }
536           };
537        }
538
539        MCE::Hobo->waitall;
540

DBM SHARING

542       Construting a shared DBM object is possible starting with the 1.827
543       release.  Supported modules are AnyDBM_File, BerkeleyDB, CDB_File,
544       DB_File, GDBM_File, NDBM_File, ODBM_File, SDBM_File, SQLite_File,
545       Tie::Array::DBD, and Tie::Hash::DBD. The list includes Tokyo Cabinet
546       <http://fallabs.com/tokyocabinet/> and Kyoto Cabinet
547       <http://fallabs.com/kyotocabinet/>. Also, see forked version by Altice
548       Labs <https://github.com/alticelabs/kyoto>. It contains an updated
549       "kyotocabinet" folder that builds successfully with recent compilers.
550
551       Freeze-thaw during "STORE"-"FETCH" (for complex data) is handled
552       automatically using Serial 3.015+ (if available) or Storable. Below,
553       are constructions for sharing various DBM modules. The construction for
554       "CDB_File" is given in the prior section.
555
556       AnyDBM_File
557
558        BEGIN { @AnyDBM_File::ISA = qw( DB_File GDBM_File NDBM_File ODBM_File ); }
559
560        use MCE::Shared;
561        use Fcntl;
562        use AnyDBM_File;
563
564        tie my %h1, 'MCE::Shared', { module => 'AnyDBM_File' },
565           'foo_a', O_CREAT|O_RDWR or die "open error: $!";
566
567       BerkeleyDB
568
569        use MCE::Shared;
570        use BerkeleyDB;
571
572        tie my %h1, 'MCE::Shared', { module => 'BerkeleyDB::Hash' },
573           -Filename => 'foo_a', -Flags => DB_CREATE
574              or die "open error: $!";
575
576        tie my %h2, 'MCE::Shared', { module => 'BerkeleyDB::Btree' },
577           -Filename => 'foo_b', -Flags => DB_CREATE
578              or die "open error: $!";
579
580        tie my @a1, 'MCE::Shared', { module => 'BerkeleyDB::Queue' },
581           -Filename => 'foo_c', -Flags => DB_CREATE
582              or die "open error: $!";
583
584        tie my @a2, 'MCE::Shared', { module => 'BerkeleyDB::Recno' },
585           -Filename => 'foo_d', -Flags => DB_CREATE -Len => 20
586              or die "open error: $!";
587
588       DB_File
589
590        use MCE::Shared;
591        use Fcntl;
592        use DB_File;
593
594        # Use pre-defined references ( $DB_HASH, $DB_BTREE, $DB_RECNO ).
595
596        tie my %h1, 'MCE::Shared', { module => 'DB_File' },
597           'foo_a', O_CREAT|O_RDWR, 0640, $DB_HASH or die "open error: $!";
598
599        tie my %h2, 'MCE::Shared', { module => 'DB_File' },
600           'foo_b', O_CREAT|O_RDWR, 0640, $DB_BTREE or die "open error: $!";
601
602        tie my @a1, 'MCE::Shared', { module => 'DB_File' },
603           'foo_c', O_CREAT|O_RDWR, 0640, $DB_RECNO or die "open error: $!";
604
605        # Changing defaults - see DB_File for valid options.
606
607        my $opt_h = DB_File::HASHINFO->new();
608        my $opt_b = DB_File::BTREEINFO->new();
609        my $opt_r = DB_File::RECNOINFO->new();
610
611        $opt_h->{'cachesize'} = 12345;
612
613        tie my %h3, 'MCE::Shared', { module => 'DB_File' },
614           'foo_d', O_CREAT|O_RDWR, 0640, $opt_h or die "open error: $!";
615
616       KyotoCabinet
617       TokyoCabinet
618
619        use MCE::Shared;
620        use KyotoCabinet;
621        use TokyoCabinet;
622
623        # file extension denotes hash database
624
625        tie my %h1, 'MCE::Shared', { module => 'KyotoCabinet::DB' }, 'foo.kch',
626           KyotoCabinet::DB::OWRITER | KyotoCabinet::DB::OCREATE
627              or die "open error: $!";
628
629        tie my %h2, 'MCE::Shared', { module => 'TokyoCabinet::HDB' }, 'foo.tch',
630           TokyoCabinet::HDB::OWRITER | TokyoCabinet::HDB::OCREAT
631              or die "open error: $!";
632
633        # file extension denotes tree database
634
635        tie my %h3, 'MCE::Shared', { module => 'KyotoCabinet::DB' }, 'foo.kct',
636           KyotoCabinet::DB::OWRITER | KyotoCabinet::DB::OCREATE
637              or die "open error: $!";
638
639        tie my %h4, 'MCE::Shared', { module => 'TokyoCabinet::BDB' }, 'foo.tcb',
640           TokyoCabinet::BDB::OWRITER | TokyoCabinet::BDB::OCREAT
641              or die "open error: $!";
642
643        # on-memory hash database
644
645        tie my %h5, 'MCE::Shared', { module => 'KyotoCabinet::DB' }, '*';
646        tie my %h6, 'MCE::Shared', { module => 'TokyoCabinet::ADB' }, '*';
647
648        # on-memory tree database
649
650        tie my %h7, 'MCE::Shared', { module => 'KyotoCabinet::DB' }, '%#pccap=256m';
651        tie my %h8, 'MCE::Shared', { module => 'TokyoCabinet::ADB' }, '+';
652
653       Tie::Array::DBD
654       Tie::Hash::DBD
655
656        use MCE::Shared;
657        use Tie::Array::DBD;
658        use Tie::Hash::DBD;
659
660        # A valid string is required for the DSN argument, not a DBI handle.
661        # Do not specify the 'str' option for Tie::(Array|Hash)::DBD.
662        # Instead, see encoder-decoder methods described under Common API.
663
664        use DBD::SQLite;
665
666        tie my @a1, 'MCE::Shared', { module => 'Tie::Array::DBD' },
667           'dbi:SQLite:dbname=foo_a.db', {
668              tbl => 't_tie_analysis',
669              key => 'h_key',
670              fld => 'h_value'
671           };
672
673        tie my %h1, 'MCE::Shared', { module => 'Tie::Hash::DBD' },
674           'dbi:SQLite:dbname=foo_h.db', {
675              tbl => 't_tie_analysis',
676              key => 'h_key',
677              fld => 'h_value'
678           };
679
680        use DBD::CSV;
681
682        tie my %h2, 'MCE::Shared', { module => 'Tie::Hash::DBD'},
683           'dbi:CSV:f_dir=.;f_ext=.csv/r;csv_null=1;csv_decode_utf8=0', {
684              tbl => 'mytable',
685              key => 'h_key',
686              fld => 'h_value'
687           };
688
689        # By default, Sereal 3.015+ is used for serialization if available.
690        # This overrides serialization from Sereal-or-Storable to JSON::XS.
691
692        use JSON::XS ();
693
694        tied(%h2)->encoder( \&JSON::XS::encode_json );
695        tied(%h2)->decoder( \&JSON::XS::decode_json );
696
697        my @pairs = ( key1 => 'val1', key2 => 'val2' );
698        my @list  = ( 1, 2, 3, 4 );
699
700        $h2{'foo'} = 'plain value';
701        $h2{'bar'} = { @pairs };
702        $h2{'baz'} = [ @list ];
703

DBM SHARING (CONT)

705       DB cursors, filters, and duplicate keys are not supported, just plain
706       array and hash functionality. The OO interface provides better
707       performance when needed.  Use "iterator" or "next" for iterating over
708       the elements.
709
710        use MCE::Hobo;
711        use MCE::Shared;
712        use Fcntl;
713        use DB_File;
714
715        unlink 'foo_a';
716
717        my $ob = tie my %h1, 'MCE::Shared', { module => 'DB_File' },
718           'foo_a', O_CREAT|O_RDWR, 0640, $DB_HASH or die "open error: $!";
719
720        $h1{key} = 'value';
721        my $val = $h1{key};
722
723        while ( my ($k, $v) = each %h1 ) {
724           print "1: $k => $v\n";
725        }
726
727        # object oriented fashion, faster
728
729        tied(%h1)->STORE( key1 => 'value1' );
730        my $val1 = tied(%h1)->FETCH('key1');
731
732        $ob->STORE( key2 => 'value2' );
733        my $val2 = $ob->FETCH('key2');
734
735        # non-parallel iteration
736
737        my $iter = $ob->iterator;
738        while ( my ($k, $v) = $iter->() ) {
739           print "2: $k => $v\n";
740        }
741
742        # parallel iteration
743
744        sub task {
745           while ( my ($k, $v) = $ob->next ) {
746              print "[$$] $k => $v\n";
747              sleep 1;
748           }
749        }
750
751        MCE::Hobo->create(\&task) for 1 .. 3;
752        MCE::Hobo->waitall;
753
754        $ob->rewind;
755
756        # undef $ob and $iter before %h1 when destroying manually
757
758        undef $ob;
759        undef $iter;
760
761        untie %h1;
762
763       See also Tie::File Demonstration, at the end of the documentation.
764

PDL SHARING

766       MCE::Shared->pdl_byte
767       MCE::Shared->pdl_short
768       MCE::Shared->pdl_ushort
769       MCE::Shared->pdl_long
770       MCE::Shared->pdl_longlong
771       MCE::Shared->pdl_float
772       MCE::Shared->pdl_double
773       MCE::Shared->pdl_ones
774       MCE::Shared->pdl_random
775       MCE::Shared->pdl_sequence
776       MCE::Shared->pdl_zeroes
777       MCE::Shared->pdl_indx
778       MCE::Shared->pdl
779
780       Sugar syntax for PDL construction to take place under the shared-
781       manager process. The helper routines are made available only if "PDL"
782       is loaded before "MCE::Shared".
783
784        use PDL;
785        use MCE::Shared;
786
787        # This makes an extra copy, transfer, including destruction.
788        my $ob1 = MCE::Shared->share( zeroes( 256, 256 ) );
789
790        # Do this instead to not involve an extra copy.
791        my $ob1 = MCE::Shared->pdl_zeroes( 256, 256 );
792
793       Below is a parallel version for a demonstration on PerlMonks.
794
795        # https://www.perlmonks.org/?node_id=1214227 (by vr)
796
797        use strict;
798        use warnings;
799        use feature 'say';
800
801        use PDL;  # must load PDL before MCE::Shared
802
803        use MCE;
804        use MCE::Shared;
805        use Time::HiRes 'time';
806
807        srand( 123 );
808
809        my $time = time;
810
811        my $n = 30000;      # input sample size
812        my $m = 10000;      # number of bootstrap repeats
813        my $r = $n;         # re-sample size
814
815        # On Windows, the non-shared piddle ($x) is unblessed in threads.
816        # Therefore, constructing the piddle inside the worker.
817        # UNIX platforms benefit from copy-on-write. Thus, one copy.
818
819        my $x   = ( $^O eq 'MSWin32' ) ? undef : random( $n );
820        my $avg = MCE::Shared->pdl_zeroes( $m );
821
822        MCE->new(
823           max_workers => 4,
824           sequence    => [ 0, $m - 1 ],
825           chunk_size  => 1,
826           user_begin  => sub {
827              $x = random( $n ) unless ( defined $x );
828           },
829           user_func   => sub {
830              my $idx  = random $r;
831              $idx    *= $n;
832              # $avg is a shared piddle which resides inside the shared-
833              # manager process or thread. The piddle is accessible via the
834              # OO interface only.
835              $avg->set( $_, $x->index( $idx )->avg );
836           }
837        )->run;
838
839        # MCE sets the seed of the base generator uniquely between workers.
840        # Unfortunately, it requires running with one worker for predictable
841        # results (i.e. no guarantee in the order which worker computes the
842        # next input chunk).
843
844        say $avg->pctover( pdl 0.05, 0.95 );
845        say time - $time, ' seconds';
846
847        __END__
848
849        # Output
850
851        [0.49387106  0.4993768]
852        1.09556317329407 seconds
853
854       ins_inplace
855
856       The "ins_inplace" method applies to shared PDL objects. It supports
857       three forms for writing elements back to the PDL object, residing under
858       the shared-manager process.
859
860        # --- action taken by the shared-manager process
861        # ins_inplace(  1 arg  ):  ins( inplace( $this ), $what, 0, 0 );
862        # ins_inplace(  2 args ):  $this->slice( $arg1 ) .= $arg2;
863        # ins_inplace( >2 args ):  ins( inplace( $this ), $what, @coords );
864
865        # --- use case
866        $o->ins_inplace( $result );                    #  1 arg
867        $o->ins_inplace( ":,$start:$stop", $result );  #  2 args
868        $o->ins_inplace( $result, 0, $seq_n );         # >2 args
869
870       Operations such as " + 5 " will not work on shared PDL objects. At this
871       time, the OO interface is the only mechanism for communicating with the
872       shared piddle. For example, call "slice", "sever", or "copy" to fetch
873       elements. Call "ins_inplace" or "set" (shown above) to update elements.
874
875        use strict;
876        use warnings;
877
878        use PDL;  # must load PDL before MCE::Shared
879        use MCE::Shared;
880
881        # make a shared piddle
882        my $b = MCE::Shared->pdl_sequence(15,15);
883
884        # fetch, add 10 to row 2 only
885        my $res1 = $b->slice(":,1:1") + 10;
886        $b->ins_inplace($res1, 0, 1);
887
888        # fetch, add 10 to rows 4 and 5
889        my $res2 = $b->slice(":,3:4") + 10;
890        $b->ins_inplace($res2, 0, 3);
891
892        # make non-shared object (i.e. export-destroy from shared)
893        $b = $b->destroy;
894
895        print "$b\n";
896
897       The following provides parallel demonstrations using "MCE::Flow".
898
899        use strict;
900        use warnings;
901
902        use PDL;  # must load PDL before MCE::Shared
903
904        use MCE::Flow;
905        use MCE::Shared;
906
907        # On Windows, the ($a) piddle is unblessed in worker threads.
908        # Therefore, constructing ($a) inside the worker versus sharing.
909        # UNIX platforms benefit from copy-on-write. Thus, one copy.
910        #
911        # Results are stored in the shared piddle ($b).
912
913        my $a = ( $^O eq 'MSWin32' ) ? undef : sequence(15,15);
914        my $b = MCE::Shared->pdl_zeroes(15,15);
915
916        MCE::Flow->init(
917           user_begin => sub {
918              $a = sequence(15,15) unless ( defined $a );
919           }
920        );
921
922        # with chunking disabled
923
924        mce_flow_s {
925           max_workers => 4, chunk_size => 1
926        },
927        sub {
928           my $row = $_;
929           my $result = $a->slice(":,$row:$row") + 5;
930           $b->ins_inplace($result, 0, $row);
931        }, 0, 15 - 1;
932
933        # with chunking enabled
934
935        mce_flow_s {
936           max_workers => 4, chunk_size => 5, bounds_only => 1
937        },
938        sub {
939           my ($row1, $row2) = @{ $_ };
940           my $result = $a->slice(":,$row1:$row2") + 5;
941           $b->ins_inplace($result, 0, $row1);
942        }, 0, 15 - 1;
943
944        # make non-shared object, export-destroy the shared object
945
946        $b = $b->destroy;
947
948        print "$b\n";
949
950       See also PDL::ParallelCPU and PDL::Parallel::threads. For further
951       reading, the MCE-Cookbook on GitHub provides two PDL demonstrations.
952
953       <https://github.com/marioroy/mce-cookbook>
954

COMMON API

956       blessed
957          Returns the real "blessed" name, provided by the shared-manager
958          process.
959
960           use MCE::Shared;
961           use Scalar::Util qw(blessed);
962
963           my $oh1 = MCE::Shared->share({ module => 'MCE::Shared::Ordhash' });
964           my $oh2 = MCE::Shared->share({ module => 'Hash::Ordered'        });
965
966           print blessed($oh1), "\n";    # MCE::Shared::Object
967           print blessed($oh2), "\n";    # MCE::Shared::Object
968
969           print $oh1->blessed(), "\n";  # MCE::Shared::Ordhash
970           print $oh2->blessed(), "\n";  # Hash::Ordered
971
972       destroy ( { unbless => 1 } )
973       destroy
974          Exports optionally, but destroys the shared object entirely from the
975          shared-manager process. The unbless option is passed to export.
976
977           my $exported_ob = $shared_ob->destroy();
978
979           $shared_ob;     # becomes undef
980
981       lock
982       unlock
983          Shared objects embed a MCE::Mutex object for locking since 1.841.
984
985           use MCE::Shared;
986
987           tie my @shared_array, 'MCE::Shared', { module => 'MCE::Shared::Array' }, 0;
988
989           tied(@shared_array)->lock;
990           $shared_array[0] += 1;
991           tied(@shared_array)->unlock;
992
993           print $shared_array[0], "\n";    # 1
994
995          Locking is not necessary typically when using the OO interface.
996          Although, exclusive access is necessary when involving a FETCH and
997          STORE.
998
999           my $shared_total = MCE::Shared->scalar(2);
1000
1001           $shared_total->lock;
1002           my $val = $shared_total->get;
1003           $shared_total->set( $val * 2 );
1004           $shared_total->unlock;
1005
1006           print $shared_total->get, "\n";  # 4
1007
1008       encoder ( CODE )
1009       decoder ( CODE )
1010          Override freeze/thaw routines. Applies to STORE and FETCH only,
1011          particularly for TIE'd objects. These are called internally for
1012          shared DB objects.
1013
1014          Current API available since 1.827.
1015
1016           use MCE::Shared;
1017           use BerkeleyDB;
1018           use DB_File;
1019
1020           my $file1 = 'file1.db';
1021           my $file2 = 'file2.db';
1022
1023           tie my @db1, 'MCE::Shared', { module => 'DB_File' }, $file1,
1024              O_RDWR|O_CREAT, 0640 or die "open error '$file1': $!";
1025
1026           tie my %db2, 'MCE::Shared', { module => 'BerkeleyDB::Hash' },
1027              -Filename => $file2, -Flags => DB_CREATE
1028              or die "open error '$file2': $!";
1029
1030           # Called automatically by MCE::Shared for DB files.
1031           # tied(@db1)->encoder( MCE::Shared::Server::_get_freeze );
1032           # tied(@db1)->decoder( MCE::Shared::Server::_get_thaw );
1033           # tied(%db2)->encoder( MCE::Shared::Server::_get_freeze );
1034           # tied(%db2)->decoder( MCE::Shared::Server::_get_thaw );
1035           # et cetera.
1036
1037           $db1[0] = 'foo';   # store plain and complex structure
1038           $db1[1] = { key => 'value' };
1039           $db1[2] = [ 'complex' ];
1040
1041           $db2{key} = 'foo'; # ditto, plain and complex structure
1042           $db2{sun} = [ 'complex' ];
1043
1044       export ( { unbless => 1 }, keys )
1045       export
1046          Exports the shared object as a non-shared object. One must export
1047          the shared object when passing into any dump routine. Otherwise, the
1048          "shared_id value" and "blessed name" is all one will see. The
1049          unbless option unblesses any shared Array, Hash, and Scalar object
1050          to a non-blessed array, hash, and scalar respectively.
1051
1052           use MCE::Shared;
1053           use MCE::Shared::Ordhash;
1054
1055           sub _dump {
1056              require Data::Dumper unless $INC{'Data/Dumper.pm'};
1057              no warnings 'once';
1058
1059              local $Data::Dumper::Varname  = 'VAR';
1060              local $Data::Dumper::Deepcopy = 1;
1061              local $Data::Dumper::Indent   = 1;
1062              local $Data::Dumper::Purity   = 1;
1063              local $Data::Dumper::Sortkeys = 0;
1064              local $Data::Dumper::Terse    = 0;
1065
1066              print Data::Dumper::Dumper($_[0]) . "\n";
1067           }
1068
1069           my $oh1 = MCE::Shared->share({ module => 'MCE::Shared::Ordhash' });
1070           my $oh2 = MCE::Shared->ordhash();  # same thing
1071
1072           _dump($oh1);
1073              # bless( [ 1, 'MCE::Shared::Ordhash' ], 'MCE::Shared::Object' )
1074
1075           _dump($oh2);
1076              # bless( [ 2, 'MCE::Shared::Ordhash' ], 'MCE::Shared::Object' )
1077
1078           _dump( $oh1->export );  # dumps object structure and content
1079           _dump( $oh2->export );  # ditto
1080
1081          "export" can optionally take a list of indices/keys for what to
1082          export.  This applies to shared array, hash, and ordhash.
1083
1084           use MCE::Shared;
1085
1086           # shared hash
1087           my $h1 = MCE::Shared->hash(
1088              qw/ I Heard The Bluebirds Sing by Marty Robbins /
1089                # k v     k   v         k    v  k     v
1090           );
1091
1092           # non-shared hash
1093           my $h2 = $h1->export( qw/ I The / );
1094
1095           _dump($h2);
1096
1097           __END__
1098
1099           # Output
1100
1101           $VAR1 = bless( {
1102             'I' => 'Heard',
1103             'The' => 'Bluebirds'
1104           }, 'MCE::Shared::Hash' );
1105
1106          Specifying the unbless option exports a non-blessed data structure
1107          instead.  The unbless option applies to shared MCE::Shared::{ Array,
1108          Hash, and Scalar } objects.
1109
1110           my $h2 = $h1->export( { unbless => 1 }, qw/ I The / );
1111           my $h3 = $h1->export( { unbless => 1 } );
1112
1113           _dump($h2);
1114           _dump($h3);
1115
1116           __END__
1117
1118           # Output
1119
1120           $VAR1 = {
1121             'The' => 'Bluebirds',
1122             'I' => 'Heard'
1123           };
1124
1125           $VAR1 = {
1126             'Marty' => 'Robbins',
1127             'Sing' => 'by',
1128             'The' => 'Bluebirds',
1129             'I' => 'Heard'
1130           };
1131
1132       next
1133          The "next" method provides parallel iteration between workers for
1134          shared "array", "hash", "ordhash", and "sequence". In list context,
1135          returns the next key-value pair or beg-end pair for sequence. In
1136          scalar context, returns the next item. The "undef" value is returned
1137          after the iteration has completed.
1138
1139          Internally, the list of keys to return is set when the closure is
1140          constructed.  Later keys added to the shared array or hash are not
1141          included. Subsequently, the "undef" value is returned for deleted
1142          keys.
1143
1144          The following example iterates through a shared array in parallel.
1145
1146           use MCE::Hobo;
1147           use MCE::Shared;
1148
1149           my $ar = MCE::Shared->array( 'a' .. 'j' );
1150
1151           sub demo1 {
1152              my ( $wid ) = @_;
1153              while ( my ( $index, $value ) = $ar->next ) {
1154                 print "$wid: [ $index ] $value\n";
1155                 sleep 1;
1156              }
1157           }
1158
1159           sub demo2 {
1160              my ( $wid ) = @_;
1161              while ( defined ( my $value = $ar->next ) ) {
1162                 print "$wid: $value\n";
1163                 sleep 1;
1164              }
1165           }
1166
1167           $ar->rewind();
1168
1169           MCE::Hobo->new( \&demo1, $_ ) for 1 .. 3;
1170           MCE::Hobo->waitall(), print "\n";
1171
1172           $ar->rewind();
1173
1174           MCE::Hobo->new( \&demo2, $_ ) for 1 .. 3;
1175           MCE::Hobo->waitall(), print "\n";
1176
1177           __END__
1178
1179           # Output
1180
1181           1: [ 0 ] a
1182           2: [ 1 ] b
1183           3: [ 2 ] c
1184           1: [ 3 ] d
1185           2: [ 5 ] f
1186           3: [ 4 ] e
1187           2: [ 8 ] i
1188           3: [ 6 ] g
1189           1: [ 7 ] h
1190           2: [ 9 ] j
1191
1192           1: a
1193           2: b
1194           3: c
1195           2: e
1196           3: f
1197           1: d
1198           3: g
1199           1: i
1200           2: h
1201           1: j
1202
1203          The form is similar for "sequence". For large sequences, the
1204          "bounds_only" option is recommended. Also, specify "chunk_size"
1205          accordingly. This reduces the amount of traffic to and from the
1206          shared-manager process.
1207
1208           use MCE::Hobo;
1209           use MCE::Shared;
1210
1211           my $N   = shift || 4_000_000;
1212           my $pi  = MCE::Shared->scalar( 0.0 );
1213
1214           my $seq = MCE::Shared->sequence(
1215              { chunk_size => 200_000, bounds_only => 1 }, 0, $N - 1
1216           );
1217
1218           sub compute_pi {
1219              my ( $wid ) = @_;
1220
1221              # Optionally, also receive the chunk_id value
1222              # while ( my ( $beg, $end, $chunk_id ) = $seq->next ) { ... }
1223
1224              while ( my ( $beg, $end ) = $seq->next ) {
1225                 my ( $_pi, $t ) = ( 0.0 );
1226                 for my $i ( $beg .. $end ) {
1227                    $t = ( $i + 0.5 ) / $N;
1228                    $_pi += 4.0 / ( 1.0 + $t * $t );
1229                 }
1230                 $pi->incrby( $_pi );
1231              }
1232
1233              return;
1234           }
1235
1236           MCE::Hobo->create( \&compute_pi, $_ ) for ( 1 .. 8 );
1237
1238           # ... do other stuff ...
1239
1240           MCE::Hobo->waitall();
1241
1242           printf "pi = %0.13f\n", $pi->get / $N;
1243
1244           __END__
1245
1246           # Output
1247
1248           3.1415926535898
1249
1250       rewind ( index, [, index, ... ] )
1251       rewind ( key, [, key, ... ] )
1252       rewind ( "query string" )
1253       rewind ( )
1254          Rewinds the parallel iterator for MCE::Shared::Array,
1255          MCE::Shared::Hash, or MCE::Shared::Ordhash when no arguments are
1256          given. Otherwise, resets the iterator with given criteria. The
1257          syntax for "query string" is described in the shared module.
1258
1259           # array
1260           $ar->rewind;
1261
1262           $ar->rewind( 0, 1 );
1263           $ar->rewind( "val eq some_value" );
1264           $ar->rewind( "key >= 50 :AND val =~ /sun|moon|air|wind/" );
1265           $ar->rewind( "val eq sun :OR val eq moon :OR val eq foo" );
1266           $ar->rewind( "key =~ /$pattern/" );
1267
1268           while ( my ( $index, $value ) = $ar->next ) {
1269              ...
1270           }
1271
1272           # hash, ordhash
1273           $oh->rewind;
1274
1275           $oh->rewind( "key1", "key2" );
1276           $oh->rewind( "val eq some_value" );
1277           $oh->rewind( "key eq some_key :AND val =~ /sun|moon|air|wind/" );
1278           $oh->rewind( "val eq sun :OR val eq moon :OR val eq foo" );
1279           $oh->rewind( "key =~ /$pattern/" );
1280
1281           while ( my ( $key, $value ) = $oh->next ) {
1282              ...
1283           }
1284
1285       rewind ( { options }, begin, end [, step, format ] )
1286       rewind ( begin, end [, step, format ] )
1287       rewind ( )
1288          Rewinds the parallel iterator for MCE::Shared::Sequence when no
1289          arguments are given. Otherwise, resets the iterator with given
1290          criteria.
1291
1292           # sequence
1293           $seq->rewind;
1294
1295           $seq->rewind( { chunk_size => 10, bounds_only => 1 }, 1, 100 );
1296
1297           while ( my ( $beg, $end ) = $seq->next ) {
1298              for my $i ( $beg .. $end ) {
1299                 ...
1300              }
1301           }
1302
1303           $seq->rewind( 1, 100 );
1304
1305           while ( defined ( my $num = $seq->next ) ) {
1306              ...
1307           }
1308
1309       store ( key, value )
1310          Deep-sharing a non-blessed structure recursively is possible with
1311          "store", an alias to "STORE".
1312
1313           use MCE::Shared;
1314
1315           my $h1 = MCE::Shared->hash();
1316           my $h2 = MCE::Shared->hash();
1317
1318           # auto-shares deeply
1319           $h1->store('key', [ 0, 2, 5, { 'foo' => 'bar' } ]);
1320           $h2->{key}[3]{foo} = 'baz';    # via auto-vivification
1321
1322           my $v1 = $h1->get('key')->get(3)->get('foo');  # bar
1323           my $v2 = $h2->get('key')->get(3)->get('foo');  # baz
1324           my $v3 = $h2->{key}[3]{foo};                   # baz
1325

SERVER API

1327       init
1328          This method is called by each MCE and Hobo worker automatically
1329          after spawning.  The effect is extra parallelism and decreased
1330          latency during inter-process communication to the shared-manager
1331          process. The optional ID (an integer) is modded internally in a
1332          round-robin fashion.
1333
1334           MCE::Shared->init();
1335           MCE::Shared->init( ID );
1336
1337       pid
1338          Returns the process ID of the shared-manager process. This class
1339          method was added in 1.849 for stopping all workers immediately when
1340          exiting a Graphics::Framebuffer application. It returns an undefined
1341          value if the shared-manager is not running. Not useful otherwise if
1342          running threads (i.e. same PID).
1343
1344           MCE::Shared->pid();
1345
1346           $SIG{INT} = $SIG{HUP} = $SIG{TERM} = sub {
1347              # Signal workers and the shared manager all at once
1348              CORE::kill('KILL', MCE::Hobo->list_pids(), MCE::Shared->pid());
1349              exec('reset');
1350           };
1351
1352       start
1353          Starts the shared-manager process. This is done automatically unless
1354          Perl lacks IO::FDPass, needed to share "condvar" and "queue" while
1355          the shared-manager is running.
1356
1357           MCE::Shared->start();
1358
1359       stop
1360          Stops the shared-manager process, wiping all shared data content.
1361          This is called by the "END" block automatically when the script
1362          terminates. However, do stop explicitly to reap the shared-manager
1363          process before exec'ing.
1364
1365           MCE::Shared->stop();
1366
1367           exec('command');
1368

LOCKING

1370       Application-level advisory locking is possible with MCE::Mutex.
1371
1372        use MCE::Hobo;
1373        use MCE::Mutex;
1374        use MCE::Shared;
1375
1376        my $mutex = MCE::Mutex->new();
1377
1378        tie my $cntr, 'MCE::Shared', 0;
1379
1380        sub work {
1381           for ( 1 .. 1000 ) {
1382              $mutex->lock;
1383
1384              # Incrementing involves 2 IPC ops ( FETCH and STORE ).
1385              # Thus, locking is required.
1386              $cntr++;
1387
1388              $mutex->unlock;
1389           }
1390        }
1391
1392        MCE::Hobo->create('work') for ( 1 .. 8 );
1393        MCE::Hobo->waitall;
1394
1395        print $cntr, "\n"; # 8000
1396
1397       Typically, locking is not necessary using the OO interface. The reason
1398       is that MCE::Shared is implemented using a single-point of entry for
1399       commands sent to the shared-manager process. Furthermore, the shared
1400       classes include sugar methods for combining set and get in a single
1401       operation.
1402
1403        use MCE::Hobo;
1404        use MCE::Shared;
1405
1406        my $cntr = MCE::Shared->scalar( 0 );
1407
1408        sub work {
1409           for ( 1 .. 1000 ) {
1410              # The next statement increments the value without having
1411              # to call set and get explicitly.
1412              $cntr->incr;
1413           }
1414        }
1415
1416        MCE::Hobo->create('work') for ( 1 .. 8 );
1417        MCE::Hobo->waitall;
1418
1419        print $cntr->get, "\n"; # 8000
1420
1421       Another possibility when running threads is locking via
1422       threads::shared.
1423
1424        use threads;
1425        use threads::shared;
1426
1427        use MCE::Flow;
1428        use MCE::Shared;
1429
1430        my $mutex : shared;
1431
1432        tie my $cntr, 'MCE::Shared', 0;
1433
1434        sub work {
1435           for ( 1 .. 1000 ) {
1436              lock $mutex;
1437
1438              # the next statement involves 2 IPC ops ( get and set )
1439              # thus, locking is required
1440              $cntr++;
1441           }
1442        }
1443
1444        MCE::Flow->run( { max_workers => 8 }, \&work );
1445        MCE::Flow->finish;
1446
1447        print $cntr, "\n"; # 8000
1448
1449       Of the three demonstrations, the OO interface yields the best
1450       performance.  This is from the lack of locking at the application
1451       level. The results were obtained from a MacBook Pro (Haswell) running
1452       at 2.6 GHz, 1600 MHz RAM.
1453
1454        CentOS 7.2 VM
1455
1456           -- Perl v5.16.3
1457           MCE::Mutex .... : 0.528 secs.
1458           OO Interface .. : 0.062 secs.
1459           threads::shared : 0.545 secs.
1460
1461        FreeBSD 10.0 VM
1462
1463           -- Perl v5.16.3
1464           MCE::Mutex .... : 0.367 secs.
1465           OO Interface .. : 0.083 secs.
1466           threads::shared : 0.593 secs.
1467
1468        Mac OS X 10.11.6 ( Host OS )
1469
1470           -- Perl v5.18.2
1471           MCE::Mutex .... : 0.397 secs.
1472           OO Interface .. : 0.070 secs.
1473           threads::shared : 0.463 secs.
1474
1475        Solaris 11.2 VM
1476
1477           -- Perl v5.12.5 installed with the OS
1478           MCE::Mutex .... : 0.895 secs.
1479           OO Interface .. : 0.099 secs.
1480           threads::shared :              Perl not built to support threads
1481
1482           -- Perl v5.22.2 built with threads support
1483           MCE::Mutex .... : 0.788 secs.
1484           OO Interface .. : 0.086 secs.
1485           threads::shared : 0.895 secs.
1486
1487        Windows 7 VM
1488
1489           -- Perl v5.22.2
1490           MCE::Mutex .... : 1.045 secs.
1491           OO Interface .. : 0.312 secs.
1492           threads::shared : 1.061 secs.
1493
1494       Beginning with MCE::Shared 1.809, the "pipeline" method provides
1495       another way.  Included in "Array", "Cache", "Hash", "Minidb", and
1496       "Ordhash", it combines multiple commands for the object to be processed
1497       serially. For shared objects, the call is made atomically due to single
1498       IPC to the shared-manager process.
1499
1500       The "pipeline" method is fully "wantarray"-aware and receives a list of
1501       commands and their arguments. In scalar or list context, it returns
1502       data from the last command in the pipeline.
1503
1504        use MCE::Mutex;
1505        use MCE::Shared;
1506
1507        my $mutex = MCE::Mutex->new();
1508        my $oh = MCE::Shared->ordhash();
1509        my @vals;
1510
1511        # mutex locking
1512
1513        $mutex->lock;
1514        $oh->set( foo => "a_a" );
1515        $oh->set( bar => "b_b" );
1516        $oh->set( baz => "c_c" );
1517        @vals = $oh->mget( qw/ foo bar baz / );
1518        $mutex->unlock;
1519
1520        # pipeline, same thing done atomically
1521
1522        @vals = $oh->pipeline(
1523           [ "set", foo => "a_a" ],
1524           [ "set", bar => "b_b" ],
1525           [ "set", baz => "c_c" ],
1526           [ "mget", qw/ foo bar baz / ]
1527        );
1528
1529        # ( "a_a", "b_b", "c_c" )
1530
1531       There is also "pipeline_ex", same as "pipeline", but returns data for
1532       every command in the pipeline.
1533
1534        @vals = $oh->pipeline_ex(
1535           [ "set", foo => "a_a" ],
1536           [ "set", bar => "b_b" ],
1537           [ "set", baz => "c_c" ]
1538        );
1539
1540        # ( "a_a", "b_b", "c_c" )
1541

PYTHON DEMONSTRATION

1543       Sharing a Python class is possible, starting with the 1.827 release.
1544       The construction is simply calling share with the module option.
1545       Methods are accessible via the OO interface.
1546
1547        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1548        # Python class.
1549        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1550
1551        package My::Class;
1552
1553        use strict;
1554        use warnings;
1555
1556        use Inline::Python qw( py_eval py_bind_class );
1557
1558        py_eval ( <<'END_OF_PYTHON_CLASS' );
1559
1560        class MyClass:
1561            def __init__(self):
1562                self.data = [0,0]
1563
1564            def set (self, key, value):
1565                self.data[key] = value
1566
1567            def get (self, key):
1568                try: return self.data[key]
1569                except KeyError: return None
1570
1571            def incr (self, key):
1572                try: self.data[key] = self.data[key] + 1
1573                except KeyError: self.data[key] = 1
1574
1575        END_OF_PYTHON_CLASS
1576
1577        # Register methods for best performance.
1578
1579        py_bind_class(
1580            'My::Class', '__main__', 'MyClass',
1581            'set', 'get', 'incr'
1582        );
1583
1584        1;
1585
1586        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1587        # Share Python class. Requires MCE::Shared 1.827 or later.
1588        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1589
1590        use strict;
1591        use warnings;
1592
1593        use MCE::Hobo;
1594        use MCE::Shared;
1595
1596        my $py1 = MCE::Shared->share({ module => 'My::Class' });
1597        my $py2 = MCE::Shared->share({ module => 'My::Class' });
1598
1599        MCE::Shared->start;
1600
1601        $py1->set(0, 100);
1602        $py2->set(1, 200);
1603
1604        die "Ooops" unless $py1->get(0) eq '100';
1605        die "Ooops" unless $py2->get(1) eq '200';
1606
1607        sub task {
1608            $py1->incr(0) for 1..50000;
1609            $py2->incr(1) for 1..50000;
1610        }
1611
1612        MCE::Hobo->create(\&task) for 1..3;
1613        MCE::Hobo->waitall;
1614
1615        print $py1->get(0), "\n";  # 150100
1616        print $py2->get(1), "\n";  # 150200
1617

LOGGER DEMONSTRATION

1619       Often, the requirement may call for concurrent logging by many workers.
1620       Calling localtime or gmtime per each log entry is expensive. This uses
1621       the old time-stamp value until one second has elapsed.
1622
1623        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1624        # Logger class.
1625        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1626
1627        package My::Logger;
1628
1629        use strict;
1630        use warnings;
1631
1632        use Time::HiRes qw( time );
1633
1634        # construction
1635
1636        sub new {
1637            my ( $class, %self ) = @_;
1638
1639            open $self{fh}, ">>", $self{path} or return '';
1640            binmode $self{fh};
1641
1642            $self{stamp} = localtime;  # or gmtime
1643            $self{time } = time;
1644
1645            bless \%self, $class;
1646        }
1647
1648        # $ob->log("message");
1649
1650        sub log {
1651            my ( $self, $stamp ) = ( shift );
1652
1653            if ( time - $self->{time} > 1.0 ) {
1654                $self->{stamp} = $stamp = localtime;  # or gmtime
1655                $self->{time } = time;
1656            }
1657            else {
1658                $stamp = $self->{stamp};
1659            }
1660
1661            print {$self->{fh}} "$stamp --- @_\n";
1662        }
1663
1664        # $ob->autoflush(0);
1665        # $ob->autoflush(1);
1666
1667        sub autoflush {
1668            my ( $self, $flag ) = @_;
1669
1670            if ( defined fileno($self->{fh}) ) {
1671                 $flag ? select(( select($self->{fh}), $| = 1 )[0])
1672                       : select(( select($self->{fh}), $| = 0 )[0]);
1673
1674                 return 1;
1675            }
1676
1677            return;
1678        }
1679
1680        # $ob->binmode($layer);
1681        # $ob->binmode();
1682
1683        sub binmode {
1684            my ( $self, $layer ) = @_;
1685
1686            if ( defined fileno($self->{fh}) ) {
1687                CORE::binmode $self->{fh}, $layer // ':raw';
1688
1689                return 1;
1690            }
1691
1692            return;
1693        }
1694
1695        # $ob->close()
1696
1697        sub close {
1698            my ( $self ) = @_;
1699
1700            if ( defined fileno($self->{fh}) ) {
1701                close $self->{'fh'};
1702            }
1703
1704            return;
1705        }
1706
1707        # $ob->flush()
1708
1709        sub flush {
1710            my ( $self ) = @_;
1711
1712            if ( defined fileno($self->{fh}) ) {
1713                my $old_fh = select $self->{fh};
1714                my $old_af = $|; $| = 1; $| = $old_af;
1715                select $old_fh;
1716
1717                return 1;
1718            }
1719
1720            return;
1721        }
1722
1723        1;
1724
1725        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1726        # Concurrent logger demo. Requires MCE::Shared 1.827 or later.
1727        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1728
1729        use strict;
1730        use warnings;
1731
1732        use MCE::Hobo;
1733        use MCE::Shared;
1734
1735        my $file = "log.txt";
1736        my $pid  = $$;
1737
1738        my $ob = MCE::Shared->share( { module => 'My::Logger' }, path => $file )
1739            or die "open error '$file': $!";
1740
1741        # $ob->autoflush(1);   # optional, flush writes immediately
1742
1743        sub work {
1744            my $id = shift;
1745            for ( 1 .. 250_000 ) {
1746                $ob->log("Hello from $id: $_");
1747            }
1748        }
1749
1750        MCE::Hobo->create('work', $_) for 1 .. 4;
1751        MCE::Hobo->waitall;
1752
1753        # Threads and multi-process safety for closing the handle.
1754
1755        sub CLONE { $pid = 0; }
1756
1757        END { $ob->close if $ob && $pid == $$; }
1758

TIE::FILE DEMONSTRATION

1760       The following presents a concurrent Tie::File demonstration. Each
1761       element in the array corresponds to a record in the text file. JSON,
1762       being readable, seems appropiate for encoding complex objects.
1763
1764        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1765        # Class extending Tie::File with two sugar methods.
1766        # Requires MCE::Shared 1.827 or later.
1767        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1768
1769        package My::File;
1770
1771        use strict;
1772        use warnings;
1773
1774        use Tie::File;
1775
1776        our @ISA = 'Tie::File';
1777
1778        # $ob->append('string');
1779
1780        sub append {
1781            my ($self, $key) = @_;
1782            my $val = $self->FETCH($key); $val .= $_[2];
1783            $self->STORE($key, $val);
1784            length $val;
1785        }
1786
1787        # $ob->incr($key);
1788
1789        sub incr {
1790            my ( $self, $key ) = @_;
1791            my $val = $self->FETCH($key); $val += 1;
1792            $self->STORE($key, $val);
1793            $val;
1794        }
1795
1796        1;
1797
1798        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1799        # The MCE::Mutex module isn't needed unless IPC involves two or
1800        # more trips for the underlying action.
1801        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1802
1803        use strict;
1804        use warnings;
1805
1806        use MCE::Hobo;
1807        use MCE::Mutex;
1808        use MCE::Shared;
1809
1810        use JSON::MaybeXS;
1811
1812        # Safety for data having line breaks.
1813        use constant EOL => "\x{0a}~\x{0a}";
1814
1815        my $file  = 'file.txt';
1816        my $mutex = MCE::Mutex->new();
1817        my $pid   = $$;
1818
1819        my $ob = tie my @db, 'MCE::Shared', { module => 'My::File' }, $file,
1820            recsep => EOL or die "open error '$file': $!";
1821
1822        $ob->encoder( \&JSON::MaybeXS::encode_json );
1823        $ob->decoder( \&JSON::MaybeXS::decode_json );
1824
1825        $db[20] = 0;  # a counter at offset 20 into the array
1826        $db[21] = [ qw/ foo bar / ];  # store complex structure
1827
1828        sub task {
1829            my $id  = sprintf "%02s", shift;
1830            my $row = int($id) - 1;
1831            my $chr = sprintf "%c", 97 + $id - 1;
1832
1833            # A mutex isn't necessary when storing a value.
1834            # Ditto for fetching a value.
1835
1836            $db[$row] = "Hello from $id: ";  # 1 trip
1837            my $val   = length $db[$row];    # 1 trip
1838
1839            # A mutex may be necessary for updates involving 2 or
1840            # more trips (FETCH and STORE) during IPC, from and to
1841            # the shared-manager process, unless a unique row.
1842
1843            for ( 1 .. 40 ) {
1844              # $db[$row] .= $id;         # 2 trips, unique row - okay
1845                $ob->append($row, $chr);  # 1 trip via the OO interface
1846
1847              # $mu->lock;
1848              # $db[20] += 1;             # incrementing counter, 2 trips
1849              # $mu->unlock;
1850
1851                $ob->incr(20);            # same thing via OO, 1 trip
1852            }
1853
1854            my $len = length $db[$row];   # 1 trip
1855
1856            printf "hobo %2d : %d\n", $id, $len;
1857        }
1858
1859        MCE::Hobo->create('task', $_) for 1 .. 20;
1860        MCE::Hobo->waitall;
1861
1862        printf "counter : %d\n", $db[20];
1863        print  $db[21]->[0], "\n";  # foo
1864
1865        # Threads and multi-process safety for closing the handle.
1866
1867        sub CLONE { $pid = 0; }
1868
1869        END {
1870            if ( $pid == $$ ) {
1871                undef $ob;  # important, undef $ob before @db
1872                untie @db;  # untie @db to flush pending writes
1873            }
1874        }
1875

REQUIREMENTS

1877       MCE::Shared requires Perl 5.10.1 or later. The IO::FDPass module is
1878       highly recommended on UNIX and Windows. This module does not install it
1879       by default.
1880

SOURCE AND FURTHER READING

1882       The source, cookbook, and examples are hosted at GitHub.
1883
1884       ·  <https://github.com/marioroy/mce-shared>
1885
1886       ·  <https://github.com/marioroy/mce-cookbook>
1887
1888       ·  <https://github.com/marioroy/mce-examples>
1889

INDEX

1891       MCE, MCE::Hobo
1892

AUTHOR

1894       Mario E. Roy, <marioeroy AT gmail DOT com>
1895

COPYRIGHT AND LICENSE

1897       Copyright (C) 2016-2019 by Mario E. Roy
1898
1899       MCE::Shared is released under the same license as Perl.
1900
1901       See <http://dev.perl.org/licenses/> for more information.
1902
1903
1904
1905perl v5.30.0                      2019-09-19                    MCE::Shared(3)