MCE::Shared(3pm)

1MCE::Shared(3)        User Contributed Perl Documentation       MCE::Shared(3)
2
3
4

NAME

6       MCE::Shared - MCE extension for sharing data supporting threads and
7       processes
8

VERSION

10       This document describes MCE::Shared version 1.873
11

SYNOPSIS

13        # OO construction.
14
15        use MCE::Shared;
16
17        my $ar = MCE::Shared->array( @list );
18        my $ca = MCE::Shared->cache( max_keys => 500, max_age => 60 );
19        my $cv = MCE::Shared->condvar( 0 );
20        my $fh = MCE::Shared->handle( '>>', \*STDOUT ) or die "$!";
21        my $ha = MCE::Shared->hash( @pairs );
22        my $oh = MCE::Shared->ordhash( @pairs );
23        my $db = MCE::Shared->minidb();
24        my $qu = MCE::Shared->queue( await => 1, fast => 0 );
25        my $va = MCE::Shared->scalar( $value );
26        my $se = MCE::Shared->sequence( $begin, $end, $step, $fmt );
27        my $ob = MCE::Shared->share( $blessed_object );
28
29        # Mutex locking is supported for all shared objects since 1.841.
30        # Previously only shared C<condvar>s allowed locking.
31
32        $ar->lock;
33        $ar->unlock;
34        ...
35        $ob->lock;
36        $ob->unlock;
37
38        # The Perl-like mce_open function is available since 1.002.
39
40        mce_open my $fh, ">>", "/foo/bar.log" or die "open error: $!";
41
42        # Tie construction. The module API option is available since 1.825.
43
44        use v5.10;
45        use MCE::Flow;
46        use MCE::Shared;
47
48        my %args  = ( max_keys => 500, max_age => 60 );
49        my @pairs = ( foo => 'bar', woo => 'baz' );
50        my @list  = ( 'a' .. 'z' );
51
52        tie my $va1, 'MCE::Shared', { module => 'MCE::Shared::Scalar' }, 0;
53        tie my @ar1, 'MCE::Shared', { module => 'MCE::Shared::Array' }, @list;
54        tie my %ca1, 'MCE::Shared', { module => 'MCE::Shared::Cache' }, %args;
55        tie my %ha1, 'MCE::Shared', { module => 'MCE::Shared::Hash' }, @pairs;
56        tie my %oh1, 'MCE::Shared', { module => 'MCE::Shared::Ordhash' }, @pairs;
57        tie my %oh2, 'MCE::Shared', { module => 'Hash::Ordered' }, @pairs;
58        tie my %oh3, 'MCE::Shared', { module => 'Tie::IxHash' }, @pairs;
59        tie my $cy1, 'MCE::Shared', { module => 'Tie::Cycle' }, [ 1 .. 8 ];
60        tie my $va2, 'MCE::Shared', { module => 'Tie::StdScalar' }, 'hello';
61        tie my @ar3, 'MCE::Shared', { module => 'Tie::StdArray' }, @list;
62        tie my %ha2, 'MCE::Shared', { module => 'Tie::StdHash' }, @pairs;
63        tie my %ha3, 'MCE::Shared', { module => 'Tie::ExtraHash' }, @pairs;
64
65        tie my $cnt, 'MCE::Shared', 0; # default MCE::Shared::Scalar
66        tie my @foo, 'MCE::Shared';    # default MCE::Shared::Array
67        tie my %bar, 'MCE::Shared';    # default MCE::Shared::Hash
68
69        tie my @ary, 'MCE::Shared', qw( a list of values );
70        tie my %ha,  'MCE::Shared', key1 => 'val1', key2 => 'val2';
71        tie my %ca,  'MCE::Shared', { max_keys => 500, max_age => 60 };
72        tie my %oh,  'MCE::Shared', { ordered => 1 }, key1 => 'value';
73
74        # Mutex locking is supported for all shared objects since 1.841.
75
76        tied($va1)->lock;
77        tied($va1)->unlock;
78        ...
79        tied(%bar)->lock;
80        tied(%bar)->unlock;
81
82        # Demonstration.
83
84        my $mutex = MCE::Mutex->new;
85
86        mce_flow {
87           max_workers => 4
88        },
89        sub {
90           my ( $mce ) = @_;
91           my ( $pid, $wid ) = ( MCE->pid, MCE->wid );
92
93           # Locking is necessary when multiple workers update the same
94           # element. The reason is that it may involve 2 trips to the
95           # shared-manager process: fetch and store in this case.
96
97           $mutex->enter( sub { $cnt += 1 } );
98
99           # Otherwise, locking is optional for unique elements.
100
101           $foo[ $wid - 1 ] = $pid;
102           $bar{ $pid }     = $wid;
103
104           # From 1.841 onwards, all shared objects include mutex locking
105           # to not need to construct MCE::Mutex separately.
106
107           tied($va1)->lock;
108           $va1 += 1;
109           tied($va1)->unlock;
110
111           return;
112        };
113
114        say "scalar : $cnt";
115        say "scalar : $va1";
116        say " array : $_" for (@foo);
117        say "  hash : $_ => $bar{$_}" for (sort keys %bar);
118
119        __END__
120
121        # Output
122
123        scalar : 4
124        scalar : 4
125         array : 37847
126         array : 37848
127         array : 37849
128         array : 37850
129          hash : 37847 => 1
130          hash : 37848 => 2
131          hash : 37849 => 3
132          hash : 37850 => 4
133

DESCRIPTION

135       This module provides data sharing capabilities for MCE supporting
136       threads and processes. MCE::Hobo provides threads-like parallelization
137       for running code asynchronously.
138

EXTRA FUNCTIONALITY

140       MCE::Shared enables extra functionality on systems with IO::FDPass
141       installed.  Without it, MCE::Shared is unable to send file descriptors
142       to the shared-manager process. The use applies to Condvar, Queue, and
143       Handle (mce_open). IO::FDpass isn't used for anything else.
144
145        use MCE::Shared;
146
147        # One may want to start the shared-manager early.
148
149        MCE::Shared->start();
150
151        # Typically, the shared-manager is started automatically when
152        # constructing a shared object.
153
154        my $ca = MCE::Shared->cache( max_keys => 500 );
155
156        # IO::FDPass is necessary for constructing a shared condvar or queue
157        # while the manager is running in order to send file descriptors
158        # associated with the object.
159
160        # Workers block using a socket handle for ->wait and ->timedwait.
161
162        my $cv = MCE::Shared->condvar();
163
164        # Workers block using a socket handle for ->dequeue and ->await.
165
166        my $q1 = MCE::Shared->queue();
167        my $q2 = MCE::Shared->queue( await => 1 );
168
169       For platforms where IO::FDPass isn't possible (e.g. Cygwin), construct
170       "condvar" and "queue" before other classes. The shared-manager process
171       will be delayed until sharing other classes (e.g. Array, Hash) or
172       starting explicitly.
173
174        use MCE::Shared;
175
176        my $has_IO_FDPass = $INC{'IO/FDPass.pm'} ? 1 : 0;
177
178        my $cv  = MCE::Shared->condvar( 0 );
179        my $que = MCE::Shared->queue( fast => 1 );
180
181        MCE::Shared->start() unless $has_IO_FDPass;
182
183        my $ha = MCE::Shared->hash();  # started implicitly
184
185       Note that MCE starts the shared-manager, prior to spawning workers, if
186       not yet started. Ditto for MCE::Hobo.
187
188       Regarding mce_open, "IO::FDPass" is needed for constructing a shared-
189       handle from a non-shared handle not yet available inside the shared-
190       manager process.  The workaround is to have the non-shared handle made
191       before the shared-manager is started. Passing a file by reference is
192       fine for the three STD* handles.
193
194        # The shared-manager knows of \*STDIN, \*STDOUT, \*STDERR.
195
196        mce_open my $shared_in,  "<",  \*STDIN;   # ok
197        mce_open my $shared_out, ">>", \*STDOUT;  # ok
198        mce_open my $shared_err, ">>", \*STDERR;  # ok
199        mce_open my $shared_fh1, "<",  "/path/to/sequence.fasta";  # ok
200        mce_open my $shared_fh2, ">>", "/path/to/results.log";     # ok
201
202        mce_open my $shared_fh, ">>", \*NON_SHARED_FH;  # requires IO::FDPass
203
204       The IO::FDPass module is known to work reliably on most platforms.
205       Install 1.1 or later to rid of limitations described above.
206
207        perl -MIO::FDPass -le "print 'Cheers! Perl has IO::FDPass.'"
208

DATA SHARING

210       MCE::Shared->array MCE::Shared::Array
211       MCE::Shared->cache MCE::Shared::Cache
212       MCE::Shared->condvar MCE::Shared::Condvar
213       MCE::Shared->handle MCE::Shared::Handle
214       MCE::Shared->hash MCE::Shared::Hash
215       MCE::Shared->minidb MCE::Shared::Minidb
216       MCE::Shared->ordhash MCE::Shared::Ordhash
217       MCE::Shared->queue MCE::Shared::Queue
218       MCE::Shared->scalar MCE::Shared::Scalar
219       MCE::Shared->sequence MCE::Shared::Sequence
220
221       Below, synopsis for sharing classes included with MCE::Shared.
222
223        use MCE::Shared;
224
225        # short form
226
227        $ar = MCE::Shared->array( @list );
228        $ca = MCE::Shared->cache( max_keys => 500, max_age => 60 );
229        $cv = MCE::Shared->condvar( 0 );
230        $fh = MCE::Shared->handle( ">>", \*STDOUT ); # see mce_open below
231        $ha = MCE::Shared->hash( @pairs );
232        $db = MCE::Shared->minidb();
233        $oh = MCE::Shared->ordhash( @pairs );
234        $qu = MCE::Shared->queue( await => 1, fast => 0 );
235        $va = MCE::Shared->scalar( $value );
236        $se = MCE::Shared->sequence( $begin, $end, $step, $fmt );
237
238        mce_open my $fh, ">>", \*STDOUT or die "open error: $!";
239
240        # long form
241
242        $ar = MCE::Shared->share( { module => 'MCE::Shared::Array'    }, ... );
243        $ca = MCE::Shared->share( { module => 'MCE::Shared::Cache'    }, ... );
244        $cv = MCE::Shared->share( { module => 'MCE::Shared::Condvar'  }, ... );
245        $fh = MCE::Shared->share( { module => 'MCE::Shared::Handle'   }, ... );
246        $ha = MCE::Shared->share( { module => 'MCE::Shared::Hash'     }, ... );
247        $db = MCE::Shared->share( { module => 'MCE::Shared::Minidb'   }, ... );
248        $oh = MCE::Shared->share( { module => 'MCE::Shared::Ordhash'  }, ... );
249        $qu = MCE::Shared->share( { module => 'MCE::Shared::Queue'    }, ... );
250        $va = MCE::Shared->share( { module => 'MCE::Shared::Scalar'   }, ... );
251        $se = MCE::Shared->share( { module => 'MCE::Shared::Sequence' }, ... );
252
253       The restriction for sharing classes not included with MCE::Shared is
254       that the object must not have file-handles nor code-blocks.
255
256        $oh = MCE::Shared->share( { module => 'Hash::Ordered' }, ... );
257
258       open ( filehandle, expr )
259       open ( filehandle, mode, expr )
260       open ( filehandle, mode, reference )
261
262       In version 1.002 and later, constructs a new object by opening the file
263       whose filename is given by "expr", and associates it with "filehandle".
264       When omitting error checking at the application level, MCE::Shared
265       emits a message and stop if open fails.
266
267       See MCE::Shared::Handle for chunk IO demonstrations.
268
269        {
270          use MCE::Shared::Handle;
271
272          # "non-shared" or "local construction" for use by a single process
273          MCE::Shared::Handle->open( my $fh, "<", "file.log" ) or die "$!";
274          MCE::Shared::Handle::open  my $fh, "<", "file.log"   or die "$!";
275
276          # mce_open is an alias for MCE::Shared::Handle::open
277          mce_open my $fh, "<", "file.log" or die "$!";
278        }
279
280        {
281          use MCE::Shared;
282
283          # construction for "sharing" with other threads and processes
284          MCE::Shared->open( my $fh, "<", "file.log" ) or die "$!";
285          MCE::Shared::open  my $fh, "<", "file.log"   or die "$!";
286
287          # mce_open is an alias for MCE::Shared::open
288          mce_open my $fh, "<", "file.log" or die "$!";
289        }
290
291       mce_open ( filehandle, expr )
292       mce_open ( filehandle, mode, expr )
293       mce_open ( filehandle, mode, reference )
294
295       Native Perl-like syntax to open a shared-file for reading:
296
297        use MCE::Shared;
298
299        # mce_open is exported by MCE::Shared or MCE::Shared::Handle.
300        # It creates a shared file handle with MCE::Shared present
301        # or a non-shared handle otherwise.
302
303        mce_open my $fh, "< input.txt"     or die "open error: $!";
304        mce_open my $fh, "<", "input.txt"  or die "open error: $!";
305        mce_open my $fh, "<", \*STDIN      or die "open error: $!";
306
307       and for writing:
308
309        mce_open my $fh, "> output.txt"    or die "open error: $!";
310        mce_open my $fh, ">", "output.txt" or die "open error: $!";
311        mce_open my $fh, ">", \*STDOUT     or die "open error: $!";
312
313       num_sequence
314
315       "num_sequence" is an alias for "sequence".
316

DEEPLY SHARING

318       The following is a demonstration for a shared tied-hash variable.
319       Before venturing into the actual code, notice the dump function making
320       a call to "export" explicitly for objects of type
321       "MCE::Shared::Object". This is necessary in order to retrieve the data
322       from the shared-manager process.
323
324       The "export" method is described later under the Common API section.
325
326        use MCE::Shared;
327
328        sub _dump {
329           require Data::Dumper unless $INC{'Data/Dumper.pm'};
330           no warnings 'once';
331
332           local $Data::Dumper::Varname  = 'VAR';
333           local $Data::Dumper::Deepcopy = 1;
334           local $Data::Dumper::Indent   = 1;
335           local $Data::Dumper::Purity   = 1;
336           local $Data::Dumper::Sortkeys = 0;
337           local $Data::Dumper::Terse    = 0;
338
339           ( ref $_[0] eq 'MCE::Shared::Object' )
340              ? print Data::Dumper::Dumper( $_[0]->export ) . "\n"
341              : print Data::Dumper::Dumper( $_[0] ) . "\n";
342        }
343
344        tie my %abc, 'MCE::Shared';
345
346        my @parents = qw( a b c );
347        my @children = qw( 1 2 3 4 );
348
349        for my $parent ( @parents ) {
350           for my $child ( @children ) {
351              $abc{ $parent }{ $child } = 1;
352           }
353        }
354
355        _dump( tied( %abc ) );
356
357        __END__
358
359        # Output
360
361        $VAR1 = bless( {
362          'c' => bless( {
363            '1' => '1',
364            '4' => '1',
365            '3' => '1',
366            '2' => '1'
367          }, 'MCE::Shared::Hash' ),
368          'a' => bless( {
369            '1' => '1',
370            '4' => '1',
371            '3' => '1',
372            '2' => '1'
373          }, 'MCE::Shared::Hash' ),
374          'b' => bless( {
375            '1' => '1',
376            '4' => '1',
377            '3' => '1',
378            '2' => '1'
379          }, 'MCE::Shared::Hash' )
380        }, 'MCE::Shared::Hash' );
381
382       Dereferencing provides hash-like behavior for "hash" and "ordhash".
383       Array-like behavior is allowed for "array", not shown below.
384
385        use MCE::Shared;
386        use Data::Dumper;
387
388        my $abc = MCE::Shared->hash;
389
390        my @parents = qw( a b c );
391        my @children = qw( 1 2 3 4 );
392
393        for my $parent ( @parents ) {
394           for my $child ( @children ) {
395              $abc->{ $parent }{ $child } = 1;
396           }
397        }
398
399        print Dumper( $abc->export({ unbless => 1 }) ), "\n";
400
401       Each level in a deeply structure requires a separate trip to the
402       shared-manager process. The included "MCE::Shared::Minidb" module
403       provides optimized methods for working with hash of hashes "HoH" and
404       hash of arrays "HoA".
405
406        use MCE::Shared;
407        use Data::Dumper;
408
409        my $abc = MCE::Shared->minidb;
410
411        my @parents = qw( a b c );
412        my @children = qw( 1 2 3 4 );
413
414        for my $parent ( @parents ) {
415           for my $child ( @children ) {
416              $abc->hset($parent, $child, 1);
417           }
418        }
419
420        print Dumper( $abc->export ), "\n";
421
422       For further reading, see MCE::Shared::Minidb.
423

OBJECT SHARING

425       MCE::Shared->share
426
427       This class method transfers the blessed-object to the shared-manager
428       process and returns a "MCE::Shared::Object" containing the "SHARED_ID".
429       Starting with the 1.827 release, the "module" option sends parameters
430       to the shared-manager, where the object is then constructed. This is
431       useful for classes involving XS code or a file handle.
432
433        use MCE::Shared;
434
435        {
436          use Math::BigFloat try => 'GMP';
437          use Math::BigInt   try => 'GMP';
438
439          my $bf  = MCE::Shared->share({ module => 'Math::BigFloat' }, 0);
440          my $bi  = MCE::Shared->share({ module => 'Math::BigInt'   }, 0);
441          my $y   = 1e9;
442
443          $bf->badd($y);  # addition (add $y to shared BigFloat object)
444          $bi->badd($y);  # addition (add $y to shared BigInt object)
445        }
446
447        {
448          use Bio::Seq;
449          use Bio::SeqIO;
450
451          my $seq_io = MCE::Shared->share({ module => 'Bio::SeqIO' },
452             -file    => ">/path/to/fasta/file.fa",
453             -format  => 'Fasta',
454             -verbose => -1,
455          );
456
457          my $seq_obj = Bio::Seq->new(
458             -display_id => "name", -desc => "desc", -seq => "seq",
459             -alphabet   => "dna"
460          );
461
462          $seq_io->write_seq($seq_obj);  # write to shared SeqIO handle
463        }
464
465        {
466          my $oh1 = MCE::Shared->share({ module => 'MCE::Shared::Ordhash' });
467          my $oh2 = MCE::Shared->ordhash();  # same thing
468
469          $oh1->assign( @pairs );
470          $oh2->assign( @pairs );
471        }
472
473        {
474          my ($ho_shared, $ho_nonshared);
475
476          $ho_shared = MCE::Shared->share({ module => 'Hash::Ordered' });
477          $ho_shared->push( @pairs );
478
479          $ho_nonshared = $ho_shared->export();   # back to non-shared
480          $ho_nonshared = $ho_shared->destroy();  # including shared destruction
481        }
482
483       The following provides long and short forms for constructing a shared
484       array, hash, or scalar object.
485
486        use MCE::Shared;
487
488        my $a1 = MCE::Shared->share( { module => 'MCE::Shared::Array' }, @list );
489        my $a2 = MCE::Shared->share( [ @list ] );
490        my $a3 = MCE::Shared->array( @list );
491
492        my $h1 = MCE::Shared->share( { module => 'MCE::Shared::Hash' }, @pairs );
493        my $h2 = MCE::Shared->share( { @pairs } );
494        my $h3 = MCE::Shared->hash( @pairs );
495
496        my $s1 = MCE::Shared->share( { module => 'MCE::Shared::Scalar' }, 20 );
497        my $s2 = MCE::Shared->share( \do{ my $o = 20 } );
498        my $s3 = MCE::Shared->scalar( 20 );
499
500       When the "module" option is given, one may optionally specify the
501       constructor function via the "new" option. This is necessary for the
502       CDB_File module, which provides two different objects. One is created
503       by new (default), and accessed by insert and finish. The other is
504       created by TIEHASH, and accessed by FETCH.
505
506        use MCE::Hobo;
507        use MCE::Shared;
508
509        # populate CDB file
510        my $cdb = MCE::Shared->share({ module => 'CDB_File' }, 't.cdb', "t.cdb.$$")
511           or die "$!\n";
512
513        $cdb->insert( $_ => $_ ) for ('aa'..'zz');
514        $cdb->finish;
515
516        # use CDB file
517        my $cdb1 = tie my %hash, 'MCE::Shared', { module => 'CDB_File' }, 't.cdb';
518
519        # same thing, without involving TIE and extra hash variable
520        my $cdb2 = MCE::Shared->share(
521           { module => 'CDB_File', new => 'TIEHASH' }, 't.cdb'
522        );
523
524        print $hash{'aa'}, "\n";
525        print $cdb1->FETCH('bb'), "\n";
526        print $cdb2->FETCH('cc'), "\n";
527
528        # rewind may be omitted on first use for parallel iteration
529        $cdb2->rewind;
530
531        for ( 1 .. 3 ) {
532           mce_async {
533              while ( my ($k,$v) = $cdb2->next ) {
534                 print "[$$] $k => $v\n";
535              }
536           };
537        }
538
539        MCE::Hobo->waitall;
540

DBM SHARING

542       Construting a shared DBM object is possible starting with the 1.827
543       release.  Supported modules are AnyDBM_File, BerkeleyDB, CDB_File,
544       DB_File, GDBM_File, NDBM_File, ODBM_File, SDBM_File, SQLite_File,
545       Tie::Array::DBD, and Tie::Hash::DBD. The list includes Tokyo Cabinet
546       <http://fallabs.com/tokyocabinet/> and Kyoto Cabinet
547       <http://fallabs.com/kyotocabinet/>. Also, see forked version by Altice
548       Labs <https://github.com/alticelabs/kyoto>. It contains an updated
549       "kyotocabinet" folder that builds successfully with recent compilers.
550
551       Freeze-thaw during "STORE"-"FETCH" (for complex data) is handled
552       automatically using Serial 3.015+ (if available) or Storable. Below,
553       are constructions for sharing various DBM modules. The construction for
554       "CDB_File" is given in the prior section.
555
556       AnyDBM_File
557
558        BEGIN { @AnyDBM_File::ISA = qw( DB_File GDBM_File NDBM_File ODBM_File ); }
559
560        use MCE::Shared;
561        use Fcntl;
562        use AnyDBM_File;
563
564        tie my %h1, 'MCE::Shared', { module => 'AnyDBM_File' },
565           'foo_a', O_CREAT|O_RDWR or die "open error: $!";
566
567       BerkeleyDB
568
569        use MCE::Shared;
570        use BerkeleyDB;
571
572        tie my %h1, 'MCE::Shared', { module => 'BerkeleyDB::Hash' },
573           -Filename => 'foo_a', -Flags => DB_CREATE
574              or die "open error: $!";
575
576        tie my %h2, 'MCE::Shared', { module => 'BerkeleyDB::Btree' },
577           -Filename => 'foo_b', -Flags => DB_CREATE
578              or die "open error: $!";
579
580        tie my @a1, 'MCE::Shared', { module => 'BerkeleyDB::Queue' },
581           -Filename => 'foo_c', -Flags => DB_CREATE
582              or die "open error: $!";
583
584        tie my @a2, 'MCE::Shared', { module => 'BerkeleyDB::Recno' },
585           -Filename => 'foo_d', -Flags => DB_CREATE -Len => 20
586              or die "open error: $!";
587
588       DB_File
589
590        use MCE::Shared;
591        use Fcntl;
592        use DB_File;
593
594        # Use pre-defined references ( $DB_HASH, $DB_BTREE, $DB_RECNO ).
595
596        tie my %h1, 'MCE::Shared', { module => 'DB_File' },
597           'foo_a', O_CREAT|O_RDWR, 0640, $DB_HASH or die "open error: $!";
598
599        tie my %h2, 'MCE::Shared', { module => 'DB_File' },
600           'foo_b', O_CREAT|O_RDWR, 0640, $DB_BTREE or die "open error: $!";
601
602        tie my @a1, 'MCE::Shared', { module => 'DB_File' },
603           'foo_c', O_CREAT|O_RDWR, 0640, $DB_RECNO or die "open error: $!";
604
605        # Changing defaults - see DB_File for valid options.
606
607        my $opt_h = DB_File::HASHINFO->new();
608        my $opt_b = DB_File::BTREEINFO->new();
609        my $opt_r = DB_File::RECNOINFO->new();
610
611        $opt_h->{'cachesize'} = 12345;
612
613        tie my %h3, 'MCE::Shared', { module => 'DB_File' },
614           'foo_d', O_CREAT|O_RDWR, 0640, $opt_h or die "open error: $!";
615
616       KyotoCabinet
617       TokyoCabinet
618
619        use MCE::Shared;
620        use KyotoCabinet;
621        use TokyoCabinet;
622
623        # file extension denotes hash database
624
625        tie my %h1, 'MCE::Shared', { module => 'KyotoCabinet::DB' }, 'foo.kch',
626           KyotoCabinet::DB::OWRITER | KyotoCabinet::DB::OCREATE
627              or die "open error: $!";
628
629        tie my %h2, 'MCE::Shared', { module => 'TokyoCabinet::HDB' }, 'foo.tch',
630           TokyoCabinet::HDB::OWRITER | TokyoCabinet::HDB::OCREAT
631              or die "open error: $!";
632
633        # file extension denotes tree database
634
635        tie my %h3, 'MCE::Shared', { module => 'KyotoCabinet::DB' }, 'foo.kct',
636           KyotoCabinet::DB::OWRITER | KyotoCabinet::DB::OCREATE
637              or die "open error: $!";
638
639        tie my %h4, 'MCE::Shared', { module => 'TokyoCabinet::BDB' }, 'foo.tcb',
640           TokyoCabinet::BDB::OWRITER | TokyoCabinet::BDB::OCREAT
641              or die "open error: $!";
642
643        # on-memory hash database
644
645        tie my %h5, 'MCE::Shared', { module => 'KyotoCabinet::DB' }, '*';
646        tie my %h6, 'MCE::Shared', { module => 'TokyoCabinet::ADB' }, '*';
647
648        # on-memory tree database
649
650        tie my %h7, 'MCE::Shared', { module => 'KyotoCabinet::DB' }, '%#pccap=256m';
651        tie my %h8, 'MCE::Shared', { module => 'TokyoCabinet::ADB' }, '+';
652
653       Tie::Array::DBD
654       Tie::Hash::DBD
655
656        use MCE::Shared;
657        use Tie::Array::DBD;
658        use Tie::Hash::DBD;
659
660        # A valid string is required for the DSN argument, not a DBI handle.
661        # Do not specify the 'str' option for Tie::(Array|Hash)::DBD.
662        # Instead, see encoder-decoder methods described under Common API.
663
664        use DBD::SQLite;
665
666        tie my @a1, 'MCE::Shared', { module => 'Tie::Array::DBD' },
667           'dbi:SQLite:dbname=foo_a.db', {
668              tbl => 't_tie_analysis',
669              key => 'h_key',
670              fld => 'h_value'
671           };
672
673        tie my %h1, 'MCE::Shared', { module => 'Tie::Hash::DBD' },
674           'dbi:SQLite:dbname=foo_h.db', {
675              tbl => 't_tie_analysis',
676              key => 'h_key',
677              fld => 'h_value'
678           };
679
680        use DBD::CSV;
681
682        tie my %h2, 'MCE::Shared', { module => 'Tie::Hash::DBD'},
683           'dbi:CSV:f_dir=.;f_ext=.csv/r;csv_null=1;csv_decode_utf8=0', {
684              tbl => 'mytable',
685              key => 'h_key',
686              fld => 'h_value'
687           };
688
689        # By default, Sereal 3.015+ is used for serialization if available.
690        # This overrides serialization from Sereal-or-Storable to JSON::XS.
691
692        use JSON::XS ();
693
694        tied(%h2)->encoder( \&JSON::XS::encode_json );
695        tied(%h2)->decoder( \&JSON::XS::decode_json );
696
697        my @pairs = ( key1 => 'val1', key2 => 'val2' );
698        my @list  = ( 1, 2, 3, 4 );
699
700        $h2{'foo'} = 'plain value';
701        $h2{'bar'} = { @pairs };
702        $h2{'baz'} = [ @list ];
703

DBM SHARING (CONT)

705       DB cursors, filters, and duplicate keys are not supported, just plain
706       array and hash functionality. The OO interface provides better
707       performance when needed.  Use "iterator" or "next" for iterating over
708       the elements.
709
710        use MCE::Hobo;
711        use MCE::Shared;
712        use Fcntl;
713        use DB_File;
714
715        unlink 'foo_a';
716
717        my $ob = tie my %h1, 'MCE::Shared', { module => 'DB_File' },
718           'foo_a', O_CREAT|O_RDWR, 0640, $DB_HASH or die "open error: $!";
719
720        $h1{key} = 'value';
721        my $val = $h1{key};
722
723        while ( my ($k, $v) = each %h1 ) {
724           print "1: $k => $v\n";
725        }
726
727        # object oriented fashion, faster
728
729        tied(%h1)->STORE( key1 => 'value1' );
730        my $val1 = tied(%h1)->FETCH('key1');
731
732        $ob->STORE( key2 => 'value2' );
733        my $val2 = $ob->FETCH('key2');
734
735        # non-parallel iteration
736
737        my $iter = $ob->iterator;
738        while ( my ($k, $v) = $iter->() ) {
739           print "2: $k => $v\n";
740        }
741
742        # parallel iteration
743
744        sub task {
745           while ( my ($k, $v) = $ob->next ) {
746              print "[$$] $k => $v\n";
747              sleep 1;
748           }
749        }
750
751        MCE::Hobo->create(\&task) for 1 .. 3;
752        MCE::Hobo->waitall;
753
754        $ob->rewind;
755
756        # undef $ob and $iter before %h1 when destroying manually
757
758        undef $ob;
759        undef $iter;
760
761        untie %h1;
762
763       See also Tie::File Demonstration, at the end of the documentation.
764

PDL SHARING

766       MCE::Shared->pdl_byte
767       MCE::Shared->pdl_short
768       MCE::Shared->pdl_ushort
769       MCE::Shared->pdl_long
770       MCE::Shared->pdl_longlong
771       MCE::Shared->pdl_float
772       MCE::Shared->pdl_double
773       MCE::Shared->pdl_ones
774       MCE::Shared->pdl_random
775       MCE::Shared->pdl_sequence
776       MCE::Shared->pdl_zeroes
777       MCE::Shared->pdl_indx
778       MCE::Shared->pdl
779
780       Sugar syntax for PDL construction to take place under the shared-
781       manager process. The helper routines are made available only if "PDL"
782       is loaded before "MCE::Shared".
783
784        use PDL;
785        use MCE::Shared;
786
787        # This makes an extra copy, transfer, including destruction.
788        my $ob1 = MCE::Shared->share( zeroes( 256, 256 ) );
789
790        # Do this instead to not involve an extra copy.
791        my $ob1 = MCE::Shared->pdl_zeroes( 256, 256 );
792
793       Below is a parallel version for a demonstration on PerlMonks.
794
795        # https://www.perlmonks.org/?node_id=1214227 (by vr)
796
797        use strict;
798        use warnings;
799        use feature 'say';
800
801        use PDL;  # must load PDL before MCE::Shared
802
803        use MCE;
804        use MCE::Shared;
805        use Time::HiRes 'time';
806
807        srand( 123 );
808
809        my $time = time;
810
811        my $n = 30000;      # input sample size
812        my $m = 10000;      # number of bootstrap repeats
813        my $r = $n;         # re-sample size
814
815        # On Windows, the non-shared piddle ($x) is unblessed in threads.
816        # Therefore, constructing the piddle inside the worker.
817        # UNIX platforms benefit from copy-on-write. Thus, one copy.
818
819        my $x   = ( $^O eq 'MSWin32' ) ? undef : random( $n );
820        my $avg = MCE::Shared->pdl_zeroes( $m );
821
822        MCE->new(
823           max_workers => 4,
824           sequence    => [ 0, $m - 1 ],
825           chunk_size  => 1,
826           user_begin  => sub {
827              $x = random( $n ) unless ( defined $x );
828           },
829           user_func   => sub {
830              my $idx  = random $r;
831              $idx    *= $n;
832              # $avg is a shared piddle which resides inside the shared-
833              # manager process or thread. The piddle is accessible via the
834              # OO interface only.
835              $avg->set( $_, $x->index( $idx )->avg );
836           }
837        )->run;
838
839        # MCE sets the seed of the base generator uniquely between workers.
840        # Unfortunately, it requires running with one worker for predictable
841        # results (i.e. no guarantee in the order which worker computes the
842        # next input chunk).
843
844        say $avg->pctover( pdl 0.05, 0.95 );
845        say time - $time, ' seconds';
846
847        __END__
848
849        # Output
850
851        [0.49387106  0.4993768]
852        1.09556317329407 seconds
853
854       ins_inplace
855
856       The "ins_inplace" method applies to shared PDL objects. It supports
857       three forms for writing elements back to the PDL object, residing under
858       the shared-manager process.
859
860        # --- action taken by the shared-manager process
861        # ins_inplace(  1 arg  ):  ins( inplace( $this ), $what, 0, 0 );
862        # ins_inplace(  2 args ):  $this->slice( $arg1 ) .= $arg2;
863        # ins_inplace( >2 args ):  ins( inplace( $this ), $what, @coords );
864
865        # --- use case
866        $o->ins_inplace( $result );                    #  1 arg
867        $o->ins_inplace( ":,$start:$stop", $result );  #  2 args
868        $o->ins_inplace( $result, 0, $seq_n );         # >2 args
869
870       Operations such as " + 5 " will not work on shared PDL objects. At this
871       time, the OO interface is the only mechanism for communicating with the
872       shared piddle. For example, call "slice", "sever", or "copy" to fetch
873       elements. Call "ins_inplace" or "set" (shown above) to update elements.
874
875        use strict;
876        use warnings;
877
878        use PDL;  # must load PDL before MCE::Shared
879        use MCE::Shared;
880
881        # make a shared piddle
882        my $b = MCE::Shared->pdl_sequence(15,15);
883
884        # fetch, add 10 to row 2 only
885        my $res1 = $b->slice(":,1:1") + 10;
886        $b->ins_inplace($res1, 0, 1);
887
888        # fetch, add 10 to rows 4 and 5
889        my $res2 = $b->slice(":,3:4") + 10;
890        $b->ins_inplace($res2, 0, 3);
891
892        # make non-shared object (i.e. export-destroy from shared)
893        $b = $b->destroy;
894
895        print "$b\n";
896
897       The following provides parallel demonstrations using "MCE::Flow".
898
899        use strict;
900        use warnings;
901
902        use PDL;  # must load PDL before MCE::Shared
903
904        use MCE::Flow;
905        use MCE::Shared;
906
907        # On Windows, the ($a) piddle is unblessed in worker threads.
908        # Therefore, constructing ($a) inside the worker versus sharing.
909        # UNIX platforms benefit from copy-on-write. Thus, one copy.
910        #
911        # Results are stored in the shared piddle ($b).
912
913        my $a = ( $^O eq 'MSWin32' ) ? undef : sequence(15,15);
914        my $b = MCE::Shared->pdl_zeroes(15,15);
915
916        MCE::Flow->init(
917           user_begin => sub {
918              $a = sequence(15,15) unless ( defined $a );
919           }
920        );
921
922        # with chunking disabled
923
924        mce_flow_s {
925           max_workers => 4, chunk_size => 1
926        },
927        sub {
928           my $row = $_;
929           my $result = $a->slice(":,$row:$row") + 5;
930           $b->ins_inplace($result, 0, $row);
931        }, 0, 15 - 1;
932
933        # with chunking enabled
934
935        mce_flow_s {
936           max_workers => 4, chunk_size => 5, bounds_only => 1
937        },
938        sub {
939           my ($row1, $row2) = @{ $_ };
940           my $result = $a->slice(":,$row1:$row2") + 5;
941           $b->ins_inplace($result, 0, $row1);
942        }, 0, 15 - 1;
943
944        # make non-shared object, export-destroy the shared object
945
946        $b = $b->destroy;
947
948        print "$b\n";
949
950       See also PDL::ParallelCPU and PDL::Parallel::threads. For further
951       reading, the MCE-Cookbook on GitHub provides two PDL demonstrations.
952
953       <https://github.com/marioroy/mce-cookbook>
954

COMMON API

956       blessed
957          Returns the real "blessed" name, provided by the shared-manager
958          process.
959
960           use MCE::Shared;
961           use Scalar::Util qw(blessed);
962
963           my $oh1 = MCE::Shared->share({ module => 'MCE::Shared::Ordhash' });
964           my $oh2 = MCE::Shared->share({ module => 'Hash::Ordered'        });
965
966           print blessed($oh1), "\n";    # MCE::Shared::Object
967           print blessed($oh2), "\n";    # MCE::Shared::Object
968
969           print $oh1->blessed(), "\n";  # MCE::Shared::Ordhash
970           print $oh2->blessed(), "\n";  # Hash::Ordered
971
972       destroy ( { unbless => 1 } )
973       destroy
974          Exports optionally, but destroys the shared object entirely from the
975          shared-manager process. The unbless option is passed to export.
976
977           my $exported_ob = $shared_ob->destroy();
978
979           $shared_ob;     # becomes undef
980
981       lock
982       unlock
983          Shared objects embed a MCE::Mutex object for locking since 1.841.
984
985           use MCE::Shared;
986
987           tie my @shared_array, 'MCE::Shared', { module => 'MCE::Shared::Array' }, 0;
988
989           tied(@shared_array)->lock;
990           $shared_array[0] += 1;
991           tied(@shared_array)->unlock;
992
993           print $shared_array[0], "\n";    # 1
994
995          Locking is not necessary typically when using the OO interface.
996          Although, exclusive access is necessary when involving a FETCH and
997          STORE.
998
999           my $shared_total = MCE::Shared->scalar(2);
1000
1001           $shared_total->lock;
1002           my $val = $shared_total->get;
1003           $shared_total->set( $val * 2 );
1004           $shared_total->unlock;
1005
1006           print $shared_total->get, "\n";  # 4
1007
1008       encoder ( CODE )
1009       decoder ( CODE )
1010          Override freeze/thaw routines. Applies to STORE and FETCH only,
1011          particularly for TIE'd objects. These are called internally for
1012          shared DB objects.
1013
1014          Current API available since 1.827.
1015
1016           use MCE::Shared;
1017           use BerkeleyDB;
1018           use DB_File;
1019
1020           my $file1 = 'file1.db';
1021           my $file2 = 'file2.db';
1022
1023           tie my @db1, 'MCE::Shared', { module => 'DB_File' }, $file1,
1024              O_RDWR|O_CREAT, 0640 or die "open error '$file1': $!";
1025
1026           tie my %db2, 'MCE::Shared', { module => 'BerkeleyDB::Hash' },
1027              -Filename => $file2, -Flags => DB_CREATE
1028              or die "open error '$file2': $!";
1029
1030           # Called automatically by MCE::Shared for DB files.
1031           # tied(@db1)->encoder( MCE::Shared::Server::_get_freeze );
1032           # tied(@db1)->decoder( MCE::Shared::Server::_get_thaw );
1033           # tied(%db2)->encoder( MCE::Shared::Server::_get_freeze );
1034           # tied(%db2)->decoder( MCE::Shared::Server::_get_thaw );
1035           # et cetera.
1036
1037           $db1[0] = 'foo';   # store plain and complex structure
1038           $db1[1] = { key => 'value' };
1039           $db1[2] = [ 'complex' ];
1040
1041           $db2{key} = 'foo'; # ditto, plain and complex structure
1042           $db2{sun} = [ 'complex' ];
1043
1044       export ( { unbless => 1 }, keys )
1045       export
1046          Exports the shared object as a non-shared object. One must export
1047          the shared object when passing into any dump routine. Otherwise, the
1048          "shared_id value" and "blessed name" is all one will see. The
1049          unbless option unblesses any shared Array, Hash, and Scalar object
1050          to a non-blessed array, hash, and scalar respectively.
1051
1052           use MCE::Shared;
1053           use MCE::Shared::Ordhash;
1054
1055           sub _dump {
1056              require Data::Dumper unless $INC{'Data/Dumper.pm'};
1057              no warnings 'once';
1058
1059              local $Data::Dumper::Varname  = 'VAR';
1060              local $Data::Dumper::Deepcopy = 1;
1061              local $Data::Dumper::Indent   = 1;
1062              local $Data::Dumper::Purity   = 1;
1063              local $Data::Dumper::Sortkeys = 0;
1064              local $Data::Dumper::Terse    = 0;
1065
1066              print Data::Dumper::Dumper($_[0]) . "\n";
1067           }
1068
1069           my $oh1 = MCE::Shared->share({ module => 'MCE::Shared::Ordhash' });
1070           my $oh2 = MCE::Shared->ordhash();  # same thing
1071
1072           _dump($oh1);
1073              # bless( [ 1, 'MCE::Shared::Ordhash' ], 'MCE::Shared::Object' )
1074
1075           _dump($oh2);
1076              # bless( [ 2, 'MCE::Shared::Ordhash' ], 'MCE::Shared::Object' )
1077
1078           _dump( $oh1->export );  # dumps object structure and content
1079           _dump( $oh2->export );  # ditto
1080
1081          "export" can optionally take a list of indices/keys for what to
1082          export.  This applies to shared array, hash, and ordhash.
1083
1084           use MCE::Shared;
1085
1086           # shared hash
1087           my $h1 = MCE::Shared->hash(
1088              qw/ I Heard The Bluebirds Sing by Marty Robbins /
1089                # k v     k   v         k    v  k     v
1090           );
1091
1092           # non-shared hash
1093           my $h2 = $h1->export( qw/ I The / );
1094
1095           _dump($h2);
1096
1097           __END__
1098
1099           # Output
1100
1101           $VAR1 = bless( {
1102             'I' => 'Heard',
1103             'The' => 'Bluebirds'
1104           }, 'MCE::Shared::Hash' );
1105
1106          Specifying the unbless option exports a non-blessed data structure
1107          instead.  The unbless option applies to shared MCE::Shared::{ Array,
1108          Hash, and Scalar } objects.
1109
1110           my $h2 = $h1->export( { unbless => 1 }, qw/ I The / );
1111           my $h3 = $h1->export( { unbless => 1 } );
1112
1113           _dump($h2);
1114           _dump($h3);
1115
1116           __END__
1117
1118           # Output
1119
1120           $VAR1 = {
1121             'The' => 'Bluebirds',
1122             'I' => 'Heard'
1123           };
1124
1125           $VAR1 = {
1126             'Marty' => 'Robbins',
1127             'Sing' => 'by',
1128             'The' => 'Bluebirds',
1129             'I' => 'Heard'
1130           };
1131
1132       next
1133          The "next" method provides parallel iteration between workers for
1134          shared "array", "hash", "ordhash", and "sequence". In list context,
1135          returns the next key-value pair or beg-end pair for sequence. In
1136          scalar context, returns the next item. The "undef" value is returned
1137          after the iteration has completed.
1138
1139          Internally, the list of keys to return is set when the closure is
1140          constructed.  Later keys added to the shared array or hash are not
1141          included. Subsequently, the "undef" value is returned for deleted
1142          keys.
1143
1144          The following example iterates through a shared array in parallel.
1145
1146           use MCE::Hobo;
1147           use MCE::Shared;
1148
1149           my $ar = MCE::Shared->array( 'a' .. 'j' );
1150
1151           sub demo1 {
1152              my ( $wid ) = @_;
1153              while ( my ( $index, $value ) = $ar->next ) {
1154                 print "$wid: [ $index ] $value\n";
1155                 sleep 1;
1156              }
1157           }
1158
1159           sub demo2 {
1160              my ( $wid ) = @_;
1161              while ( defined ( my $value = $ar->next ) ) {
1162                 print "$wid: $value\n";
1163                 sleep 1;
1164              }
1165           }
1166
1167           $ar->rewind();
1168
1169           MCE::Hobo->new( \&demo1, $_ ) for 1 .. 3;
1170           MCE::Hobo->waitall(), print "\n";
1171
1172           $ar->rewind();
1173
1174           MCE::Hobo->new( \&demo2, $_ ) for 1 .. 3;
1175           MCE::Hobo->waitall(), print "\n";
1176
1177           __END__
1178
1179           # Output
1180
1181           1: [ 0 ] a
1182           2: [ 1 ] b
1183           3: [ 2 ] c
1184           1: [ 3 ] d
1185           2: [ 5 ] f
1186           3: [ 4 ] e
1187           2: [ 8 ] i
1188           3: [ 6 ] g
1189           1: [ 7 ] h
1190           2: [ 9 ] j
1191
1192           1: a
1193           2: b
1194           3: c
1195           2: e
1196           3: f
1197           1: d
1198           3: g
1199           1: i
1200           2: h
1201           1: j
1202
1203          The form is similar for "sequence". For large sequences, the
1204          "bounds_only" option is recommended. Also, specify "chunk_size"
1205          accordingly. This reduces the amount of traffic to and from the
1206          shared-manager process.
1207
1208           use MCE::Hobo;
1209           use MCE::Shared;
1210
1211           my $N   = shift || 4_000_000;
1212           my $pi  = MCE::Shared->scalar( 0.0 );
1213
1214           my $seq = MCE::Shared->sequence(
1215              { chunk_size => 200_000, bounds_only => 1 }, 0, $N - 1
1216           );
1217
1218           sub compute_pi {
1219              my ( $wid ) = @_;
1220
1221              # Optionally, also receive the chunk_id value
1222              # while ( my ( $beg, $end, $chunk_id ) = $seq->next ) { ... }
1223
1224              while ( my ( $beg, $end ) = $seq->next ) {
1225                 my ( $_pi, $t ) = ( 0.0 );
1226                 for my $i ( $beg .. $end ) {
1227                    $t = ( $i + 0.5 ) / $N;
1228                    $_pi += 4.0 / ( 1.0 + $t * $t );
1229                 }
1230                 $pi->incrby( $_pi );
1231              }
1232
1233              return;
1234           }
1235
1236           MCE::Hobo->create( \&compute_pi, $_ ) for ( 1 .. 8 );
1237
1238           # ... do other stuff ...
1239
1240           MCE::Hobo->waitall();
1241
1242           printf "pi = %0.13f\n", $pi->get / $N;
1243
1244           __END__
1245
1246           # Output
1247
1248           3.1415926535898
1249
1250       rewind ( index, [, index, ... ] )
1251       rewind ( key, [, key, ... ] )
1252       rewind ( "query string" )
1253       rewind ( )
1254          Rewinds the parallel iterator for MCE::Shared::Array,
1255          MCE::Shared::Hash, or MCE::Shared::Ordhash when no arguments are
1256          given. Otherwise, resets the iterator with given criteria. The
1257          syntax for "query string" is described in the shared module.
1258
1259           # array
1260           $ar->rewind;
1261
1262           $ar->rewind( 0, 1 );
1263           $ar->rewind( "val eq some_value" );
1264           $ar->rewind( "key >= 50 :AND val =~ /sun|moon|air|wind/" );
1265           $ar->rewind( "val eq sun :OR val eq moon :OR val eq foo" );
1266           $ar->rewind( "key =~ /$pattern/" );
1267
1268           while ( my ( $index, $value ) = $ar->next ) {
1269              ...
1270           }
1271
1272           # hash, ordhash
1273           $oh->rewind;
1274
1275           $oh->rewind( "key1", "key2" );
1276           $oh->rewind( "val eq some_value" );
1277           $oh->rewind( "key eq some_key :AND val =~ /sun|moon|air|wind/" );
1278           $oh->rewind( "val eq sun :OR val eq moon :OR val eq foo" );
1279           $oh->rewind( "key =~ /$pattern/" );
1280
1281           while ( my ( $key, $value ) = $oh->next ) {
1282              ...
1283           }
1284
1285       rewind ( { options }, begin, end [, step, format ] )
1286       rewind ( begin, end [, step, format ] )
1287       rewind ( )
1288          Rewinds the parallel iterator for MCE::Shared::Sequence when no
1289          arguments are given. Otherwise, resets the iterator with given
1290          criteria.
1291
1292           # sequence
1293           $seq->rewind;
1294
1295           $seq->rewind( { chunk_size => 10, bounds_only => 1 }, 1, 100 );
1296
1297           while ( my ( $beg, $end ) = $seq->next ) {
1298              for my $i ( $beg .. $end ) {
1299                 ...
1300              }
1301           }
1302
1303           $seq->rewind( 1, 100 );
1304
1305           while ( defined ( my $num = $seq->next ) ) {
1306              ...
1307           }
1308
1309       store ( key, value )
1310          Deeply sharing a non-blessed structure recursively is possible with
1311          "store", an alias to "STORE".
1312
1313           use MCE::Shared;
1314
1315           my $h1 = MCE::Shared->hash();
1316           my $h2 = MCE::Shared->hash();
1317
1318           # auto-shares deeply
1319           $h1->store('key', [ 0, 2, 5, { 'foo' => 'bar' } ]);
1320           $h2->{key}[3]{foo} = 'baz';    # via auto-vivification
1321
1322           my $v1 = $h1->get('key')->get(3)->get('foo');  # bar
1323           my $v2 = $h2->get('key')->get(3)->get('foo');  # baz
1324           my $v3 = $h2->{key}[3]{foo};                   # baz
1325

SERVER API

1327       init
1328          This method is called by each MCE and Hobo worker automatically
1329          after spawning.  The effect is extra parallelism and decreased
1330          latency during inter-process communication to the shared-manager
1331          process. The optional ID (an integer) is modded internally in a
1332          round-robin fashion.
1333
1334           MCE::Shared->init();
1335           MCE::Shared->init( ID );
1336
1337       pid
1338          Returns the process ID of the shared-manager process. This class
1339          method was added in 1.849 for stopping all workers immediately when
1340          exiting a Graphics::Framebuffer application. It returns an undefined
1341          value if the shared-manager is not running. Not useful otherwise if
1342          running threads (i.e. same PID).
1343
1344           MCE::Shared->pid();
1345
1346           $SIG{INT} = $SIG{HUP} = $SIG{TERM} = sub {
1347              # Signal workers and the shared manager all at once
1348              CORE::kill('KILL', MCE::Hobo->list_pids(), MCE::Shared->pid());
1349              exec('reset');
1350           };
1351
1352       start
1353          Starts the shared-manager process. This is done automatically unless
1354          Perl lacks IO::FDPass, needed to share "condvar" and "queue" while
1355          the shared-manager is running.
1356
1357           MCE::Shared->start();
1358
1359       stop
1360          Stops the shared-manager process, wiping all shared data content.
1361          This is called by the "END" block automatically when the script
1362          terminates. However, do stop explicitly to reap the shared-manager
1363          process before exec'ing.
1364
1365           MCE::Shared->stop();
1366
1367           exec('command');
1368

LOCKING

1370       Application-level advisory locking is possible with MCE::Mutex.
1371
1372        use MCE::Hobo;
1373        use MCE::Mutex;
1374        use MCE::Shared;
1375
1376        my $mutex = MCE::Mutex->new();
1377
1378        tie my $cntr, 'MCE::Shared', 0;
1379
1380        sub work {
1381           for ( 1 .. 1000 ) {
1382              $mutex->lock;
1383
1384              # Incrementing involves 2 IPC ops ( FETCH and STORE ).
1385              # Thus, locking is required.
1386              $cntr++;
1387
1388              $mutex->unlock;
1389           }
1390        }
1391
1392        MCE::Hobo->create('work') for ( 1 .. 8 );
1393        MCE::Hobo->waitall;
1394
1395        print $cntr, "\n"; # 8000
1396
1397       Locking is available for shared objects via "lock" and "unlock" methods
1398       since 1.841. Previously, for "condvar" only.
1399
1400        use MCE::Hobo;
1401        use MCE::Shared;
1402
1403        tie my $cntr, 'MCE::Shared', 0;
1404
1405        sub work {
1406           for ( 1 .. 1000 ) {
1407              tied($cntr)->lock;
1408
1409              # Incrementing involves 2 IPC ops ( FETCH and STORE ).
1410              # Thus, locking is required.
1411              $cntr++;
1412
1413              tied($cntr)->unlock;
1414           }
1415        }
1416
1417        MCE::Hobo->create('work') for ( 1 .. 8 );
1418        MCE::Hobo->waitall;
1419
1420        print $cntr, "\n"; # 8000
1421
1422       Typically, locking is not necessary using the OO interface. The reason
1423       is that MCE::Shared is implemented using a single-point of entry for
1424       commands sent to the shared-manager process. Furthermore, the shared
1425       classes include sugar methods for combining set and get in a single
1426       operation.
1427
1428        use MCE::Hobo;
1429        use MCE::Shared;
1430
1431        my $cntr = MCE::Shared->scalar( 0 );
1432
1433        sub work {
1434           for ( 1 .. 1000 ) {
1435              # The next statement increments the value without having
1436              # to call set and get explicitly.
1437              $cntr->incr;
1438           }
1439        }
1440
1441        MCE::Hobo->create('work') for ( 1 .. 8 );
1442        MCE::Hobo->waitall;
1443
1444        print $cntr->get, "\n"; # 8000
1445
1446       Another possibility when running threads is locking via
1447       threads::shared.
1448
1449        use threads;
1450        use threads::shared;
1451
1452        use MCE::Flow;
1453        use MCE::Shared;
1454
1455        my $mutex : shared;
1456
1457        tie my $cntr, 'MCE::Shared', 0;
1458
1459        sub work {
1460           for ( 1 .. 1000 ) {
1461              lock $mutex;
1462
1463              # the next statement involves 2 IPC ops ( get and set )
1464              # thus, locking is required
1465              $cntr++;
1466           }
1467        }
1468
1469        MCE::Flow->run( { max_workers => 8 }, \&work );
1470        MCE::Flow->finish;
1471
1472        print $cntr, "\n"; # 8000
1473
1474       Of the three demonstrations, the OO interface yields the best
1475       performance.  This is from the lack of locking at the application
1476       level. The results were obtained from a MacBook Pro (Haswell) running
1477       at 2.6 GHz, 1600 MHz RAM.
1478
1479        CentOS 7.2 VM
1480
1481           -- Perl v5.16.3
1482           MCE::Mutex .... : 0.528 secs.
1483           OO Interface .. : 0.062 secs.
1484           threads::shared : 0.545 secs.
1485
1486        FreeBSD 10.0 VM
1487
1488           -- Perl v5.16.3
1489           MCE::Mutex .... : 0.367 secs.
1490           OO Interface .. : 0.083 secs.
1491           threads::shared : 0.593 secs.
1492
1493        Mac OS X 10.11.6 ( Host OS )
1494
1495           -- Perl v5.18.2
1496           MCE::Mutex .... : 0.397 secs.
1497           OO Interface .. : 0.070 secs.
1498           threads::shared : 0.463 secs.
1499
1500        Solaris 11.2 VM
1501
1502           -- Perl v5.12.5 installed with the OS
1503           MCE::Mutex .... : 0.895 secs.
1504           OO Interface .. : 0.099 secs.
1505           threads::shared :              Perl not built to support threads
1506
1507           -- Perl v5.22.2 built with threads support
1508           MCE::Mutex .... : 0.788 secs.
1509           OO Interface .. : 0.086 secs.
1510           threads::shared : 0.895 secs.
1511
1512        Windows 7 VM
1513
1514           -- Perl v5.22.2
1515           MCE::Mutex .... : 1.045 secs.
1516           OO Interface .. : 0.312 secs.
1517           threads::shared : 1.061 secs.
1518
1519       Beginning with MCE::Shared 1.809, the "pipeline" method provides
1520       another way.  Included in "Array", "Cache", "Hash", "Minidb", and
1521       "Ordhash", it combines multiple commands for the object to be processed
1522       serially. For shared objects, the call is made atomically due to single
1523       IPC to the shared-manager process.
1524
1525       The "pipeline" method is fully "wantarray"-aware and receives a list of
1526       commands and their arguments. In scalar or list context, it returns
1527       data from the last command in the pipeline.
1528
1529        use MCE::Mutex;
1530        use MCE::Shared;
1531
1532        my $mutex = MCE::Mutex->new();
1533        my $oh = MCE::Shared->ordhash();
1534        my @vals;
1535
1536        # mutex locking
1537
1538        $mutex->lock;
1539        $oh->set( foo => "a_a" );
1540        $oh->set( bar => "b_b" );
1541        $oh->set( baz => "c_c" );
1542        @vals = $oh->mget( qw/ foo bar baz / );
1543        $mutex->unlock;
1544
1545        # pipeline, same thing done atomically
1546
1547        @vals = $oh->pipeline(
1548           [ "set", foo => "a_a" ],
1549           [ "set", bar => "b_b" ],
1550           [ "set", baz => "c_c" ],
1551           [ "mget", qw/ foo bar baz / ]
1552        );
1553
1554        # ( "a_a", "b_b", "c_c" )
1555
1556       There is also "pipeline_ex", same as "pipeline", but returns data for
1557       every command in the pipeline.
1558
1559        @vals = $oh->pipeline_ex(
1560           [ "set", foo => "a_a" ],
1561           [ "set", bar => "b_b" ],
1562           [ "set", baz => "c_c" ]
1563        );
1564
1565        # ( "a_a", "b_b", "c_c" )
1566

PYTHON DEMONSTRATION

1568       Sharing a Python class is possible, starting with the 1.827 release.
1569       The construction is simply calling share with the module option.
1570       Methods are accessible via the OO interface.
1571
1572        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1573        # Python class.
1574        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1575
1576        package My::Class;
1577
1578        use strict;
1579        use warnings;
1580
1581        use Inline::Python qw( py_eval py_bind_class );
1582
1583        py_eval ( <<'END_OF_PYTHON_CLASS' );
1584
1585        class MyClass:
1586            def __init__(self):
1587                self.data = [0,0]
1588
1589            def set (self, key, value):
1590                self.data[key] = value
1591
1592            def get (self, key):
1593                try: return self.data[key]
1594                except KeyError: return None
1595
1596            def incr (self, key):
1597                try: self.data[key] = self.data[key] + 1
1598                except KeyError: self.data[key] = 1
1599
1600        END_OF_PYTHON_CLASS
1601
1602        # Register methods for best performance.
1603
1604        py_bind_class(
1605            'My::Class', '__main__', 'MyClass',
1606            'set', 'get', 'incr'
1607        );
1608
1609        1;
1610
1611        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1612        # Share Python class. Requires MCE::Shared 1.827 or later.
1613        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1614
1615        use strict;
1616        use warnings;
1617
1618        use MCE::Hobo;
1619        use MCE::Shared;
1620
1621        my $py1 = MCE::Shared->share({ module => 'My::Class' });
1622        my $py2 = MCE::Shared->share({ module => 'My::Class' });
1623
1624        MCE::Shared->start;
1625
1626        $py1->set(0, 100);
1627        $py2->set(1, 200);
1628
1629        die "Ooops" unless $py1->get(0) eq '100';
1630        die "Ooops" unless $py2->get(1) eq '200';
1631
1632        sub task {
1633            $py1->incr(0) for 1..50000;
1634            $py2->incr(1) for 1..50000;
1635        }
1636
1637        MCE::Hobo->create(\&task) for 1..3;
1638        MCE::Hobo->waitall;
1639
1640        print $py1->get(0), "\n";  # 150100
1641        print $py2->get(1), "\n";  # 150200
1642

LOGGER DEMONSTRATION

1644       Often, the requirement may call for concurrent logging by many workers.
1645       Calling localtime or gmtime per each log entry is expensive. This uses
1646       the old time-stamp value until one second has elapsed.
1647
1648        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1649        # Logger class.
1650        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1651
1652        package My::Logger;
1653
1654        use strict;
1655        use warnings;
1656
1657        use Time::HiRes qw( time );
1658
1659        # construction
1660
1661        sub new {
1662            my ( $class, %self ) = @_;
1663
1664            open $self{fh}, ">>", $self{path} or return '';
1665            binmode $self{fh};
1666
1667            $self{stamp} = localtime;  # or gmtime
1668            $self{time } = time;
1669
1670            bless \%self, $class;
1671        }
1672
1673        # $ob->log("message");
1674
1675        sub log {
1676            my ( $self, $stamp ) = ( shift );
1677
1678            if ( time - $self->{time} > 1.0 ) {
1679                $self->{stamp} = $stamp = localtime;  # or gmtime
1680                $self->{time } = time;
1681            }
1682            else {
1683                $stamp = $self->{stamp};
1684            }
1685
1686            print {$self->{fh}} "$stamp --- @_\n";
1687        }
1688
1689        # $ob->autoflush(0);
1690        # $ob->autoflush(1);
1691
1692        sub autoflush {
1693            my ( $self, $flag ) = @_;
1694
1695            if ( defined fileno($self->{fh}) ) {
1696                 $flag ? select(( select($self->{fh}), $| = 1 )[0])
1697                       : select(( select($self->{fh}), $| = 0 )[0]);
1698
1699                 return 1;
1700            }
1701
1702            return;
1703        }
1704
1705        # $ob->binmode($layer);
1706        # $ob->binmode();
1707
1708        sub binmode {
1709            my ( $self, $layer ) = @_;
1710
1711            if ( defined fileno($self->{fh}) ) {
1712                CORE::binmode $self->{fh}, $layer // ':raw';
1713
1714                return 1;
1715            }
1716
1717            return;
1718        }
1719
1720        # $ob->close()
1721
1722        sub close {
1723            my ( $self ) = @_;
1724
1725            if ( defined fileno($self->{fh}) ) {
1726                close $self->{'fh'};
1727            }
1728
1729            return;
1730        }
1731
1732        # $ob->flush()
1733
1734        sub flush {
1735            my ( $self ) = @_;
1736
1737            if ( defined fileno($self->{fh}) ) {
1738                my $old_fh = select $self->{fh};
1739                my $old_af = $|; $| = 1; $| = $old_af;
1740                select $old_fh;
1741
1742                return 1;
1743            }
1744
1745            return;
1746        }
1747
1748        1;
1749
1750        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1751        # Concurrent logger demo. Requires MCE::Shared 1.827 or later.
1752        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1753
1754        use strict;
1755        use warnings;
1756
1757        use MCE::Hobo;
1758        use MCE::Shared;
1759
1760        my $file = "log.txt";
1761        my $pid  = $$;
1762
1763        my $ob = MCE::Shared->share( { module => 'My::Logger' }, path => $file )
1764            or die "open error '$file': $!";
1765
1766        # $ob->autoflush(1);   # optional, flush writes immediately
1767
1768        sub work {
1769            my $id = shift;
1770            for ( 1 .. 250_000 ) {
1771                $ob->log("Hello from $id: $_");
1772            }
1773        }
1774
1775        MCE::Hobo->create('work', $_) for 1 .. 4;
1776        MCE::Hobo->waitall;
1777
1778        # Threads and multi-process safety for closing the handle.
1779
1780        sub CLONE { $pid = 0; }
1781
1782        END { $ob->close if $ob && $pid == $$; }
1783

TIE::FILE DEMONSTRATION

1785       The following presents a concurrent Tie::File demonstration. Each
1786       element in the array corresponds to a record in the text file. JSON,
1787       being readable, seems appropiate for encoding complex objects.
1788
1789        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1790        # Class extending Tie::File with two sugar methods.
1791        # Requires MCE::Shared 1.827 or later.
1792        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1793
1794        package My::File;
1795
1796        use strict;
1797        use warnings;
1798
1799        use Tie::File;
1800
1801        our @ISA = 'Tie::File';
1802
1803        # $ob->append('string');
1804
1805        sub append {
1806            my ($self, $key) = @_;
1807            my $val = $self->FETCH($key); $val .= $_[2];
1808            $self->STORE($key, $val);
1809            length $val;
1810        }
1811
1812        # $ob->incr($key);
1813
1814        sub incr {
1815            my ( $self, $key ) = @_;
1816            my $val = $self->FETCH($key); $val += 1;
1817            $self->STORE($key, $val);
1818            $val;
1819        }
1820
1821        1;
1822
1823        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1824        # The MCE::Mutex module isn't needed unless IPC involves two or
1825        # more trips for the underlying action.
1826        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1827
1828        use strict;
1829        use warnings;
1830
1831        use MCE::Hobo;
1832        use MCE::Mutex;
1833        use MCE::Shared;
1834
1835        use JSON::MaybeXS;
1836
1837        # Safety for data having line breaks.
1838        use constant EOL => "\x{0a}~\x{0a}";
1839
1840        my $file  = 'file.txt';
1841        my $mutex = MCE::Mutex->new();
1842        my $pid   = $$;
1843
1844        my $ob = tie my @db, 'MCE::Shared', { module => 'My::File' }, $file,
1845            recsep => EOL or die "open error '$file': $!";
1846
1847        $ob->encoder( \&JSON::MaybeXS::encode_json );
1848        $ob->decoder( \&JSON::MaybeXS::decode_json );
1849
1850        $db[20] = 0;  # a counter at offset 20 into the array
1851        $db[21] = [ qw/ foo bar / ];  # store complex structure
1852
1853        sub task {
1854            my $id  = sprintf "%02s", shift;
1855            my $row = int($id) - 1;
1856            my $chr = sprintf "%c", 97 + $id - 1;
1857
1858            # A mutex isn't necessary when storing a value.
1859            # Ditto for fetching a value.
1860
1861            $db[$row] = "Hello from $id: ";  # 1 trip
1862            my $val   = length $db[$row];    # 1 trip
1863
1864            # A mutex may be necessary for updates involving 2 or
1865            # more trips (FETCH and STORE) during IPC, from and to
1866            # the shared-manager process, unless a unique row.
1867
1868            for ( 1 .. 40 ) {
1869              # $db[$row] .= $id;         # 2 trips, unique row - okay
1870                $ob->append($row, $chr);  # 1 trip via the OO interface
1871
1872              # $mu->lock;
1873              # $db[20] += 1;             # incrementing counter, 2 trips
1874              # $mu->unlock;
1875
1876                $ob->incr(20);            # same thing via OO, 1 trip
1877            }
1878
1879            my $len = length $db[$row];   # 1 trip
1880
1881            printf "hobo %2d : %d\n", $id, $len;
1882        }
1883
1884        MCE::Hobo->create('task', $_) for 1 .. 20;
1885        MCE::Hobo->waitall;
1886
1887        printf "counter : %d\n", $db[20];
1888        print  $db[21]->[0], "\n";  # foo
1889
1890        # Threads and multi-process safety for closing the handle.
1891
1892        sub CLONE { $pid = 0; }
1893
1894        END {
1895            if ( $pid == $$ ) {
1896                undef $ob;  # important, undef $ob before @db
1897                untie @db;  # untie @db to flush pending writes
1898            }
1899        }
1900

REQUIREMENTS

1902       MCE::Shared requires Perl 5.10.1 or later. The IO::FDPass module is
1903       highly recommended on UNIX and Windows. This module does not install it
1904       by default.
1905

SOURCE AND FURTHER READING

1907       The source, cookbook, and examples are hosted at GitHub.
1908
1909       ·  <https://github.com/marioroy/mce-shared>
1910
1911       ·  <https://github.com/marioroy/mce-cookbook>
1912
1913       ·  <https://github.com/marioroy/mce-examples>
1914

INDEX

1916       MCE, MCE::Hobo
1917

AUTHOR

1919       Mario E. Roy, <marioeroy AT gmail DOT com>
1920

COPYRIGHT AND LICENSE

1922       Copyright (C) 2016-2020 by Mario E. Roy
1923
1924       MCE::Shared is released under the same license as Perl.
1925
1926       See <http://dev.perl.org/licenses/> for more information.
1927
1928
1929
1930perl v5.32.0                      2020-08-02                    MCE::Shared(3)