MCE::Shared(3pm)

1MCE::Shared(3)        User Contributed Perl Documentation       MCE::Shared(3)
2
3
4

NAME

6       MCE::Shared - MCE extension for sharing data supporting threads and
7       processes
8

VERSION

10       This document describes MCE::Shared version 1.877
11

SYNOPSIS

13        # OO construction.
14
15        use MCE::Shared;
16
17        my $ar = MCE::Shared->array( @list );
18        my $ca = MCE::Shared->cache( max_keys => 500, max_age => 60 );
19        my $cv = MCE::Shared->condvar( 0 );
20        my $fh = MCE::Shared->handle( '>>', \*STDOUT ) or die "$!";
21        my $ha = MCE::Shared->hash( @pairs );
22        my $oh = MCE::Shared->ordhash( @pairs );
23        my $db = MCE::Shared->minidb();
24        my $qu = MCE::Shared->queue( await => 1, fast => 0 );
25        my $va = MCE::Shared->scalar( $value );
26        my $se = MCE::Shared->sequence( $begin, $end, $step, $fmt );
27        my $ob = MCE::Shared->share( $blessed_object );
28
29        # Mutex locking is supported for all shared objects since 1.841.
30        # Previously only shared C<condvar>s allowed locking.
31
32        $ar->lock;
33        $ar->unlock;
34        ...
35        $ob->lock;
36        $ob->unlock;
37
38        # The Perl-like mce_open function is available since 1.002.
39
40        mce_open my $fh, ">>", "/foo/bar.log" or die "open error: $!";
41
42        # Tie construction. The module API option is available since 1.825.
43
44        use v5.10;
45        use MCE::Flow;
46        use MCE::Shared;
47
48        my %args  = ( max_keys => 500, max_age => 60 );
49        my @pairs = ( foo => 'bar', woo => 'baz' );
50        my @list  = ( 'a' .. 'z' );
51
52        tie my $va1, 'MCE::Shared', { module => 'MCE::Shared::Scalar' }, 0;
53        tie my @ar1, 'MCE::Shared', { module => 'MCE::Shared::Array' }, @list;
54        tie my %ca1, 'MCE::Shared', { module => 'MCE::Shared::Cache' }, %args;
55        tie my %ha1, 'MCE::Shared', { module => 'MCE::Shared::Hash' }, @pairs;
56        tie my %oh1, 'MCE::Shared', { module => 'MCE::Shared::Ordhash' }, @pairs;
57        tie my %oh2, 'MCE::Shared', { module => 'Hash::Ordered' }, @pairs;
58        tie my %oh3, 'MCE::Shared', { module => 'Tie::IxHash' }, @pairs;
59        tie my $cy1, 'MCE::Shared', { module => 'Tie::Cycle' }, [ 1 .. 8 ];
60        tie my $va2, 'MCE::Shared', { module => 'Tie::StdScalar' }, 'hello';
61        tie my @ar3, 'MCE::Shared', { module => 'Tie::StdArray' }, @list;
62        tie my %ha2, 'MCE::Shared', { module => 'Tie::StdHash' }, @pairs;
63        tie my %ha3, 'MCE::Shared', { module => 'Tie::ExtraHash' }, @pairs;
64
65        tie my $cnt, 'MCE::Shared', 0; # default MCE::Shared::Scalar
66        tie my @foo, 'MCE::Shared';    # default MCE::Shared::Array
67        tie my %bar, 'MCE::Shared';    # default MCE::Shared::Hash
68
69        tie my @ary, 'MCE::Shared', qw( a list of values );
70        tie my %ha,  'MCE::Shared', key1 => 'val1', key2 => 'val2';
71        tie my %ca,  'MCE::Shared', { max_keys => 500, max_age => 60 };
72        tie my %oh,  'MCE::Shared', { ordered => 1 }, key1 => 'value';
73
74        # Mutex locking is supported for all shared objects since 1.841.
75
76        tied($va1)->lock;
77        tied($va1)->unlock;
78        ...
79        tied(%bar)->lock;
80        tied(%bar)->unlock;
81
82        # Demonstration.
83
84        my $mutex = MCE::Mutex->new;
85
86        mce_flow {
87           max_workers => 4
88        },
89        sub {
90           my ( $mce ) = @_;
91           my ( $pid, $wid ) = ( MCE->pid, MCE->wid );
92
93           # Locking is necessary when multiple workers update the same
94           # element. The reason is that it may involve 2 trips to the
95           # shared-manager process: fetch and store in this case.
96
97           $mutex->enter( sub { $cnt += 1 } );
98
99           # Otherwise, locking is optional for unique elements.
100
101           $foo[ $wid - 1 ] = $pid;
102           $bar{ $pid }     = $wid;
103
104           # From 1.841 onwards, all shared objects include mutex locking
105           # to not need to construct MCE::Mutex separately.
106
107           tied($va1)->lock;
108           $va1 += 1;
109           tied($va1)->unlock;
110
111           return;
112        };
113
114        say "scalar : $cnt";
115        say "scalar : $va1";
116        say " array : $_" for (@foo);
117        say "  hash : $_ => $bar{$_}" for (sort keys %bar);
118
119        __END__
120
121        # Output
122
123        scalar : 4
124        scalar : 4
125         array : 37847
126         array : 37848
127         array : 37849
128         array : 37850
129          hash : 37847 => 1
130          hash : 37848 => 2
131          hash : 37849 => 3
132          hash : 37850 => 4
133

DESCRIPTION

135       This module provides data sharing capabilities for MCE supporting
136       threads and processes. MCE::Hobo provides threads-like parallelization
137       for running code asynchronously.
138

EXTRA FUNCTIONALITY

140       MCE::Shared enables extra functionality on systems with IO::FDPass
141       installed.  Without it, MCE::Shared is unable to send file descriptors
142       to the shared-manager process. The use applies to Condvar, Queue, and
143       Handle (mce_open). IO::FDpass isn't used for anything else.
144
145        use MCE::Shared;
146
147        # One may want to start the shared-manager early.
148
149        MCE::Shared->start();
150
151        # Typically, the shared-manager is started automatically when
152        # constructing a shared object.
153
154        my $ca = MCE::Shared->cache( max_keys => 500 );
155
156        # IO::FDPass is necessary for constructing a shared condvar or queue
157        # while the manager is running in order to send file descriptors
158        # associated with the object.
159
160        # Workers block using a socket handle for ->wait and ->timedwait.
161
162        my $cv = MCE::Shared->condvar();
163
164        # Workers block using a socket handle for ->dequeue and ->await.
165
166        my $q1 = MCE::Shared->queue();
167        my $q2 = MCE::Shared->queue( await => 1 );
168
169       For platforms where IO::FDPass isn't possible (e.g. Cygwin), construct
170       "condvar" and "queue" before other classes. The shared-manager process
171       will be delayed until sharing other classes (e.g. Array, Hash) or
172       starting explicitly.
173
174        use MCE::Shared;
175
176        my $has_IO_FDPass = $INC{'IO/FDPass.pm'} ? 1 : 0;
177
178        my $cv  = MCE::Shared->condvar( 0 );
179        my $que = MCE::Shared->queue( fast => 1 );
180
181        MCE::Shared->start() unless $has_IO_FDPass;
182
183        my $ha = MCE::Shared->hash();  # started implicitly
184
185       Note that MCE starts the shared-manager, prior to spawning workers, if
186       not yet started. Ditto for MCE::Hobo.
187
188       Regarding mce_open, "IO::FDPass" is needed for constructing a shared-
189       handle from a non-shared handle not yet available inside the shared-
190       manager process.  The workaround is to have the non-shared handle made
191       before the shared-manager is started. Passing a file by reference is
192       fine for the three STD* handles.
193
194        # The shared-manager knows of \*STDIN, \*STDOUT, \*STDERR.
195
196        mce_open my $shared_in,  "<",  \*STDIN;   # ok
197        mce_open my $shared_out, ">>", \*STDOUT;  # ok
198        mce_open my $shared_err, ">>", \*STDERR;  # ok
199        mce_open my $shared_fh1, "<",  "/path/to/sequence.fasta";  # ok
200        mce_open my $shared_fh2, ">>", "/path/to/results.log";     # ok
201
202        mce_open my $shared_fh, ">>", \*NON_SHARED_FH;  # requires IO::FDPass
203
204       The IO::FDPass module is known to work reliably on most platforms.
205       Install 1.1 or later to rid of limitations described above.
206
207        perl -MIO::FDPass -le "print 'Cheers! Perl has IO::FDPass.'"
208

DATA SHARING

210       MCE::Shared->array MCE::Shared::Array
211       MCE::Shared->cache MCE::Shared::Cache
212       MCE::Shared->condvar MCE::Shared::Condvar
213       MCE::Shared->handle MCE::Shared::Handle
214       MCE::Shared->hash MCE::Shared::Hash
215       MCE::Shared->minidb MCE::Shared::Minidb
216       MCE::Shared->ordhash MCE::Shared::Ordhash
217       MCE::Shared->queue MCE::Shared::Queue
218       MCE::Shared->scalar MCE::Shared::Scalar
219       MCE::Shared->sequence MCE::Shared::Sequence
220
221       Below, synopsis for sharing classes included with MCE::Shared.
222
223        use MCE::Shared;
224
225        # short form
226
227        $ar = MCE::Shared->array( @list );
228        $ca = MCE::Shared->cache( max_keys => 500, max_age => 60 );
229        $cv = MCE::Shared->condvar( 0 );
230        $fh = MCE::Shared->handle( ">>", \*STDOUT ); # see mce_open below
231        $ha = MCE::Shared->hash( @pairs );
232        $db = MCE::Shared->minidb();
233        $oh = MCE::Shared->ordhash( @pairs );
234        $qu = MCE::Shared->queue( await => 1, fast => 0 );
235        $va = MCE::Shared->scalar( $value );
236        $se = MCE::Shared->sequence( $begin, $end, $step, $fmt );
237
238        mce_open my $fh, ">>", \*STDOUT or die "open error: $!";
239
240        # long form
241
242        $ar = MCE::Shared->share( { module => 'MCE::Shared::Array'    }, ... );
243        $ca = MCE::Shared->share( { module => 'MCE::Shared::Cache'    }, ... );
244        $cv = MCE::Shared->share( { module => 'MCE::Shared::Condvar'  }, ... );
245        $fh = MCE::Shared->share( { module => 'MCE::Shared::Handle'   }, ... );
246        $ha = MCE::Shared->share( { module => 'MCE::Shared::Hash'     }, ... );
247        $db = MCE::Shared->share( { module => 'MCE::Shared::Minidb'   }, ... );
248        $oh = MCE::Shared->share( { module => 'MCE::Shared::Ordhash'  }, ... );
249        $qu = MCE::Shared->share( { module => 'MCE::Shared::Queue'    }, ... );
250        $va = MCE::Shared->share( { module => 'MCE::Shared::Scalar'   }, ... );
251        $se = MCE::Shared->share( { module => 'MCE::Shared::Sequence' }, ... );
252
253       The restriction for sharing classes not included with MCE::Shared is
254       that the object must not have file-handles nor code-blocks.
255
256        $oh = MCE::Shared->share( { module => 'Hash::Ordered' }, ... );
257
258       open ( filehandle, expr )
259       open ( filehandle, mode, expr )
260       open ( filehandle, mode, reference )
261
262       In version 1.002 and later, constructs a new object by opening the file
263       whose filename is given by "expr", and associates it with "filehandle".
264       When omitting error checking at the application level, MCE::Shared
265       emits a message and stop if open fails.
266
267       See MCE::Shared::Handle for chunk IO demonstrations.
268
269        {
270          use MCE::Shared::Handle;
271
272          # "non-shared" or "local construction" for use by a single process
273          MCE::Shared::Handle->open( my $fh, "<", "file.log" ) or die "$!";
274          MCE::Shared::Handle::open  my $fh, "<", "file.log"   or die "$!";
275
276          # mce_open is an alias for MCE::Shared::Handle::open
277          mce_open my $fh, "<", "file.log" or die "$!";
278        }
279
280        {
281          use MCE::Shared;
282
283          # construction for "sharing" with other threads and processes
284          MCE::Shared->open( my $fh, "<", "file.log" ) or die "$!";
285          MCE::Shared::open  my $fh, "<", "file.log"   or die "$!";
286
287          # mce_open is an alias for MCE::Shared::open
288          mce_open my $fh, "<", "file.log" or die "$!";
289        }
290
291       mce_open ( filehandle, expr )
292       mce_open ( filehandle, mode, expr )
293       mce_open ( filehandle, mode, reference )
294
295       Native Perl-like syntax to open a shared-file for reading:
296
297        use MCE::Shared;
298
299        # mce_open is exported by MCE::Shared or MCE::Shared::Handle.
300        # It creates a shared file handle with MCE::Shared present
301        # or a non-shared handle otherwise.
302
303        mce_open my $fh, "< input.txt"     or die "open error: $!";
304        mce_open my $fh, "<", "input.txt"  or die "open error: $!";
305        mce_open my $fh, "<", \*STDIN      or die "open error: $!";
306
307       and for writing:
308
309        mce_open my $fh, "> output.txt"    or die "open error: $!";
310        mce_open my $fh, ">", "output.txt" or die "open error: $!";
311        mce_open my $fh, ">", \*STDOUT     or die "open error: $!";
312
313       num_sequence
314
315       "num_sequence" is an alias for "sequence".
316

DEEPLY SHARING

318       The following is a demonstration for a shared tied-hash variable.
319       Before venturing into the actual code, notice the dump function making
320       a call to "export" explicitly for objects of type
321       "MCE::Shared::Object". This is necessary in order to retrieve the data
322       from the shared-manager process.
323
324       The "export" method is described later under the Common API section.
325
326        use MCE::Shared;
327
328        sub _dump {
329           require Data::Dumper unless $INC{'Data/Dumper.pm'};
330           no warnings 'once';
331
332           local $Data::Dumper::Varname  = 'VAR';
333           local $Data::Dumper::Deepcopy = 1;
334           local $Data::Dumper::Indent   = 1;
335           local $Data::Dumper::Purity   = 1;
336           local $Data::Dumper::Sortkeys = 0;
337           local $Data::Dumper::Terse    = 0;
338
339           ( ref $_[0] eq 'MCE::Shared::Object' )
340              ? print Data::Dumper::Dumper( $_[0]->export ) . "\n"
341              : print Data::Dumper::Dumper( $_[0] ) . "\n";
342        }
343
344        tie my %abc, 'MCE::Shared';
345
346        my @parents = qw( a b c );
347        my @children = qw( 1 2 3 4 );
348
349        for my $parent ( @parents ) {
350           for my $child ( @children ) {
351              $abc{ $parent }{ $child } = 1;
352           }
353        }
354
355        _dump( tied( %abc ) );
356
357        __END__
358
359        # Output
360
361        $VAR1 = bless( {
362          'c' => bless( {
363            '1' => '1',
364            '4' => '1',
365            '3' => '1',
366            '2' => '1'
367          }, 'MCE::Shared::Hash' ),
368          'a' => bless( {
369            '1' => '1',
370            '4' => '1',
371            '3' => '1',
372            '2' => '1'
373          }, 'MCE::Shared::Hash' ),
374          'b' => bless( {
375            '1' => '1',
376            '4' => '1',
377            '3' => '1',
378            '2' => '1'
379          }, 'MCE::Shared::Hash' )
380        }, 'MCE::Shared::Hash' );
381
382       Dereferencing provides hash-like behavior for "hash" and "ordhash".
383       Array-like behavior is allowed for "array", not shown below.
384
385        use MCE::Shared;
386        use Data::Dumper;
387
388        my $abc = MCE::Shared->hash;
389
390        my @parents = qw( a b c );
391        my @children = qw( 1 2 3 4 );
392
393        for my $parent ( @parents ) {
394           for my $child ( @children ) {
395              $abc->{ $parent }{ $child } = 1;
396           }
397        }
398
399        print Dumper( $abc->export({ unbless => 1 }) ), "\n";
400
401       Each level in a deeply structure requires a separate trip to the
402       shared-manager process. The included "MCE::Shared::Minidb" module
403       provides optimized methods for working with hash of hashes "HoH" and
404       hash of arrays "HoA".
405
406        use MCE::Shared;
407        use Data::Dumper;
408
409        my $abc = MCE::Shared->minidb;
410
411        my @parents = qw( a b c );
412        my @children = qw( 1 2 3 4 );
413
414        for my $parent ( @parents ) {
415           for my $child ( @children ) {
416              $abc->hset($parent, $child, 1);
417           }
418        }
419
420        print Dumper( $abc->export ), "\n";
421
422       For further reading, see MCE::Shared::Minidb.
423

OBJECT SHARING

425       MCE::Shared->share
426
427       This class method transfers the blessed-object to the shared-manager
428       process and returns a "MCE::Shared::Object" containing the "SHARED_ID".
429       Starting with the 1.827 release, the "module" option sends parameters
430       to the shared-manager, where the object is then constructed. This is
431       useful for classes involving XS code or a file handle.
432
433        use MCE::Shared;
434
435        {
436          use Math::BigFloat try => 'GMP';
437          use Math::BigInt   try => 'GMP';
438
439          my $bf  = MCE::Shared->share({ module => 'Math::BigFloat' }, 0);
440          my $bi  = MCE::Shared->share({ module => 'Math::BigInt'   }, 0);
441          my $y   = 1e9;
442
443          $bf->badd($y);  # addition (add $y to shared BigFloat object)
444          $bi->badd($y);  # addition (add $y to shared BigInt object)
445        }
446
447        {
448          use Bio::Seq;
449          use Bio::SeqIO;
450
451          my $seq_io = MCE::Shared->share({ module => 'Bio::SeqIO' },
452             -file    => ">/path/to/fasta/file.fa",
453             -format  => 'Fasta',
454             -verbose => -1,
455          );
456
457          my $seq_obj = Bio::Seq->new(
458             -display_id => "name", -desc => "desc", -seq => "seq",
459             -alphabet   => "dna"
460          );
461
462          $seq_io->write_seq($seq_obj);  # write to shared SeqIO handle
463        }
464
465        {
466          my $oh1 = MCE::Shared->share({ module => 'MCE::Shared::Ordhash' });
467          my $oh2 = MCE::Shared->ordhash();  # same thing
468
469          $oh1->assign( @pairs );
470          $oh2->assign( @pairs );
471        }
472
473        {
474          my ($ho_shared, $ho_nonshared);
475
476          $ho_shared = MCE::Shared->share({ module => 'Hash::Ordered' });
477          $ho_shared->push( @pairs );
478
479          $ho_nonshared = $ho_shared->export();   # back to non-shared
480          $ho_nonshared = $ho_shared->destroy();  # including shared destruction
481        }
482
483       The following provides long and short forms for constructing a shared
484       array, hash, or scalar object.
485
486        use MCE::Shared;
487
488        my $a1 = MCE::Shared->share( { module => 'MCE::Shared::Array' }, @list );
489        my $a2 = MCE::Shared->share( [ @list ] );
490        my $a3 = MCE::Shared->array( @list );
491
492        my $h1 = MCE::Shared->share( { module => 'MCE::Shared::Hash' }, @pairs );
493        my $h2 = MCE::Shared->share( { @pairs } );
494        my $h3 = MCE::Shared->hash( @pairs );
495
496        my $s1 = MCE::Shared->share( { module => 'MCE::Shared::Scalar' }, 20 );
497        my $s2 = MCE::Shared->share( \do{ my $o = 20 } );
498        my $s3 = MCE::Shared->scalar( 20 );
499
500       When the "module" option is given, one may optionally specify the
501       constructor function via the "new" option. This is necessary for the
502       CDB_File module, which provides two different objects. One is created
503       by new (default), and accessed by insert and finish. The other is
504       created by TIEHASH, and accessed by FETCH.
505
506        use MCE::Hobo;
507        use MCE::Shared;
508
509        # populate CDB file
510        my $cdb = MCE::Shared->share({ module => 'CDB_File' }, 't.cdb', "t.cdb.$$")
511           or die "$!\n";
512
513        $cdb->insert( $_ => $_ ) for ('aa'..'zz');
514        $cdb->finish;
515
516        # use CDB file
517        my $cdb1 = tie my %hash, 'MCE::Shared', { module => 'CDB_File' }, 't.cdb';
518
519        # same thing, without involving TIE and extra hash variable
520        my $cdb2 = MCE::Shared->share(
521           { module => 'CDB_File', new => 'TIEHASH' }, 't.cdb'
522        );
523
524        print $hash{'aa'}, "\n";
525        print $cdb1->FETCH('bb'), "\n";
526        print $cdb2->FETCH('cc'), "\n";
527
528        # rewind may be omitted on first use for parallel iteration
529        $cdb2->rewind;
530
531        for ( 1 .. 3 ) {
532           mce_async {
533              while ( my ($k,$v) = $cdb2->next ) {
534                 print "[$$] $k => $v\n";
535              }
536           };
537        }
538
539        MCE::Hobo->waitall;
540

DBM SHARING

542       Construting a shared DBM object is possible starting with the 1.827
543       release.  Supported modules are AnyDBM_File, BerkeleyDB, CDB_File,
544       DB_File, GDBM_File, NDBM_File, ODBM_File, SDBM_File, SQLite_File,
545       Tie::Array::DBD, and Tie::Hash::DBD. The list includes Tokyo Cabinet
546       <http://fallabs.com/tokyocabinet/> and Kyoto Cabinet
547       <http://fallabs.com/kyotocabinet/>. Also, see forked version by Altice
548       Labs <https://github.com/alticelabs/kyoto>. It contains an updated
549       "kyotocabinet" folder that builds successfully with recent compilers.
550
551       Freeze-thaw during "STORE"-"FETCH" (for complex data) is handled
552       automatically using Serial 3.015+ (if available) or Storable. Below,
553       are constructions for sharing various DBM modules. The construction for
554       "CDB_File" is given in the prior section.
555
556       AnyDBM_File
557
558        BEGIN { @AnyDBM_File::ISA = qw( DB_File GDBM_File NDBM_File ODBM_File ); }
559
560        use MCE::Shared;
561        use Fcntl;
562        use AnyDBM_File;
563
564        tie my %h1, 'MCE::Shared', { module => 'AnyDBM_File' },
565           'foo_a', O_CREAT|O_RDWR or die "open error: $!";
566
567       BerkeleyDB
568
569        use MCE::Shared;
570        use BerkeleyDB;
571
572        tie my %h1, 'MCE::Shared', { module => 'BerkeleyDB::Hash' },
573           -Filename => 'foo_a', -Flags => DB_CREATE
574              or die "open error: $!";
575
576        tie my %h2, 'MCE::Shared', { module => 'BerkeleyDB::Btree' },
577           -Filename => 'foo_b', -Flags => DB_CREATE
578              or die "open error: $!";
579
580        tie my @a1, 'MCE::Shared', { module => 'BerkeleyDB::Queue' },
581           -Filename => 'foo_c', -Flags => DB_CREATE
582              or die "open error: $!";
583
584        tie my @a2, 'MCE::Shared', { module => 'BerkeleyDB::Recno' },
585           -Filename => 'foo_d', -Flags => DB_CREATE -Len => 20
586              or die "open error: $!";
587
588       DB_File
589
590        use MCE::Shared;
591        use Fcntl;
592        use DB_File;
593
594        # Use pre-defined references ( $DB_HASH, $DB_BTREE, $DB_RECNO ).
595
596        tie my %h1, 'MCE::Shared', { module => 'DB_File' },
597           'foo_a', O_CREAT|O_RDWR, 0640, $DB_HASH or die "open error: $!";
598
599        tie my %h2, 'MCE::Shared', { module => 'DB_File' },
600           'foo_b', O_CREAT|O_RDWR, 0640, $DB_BTREE or die "open error: $!";
601
602        tie my @a1, 'MCE::Shared', { module => 'DB_File' },
603           'foo_c', O_CREAT|O_RDWR, 0640, $DB_RECNO or die "open error: $!";
604
605        # Changing defaults - see DB_File for valid options.
606
607        my $opt_h = DB_File::HASHINFO->new();
608        my $opt_b = DB_File::BTREEINFO->new();
609        my $opt_r = DB_File::RECNOINFO->new();
610
611        $opt_h->{'cachesize'} = 12345;
612
613        tie my %h3, 'MCE::Shared', { module => 'DB_File' },
614           'foo_d', O_CREAT|O_RDWR, 0640, $opt_h or die "open error: $!";
615
616       KyotoCabinet
617       TokyoCabinet
618
619        use MCE::Shared;
620        use KyotoCabinet;
621        use TokyoCabinet;
622
623        # file extension denotes hash database
624
625        tie my %h1, 'MCE::Shared', { module => 'KyotoCabinet::DB' }, 'foo.kch',
626           KyotoCabinet::DB::OWRITER | KyotoCabinet::DB::OCREATE
627              or die "open error: $!";
628
629        tie my %h2, 'MCE::Shared', { module => 'TokyoCabinet::HDB' }, 'foo.tch',
630           TokyoCabinet::HDB::OWRITER | TokyoCabinet::HDB::OCREAT
631              or die "open error: $!";
632
633        # file extension denotes tree database
634
635        tie my %h3, 'MCE::Shared', { module => 'KyotoCabinet::DB' }, 'foo.kct',
636           KyotoCabinet::DB::OWRITER | KyotoCabinet::DB::OCREATE
637              or die "open error: $!";
638
639        tie my %h4, 'MCE::Shared', { module => 'TokyoCabinet::BDB' }, 'foo.tcb',
640           TokyoCabinet::BDB::OWRITER | TokyoCabinet::BDB::OCREAT
641              or die "open error: $!";
642
643        # on-memory hash database
644
645        tie my %h5, 'MCE::Shared', { module => 'KyotoCabinet::DB' }, '*';
646        tie my %h6, 'MCE::Shared', { module => 'TokyoCabinet::ADB' }, '*';
647
648        # on-memory tree database
649
650        tie my %h7, 'MCE::Shared', { module => 'KyotoCabinet::DB' }, '%#pccap=256m';
651        tie my %h8, 'MCE::Shared', { module => 'TokyoCabinet::ADB' }, '+';
652
653       Tie::Array::DBD
654       Tie::Hash::DBD
655
656        use MCE::Shared;
657        use Tie::Array::DBD;
658        use Tie::Hash::DBD;
659
660        # A valid string is required for the DSN argument, not a DBI handle.
661        # Do not specify the 'str' option for Tie::(Array|Hash)::DBD.
662        # Instead, see encoder-decoder methods described under Common API.
663
664        use DBD::SQLite;
665
666        tie my @a1, 'MCE::Shared', { module => 'Tie::Array::DBD' },
667           'dbi:SQLite:dbname=foo_a.db', {
668              tbl => 't_tie_analysis',
669              key => 'h_key',
670              fld => 'h_value'
671           };
672
673        tie my %h1, 'MCE::Shared', { module => 'Tie::Hash::DBD' },
674           'dbi:SQLite:dbname=foo_h.db', {
675              tbl => 't_tie_analysis',
676              key => 'h_key',
677              fld => 'h_value'
678           };
679
680        use DBD::CSV;
681
682        tie my %h2, 'MCE::Shared', { module => 'Tie::Hash::DBD'},
683           'dbi:CSV:f_dir=.;f_ext=.csv/r;csv_null=1;csv_decode_utf8=0', {
684              tbl => 'mytable',
685              key => 'h_key',
686              fld => 'h_value'
687           };
688
689        # By default, Sereal 3.015+ is used for serialization if available.
690        # This overrides serialization from Sereal-or-Storable to JSON::XS.
691
692        use JSON::XS ();
693
694        tied(%h2)->encoder( \&JSON::XS::encode_json );
695        tied(%h2)->decoder( \&JSON::XS::decode_json );
696
697        my @pairs = ( key1 => 'val1', key2 => 'val2' );
698        my @list  = ( 1, 2, 3, 4 );
699
700        $h2{'foo'} = 'plain value';
701        $h2{'bar'} = { @pairs };
702        $h2{'baz'} = [ @list ];
703

DBM SHARING (CONT)

705       DB cursors, filters, and duplicate keys are not supported, just plain
706       array and hash functionality. The OO interface provides better
707       performance when needed.  Use "iterator" or "next" for iterating over
708       the elements.
709
710        use MCE::Hobo;
711        use MCE::Shared;
712        use Fcntl;
713        use DB_File;
714
715        unlink 'foo_a';
716
717        my $ob = tie my %h1, 'MCE::Shared', { module => 'DB_File' },
718           'foo_a', O_CREAT|O_RDWR, 0640, $DB_HASH or die "open error: $!";
719
720        $h1{key} = 'value';
721        my $val = $h1{key};
722
723        while ( my ($k, $v) = each %h1 ) {
724           print "1: $k => $v\n";
725        }
726
727        # object oriented fashion, faster
728
729        tied(%h1)->STORE( key1 => 'value1' );
730        my $val1 = tied(%h1)->FETCH('key1');
731
732        $ob->STORE( key2 => 'value2' );
733        my $val2 = $ob->FETCH('key2');
734
735        # non-parallel iteration
736
737        my $iter = $ob->iterator;
738        while ( my ($k, $v) = $iter->() ) {
739           print "2: $k => $v\n";
740        }
741
742        # parallel iteration
743
744        sub task {
745           while ( my ($k, $v) = $ob->next ) {
746              print "[$$] $k => $v\n";
747              sleep 1;
748           }
749        }
750
751        MCE::Hobo->create(\&task) for 1 .. 3;
752        MCE::Hobo->waitall;
753
754        $ob->rewind;
755
756        # undef $ob and $iter before %h1 when destroying manually
757
758        undef $ob;
759        undef $iter;
760
761        untie %h1;
762
763       See also Tie::File Demonstration, at the end of the documentation.
764

PDL SHARING

766       MCE::Shared->pdl_sbyte
767       MCE::Shared->pdl_byte
768       MCE::Shared->pdl_short
769       MCE::Shared->pdl_ushort
770       MCE::Shared->pdl_long
771       MCE::Shared->pdl_ulong
772       MCE::Shared->pdl_indx
773       MCE::Shared->pdl_longlong
774       MCE::Shared->pdl_ulonglong
775       MCE::Shared->pdl_float
776       MCE::Shared->pdl_double
777       MCE::Shared->pdl_ldouble
778       MCE::Shared->pdl_sequence
779       MCE::Shared->pdl_zeroes
780       MCE::Shared->pdl_zeros
781       MCE::Shared->pdl_ones
782       MCE::Shared->pdl_random
783       MCE::Shared->pdl_grandom
784       MCE::Shared->pdl
785
786       Sugar syntax for PDL construction to take place under the shared-
787       manager process. The helper routines are made available only if "PDL"
788       is loaded before "MCE::Shared".
789
790        use PDL;
791        use MCE::Shared;
792
793        # This makes an extra copy, transfer, including destruction.
794        my $ob1 = MCE::Shared->share( zeroes( 256, 256 ) );
795
796        # Do this instead to not involve an extra copy.
797        my $ob1 = MCE::Shared->pdl_zeroes( 256, 256 );
798
799       Below is a parallel version for a demonstration on PerlMonks.
800
801        # https://www.perlmonks.org/?node_id=1214227 (by vr)
802
803        use strict;
804        use warnings;
805        use feature 'say';
806
807        use PDL;  # must load PDL before MCE::Shared
808
809        use MCE;
810        use MCE::Shared;
811        use Time::HiRes 'time';
812
813        srand( 123 );
814
815        my $time = time;
816
817        my $n = 30000;      # input sample size
818        my $m = 10000;      # number of bootstrap repeats
819        my $r = $n;         # re-sample size
820
821        # On Windows, the non-shared piddle ($x) is unblessed in threads.
822        # Therefore, constructing the piddle inside the worker.
823        # UNIX platforms benefit from copy-on-write. Thus, one copy.
824
825        my $x   = ( $^O eq 'MSWin32' ) ? undef : random( $n );
826        my $avg = MCE::Shared->pdl_zeroes( $m );
827
828        MCE->new(
829           max_workers => 4,
830           sequence    => [ 0, $m - 1 ],
831           chunk_size  => 1,
832           user_begin  => sub {
833              $x = random( $n ) unless ( defined $x );
834           },
835           user_func   => sub {
836              my $idx  = random $r;
837              $idx    *= $n;
838              # $avg is a shared piddle which resides inside the shared-
839              # manager process or thread. The piddle is accessible via the
840              # OO interface only.
841              $avg->set( $_, $x->index( $idx )->avg );
842           }
843        )->run;
844
845        # MCE sets the seed of the base generator uniquely between workers.
846        # Unfortunately, it requires running with one worker for predictable
847        # results (i.e. no guarantee in the order which worker computes the
848        # next input chunk).
849
850        say $avg->pctover( pdl 0.05, 0.95 );
851        say time - $time, ' seconds';
852
853        __END__
854
855        # Output
856
857        [0.49387106  0.4993768]
858        1.09556317329407 seconds
859
860       ins_inplace
861
862       The "ins_inplace" method applies to shared PDL objects. It supports
863       three forms for writing elements back to the PDL object, residing under
864       the shared-manager process.
865
866        # --- action taken by the shared-manager process
867        # ins_inplace(  1 arg  ):  ins( inplace( $this ), $what, 0, 0 );
868        # ins_inplace(  2 args ):  $this->slice( $arg1 ) .= $arg2;
869        # ins_inplace( >2 args ):  ins( inplace( $this ), $what, @coords );
870
871        # --- use case
872        $o->ins_inplace( $result );                    #  1 arg
873        $o->ins_inplace( ":,$start:$stop", $result );  #  2 args
874        $o->ins_inplace( $result, 0, $seq_n );         # >2 args
875
876       Operations such as " + 5 " will not work on shared PDL objects. At this
877       time, the OO interface is the only mechanism for communicating with the
878       shared piddle. For example, call "slice", "sever", or "copy" to fetch
879       elements. Call "ins_inplace" or "set" (shown above) to update elements.
880
881        use strict;
882        use warnings;
883
884        use PDL;  # must load PDL before MCE::Shared
885        use MCE::Shared;
886
887        # make a shared piddle
888        my $b = MCE::Shared->pdl_sequence(15,15);
889
890        # fetch, add 10 to row 2 only
891        my $res1 = $b->slice(":,1:1") + 10;
892        $b->ins_inplace($res1, 0, 1);
893
894        # fetch, add 10 to rows 4 and 5
895        my $res2 = $b->slice(":,3:4") + 10;
896        $b->ins_inplace($res2, 0, 3);
897
898        # make non-shared object (i.e. export-destroy from shared)
899        $b = $b->destroy;
900
901        print "$b\n";
902
903       The following provides parallel demonstrations using "MCE::Flow".
904
905        use strict;
906        use warnings;
907
908        use PDL;  # must load PDL before MCE::Shared
909
910        use MCE::Flow;
911        use MCE::Shared;
912
913        # On Windows, the ($a) piddle is unblessed in worker threads.
914        # Therefore, constructing ($a) inside the worker versus sharing.
915        # UNIX platforms benefit from copy-on-write. Thus, one copy.
916        #
917        # Results are stored in the shared piddle ($b).
918
919        my $a = ( $^O eq 'MSWin32' ) ? undef : sequence(15,15);
920        my $b = MCE::Shared->pdl_zeroes(15,15);
921
922        MCE::Flow->init(
923           user_begin => sub {
924              $a = sequence(15,15) unless ( defined $a );
925           }
926        );
927
928        # with chunking disabled
929
930        mce_flow_s {
931           max_workers => 4, chunk_size => 1
932        },
933        sub {
934           my $row = $_;
935           my $result = $a->slice(":,$row:$row") + 5;
936           $b->ins_inplace($result, 0, $row);
937        }, 0, 15 - 1;
938
939        # with chunking enabled
940
941        mce_flow_s {
942           max_workers => 4, chunk_size => 5, bounds_only => 1
943        },
944        sub {
945           my ($row1, $row2) = @{ $_ };
946           my $result = $a->slice(":,$row1:$row2") + 5;
947           $b->ins_inplace($result, 0, $row1);
948        }, 0, 15 - 1;
949
950        # make non-shared object, export-destroy the shared object
951
952        $b = $b->destroy;
953
954        print "$b\n";
955
956       See also, PDL::ParallelCPU and PDL::Parallel::threads.
957

COMMON API

959       blessed
960          Returns the real "blessed" name, provided by the shared-manager
961          process.
962
963           use MCE::Shared;
964           use Scalar::Util qw(blessed);
965
966           my $oh1 = MCE::Shared->share({ module => 'MCE::Shared::Ordhash' });
967           my $oh2 = MCE::Shared->share({ module => 'Hash::Ordered'        });
968
969           print blessed($oh1), "\n";    # MCE::Shared::Object
970           print blessed($oh2), "\n";    # MCE::Shared::Object
971
972           print $oh1->blessed(), "\n";  # MCE::Shared::Ordhash
973           print $oh2->blessed(), "\n";  # Hash::Ordered
974
975       destroy ( { unbless => 1 } )
976       destroy
977          Exports optionally, but destroys the shared object entirely from the
978          shared-manager process. The unbless option is passed to export.
979
980           my $exported_ob = $shared_ob->destroy();
981
982           $shared_ob;     # becomes undef
983
984       lock
985       unlock
986          Shared objects embed a MCE::Mutex object for locking since 1.841.
987
988           use MCE::Shared;
989
990           tie my @shared_array, 'MCE::Shared', { module => 'MCE::Shared::Array' }, 0;
991
992           tied(@shared_array)->lock;
993           $shared_array[0] += 1;
994           tied(@shared_array)->unlock;
995
996           print $shared_array[0], "\n";    # 1
997
998          Locking is not necessary typically when using the OO interface.
999          Although, exclusive access is necessary when involving a FETCH and
1000          STORE.
1001
1002           my $shared_total = MCE::Shared->scalar(2);
1003
1004           $shared_total->lock;
1005           my $val = $shared_total->get;
1006           $shared_total->set( $val * 2 );
1007           $shared_total->unlock;
1008
1009           print $shared_total->get, "\n";  # 4
1010
1011       encoder ( CODE )
1012       decoder ( CODE )
1013          Override freeze/thaw routines. Applies to STORE and FETCH only,
1014          particularly for TIE'd objects. These are called internally for
1015          shared DB objects.
1016
1017          Current API available since 1.827.
1018
1019           use MCE::Shared;
1020           use BerkeleyDB;
1021           use DB_File;
1022
1023           my $file1 = 'file1.db';
1024           my $file2 = 'file2.db';
1025
1026           tie my @db1, 'MCE::Shared', { module => 'DB_File' }, $file1,
1027              O_RDWR|O_CREAT, 0640 or die "open error '$file1': $!";
1028
1029           tie my %db2, 'MCE::Shared', { module => 'BerkeleyDB::Hash' },
1030              -Filename => $file2, -Flags => DB_CREATE
1031              or die "open error '$file2': $!";
1032
1033           # Called automatically by MCE::Shared for DB files.
1034           # tied(@db1)->encoder( MCE::Shared::Server::_get_freeze );
1035           # tied(@db1)->decoder( MCE::Shared::Server::_get_thaw );
1036           # tied(%db2)->encoder( MCE::Shared::Server::_get_freeze );
1037           # tied(%db2)->decoder( MCE::Shared::Server::_get_thaw );
1038           # et cetera.
1039
1040           $db1[0] = 'foo';   # store plain and complex structure
1041           $db1[1] = { key => 'value' };
1042           $db1[2] = [ 'complex' ];
1043
1044           $db2{key} = 'foo'; # ditto, plain and complex structure
1045           $db2{sun} = [ 'complex' ];
1046
1047       export ( { unbless => 1 }, keys )
1048       export
1049          Exports the shared object as a non-shared object. One must export
1050          the shared object when passing into any dump routine. Otherwise, the
1051          "shared_id value" and "blessed name" is all one will see. The
1052          unbless option unblesses any shared Array, Hash, and Scalar object
1053          to a non-blessed array, hash, and scalar respectively.
1054
1055           use MCE::Shared;
1056           use MCE::Shared::Ordhash;
1057
1058           sub _dump {
1059              require Data::Dumper unless $INC{'Data/Dumper.pm'};
1060              no warnings 'once';
1061
1062              local $Data::Dumper::Varname  = 'VAR';
1063              local $Data::Dumper::Deepcopy = 1;
1064              local $Data::Dumper::Indent   = 1;
1065              local $Data::Dumper::Purity   = 1;
1066              local $Data::Dumper::Sortkeys = 0;
1067              local $Data::Dumper::Terse    = 0;
1068
1069              print Data::Dumper::Dumper($_[0]) . "\n";
1070           }
1071
1072           my $oh1 = MCE::Shared->share({ module => 'MCE::Shared::Ordhash' });
1073           my $oh2 = MCE::Shared->ordhash();  # same thing
1074
1075           _dump($oh1);
1076              # bless( [ 1, 'MCE::Shared::Ordhash' ], 'MCE::Shared::Object' )
1077
1078           _dump($oh2);
1079              # bless( [ 2, 'MCE::Shared::Ordhash' ], 'MCE::Shared::Object' )
1080
1081           _dump( $oh1->export );  # dumps object structure and content
1082           _dump( $oh2->export );  # ditto
1083
1084          "export" can optionally take a list of indices/keys for what to
1085          export.  This applies to shared array, hash, and ordhash.
1086
1087           use MCE::Shared;
1088
1089           # shared hash
1090           my $h1 = MCE::Shared->hash(
1091              qw/ I Heard The Bluebirds Sing by Marty Robbins /
1092                # k v     k   v         k    v  k     v
1093           );
1094
1095           # non-shared hash
1096           my $h2 = $h1->export( qw/ I The / );
1097
1098           _dump($h2);
1099
1100           __END__
1101
1102           # Output
1103
1104           $VAR1 = bless( {
1105             'I' => 'Heard',
1106             'The' => 'Bluebirds'
1107           }, 'MCE::Shared::Hash' );
1108
1109          Specifying the unbless option exports a non-blessed data structure
1110          instead.  The unbless option applies to shared MCE::Shared::{ Array,
1111          Hash, and Scalar } objects.
1112
1113           my $h2 = $h1->export( { unbless => 1 }, qw/ I The / );
1114           my $h3 = $h1->export( { unbless => 1 } );
1115
1116           _dump($h2);
1117           _dump($h3);
1118
1119           __END__
1120
1121           # Output
1122
1123           $VAR1 = {
1124             'The' => 'Bluebirds',
1125             'I' => 'Heard'
1126           };
1127
1128           $VAR1 = {
1129             'Marty' => 'Robbins',
1130             'Sing' => 'by',
1131             'The' => 'Bluebirds',
1132             'I' => 'Heard'
1133           };
1134
1135       next
1136          The "next" method provides parallel iteration between workers for
1137          shared "array", "hash", "ordhash", and "sequence". In list context,
1138          returns the next key-value pair or beg-end pair for sequence. In
1139          scalar context, returns the next item. The "undef" value is returned
1140          after the iteration has completed.
1141
1142          Internally, the list of keys to return is set when the closure is
1143          constructed.  Later keys added to the shared array or hash are not
1144          included. Subsequently, the "undef" value is returned for deleted
1145          keys.
1146
1147          The following example iterates through a shared array in parallel.
1148
1149           use MCE::Hobo;
1150           use MCE::Shared;
1151
1152           my $ar = MCE::Shared->array( 'a' .. 'j' );
1153
1154           sub demo1 {
1155              my ( $wid ) = @_;
1156              while ( my ( $index, $value ) = $ar->next ) {
1157                 print "$wid: [ $index ] $value\n";
1158                 sleep 1;
1159              }
1160           }
1161
1162           sub demo2 {
1163              my ( $wid ) = @_;
1164              while ( defined ( my $value = $ar->next ) ) {
1165                 print "$wid: $value\n";
1166                 sleep 1;
1167              }
1168           }
1169
1170           $ar->rewind();
1171
1172           MCE::Hobo->new( \&demo1, $_ ) for 1 .. 3;
1173           MCE::Hobo->waitall(), print "\n";
1174
1175           $ar->rewind();
1176
1177           MCE::Hobo->new( \&demo2, $_ ) for 1 .. 3;
1178           MCE::Hobo->waitall(), print "\n";
1179
1180           __END__
1181
1182           # Output
1183
1184           1: [ 0 ] a
1185           2: [ 1 ] b
1186           3: [ 2 ] c
1187           1: [ 3 ] d
1188           2: [ 5 ] f
1189           3: [ 4 ] e
1190           2: [ 8 ] i
1191           3: [ 6 ] g
1192           1: [ 7 ] h
1193           2: [ 9 ] j
1194
1195           1: a
1196           2: b
1197           3: c
1198           2: e
1199           3: f
1200           1: d
1201           3: g
1202           1: i
1203           2: h
1204           1: j
1205
1206          The form is similar for "sequence". For large sequences, the
1207          "bounds_only" option is recommended. Also, specify "chunk_size"
1208          accordingly. This reduces the amount of traffic to and from the
1209          shared-manager process.
1210
1211           use MCE::Hobo;
1212           use MCE::Shared;
1213
1214           my $N   = shift || 4_000_000;
1215           my $pi  = MCE::Shared->scalar( 0.0 );
1216
1217           my $seq = MCE::Shared->sequence(
1218              { chunk_size => 200_000, bounds_only => 1 }, 0, $N - 1
1219           );
1220
1221           sub compute_pi {
1222              my ( $wid ) = @_;
1223
1224              # Optionally, also receive the chunk_id value
1225              # while ( my ( $beg, $end, $chunk_id ) = $seq->next ) { ... }
1226
1227              while ( my ( $beg, $end ) = $seq->next ) {
1228                 my ( $_pi, $t ) = ( 0.0 );
1229                 for my $i ( $beg .. $end ) {
1230                    $t = ( $i + 0.5 ) / $N;
1231                    $_pi += 4.0 / ( 1.0 + $t * $t );
1232                 }
1233                 $pi->incrby( $_pi );
1234              }
1235
1236              return;
1237           }
1238
1239           MCE::Hobo->create( \&compute_pi, $_ ) for ( 1 .. 8 );
1240
1241           # ... do other stuff ...
1242
1243           MCE::Hobo->waitall();
1244
1245           printf "pi = %0.13f\n", $pi->get / $N;
1246
1247           __END__
1248
1249           # Output
1250
1251           3.1415926535898
1252
1253       rewind ( index, [, index, ... ] )
1254       rewind ( key, [, key, ... ] )
1255       rewind ( "query string" )
1256       rewind ( )
1257          Rewinds the parallel iterator for MCE::Shared::Array,
1258          MCE::Shared::Hash, or MCE::Shared::Ordhash when no arguments are
1259          given. Otherwise, resets the iterator with given criteria. The
1260          syntax for "query string" is described in the shared module.
1261
1262           # array
1263           $ar->rewind;
1264
1265           $ar->rewind( 0, 1 );
1266           $ar->rewind( "val eq some_value" );
1267           $ar->rewind( "key >= 50 :AND val =~ /sun|moon|air|wind/" );
1268           $ar->rewind( "val eq sun :OR val eq moon :OR val eq foo" );
1269           $ar->rewind( "key =~ /$pattern/" );
1270
1271           while ( my ( $index, $value ) = $ar->next ) {
1272              ...
1273           }
1274
1275           # hash, ordhash
1276           $oh->rewind;
1277
1278           $oh->rewind( "key1", "key2" );
1279           $oh->rewind( "val eq some_value" );
1280           $oh->rewind( "key eq some_key :AND val =~ /sun|moon|air|wind/" );
1281           $oh->rewind( "val eq sun :OR val eq moon :OR val eq foo" );
1282           $oh->rewind( "key =~ /$pattern/" );
1283
1284           while ( my ( $key, $value ) = $oh->next ) {
1285              ...
1286           }
1287
1288       rewind ( { options }, begin, end [, step, format ] )
1289       rewind ( begin, end [, step, format ] )
1290       rewind ( )
1291          Rewinds the parallel iterator for MCE::Shared::Sequence when no
1292          arguments are given. Otherwise, resets the iterator with given
1293          criteria.
1294
1295           # sequence
1296           $seq->rewind;
1297
1298           $seq->rewind( { chunk_size => 10, bounds_only => 1 }, 1, 100 );
1299
1300           while ( my ( $beg, $end ) = $seq->next ) {
1301              for my $i ( $beg .. $end ) {
1302                 ...
1303              }
1304           }
1305
1306           $seq->rewind( 1, 100 );
1307
1308           while ( defined ( my $num = $seq->next ) ) {
1309              ...
1310           }
1311
1312       store ( key, value )
1313          Deeply sharing a non-blessed structure recursively is possible with
1314          "store", an alias to "STORE".
1315
1316           use MCE::Shared;
1317
1318           my $h1 = MCE::Shared->hash();
1319           my $h2 = MCE::Shared->hash();
1320
1321           # auto-shares deeply
1322           $h1->store('key', [ 0, 2, 5, { 'foo' => 'bar' } ]);
1323           $h2->{key}[3]{foo} = 'baz';    # via auto-vivification
1324
1325           my $v1 = $h1->get('key')->get(3)->get('foo');  # bar
1326           my $v2 = $h2->get('key')->get(3)->get('foo');  # baz
1327           my $v3 = $h2->{key}[3]{foo};                   # baz
1328

SERVER API

1330       init
1331          This method is called by each MCE and Hobo worker automatically
1332          after spawning.  The effect is extra parallelism and decreased
1333          latency during inter-process communication to the shared-manager
1334          process. The optional ID (an integer) is modded internally in a
1335          round-robin fashion.
1336
1337           MCE::Shared->init();
1338           MCE::Shared->init( ID );
1339
1340       pid
1341          Returns the process ID of the shared-manager process. This class
1342          method was added in 1.849 for stopping all workers immediately when
1343          exiting a Graphics::Framebuffer application. It returns an undefined
1344          value if the shared-manager is not running. Not useful otherwise if
1345          running threads (i.e. same PID).
1346
1347           MCE::Shared->pid();
1348
1349           $SIG{INT} = $SIG{HUP} = $SIG{TERM} = sub {
1350              # Signal workers and the shared manager all at once
1351              CORE::kill('KILL', MCE::Hobo->list_pids(), MCE::Shared->pid());
1352              exec('reset');
1353           };
1354
1355       start
1356          Starts the shared-manager process. This is done automatically unless
1357          Perl lacks IO::FDPass, needed to share "condvar" and "queue" while
1358          the shared-manager is running.
1359
1360           MCE::Shared->start();
1361
1362       stop
1363          Stops the shared-manager process, wiping all shared data content.
1364          This is called by the "END" block automatically when the script
1365          terminates. However, do stop explicitly to reap the shared-manager
1366          process before exec'ing.
1367
1368           MCE::Shared->stop();
1369
1370           exec('command');
1371

LOCKING

1373       Application-level advisory locking is possible with MCE::Mutex.
1374
1375        use MCE::Hobo;
1376        use MCE::Mutex;
1377        use MCE::Shared;
1378
1379        my $mutex = MCE::Mutex->new();
1380
1381        tie my $cntr, 'MCE::Shared', 0;
1382
1383        sub work {
1384           for ( 1 .. 1000 ) {
1385              $mutex->lock;
1386
1387              # Incrementing involves 2 IPC ops ( FETCH and STORE ).
1388              # Thus, locking is required.
1389              $cntr++;
1390
1391              $mutex->unlock;
1392           }
1393        }
1394
1395        MCE::Hobo->create('work') for ( 1 .. 8 );
1396        MCE::Hobo->waitall;
1397
1398        print $cntr, "\n"; # 8000
1399
1400       Locking is available for shared objects via "lock" and "unlock" methods
1401       since 1.841. Previously, for "condvar" only.
1402
1403        use MCE::Hobo;
1404        use MCE::Shared;
1405
1406        tie my $cntr, 'MCE::Shared', 0;
1407
1408        sub work {
1409           for ( 1 .. 1000 ) {
1410              tied($cntr)->lock;
1411
1412              # Incrementing involves 2 IPC ops ( FETCH and STORE ).
1413              # Thus, locking is required.
1414              $cntr++;
1415
1416              tied($cntr)->unlock;
1417           }
1418        }
1419
1420        MCE::Hobo->create('work') for ( 1 .. 8 );
1421        MCE::Hobo->waitall;
1422
1423        print $cntr, "\n"; # 8000
1424
1425       Typically, locking is not necessary using the OO interface. The reason
1426       is that MCE::Shared is implemented using a single-point of entry for
1427       commands sent to the shared-manager process. Furthermore, the shared
1428       classes include sugar methods for combining set and get in a single
1429       operation.
1430
1431        use MCE::Hobo;
1432        use MCE::Shared;
1433
1434        my $cntr = MCE::Shared->scalar( 0 );
1435
1436        sub work {
1437           for ( 1 .. 1000 ) {
1438              # The next statement increments the value without having
1439              # to call set and get explicitly.
1440              $cntr->incr;
1441           }
1442        }
1443
1444        MCE::Hobo->create('work') for ( 1 .. 8 );
1445        MCE::Hobo->waitall;
1446
1447        print $cntr->get, "\n"; # 8000
1448
1449       Another possibility when running threads is locking via
1450       threads::shared.
1451
1452        use threads;
1453        use threads::shared;
1454
1455        use MCE::Flow;
1456        use MCE::Shared;
1457
1458        my $mutex : shared;
1459
1460        tie my $cntr, 'MCE::Shared', 0;
1461
1462        sub work {
1463           for ( 1 .. 1000 ) {
1464              lock $mutex;
1465
1466              # the next statement involves 2 IPC ops ( get and set )
1467              # thus, locking is required
1468              $cntr++;
1469           }
1470        }
1471
1472        MCE::Flow->run( { max_workers => 8 }, \&work );
1473        MCE::Flow->finish;
1474
1475        print $cntr, "\n"; # 8000
1476
1477       Of the three demonstrations, the OO interface yields the best
1478       performance.  This is from the lack of locking at the application
1479       level. The results were obtained from a MacBook Pro (Haswell) running
1480       at 2.6 GHz, 1600 MHz RAM.
1481
1482        CentOS 7.2 VM
1483
1484           -- Perl v5.16.3
1485           MCE::Mutex .... : 0.528 secs.
1486           OO Interface .. : 0.062 secs.
1487           threads::shared : 0.545 secs.
1488
1489        FreeBSD 10.0 VM
1490
1491           -- Perl v5.16.3
1492           MCE::Mutex .... : 0.367 secs.
1493           OO Interface .. : 0.083 secs.
1494           threads::shared : 0.593 secs.
1495
1496        Mac OS X 10.11.6 ( Host OS )
1497
1498           -- Perl v5.18.2
1499           MCE::Mutex .... : 0.397 secs.
1500           OO Interface .. : 0.070 secs.
1501           threads::shared : 0.463 secs.
1502
1503        Solaris 11.2 VM
1504
1505           -- Perl v5.12.5 installed with the OS
1506           MCE::Mutex .... : 0.895 secs.
1507           OO Interface .. : 0.099 secs.
1508           threads::shared :              Perl not built to support threads
1509
1510           -- Perl v5.22.2 built with threads support
1511           MCE::Mutex .... : 0.788 secs.
1512           OO Interface .. : 0.086 secs.
1513           threads::shared : 0.895 secs.
1514
1515        Windows 7 VM
1516
1517           -- Perl v5.22.2
1518           MCE::Mutex .... : 1.045 secs.
1519           OO Interface .. : 0.312 secs.
1520           threads::shared : 1.061 secs.
1521
1522       Beginning with MCE::Shared 1.809, the "pipeline" method provides
1523       another way.  Included in "Array", "Cache", "Hash", "Minidb", and
1524       "Ordhash", it combines multiple commands for the object to be processed
1525       serially. For shared objects, the call is made atomically due to single
1526       IPC to the shared-manager process.
1527
1528       The "pipeline" method is fully "wantarray"-aware and receives a list of
1529       commands and their arguments. In scalar or list context, it returns
1530       data from the last command in the pipeline.
1531
1532        use MCE::Mutex;
1533        use MCE::Shared;
1534
1535        my $mutex = MCE::Mutex->new();
1536        my $oh = MCE::Shared->ordhash();
1537        my @vals;
1538
1539        # mutex locking
1540
1541        $mutex->lock;
1542        $oh->set( foo => "a_a" );
1543        $oh->set( bar => "b_b" );
1544        $oh->set( baz => "c_c" );
1545        @vals = $oh->mget( qw/ foo bar baz / );
1546        $mutex->unlock;
1547
1548        # pipeline, same thing done atomically
1549
1550        @vals = $oh->pipeline(
1551           [ "set", foo => "a_a" ],
1552           [ "set", bar => "b_b" ],
1553           [ "set", baz => "c_c" ],
1554           [ "mget", qw/ foo bar baz / ]
1555        );
1556
1557        # ( "a_a", "b_b", "c_c" )
1558
1559       There is also "pipeline_ex", same as "pipeline", but returns data for
1560       every command in the pipeline.
1561
1562        @vals = $oh->pipeline_ex(
1563           [ "set", foo => "a_a" ],
1564           [ "set", bar => "b_b" ],
1565           [ "set", baz => "c_c" ]
1566        );
1567
1568        # ( "a_a", "b_b", "c_c" )
1569

PYTHON DEMONSTRATION

1571       Sharing a Python class is possible, starting with the 1.827 release.
1572       The construction is simply calling share with the module option.
1573       Methods are accessible via the OO interface.
1574
1575        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1576        # Python class.
1577        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1578
1579        package My::Class;
1580
1581        use strict;
1582        use warnings;
1583
1584        use Inline::Python qw( py_eval py_bind_class );
1585
1586        py_eval ( <<'END_OF_PYTHON_CLASS' );
1587
1588        class MyClass:
1589            def __init__(self):
1590                self.data = [0,0]
1591
1592            def set (self, key, value):
1593                self.data[key] = value
1594
1595            def get (self, key):
1596                try: return self.data[key]
1597                except KeyError: return None
1598
1599            def incr (self, key):
1600                try: self.data[key] = self.data[key] + 1
1601                except KeyError: self.data[key] = 1
1602
1603        END_OF_PYTHON_CLASS
1604
1605        # Register methods for best performance.
1606
1607        py_bind_class(
1608            'My::Class', '__main__', 'MyClass',
1609            'set', 'get', 'incr'
1610        );
1611
1612        1;
1613
1614        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1615        # Share Python class. Requires MCE::Shared 1.827 or later.
1616        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1617
1618        use strict;
1619        use warnings;
1620
1621        use MCE::Hobo;
1622        use MCE::Shared;
1623
1624        my $py1 = MCE::Shared->share({ module => 'My::Class' });
1625        my $py2 = MCE::Shared->share({ module => 'My::Class' });
1626
1627        MCE::Shared->start;
1628
1629        $py1->set(0, 100);
1630        $py2->set(1, 200);
1631
1632        die "Ooops" unless $py1->get(0) eq '100';
1633        die "Ooops" unless $py2->get(1) eq '200';
1634
1635        sub task {
1636            $py1->incr(0) for 1..50000;
1637            $py2->incr(1) for 1..50000;
1638        }
1639
1640        MCE::Hobo->create(\&task) for 1..3;
1641        MCE::Hobo->waitall;
1642
1643        print $py1->get(0), "\n";  # 150100
1644        print $py2->get(1), "\n";  # 150200
1645

LOGGER DEMONSTRATION

1647       Often, the requirement may call for concurrent logging by many workers.
1648       Calling localtime or gmtime per each log entry is expensive. This uses
1649       the old time-stamp value until one second has elapsed.
1650
1651        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1652        # Logger class.
1653        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1654
1655        package My::Logger;
1656
1657        use strict;
1658        use warnings;
1659
1660        use Time::HiRes qw( time );
1661
1662        # construction
1663
1664        sub new {
1665            my ( $class, %self ) = @_;
1666
1667            open $self{fh}, ">>", $self{path} or return '';
1668            binmode $self{fh};
1669
1670            $self{stamp} = localtime;  # or gmtime
1671            $self{time } = time;
1672
1673            bless \%self, $class;
1674        }
1675
1676        # $ob->log("message");
1677
1678        sub log {
1679            my ( $self, $stamp ) = ( shift );
1680
1681            if ( time - $self->{time} > 1.0 ) {
1682                $self->{stamp} = $stamp = localtime;  # or gmtime
1683                $self->{time } = time;
1684            }
1685            else {
1686                $stamp = $self->{stamp};
1687            }
1688
1689            print {$self->{fh}} "$stamp --- @_\n";
1690        }
1691
1692        # $ob->autoflush(0);
1693        # $ob->autoflush(1);
1694
1695        sub autoflush {
1696            my ( $self, $flag ) = @_;
1697
1698            if ( defined fileno($self->{fh}) ) {
1699                 $flag ? select(( select($self->{fh}), $| = 1 )[0])
1700                       : select(( select($self->{fh}), $| = 0 )[0]);
1701
1702                 return 1;
1703            }
1704
1705            return;
1706        }
1707
1708        # $ob->binmode($layer);
1709        # $ob->binmode();
1710
1711        sub binmode {
1712            my ( $self, $layer ) = @_;
1713
1714            if ( defined fileno($self->{fh}) ) {
1715                CORE::binmode $self->{fh}, $layer // ':raw';
1716
1717                return 1;
1718            }
1719
1720            return;
1721        }
1722
1723        # $ob->close()
1724
1725        sub close {
1726            my ( $self ) = @_;
1727
1728            if ( defined fileno($self->{fh}) ) {
1729                close $self->{'fh'};
1730            }
1731
1732            return;
1733        }
1734
1735        # $ob->flush()
1736
1737        sub flush {
1738            my ( $self ) = @_;
1739
1740            if ( defined fileno($self->{fh}) ) {
1741                my $old_fh = select $self->{fh};
1742                my $old_af = $|; $| = 1; $| = $old_af;
1743                select $old_fh;
1744
1745                return 1;
1746            }
1747
1748            return;
1749        }
1750
1751        1;
1752
1753        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1754        # Concurrent logger demo. Requires MCE::Shared 1.827 or later.
1755        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1756
1757        use strict;
1758        use warnings;
1759
1760        use MCE::Hobo;
1761        use MCE::Shared;
1762
1763        my $file = "log.txt";
1764        my $pid  = $$;
1765
1766        my $ob = MCE::Shared->share( { module => 'My::Logger' }, path => $file )
1767            or die "open error '$file': $!";
1768
1769        # $ob->autoflush(1);   # optional, flush writes immediately
1770
1771        sub work {
1772            my $id = shift;
1773            for ( 1 .. 250_000 ) {
1774                $ob->log("Hello from $id: $_");
1775            }
1776        }
1777
1778        MCE::Hobo->create('work', $_) for 1 .. 4;
1779        MCE::Hobo->waitall;
1780
1781        # Threads and multi-process safety for closing the handle.
1782
1783        sub CLONE { $pid = 0; }
1784
1785        END { $ob->close if $ob && $pid == $$; }
1786

TIE::FILE DEMONSTRATION

1788       The following presents a concurrent Tie::File demonstration. Each
1789       element in the array corresponds to a record in the text file. JSON,
1790       being readable, seems appropiate for encoding complex objects.
1791
1792        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1793        # Class extending Tie::File with two sugar methods.
1794        # Requires MCE::Shared 1.827 or later.
1795        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1796
1797        package My::File;
1798
1799        use strict;
1800        use warnings;
1801
1802        use Tie::File;
1803
1804        our @ISA = 'Tie::File';
1805
1806        # $ob->append('string');
1807
1808        sub append {
1809            my ($self, $key) = @_;
1810            my $val = $self->FETCH($key); $val .= $_[2];
1811            $self->STORE($key, $val);
1812            length $val;
1813        }
1814
1815        # $ob->incr($key);
1816
1817        sub incr {
1818            my ( $self, $key ) = @_;
1819            my $val = $self->FETCH($key); $val += 1;
1820            $self->STORE($key, $val);
1821            $val;
1822        }
1823
1824        1;
1825
1826        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1827        # The MCE::Mutex module isn't needed unless IPC involves two or
1828        # more trips for the underlying action.
1829        #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1830
1831        use strict;
1832        use warnings;
1833
1834        use MCE::Hobo;
1835        use MCE::Mutex;
1836        use MCE::Shared;
1837
1838        use JSON::MaybeXS;
1839
1840        # Safety for data having line breaks.
1841        use constant EOL => "\x{0a}~\x{0a}";
1842
1843        my $file  = 'file.txt';
1844        my $mutex = MCE::Mutex->new();
1845        my $pid   = $$;
1846
1847        my $ob = tie my @db, 'MCE::Shared', { module => 'My::File' }, $file,
1848            recsep => EOL or die "open error '$file': $!";
1849
1850        $ob->encoder( \&JSON::MaybeXS::encode_json );
1851        $ob->decoder( \&JSON::MaybeXS::decode_json );
1852
1853        $db[20] = 0;  # a counter at offset 20 into the array
1854        $db[21] = [ qw/ foo bar / ];  # store complex structure
1855
1856        sub task {
1857            my $id  = sprintf "%02s", shift;
1858            my $row = int($id) - 1;
1859            my $chr = sprintf "%c", 97 + $id - 1;
1860
1861            # A mutex isn't necessary when storing a value.
1862            # Ditto for fetching a value.
1863
1864            $db[$row] = "Hello from $id: ";  # 1 trip
1865            my $val   = length $db[$row];    # 1 trip
1866
1867            # A mutex may be necessary for updates involving 2 or
1868            # more trips (FETCH and STORE) during IPC, from and to
1869            # the shared-manager process, unless a unique row.
1870
1871            for ( 1 .. 40 ) {
1872              # $db[$row] .= $id;         # 2 trips, unique row - okay
1873                $ob->append($row, $chr);  # 1 trip via the OO interface
1874
1875              # $mu->lock;
1876              # $db[20] += 1;             # incrementing counter, 2 trips
1877              # $mu->unlock;
1878
1879                $ob->incr(20);            # same thing via OO, 1 trip
1880            }
1881
1882            my $len = length $db[$row];   # 1 trip
1883
1884            printf "hobo %2d : %d\n", $id, $len;
1885        }
1886
1887        MCE::Hobo->create('task', $_) for 1 .. 20;
1888        MCE::Hobo->waitall;
1889
1890        printf "counter : %d\n", $db[20];
1891        print  $db[21]->[0], "\n";  # foo
1892
1893        # Threads and multi-process safety for closing the handle.
1894
1895        sub CLONE { $pid = 0; }
1896
1897        END {
1898            if ( $pid == $$ ) {
1899                undef $ob;  # important, undef $ob before @db
1900                untie @db;  # untie @db to flush pending writes
1901            }
1902        }
1903

REQUIREMENTS

1905       MCE::Shared requires Perl 5.10.1 or later. The IO::FDPass module is
1906       highly recommended on UNIX and Windows. This module does not install it
1907       by default.
1908

SOURCE AND FURTHER READING

1910       The source and examples are hosted at GitHub.
1911
1912       •  <https://github.com/marioroy/mce-shared>
1913
1914       •  <https://github.com/marioroy/mce-examples>
1915

INDEX

1917       MCE, MCE::Hobo
1918

AUTHOR

1920       Mario E. Roy, <marioeroy AT gmail DOT com>
1921

COPYRIGHT AND LICENSE

1923       Copyright (C) 2016-2022 by Mario E. Roy
1924
1925       MCE::Shared is released under the same license as Perl.
1926
1927       See <https://dev.perl.org/licenses/> for more information.
1928
1929
1930
1931perl v5.36.0                      2022-07-22                    MCE::Shared(3)