1MCE::Shared::Handle(3)User Contributed Perl DocumentationMCE::Shared::Handle(3)
2
3
4

NAME

6       MCE::Shared::Handle - Handle helper class
7

VERSION

9       This document describes MCE::Shared::Handle version 1.876
10

DESCRIPTION

12       A handle helper class for use as a standalone or managed by
13       MCE::Shared.
14

SYNOPSIS

16        # non-shared or local construction for use by a single process
17        # shorter, mce_open is an alias for MCE::Shared::Handle::open
18
19        use MCE::Shared::Handle;
20
21        MCE::Shared::Handle->open( my $fh, "<", "bio.fasta" )
22           or die "open error: $!";
23        MCE::Shared::Handle::open  my $fh, "<", "bio.fasta"
24           or die "open error: $!";
25
26        mce_open my $fh, "<", "bio.fasta" or die "open error: $!";
27
28        # construction for sharing with other threads and processes
29        # shorter, mce_open is an alias for MCE::Shared::open
30
31        use MCE::Shared;
32
33        MCE::Shared->open( my $fh, "<", "bio.fasta" )
34           or die "open error: $!";
35        MCE::Shared::open  my $fh, "<", "bio.fasta"
36           or die "open error: $!";
37
38        mce_open my $fh, "<", "bio.fasta" or die "open error: $!";
39
40        # example, output is serialized, not garbled
41
42        use MCE::Hobo;
43        use MCE::Shared;
44
45        mce_open my $ofh, ">>", \*STDOUT  or die "open error: $!";
46        mce_open my $ifh, "<", "file.log" or die "open error: $!";
47
48        sub parallel {
49           $/ = "\n"; # can set the input record separator
50           while (my $line = <$ifh>) {
51              printf {$ofh} "[%5d] %s", $., $line;
52           }
53        }
54
55        MCE::Hobo->create( \&parallel ) for 1 .. 4;
56
57        $_->join() for MCE::Hobo->list();
58
59        # handle functions
60
61        my $bool = eof($ifh);
62        my $off  = tell($ifh);
63        my $fd   = fileno($ifh);
64        my $char = getc($ifh);
65        my $line = readline($ifh);
66
67        binmode $ifh;
68        seek $ifh, 10, 0;
69        read $ifh, my($buf), 80;
70
71        print  {$ofh} "foo\n";
72        printf {$ofh} "%s\n", "bar";
73
74        open $ofh, ">>", \*STDERR;
75        syswrite $ofh, "shared handle to STDERR\n";
76
77        close $ifh;
78        close $ofh;
79

API DOCUMENTATION

81   MCE::Shared::Handle->new ( )
82       Called by MCE::Shared for constructing a shared-handle object.
83
84   open ( filehandle, expr )
85   open ( filehandle, mode, expr )
86   open ( filehandle, mode, reference )
87       In version 1.007 and later, constructs a new object by opening the file
88       whose filename is given by "expr", and associates it with "filehandle".
89       When omitting error checking at the application level, MCE::Shared
90       emits a message and stop if open fails.
91
92        # non-shared or local construction for use by a single process
93
94        use MCE::Shared::Handle;
95
96        MCE::Shared::Handle->open( my $fh, "<", "file.log" ) or die "$!";
97        MCE::Shared::Handle::open  my $fh, "<", "file.log"   or die "$!";
98
99        mce_open my $fh, "<", "file.log" or die "$!"; # ditto
100
101        # construction for sharing with other threads and processes
102
103        use MCE::Shared;
104
105        MCE::Shared->open( my $fh, "<", "file.log" ) or die "$!";
106        MCE::Shared::open  my $fh, "<", "file.log"   or die "$!";
107
108        mce_open my $fh, "<", "file.log" or die "$!"; # ditto
109
110   mce_open ( filehandle, expr )
111   mce_open ( filehandle, mode, expr )
112   mce_open ( filehandle, mode, reference )
113       Native Perl-like syntax to open a file for reading:
114
115        # mce_open is exported by MCE::Shared or MCE::Shared::Handle.
116        # It creates a shared file handle with MCE::Shared present
117        # or a non-shared handle otherwise.
118
119        mce_open my $fh, "< input.txt"     or die "open error: $!";
120        mce_open my $fh, "<", "input.txt"  or die "open error: $!";
121        mce_open my $fh, "<", \*STDIN      or die "open error: $!";
122
123       and for writing:
124
125        mce_open my $fh, "> output.txt"    or die "open error: $!";
126        mce_open my $fh, ">", "output.txt" or die "open error: $!";
127        mce_open my $fh, ">", \*STDOUT     or die "open error: $!";
128

CHUNK IO

130       Starting with "MCE::Shared" v1.007, chunk IO is possible for both non-
131       shared and shared handles. Chunk IO is enabled by the trailing 'k' or
132       'm' for read size. Also, chunk IO supports the special "\n>"-like
133       record separator.  That anchors ">" at the start of the line. Workers
134       receive record(s) beginning with ">" and ending with "\n".
135
136        # non-shared handle ---------------------------------------------
137
138        use MCE::Shared::Handle;
139
140        mce_open my $fh, '<', 'bio.fasta' or die "open error: $!";
141
142        # shared handle -------------------------------------------------
143
144        use MCE::Shared;
145
146        mce_open my $fh, '<', 'bio.fasta' or die "open error: $!";
147
148        # 'k' or 'm' indicates kibiBytes (KiB) or mebiBytes (MiB) respectively.
149        # Read continues reading until reaching the record separator or EOF.
150        # Optionally, one may specify the record separator.
151
152        $/ = "\n>";
153
154        while ( read($fh, my($buf), '2k') ) {
155           print "# chunk number: $.\n";
156           print "$buf\n";
157        }
158
159       $. contains the chunk_id above or the record_number below.
160       "readline($fh)" or $fh may be used for reading a single record.
161
162        while ( my $buf = <$fh> ) {
163           print "# record number: $.\n";
164           print "$buf\n";
165        }
166
167       The following provides a parallel demonstration. Workers receive the
168       next chunk from the shared-manager process where the actual read takes
169       place. MCE::Shared also works with "threads", "forks", and likely other
170       parallel modules.
171
172        use MCE::Hobo;       # (change to) use threads; (or) use forks;
173        use MCE::Shared;
174        use feature qw( say );
175
176        my $pattern  = 'something';
177        my $hugefile = 'somehuge.log';
178
179        my $result = MCE::Shared->array();
180        mce_open my $fh, "<", $hugefile or die "open error: $!";
181
182        sub task {
183           # the trailing 'k' or 'm' for size enables chunk IO
184           while ( read $fh, my( $slurp_chunk ), "640k" ) {
185              my $chunk_id = $.;
186              # process chunk only if a match is found; ie. fast scan
187              # optionally, comment out the if statement and closing brace
188              if ( $slurp_chunk =~ /$pattern/m ) {
189                 my @matches;
190                 while ( $slurp_chunk =~ /([^\n]+\n)/mg ) {
191                    my $line = $1; # save $1 to not lose the value
192                    push @matches, $line if ( $line =~ /$pattern/ );
193                 }
194                 $result->push( @matches ) if @matches;
195              }
196           }
197        }
198
199        MCE::Hobo->create('task') for 1 .. 4;
200
201        # do something else
202
203        MCE::Hobo->waitall();
204
205        say $result->len();
206
207       For comparison, the same thing using "MCE::Flow". MCE workers read the
208       file directly when given a plain path, so will have lesser overhead.
209       However, the run time is similar if one were to pass a file handle
210       instead to mce_flow_f.
211
212       The benefit of chunk IO is from lesser IPC for the shared-manager
213       process (above). Likewise, for the mce-manager process (below).
214
215        use MCE::Flow;
216        use feature qw( say );
217
218        my $pattern  = 'something';
219        my $hugefile = 'somehuge.log';
220
221        my @result = mce_flow_f {
222           max_workers => 4, chunk_size => '640k',
223           use_slurpio => 1,
224        },
225        sub {
226           my ( $mce, $slurp_ref, $chunk_id ) = @_;
227           # process chunk only if a match is found; ie. fast scan
228           # optionally, comment out the if statement and closing brace
229           if ( $$slurp_ref =~ /$pattern/m ) {
230              my @matches;
231              while ( $$slurp_ref =~ /([^\n]+\n)/mg ) {
232                 my $line = $1; # save $1 to not lose the value
233                 push @matches, $line if ( $line =~ /$pattern/ );
234              }
235              MCE->gather( @matches ) if @matches;
236           }
237        }, $hugefile;
238
239        say scalar( @result );
240

CREDITS

242       Implementation inspired by Tie::StdHandle.
243

LIMITATIONS

245       Perl must have IO::FDPass for constructing a shared "condvar" or
246       "queue" while the shared-manager process is running. For platforms
247       where IO::FDPass isn't possible, construct "condvar" and "queue" before
248       other classes.  On systems without "IO::FDPass", the manager process is
249       delayed until sharing other classes or started explicitly.
250
251        use MCE::Shared;
252
253        my $has_IO_FDPass = $INC{'IO/FDPass.pm'} ? 1 : 0;
254
255        my $cv  = MCE::Shared->condvar();
256        my $que = MCE::Shared->queue();
257
258        MCE::Shared->start() unless $has_IO_FDPass;
259
260       Regarding mce_open, "IO::FDPass" is needed for constructing a shared-
261       handle from a non-shared handle not yet available inside the shared-
262       manager process.  The workaround is to have the non-shared handle made
263       before the shared-manager is started. Passing a file by reference is
264       fine for the three STD* handles.
265
266        # The shared-manager knows of \*STDIN, \*STDOUT, \*STDERR.
267
268        mce_open my $shared_in,  "<",  \*STDIN;   # ok
269        mce_open my $shared_out, ">>", \*STDOUT;  # ok
270        mce_open my $shared_err, ">>", \*STDERR;  # ok
271        mce_open my $shared_fh1, "<",  "/path/to/sequence.fasta";  # ok
272        mce_open my $shared_fh2, ">>", "/path/to/results.log";     # ok
273
274        mce_open my $shared_fh, ">>", \*NON_SHARED_FH;  # requires IO::FDPass
275
276       The IO::FDPass module is known to work reliably on most platforms.
277       Install 1.1 or later to rid of limitations described above.
278
279        perl -MIO::FDPass -le "print 'Cheers! Perl has IO::FDPass.'"
280

INDEX

282       MCE, MCE::Hobo, MCE::Shared
283

AUTHOR

285       Mario E. Roy, <marioeroy AT gmail DOT com>
286
287
288
289perl v5.34.0                      2022-02-20            MCE::Shared::Handle(3)
Impressum