1MCE::Shared::Handle(3)User Contributed Perl DocumentationMCE::Shared::Handle(3)
2
6 MCE::Shared::Handle - Handle helper class
7
9 This document describes MCE::Shared::Handle version 1.873
10
12 A handle helper class for use as a standalone or managed by
13 MCE::Shared.
14
16 # non-shared or local construction for use by a single process
17 # shorter, mce_open is an alias for MCE::Shared::Handle::open
18 use MCE::Shared::Handle;
20 MCE::Shared::Handle->open( my $fh, "<", "bio.fasta" )
22 or die "open error: $!";
23 MCE::Shared::Handle::open my $fh, "<", "bio.fasta"
24 or die "open error: $!";
25 mce_open my $fh, "<", "bio.fasta" or die "open error: $!";
27 # construction for sharing with other threads and processes
29 # shorter, mce_open is an alias for MCE::Shared::open
30 use MCE::Shared;
32 MCE::Shared->open( my $fh, "<", "bio.fasta" )
34 or die "open error: $!";
35 MCE::Shared::open my $fh, "<", "bio.fasta"
36 or die "open error: $!";
37 mce_open my $fh, "<", "bio.fasta" or die "open error: $!";
39 # example, output is serialized, not garbled
41 use MCE::Hobo;
43 use MCE::Shared;
44 mce_open my $ofh, ">>", \*STDOUT or die "open error: $!";
46 mce_open my $ifh, "<", "file.log" or die "open error: $!";
47 sub parallel {
49 $/ = "\n"; # can set the input record separator
50 while (my $line = <$ifh>) {
51 printf {$ofh} "[%5d] %s", $., $line;
52 }
53 }
54 MCE::Hobo->create( \¶llel ) for 1 .. 4;
56 $_->join() for MCE::Hobo->list();
58 # handle functions
60 my $bool = eof($ifh);
62 my $off = tell($ifh);
63 my $fd = fileno($ifh);
64 my $char = getc($ifh);
65 my $line = readline($ifh);
66 binmode $ifh;
68 seek $ifh, 10, 0;
69 read $ifh, my($buf), 80;
70 print {$ofh} "foo\n";
72 printf {$ofh} "%s\n", "bar";
73 open $ofh, ">>", \*STDERR;
75 syswrite $ofh, "shared handle to STDERR\n";
76 close $ifh;
78 close $ofh;
79
81 MCE::Shared::Handle->new ( )
82 Called by MCE::Shared for constructing a shared-handle object.
83 open ( filehandle, expr )
85 open ( filehandle, mode, expr )
86 open ( filehandle, mode, reference )
87 In version 1.007 and later, constructs a new object by opening the file
88 whose filename is given by "expr", and associates it with "filehandle".
89 When omitting error checking at the application level, MCE::Shared
90 emits a message and stop if open fails.
91 # non-shared or local construction for use by a single process
93 use MCE::Shared::Handle;
95 MCE::Shared::Handle->open( my $fh, "<", "file.log" ) or die "$!";
97 MCE::Shared::Handle::open my $fh, "<", "file.log" or die "$!";
98 mce_open my $fh, "<", "file.log" or die "$!"; # ditto
100 # construction for sharing with other threads and processes
102 use MCE::Shared;
104 MCE::Shared->open( my $fh, "<", "file.log" ) or die "$!";
106 MCE::Shared::open my $fh, "<", "file.log" or die "$!";
107 mce_open my $fh, "<", "file.log" or die "$!"; # ditto
109 mce_open ( filehandle, expr )
111 mce_open ( filehandle, mode, expr )
112 mce_open ( filehandle, mode, reference )
113 Native Perl-like syntax to open a file for reading:
114 # mce_open is exported by MCE::Shared or MCE::Shared::Handle.
116 # It creates a shared file handle with MCE::Shared present
117 # or a non-shared handle otherwise.
118 mce_open my $fh, "< input.txt" or die "open error: $!";
120 mce_open my $fh, "<", "input.txt" or die "open error: $!";
121 mce_open my $fh, "<", \*STDIN or die "open error: $!";
122 and for writing:
124 mce_open my $fh, "> output.txt" or die "open error: $!";
126 mce_open my $fh, ">", "output.txt" or die "open error: $!";
127 mce_open my $fh, ">", \*STDOUT or die "open error: $!";
128
130 Starting with "MCE::Shared" v1.007, chunk IO is possible for both non-
131 shared and shared handles. Chunk IO is enabled by the trailing 'k' or
132 'm' for read size. Also, chunk IO supports the special "\n>"-like
133 record separator. That anchors ">" at the start of the line. Workers
134 receive record(s) beginning with ">" and ending with "\n".
135 # non-shared handle ---------------------------------------------
137 use MCE::Shared::Handle;
139 mce_open my $fh, '<', 'bio.fasta' or die "open error: $!";
141 # shared handle -------------------------------------------------
143 use MCE::Shared;
145 mce_open my $fh, '<', 'bio.fasta' or die "open error: $!";
147 # 'k' or 'm' indicates kibiBytes (KiB) or mebiBytes (MiB) respectively.
149 # Read continues reading until reaching the record separator or EOF.
150 # Optionally, one may specify the record separator.
151 $/ = "\n>";
153 while ( read($fh, my($buf), '2k') ) {
155 print "# chunk number: $.\n";
156 print "$buf\n";
157 }
158 $. contains the chunk_id above or the record_number below.
160 "readline($fh)" or $fh may be used for reading a single record.
161 while ( my $buf = <$fh> ) {
163 print "# record number: $.\n";
164 print "$buf\n";
165 }
166 The following provides a parallel demonstration. Workers receive the
168 next chunk from the shared-manager process where the actual read takes
169 place. MCE::Shared also works with "threads", "forks", and likely other
170 parallel modules.
171 use MCE::Hobo; # (change to) use threads; (or) use forks;
173 use MCE::Shared;
174 use feature qw( say );
175 my $pattern = 'something';
177 my $hugefile = 'somehuge.log';
178 my $result = MCE::Shared->array();
180 mce_open my $fh, "<", $hugefile or die "open error: $!";
181 sub task {
183 # the trailing 'k' or 'm' for size enables chunk IO
184 while ( read $fh, my( $slurp_chunk ), "640k" ) {
185 my $chunk_id = $.;
186 # process chunk only if a match is found; ie. fast scan
187 # optionally, comment out the if statement and closing brace
188 if ( $slurp_chunk =~ /$pattern/m ) {
189 my @matches;
190 while ( $slurp_chunk =~ /([^\n]+\n)/mg ) {
191 my $line = $1; # save $1 to not lose the value
192 push @matches, $line if ( $line =~ /$pattern/ );
193 }
194 $result->push( @matches ) if @matches;
195 }
196 }
197 }
198 MCE::Hobo->create('task') for 1 .. 4;
200 # do something else
202 MCE::Hobo->waitall();
204 say $result->len();
206 For comparison, the same thing using "MCE::Flow". MCE workers read the
208 file directly when given a plain path, so will have lesser overhead.
209 However, the run time is similar if one were to pass a file handle
210 instead to mce_flow_f.
211 The benefit of chunk IO is from lesser IPC for the shared-manager
213 process (above). Likewise, for the mce-manager process (below).
214 use MCE::Flow;
216 use feature qw( say );
217 my $pattern = 'something';
219 my $hugefile = 'somehuge.log';
220 my @result = mce_flow_f {
222 max_workers => 4, chunk_size => '640k',
223 use_slurpio => 1,
224 },
225 sub {
226 my ( $mce, $slurp_ref, $chunk_id ) = @_;
227 # process chunk only if a match is found; ie. fast scan
228 # optionally, comment out the if statement and closing brace
229 if ( $$slurp_ref =~ /$pattern/m ) {
230 my @matches;
231 while ( $$slurp_ref =~ /([^\n]+\n)/mg ) {
232 my $line = $1; # save $1 to not lose the value
233 push @matches, $line if ( $line =~ /$pattern/ );
234 }
235 MCE->gather( @matches ) if @matches;
236 }
237 }, $hugefile;
238 say scalar( @result );
240
242 Implementation inspired by Tie::StdHandle.
243
245 Perl must have IO::FDPass for constructing a shared "condvar" or
246 "queue" while the shared-manager process is running. For platforms
247 where IO::FDPass isn't possible, construct "condvar" and "queue" before
248 other classes. On systems without "IO::FDPass", the manager process is
249 delayed until sharing other classes or started explicitly.
250 use MCE::Shared;
252 my $has_IO_FDPass = $INC{'IO/FDPass.pm'} ? 1 : 0;
254 my $cv = MCE::Shared->condvar();
256 my $que = MCE::Shared->queue();
257 MCE::Shared->start() unless $has_IO_FDPass;
259 Regarding mce_open, "IO::FDPass" is needed for constructing a shared-
261 handle from a non-shared handle not yet available inside the shared-
262 manager process. The workaround is to have the non-shared handle made
263 before the shared-manager is started. Passing a file by reference is
264 fine for the three STD* handles.
265 # The shared-manager knows of \*STDIN, \*STDOUT, \*STDERR.
267 mce_open my $shared_in, "<", \*STDIN; # ok
269 mce_open my $shared_out, ">>", \*STDOUT; # ok
270 mce_open my $shared_err, ">>", \*STDERR; # ok
271 mce_open my $shared_fh1, "<", "/path/to/sequence.fasta"; # ok
272 mce_open my $shared_fh2, ">>", "/path/to/results.log"; # ok
273 mce_open my $shared_fh, ">>", \*NON_SHARED_FH; # requires IO::FDPass
275 The IO::FDPass module is known to work reliably on most platforms.
277 Install 1.1 or later to rid of limitations described above.
278 perl -MIO::FDPass -le "print 'Cheers! Perl has IO::FDPass.'"
280
282 MCE, MCE::Hobo, MCE::Shared
283
285 Mario E. Roy, <marioeroy AT gmail DOT com>
286perl v5.32.0 2020-08-02 MCE::Shared::Handle(3)