1LIBPIPELINE(3)           GNU Library Functions Manual           LIBPIPELINE(3)
2

NAME

4     libpipeline — pipeline manipulation library
5

SYNOPSIS

7     #include <pipeline.h>
8

DESCRIPTION

10     libpipeline is a C library for setting up and running pipelines of pro‐
11     cesses, without needing to involve shell command-line parsing which is
12     often error-prone and insecure.  This relieves programmers of the need to
13     laboriously construct pipelines using lower-level primitives such as fork
14     and execve.
15
16     The general way to use libpipeline involves constructing a pipeline
17     structure and adding one or more pipecmd structures to it.  A pipecmd
18     represents a subprocess (or “command”), while a pipeline represents a
19     sequence of subprocesses each of whose outputs is connected to the next
20     one's input, as in the example ls | grep pattern | less.  The calling
21     program may adjust certain properties of each command independently, such
22     as its environment and nice(3) priority, as well as properties of the
23     entire pipeline such as its input and output and the way signals are han‐
24     dled while executing it.  The calling program may then start the pipe‐
25     line, read output from it, wait for it to complete, and gather its exit
26     status.
27
28     Strings passed as const char * function arguments will be copied by the
29     library.
30
31   Functions to build individual commands
32     pipecmd *pipecmd_new(const char *name)
33
34           Construct a new command representing execution of a program called
35           name.
36
37     pipecmd *pipecmd_new_argv(const char *name, va_list argv)
38     pipecmd *pipecmd_new_args(const char *name, ...)
39
40           Convenience constructors wrapping pipecmd_new() and pipecmd_arg().
41           Construct a new command representing execution of a program called
42           name with arguments.  Terminate arguments with NULL.
43
44     pipecmd *pipecmd_new_argstr(const char *argstr)
45
46           Split argstr on whitespace to construct a command and arguments,
47           honouring shell-style single-quoting, double-quoting, and back‐
48           slashes, but not other shell evilness like wildcards, semicolons,
49           or backquotes.  This is included only to support situations where
50           command arguments are encoded into configuration files and the
51           like.  While it is safer than system(3), it still involves signifi‐
52           cant string parsing which is inherently riskier than avoiding it
53           altogether.  Please try to avoid using it in new code.
54
55     typedef void pipecmd_function_type (void *);
56     typedef void pipecmd_function_free_type (void *);
57     pipecmd *pipecmd_new_function(const char *name,
58           pipecmd_function_type *func, pipecmd_function_free_type *free_func,
59           void *data)
60
61           Construct a new command that calls a given function rather than
62           executing a process.
63
64           The data argument is passed as the function's only argument, and
65           will be freed before returning using free_func (if non-NULL).
66
67           pipecmd_* functions that deal with arguments cannot be used with
68           the command returned by this function.
69
70     pipecmd *pipecmd_new_sequencev(const char *name, va_list cmdv)
71     pipecmd *pipecmd_new_sequence(const char *name, ...)
72
73           Construct a new command that itself runs a sequence of commands,
74           supplied as command * arguments following name and terminated by
75           NULL.  The commands will be executed in forked children; if any
76           exits non-zero then it will terminate the sequence, as with "&&" in
77           shell.
78
79           pipecmd_* functions that deal with arguments cannot be used with
80           the command returned by this function.
81
82     pipecmd *pipecmd_new_passthrough(void)
83
84           Return a new command that just passes data from its input to its
85           output.
86
87     pipecmd *pipecmd_dup(pipecmd *cmd)
88
89           Return a duplicate of a command.
90
91     void pipecmd_arg(pipecmd *cmd, const char *arg)
92
93           Add an argument to a command.
94
95     void pipecmd_argf(pipecmd *cmd, const char *format, ...)
96
97           Convenience function to add an argument with printf substitutions.
98
99     void pipecmd_argv(pipecmd *cmd, va_list argv)
100     void pipecmd_args(pipecmd *cmd, ...)
101
102           Convenience functions wrapping pipecmd_arg() to add multiple argu‐
103           ments at once.  Terminate arguments with NULL.
104
105     void pipecmd_argstr(pipecmd *cmd, const char *argstr)
106
107           Split argstr on whitespace to add a list of arguments, honouring
108           shell-style single-quoting, double-quoting, and backslashes, but
109           not other shell evilness like wildcards, semicolons, or backquotes.
110           This is included only to support situations where command arguments
111           are encoded into configuration files and the like.  While it is
112           safer than system(3), it still involves significant string parsing
113           which is inherently riskier than avoiding it altogether.  Please
114           try to avoid using it in new code.
115
116     void pipecmd_get_nargs(pipecmd *cmd)
117
118           Return the number of arguments to this command.  Note that this
119           includes the command name as the first argument, so the command
120           ‘echo foo bar’ is counted as having three arguments.
121
122     void pipecmd_nice(pipecmd *cmd, int value)
123
124           Set the nice(3) value for this command.  Defaults to 0.  Errors
125           while attempting to set the nice value are ignored, aside from
126           emitting a debug message.
127
128     void pipecmd_discard_err(pipecmd *cmd, int discard_err)
129
130           If discard_err is non-zero, redirect this command's standard error
131           to /dev/null.  Otherwise, and by default, pass it through.  This is
132           usually a bad idea.
133
134     void pipecmd_setenv(pipecmd *cmd, const char *name, const char *value)
135
136           Set environment variable name to value while running this command.
137
138     void pipecmd_unsetenv(pipecmd *cmd, const char *name)
139
140           Unset environment variable name while running this command.
141
142     void pipecmd_clearenv(pipecmd *cmd)
143
144           Clear the environment while running this command.  (Note that envi‐
145           ronment operations work in sequence; pipecmd_clearenv followed by
146           pipecmd_setenv causes the command to have just a single environment
147           variable set.)
148
149     void pipecmd_sequence_command(pipecmd *cmd, pipecmd *child)
150
151           Add a command to a sequence created using pipecmd_new_sequence().
152
153     void pipecmd_dump(pipecmd *cmd, FILE *stream)
154
155           Dump a string representation of a command to stream.
156
157     char *pipecmd_tostring(pipecmd *cmd)
158
159           Return a string representation of a command.  The caller should
160           free the result.
161
162     void pipecmd_exec(pipecmd *cmd)
163
164           Execute a single command, replacing the current process.  Never
165           returns, instead exiting non-zero on failure.
166
167     void pipecmd_free(pipecmd *cmd)
168
169           Destroy a command.  Safely does nothing if cmd is NULL.
170
171   Functions to build pipelines
172     pipeline *pipeline_new(void)
173
174           Construct a new pipeline.
175
176     pipeline *pipeline_new_commandv(pipecmd *cmd1, va_list cmdv)
177     pipeline *pipeline_new_commands(pipecmd *cmd1, ...)
178
179           Convenience constructors wrapping pipeline_new() and
180           pipeline_command().  Construct a new pipeline consisting of the
181           given list of commands.  Terminate commands with NULL.
182
183     pipeline *pipeline_new_command_argv(const char *name, va_list argv)
184     pipeline *pipeline_new_command_args(const char *name, ...)
185
186           Construct a new pipeline and add a single command to it.
187
188     pipeline *pipeline_join(pipeline *p1, pipeline *p2)
189
190           Joins two pipelines, neither of which are allowed to be started.
191           Discards want_out, want_outfile, and outfd from p1, and want_in,
192           want_infile, and infd from p2.
193
194     void pipeline_connect(pipeline *source, pipeline *sink, ...)
195
196           Connect the input of one or more sink pipelines to the output of a
197           source pipeline.  The source pipeline may be started, but in that
198           case pipeline_want_out() must have been called with a negative fd;
199           otherwise, calls pipeline_want_out(source, -1).  In any event,
200           calls pipeline_want_in(sink, -1) on all sinks, none of which are
201           allowed to be started.  Terminate arguments with NULL.
202
203           This is an application-level connection; data may be intercepted
204           between the pipelines by the program before calling
205           pipeline_pump(), which sets data flowing from the source to the
206           sinks.  It is primarily useful when more than one sink pipeline is
207           involved, in which case the pipelines cannot simply be concatenated
208           into one.
209
210           The result is similar to tee(1), except that output can be sent to
211           more than two places and can easily be sent to multiple processes.
212
213     void pipeline_command(pipeline *p, pipecmd *cmd)
214
215           Add a command to a pipeline.
216
217     void pipeline_command_argv(pipeline *p, const char *name, va_list argv)
218     void pipeline_command_args(pipeline *p, const char *name, ...)
219
220           Construct a new command and add it to a pipeline in one go.
221
222     void pipeline_command_argstr(pipeline *p, const char *argstr)
223
224           Construct a new command from a shell-quoted string and add it to a
225           pipeline in one go.  See the comment against pipecmd_new_argstr()
226           above if you're tempted to use this function.
227
228     void pipeline_commandv(pipeline *p, va_list cmdv)
229     void pipeline_commands(pipeline *p, ...)
230
231           Convenience functions wrapping pipeline_command() to add multiple
232           commands at once.  Terminate arguments with NULL.
233
234     void pipeline_want_in(pipeline *p, int fd)
235     void pipeline_want_out(pipeline *p, int fd)
236
237           Set file descriptors to use as the input and output of the whole
238           pipeline.  If non-negative, fd is used directly as a file descrip‐
239           tor.  If negative, pipeline_start() will create pipes and store the
240           input writing half and the output reading half in the pipeline's
241           infd or outfd field as appropriate.  The default is to leave input
242           and output as stdin and stdout unless pipeline_want_infile() or
243           pipeline_want_outfile() respectively has been called.
244
245           Calling these functions supersedes any previous call to
246           pipeline_want_infile() or pipeline_want_outfile() respectively.
247
248     void pipeline_want_infile(pipeline *p, const char *file)
249     void pipeline_want_outfile(pipeline *p, const char *file)
250
251           Set file names to open and use as the input and output of the whole
252           pipeline.  This may be more convenient than supplying file descrip‐
253           tors, and guarantees that the files are opened with the same privi‐
254           leges under which the pipeline is run.
255
256           Calling these functions (even with NULL, which returns to the
257           default of leaving input and output as stdin and stdout) supersedes
258           any previous call to pipeline_want_in() or pipeline_want_outfile()
259           respectively.
260
261           The given files will be opened when the pipeline is started.  If an
262           output file does not already exist, it is created (with mode 0666
263           modified in the usual way by umask); if it does exist, then it is
264           truncated.
265
266     void pipeline_ignore_signals(pipeline *p, int ignore_signals)
267
268           If ignore_signals is non-zero, ignore SIGINT and SIGQUIT in the
269           calling process while the pipeline is running, like system(3).
270           Otherwise, and by default, leave their dispositions unchanged.
271
272     int pipeline_get_ncommands(pipeline *p)
273
274           Return the number of commands in this pipeline.
275
276     pipecmd *pipeline_get_command(pipeline *p, int n)
277
278           Return command number n from this pipeline, counting from zero, or
279           NULL if n is out of range.
280
281     pipecmd *pipeline_set_command(pipeline *p, int n, pipecmd *cmd)
282
283           Set command number n in this pipeline, counting from zero, to cmd,
284           and return the previous command in that position.  Do nothing and
285           return NULL if n is out of range.
286
287     pid_t pipeline_get_pid(pipeline *p, int n)
288
289           Return the process ID of command number n from this pipeline,
290           counting from zero.  The pipeline must be started.  Return -1 if n
291           is out of range or if the command has already exited and been
292           reaped.
293
294     FILE *pipeline_get_infile(pipeline *p)
295     FILE *pipeline_get_outfile(pipeline *p)
296
297           Get streams corresponding to infd and outfd respectively.  The
298           pipeline must be started.
299
300     void pipeline_dump(pipeline *p, FILE *stream)
301
302           Dump a string representation of p to stream.
303
304     char *pipeline_tostring(pipeline *p)
305
306           Return a string representation of p.  The caller should free the
307           result.
308
309     void pipeline_free(pipeline *p)
310
311           Destroy a pipeline and all its commands.  Safely does nothing if p
312           is NULL.  May wait for the pipeline to complete if it has not
313           already done so.
314
315   Functions to run pipelines and handle signals
316     typedef void pipeline_post_fork_fn (void);
317     void pipeline_install_post_fork(pipeline_post_fork_fn *fn)
318
319           Install a post-fork handler.  This will be run in any child process
320           immediately after it is forked.  For instance, this may be used for
321           cleaning up application-specific signal handlers.  Pass NULL to
322           clear any existing post-fork handler.
323
324     void pipeline_start(pipeline *p)
325
326           Start the processes in a pipeline.  Installs this library's SIGCHLD
327           handler if not already installed.  Calls error (FATAL) on error.
328
329     int pipeline_wait_all(pipeline *p, int **statuses, int *n_statuses)
330
331           Wait for a pipeline to complete.  Set *statuses to a newly-allo‐
332           cated array of wait statuses, as returned by waitpid(2), and
333           *n_statuses to the length of that array.  The return value is simi‐
334           lar to the exit status that a shell would return, with some modifi‐
335           cations.  If the last command exits with a signal (other than
336           SIGPIPE, which is considered equivalent to exiting zero), then the
337           return value is 128 plus the signal number; if the last command
338           exits normally but non-zero, then the return value is its exit sta‐
339           tus; if any other command exits non-zero, then the return value is
340           127; otherwise, the return value is 0.  This means that the return
341           value is only 0 if all commands in the pipeline exit successfully.
342
343     int pipeline_wait(pipeline *p)
344
345           Wait for a pipeline to complete and return the exit status.
346
347     int pipeline_run(pipeline *p)
348
349           Start a pipeline, wait for it to complete, and free it, all in one
350           go.
351
352     void pipeline_pump(pipeline *p, ...)
353
354           Pump data among one or more pipelines connected using
355           pipeline_connect() until all source pipelines have reached end-of-
356           file and all data has been written to all sinks (or failed).  All
357           relevant pipelines must be supplied: that is, no pipeline that has
358           been connected to a source pipeline may be supplied unless that
359           source pipeline is also supplied.  Automatically starts all pipe‐
360           lines if they are not already started, but does not wait for them.
361           Terminate arguments with NULL.
362
363   Functions to read output from pipelines
364     In general, output is returned as a pointer into a buffer owned by the
365     pipeline, which is automatically freed when pipeline_free() is called.
366     This saves the caller from having to explicitly free individual blocks of
367     output data.
368
369     const char *pipeline_read(pipeline *p, size_t *len)
370
371           Read len bytes of data from the pipeline, returning the data block.
372           len is updated with the number of bytes read.
373
374     const char *pipeline_peek(pipeline *p, size_t *len)
375
376           Look ahead in the pipeline's output for len bytes of data, return‐
377           ing the data block.  len is updated with the number of bytes read.
378           The starting position of the next read or peek is not affected by
379           this call.
380
381     size_t pipeline_peek_size(pipeline *p)
382
383           Return the number of bytes of data that can be read using
384           pipeline_read() or pipeline_peek() solely from the peek cache,
385           without having to read from the pipeline itself (and thus poten‐
386           tially block).
387
388     void pipeline_peek_skip(pipeline *p, size_t len)
389
390           Skip over and discard len bytes of data from the peek cache.
391           Asserts that enough data is available to skip, so you may want to
392           check using pipeline_peek_size() first.
393
394     const char *pipeline_readline(pipeline *p)
395
396           Read a line of data from the pipeline, returning it.
397
398     const char *pipeline_peekline(pipeline *p)
399
400           Look ahead in the pipeline's output for a line of data, returning
401           it.  The starting position of the next read or peek is not affected
402           by this call.
403
404   Signal handling
405     libpipeline installs a signal handler for SIGCHLD, and collects the exit
406     status of child processes in pipeline_wait().  Applications using this
407     library must either refrain from changing the disposition of SIGCHLD (in
408     other words, must rely on libpipeline for all child process handling) or
409     else must make sure to restore libpipeline's SIGCHLD handler before call‐
410     ing any of its functions.
411
412     If the ignore_signals flag is set in a pipeline (which is the default),
413     then the SIGINT and SIGQUIT signals will be ignored in the parent process
414     while child processes are running.  This mirrors the behaviour of
415     system(3).
416
417     libpipeline leaves child processes with the default disposition of
418     SIGPIPE, namely to terminate the process.  It ignores SIGPIPE in the par‐
419     ent process while running pipeline_pump().
420
421   Reaping of child processes
422     libpipeline installs a SIGCHLD handler that will attempt to reap child
423     processes which have exited.  This calls waitpid(2) with -1, so it will
424     reap any child process, not merely those created by way of this library.
425     At present, this means that if the calling program forks other child pro‐
426     cesses which may exit while a pipeline is running, the program is not
427     guaranteed to be able to collect exit statuses of those processes.
428
429     You should not rely on this behaviour, and in future it may be modified
430     either to reap only child processes created by this library or to provide
431     a way to return foreign statuses to the application.  Please contact the
432     author if you have an example application and would like to help design
433     such an interface.
434

ENVIRONMENT

436     If the PIPELINE_DEBUG environment variable is set to “1”, then
437     libpipeline will emit debugging messages on standard error.
438

EXAMPLES

440     In the following examples, function names starting with pipecmd_ or
441     pipeline_ are real libpipeline functions, while any other function names
442     are pseudocode.
443
444     The simplest case is simple.  To run a single command, such as mv source
445     dest:
446
447           pipeline *p = pipeline_new_command_args ("mv", source, dest, NULL);
448           int status = pipeline_run (p);
449
450     libpipeline is often used to mimic shell pipelines, such as the following
451     example:
452
453           zsoelim < input-file | tbl | nroff -mandoc -Tutf8
454
455     The code to construct this would be:
456
457           pipeline *p;
458           int status;
459
460           p = pipeline_new ();
461           pipeline_want_infile (p, "input-file");
462           pipeline_command_args (p, "zsoelim", NULL);
463           pipeline_command_args (p, "tbl", NULL);
464           pipeline_command_args (p, "nroff", "-mandoc", "-Tutf8", NULL);
465           status = pipeline_run (p);
466
467     You might want to construct a command more dynamically:
468
469           pipecmd *manconv = pipecmd_new_args ("manconv", "-f", from_code,
470                                                "-t", "UTF-8", NULL);
471           if (quiet)
472                   pipecmd_arg (manconv, "-q");
473           pipeline_command (p, manconv);
474
475     Perhaps you want an environment variable set only while running a certain
476     command:
477
478           pipecmd *less = pipecmd_new ("less");
479           pipecmd_setenv (less, "LESSCHARSET", lesscharset);
480
481     You might find yourself needing to pass the output of one pipeline to
482     several other pipelines, in a “tee” arrangement:
483
484           pipeline *source, *sink1, *sink2;
485
486           source = make_source ();
487           sink1 = make_sink1 ();
488           sink2 = make_sink2 ();
489           pipeline_connect (source, sink1, sink2, NULL);
490           /* Pump data among these pipelines until there's nothing left. */
491           pipeline_pump (source, sink1, sink2, NULL);
492           pipeline_free (sink2);
493           pipeline_free (sink1);
494           pipeline_free (source);
495
496     Maybe one of your commands is actually an in-process function, rather
497     than an external program:
498
499           pipecmd *inproc = pipecmd_new_function ("in-process", &func,
500                                                   NULL, NULL);
501           pipeline_command (p, inproc);
502
503     Sometimes your program needs to consume the output of a pipeline, rather
504     than sending it all to some other subprocess:
505
506           pipeline *p = make_pipeline ();
507           const char *line;
508
509           pipeline_want_out (p, -1);
510           pipeline_start (p);
511           line = pipeline_peekline (p);
512           if (!strstr (line, "coding: UTF-8"))
513                   printf ("Unicode text follows:0);
514           while (line = pipeline_readline (p))
515                   printf ("  %s", line);
516           pipeline_free (p);
517

SEE ALSO

519     fork(2), execve(2), system(3), popen(3).
520

AUTHORS

522     Most of libpipeline was written by Colin Watson <cjwatson@debian.org>,
523     originally for use in man-db.  The initial version was based very loosely
524     on the run_pipeline() function in GNU groff, written by James Clark
525     <jjc@jclark.com>.  It also contains library code by Markus Armbruster,
526     and by various contributors to Gnulib.
527
528     libpipeline is licensed under the GNU General Public License, version 3
529     or later.  See the README file for full details.
530

BUGS

532     Using this library in a program which runs any other child processes
533     and/or installs its own SIGCHLD handler is unlikely to work.
534
535GNU                            October 11, 2010                            GNU
Impressum