1JSON::Tokenize(3)     User Contributed Perl Documentation    JSON::Tokenize(3)
2
3
4

NAME

6       JSON::Tokenize - Tokenize JSON
7

SYNOPSIS

9           use JSON::Tokenize ':all';
10
11           my $input = '{"tuttie":["fruity", true, 100]}';
12           my $token = tokenize_json ($input);
13           print_tokens ($token, 0);
14
15           sub print_tokens
16           {
17               my ($token, $depth) = @_;
18               while ($token) {
19                   my $start = tokenize_start ($token);
20                   my $end = tokenize_end ($token);
21                   my $type = tokenize_type ($token);
22                   print "   " x $depth;
23                   my $value = substr ($input, $start, $end - $start);
24                   print "'$value' has type '$type'.\n";
25                   my $child = tokenize_child ($token);
26                   if ($child) {
27                       print_tokens ($child, $depth+1);
28                   }
29                   my $next = tokenize_next ($token);
30                   $token = $next;
31               }
32           }
33
34       This outputs
35
36           '{"tuttie":["fruity", true, 100]}' has type 'object'.
37              '"tuttie"' has type 'string'.
38              ':' has type 'colon'.
39              '["fruity", true, 100]' has type 'array'.
40                 '"fruity"' has type 'string'.
41                 ',' has type 'comma'.
42                 'true' has type 'literal'.
43                 ',' has type 'comma'.
44                 '100' has type 'number'.
45

VERSION

47       This documents version 0.61 of JSON::Tokenize corresponding to git
48       commit 033269fa8972fdce8626aa65cd11a5394ab50492
49       <https://github.com/benkasminbullock/JSON-
50       Parse/commit/033269fa8972fdce8626aa65cd11a5394ab50492> released on Thu
51       Feb 11 09:14:04 2021 +0900.
52

DESCRIPTION

54       This is a module for tokenizing a JSON string. "Tokenizing" means
55       breaking the string into individual tokens, without creating any Perl
56       structures. It uses the same underlying code as JSON::Parse. Tokenizing
57       can be used for tasks such as picking out or searching through parts of
58       a large JSON structure without storing each part of the entire
59       structure in memory.
60
61       This module is an experimental part of JSON::Parse and its interface is
62       likely to change. The tokenizing functions are currently written in a
63       very primitive way.
64

FUNCTIONS

66   tokenize_child
67           my $child = tokenize_child ($child);
68
69       Walk the tree of tokens.
70
71   tokenize_end
72           my $end = tokenize_end ($token);
73
74       Get the end of the token as a byte offset from the start of the string.
75       Note this is a byte offset not a character offset.
76
77   tokenize_json
78           my $token = tokenize_json ($json);
79
80   tokenize_next
81           my $next = tokenize_next ($token);
82
83       Walk the tree of tokens.
84
85   tokenize_start
86           my $start = tokenize_start ($token);
87
88       Get the start of the token as a byte offset from the start of the
89       string. Note this is a byte offset not a character offset.
90
91   tokenize_text
92           my $text = tokenize_text ($json, $token);
93
94       Given a token $token from this parsing and the JSON in $json, return
95       the text which corresponds to the token. This is a convenience function
96       written in Perl which uses "tokenize_start" and "tokenize_end" and
97       "substr" to get the string from $json.
98
99   tokenize_type
100           my $type = tokenize_type ($token);
101
102       Get the type of the token as a string. The possible return values are
103
104           "array",
105           "initial state",
106           "invalid",
107           "literal",
108           "number",
109           "object",
110           "string",
111           "unicode escape"
112

AUTHOR

114       Ben Bullock, <bkb@cpan.org>
115

117       This package and associated files are copyright (C) 2016-2021 Ben
118       Bullock.
119
120       You can use, copy, modify and redistribute this package and associated
121       files under the Perl Artistic Licence or the GNU General Public
122       Licence.
123
124
125
126perl v5.34.0                      2022-01-21                 JSON::Tokenize(3)
Impressum