1 package org.modeshape.cnd;
2
3 import org.modeshape.common.CommonI18n;
4 import org.modeshape.common.text.ParsingException;
5 import org.modeshape.common.text.Position;
6 import org.modeshape.common.text.TokenStream.CharacterStream;
7 import org.modeshape.common.text.TokenStream.Tokenizer;
8 import org.modeshape.common.text.TokenStream.Tokens;
9
10 /**
11 * A {@link Tokenizer} implementation that adheres to the CND format by ignoring whitespace while including tokens for individual
12 * symbols, the period ('.'), single-quoted strings, double-quoted strings, whitespace-delimited words, and optionally comments.
13 * This tokenizer optionally includes comments and vendor extensions.
14 */
15 public class CndTokenizer implements Tokenizer {
16 /**
17 * The token type for tokens that represent an unquoted string containing a character sequence made up of non-whitespace and
18 * non-symbol characters.
19 */
20 public static final int WORD = 1;
21 /**
22 * The token type for tokens that consist of an individual "symbol" character. The set of characters includes:
23 * <code>[]<>=-+(),</code>
24 */
25 public static final int SYMBOL = 2;
26 /**
27 * The token type for tokens that consist of an individual '.' character.
28 */
29 public static final int DECIMAL = 3;
30 /**
31 * The token type for tokens that consist of all the characters within single-quotes. Single quote characters are included if
32 * they are preceded (escaped) by a '\' character.
33 */
34 public static final int SINGLE_QUOTED_STRING = 4;
35 /**
36 * The token type for tokens that consist of all the characters within double-quotes. Double quote characters are included if
37 * they are preceded (escaped) by a '\' character.
38 */
39 public static final int DOUBLE_QUOTED_STRING = 5;
40 /**
41 * The token type for tokens that consist of all the characters between "/*" and "*/" or between "//" and the next line
42 * terminator (e.g., '\n', '\r' or "\r\n").
43 */
44 public static final int COMMENT = 6;
45 /**
46 * The token type for the token containing a vendor extension block.
47 */
48 public static final int VENDOR_EXTENSION = 7;
49
50 private final boolean useComments;
51 private final boolean useVendorExtensions;
52
53 public CndTokenizer( boolean useComments,
54 boolean useVendorExtensions ) {
55 this.useComments = useComments;
56 this.useVendorExtensions = useVendorExtensions;
57 }
58
59 /**
60 * {@inheritDoc}
61 *
62 * @see org.modeshape.common.text.TokenStream.Tokenizer#tokenize(CharacterStream, Tokens)
63 */
64 public void tokenize( CharacterStream input,
65 Tokens tokens ) throws ParsingException {
66 while (input.hasNext()) {
67 char c = input.next();
68 switch (c) {
69 case ' ':
70 case '\t':
71 case '\n':
72 case '\r':
73 // Just skip these whitespace characters ...
74 break;
75 case '[':
76 case ']':
77 case '<':
78 case '>':
79 case '=':
80 case '-':
81 case '+':
82 case '(':
83 case ')':
84 case ',':
85 tokens.addToken(input.position(input.index()), input.index(), input.index() + 1, SYMBOL);
86 break;
87 // case '.':
88 // tokens.addToken(input.position(), input.index(), input.index() + 1, DECIMAL);
89 // break;
90 case '{':
91 // Vendor extension, meant to be excluded
92 int startIndex = input.index();
93 Position startingPosition = input.position(startIndex);
94 boolean foundClosingBrace = false;
95 String vendorName = "";
96 while (input.hasNext()) {
97 c = input.next();
98 if (c == '\\' && input.isNext('}')) {
99 c = input.next(); // consume the '}' character since it is escaped
100 } else if (c == '}') {
101 foundClosingBrace = true;
102 break;
103 }
104 }
105 if (!foundClosingBrace) {
106 String msg = CndI18n.vendorBlockWasNotClosed.text(startingPosition.getLine(),
107 startingPosition.getColumn());
108 throw new ParsingException(startingPosition, msg);
109 }
110 int endIndex = input.index() + 1; // beyond last character read
111 if (useVendorExtensions) {
112 tokens.addToken(startingPosition, startIndex, endIndex, VENDOR_EXTENSION);
113 }
114 break;
115 case '\"':
116 startIndex = input.index();
117 startingPosition = input.position(startIndex);
118 boolean foundClosingQuote = false;
119 while (input.hasNext()) {
120 c = input.next();
121 if (c == '\\' && input.isNext('"')) {
122 c = input.next(); // consume the ' character since it is escaped
123 } else if (c == '"') {
124 foundClosingQuote = true;
125 break;
126 }
127 }
128 if (!foundClosingQuote) {
129 String msg = CommonI18n.noMatchingDoubleQuoteFound.text(startingPosition.getLine(),
130 startingPosition.getColumn());
131 throw new ParsingException(startingPosition, msg);
132 }
133 endIndex = input.index() + 1; // beyond last character read
134 tokens.addToken(startingPosition, startIndex, endIndex, DOUBLE_QUOTED_STRING);
135 break;
136 case '\'':
137 startIndex = input.index();
138 startingPosition = input.position(startIndex);
139 foundClosingQuote = false;
140 while (input.hasNext()) {
141 c = input.next();
142 if (c == '\\' && input.isNext('\'')) {
143 c = input.next(); // consume the ' character since it is escaped
144 } else if (c == '\'') {
145 foundClosingQuote = true;
146 break;
147 }
148 }
149 if (!foundClosingQuote) {
150 String msg = CommonI18n.noMatchingSingleQuoteFound.text(startingPosition.getLine(),
151 startingPosition.getColumn());
152 throw new ParsingException(startingPosition, msg);
153 }
154 endIndex = input.index() + 1; // beyond last character read
155 tokens.addToken(startingPosition, startIndex, endIndex, SINGLE_QUOTED_STRING);
156 break;
157 case '/':
158 startIndex = input.index();
159 startingPosition = input.position(startIndex);
160 if (input.isNext('/')) {
161 // End-of-line comment ...
162 boolean foundLineTerminator = false;
163 while (input.hasNext()) {
164 c = input.next();
165 if (c == '\n' || c == '\r') {
166 foundLineTerminator = true;
167 break;
168 }
169 }
170 endIndex = input.index(); // the token won't include the '\n' or '\r' character(s)
171 if (!foundLineTerminator) ++endIndex; // must point beyond last char
172 if (c == '\r' && input.isNext('\n')) input.next();
173 if (useComments) {
174 tokens.addToken(startingPosition, startIndex, endIndex, COMMENT);
175 }
176 } else if (input.isNext('*')) {
177 // Multi-line comment ...
178 while (input.hasNext() && !input.isNext('*', '/')) {
179 c = input.next();
180 }
181 if (input.hasNext()) input.next(); // consume the '*'
182 if (input.hasNext()) input.next(); // consume the '/'
183 if (useComments) {
184 endIndex = input.index() + 1; // the token will include the '/' and '*' characters
185 tokens.addToken(startingPosition, startIndex, endIndex, COMMENT);
186 }
187 } else {
188 continue;
189 }
190 break;
191 default:
192 // The JCR 2.0 Public Final Draft is very unclear about what exactly a string is defined to be,
193 // and in fact the reference implementation (all versions) basically just treat an unquoted string
194 // to be defined as
195 // - unquoted_string ::= [A-Za-z0-9:_]+
196 // But this doesn't really seem to align very well with the spec, which alludes to any number
197 // of XmlChar:
198 // - unquoted_string ::= XmlChar { XmlChar }
199 // - XmlChar ::= /* see ยง3.2.2 Local Names */
200 // Then in Section 3.2.2, there is this rule:
201 // - XmlChar ::= /* Any character that matches the Char production at http://www.w3.org/TR/xml/" target="alexandria_uri">http://www.w3.org/TR/xml/#NT-Char */
202 // This doesn't really make sense, because even whitespace is valid in Char.
203 //
204 // Could the CND grammar instead reference 3.2.5.2 (rather than 3.2.2)? This refers to qualified
205 // names, and appears to be much closer to the examples and reference implementation.
206 //
207 // What we're doing is basically reading all subsequent characters until we find a whitespace,
208 // one of the SYMBOL characters, a single- or double-quote character, a slash, or an open brace
209 // (since these are all the basis for other tokenization rules above). Also, the '*' and '|'
210 // characters terminate a WORD token, since these cannot appear unescaped within local names;
211 // since these do not appear in other rules above, they will result in one-character tokens.
212 //
213 startIndex = input.index();
214 startingPosition = input.position(startIndex);
215 // Read as long as there is a valid XML character ...
216 while (input.hasNext() && !(input.isNextWhitespace() || input.isNextAnyOf("[]<>=-+(),\"'/{*|"))) {
217 c = input.next();
218 }
219 endIndex = input.index() + 1; // beyond last character that was included
220 tokens.addToken(startingPosition, startIndex, endIndex, WORD);
221 }
222 }
223 }
224 }