scantoken.cpp 10.3 KB
Newer Older
1
2
3
4
5
#include "scanner.h"
#include "token.h"
#include "exceptions.h"
#include "exp.h"
#include "scanscalar.h"
6
7
#include "scantag.h"
#include "tag.h"
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
#include <sstream>

namespace YAML
{
	///////////////////////////////////////////////////////////////////////
	// Specialization for scanning specific tokens

	// Directive
	// . Note: no semantic checking is done here (that's for the parser to do)
	void Scanner::ScanDirective()
	{
		std::string name;
		std::vector <std::string> params;

		// pop indents and simple keys
23
		PopAllIndents();
24
		PopAllSimpleKeys();
25
26
27
28

		m_simpleKeyAllowed = false;

		// store pos and eat indicator
29
		Token token(Token::DIRECTIVE, INPUT.mark());
30
31
32
		INPUT.eat(1);

		// read name
33
		while(INPUT && !Exp::BlankOrBreak.Matches(INPUT))
34
			token.value += INPUT.get();
35
36
37
38
39
40
41
42

		// read parameters
		while(1) {
			// first get rid of whitespace
			while(Exp::Blank.Matches(INPUT))
				INPUT.eat(1);

			// break on newline or comment
43
			if(!INPUT || Exp::Break.Matches(INPUT) || Exp::Comment.Matches(INPUT))
44
45
46
47
				break;

			// now read parameter
			std::string param;
48
			while(INPUT && !Exp::BlankOrBreak.Matches(INPUT))
49
50
				param += INPUT.get();

51
			token.params.push_back(param);
52
53
54
55
56
57
58
59
		}
		
		m_tokens.push(token);
	}

	// DocStart
	void Scanner::ScanDocStart()
	{
60
		PopAllIndents();
61
		PopAllSimpleKeys();
62
63
64
		m_simpleKeyAllowed = false;

		// eat
65
		Mark mark = INPUT.mark();
66
		INPUT.eat(3);
67
		m_tokens.push(Token(Token::DOC_START, mark));
68
69
70
71
72
	}

	// DocEnd
	void Scanner::ScanDocEnd()
	{
73
		PopAllIndents();
74
		PopAllSimpleKeys();
75
76
77
		m_simpleKeyAllowed = false;

		// eat
78
		Mark mark = INPUT.mark();
79
		INPUT.eat(3);
80
		m_tokens.push(Token(Token::DOC_END, mark));
81
82
83
84
85
86
	}

	// FlowStart
	void Scanner::ScanFlowStart()
	{
		// flows can be simple keys
87
		InsertPotentialSimpleKey();
88
89
90
		m_simpleKeyAllowed = true;

		// eat
91
		Mark mark = INPUT.mark();
92
		char ch = INPUT.get();
93
94
95
		FLOW_MARKER flowType = (ch == Keys::FlowSeqStart ? FLOW_SEQ : FLOW_MAP);
		m_flows.push(flowType);
		Token::TYPE type = (flowType == FLOW_SEQ ? Token::FLOW_SEQ_START : Token::FLOW_MAP_START);
96
		m_tokens.push(Token(type, mark));
97
98
99
100
101
	}

	// FlowEnd
	void Scanner::ScanFlowEnd()
	{
102
		if(InBlockContext())
103
			throw ParserException(INPUT.mark(), ErrorMsg::FLOW_END);
104

105
		// we might have a solo entry in the flow context
106
107
108
109
110
111
		if(InFlowContext()) {
			if(m_flows.top() == FLOW_MAP && VerifySimpleKey())
				m_tokens.push(Token(Token::VALUE, INPUT.mark()));
			else if(m_flows.top() == FLOW_SEQ)
				InvalidateSimpleKey();
		}
112

113
114
115
		m_simpleKeyAllowed = false;

		// eat
116
		Mark mark = INPUT.mark();
117
		char ch = INPUT.get();
118
119
120
121
122
123
124
125

		// check that it matches the start
		FLOW_MARKER flowType = (ch == Keys::FlowSeqEnd ? FLOW_SEQ : FLOW_MAP);
		if(m_flows.top() != flowType)
			throw ParserException(mark, ErrorMsg::FLOW_END);
		m_flows.pop();
		
		Token::TYPE type = (flowType ? Token::FLOW_SEQ_END : Token::FLOW_MAP_END);
126
		m_tokens.push(Token(type, mark));
127
128
129
130
131
	}

	// FlowEntry
	void Scanner::ScanFlowEntry()
	{
132
133
134
135
136
137
138
		// we might have a solo entry in the flow context
		if(InFlowContext()) {
			if(m_flows.top() == FLOW_MAP && VerifySimpleKey())
				m_tokens.push(Token(Token::VALUE, INPUT.mark()));
			else if(m_flows.top() == FLOW_SEQ)
				InvalidateSimpleKey();
		}
139
		
140
141
142
		m_simpleKeyAllowed = true;

		// eat
143
		Mark mark = INPUT.mark();
144
		INPUT.eat(1);
145
		m_tokens.push(Token(Token::FLOW_ENTRY, mark));
146
147
148
149
150
151
	}

	// BlockEntry
	void Scanner::ScanBlockEntry()
	{
		// we better be in the block context!
152
		if(InFlowContext())
153
			throw ParserException(INPUT.mark(), ErrorMsg::BLOCK_ENTRY);
154
155
156

		// can we put it here?
		if(!m_simpleKeyAllowed)
157
			throw ParserException(INPUT.mark(), ErrorMsg::BLOCK_ENTRY);
158

159
		PushIndentTo(INPUT.column(), IndentMarker::SEQ);
160
161
162
		m_simpleKeyAllowed = true;

		// eat
163
		Mark mark = INPUT.mark();
164
		INPUT.eat(1);
165
		m_tokens.push(Token(Token::BLOCK_ENTRY, mark));
166
167
168
169
170
171
	}

	// Key
	void Scanner::ScanKey()
	{
		// handle keys diffently in the block context (and manage indents)
172
		if(InBlockContext()) {
173
			if(!m_simpleKeyAllowed)
174
				throw ParserException(INPUT.mark(), ErrorMsg::MAP_KEY);
175

176
			PushIndentTo(INPUT.column(), IndentMarker::MAP);
177
178
179
		}

		// can only put a simple key here if we're in block context
180
		m_simpleKeyAllowed = InBlockContext();
181
182

		// eat
183
		Mark mark = INPUT.mark();
184
		INPUT.eat(1);
185
		m_tokens.push(Token(Token::KEY, mark));
186
187
188
189
190
	}

	// Value
	void Scanner::ScanValue()
	{
191
192
193
194
		// and check that simple key
		bool isSimpleKey = VerifySimpleKey();
		
		if(isSimpleKey) {
195
196
197
198
			// can't follow a simple key with another simple key (dunno why, though - it seems fine)
			m_simpleKeyAllowed = false;
		} else {
			// handle values diffently in the block context (and manage indents)
199
			if(InBlockContext()) {
200
				if(!m_simpleKeyAllowed)
201
					throw ParserException(INPUT.mark(), ErrorMsg::MAP_VALUE);
202

203
				PushIndentTo(INPUT.column(), IndentMarker::MAP);
204
205
206
			}

			// can only put a simple key here if we're in block context
207
			m_simpleKeyAllowed = InBlockContext();
208
209
210
		}

		// eat
211
		Mark mark = INPUT.mark();
212
		INPUT.eat(1);
213
		m_tokens.push(Token(Token::VALUE, mark));
214
215
216
217
218
219
220
221
222
	}

	// AnchorOrAlias
	void Scanner::ScanAnchorOrAlias()
	{
		bool alias;
		std::string name;

		// insert a potential simple key
223
		InsertPotentialSimpleKey();
224
225
226
		m_simpleKeyAllowed = false;

		// eat the indicator
227
		Mark mark = INPUT.mark();
228
229
230
231
232
233
234
235
236
		char indicator = INPUT.get();
		alias = (indicator == Keys::Alias);

		// now eat the content
		while(Exp::AlphaNumeric.Matches(INPUT))
			name += INPUT.get();

		// we need to have read SOMETHING!
		if(name.empty())
237
			throw ParserException(INPUT.mark(), alias ? ErrorMsg::ALIAS_NOT_FOUND : ErrorMsg::ANCHOR_NOT_FOUND);
238
239

		// and needs to end correctly
240
		if(INPUT && !Exp::AnchorEnd.Matches(INPUT))
241
			throw ParserException(INPUT.mark(), alias ? ErrorMsg::CHAR_IN_ALIAS : ErrorMsg::CHAR_IN_ANCHOR);
242
243

		// and we're done
244
		Token token(alias ? Token::ALIAS : Token::ANCHOR, mark);
245
246
247
248
249
250
251
252
		token.value = name;
		m_tokens.push(token);
	}

	// Tag
	void Scanner::ScanTag()
	{
		// insert a potential simple key
253
		InsertPotentialSimpleKey();
254
255
		m_simpleKeyAllowed = false;

256
		Token token(Token::TAG, INPUT.mark());
257

258
259
260
261
262
		// eat the indicator
		INPUT.get();
		
		if(INPUT && INPUT.peek() == Keys::VerbatimTagStart){
			std::string tag = ScanVerbatimTag(INPUT);
263

264
265
			token.value = tag;
			token.data = Tag::VERBATIM;
266
		} else {
267
268
269
270
271
272
273
274
275
276
277
			bool canBeHandle;
			token.value = ScanTagHandle(INPUT, canBeHandle);
			token.data = (token.value.empty() ? Tag::SECONDARY_HANDLE : Tag::PRIMARY_HANDLE);
			
			// is there a suffix?
			if(canBeHandle && INPUT.peek() == Keys::Tag) {
				// eat the indicator
				INPUT.get();
				token.params.push_back(ScanTagSuffix(INPUT));
				token.data = Tag::NAMED_HANDLE;
			}
278
279
280
281
282
283
284
285
286
287
288
289
		}

		m_tokens.push(token);
	}

	// PlainScalar
	void Scanner::ScanPlainScalar()
	{
		std::string scalar;

		// set up the scanning parameters
		ScanScalarParams params;
290
		params.end = (InFlowContext() ? Exp::EndScalarInFlow : Exp::EndScalar) || (Exp::BlankOrBreak + Exp::Comment);
291
		params.eatEnd = false;
292
		params.indent = (InFlowContext() ? 0 : GetTopIndent() + 1);
293
		params.fold = FOLD_FLOW;
294
295
		params.eatLeadingWhitespace = true;
		params.trimTrailingSpaces = true;
296
		params.chomp = STRIP;
297
298
299
300
		params.onDocIndicator = BREAK;
		params.onTabInIndentation = THROW;

		// insert a potential simple key
301
		InsertPotentialSimpleKey();
302

303
		Mark mark = INPUT.mark();
304
305
306
307
308
309
310
		scalar = ScanScalar(INPUT, params);

		// can have a simple key only if we ended the scalar by starting a new line
		m_simpleKeyAllowed = params.leadingSpaces;

		// finally, check and see if we ended on an illegal character
		//if(Exp::IllegalCharInScalar.Matches(INPUT))
311
		//	throw ParserException(INPUT.mark(), ErrorMsg::CHAR_IN_SCALAR);
312

313
		Token token(Token::SCALAR, mark);
314
315
316
317
318
319
320
321
322
		token.value = scalar;
		m_tokens.push(token);
	}

	// QuotedScalar
	void Scanner::ScanQuotedScalar()
	{
		std::string scalar;

323
324
		// peek at single or double quote (don't eat because we need to preserve (for the time being) the input position)
		char quote = INPUT.peek();
325
326
327
328
329
330
331
332
		bool single = (quote == '\'');

		// setup the scanning parameters
		ScanScalarParams params;
		params.end = (single ? RegEx(quote) && !Exp::EscSingleQuote : RegEx(quote));
		params.eatEnd = true;
		params.escape = (single ? '\'' : '\\');
		params.indent = 0;
333
		params.fold = FOLD_FLOW;
334
335
336
337
338
339
		params.eatLeadingWhitespace = true;
		params.trimTrailingSpaces = false;
		params.chomp = CLIP;
		params.onDocIndicator = THROW;

		// insert a potential simple key
340
		InsertPotentialSimpleKey();
341

342
		Mark mark = INPUT.mark();
343
344
345
346
347

		// now eat that opening quote
		INPUT.get();
		
		// and scan
348
349
350
		scalar = ScanScalar(INPUT, params);
		m_simpleKeyAllowed = false;

351
		Token token(Token::SCALAR, mark);
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
		token.value = scalar;
		m_tokens.push(token);
	}

	// BlockScalarToken
	// . These need a little extra processing beforehand.
	// . We need to scan the line where the indicator is (this doesn't count as part of the scalar),
	//   and then we need to figure out what level of indentation we'll be using.
	void Scanner::ScanBlockScalar()
	{
		std::string scalar;

		ScanScalarParams params;
		params.indent = 1;
		params.detectIndent = true;

		// eat block indicator ('|' or '>')
369
		Mark mark = INPUT.mark();
370
		char indicator = INPUT.get();
371
		params.fold = (indicator == Keys::FoldedScalar ? FOLD_BLOCK : DONT_FOLD);
372
373

		// eat chomping/indentation indicators
374
		params.chomp = CLIP;
375
376
377
378
379
380
381
382
383
		int n = Exp::Chomp.Match(INPUT);
		for(int i=0;i<n;i++) {
			char ch = INPUT.get();
			if(ch == '+')
				params.chomp = KEEP;
			else if(ch == '-')
				params.chomp = STRIP;
			else if(Exp::Digit.Matches(ch)) {
				if(ch == '0')
384
					throw ParserException(INPUT.mark(), ErrorMsg::ZERO_INDENT_IN_BLOCK);
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401

				params.indent = ch - '0';
				params.detectIndent = false;
			}
		}

		// now eat whitespace
		while(Exp::Blank.Matches(INPUT))
			INPUT.eat(1);

		// and comments to the end of the line
		if(Exp::Comment.Matches(INPUT))
			while(INPUT && !Exp::Break.Matches(INPUT))
				INPUT.eat(1);

		// if it's not a line break, then we ran into a bad character inline
		if(INPUT && !Exp::Break.Matches(INPUT))
402
			throw ParserException(INPUT.mark(), ErrorMsg::CHAR_IN_BLOCK);
403
404

		// set the initial indentation
405
406
		if(GetTopIndent() >= 0)
			params.indent += GetTopIndent();
407
408
409
410
411
412
413
414
415
416

		params.eatLeadingWhitespace = false;
		params.trimTrailingSpaces = false;
		params.onTabInIndentation = THROW;

		scalar = ScanScalar(INPUT, params);

		// simple keys always ok after block scalars (since we're gonna start a new line anyways)
		m_simpleKeyAllowed = true;

417
		Token token(Token::SCALAR, mark);
418
419
420
421
		token.value = scalar;
		m_tokens.push(token);
	}
}