"vscode:/vscode.git/clone" did not exist on "cc9d2a84bf3e49482416e0fcf1f8a5cc5d11730c"
scantoken.cpp 9.64 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
#include "crt.h"
#include "scanner.h"
#include "token.h"
#include "exceptions.h"
#include "exp.h"
#include "scanscalar.h"
#include <sstream>

namespace YAML
{
	///////////////////////////////////////////////////////////////////////
	// Specialization for scanning specific tokens

	// Directive
	// . Note: no semantic checking is done here (that's for the parser to do)
	void Scanner::ScanDirective()
	{
		std::string name;
		std::vector <std::string> params;

		// pop indents and simple keys
		PopIndentTo(-1);
		VerifyAllSimpleKeys();

		m_simpleKeyAllowed = false;

		// store pos and eat indicator
28
		Mark mark = INPUT.mark();
29
30
31
		INPUT.eat(1);

		// read name
32
		while(INPUT && !Exp::BlankOrBreak.Matches(INPUT))
33
34
35
36
37
38
39
40
41
			name += INPUT.get();

		// read parameters
		while(1) {
			// first get rid of whitespace
			while(Exp::Blank.Matches(INPUT))
				INPUT.eat(1);

			// break on newline or comment
42
			if(!INPUT || Exp::Break.Matches(INPUT) || Exp::Comment.Matches(INPUT))
43
44
45
46
				break;

			// now read parameter
			std::string param;
47
			while(INPUT && !Exp::BlankOrBreak.Matches(INPUT))
48
49
50
51
52
				param += INPUT.get();

			params.push_back(param);
		}
		
53
		Token token(TT_DIRECTIVE, mark);
54
55
56
57
58
59
60
61
		token.value = name;
		token.params = params;
		m_tokens.push(token);
	}

	// DocStart
	void Scanner::ScanDocStart()
	{
62
		PopIndentTo(-1);
63
64
65
66
		VerifyAllSimpleKeys();
		m_simpleKeyAllowed = false;

		// eat
67
		Mark mark = INPUT.mark();
68
		INPUT.eat(3);
69
		m_tokens.push(Token(TT_DOC_START, mark));
70
71
72
73
74
75
76
77
78
79
	}

	// DocEnd
	void Scanner::ScanDocEnd()
	{
		PopIndentTo(-1);
		VerifyAllSimpleKeys();
		m_simpleKeyAllowed = false;

		// eat
80
		Mark mark = INPUT.mark();
81
		INPUT.eat(3);
82
		m_tokens.push(Token(TT_DOC_END, mark));
83
84
85
86
87
88
89
90
91
92
93
	}

	// FlowStart
	void Scanner::ScanFlowStart()
	{
		// flows can be simple keys
		InsertSimpleKey();
		m_flowLevel++;
		m_simpleKeyAllowed = true;

		// eat
94
		Mark mark = INPUT.mark();
95
96
		char ch = INPUT.get();
		TOKEN_TYPE type = (ch == Keys::FlowSeqStart ? TT_FLOW_SEQ_START : TT_FLOW_MAP_START);
97
		m_tokens.push(Token(type, mark));
98
99
100
101
102
103
	}

	// FlowEnd
	void Scanner::ScanFlowEnd()
	{
		if(m_flowLevel == 0)
104
			throw ParserException(INPUT.mark(), ErrorMsg::FLOW_END);
105
106
107
108
109

		m_flowLevel--;
		m_simpleKeyAllowed = false;

		// eat
110
		Mark mark = INPUT.mark();
111
112
		char ch = INPUT.get();
		TOKEN_TYPE type = (ch == Keys::FlowSeqEnd ? TT_FLOW_SEQ_END : TT_FLOW_MAP_END);
113
		m_tokens.push(Token(type, mark));
114
115
116
117
118
119
120
121
	}

	// FlowEntry
	void Scanner::ScanFlowEntry()
	{
		m_simpleKeyAllowed = true;

		// eat
122
		Mark mark = INPUT.mark();
123
		INPUT.eat(1);
124
		m_tokens.push(Token(TT_FLOW_ENTRY, mark));
125
126
127
128
129
130
131
	}

	// BlockEntry
	void Scanner::ScanBlockEntry()
	{
		// we better be in the block context!
		if(m_flowLevel > 0)
132
			throw ParserException(INPUT.mark(), ErrorMsg::BLOCK_ENTRY);
133
134
135

		// can we put it here?
		if(!m_simpleKeyAllowed)
136
			throw ParserException(INPUT.mark(), ErrorMsg::BLOCK_ENTRY);
137

138
		PushIndentTo(INPUT.column(), true);
139
140
141
		m_simpleKeyAllowed = true;

		// eat
142
		Mark mark = INPUT.mark();
143
		INPUT.eat(1);
144
		m_tokens.push(Token(TT_BLOCK_ENTRY, mark));
145
146
147
148
149
150
151
152
	}

	// Key
	void Scanner::ScanKey()
	{
		// handle keys diffently in the block context (and manage indents)
		if(m_flowLevel == 0) {
			if(!m_simpleKeyAllowed)
153
				throw ParserException(INPUT.mark(), ErrorMsg::MAP_KEY);
154

155
			PushIndentTo(INPUT.column(), false);
156
157
158
159
160
161
162
163
164
		}

		// can only put a simple key here if we're in block context
		if(m_flowLevel == 0)
			m_simpleKeyAllowed = true;
		else
			m_simpleKeyAllowed = false;

		// eat
165
		Mark mark = INPUT.mark();
166
		INPUT.eat(1);
167
		m_tokens.push(Token(TT_KEY, mark));
168
169
170
171
172
173
174
175
176
177
178
179
180
	}

	// Value
	void Scanner::ScanValue()
	{
		// does this follow a simple key?
		if(m_isLastKeyValid) {
			// can't follow a simple key with another simple key (dunno why, though - it seems fine)
			m_simpleKeyAllowed = false;
		} else {
			// handle values diffently in the block context (and manage indents)
			if(m_flowLevel == 0) {
				if(!m_simpleKeyAllowed)
181
					throw ParserException(INPUT.mark(), ErrorMsg::MAP_VALUE);
182

183
				PushIndentTo(INPUT.column(), false);
184
185
186
187
188
189
190
191
192
193
			}

			// can only put a simple key here if we're in block context
			if(m_flowLevel == 0)
				m_simpleKeyAllowed = true;
			else
				m_simpleKeyAllowed = false;
		}

		// eat
194
		Mark mark = INPUT.mark();
195
		INPUT.eat(1);
196
		m_tokens.push(Token(TT_VALUE, mark));
197
198
199
200
201
202
203
204
205
206
207
208
209
210
	}

	// AnchorOrAlias
	void Scanner::ScanAnchorOrAlias()
	{
		bool alias;
		std::string name;

		// insert a potential simple key
		if(m_simpleKeyAllowed)
			InsertSimpleKey();
		m_simpleKeyAllowed = false;

		// eat the indicator
211
		Mark mark = INPUT.mark();
212
213
214
215
216
217
218
219
220
		char indicator = INPUT.get();
		alias = (indicator == Keys::Alias);

		// now eat the content
		while(Exp::AlphaNumeric.Matches(INPUT))
			name += INPUT.get();

		// we need to have read SOMETHING!
		if(name.empty())
221
			throw ParserException(INPUT.mark(), alias ? ErrorMsg::ALIAS_NOT_FOUND : ErrorMsg::ANCHOR_NOT_FOUND);
222
223

		// and needs to end correctly
224
		if(INPUT && !Exp::AnchorEnd.Matches(INPUT))
225
			throw ParserException(INPUT.mark(), alias ? ErrorMsg::CHAR_IN_ALIAS : ErrorMsg::CHAR_IN_ANCHOR);
226
227

		// and we're done
228
		Token token(alias ? TT_ALIAS : TT_ANCHOR, mark);
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
		token.value = name;
		m_tokens.push(token);
	}

	// Tag
	void Scanner::ScanTag()
	{
		std::string handle, suffix;

		// insert a potential simple key
		if(m_simpleKeyAllowed)
			InsertSimpleKey();
		m_simpleKeyAllowed = false;

		// eat the indicator
244
		Mark mark = INPUT.mark();
245
246
247
		handle += INPUT.get();

		// read the handle
248
		while(INPUT && INPUT.peek() != Keys::Tag && !Exp::BlankOrBreak.Matches(INPUT))
249
250
251
252
253
254
255
256
			handle += INPUT.get();

		// is there a suffix?
		if(INPUT.peek() == Keys::Tag) {
			// eat the indicator
			handle += INPUT.get();

			// then read it
257
			while(INPUT && !Exp::BlankOrBreak.Matches(INPUT))
258
259
260
261
262
263
264
				suffix += INPUT.get();
		} else {
			// this is a bit weird: we keep just the '!' as the handle and move the rest to the suffix
			suffix = handle.substr(1);
			handle = "!";
		}

265
		Token token(TT_TAG, mark);
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
		token.value = handle;
		token.params.push_back(suffix);
		m_tokens.push(token);
	}

	// PlainScalar
	void Scanner::ScanPlainScalar()
	{
		std::string scalar;

		// set up the scanning parameters
		ScanScalarParams params;
		params.end = (m_flowLevel > 0 ? Exp::EndScalarInFlow : Exp::EndScalar) || (Exp::BlankOrBreak + Exp::Comment);
		params.eatEnd = false;
		params.indent = (m_flowLevel > 0 ? 0 : m_indents.top() + 1);
		params.fold = true;
		params.eatLeadingWhitespace = true;
		params.trimTrailingSpaces = true;
		params.chomp = CLIP;
		params.onDocIndicator = BREAK;
		params.onTabInIndentation = THROW;

		// insert a potential simple key
		if(m_simpleKeyAllowed)
			InsertSimpleKey();

292
		Mark mark = INPUT.mark();
293
294
295
296
297
298
299
		scalar = ScanScalar(INPUT, params);

		// can have a simple key only if we ended the scalar by starting a new line
		m_simpleKeyAllowed = params.leadingSpaces;

		// finally, check and see if we ended on an illegal character
		//if(Exp::IllegalCharInScalar.Matches(INPUT))
300
		//	throw ParserException(INPUT.mark(), ErrorMsg::CHAR_IN_SCALAR);
301

302
		Token token(TT_SCALAR, mark);
303
304
305
306
307
308
309
310
311
		token.value = scalar;
		m_tokens.push(token);
	}

	// QuotedScalar
	void Scanner::ScanQuotedScalar()
	{
		std::string scalar;

312
313
		// peek at single or double quote (don't eat because we need to preserve (for the time being) the input position)
		char quote = INPUT.peek();
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
		bool single = (quote == '\'');

		// setup the scanning parameters
		ScanScalarParams params;
		params.end = (single ? RegEx(quote) && !Exp::EscSingleQuote : RegEx(quote));
		params.eatEnd = true;
		params.escape = (single ? '\'' : '\\');
		params.indent = 0;
		params.fold = true;
		params.eatLeadingWhitespace = true;
		params.trimTrailingSpaces = false;
		params.chomp = CLIP;
		params.onDocIndicator = THROW;

		// insert a potential simple key
		if(m_simpleKeyAllowed)
			InsertSimpleKey();

332
		Mark mark = INPUT.mark();
333
334
335
336
337

		// now eat that opening quote
		INPUT.get();
		
		// and scan
338
339
340
		scalar = ScanScalar(INPUT, params);
		m_simpleKeyAllowed = false;

341
		Token token(TT_SCALAR, mark);
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
		token.value = scalar;
		m_tokens.push(token);
	}

	// BlockScalarToken
	// . These need a little extra processing beforehand.
	// . We need to scan the line where the indicator is (this doesn't count as part of the scalar),
	//   and then we need to figure out what level of indentation we'll be using.
	void Scanner::ScanBlockScalar()
	{
		std::string scalar;

		ScanScalarParams params;
		params.indent = 1;
		params.detectIndent = true;

		// eat block indicator ('|' or '>')
359
		Mark mark = INPUT.mark();
360
361
362
363
364
365
366
367
368
369
370
371
372
		char indicator = INPUT.get();
		params.fold = (indicator == Keys::FoldedScalar);

		// eat chomping/indentation indicators
		int n = Exp::Chomp.Match(INPUT);
		for(int i=0;i<n;i++) {
			char ch = INPUT.get();
			if(ch == '+')
				params.chomp = KEEP;
			else if(ch == '-')
				params.chomp = STRIP;
			else if(Exp::Digit.Matches(ch)) {
				if(ch == '0')
373
					throw ParserException(INPUT.mark(), ErrorMsg::ZERO_INDENT_IN_BLOCK);
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390

				params.indent = ch - '0';
				params.detectIndent = false;
			}
		}

		// now eat whitespace
		while(Exp::Blank.Matches(INPUT))
			INPUT.eat(1);

		// and comments to the end of the line
		if(Exp::Comment.Matches(INPUT))
			while(INPUT && !Exp::Break.Matches(INPUT))
				INPUT.eat(1);

		// if it's not a line break, then we ran into a bad character inline
		if(INPUT && !Exp::Break.Matches(INPUT))
391
			throw ParserException(INPUT.mark(), ErrorMsg::CHAR_IN_BLOCK);
392
393
394
395
396
397
398
399
400
401
402
403
404
405

		// set the initial indentation
		if(m_indents.top() >= 0)
			params.indent += m_indents.top();

		params.eatLeadingWhitespace = false;
		params.trimTrailingSpaces = false;
		params.onTabInIndentation = THROW;

		scalar = ScanScalar(INPUT, params);

		// simple keys always ok after block scalars (since we're gonna start a new line anyways)
		m_simpleKeyAllowed = true;

406
		Token token(TT_SCALAR, mark);
407
408
409
410
		token.value = scalar;
		m_tokens.push(token);
	}
}