scanner.cpp 7.99 KB
Newer Older
Jesse Beder's avatar
Jesse Beder committed
1
2
#include "scanner.h"
#include "token.h"
Jesse Beder's avatar
Jesse Beder committed
3
#include "exceptions.h"
4
#include "exp.h"
Jesse Beder's avatar
Jesse Beder committed
5
#include <iostream>
Jesse Beder's avatar
Jesse Beder committed
6
7
8
9

namespace YAML
{
	Scanner::Scanner(std::istream& in)
Jesse Beder's avatar
Jesse Beder committed
10
11
		: INPUT(in), m_startedStream(false), m_endedStream(false), m_simpleKeyAllowed(false), m_flowLevel(0),
		m_line(0), m_column(0)
Jesse Beder's avatar
Jesse Beder committed
12
13
14
15
16
	{
	}

	Scanner::~Scanner()
	{
Jesse Beder's avatar
Jesse Beder committed
17
18
19
20
21
22
23
24
		while(!m_tokens.empty()) {
			delete m_tokens.front();
			m_tokens.pop();
		}

		// delete limbo tokens (they're here for RAII)
		for(std::set <Token *>::const_iterator it=m_limboTokens.begin();it!=m_limboTokens.end();++it)
			delete *it;
Jesse Beder's avatar
Jesse Beder committed
25
26
27
28
29
30
31
32
33
34
	}

	///////////////////////////////////////////////////////////////////////
	// Misc. helpers

	// GetChar
	// . Extracts a character from the stream and updates our position
	char Scanner::GetChar()
	{
		m_column++;
35
		char ch = INPUT.get();
Jesse Beder's avatar
Jesse Beder committed
36
		if(ch == '\n') {
37
			m_column = 0;
Jesse Beder's avatar
Jesse Beder committed
38
39
			m_line++;
		}
40
		return ch;
Jesse Beder's avatar
Jesse Beder committed
41
42
	}

43
44
45
46
47
48
49
50
51
52
	// GetChar
	// . Extracts 'n' characters from the stream and updates our position
	std::string Scanner::GetChar(int n)
	{
		std::string ret;
		for(int i=0;i<n;i++)
			ret += GetChar();
		return ret;
	}

Jesse Beder's avatar
Jesse Beder committed
53
54
55
	// Eat
	// . Eats 'n' characters and updates our position.
	void Scanner::Eat(int n)
Jesse Beder's avatar
Jesse Beder committed
56
	{
Jesse Beder's avatar
Jesse Beder committed
57
58
		for(int i=0;i<n;i++)
			GetChar();
Jesse Beder's avatar
Jesse Beder committed
59
60
	}

Jesse Beder's avatar
Jesse Beder committed
61
	// GetLineBreak
Jesse Beder's avatar
Jesse Beder committed
62
	// . Eats with no checking
Jesse Beder's avatar
Jesse Beder committed
63
	void Scanner::EatLineBreak()
Jesse Beder's avatar
Jesse Beder committed
64
	{
Jesse Beder's avatar
Jesse Beder committed
65
66
		Eat(1);
		m_column = 0;
Jesse Beder's avatar
Jesse Beder committed
67
68
69
70
71
72
73
74
75
	}

	// IsWhitespaceToBeEaten
	// . We can eat whitespace if:
	//   1. It's a space
	//   2. It's a tab, and we're either:
	//      a. In the flow context
	//      b. In the block context but not where a simple key could be allowed
	//         (i.e., not at the beginning of a line, or following '-', '?', or ':')
Jesse Beder's avatar
Jesse Beder committed
76
	bool Scanner::IsWhitespaceToBeEaten(char ch)
Jesse Beder's avatar
Jesse Beder committed
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
	{
		if(ch == ' ')
			return true;

		if(ch == '\t' && (m_flowLevel >= 0 || !m_simpleKeyAllowed))
			return true;

		return false;
	}

	// IsDocumentStart
	bool Scanner::IsDocumentStart()
	{
		// needs to be at the start of a new line
		if(m_column != 0)
			return false;

94
		return Exp::DocStart.Matches(INPUT);
Jesse Beder's avatar
Jesse Beder committed
95
96
97
98
99
100
101
102
103
	}

	// IsDocumentEnd
	bool Scanner::IsDocumentEnd()
	{
		// needs to be at the start of a new line
		if(m_column != 0)
			return false;

104
		return Exp::DocEnd.Matches(INPUT);
Jesse Beder's avatar
Jesse Beder committed
105
106
	}

Jesse Beder's avatar
Jesse Beder committed
107
108
109
	// IsBlockEntry
	bool Scanner::IsBlockEntry()
	{
110
		return Exp::BlockEntry.Matches(INPUT);
Jesse Beder's avatar
Jesse Beder committed
111
112
113
114
115
	}

	// IsKey
	bool Scanner::IsKey()
	{
116
		if(m_flowLevel > 0)
117
118
			return Exp::KeyInFlow.Matches(INPUT);
		return Exp::Key.Matches(INPUT);
Jesse Beder's avatar
Jesse Beder committed
119
120
121
122
123
	}

	// IsValue
	bool Scanner::IsValue()
	{
124
		if(m_flowLevel > 0)
125
126
			return Exp::ValueInFlow.Matches(INPUT);
		return Exp::Value.Matches(INPUT);
Jesse Beder's avatar
Jesse Beder committed
127
128
129
130
131
	}

	// IsPlainScalar
	bool Scanner::IsPlainScalar()
	{
132
		if(m_flowLevel > 0)
133
134
			return Exp::PlainScalarInFlow.Matches(INPUT);
		return Exp::PlainScalar.Matches(INPUT);
Jesse Beder's avatar
Jesse Beder committed
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
	}

	// ScanAndEnqueue
	// . Scans the token, then pushes it in the queue.
	// . Note: we also use a set of "limbo tokens", i.e., tokens
	//   that haven't yet been pushed. This way, if ScanToken()
	//   throws an exception, we'll be keeping track of 'pToken'
	//   somewhere, and it will be automatically cleaned up when
	//   the Scanner destructs.
	template <typename T> void Scanner::ScanAndEnqueue(T *pToken)
	{
		m_limboTokens.insert(pToken);
		m_tokens.push(ScanToken(pToken));
		m_limboTokens.erase(pToken);
	}

Jesse Beder's avatar
Jesse Beder committed
151
152
153
	///////////////////////////////////////////////////////////////////////
	// The main scanning function

Jesse Beder's avatar
Jesse Beder committed
154
	void Scanner::ScanNextToken()
Jesse Beder's avatar
Jesse Beder committed
155
	{
156
157
158
		if(m_endedStream)
			return;

Jesse Beder's avatar
Jesse Beder committed
159
		if(!m_startedStream)
Jesse Beder's avatar
Jesse Beder committed
160
			return ScanAndEnqueue(new StreamStartToken);
Jesse Beder's avatar
Jesse Beder committed
161
162

		ScanToNextToken();
163
		ValidateSimpleKey();
Jesse Beder's avatar
Jesse Beder committed
164
		PopIndentTo(m_column);
Jesse Beder's avatar
Jesse Beder committed
165
166

		if(INPUT.peek() == EOF)
Jesse Beder's avatar
Jesse Beder committed
167
			return ScanAndEnqueue(new StreamEndToken);
Jesse Beder's avatar
Jesse Beder committed
168

Jesse Beder's avatar
Jesse Beder committed
169
		// are we at a document token?
Jesse Beder's avatar
Jesse Beder committed
170
		if(IsDocumentStart())
Jesse Beder's avatar
Jesse Beder committed
171
			return ScanAndEnqueue(new DocumentStartToken);
Jesse Beder's avatar
Jesse Beder committed
172
173

		if(IsDocumentEnd())
Jesse Beder's avatar
Jesse Beder committed
174
			return ScanAndEnqueue(new DocumentEndToken);
Jesse Beder's avatar
Jesse Beder committed
175

Jesse Beder's avatar
Jesse Beder committed
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
		// are we at a flow start/end/entry?
		if(INPUT.peek() == Keys::FlowSeqStart)
			return ScanAndEnqueue(new FlowSeqStartToken);

		if(INPUT.peek() == Keys::FlowSeqEnd)
			return ScanAndEnqueue(new FlowSeqEndToken);
		
		if(INPUT.peek() == Keys::FlowMapStart)
			return ScanAndEnqueue(new FlowMapStartToken);
		
		if(INPUT.peek() == Keys::FlowMapEnd)
			return ScanAndEnqueue(new FlowMapEndToken);

		if(INPUT.peek() == Keys::FlowEntry)
			return ScanAndEnqueue(new FlowEntryToken);

		// block/map stuff?
		if(IsBlockEntry())
			return ScanAndEnqueue(new BlockEntryToken);

		if(IsKey())
			return ScanAndEnqueue(new KeyToken);

		if(IsValue())
			return ScanAndEnqueue(new ValueToken);

		// TODO: alias/anchor/tag

		// TODO: special scalars
		if(INPUT.peek() == Keys::LiteralScalar && m_flowLevel == 0)
			return;

		if(INPUT.peek() == Keys::FoldedScalar && m_flowLevel == 0)
			return;

211
212
		if(INPUT.peek() == '\'' || INPUT.peek() == '\"')
			return ScanAndEnqueue(new QuotedScalarToken);
Jesse Beder's avatar
Jesse Beder committed
213
214
215
216
217
218
219

		// plain scalars
		if(IsPlainScalar())
			return ScanAndEnqueue(new PlainScalarToken);

		// don't know what it is!
		throw UnknownToken();
Jesse Beder's avatar
Jesse Beder committed
220
221
222
223
224
225
226
227
	}

	// ScanToNextToken
	// . Eats input until we reach the next token-like thing.
	void Scanner::ScanToNextToken()
	{
		while(1) {
			// first eat whitespace
Jesse Beder's avatar
Jesse Beder committed
228
229
			while(IsWhitespaceToBeEaten(INPUT.peek()))
				Eat(1);
Jesse Beder's avatar
Jesse Beder committed
230
231

			// then eat a comment
Jesse Beder's avatar
Jesse Beder committed
232
			if(Exp::Comment.Matches(INPUT)) {
Jesse Beder's avatar
Jesse Beder committed
233
				// eat until line break
Jesse Beder's avatar
Jesse Beder committed
234
				while(INPUT && !Exp::Break.Matches(INPUT))
Jesse Beder's avatar
Jesse Beder committed
235
					Eat(1);
Jesse Beder's avatar
Jesse Beder committed
236
237
238
			}

			// if it's NOT a line break, then we're done!
Jesse Beder's avatar
Jesse Beder committed
239
			if(!Exp::Break.Matches(INPUT))
Jesse Beder's avatar
Jesse Beder committed
240
241
242
243
244
				break;

			// otherwise, let's eat the line break and keep going
			EatLineBreak();

Jesse Beder's avatar
Jesse Beder committed
245
246
247
			// oh yeah, and let's get rid of that simple key
			ValidateSimpleKey();

Jesse Beder's avatar
Jesse Beder committed
248
249
250
251
252
253
			// new line - we may be able to accept a simple key now
			if(m_flowLevel == 0)
				m_simpleKeyAllowed = true;
        }
	}

Jesse Beder's avatar
Jesse Beder committed
254
255
256
	// PushIndentTo
	// . Pushes an indentation onto the stack, and enqueues the
	//   proper token (sequence start or mapping start).
Jesse Beder's avatar
Jesse Beder committed
257
258
	// . Returns the token it generates (if any).
	Token *Scanner::PushIndentTo(int column, bool sequence)
Jesse Beder's avatar
Jesse Beder committed
259
260
261
	{
		// are we in flow?
		if(m_flowLevel > 0)
Jesse Beder's avatar
Jesse Beder committed
262
			return 0;
Jesse Beder's avatar
Jesse Beder committed
263
264
265

		// is this actually an indentation?
		if(column <= m_indents.top())
Jesse Beder's avatar
Jesse Beder committed
266
			return 0;
Jesse Beder's avatar
Jesse Beder committed
267
268
269
270
271
272
273

		// now push
		m_indents.push(column);
		if(sequence)
			m_tokens.push(new BlockSeqStartToken);
		else
			m_tokens.push(new BlockMapStartToken);
Jesse Beder's avatar
Jesse Beder committed
274
275

		return m_tokens.front();
Jesse Beder's avatar
Jesse Beder committed
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
	}

	// PopIndentTo
	// . Pops indentations off the stack until we reach 'column' indentation,
	//   and enqueues the proper token each time.
	void Scanner::PopIndentTo(int column)
	{
		// are we in flow?
		if(m_flowLevel > 0)
			return;

		// now pop away
		while(!m_indents.empty() && m_indents.top() > column) {
			m_indents.pop();
			m_tokens.push(new BlockEndToken);
		}
	}

294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
	// IncreaseFlowLevel
	void Scanner::IncreaseFlowLevel()
	{
		// TODO: Push simple key
		m_flowLevel++;
	}

	// DecreaseFlowLevel
	void Scanner::DecreaseFlowLevel()
	{
		if(m_flowLevel > 0) {
			m_flowLevel--;
			// TODO: Pop simple key
		}
	}

310
311
312
	// GetNextToken
	// . Returns the next token on the queue, and scans if only we need to.
	Token *Scanner::GetNextToken()
Jesse Beder's avatar
Jesse Beder committed
313
	{
314
		while(1) {
315
			Token *pToken = 0;
Jesse Beder's avatar
Jesse Beder committed
316

317
318
319
			// is there a token in the queue?
			if(!m_tokens.empty())
				pToken = m_tokens.front();
Jesse Beder's avatar
Jesse Beder committed
320

321
322
323
			// ... that's possible
			// (here's where we clean up the impossible tokens)
			if(pToken && !pToken->isPossible) {
Jesse Beder's avatar
Jesse Beder committed
324
325
				m_tokens.pop();
				delete pToken;
326
				continue;
Jesse Beder's avatar
Jesse Beder committed
327
			}
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359

			// and valid
			if(pToken && !pToken->isValid)
				pToken = 0;

			// then that's what we want
			if(pToken) {
				m_tokens.pop();
				return pToken;
			}

			// no token? maybe we've actually finished
			if(m_endedStream)
				break;

			// no? then scan...
			ScanNextToken();
		}

		return 0;
	}

	// temporary function for testing
	void Scanner::Scan()
	{
		while(1) {
			Token *pToken = GetNextToken();
			if(!pToken)
				break;

			std::cout << typeid(*pToken).name() << ": " << *pToken << std::endl;
			delete pToken;
Jesse Beder's avatar
Jesse Beder committed
360
		}
Jesse Beder's avatar
Jesse Beder committed
361
362
	}
}