Skip to content

GitLab

Commit ed6c2947 authored by Jesse Beder's avatar Jesse Beder
Browse files

Added a peek token command (for the parser to use).

parent 07d4cac4
Hide whitespace changes
Inline Side-by-side
Showing with 108 additions and 112 deletions
document.cpp
View file @ ed6c2947
......@@ -5,6 +5,9 @@
#include "exceptions.h"
#include <fstream>
#include <iostream>
#include "token.h"
namespace YAML
{
Document::Document(): m_pRoot(0)
......@@ -34,10 +37,16 @@ namespace YAML
std::ifstream fin(fileName.c_str());
Scanner scanner(fin);
scanner.Scan();
// scan and output, for now
while(1) {
Token *pToken = scanner.GetNextToken();
if(!pToken)
break;
std::cout << typeid(*pToken).name() << ": " << *pToken << std::endl;
delete pToken;
}
getchar();
// if(!scanner)
// return;
// m_pRoot = parser.ReadNextNode();
}
......
exceptions.h
View file @ ed6c2947
......@@ -5,33 +5,34 @@
namespace YAML
{
class Exception: public std::exception {};
class ScannerException: public Exception {};
class UnknownToken: public Exception {};
class IllegalBlockEntry: public Exception {};
class IllegalMapKey: public Exception {};
class IllegalMapValue: public Exception {};
class IllegalScalar: public Exception {};
class IllegalTabInIndentation: public Exception {};
class IllegalFlowEnd: public Exception {};
class IllegalDocIndicator: public Exception {};
class IllegalEOF: public Exception {};
class RequiredSimpleKeyNotFound: public Exception {};
class ZeroIndentationInBlockScalar: public Exception {};
class UnexpectedCharacterInBlockScalar: public Exception {};
class AnchorNotFound: public Exception {};
class IllegalCharacterInAnchor: public Exception {};
class UnknownToken: public ScannerException {};
class IllegalBlockEntry: public ScannerException {};
class IllegalMapKey: public ScannerException {};
class IllegalMapValue: public ScannerException {};
class IllegalScalar: public ScannerException {};
class IllegalTabInIndentation: public ScannerException {};
class IllegalFlowEnd: public ScannerException {};
class IllegalDocIndicator: public ScannerException {};
class IllegalEOF: public ScannerException {};
class RequiredSimpleKeyNotFound: public ScannerException {};
class ZeroIndentationInBlockScalar: public ScannerException {};
class UnexpectedCharacterInBlockScalar: public ScannerException {};
class AnchorNotFound: public ScannerException {};
class IllegalCharacterInAnchor: public ScannerException {};
class UnknownEscapeSequence: public Exception {
class UnknownEscapeSequence: public ScannerException {
public:
UnknownEscapeSequence(char ch_): ch(ch_) {}
char ch;
};
class NonHexNumber: public Exception {
class NonHexNumber: public ScannerException {
public:
NonHexNumber(char ch_): ch(ch_) {}
char ch;
};
class InvalidUnicode: public Exception {
class InvalidUnicode: public ScannerException {
public:
InvalidUnicode(unsigned value_): value(value_) {}
unsigned value;
......
scanner.cpp
View file @ ed6c2947
......@@ -2,7 +2,6 @@
#include "token.h"
#include "exceptions.h"
#include "exp.h"
#include <iostream>
namespace YAML
{
......@@ -23,39 +22,51 @@ namespace YAML
delete *it;
}
///////////////////////////////////////////////////////////////////////
// Misc. helpers
// GetNextToken
// . Removes and returns the next token on the queue.
Token *Scanner::GetNextToken()
{
Token *pToken = PeekNextToken();
if(!m_tokens.empty())
m_tokens.pop();
return pToken;
}
// IsWhitespaceToBeEaten
// . We can eat whitespace if:
// 1. It's a space
// 2. It's a tab, and we're either:
// a. In the flow context
// b. In the block context but not where a simple key could be allowed
// (i.e., not at the beginning of a line, or following '-', '?', or ':')
bool Scanner::IsWhitespaceToBeEaten(char ch)
// PeekNextToken
// . Returns (but does not remove) the next token on the queue, and scans if only we need to.
Token *Scanner::PeekNextToken()
{
if(ch == ' ')
return true;
while(1) {
Token *pToken = 0;
if(ch == '\t' && (m_flowLevel >= 0 || !m_simpleKeyAllowed))
return true;
// is there a token in the queue?
if(!m_tokens.empty())
pToken = m_tokens.front();
return false;
}
// (here's where we clean up the impossible tokens)
if(pToken && pToken->status == TS_INVALID) {
m_tokens.pop();
delete pToken;
continue;
}
// ScanAndEnqueue
// . Scans the token, then pushes it in the queue.
// . Note: we also use a set of "limbo tokens", i.e., tokens
// that haven't yet been pushed. This way, if ScanToken()
// throws an exception, we'll be keeping track of 'pToken'
// somewhere, and it will be automatically cleaned up when
// the Scanner destructs.
template <typename T> void Scanner::ScanAndEnqueue(T *pToken)
{
m_limboTokens.insert(pToken);
m_tokens.push(ScanToken(pToken));
m_limboTokens.erase(pToken);
// on unverified tokens, we just have to wait
if(pToken && pToken->status == TS_UNVERIFIED)
pToken = 0;
// then that's what we want
if(pToken)
return pToken;
// no token? maybe we've actually finished
if(m_endedStream)
break;
// no? then scan...
ScanNextToken();
}
return 0;
}
// ScanNextToken
......@@ -166,7 +177,8 @@ namespace YAML
break;
// otherwise, let's eat the line break and keep going
INPUT.EatLineBreak();
int n = Exp::Break.Match(INPUT);
INPUT.Eat(n);
// oh yeah, and let's get rid of that simple key
VerifySimpleKey();
......@@ -177,6 +189,41 @@ namespace YAML
}
}
///////////////////////////////////////////////////////////////////////
// Misc. helpers
// IsWhitespaceToBeEaten
// . We can eat whitespace if:
// 1. It's a space
// 2. It's a tab, and we're either:
// a. In the flow context
// b. In the block context but not where a simple key could be allowed
// (i.e., not at the beginning of a line, or following '-', '?', or ':')
bool Scanner::IsWhitespaceToBeEaten(char ch)
{
if(ch == ' ')
return true;
if(ch == '\t' && (m_flowLevel >= 0 || !m_simpleKeyAllowed))
return true;
return false;
}
// ScanAndEnqueue
// . Scans the token, then pushes it in the queue.
// . Note: we also use a set of "limbo tokens", i.e., tokens
// that haven't yet been pushed. This way, if ScanToken()
// throws an exception, we'll be keeping track of 'pToken'
// somewhere, and it will be automatically cleaned up when
// the Scanner destructs.
template <typename T> void Scanner::ScanAndEnqueue(T *pToken)
{
m_limboTokens.insert(pToken);
m_tokens.push(ScanToken(pToken));
m_limboTokens.erase(pToken);
}
// PushIndentTo
// . Pushes an indentation onto the stack, and enqueues the
// proper token (sequence start or mapping start).
......@@ -216,56 +263,4 @@ namespace YAML
m_tokens.push(new BlockEndToken);
}
}
// GetNextToken
// . Returns the next token on the queue, and scans if only we need to.
Token *Scanner::GetNextToken()
{
while(1) {
Token *pToken = 0;
// is there a token in the queue?
if(!m_tokens.empty())
pToken = m_tokens.front();
// (here's where we clean up the impossible tokens)
if(pToken && pToken->status == TS_INVALID) {
m_tokens.pop();
delete pToken;
continue;
}
// on unverified tokens, we just have to wait
if(pToken && pToken->status == TS_UNVERIFIED)
pToken = 0;
// then that's what we want
if(pToken) {
m_tokens.pop();
return pToken;
}
// no token? maybe we've actually finished
if(m_endedStream)
break;
// no? then scan...
ScanNextToken();
}
return 0;
}
// temporary function for testing
void Scanner::Scan()
{
while(1) {
Token *pToken = GetNextToken();
if(!pToken)
break;
std::cout << typeid(*pToken).name() << ": " << *pToken << std::endl;
delete pToken;
}
}
}
scanner.h
View file @ ed6c2947
......@@ -19,7 +19,7 @@ namespace YAML
~Scanner();
Token *GetNextToken();
void Scan();
Token *PeekNextToken();
private:
// scanning
......
stream.cpp
View file @ ed6c2947
......@@ -32,12 +32,4 @@ namespace YAML
for(int i=0;i<n;i++)
GetChar();
}
// GetLineBreak
// . Eats with no checking
void Stream::EatLineBreak()
{
Eat(1);
column = 0;
}
}
stream.h
View file @ ed6c2947
......@@ -18,7 +18,6 @@ namespace YAML
char GetChar();
std::string GetChar(int n);
void Eat(int n = 1);
void EatLineBreak();
std::istream& input;
int line, column;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment