Skip to content

GitLab

Commit 11706abb authored by Jesse Beder's avatar Jesse Beder
Browse files

Added simple keys. 

There's a bug (and question): should we test simple keys' validity BEFORE stuff or AFTER stuff?
parent 49a75b2d
Hide whitespace changes
Inline Side-by-side
Showing with 321 additions and 79 deletions
exceptions.h
View file @ 11706abb
......@@ -14,6 +14,8 @@ namespace YAML
class IllegalTabInScalar: public Exception {};
class DocIndicatorInQuote: public Exception {};
class EOFInQuote: public Exception {};
class RequiredSimpleKeyNotFound: public Exception {};
class UnknownEscapeSequence: public Exception {
public:
UnknownEscapeSequence(char ch_): ch(ch_) {}
......
exp.cpp
View file @ 11706abb
......@@ -46,18 +46,18 @@ namespace YAML
unsigned value = ParseHex(str);
// legal unicode?
if((value >= 0xD800 && value <= 0xDFFF) || value > 0x10FFFF)
throw InvalidUnicode(value);
// now break it up into chars
if(value <= 0x7F)
return Str(value);
else if(value <= 0x7FF)
return Str(0xC0 + (value >> 6)) + Str(0x80 + (value & 0x3F));
else if(value <= 0xFFFF)
return Str(0xE0 + (value >> 12)) + Str(0x80 + ((value >> 6) & 0x3F)) + Str(0x80 + (value & 0x3F));
else
return Str(0xF0 + (value >> 18)) + Str(0x80 + ((value >> 12) & 0x3F)) +
if((value >= 0xD800 && value <= 0xDFFF) || value > 0x10FFFF)
throw InvalidUnicode(value);
// now break it up into chars
if(value <= 0x7F)
return Str(value);
else if(value <= 0x7FF)
return Str(0xC0 + (value >> 6)) + Str(0x80 + (value & 0x3F));
else if(value <= 0xFFFF)
return Str(0xE0 + (value >> 12)) + Str(0x80 + ((value >> 6) & 0x3F)) + Str(0x80 + (value & 0x3F));
else
return Str(0xF0 + (value >> 18)) + Str(0x80 + ((value >> 12) & 0x3F)) +
Str(0x80 + ((value >> 6) & 0x3F)) + Str(0x80 + (value & 0x3F));
}
......@@ -76,27 +76,27 @@ namespace YAML
// switch on escape character
char ch = in.get();
switch(ch) {
case '0': return "\0";
case 'a': return "\x07";
case 'b': return "\x08";
case 't':
case '\t': return "\x09";
case 'n': return "\x0A";
case 'v': return "\x0B";
case 'f': return "\x0C";
case 'r': return "\x0D";
case 'e': return "\x1B";
case ' ': return "\x20";
case '\"': return "\"";
case '\'': return "\'";
case '\\': return "\\";
case 'N': return "\xC2\x85"; // NEL (#x85)
case '_': return "\xC2\xA0"; // #xA0
case 'L': return "\xE2\x80\xA8"; // LS (#x2028)
case 'P': return "\xE2\x80\xA9"; // PS (#x2029)
case 'x': return Escape(in, length, 2);
case 'u': return Escape(in, length, 4);
switch(ch) {
case '0': return "\0";
case 'a': return "\x07";
case 'b': return "\x08";
case 't':
case '\t': return "\x09";
case 'n': return "\x0A";
case 'v': return "\x0B";
case 'f': return "\x0C";
case 'r': return "\x0D";
case 'e': return "\x1B";
case ' ': return "\x20";
case '\"': return "\"";
case '\'': return "\'";
case '\\': return "\\";
case 'N': return "\xC2\x85"; // NEL (#x85)
case '_': return "\xC2\xA0"; // #xA0
case 'L': return "\xE2\x80\xA8"; // LS (#x2028)
case 'P': return "\xE2\x80\xA9"; // PS (#x2029)
case 'x': return Escape(in, length, 2);
case 'u': return Escape(in, length, 4);
case 'U': return Escape(in, length, 8);
}
......
scanner.cpp
View file @ 11706abb
......@@ -7,7 +7,8 @@
namespace YAML
{
Scanner::Scanner(std::istream& in)
: INPUT(in), m_startedStream(false), m_endedStream(false), m_simpleKeyAllowed(false), m_flowLevel(0), m_column(0)
: INPUT(in), m_startedStream(false), m_endedStream(false), m_simpleKeyAllowed(false), m_flowLevel(0),
m_line(0), m_column(0)
{
}
......@@ -32,8 +33,10 @@ namespace YAML
{
m_column++;
char ch = INPUT.get();
if(ch == '\n')
if(ch == '\n') {
m_column = 0;
m_line++;
}
return ch;
}
......@@ -51,12 +54,8 @@ namespace YAML
// . Eats 'n' characters and updates our position.
void Scanner::Eat(int n)
{
for(int i=0;i<n;i++) {
m_column++;
char ch = INPUT.get();
if(ch == '\n')
m_column = 0;
}
for(int i=0;i<n;i++)
GetChar();
}
// GetLineBreak
......@@ -147,6 +146,17 @@ namespace YAML
m_limboTokens.insert(pToken);
m_tokens.push(ScanToken(pToken));
m_limboTokens.erase(pToken);
// then remove impossible tokens
std::queue <Token *> temp;
while(!m_tokens.empty()) {
Token *pToken = m_tokens.front();
m_tokens.pop();
if(pToken->isPossible)
temp.push(pToken);
}
m_tokens = temp;
}
///////////////////////////////////////////////////////////////////////
......@@ -243,6 +253,9 @@ namespace YAML
// otherwise, let's eat the line break and keep going
EatLineBreak();
// oh yeah, and let's get rid of that simple key
ValidateSimpleKey();
// new line - we may be able to accept a simple key now
if(m_flowLevel == 0)
m_simpleKeyAllowed = true;
......@@ -252,15 +265,16 @@ namespace YAML
// PushIndentTo
// . Pushes an indentation onto the stack, and enqueues the
// proper token (sequence start or mapping start).
void Scanner::PushIndentTo(int column, bool sequence)
// . Returns the token it generates (if any).
Token *Scanner::PushIndentTo(int column, bool sequence)
{
// are we in flow?
if(m_flowLevel > 0)
return;
return 0;
// is this actually an indentation?
if(column <= m_indents.top())
return;
return 0;
// now push
m_indents.push(column);
......@@ -268,6 +282,8 @@ namespace YAML
m_tokens.push(new BlockSeqStartToken);
else
m_tokens.push(new BlockMapStartToken);
return m_tokens.front();
}
// PopIndentTo
......@@ -312,6 +328,9 @@ namespace YAML
while(!m_tokens.empty()) {
Token *pToken = m_tokens.front();
if(!pToken->isValid) // gotta wait on the invalid tokens - they might become valid!
break;
m_tokens.pop();
std::cout << typeid(*pToken).name() << ": " << *pToken << std::endl;
delete pToken;
......
scanner.h
View file @ 11706abb
......@@ -18,11 +18,15 @@ namespace YAML
void ScanNextToken();
void ScanToNextToken();
void PushIndentTo(int column, bool sequence);
Token *PushIndentTo(int column, bool sequence);
void PopIndentTo(int column);
void IncreaseFlowLevel();
void DecreaseFlowLevel();
void InsertSimpleKey();
bool ValidateSimpleKey();
void ValidateAllSimpleKeys();
void Scan();
private:
......@@ -41,6 +45,7 @@ namespace YAML
struct WhitespaceInfo {
WhitespaceInfo();
void AddBlank(char ch);
void AddBreak(const std::string& line);
std::string Join();
......@@ -49,13 +54,24 @@ namespace YAML
std::string whitespace, leadingBreaks, trailingBreaks;
};
struct SimpleKey {
SimpleKey(int pos_, int line_, int column_);
void Validate();
void Invalidate();
int pos, line, column;
bool required;
Token *pMapStart, *pKey;
};
template <typename T> void ScanAndEnqueue(T *pToken);
template <typename T> T *ScanToken(T *pToken);
private:
// the stream
std::istream& INPUT;
int m_column;
int m_line, m_column;
// the output (tokens)
std::queue <Token *> m_tokens;
......@@ -65,6 +81,7 @@ namespace YAML
bool m_startedStream, m_endedStream;
bool m_simpleKeyAllowed;
int m_flowLevel; // number of unclosed '[' and '{' indicators
std::stack <SimpleKey> m_simpleKeys;
std::stack <int> m_indents;
};
}
scantoken.cpp
View file @ 11706abb
......@@ -26,7 +26,7 @@ namespace YAML
m_column = 0;
PopIndentTo(-1);
// TODO: "reset simple keys"
ValidateAllSimpleKeys();
m_simpleKeyAllowed = false;
m_endedStream = true;
......@@ -38,8 +38,7 @@ namespace YAML
template <> DocumentStartToken *Scanner::ScanToken(DocumentStartToken *pToken)
{
PopIndentTo(m_column);
// TODO: "reset simple keys"
ValidateAllSimpleKeys();
m_simpleKeyAllowed = false;
// eat
......@@ -51,8 +50,7 @@ namespace YAML
template <> DocumentEndToken *Scanner::ScanToken(DocumentEndToken *pToken)
{
PopIndentTo(-1);
// TODO: "reset simple keys"
ValidateAllSimpleKeys();
m_simpleKeyAllowed = false;
// eat
......@@ -63,8 +61,8 @@ namespace YAML
// FlowSeqStartToken
template <> FlowSeqStartToken *Scanner::ScanToken(FlowSeqStartToken *pToken)
{
// TODO: "save simple key"
// flow sequences can be simple keys
InsertSimpleKey();
IncreaseFlowLevel();
m_simpleKeyAllowed = true;
......@@ -76,8 +74,8 @@ namespace YAML
// FlowMapStartToken
template <> FlowMapStartToken *Scanner::ScanToken(FlowMapStartToken *pToken)
{
// TODO: "save simple key"
// flow maps can be simple keys
InsertSimpleKey();
IncreaseFlowLevel();
m_simpleKeyAllowed = true;
......@@ -89,8 +87,7 @@ namespace YAML
// FlowSeqEndToken
template <> FlowSeqEndToken *Scanner::ScanToken(FlowSeqEndToken *pToken)
{
// TODO: "remove simple key"
// ValidateSimpleKey();
DecreaseFlowLevel();
m_simpleKeyAllowed = false;
......@@ -102,8 +99,7 @@ namespace YAML
// FlowMapEndToken
template <> FlowMapEndToken *Scanner::ScanToken(FlowMapEndToken *pToken)
{
// TODO: "remove simple key"
ValidateSimpleKey();
DecreaseFlowLevel();
m_simpleKeyAllowed = false;
......@@ -115,8 +111,7 @@ namespace YAML
// FlowEntryToken
template <> FlowEntryToken *Scanner::ScanToken(FlowEntryToken *pToken)
{
// TODO: "remove simple key"
ValidateSimpleKey();
m_simpleKeyAllowed = true;
// eat
......@@ -127,6 +122,8 @@ namespace YAML
// BlockEntryToken
template <> BlockEntryToken *Scanner::ScanToken(BlockEntryToken *pToken)
{
ValidateSimpleKey();
// we better be in the block context!
if(m_flowLevel == 0) {
// can we put it here?
......@@ -138,8 +135,6 @@ namespace YAML
// TODO: throw?
}
// TODO: "remove simple key"
m_simpleKeyAllowed = true;
// eat
......@@ -174,10 +169,13 @@ namespace YAML
// ValueToken
template <> ValueToken *Scanner::ScanToken(ValueToken *pToken)
{
// TODO: Is it a simple key?
if(false) {
// does this follow a simple key?
bool isValidKey = ValidateSimpleKey();
if(isValidKey) {
// can't follow a simple key with another simple key (dunno why, though - it seems fine)
m_simpleKeyAllowed = false;
} else {
// If not, ...
// are we in block context?
if(m_flowLevel == 0) {
if(!m_simpleKeyAllowed)
......@@ -185,13 +183,13 @@ namespace YAML
PushIndentTo(m_column, false);
}
}
// can only put a simple key here if we're in block context
if(m_flowLevel == 0)
m_simpleKeyAllowed = true;
else
m_simpleKeyAllowed = false;
// can only put a simple key here if we're in block context
if(m_flowLevel == 0)
m_simpleKeyAllowed = true;
else
m_simpleKeyAllowed = false;
}
// eat
Eat(1);
......@@ -205,8 +203,8 @@ namespace YAML
// and in-line whitespace (which is kept) separately.
template <> PlainScalarToken *Scanner::ScanToken(PlainScalarToken *pToken)
{
// TODO: "save simple key"
// insert a potential simple key
InsertSimpleKey();
m_simpleKeyAllowed = false;
// now eat and store the scalar
......@@ -255,6 +253,9 @@ namespace YAML
int n = Exp::Break.Match(INPUT);
std::string line = GetChar(n);
info.AddBreak(line);
// and we can't continue a simple key to the next line
ValidateSimpleKey();
}
}
......@@ -277,8 +278,8 @@ namespace YAML
// QuotedScalarToken
template <> QuotedScalarToken *Scanner::ScanToken(QuotedScalarToken *pToken)
{
// TODO: "save simple key"
// insert a potential simple key
InsertSimpleKey();
m_simpleKeyAllowed = false;
// eat single or double quote
......@@ -341,6 +342,9 @@ namespace YAML
int n = Exp::Break.Match(INPUT);
std::string line = GetChar(n);
info.AddBreak(line);
// and we can't continue a simple key to the next line
ValidateSimpleKey();
}
}
......
simplekey.cpp 0 → 100644
View file @ 11706abb
#include "scanner.h"
#include "token.h"
#include "exceptions.h"
#include "exp.h"
namespace YAML
{
Scanner::SimpleKey::SimpleKey(int pos_, int line_, int column_)
: pos(pos_), line(line_), column(column_), required(false), pMapStart(0), pKey(0)
{
}
void Scanner::SimpleKey::Validate()
{
if(pMapStart)
pMapStart->isValid = true;
if(pKey)
pKey->isValid = true;
}
void Scanner::SimpleKey::Invalidate()
{
if(required)
throw RequiredSimpleKeyNotFound();
if(pMapStart)
pMapStart->isPossible = false;
if(pKey)
pKey->isPossible = false;
}
// InsertSimpleKey
// . Adds a potential simple key to the queue,
// and saves it on a stack.
void Scanner::InsertSimpleKey()
{
SimpleKey key(INPUT.tellg(), m_line, m_column);
// first add a map start, if necessary
key.pMapStart = PushIndentTo(m_column, false);
if(key.pMapStart)
key.pMapStart->isValid = false;
// else
// key.required = true; // TODO: is this correct?
// then add the (now invalid) key
key.pKey = new KeyToken;
key.pKey->isValid = false;
m_tokens.push(key.pKey);
m_simpleKeys.push(key);
}
// ValidateSimpleKey
// . Determines whether the latest simple key to be added is valid,
// and if so, makes it valid.
bool Scanner::ValidateSimpleKey()
{
if(m_simpleKeys.empty())
return false;
// grab top key
SimpleKey key = m_simpleKeys.top();
m_simpleKeys.pop();
bool isValid = true;
// needs to be followed immediately by a value
if(m_flowLevel > 0 && !Exp::ValueInFlow.Matches(INPUT))
isValid = false;
if(m_flowLevel == 0 && !Exp::Value.Matches(INPUT))
isValid = false;
// also needs to be less than 1024 characters and inline
if(m_line != key.line || (int) INPUT.tellg() - key.pos > 1024)
isValid = false;
// invalidate key
if(isValid)
key.Validate();
else
key.Invalidate();
// In block style, remember that we've pushed an indent for this potential simple key (if it was starting).
// If it was invalid, then we need to pop it off.
// Note: we're guaranteed to be popping the right one (i.e., there couldn't have been anything in
// between) since keys have to be inline, and will be invalidated immediately on a newline.
if(!isValid && m_flowLevel == 0)
m_indents.pop();
return isValid;
}
void Scanner::ValidateAllSimpleKeys()
{
while(!m_simpleKeys.empty())
ValidateSimpleKey();
}
}
test.yaml
View file @ 11706abb
---
- milk and eggs
- [cheddar, american, swiss]
...
\ No newline at end of file
model:
file: data/models/compound.model
textures: data/materials/compound
rooms:
- name: "Room #1"
pos: [0, 0, 0]
size: [1000, 1000, 500]
height: 500
stairtype: none
display: []
pathfinding:
tilesize: 50
size: [24, 24]
map:
-----------------------
-+++++++++++++++++++++-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+---------------------
-+---------------------
-+---------------------
-+---------------------
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+++++++++++++++++++++-
-----------------------
- name: Doorway
pos: [1000, 400, 0]
size: [50, 200, 500]
height: 500
stairtype: none
display: []
pathfinding:
tilesize: 50
size: [5, 9]
map:
-----
-+++-
-----
-----
-----
-----
-----
-+++-
-----
- name: "Room #2"
pos: [1050, 0, 0]
size: [1000, 1000, 500]
height: 500
stairtype: none
display: []
pathfinding:
tilesize: 50
size: [24, 24]
map:
-----------------------
-+++++++++++++++++++++-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
---------------------+-
---------------------+-
---------------------+-
---------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+++++++++++++++++++++-
-----------------------
exits:
- room1: "Room #1"
room2: "Room #2"
dir: e
pos: [400, 600]
...
token.h
View file @ 11706abb
......@@ -5,10 +5,12 @@
namespace YAML
{
struct Token {
Token(): isValid(true), isPossible(true) {}
virtual ~Token() {}
virtual void Write(std::ostream& out) const {}
friend std::ostream& operator << (std::ostream& out, const Token& token) { token.Write(out); return out; }
bool isValid, isPossible;
};
struct StreamStartToken: public Token {};
......
yaml-reader.vcproj
View file @ 11706abb
......@@ -209,6 +209,10 @@
RelativePath=".\sequence.cpp"
>
</File>
<File
RelativePath=".\simplekey.cpp"
>
</File>
</Filter>
<Filter
Name="Header Files"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment