Skip to content

GitLab

Commit 2b862892 authored by Jesse Beder's avatar Jesse Beder
Browse files

Moved the three scalar token scanning functions back to scantoken.cpp, so... 

Moved the three scalar token scanning functions back to scantoken.cpp, so scanscalar.cpp now only has the main scalar scanning function.
Renamed ScanScalarInfo to ScanScalarParams.
parent 0683cbf8
Hide whitespace changes
Inline Side-by-side
Showing with 252 additions and 161 deletions
scanscalar.cpp
View file @ 2b862892
...@@ -7,39 +7,48 @@ ...@@ -7,39 +7,48 @@
namespace YAML namespace YAML
{ {
// ScanScalar // ScanScalar
std::string ScanScalar(Stream& INPUT, ScanScalarInfo& info) // . This is where the scalar magic happens.
//
// . We do the scanning in three phases:
// 1. Scan until newline
// 2. Eat newline
// 3. Scan leading blanks.
//
// . Depending on the parameters given, we store or stop
// and different places in the above flow.
std::string ScanScalar(Stream& INPUT, ScanScalarParams& params)
{ {
bool foundNonEmptyLine = false; bool foundNonEmptyLine = false;
bool emptyLine = false, moreIndented = false; bool emptyLine = false, moreIndented = false;
std::string scalar; std::string scalar;
info.leadingSpaces = false; params.leadingSpaces = false;
while(INPUT) { while(INPUT) {
// ******************************** // ********************************
// Phase #1: scan until line ending // Phase #1: scan until line ending
while(!info.end.Matches(INPUT) && !Exp::Break.Matches(INPUT)) { while(!params.end.Matches(INPUT) && !Exp::Break.Matches(INPUT)) {
if(INPUT.peek() == EOF) if(INPUT.peek() == EOF)
break; break;
// document indicator? // document indicator?
if(INPUT.column == 0 && Exp::DocIndicator.Matches(INPUT)) { if(INPUT.column == 0 && Exp::DocIndicator.Matches(INPUT)) {
if(info.onDocIndicator == BREAK) if(params.onDocIndicator == BREAK)
break; break;
else if(info.onDocIndicator == THROW) else if(params.onDocIndicator == THROW)
throw IllegalDocIndicator(); throw IllegalDocIndicator();
} }
foundNonEmptyLine = true; foundNonEmptyLine = true;
// escaped newline? (only if we're escaping on slash) // escaped newline? (only if we're escaping on slash)
if(info.escape == '\\' && Exp::EscBreak.Matches(INPUT)) { if(params.escape == '\\' && Exp::EscBreak.Matches(INPUT)) {
int n = Exp::EscBreak.Match(INPUT); int n = Exp::EscBreak.Match(INPUT);
INPUT.Eat(n); INPUT.Eat(n);
continue; continue;
} }
// escape this? // escape this?
if(INPUT.peek() == info.escape) { if(INPUT.peek() == params.escape) {
scalar += Exp::Escape(INPUT); scalar += Exp::Escape(INPUT);
continue; continue;
} }
...@@ -50,19 +59,19 @@ namespace YAML ...@@ -50,19 +59,19 @@ namespace YAML
// eof? if we're looking to eat something, then we throw // eof? if we're looking to eat something, then we throw
if(INPUT.peek() == EOF) { if(INPUT.peek() == EOF) {
if(info.eatEnd) if(params.eatEnd)
throw IllegalEOF(); throw IllegalEOF();
break; break;
} }
// doc indicator? // doc indicator?
if(info.onDocIndicator == BREAK && INPUT.column == 0 && Exp::DocIndicator.Matches(INPUT)) if(params.onDocIndicator == BREAK && INPUT.column == 0 && Exp::DocIndicator.Matches(INPUT))
break; break;
// are we done via character match? // are we done via character match?
int n = info.end.Match(INPUT); int n = params.end.Match(INPUT);
if(n >= 0) { if(n >= 0) {
if(info.eatEnd) if(params.eatEnd)
INPUT.Eat(n); INPUT.Eat(n);
break; break;
} }
...@@ -76,20 +85,20 @@ namespace YAML ...@@ -76,20 +85,20 @@ namespace YAML
// Phase #3: scan initial spaces // Phase #3: scan initial spaces
// first the required indentation // first the required indentation
while(INPUT.peek() == ' ' && (INPUT.column < info.indent || (info.detectIndent && !foundNonEmptyLine))) while(INPUT.peek() == ' ' && (INPUT.column < params.indent || (params.detectIndent && !foundNonEmptyLine)))
INPUT.Eat(1); INPUT.Eat(1);
// update indent if we're auto-detecting // update indent if we're auto-detecting
if(info.detectIndent && !foundNonEmptyLine) if(params.detectIndent && !foundNonEmptyLine)
info.indent = std::max(info.indent, INPUT.column); params.indent = std::max(params.indent, INPUT.column);
// and then the rest of the whitespace // and then the rest of the whitespace
while(Exp::Blank.Matches(INPUT)) { while(Exp::Blank.Matches(INPUT)) {
// we check for tabs that masquerade as indentation // we check for tabs that masquerade as indentation
if(INPUT.peek() == '\t'&& INPUT.column < info.indent && info.onTabInIndentation == THROW) if(INPUT.peek() == '\t'&& INPUT.column < params.indent && params.onTabInIndentation == THROW)
throw IllegalTabInIndentation(); throw IllegalTabInIndentation();
if(!info.eatLeadingWhitespace) if(!params.eatLeadingWhitespace)
break; break;
INPUT.Eat(1); INPUT.Eat(1);
...@@ -101,7 +110,7 @@ namespace YAML ...@@ -101,7 +110,7 @@ namespace YAML
// TODO: for block scalars, we always start with a newline, so we should fold OR keep that // TODO: for block scalars, we always start with a newline, so we should fold OR keep that
if(info.fold && !emptyLine && !nextEmptyLine && !moreIndented && !nextMoreIndented) if(params.fold && !emptyLine && !nextEmptyLine && !moreIndented && !nextMoreIndented)
scalar += " "; scalar += " ";
else else
scalar += "\n"; scalar += "\n";
...@@ -110,146 +119,27 @@ namespace YAML ...@@ -110,146 +119,27 @@ namespace YAML
moreIndented = nextMoreIndented; moreIndented = nextMoreIndented;
// are we done via indentation? // are we done via indentation?
if(!emptyLine && INPUT.column < info.indent) { if(!emptyLine && INPUT.column < params.indent) {
info.leadingSpaces = true; params.leadingSpaces = true;
break; break;
} }
} }
// post-processing // post-processing
if(info.trimTrailingSpaces) { if(params.trimTrailingSpaces) {
unsigned pos = scalar.find_last_not_of(' '); unsigned pos = scalar.find_last_not_of(' ');
if(pos < scalar.size()) if(pos < scalar.size())
scalar.erase(pos + 1); scalar.erase(pos + 1);
} }
if(info.chomp <= 0) { if(params.chomp <= 0) {
unsigned pos = scalar.find_last_not_of('\n'); unsigned pos = scalar.find_last_not_of('\n');
if(info.chomp == 0 && pos + 1 < scalar.size()) if(params.chomp == 0 && pos + 1 < scalar.size())
scalar.erase(pos + 2); scalar.erase(pos + 2);
else if(info.chomp == -1 && pos < scalar.size()) else if(params.chomp == -1 && pos < scalar.size())
scalar.erase(pos + 1); scalar.erase(pos + 1);
} }
return scalar; return scalar;
} }
// PlainScalarToken
template <> PlainScalarToken *Scanner::ScanToken(PlainScalarToken *pToken)
{
// set up the scanning parameters
ScanScalarInfo info;
info.end = (m_flowLevel > 0 ? Exp::EndScalarInFlow : Exp::EndScalar) || (RegEx(' ') + Exp::Comment);
info.eatEnd = false;
info.indent = (m_flowLevel > 0 ? 0 : m_indents.top() + 1);
info.fold = true;
info.eatLeadingWhitespace = true;
info.trimTrailingSpaces = true;
info.chomp = CLIP;
info.onDocIndicator = BREAK;
info.onTabInIndentation = THROW;
// insert a potential simple key
if(m_simpleKeyAllowed)
InsertSimpleKey();
pToken->value = ScanScalar(INPUT, info);
// can have a simple key only if we ended the scalar by starting a new line
m_simpleKeyAllowed = info.leadingSpaces;
// finally, we can't have any colons in a scalar, so if we ended on a colon, there
// had better be a break after it
if(Exp::IllegalColonInScalar.Matches(INPUT))
throw IllegalScalar();
return pToken;
}
// QuotedScalarToken
template <> QuotedScalarToken *Scanner::ScanToken(QuotedScalarToken *pToken)
{
// eat single or double quote
char quote = INPUT.GetChar();
pToken->single = (quote == '\'');
// setup the scanning parameters
ScanScalarInfo info;
info.end = (pToken->single ? RegEx(quote) && !Exp::EscSingleQuote : RegEx(quote));
info.eatEnd = true;
info.escape = (pToken->single ? '\'' : '\\');
info.indent = 0;
info.fold = true;
info.eatLeadingWhitespace = true;
info.trimTrailingSpaces = false;
info.chomp = CLIP;
info.onDocIndicator = THROW;
// insert a potential simple key
if(m_simpleKeyAllowed)
InsertSimpleKey();
pToken->value = ScanScalar(INPUT, info);
m_simpleKeyAllowed = false;
return pToken;
}
// BlockScalarToken
// . These need a little extra processing beforehand.
// . We need to scan the line where the indicator is (this doesn't count as part of the scalar),
// and then we need to figure out what level of indentation we'll be using.
template <> BlockScalarToken *Scanner::ScanToken(BlockScalarToken *pToken)
{
ScanScalarInfo info;
info.indent = 1;
info.detectIndent = true;
// eat block indicator ('|' or '>')
char indicator = INPUT.GetChar();
info.fold = (indicator == Keys::FoldedScalar);
// eat chomping/indentation indicators
int n = Exp::Chomp.Match(INPUT);
for(int i=0;i<n;i++) {
char ch = INPUT.GetChar();
if(ch == '+')
info.chomp = KEEP;
else if(ch == '-')
info.chomp = STRIP;
else if(Exp::Digit.Matches(ch)) {
info.indent = ch - '0';
info.detectIndent = false;
if(info.indent == 0)
throw ZeroIndentationInBlockScalar();
}
}
// now eat whitespace
while(Exp::Blank.Matches(INPUT))
INPUT.Eat(1);
// and comments to the end of the line
if(Exp::Comment.Matches(INPUT))
while(INPUT && !Exp::Break.Matches(INPUT))
INPUT.Eat(1);
// if it's not a line break, then we ran into a bad character inline
if(INPUT && !Exp::Break.Matches(INPUT))
throw UnexpectedCharacterInBlockScalar();
// set the initial indentation
if(m_indents.top() >= 0)
info.indent += m_indents.top();
info.eatLeadingWhitespace = false;
info.trimTrailingSpaces = false;
info.onTabInIndentation = THROW;
pToken->value = ScanScalar(INPUT, info);
// simple keys always ok after block scalars (since we're gonna start a new line anyways)
m_simpleKeyAllowed = true;
return pToken;
}
} }
scanscalar.h
View file @ 2b862892
...@@ -9,8 +9,8 @@ namespace YAML ...@@ -9,8 +9,8 @@ namespace YAML
enum CHOMP { STRIP = -1, CLIP, KEEP }; enum CHOMP { STRIP = -1, CLIP, KEEP };
enum ACTION { NONE, BREAK, THROW }; enum ACTION { NONE, BREAK, THROW };
struct ScanScalarInfo { struct ScanScalarParams {
ScanScalarInfo(): eatEnd(false), indent(0), detectIndent(false), eatLeadingWhitespace(0), escape(0), fold(false), ScanScalarParams(): eatEnd(false), indent(0), detectIndent(false), eatLeadingWhitespace(0), escape(0), fold(false),
trimTrailingSpaces(0), chomp(CLIP), onDocIndicator(NONE), onTabInIndentation(NONE), leadingSpaces(false) {} trimTrailingSpaces(0), chomp(CLIP), onDocIndicator(NONE), onTabInIndentation(NONE), leadingSpaces(false) {}
// input: // input:
...@@ -31,5 +31,5 @@ namespace YAML ...@@ -31,5 +31,5 @@ namespace YAML
bool leadingSpaces; bool leadingSpaces;
}; };
std::string ScanScalar(Stream& INPUT, ScanScalarInfo& info); std::string ScanScalar(Stream& INPUT, ScanScalarParams& info);
} }
scantoken.cpp
View file @ 2b862892
...@@ -2,6 +2,7 @@ ...@@ -2,6 +2,7 @@
#include "token.h" #include "token.h"
#include "exceptions.h" #include "exceptions.h"
#include "exp.h" #include "exp.h"
#include "scanscalar.h"
namespace YAML namespace YAML
{ {
...@@ -220,4 +221,124 @@ namespace YAML ...@@ -220,4 +221,124 @@ namespace YAML
pToken->value = tag; pToken->value = tag;
return pToken; return pToken;
} }
// PlainScalarToken
template <> PlainScalarToken *Scanner::ScanToken(PlainScalarToken *pToken)
{
// set up the scanning parameters
ScanScalarParams params;
params.end = (m_flowLevel > 0 ? Exp::EndScalarInFlow : Exp::EndScalar) || (RegEx(' ') + Exp::Comment);
params.eatEnd = false;
params.indent = (m_flowLevel > 0 ? 0 : m_indents.top() + 1);
params.fold = true;
params.eatLeadingWhitespace = true;
params.trimTrailingSpaces = true;
params.chomp = CLIP;
params.onDocIndicator = BREAK;
params.onTabInIndentation = THROW;
// insert a potential simple key
if(m_simpleKeyAllowed)
InsertSimpleKey();
pToken->value = ScanScalar(INPUT, params);
// can have a simple key only if we ended the scalar by starting a new line
m_simpleKeyAllowed = params.leadingSpaces;
// finally, we can't have any colons in a scalar, so if we ended on a colon, there
// had better be a break after it
if(Exp::IllegalColonInScalar.Matches(INPUT))
throw IllegalScalar();
return pToken;
}
// QuotedScalarToken
template <> QuotedScalarToken *Scanner::ScanToken(QuotedScalarToken *pToken)
{
// eat single or double quote
char quote = INPUT.GetChar();
pToken->single = (quote == '\'');
// setup the scanning parameters
ScanScalarParams params;
params.end = (pToken->single ? RegEx(quote) && !Exp::EscSingleQuote : RegEx(quote));
params.eatEnd = true;
params.escape = (pToken->single ? '\'' : '\\');
params.indent = 0;
params.fold = true;
params.eatLeadingWhitespace = true;
params.trimTrailingSpaces = false;
params.chomp = CLIP;
params.onDocIndicator = THROW;
// insert a potential simple key
if(m_simpleKeyAllowed)
InsertSimpleKey();
pToken->value = ScanScalar(INPUT, params);
m_simpleKeyAllowed = false;
return pToken;
}
// BlockScalarToken
// . These need a little extra processing beforehand.
// . We need to scan the line where the indicator is (this doesn't count as part of the scalar),
// and then we need to figure out what level of indentation we'll be using.
template <> BlockScalarToken *Scanner::ScanToken(BlockScalarToken *pToken)
{
ScanScalarParams params;
params.indent = 1;
params.detectIndent = true;
// eat block indicator ('|' or '>')
char indicator = INPUT.GetChar();
params.fold = (indicator == Keys::FoldedScalar);
// eat chomping/indentation indicators
int n = Exp::Chomp.Match(INPUT);
for(int i=0;i<n;i++) {
char ch = INPUT.GetChar();
if(ch == '+')
params.chomp = KEEP;
else if(ch == '-')
params.chomp = STRIP;
else if(Exp::Digit.Matches(ch)) {
if(ch == '0')
throw ZeroIndentationInBlockScalar();
params.indent = ch - '0';
params.detectIndent = false;
}
}
// now eat whitespace
while(Exp::Blank.Matches(INPUT))
INPUT.Eat(1);
// and comments to the end of the line
if(Exp::Comment.Matches(INPUT))
while(INPUT && !Exp::Break.Matches(INPUT))
INPUT.Eat(1);
// if it's not a line break, then we ran into a bad character inline
if(INPUT && !Exp::Break.Matches(INPUT))
throw UnexpectedCharacterInBlockScalar();
// set the initial indentation
if(m_indents.top() >= 0)
params.indent += m_indents.top();
params.eatLeadingWhitespace = false;
params.trimTrailingSpaces = false;
params.onTabInIndentation = THROW;
pToken->value = ScanScalar(INPUT, params);
// simple keys always ok after block scalars (since we're gonna start a new line anyways)
m_simpleKeyAllowed = true;
return pToken;
}
} }
test.yaml
View file @ 2b862892
--- ---
- here's a key: value model:
here's the first block: | file: data/models/compound.model
after the block: value textures: data/materials/compound
and here's a block: |- rooms:
What's going on? - name: "Room #1"
How are you doing? pos: [0, 0, 0]
Here's some code: size: [1000, 1000, 500]
height: 500
#include <iostream> stairtype: none
int main() display: []
{ pathfinding:
std::cout << "Hello World!\n"; tilesize: 50
} size: [24, 24]
map: |
I'm doing fine! -----------------------
and last key: value -+++++++++++++++++++++-
\ No newline at end of file -+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+---------------------
-+---------------------
-+---------------------
-+---------------------
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+++++++++++++++++++++-
-----------------------
- name: Doorway
pos: [1000, 400, 0]
size: [50, 200, 500]
height: 500
stairtype: none
display: []
pathfinding:
tilesize: 50
size: [5, 9]
map: |
-----
-+++-
-----
-----
-----
-----
-----
-+++-
-----
- name: "Room #2"
pos: [1050, 0, 0]
size: [1000, 1000, 500]
height: 500
stairtype: none
display: []
pathfinding:
tilesize: 50
size: [24, 24]
map: |
-----------------------
-+++++++++++++++++++++-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
---------------------+-
---------------------+-
---------------------+-
---------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+++++++++++++++++++++-
-----------------------
exits:
- room1: "Room #1"
room2: "Room #2"
dir: e
pos: [400, 600]
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment