Skip to content

GitLab

"vscode:/vscode.git/clone" did not exist on "3be9fa97d6d3829c0678265b3fdb3563f6bad101"
Commit 2b862892 authored by Jesse Beder's avatar Jesse Beder
Browse files

Moved the three scalar token scanning functions back to scantoken.cpp, so... 

Moved the three scalar token scanning functions back to scantoken.cpp, so scanscalar.cpp now only has the main scalar scanning function.
Renamed ScanScalarInfo to ScanScalarParams.
parent 0683cbf8
Hide whitespace changes
Inline Side-by-side
Showing with 252 additions and 161 deletions
scanscalar.cpp
View file @ 2b862892
......@@ -7,39 +7,48 @@
namespace YAML
{
// ScanScalar
std::string ScanScalar(Stream& INPUT, ScanScalarInfo& info)
// . This is where the scalar magic happens.
//
// . We do the scanning in three phases:
// 1. Scan until newline
// 2. Eat newline
// 3. Scan leading blanks.
//
// . Depending on the parameters given, we store or stop
// and different places in the above flow.
std::string ScanScalar(Stream& INPUT, ScanScalarParams& params)
{
bool foundNonEmptyLine = false;
bool emptyLine = false, moreIndented = false;
std::string scalar;
info.leadingSpaces = false;
params.leadingSpaces = false;
while(INPUT) {
// ********************************
// Phase #1: scan until line ending
while(!info.end.Matches(INPUT) && !Exp::Break.Matches(INPUT)) {
while(!params.end.Matches(INPUT) && !Exp::Break.Matches(INPUT)) {
if(INPUT.peek() == EOF)
break;
// document indicator?
if(INPUT.column == 0 && Exp::DocIndicator.Matches(INPUT)) {
if(info.onDocIndicator == BREAK)
if(params.onDocIndicator == BREAK)
break;
else if(info.onDocIndicator == THROW)
else if(params.onDocIndicator == THROW)
throw IllegalDocIndicator();
}
foundNonEmptyLine = true;
// escaped newline? (only if we're escaping on slash)
if(info.escape == '\\' && Exp::EscBreak.Matches(INPUT)) {
if(params.escape == '\\' && Exp::EscBreak.Matches(INPUT)) {
int n = Exp::EscBreak.Match(INPUT);
INPUT.Eat(n);
continue;
}
// escape this?
if(INPUT.peek() == info.escape) {
if(INPUT.peek() == params.escape) {
scalar += Exp::Escape(INPUT);
continue;
}
......@@ -50,19 +59,19 @@ namespace YAML
// eof? if we're looking to eat something, then we throw
if(INPUT.peek() == EOF) {
if(info.eatEnd)
if(params.eatEnd)
throw IllegalEOF();
break;
}
// doc indicator?
if(info.onDocIndicator == BREAK && INPUT.column == 0 && Exp::DocIndicator.Matches(INPUT))
if(params.onDocIndicator == BREAK && INPUT.column == 0 && Exp::DocIndicator.Matches(INPUT))
break;
// are we done via character match?
int n = info.end.Match(INPUT);
int n = params.end.Match(INPUT);
if(n >= 0) {
if(info.eatEnd)
if(params.eatEnd)
INPUT.Eat(n);
break;
}
......@@ -76,20 +85,20 @@ namespace YAML
// Phase #3: scan initial spaces
// first the required indentation
while(INPUT.peek() == ' ' && (INPUT.column < info.indent || (info.detectIndent && !foundNonEmptyLine)))
while(INPUT.peek() == ' ' && (INPUT.column < params.indent || (params.detectIndent && !foundNonEmptyLine)))
INPUT.Eat(1);
// update indent if we're auto-detecting
if(info.detectIndent && !foundNonEmptyLine)
info.indent = std::max(info.indent, INPUT.column);
if(params.detectIndent && !foundNonEmptyLine)
params.indent = std::max(params.indent, INPUT.column);
// and then the rest of the whitespace
while(Exp::Blank.Matches(INPUT)) {
// we check for tabs that masquerade as indentation
if(INPUT.peek() == '\t'&& INPUT.column < info.indent && info.onTabInIndentation == THROW)
if(INPUT.peek() == '\t'&& INPUT.column < params.indent && params.onTabInIndentation == THROW)
throw IllegalTabInIndentation();
if(!info.eatLeadingWhitespace)
if(!params.eatLeadingWhitespace)
break;
INPUT.Eat(1);
......@@ -101,7 +110,7 @@ namespace YAML
// TODO: for block scalars, we always start with a newline, so we should fold OR keep that
if(info.fold && !emptyLine && !nextEmptyLine && !moreIndented && !nextMoreIndented)
if(params.fold && !emptyLine && !nextEmptyLine && !moreIndented && !nextMoreIndented)
scalar += " ";
else
scalar += "\n";
......@@ -110,146 +119,27 @@ namespace YAML
moreIndented = nextMoreIndented;
// are we done via indentation?
if(!emptyLine && INPUT.column < info.indent) {
info.leadingSpaces = true;
if(!emptyLine && INPUT.column < params.indent) {
params.leadingSpaces = true;
break;
}
}
// post-processing
if(info.trimTrailingSpaces) {
if(params.trimTrailingSpaces) {
unsigned pos = scalar.find_last_not_of(' ');
if(pos < scalar.size())
scalar.erase(pos + 1);
}
if(info.chomp <= 0) {
if(params.chomp <= 0) {
unsigned pos = scalar.find_last_not_of('\n');
if(info.chomp == 0 && pos + 1 < scalar.size())
if(params.chomp == 0 && pos + 1 < scalar.size())
scalar.erase(pos + 2);
else if(info.chomp == -1 && pos < scalar.size())
else if(params.chomp == -1 && pos < scalar.size())
scalar.erase(pos + 1);
}
return scalar;
}
// PlainScalarToken
template <> PlainScalarToken *Scanner::ScanToken(PlainScalarToken *pToken)
{
// set up the scanning parameters
ScanScalarInfo info;
info.end = (m_flowLevel > 0 ? Exp::EndScalarInFlow : Exp::EndScalar) || (RegEx(' ') + Exp::Comment);
info.eatEnd = false;
info.indent = (m_flowLevel > 0 ? 0 : m_indents.top() + 1);
info.fold = true;
info.eatLeadingWhitespace = true;
info.trimTrailingSpaces = true;
info.chomp = CLIP;
info.onDocIndicator = BREAK;
info.onTabInIndentation = THROW;
// insert a potential simple key
if(m_simpleKeyAllowed)
InsertSimpleKey();
pToken->value = ScanScalar(INPUT, info);
// can have a simple key only if we ended the scalar by starting a new line
m_simpleKeyAllowed = info.leadingSpaces;
// finally, we can't have any colons in a scalar, so if we ended on a colon, there
// had better be a break after it
if(Exp::IllegalColonInScalar.Matches(INPUT))
throw IllegalScalar();
return pToken;
}
// QuotedScalarToken
template <> QuotedScalarToken *Scanner::ScanToken(QuotedScalarToken *pToken)
{
// eat single or double quote
char quote = INPUT.GetChar();
pToken->single = (quote == '\'');
// setup the scanning parameters
ScanScalarInfo info;
info.end = (pToken->single ? RegEx(quote) && !Exp::EscSingleQuote : RegEx(quote));
info.eatEnd = true;
info.escape = (pToken->single ? '\'' : '\\');
info.indent = 0;
info.fold = true;
info.eatLeadingWhitespace = true;
info.trimTrailingSpaces = false;
info.chomp = CLIP;
info.onDocIndicator = THROW;
// insert a potential simple key
if(m_simpleKeyAllowed)
InsertSimpleKey();
pToken->value = ScanScalar(INPUT, info);
m_simpleKeyAllowed = false;
return pToken;
}
// BlockScalarToken
// . These need a little extra processing beforehand.
// . We need to scan the line where the indicator is (this doesn't count as part of the scalar),
// and then we need to figure out what level of indentation we'll be using.
template <> BlockScalarToken *Scanner::ScanToken(BlockScalarToken *pToken)
{
ScanScalarInfo info;
info.indent = 1;
info.detectIndent = true;
// eat block indicator ('|' or '>')
char indicator = INPUT.GetChar();
info.fold = (indicator == Keys::FoldedScalar);
// eat chomping/indentation indicators
int n = Exp::Chomp.Match(INPUT);
for(int i=0;i<n;i++) {
char ch = INPUT.GetChar();
if(ch == '+')
info.chomp = KEEP;
else if(ch == '-')
info.chomp = STRIP;
else if(Exp::Digit.Matches(ch)) {
info.indent = ch - '0';
info.detectIndent = false;
if(info.indent == 0)
throw ZeroIndentationInBlockScalar();
}
}
// now eat whitespace
while(Exp::Blank.Matches(INPUT))
INPUT.Eat(1);
// and comments to the end of the line
if(Exp::Comment.Matches(INPUT))
while(INPUT && !Exp::Break.Matches(INPUT))
INPUT.Eat(1);
// if it's not a line break, then we ran into a bad character inline
if(INPUT && !Exp::Break.Matches(INPUT))
throw UnexpectedCharacterInBlockScalar();
// set the initial indentation
if(m_indents.top() >= 0)
info.indent += m_indents.top();
info.eatLeadingWhitespace = false;
info.trimTrailingSpaces = false;
info.onTabInIndentation = THROW;
pToken->value = ScanScalar(INPUT, info);
// simple keys always ok after block scalars (since we're gonna start a new line anyways)
m_simpleKeyAllowed = true;
return pToken;
}
}
scanscalar.h
View file @ 2b862892
......@@ -9,8 +9,8 @@ namespace YAML
enum CHOMP { STRIP = -1, CLIP, KEEP };
enum ACTION { NONE, BREAK, THROW };
struct ScanScalarInfo {
ScanScalarInfo(): eatEnd(false), indent(0), detectIndent(false), eatLeadingWhitespace(0), escape(0), fold(false),
struct ScanScalarParams {
ScanScalarParams(): eatEnd(false), indent(0), detectIndent(false), eatLeadingWhitespace(0), escape(0), fold(false),
trimTrailingSpaces(0), chomp(CLIP), onDocIndicator(NONE), onTabInIndentation(NONE), leadingSpaces(false) {}
// input:
......@@ -31,5 +31,5 @@ namespace YAML
bool leadingSpaces;
};
std::string ScanScalar(Stream& INPUT, ScanScalarInfo& info);
std::string ScanScalar(Stream& INPUT, ScanScalarParams& info);
}
scantoken.cpp
View file @ 2b862892
......@@ -2,6 +2,7 @@
#include "token.h"
#include "exceptions.h"
#include "exp.h"
#include "scanscalar.h"
namespace YAML
{
......@@ -220,4 +221,124 @@ namespace YAML
pToken->value = tag;
return pToken;
}
// PlainScalarToken
template <> PlainScalarToken *Scanner::ScanToken(PlainScalarToken *pToken)
{
// set up the scanning parameters
ScanScalarParams params;
params.end = (m_flowLevel > 0 ? Exp::EndScalarInFlow : Exp::EndScalar) || (RegEx(' ') + Exp::Comment);
params.eatEnd = false;
params.indent = (m_flowLevel > 0 ? 0 : m_indents.top() + 1);
params.fold = true;
params.eatLeadingWhitespace = true;
params.trimTrailingSpaces = true;
params.chomp = CLIP;
params.onDocIndicator = BREAK;
params.onTabInIndentation = THROW;
// insert a potential simple key
if(m_simpleKeyAllowed)
InsertSimpleKey();
pToken->value = ScanScalar(INPUT, params);
// can have a simple key only if we ended the scalar by starting a new line
m_simpleKeyAllowed = params.leadingSpaces;
// finally, we can't have any colons in a scalar, so if we ended on a colon, there
// had better be a break after it
if(Exp::IllegalColonInScalar.Matches(INPUT))
throw IllegalScalar();
return pToken;
}
// QuotedScalarToken
template <> QuotedScalarToken *Scanner::ScanToken(QuotedScalarToken *pToken)
{
// eat single or double quote
char quote = INPUT.GetChar();
pToken->single = (quote == '\'');
// setup the scanning parameters
ScanScalarParams params;
params.end = (pToken->single ? RegEx(quote) && !Exp::EscSingleQuote : RegEx(quote));
params.eatEnd = true;
params.escape = (pToken->single ? '\'' : '\\');
params.indent = 0;
params.fold = true;
params.eatLeadingWhitespace = true;
params.trimTrailingSpaces = false;
params.chomp = CLIP;
params.onDocIndicator = THROW;
// insert a potential simple key
if(m_simpleKeyAllowed)
InsertSimpleKey();
pToken->value = ScanScalar(INPUT, params);
m_simpleKeyAllowed = false;
return pToken;
}
// BlockScalarToken
// . These need a little extra processing beforehand.
// . We need to scan the line where the indicator is (this doesn't count as part of the scalar),
// and then we need to figure out what level of indentation we'll be using.
template <> BlockScalarToken *Scanner::ScanToken(BlockScalarToken *pToken)
{
ScanScalarParams params;
params.indent = 1;
params.detectIndent = true;
// eat block indicator ('|' or '>')
char indicator = INPUT.GetChar();
params.fold = (indicator == Keys::FoldedScalar);
// eat chomping/indentation indicators
int n = Exp::Chomp.Match(INPUT);
for(int i=0;i<n;i++) {
char ch = INPUT.GetChar();
if(ch == '+')
params.chomp = KEEP;
else if(ch == '-')
params.chomp = STRIP;
else if(Exp::Digit.Matches(ch)) {
if(ch == '0')
throw ZeroIndentationInBlockScalar();
params.indent = ch - '0';
params.detectIndent = false;
}
}
// now eat whitespace
while(Exp::Blank.Matches(INPUT))
INPUT.Eat(1);
// and comments to the end of the line
if(Exp::Comment.Matches(INPUT))
while(INPUT && !Exp::Break.Matches(INPUT))
INPUT.Eat(1);
// if it's not a line break, then we ran into a bad character inline
if(INPUT && !Exp::Break.Matches(INPUT))
throw UnexpectedCharacterInBlockScalar();
// set the initial indentation
if(m_indents.top() >= 0)
params.indent += m_indents.top();
params.eatLeadingWhitespace = false;
params.trimTrailingSpaces = false;
params.onTabInIndentation = THROW;
pToken->value = ScanScalar(INPUT, params);
// simple keys always ok after block scalars (since we're gonna start a new line anyways)
m_simpleKeyAllowed = true;
return pToken;
}
}
test.yaml
View file @ 2b862892
---
- here's a key: value
here's the first block: |
after the block: value
and here's a block: |-
What's going on?
How are you doing?
Here's some code:
#include <iostream>
int main()
{
std::cout << "Hello World!\n";
}
I'm doing fine!
and last key: value
\ No newline at end of file
model:
file: data/models/compound.model
textures: data/materials/compound
rooms:
- name: "Room #1"
pos: [0, 0, 0]
size: [1000, 1000, 500]
height: 500
stairtype: none
display: []
pathfinding:
tilesize: 50
size: [24, 24]
map: |
-----------------------
-+++++++++++++++++++++-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+---------------------
-+---------------------
-+---------------------
-+---------------------
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+++++++++++++++++++++-
-----------------------
- name: Doorway
pos: [1000, 400, 0]
size: [50, 200, 500]
height: 500
stairtype: none
display: []
pathfinding:
tilesize: 50
size: [5, 9]
map: |
-----
-+++-
-----
-----
-----
-----
-----
-+++-
-----
- name: "Room #2"
pos: [1050, 0, 0]
size: [1000, 1000, 500]
height: 500
stairtype: none
display: []
pathfinding:
tilesize: 50
size: [24, 24]
map: |
-----------------------
-+++++++++++++++++++++-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
---------------------+-
---------------------+-
---------------------+-
---------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+-------------------+-
-+++++++++++++++++++++-
-----------------------
exits:
- room1: "Room #1"
room2: "Room #2"
dir: e
pos: [400, 600]
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment