Skip to content

GitLab

Commit 0683cbf8 authored by Jesse Beder's avatar Jesse Beder
Browse files

Mostly finished refactoring the scalar scanning.

parent 5f8252ee
Show whitespace changes
Inline Side-by-side
Showing with 192 additions and 378 deletions
exceptions.h
View file @ 0683cbf8
......@@ -11,10 +11,10 @@ namespace YAML
class IllegalMapKey: public Exception {};
class IllegalMapValue: public Exception {};
class IllegalScalar: public Exception {};
class IllegalTabInScalar: public Exception {};
class IllegalTabInIndentation: public Exception {};
class IllegalFlowEnd: public Exception {};
class DocIndicatorInQuote: public Exception {};
class EOFInQuote: public Exception {};
class IllegalDocIndicator: public Exception {};
class IllegalEOF: public Exception {};
class RequiredSimpleKeyNotFound: public Exception {};
class ZeroIndentationInBlockScalar: public Exception {};
class UnexpectedCharacterInBlockScalar: public Exception {};
......
exp.h
View file @ 0683cbf8
......@@ -25,6 +25,7 @@ namespace YAML
const RegEx DocStart = RegEx("---") + (BlankOrBreak || RegEx(EOF) || RegEx());
const RegEx DocEnd = RegEx("...") + (BlankOrBreak || RegEx(EOF) || RegEx());
const RegEx DocIndicator = DocStart || DocEnd;
const RegEx BlockEntry = RegEx('-') + (BlankOrBreak || RegEx(EOF));
const RegEx Key = RegEx('?'),
KeyInFlow = RegEx('?') + BlankOrBreak;
......
scanner.h
View file @ 0683cbf8
......@@ -19,19 +19,20 @@ namespace YAML
~Scanner();
Token *GetNextToken();
void Scan();
private:
// scanning
void ScanNextToken();
void ScanToNextToken();
Token *PushIndentTo(int column, bool sequence);
void PopIndentTo(int column);
// checking input
void InsertSimpleKey();
bool VerifySimpleKey();
void VerifyAllSimpleKeys();
void Scan();
private:
bool IsWhitespaceToBeEaten(char ch);
bool IsDocumentStart();
bool IsDocumentEnd();
......
scanscalar.cpp
View file @ 0683cbf8
......@@ -6,135 +6,138 @@
namespace YAML
{
//////////////////////////////////////////////////////////
// WhitespaceInfo
WhitespaceInfo::WhitespaceInfo(): leadingBlanks(false), fold(true), chomp(0), increment(0)
// ScanScalar
std::string ScanScalar(Stream& INPUT, ScanScalarInfo& info)
{
bool foundNonEmptyLine = false;
bool emptyLine = false, moreIndented = false;
std::string scalar;
info.leadingSpaces = false;
while(INPUT) {
// ********************************
// Phase #1: scan until line ending
while(!info.end.Matches(INPUT) && !Exp::Break.Matches(INPUT)) {
if(INPUT.peek() == EOF)
break;
// document indicator?
if(INPUT.column == 0 && Exp::DocIndicator.Matches(INPUT)) {
if(info.onDocIndicator == BREAK)
break;
else if(info.onDocIndicator == THROW)
throw IllegalDocIndicator();
}
void WhitespaceInfo::SetChompers(char ch)
{
if(ch == '+')
chomp = 1;
else if(ch == '-')
chomp = -1;
else if(Exp::Digit.Matches(ch)) {
increment = ch - '0';
if(increment == 0)
throw ZeroIndentationInBlockScalar();
foundNonEmptyLine = true;
// escaped newline? (only if we're escaping on slash)
if(info.escape == '\\' && Exp::EscBreak.Matches(INPUT)) {
int n = Exp::EscBreak.Match(INPUT);
INPUT.Eat(n);
continue;
}
// escape this?
if(INPUT.peek() == info.escape) {
scalar += Exp::Escape(INPUT);
continue;
}
void WhitespaceInfo::AddBlank(char ch)
{
if(!leadingBlanks)
whitespace += ch;
// otherwise, just add the damn character
scalar += INPUT.GetChar();
}
void WhitespaceInfo::AddBreak(const std::string& line)
{
// where to store this character?
if(!leadingBlanks) {
leadingBlanks = true;
whitespace = "";
leadingBreaks += line;
} else
trailingBreaks += line;
// eof? if we're looking to eat something, then we throw
if(INPUT.peek() == EOF) {
if(info.eatEnd)
throw IllegalEOF();
break;
}
std::string WhitespaceInfo::Join(bool lastLine)
{
std::string ret;
if(leadingBlanks) {
// fold line break?
if(fold && Exp::Break.Matches(leadingBreaks) && trailingBreaks.empty() && !lastLine)
ret = " ";
else if(!lastLine || chomp != -1)
ret = leadingBreaks;
if(!lastLine || chomp == 1)
ret += trailingBreaks;
leadingBlanks = false;
leadingBreaks = "";
trailingBreaks = "";
} else if(!whitespace.empty()) {
ret = whitespace;
whitespace = "";
// doc indicator?
if(info.onDocIndicator == BREAK && INPUT.column == 0 && Exp::DocIndicator.Matches(INPUT))
break;
// are we done via character match?
int n = info.end.Match(INPUT);
if(n >= 0) {
if(info.eatEnd)
INPUT.Eat(n);
break;
}
return ret;
// ********************************
// Phase #2: eat line ending
n = Exp::Break.Match(INPUT);
INPUT.Eat(n);
// ********************************
// Phase #3: scan initial spaces
// first the required indentation
while(INPUT.peek() == ' ' && (INPUT.column < info.indent || (info.detectIndent && !foundNonEmptyLine)))
INPUT.Eat(1);
// update indent if we're auto-detecting
if(info.detectIndent && !foundNonEmptyLine)
info.indent = std::max(info.indent, INPUT.column);
// and then the rest of the whitespace
while(Exp::Blank.Matches(INPUT)) {
// we check for tabs that masquerade as indentation
if(INPUT.peek() == '\t'&& INPUT.column < info.indent && info.onTabInIndentation == THROW)
throw IllegalTabInIndentation();
if(!info.eatLeadingWhitespace)
break;
INPUT.Eat(1);
}
// was this an empty line?
bool nextEmptyLine = Exp::Break.Matches(INPUT);
bool nextMoreIndented = (INPUT.peek() == ' ');
// TODO: for block scalars, we always start with a newline, so we should fold OR keep that
if(info.fold && !emptyLine && !nextEmptyLine && !moreIndented && !nextMoreIndented)
scalar += " ";
else
scalar += "\n";
emptyLine = nextEmptyLine;
moreIndented = nextMoreIndented;
// are we done via indentation?
if(!emptyLine && INPUT.column < info.indent) {
info.leadingSpaces = true;
break;
}
}
// post-processing
if(info.trimTrailingSpaces) {
unsigned pos = scalar.find_last_not_of(' ');
if(pos < scalar.size())
scalar.erase(pos + 1);
}
if(info.chomp <= 0) {
unsigned pos = scalar.find_last_not_of('\n');
if(info.chomp == 0 && pos + 1 < scalar.size())
scalar.erase(pos + 2);
else if(info.chomp == -1 && pos < scalar.size())
scalar.erase(pos + 1);
}
return scalar;
}
// PlainScalarToken
// . We scan these in passes of two steps each: First, grab all non-whitespace
// characters we can, and then grab all whitespace characters we can.
// . This has the benefit of letting us handle leading whitespace (which is chomped)
// and in-line whitespace (which is kept) separately.
template <> PlainScalarToken *Scanner::ScanToken(PlainScalarToken *pToken)
{
//// now eat and store the scalar
//std::string scalar;
//WhitespaceInfo info;
//while(INPUT) {
// // doc start/end tokens
// if(IsDocumentStart() || IsDocumentEnd())
// break;
// // comment
// if(Exp::Comment.Matches(INPUT))
// break;
// // first eat non-blanks
// while(INPUT && !Exp::BlankOrBreak.Matches(INPUT)) {
// // illegal colon in flow context
// if(m_flowLevel > 0 && Exp::IllegalColonInScalar.Matches(INPUT))
// throw IllegalScalar();
// // characters that might end the scalar
// if(m_flowLevel > 0 && Exp::EndScalarInFlow.Matches(INPUT))
// break;
// if(m_flowLevel == 0 && Exp::EndScalar.Matches(INPUT))
// break;
// // finally, read the character!
// scalar += GetChar();
// }
// // did we hit a non-blank character that ended us?
// if(!Exp::BlankOrBreak.Matches(INPUT))
// break;
// // now eat blanks
// while(INPUT && Exp::BlankOrBreak.Matches(INPUT)) {
// if(Exp::Blank.Matches(INPUT)) {
// // can't use tabs as indentation! only spaces!
// if(INPUT.peek() == '\t' && info.leadingBlanks && m_column <= m_indents.top())
// throw IllegalTabInScalar();
// info.AddBlank(GetChar());
// } else {
// // we know it's a line break; see how many characters to read
// int n = Exp::Break.Match(INPUT);
// std::string line = GetChar(n);
// info.AddBreak(line);
// // and we can't continue a simple key to the next line
// ValidateSimpleKey();
// }
// }
// // break if we're below the indentation level
// if(m_flowLevel == 0 && m_column <= m_indents.top())
// break;
// // finally join whitespace
// scalar += info.Join();
//}
// set up the scanning parameters
ScanScalarInfo info;
info.end = (m_flowLevel > 0 ? Exp::EndScalarInFlow : Exp::EndScalar) || (RegEx(' ') + Exp::Comment);
info.eatEnd = false;
......@@ -143,6 +146,8 @@ namespace YAML
info.eatLeadingWhitespace = true;
info.trimTrailingSpaces = true;
info.chomp = CLIP;
info.onDocIndicator = BREAK;
info.onTabInIndentation = THROW;
// insert a potential simple key
if(m_simpleKeyAllowed)
......@@ -150,9 +155,13 @@ namespace YAML
pToken->value = ScanScalar(INPUT, info);
m_simpleKeyAllowed = false;
if(true/*info.leadingBlanks*/)
m_simpleKeyAllowed = true;
// can have a simple key only if we ended the scalar by starting a new line
m_simpleKeyAllowed = info.leadingSpaces;
// finally, we can't have any colons in a scalar, so if we ended on a colon, there
// had better be a break after it
if(Exp::IllegalColonInScalar.Matches(INPUT))
throw IllegalScalar();
return pToken;
}
......@@ -160,76 +169,11 @@ namespace YAML
// QuotedScalarToken
template <> QuotedScalarToken *Scanner::ScanToken(QuotedScalarToken *pToken)
{
//// now eat and store the scalar
//std::string scalar;
//WhitespaceInfo info;
//while(INPUT) {
// if(IsDocumentStart() || IsDocumentEnd())
// throw DocIndicatorInQuote();
// if(INPUT.peek() == EOF)
// throw EOFInQuote();
// // first eat non-blanks
// while(INPUT && !Exp::BlankOrBreak.Matches(INPUT)) {
// // escaped single quote?
// if(pToken->single && Exp::EscSingleQuote.Matches(INPUT)) {
// int n = Exp::EscSingleQuote.Match(INPUT);
// scalar += GetChar(n);
// continue;
// }
// // is the quote ending?
// if(INPUT.peek() == quote)
// break;
// // escaped newline?
// if(Exp::EscBreak.Matches(INPUT))
// break;
// // other escape sequence
// if(INPUT.peek() == '\\') {
// int length = 0;
// scalar += Exp::Escape(INPUT, length);
// m_column += length;
// continue;
// }
// // and finally, just add the damn character
// scalar += GetChar();
// }
// // is the quote ending?
// if(INPUT.peek() == quote) {
// // eat and go
// GetChar();
// break;
// }
// // now we eat blanks
// while(Exp::BlankOrBreak.Matches(INPUT)) {
// if(Exp::Blank.Matches(INPUT)) {
// info.AddBlank(GetChar());
// } else {
// // we know it's a line break; see how many characters to read
// int n = Exp::Break.Match(INPUT);
// std::string line = GetChar(n);
// info.AddBreak(line);
// // and we can't continue a simple key to the next line
// ValidateSimpleKey();
// }
// }
// // and finally join the whitespace
// scalar += info.Join();
//}
// eat single or double quote
char quote = INPUT.GetChar();
pToken->single = (quote == '\'');
// setup the scanning parameters
ScanScalarInfo info;
info.end = (pToken->single ? RegEx(quote) && !Exp::EscSingleQuote : RegEx(quote));
info.eatEnd = true;
......@@ -239,6 +183,7 @@ namespace YAML
info.eatLeadingWhitespace = true;
info.trimTrailingSpaces = false;
info.chomp = CLIP;
info.onDocIndicator = THROW;
// insert a potential simple key
if(m_simpleKeyAllowed)
......@@ -251,9 +196,14 @@ namespace YAML
}
// BlockScalarToken
// . These need a little extra processing beforehand.
// . We need to scan the line where the indicator is (this doesn't count as part of the scalar),
// and then we need to figure out what level of indentation we'll be using.
template <> BlockScalarToken *Scanner::ScanToken(BlockScalarToken *pToken)
{
WhitespaceInfo info;
ScanScalarInfo info;
info.indent = 1;
info.detectIndent = true;
// eat block indicator ('|' or '>')
char indicator = INPUT.GetChar();
......@@ -261,10 +211,21 @@ namespace YAML
// eat chomping/indentation indicators
int n = Exp::Chomp.Match(INPUT);
for(int i=0;i<n;i++)
info.SetChompers(INPUT.GetChar());
for(int i=0;i<n;i++) {
char ch = INPUT.GetChar();
if(ch == '+')
info.chomp = KEEP;
else if(ch == '-')
info.chomp = STRIP;
else if(Exp::Digit.Matches(ch)) {
info.indent = ch - '0';
info.detectIndent = false;
if(info.indent == 0)
throw ZeroIndentationInBlockScalar();
}
}
// first eat whitespace
// now eat whitespace
while(Exp::Blank.Matches(INPUT))
INPUT.Eat(1);
......@@ -277,164 +238,18 @@ namespace YAML
if(INPUT && !Exp::Break.Matches(INPUT))
throw UnexpectedCharacterInBlockScalar();
// and eat that baby
INPUT.EatLineBreak();
// set the initial indentation
int indent = info.increment;
if(info.increment && m_indents.top() >= 0)
indent += m_indents.top();
GetBlockIndentation(INPUT, indent, info.trailingBreaks, m_indents.top());
if(m_indents.top() >= 0)
info.indent += m_indents.top();
ScanScalarInfo sinfo;
sinfo.indent = indent;
sinfo.fold = info.fold;
sinfo.eatLeadingWhitespace = false;
sinfo.trimTrailingSpaces = false;
sinfo.chomp = (CHOMP) info.chomp;
info.eatLeadingWhitespace = false;
info.trimTrailingSpaces = false;
info.onTabInIndentation = THROW;
pToken->value = ScanScalar(INPUT, sinfo);
pToken->value = ScanScalar(INPUT, info);
// simple keys always ok after block scalars (since we're gonna start a new line anyways)
m_simpleKeyAllowed = true;
return pToken;
}
// GetBlockIndentation
// . Helper to scanning a block scalar.
// . Eats leading *indentation* zeros (i.e., those that come before 'indent'),
// and updates 'indent' (if it hasn't been set yet).
void GetBlockIndentation(Stream& INPUT, int& indent, std::string& breaks, int topIndent)
{
int maxIndent = 0;
while(1) {
// eat as many indentation spaces as we can
while((indent == 0 || INPUT.column < indent) && INPUT.peek() == ' ')
INPUT.Eat(1);
if(INPUT.column > maxIndent)
maxIndent = INPUT.column;
// do we need more indentation, but we've got a tab?
if((indent == 0 || INPUT.column < indent) && INPUT.peek() == '\t')
throw IllegalTabInScalar(); // TODO: are literal scalar lines allowed to have tabs here?
// is this a non-empty line?
if(!Exp::Break.Matches(INPUT))
break;
// otherwise, eat the line break and move on
int n = Exp::Break.Match(INPUT);
breaks += INPUT.GetChar(n);
}
// finally, set the indentation
if(indent == 0) {
indent = maxIndent;
if(indent < topIndent + 1)
indent = topIndent + 1;
if(indent < 1)
indent = 1;
}
}
// ScanScalar
std::string ScanScalar(Stream& INPUT, ScanScalarInfo info)
{
bool emptyLine = false, moreIndented = false;
std::string scalar;
while(INPUT) {
// ********************************
// Phase #1: scan until line ending
while(!info.end.Matches(INPUT) && !Exp::Break.Matches(INPUT)) {
if(INPUT.peek() == EOF)
break;
// escaped newline? (only if we're escaping on slash)
if(info.escape == '\\' && Exp::EscBreak.Matches(INPUT)) {
int n = Exp::EscBreak.Match(INPUT);
INPUT.Eat(n);
continue;
}
// escape this?
if(INPUT.peek() == info.escape) {
scalar += Exp::Escape(INPUT);
continue;
}
// otherwise, just add the damn character
scalar += INPUT.GetChar();
}
// eof? if we're looking to eat something, then we throw
if(INPUT.peek() == EOF) {
if(info.eatEnd)
throw EOFInQuote();
break;
}
// are we done via character match?
int n = info.end.Match(INPUT);
if(n >= 0) {
if(info.eatEnd)
INPUT.Eat(n);
break;
}
// ********************************
// Phase #2: eat line ending
n = Exp::Break.Match(INPUT);
INPUT.Eat(n);
// ********************************
// Phase #3: scan initial spaces
// first the required indentation
while(INPUT.peek() == ' ' && INPUT.column < info.indent)
INPUT.Eat(1);
// and then the rest of the whitespace
if(info.eatLeadingWhitespace) {
while(Exp::Blank.Matches(INPUT))
INPUT.Eat(1);
}
// was this an empty line?
bool nextEmptyLine = Exp::Break.Matches(INPUT);
bool nextMoreIndented = (INPUT.peek() == ' ');
if(info.fold && !emptyLine && !nextEmptyLine && !moreIndented && !nextMoreIndented)
scalar += " ";
else
scalar += "\n";
emptyLine = nextEmptyLine;
moreIndented = nextMoreIndented;
// are we done via indentation?
if(!emptyLine && INPUT.column < info.indent)
break;
}
// post-processing
if(info.trimTrailingSpaces) {
unsigned pos = scalar.find_last_not_of(' ');
if(pos < scalar.size())
scalar.erase(pos + 1);
}
if(info.chomp <= 0) {
unsigned pos = scalar.find_last_not_of('\n');
if(info.chomp == 0 && pos + 1 < scalar.size())
scalar.erase(pos + 2);
else if(info.chomp == -1 && pos < scalar.size())
scalar.erase(pos + 1);
}
return scalar;
}
}
scanscalar.h
View file @ 0683cbf8
......@@ -7,35 +7,29 @@
namespace YAML
{
enum CHOMP { STRIP = -1, CLIP, KEEP };
enum ACTION { NONE, BREAK, THROW };
struct ScanScalarInfo {
ScanScalarInfo(): eatEnd(false), indent(0), eatLeadingWhitespace(0), escape(0), fold(false), trimTrailingSpaces(0), chomp(CLIP) {}
ScanScalarInfo(): eatEnd(false), indent(0), detectIndent(false), eatLeadingWhitespace(0), escape(0), fold(false),
trimTrailingSpaces(0), chomp(CLIP), onDocIndicator(NONE), onTabInIndentation(NONE), leadingSpaces(false) {}
// input:
RegEx end; // what condition ends this scalar?
bool eatEnd; // should we eat that condition when we see it?
int indent; // what level of indentation should be eaten and ignored?
bool detectIndent; // should we try to autodetect the indent?
bool eatLeadingWhitespace; // should we continue eating this delicious indentation after 'indent' spaces?
char escape; // what character do we escape on (i.e., slash or single quote) (0 for none)
bool fold; // do we fold line ends?
bool trimTrailingSpaces; // do we remove all trailing spaces (at the very end)
CHOMP chomp; // do we strip, clip, or keep trailing newlines (at the very end)
// Note: strip means kill all, clip means keep at most one, keep means keep all
};
void GetBlockIndentation(Stream& INPUT, int& indent, std::string& breaks, int topIndent);
std::string ScanScalar(Stream& INPUT, ScanScalarInfo info);
struct WhitespaceInfo {
WhitespaceInfo();
ACTION onDocIndicator; // what do we do if we see a document indicator?
ACTION onTabInIndentation; // what do we do if we see a tab where we should be seeing indentation spaces
void SetChompers(char ch);
void AddBlank(char ch);
void AddBreak(const std::string& line);
std::string Join(bool lastline = false);
bool leadingBlanks;
bool fold;
std::string whitespace, leadingBreaks, trailingBreaks;
int chomp, increment;
// output:
bool leadingSpaces;
};
std::string ScanScalar(Stream& INPUT, ScanScalarInfo& info);
}
test.yaml
View file @ 0683cbf8
---
- "quoted scalar\twith a tab\nand a newline"
- 'This is Jesse''s single quote!'
- |
here's a literal:
#include <iostream>
- here's a key: value
here's the first block: |
after the block: value
and here's a block: |-
What's going on?
How are you doing?
Here's some code:
#include <iostream>
int main()
{
std::cout << "Hello World!\n";
return 0;
}
- key1: value1
key2: value2
\ No newline at end of file
I'm doing fine!
and last key: value
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment