Moved the simple key validation to before each token scan (plus at newlines of scalars).

scanner.cpp

@@ -146,17 +146,6 @@ namespace YAML
 		m_limboTokens.insert(pToken);
 		m_tokens.push(ScanToken(pToken));
 		m_limboTokens.erase(pToken);
-
-		// then remove impossible tokens
-		std::queue <Token *> temp;
-		while(!m_tokens.empty()) {
-			Token *pToken = m_tokens.front();
-			m_tokens.pop();
-			if(pToken->isPossible)
-				temp.push(pToken);
-		}
-
-		m_tokens = temp;
 	}
 
 	///////////////////////////////////////////////////////////////////////
@@ -171,7 +160,7 @@ namespace YAML
 			return ScanAndEnqueue(new StreamStartToken);
 
 		ScanToNextToken();
-		// TODO: remove "obsolete potential simple keys"
+		ValidateSimpleKey();
 		PopIndentTo(m_column);
 
 		if(INPUT.peek() == EOF)
@@ -318,23 +307,56 @@ namespace YAML
 		}
 	}
 
-	// temporary function for testing
-	void Scanner::Scan()
+	// GetNextToken
+	// . Returns the next token on the queue, and scans if only we need to.
+	Token *Scanner::GetNextToken()
 	{
 		while(1) {
-			ScanNextToken();
-			if(m_tokens.empty())
-				break;
+			Token *pToken = 0;
 
-			while(!m_tokens.empty()) {
-				Token *pToken = m_tokens.front();
-				if(!pToken->isValid)   // gotta wait on the invalid tokens - they might become valid!
-					break;
+			// is there a token in the queue?
+			if(!m_tokens.empty())
+				pToken = m_tokens.front();
 
+			// ... that's possible
+			// (here's where we clean up the impossible tokens)
+			if(pToken && !pToken->isPossible) {
 				m_tokens.pop();
-				std::cout << typeid(*pToken).name() << ": " << *pToken << std::endl;
 				delete pToken;
+				continue;
 			}
+
+			// and valid
+			if(pToken && !pToken->isValid)
+				pToken = 0;
+
+			// then that's what we want
+			if(pToken) {
+				m_tokens.pop();
+				return pToken;
+			}
+
+			// no token? maybe we've actually finished
+			if(m_endedStream)
+				break;
+
+			// no? then scan...
+			ScanNextToken();
+		}
+
+		return 0;
+	}
+
+	// temporary function for testing
+	void Scanner::Scan()
+	{
+		while(1) {
+			Token *pToken = GetNextToken();
+			if(!pToken)
+				break;
+
+			std::cout << typeid(*pToken).name() << ": " << *pToken << std::endl;
+			delete pToken;
 		}
 	}
 }

scanner.h

@@ -16,6 +16,8 @@ namespace YAML
 		Scanner(std::istream& in);
 		~Scanner();
 
+		Token *GetNextToken();
+
 		void ScanNextToken();
 		void ScanToNextToken();
 		Token *PushIndentTo(int column, bool sequence);
@@ -55,12 +57,12 @@ namespace YAML
 		};
 
 		struct SimpleKey {
-			SimpleKey(int pos_, int line_, int column_);
+			SimpleKey(int pos_, int line_, int column_, int flowLevel_);
 
 			void Validate();
 			void Invalidate();
 
-			int pos, line, column;
+			int pos, line, column, flowLevel;
 			bool required;
 			Token *pMapStart, *pKey;
 		};
@@ -81,6 +83,7 @@ namespace YAML
 		bool m_startedStream, m_endedStream;
 		bool m_simpleKeyAllowed;
 		int m_flowLevel;                // number of unclosed '[' and '{' indicators
+		bool m_isLastKeyValid;
 		std::stack <SimpleKey> m_simpleKeys;
 		std::stack <int> m_indents;
 	};

scantoken.cpp

@@ -99,7 +99,7 @@ namespace YAML
 	// FlowMapEndToken
 	template <> FlowMapEndToken *Scanner::ScanToken(FlowMapEndToken *pToken)
 	{
-		ValidateSimpleKey();
+		//ValidateSimpleKey();
 		DecreaseFlowLevel();
 		m_simpleKeyAllowed = false;
 
@@ -111,7 +111,7 @@ namespace YAML
 	// FlowEntryToken
 	template <> FlowEntryToken *Scanner::ScanToken(FlowEntryToken *pToken)
 	{
-		ValidateSimpleKey();
+		//ValidateSimpleKey();
 		m_simpleKeyAllowed = true;
 
 		// eat
@@ -122,7 +122,7 @@ namespace YAML
 	// BlockEntryToken
 	template <> BlockEntryToken *Scanner::ScanToken(BlockEntryToken *pToken)
 	{
-		ValidateSimpleKey();
+		//ValidateSimpleKey();
 
 		// we better be in the block context!
 		if(m_flowLevel == 0) {
@@ -170,9 +170,9 @@ namespace YAML
 	template <> ValueToken *Scanner::ScanToken(ValueToken *pToken)
 	{
 		// does this follow a simple key?
-		bool isValidKey = ValidateSimpleKey();
+//		bool isValidKey = ValidateSimpleKey();
 
-		if(isValidKey) {
+		if(m_isLastKeyValid) {
 			// can't follow a simple key with another simple key (dunno why, though - it seems fine)
 			m_simpleKeyAllowed = false;
 		} else {

simplekey.cpp

@@ -5,8 +5,8 @@
 
 namespace YAML
 {
-	Scanner::SimpleKey::SimpleKey(int pos_, int line_, int column_)
-		: pos(pos_), line(line_), column(column_), required(false), pMapStart(0), pKey(0)
+	Scanner::SimpleKey::SimpleKey(int pos_, int line_, int column_, int flowLevel_)
+		: pos(pos_), line(line_), column(column_), flowLevel(flowLevel_), required(false), pMapStart(0), pKey(0)
 	{
 	}
 
@@ -34,7 +34,7 @@ namespace YAML
 	//   and saves it on a stack.
 	void Scanner::InsertSimpleKey()
 	{
-		SimpleKey key(INPUT.tellg(), m_line, m_column);
+		SimpleKey key(INPUT.tellg(), m_line, m_column, m_flowLevel);
 
 		// first add a map start, if necessary
 		key.pMapStart = PushIndentTo(m_column, false);
@@ -57,11 +57,17 @@ namespace YAML
 	//   and if so, makes it valid.
 	bool Scanner::ValidateSimpleKey()
 	{
+		m_isLastKeyValid = false;
 		if(m_simpleKeys.empty())
-			return false;
+			return m_isLastKeyValid;
 
 		// grab top key
 		SimpleKey key = m_simpleKeys.top();
+
+		// only validate if we're in the correct flow level
+		if(key.flowLevel != m_flowLevel)
+			return false;
+
 		m_simpleKeys.pop();
 
 		bool isValid = true;
@@ -89,6 +95,7 @@ namespace YAML
 		if(!isValid && m_flowLevel == 0)
 			m_indents.pop();
 
+		m_isLastKeyValid = isValid;
 		return isValid;
 	}