From a1460169e624e78aec64c0a3c620c479fb0b0f8e Mon Sep 17 00:00:00 2001 From: Jesse Beder Date: Mon, 24 Aug 2009 22:56:54 +0000 Subject: [PATCH] Fixed bug in anchors with no content. This involved refactoring the 'implicit sequence' concept (where a map and a sequence start on the same indent, but we read the sequence as more indented since the '-' is visually an indent). --- src/map.cpp | 4 +- src/node.cpp | 3 +- src/scanner.cpp | 75 +++++++++++++++++++++++++++++------- src/scanner.h | 18 +++++++-- src/scantoken.cpp | 18 ++++----- src/sequence.cpp | 6 +-- src/simplekey.cpp | 2 +- src/token.h | 6 ++- yaml-reader/emittertests.cpp | 10 +++++ yaml-reader/parsertests.cpp | 69 +++++++++++++++++++++++++++++++++ yaml-reader/tests.cpp | 4 ++ yaml-reader/tests.h | 4 ++ 12 files changed, 183 insertions(+), 36 deletions(-) diff --git a/src/map.cpp b/src/map.cpp index f0038fc..ce24978 100644 --- a/src/map.cpp +++ b/src/map.cpp @@ -75,10 +75,10 @@ namespace YAML throw ParserException(Mark::null(), ErrorMsg::END_OF_MAP); Token token = pScanner->peek(); - if(token.type != TT_KEY && token.type != TT_VALUE && token.type != TT_BLOCK_END) + if(token.type != TT_KEY && token.type != TT_VALUE && token.type != TT_BLOCK_MAP_END) throw ParserException(token.mark, ErrorMsg::END_OF_MAP); - if(token.type == TT_BLOCK_END) { + if(token.type == TT_BLOCK_MAP_END) { pScanner->pop(); break; } diff --git a/src/node.cpp b/src/node.cpp index 7a460f3..7223bed 100644 --- a/src/node.cpp +++ b/src/node.cpp @@ -93,7 +93,6 @@ namespace YAML break; case TT_FLOW_SEQ_START: case TT_BLOCK_SEQ_START: - case TT_BLOCK_ENTRY: m_pContent = new Sequence; break; case TT_FLOW_MAP_START: @@ -265,7 +264,7 @@ namespace YAML // write content if(node.m_pContent) node.m_pContent->Write(out); - else + else if(!node.m_alias) out << Null; return out; diff --git a/src/scanner.cpp b/src/scanner.cpp index 5848d77..83a16fb 100644 --- a/src/scanner.cpp +++ b/src/scanner.cpp @@ -46,6 +46,7 @@ namespace YAML assert(!m_tokens.empty()); // should we be asserting here? I mean, we really just be checking // if it's empty before peeking. +// std::cerr << "peek: (" << &m_tokens.front() << ") " << m_tokens.front() << "\n"; return m_tokens.front(); } @@ -98,7 +99,7 @@ namespace YAML VerifySimpleKey(); // maybe need to end some blocks - PopIndentTo(INPUT.column()); + PopIndentToHere(); // ***** // And now branch based on the next few characters! @@ -221,7 +222,7 @@ namespace YAML { m_startedStream = true; m_simpleKeyAllowed = true; - m_indents.push(-1); + m_indents.push(IndentMarker(-1, IndentMarker::NONE)); m_anchors.clear(); } @@ -233,7 +234,7 @@ namespace YAML if(INPUT.column() > 0) INPUT.ResetColumn(); - PopIndentTo(-1); + PopAllIndents(); VerifyAllSimpleKeys(); m_simpleKeyAllowed = false; @@ -244,41 +245,87 @@ namespace YAML // . Pushes an indentation onto the stack, and enqueues the // proper token (sequence start or mapping start). // . Returns the token it generates (if any). - Token *Scanner::PushIndentTo(int column, bool sequence) + Token *Scanner::PushIndentTo(int column, IndentMarker::INDENT_TYPE type) { // are we in flow? if(m_flowLevel > 0) return 0; + + IndentMarker indent(column, type); + const IndentMarker& lastIndent = m_indents.top(); // is this actually an indentation? - if(column <= m_indents.top()) + if(indent.column < lastIndent.column) + return 0; + if(indent.column == lastIndent.column && !(indent.type == IndentMarker::SEQ && lastIndent.type == IndentMarker::MAP)) return 0; // now push - m_indents.push(column); - if(sequence) + m_indents.push(indent); + if(type == IndentMarker::SEQ) m_tokens.push(Token(TT_BLOCK_SEQ_START, INPUT.mark())); - else + else if(type == IndentMarker::MAP) m_tokens.push(Token(TT_BLOCK_MAP_START, INPUT.mark())); + else + assert(false); return &m_tokens.back(); } - // PopIndentTo - // . Pops indentations off the stack until we reach 'column' indentation, + // PopIndentToHere + // . Pops indentations off the stack until we reach the current indentation level, // and enqueues the proper token each time. - void Scanner::PopIndentTo(int column) + void Scanner::PopIndentToHere() { // are we in flow? if(m_flowLevel > 0) return; // now pop away - while(!m_indents.empty() && m_indents.top() > column) { - m_indents.pop(); - m_tokens.push(Token(TT_BLOCK_END, INPUT.mark())); + while(!m_indents.empty()) { + const IndentMarker& indent = m_indents.top(); + if(indent.column < INPUT.column()) + break; + if(indent.column == INPUT.column() && !(indent.type == IndentMarker::SEQ && !Exp::BlockEntry.Matches(INPUT))) + break; + + PopIndent(); } } + + // PopAllIndents + // . Pops all indentations off the stack, + // and enqueues the proper token each time. + void Scanner::PopAllIndents() + { + // are we in flow? + if(m_flowLevel > 0) + return; + + // now pop away + while(!m_indents.empty()) + PopIndent(); + } + + // PopIndent + // . Pops a single indent, pushing the proper token + void Scanner::PopIndent() + { + IndentMarker::INDENT_TYPE type = m_indents.top().type; + m_indents.pop(); + if(type == IndentMarker::SEQ) + m_tokens.push(Token(TT_BLOCK_SEQ_END, INPUT.mark())); + else if(type == IndentMarker::MAP) + m_tokens.push(Token(TT_BLOCK_MAP_END, INPUT.mark())); + } + + // GetTopIndent + int Scanner::GetTopIndent() const + { + if(m_indents.empty()) + return 0; + return m_indents.top().column; + } // Save // . Saves a pointer to the Node object referenced by a particular anchor diff --git a/src/scanner.h b/src/scanner.h index 36aa917..853fa64 100644 --- a/src/scanner.h +++ b/src/scanner.h @@ -34,14 +34,26 @@ namespace YAML void ClearAnchors(); private: + struct IndentMarker { + enum INDENT_TYPE { MAP, SEQ, NONE }; + IndentMarker(int column_, INDENT_TYPE type_): column(column_), type(type_) {} + + int column; + INDENT_TYPE type; + }; + + private: // scanning void EnsureTokensInQueue(); void ScanNextToken(); void ScanToNextToken(); void StartStream(); void EndStream(); - Token *PushIndentTo(int column, bool sequence); - void PopIndentTo(int column); + Token *PushIndentTo(int column, IndentMarker::INDENT_TYPE type); + void PopIndentToHere(); + void PopAllIndents(); + void PopIndent(); + int GetTopIndent() const; // checking input void InsertSimpleKey(); @@ -94,7 +106,7 @@ namespace YAML int m_flowLevel; // number of unclosed '[' and '{' indicators bool m_isLastKeyValid; std::stack m_simpleKeys; - std::stack m_indents; + std::stack m_indents; std::map m_anchors; }; } diff --git a/src/scantoken.cpp b/src/scantoken.cpp index 3699b4d..22b9712 100644 --- a/src/scantoken.cpp +++ b/src/scantoken.cpp @@ -19,7 +19,7 @@ namespace YAML std::vector params; // pop indents and simple keys - PopIndentTo(-1); + PopAllIndents(); VerifyAllSimpleKeys(); m_simpleKeyAllowed = false; @@ -59,7 +59,7 @@ namespace YAML // DocStart void Scanner::ScanDocStart() { - PopIndentTo(-1); + PopAllIndents(); VerifyAllSimpleKeys(); m_simpleKeyAllowed = false; @@ -72,7 +72,7 @@ namespace YAML // DocEnd void Scanner::ScanDocEnd() { - PopIndentTo(-1); + PopAllIndents(); VerifyAllSimpleKeys(); m_simpleKeyAllowed = false; @@ -135,7 +135,7 @@ namespace YAML if(!m_simpleKeyAllowed) throw ParserException(INPUT.mark(), ErrorMsg::BLOCK_ENTRY); - PushIndentTo(INPUT.column(), true); + PushIndentTo(INPUT.column(), IndentMarker::SEQ); m_simpleKeyAllowed = true; // eat @@ -152,7 +152,7 @@ namespace YAML if(!m_simpleKeyAllowed) throw ParserException(INPUT.mark(), ErrorMsg::MAP_KEY); - PushIndentTo(INPUT.column(), false); + PushIndentTo(INPUT.column(), IndentMarker::MAP); } // can only put a simple key here if we're in block context @@ -180,7 +180,7 @@ namespace YAML if(!m_simpleKeyAllowed) throw ParserException(INPUT.mark(), ErrorMsg::MAP_VALUE); - PushIndentTo(INPUT.column(), false); + PushIndentTo(INPUT.column(), IndentMarker::MAP); } // can only put a simple key here if we're in block context @@ -277,7 +277,7 @@ namespace YAML ScanScalarParams params; params.end = (m_flowLevel > 0 ? Exp::EndScalarInFlow : Exp::EndScalar) || (Exp::BlankOrBreak + Exp::Comment); params.eatEnd = false; - params.indent = (m_flowLevel > 0 ? 0 : m_indents.top() + 1); + params.indent = (m_flowLevel > 0 ? 0 : GetTopIndent() + 1); params.fold = true; params.eatLeadingWhitespace = true; params.trimTrailingSpaces = true; @@ -391,8 +391,8 @@ namespace YAML throw ParserException(INPUT.mark(), ErrorMsg::CHAR_IN_BLOCK); // set the initial indentation - if(m_indents.top() >= 0) - params.indent += m_indents.top(); + if(GetTopIndent() >= 0) + params.indent += GetTopIndent(); params.eatLeadingWhitespace = false; params.trimTrailingSpaces = false; diff --git a/src/sequence.cpp b/src/sequence.cpp index dde05c6..5073f08 100644 --- a/src/sequence.cpp +++ b/src/sequence.cpp @@ -83,11 +83,11 @@ namespace YAML throw ParserException(Mark::null(), ErrorMsg::END_OF_SEQ); Token token = pScanner->peek(); - if(token.type != TT_BLOCK_ENTRY && token.type != TT_BLOCK_END) + if(token.type != TT_BLOCK_ENTRY && token.type != TT_BLOCK_SEQ_END) throw ParserException(token.mark, ErrorMsg::END_OF_SEQ); pScanner->pop(); - if(token.type == TT_BLOCK_END) + if(token.type == TT_BLOCK_SEQ_END) break; Node *pNode = new Node; @@ -96,7 +96,7 @@ namespace YAML // check for null if(!pScanner->empty()) { const Token& token = pScanner->peek(); - if(token.type == TT_BLOCK_ENTRY || token.type == TT_BLOCK_END) + if(token.type == TT_BLOCK_ENTRY || token.type == TT_BLOCK_SEQ_END) continue; } diff --git a/src/simplekey.cpp b/src/simplekey.cpp index f1c0a2b..85dc7f8 100644 --- a/src/simplekey.cpp +++ b/src/simplekey.cpp @@ -35,7 +35,7 @@ namespace YAML SimpleKey key(INPUT.mark(), m_flowLevel); // first add a map start, if necessary - key.pMapStart = PushIndentTo(INPUT.column(), false); + key.pMapStart = PushIndentTo(INPUT.column(), IndentMarker::MAP); if(key.pMapStart) key.pMapStart->status = TS_UNVERIFIED; diff --git a/src/token.h b/src/token.h index 060861a..c21524d 100644 --- a/src/token.h +++ b/src/token.h @@ -18,7 +18,8 @@ namespace YAML TT_DOC_END, TT_BLOCK_SEQ_START, TT_BLOCK_MAP_START, - TT_BLOCK_END, + TT_BLOCK_SEQ_END, + TT_BLOCK_MAP_END, TT_BLOCK_ENTRY, TT_FLOW_SEQ_START, TT_FLOW_MAP_START, @@ -39,7 +40,8 @@ namespace YAML "DOC_END", "BLOCK_SEQ_START", "BLOCK_MAP_START", - "BLOCK_END", + "BLOCK_SEQ_END", + "BLOCK_MAP_END", "BLOCK_ENTRY", "FLOW_SEQ_START", "FLOW_MAP_START", diff --git a/yaml-reader/emittertests.cpp b/yaml-reader/emittertests.cpp index a016bff..95c0ac4 100644 --- a/yaml-reader/emittertests.cpp +++ b/yaml-reader/emittertests.cpp @@ -272,6 +272,16 @@ namespace Test desiredOutput = "- &fred\n name: Fred\n age: 42\n- *fred"; } + void AliasAndAnchorWithNull(YAML::Emitter& out, std::string& desiredOutput) + { + out << YAML::BeginSeq; + out << YAML::Anchor("fred") << YAML::Null; + out << YAML::Alias("fred"); + out << YAML::EndSeq; + + desiredOutput = "- &fred ~\n- *fred"; + } + void ComplexDoc(YAML::Emitter& out, std::string& desiredOutput) { out << YAML::BeginMap; diff --git a/yaml-reader/parsertests.cpp b/yaml-reader/parsertests.cpp index 3b2e28d..03c833e 100644 --- a/yaml-reader/parsertests.cpp +++ b/yaml-reader/parsertests.cpp @@ -248,6 +248,30 @@ namespace Test return true; } + bool CompressedMapAndSeq() + { + std::string input = "key:\n- one\n- two"; + + std::stringstream stream(input); + YAML::Parser parser(stream); + YAML::Node doc; + parser.GetNextDocument(doc); + + const YAML::Node& seq = doc["key"]; + if(seq.size() != 2) + return false; + + std::string output; + seq[0] >> output; + if(output != "one") + return false; + seq[1] >> output; + if(output != "two") + return false; + + return true; + } + bool NullBlockSeqEntry() { std::string input = "- hello\n-\n- world"; @@ -301,5 +325,50 @@ namespace Test return true; } + + bool SimpleAlias() + { + std::string input = "- &alias test\n- *alias"; + + std::stringstream stream(input); + YAML::Parser parser(stream); + YAML::Node doc; + parser.GetNextDocument(doc); + + std::string output; + doc[0] >> output; + if(output != "test") + return false; + + doc[1] >> output; + if(output != "test") + return false; + + if(doc.size() != 2) + return false; + + return true; + } + + bool AliasWithNull() + { + std::string input = "- &alias\n- *alias"; + + std::stringstream stream(input); + YAML::Parser parser(stream); + YAML::Node doc; + parser.GetNextDocument(doc); + + if(!IsNull(doc[0])) + return false; + + if(!IsNull(doc[1])) + return false; + + if(doc.size() != 2) + return false; + + return true; + } } } diff --git a/yaml-reader/tests.cpp b/yaml-reader/tests.cpp index f434c03..00c5132 100644 --- a/yaml-reader/tests.cpp +++ b/yaml-reader/tests.cpp @@ -263,9 +263,12 @@ namespace Test RunParserTest(&Parser::FlowSeq, "flow seq", passed); RunParserTest(&Parser::FlowMap, "flow map", passed); RunParserTest(&Parser::QuotedSimpleKeys, "quoted simple keys", passed); + RunParserTest(&Parser::CompressedMapAndSeq, "compressed map and seq", passed); RunParserTest(&Parser::NullBlockSeqEntry, "null block seq entry", passed); RunParserTest(&Parser::NullBlockMapKey, "null block map key", passed); RunParserTest(&Parser::NullBlockMapValue, "null block map value", passed); + RunParserTest(&Parser::SimpleAlias, "simple alias", passed); + RunParserTest(&Parser::AliasWithNull, "alias with null", passed); RunEncodingTest(&EncodeToUtf8, false, "UTF-8, no BOM", passed); RunEncodingTest(&EncodeToUtf8, true, "UTF-8 with BOM", passed); @@ -346,6 +349,7 @@ namespace Test RunEmitterTest(&Emitter::LongKeyFlowMap, "long key flow map", passed); RunEmitterTest(&Emitter::BlockMapAsKey, "block map as key", passed); RunEmitterTest(&Emitter::AliasAndAnchor, "alias and anchor", passed); + RunEmitterTest(&Emitter::AliasAndAnchorWithNull, "alias and anchor with null", passed); RunEmitterTest(&Emitter::ComplexDoc, "complex doc", passed); RunEmitterTest(&Emitter::STLContainers, "STL containers", passed); RunEmitterTest(&Emitter::SimpleComment, "simple comment", passed); diff --git a/yaml-reader/tests.h b/yaml-reader/tests.h index e205166..f8b3205 100644 --- a/yaml-reader/tests.h +++ b/yaml-reader/tests.h @@ -34,9 +34,12 @@ namespace Test { bool FlowSeq(); bool FlowMap(); bool QuotedSimpleKeys(); + bool CompressedMapAndSeq(); bool NullBlockSeqEntry(); bool NullBlockMapKey(); bool NullBlockMapValue(); + bool SimpleAlias(); + bool AliasWithNull(); } namespace Emitter { @@ -63,6 +66,7 @@ namespace Test { void LongKeyFlowMap(YAML::Emitter& out, std::string& desiredOutput); void BlockMapAsKey(YAML::Emitter& out, std::string& desiredOutput); void AliasAndAnchor(YAML::Emitter& out, std::string& desiredOutput); + void AliasAndAnchorWithNull(YAML::Emitter& out, std::string& desiredOutput); void ComplexDoc(YAML::Emitter& out, std::string& desiredOutput); void STLContainers(YAML::Emitter& out, std::string& desiredOutput); void SimpleComment(YAML::Emitter& out, std::string& desiredOutput);