From 0c101d89f81a890372d96521289a8b921e0b2f13 Mon Sep 17 00:00:00 2001 From: Lefteris Karapetsas Date: Tue, 18 Nov 2014 18:50:40 +0100 Subject: [PATCH 1/7] solidity scanner takes triple slash doc comments into account - Conditionally scanning for the documentation comments and gettings their contents. - Adding tests for this functionality of the scanner --- libsolidity/Scanner.cpp | 36 +++++++++++++++++++++++++------- libsolidity/Scanner.h | 19 ++++++++++++----- libsolidity/Token.h | 1 + test/solidityScanner.cpp | 44 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+), 12 deletions(-) diff --git a/libsolidity/Scanner.cpp b/libsolidity/Scanner.cpp index b13e52d7e..382d07a99 100644 --- a/libsolidity/Scanner.cpp +++ b/libsolidity/Scanner.cpp @@ -102,13 +102,13 @@ int HexValue(char c) } } // end anonymous namespace -void Scanner::reset(CharStream const& _source) +void Scanner::reset(CharStream const& _source, bool _skipDocumentationComments) { m_source = _source; m_char = m_source.get(); skipWhitespace(); - scanToken(); - next(); + scanToken(_skipDocumentationComments); + next(_skipDocumentationComments); } @@ -134,10 +134,10 @@ bool Scanner::scanHexByte(char& o_scannedByte) // Ensure that tokens can be stored in a byte. BOOST_STATIC_ASSERT(Token::NUM_TOKENS <= 0x100); -Token::Value Scanner::next() +Token::Value Scanner::next(bool _skipDocumentationComments) { m_current_token = m_next_token; - scanToken(); + scanToken(_skipDocumentationComments); return m_current_token.token; } @@ -171,6 +171,21 @@ Token::Value Scanner::skipSingleLineComment() return Token::WHITESPACE; } +// For the moment this function simply consumes a single line triple slash doc comment +Token::Value Scanner::scanDocumentationComment() +{ + LiteralScope literal(this); + advance(); //consume the last '/' + while (!isSourcePastEndOfInput() && !IsLineTerminator(m_char)) + { + char c = m_char; + advance(); + addLiteralChar(c); + } + literal.Complete(); + return Token::COMMENT_LITERAL; +} + Token::Value Scanner::skipMultiLineComment() { if (asserts(m_char == '*')) @@ -194,7 +209,7 @@ Token::Value Scanner::skipMultiLineComment() return Token::ILLEGAL; } -void Scanner::scanToken() +void Scanner::scanToken(bool _skipDocumentationComments) { m_next_token.literal.clear(); Token::Value token; @@ -297,7 +312,14 @@ void Scanner::scanToken() // / // /* /= advance(); if (m_char == '/') - token = skipSingleLineComment(); + { + if (!advance()) /* double slash comment directly before EOS */ + token = Token::WHITESPACE; + else if (!_skipDocumentationComments) + token = scanDocumentationComment(); + else + token = skipSingleLineComment(); + } else if (m_char == '*') token = skipMultiLineComment(); else if (m_char == '=') diff --git a/libsolidity/Scanner.h b/libsolidity/Scanner.h index 997365f3c..fd48d5698 100644 --- a/libsolidity/Scanner.h +++ b/libsolidity/Scanner.h @@ -111,19 +111,27 @@ public: }; Scanner() { reset(CharStream()); } - explicit Scanner(CharStream const& _source) { reset(_source); } + explicit Scanner(CharStream const& _source, bool _skipDocumentationComments = true) + { + reset(_source, _skipDocumentationComments); + } /// Resets the scanner as if newly constructed with _input as input. - void reset(CharStream const& _source); + void reset(CharStream const& _source, bool _skipDocumentationComments = true); /// Returns the next token and advances input. - Token::Value next(); + Token::Value next(bool _skipDocumentationComments = true); ///@{ ///@name Information about the current token /// Returns the current token - Token::Value getCurrentToken() { return m_current_token.token; } + Token::Value getCurrentToken(bool _skipDocumentationComments = true) + { + if (!_skipDocumentationComments) + next(_skipDocumentationComments); + return m_current_token.token; + } Location getCurrentLocation() const { return m_current_token.location; } std::string const& getCurrentLiteral() const { return m_current_token.literal; } ///@} @@ -172,7 +180,7 @@ private: bool scanHexByte(char& o_scannedByte); /// Scans a single JavaScript token. - void scanToken(); + void scanToken(bool _skipDocumentationComments = true); /// Skips all whitespace and @returns true if something was skipped. bool skipWhitespace(); @@ -184,6 +192,7 @@ private: Token::Value scanIdentifierOrKeyword(); Token::Value scanString(); + Token::Value scanDocumentationComment(); /// Scans an escape-sequence which is part of a string and adds the /// decoded character to the current literal. Returns true if a pattern diff --git a/libsolidity/Token.h b/libsolidity/Token.h index 67971c3d0..39a84bcc2 100644 --- a/libsolidity/Token.h +++ b/libsolidity/Token.h @@ -281,6 +281,7 @@ namespace solidity K(FALSE_LITERAL, "false", 0) \ T(NUMBER, NULL, 0) \ T(STRING_LITERAL, NULL, 0) \ + T(COMMENT_LITERAL, NULL, 0) \ \ /* Identifiers (not keywords or future reserved words). */ \ T(IDENTIFIER, NULL, 0) \ diff --git a/test/solidityScanner.cpp b/test/solidityScanner.cpp index d714699a0..28f52d3af 100644 --- a/test/solidityScanner.cpp +++ b/test/solidityScanner.cpp @@ -153,6 +153,50 @@ BOOST_AUTO_TEST_CASE(ambiguities) BOOST_CHECK_EQUAL(scanner.next(), Token::SHL); } +BOOST_AUTO_TEST_CASE(documentation_comments_parsed_begin) +{ + Scanner scanner(CharStream("/// Send $(value / 1000) chocolates to the user"), false); + BOOST_CHECK_EQUAL(scanner.getCurrentToken(), Token::COMMENT_LITERAL); + BOOST_CHECK_EQUAL(scanner.getCurrentLiteral(), " Send $(value / 1000) chocolates to the user"); +} + +BOOST_AUTO_TEST_CASE(documentation_comments_skipped_begin) +{ + Scanner scanner(CharStream("/// Send $(value / 1000) chocolates to the user")); + BOOST_CHECK_EQUAL(scanner.getCurrentToken(), Token::EOS); +} + +BOOST_AUTO_TEST_CASE(documentation_comments_parsed) +{ + Scanner scanner(CharStream("some other tokens /// Send $(value / 1000) chocolates to the user")); + BOOST_CHECK_EQUAL(scanner.getCurrentToken(), Token::IDENTIFIER); + BOOST_CHECK_EQUAL(scanner.next(), Token::IDENTIFIER); + BOOST_CHECK_EQUAL(scanner.next(false), Token::IDENTIFIER); + BOOST_CHECK_EQUAL(scanner.next(false), Token::COMMENT_LITERAL); + BOOST_CHECK_EQUAL(scanner.getCurrentLiteral(), " Send $(value / 1000) chocolates to the user"); +} + +BOOST_AUTO_TEST_CASE(documentation_comments_skipped) +{ + Scanner scanner(CharStream("some other tokens /// Send $(value / 1000) chocolates to the user")); + BOOST_CHECK_EQUAL(scanner.getCurrentToken(), Token::IDENTIFIER); + BOOST_CHECK_EQUAL(scanner.next(), Token::IDENTIFIER); + BOOST_CHECK_EQUAL(scanner.next(), Token::IDENTIFIER); + BOOST_CHECK_EQUAL(scanner.next(), Token::EOS); +} + +BOOST_AUTO_TEST_CASE(comment_before_eos) +{ + Scanner scanner(CharStream("//")); + BOOST_CHECK_EQUAL(scanner.getCurrentToken(), Token::EOS); +} + +BOOST_AUTO_TEST_CASE(documentation_comment_before_eos) +{ + Scanner scanner(CharStream("///"), false); + BOOST_CHECK_EQUAL(scanner.getCurrentToken(), Token::COMMENT_LITERAL); + BOOST_CHECK_EQUAL(scanner.getCurrentLiteral(), ""); +} BOOST_AUTO_TEST_SUITE_END() From 0788e326ced4e1deeb0514133bcc67dd7bb71c25 Mon Sep 17 00:00:00 2001 From: Lefteris Karapetsas Date: Wed, 19 Nov 2014 02:02:30 +0100 Subject: [PATCH 2/7] fixing typo and alignment --- libsolidity/Scanner.h | 2 +- libsolidity/Token.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libsolidity/Scanner.h b/libsolidity/Scanner.h index fd48d5698..402f1aea8 100644 --- a/libsolidity/Scanner.h +++ b/libsolidity/Scanner.h @@ -179,7 +179,7 @@ private: bool scanHexByte(char& o_scannedByte); - /// Scans a single JavaScript token. + /// Scans a single Solidity token. void scanToken(bool _skipDocumentationComments = true); /// Skips all whitespace and @returns true if something was skipped. diff --git a/libsolidity/Token.h b/libsolidity/Token.h index 39a84bcc2..524487521 100644 --- a/libsolidity/Token.h +++ b/libsolidity/Token.h @@ -281,7 +281,7 @@ namespace solidity K(FALSE_LITERAL, "false", 0) \ T(NUMBER, NULL, 0) \ T(STRING_LITERAL, NULL, 0) \ - T(COMMENT_LITERAL, NULL, 0) \ + T(COMMENT_LITERAL, NULL, 0) \ \ /* Identifiers (not keywords or future reserved words). */ \ T(IDENTIFIER, NULL, 0) \ From be1d8881af0271644395f94f7988f627edf4c960 Mon Sep 17 00:00:00 2001 From: Lefteris Karapetsas Date: Wed, 19 Nov 2014 16:21:42 +0100 Subject: [PATCH 3/7] documentation comments are now always skipped but saved as special tokens at the Scanner --- libsolidity/Scanner.cpp | 31 ++++++++++++++++++++++--------- libsolidity/Scanner.h | 34 +++++++++++++++++++++------------- test/solidityScanner.cpp | 25 +++++-------------------- 3 files changed, 48 insertions(+), 42 deletions(-) diff --git a/libsolidity/Scanner.cpp b/libsolidity/Scanner.cpp index 382d07a99..cddc687a9 100644 --- a/libsolidity/Scanner.cpp +++ b/libsolidity/Scanner.cpp @@ -102,13 +102,14 @@ int HexValue(char c) } } // end anonymous namespace -void Scanner::reset(CharStream const& _source, bool _skipDocumentationComments) +void Scanner::reset(CharStream const& _source) { + bool found_doc_comment; m_source = _source; m_char = m_source.get(); skipWhitespace(); - scanToken(_skipDocumentationComments); - next(_skipDocumentationComments); + found_doc_comment = scanToken(); + next(found_doc_comment); } @@ -134,10 +135,11 @@ bool Scanner::scanHexByte(char& o_scannedByte) // Ensure that tokens can be stored in a byte. BOOST_STATIC_ASSERT(Token::NUM_TOKENS <= 0x100); -Token::Value Scanner::next(bool _skipDocumentationComments) +Token::Value Scanner::next(bool _change_skipped_comment) { m_current_token = m_next_token; - scanToken(_skipDocumentationComments); + if (scanToken() || _change_skipped_comment) + m_skipped_comment = m_next_skipped_comment; return m_current_token.token; } @@ -180,7 +182,7 @@ Token::Value Scanner::scanDocumentationComment() { char c = m_char; advance(); - addLiteralChar(c); + addCommentLiteralChar(c); } literal.Complete(); return Token::COMMENT_LITERAL; @@ -209,8 +211,9 @@ Token::Value Scanner::skipMultiLineComment() return Token::ILLEGAL; } -void Scanner::scanToken(bool _skipDocumentationComments) +bool Scanner::scanToken() { + bool found_doc_comment = false; m_next_token.literal.clear(); Token::Value token; do @@ -315,8 +318,16 @@ void Scanner::scanToken(bool _skipDocumentationComments) { if (!advance()) /* double slash comment directly before EOS */ token = Token::WHITESPACE; - else if (!_skipDocumentationComments) - token = scanDocumentationComment(); + else if (m_char == '/') + { + Token::Value comment; + m_next_skipped_comment.location.start = getSourcePos(); + comment = scanDocumentationComment(); + m_next_skipped_comment.location.end = getSourcePos(); + m_next_skipped_comment.token = comment; + token = Token::WHITESPACE; + found_doc_comment = true; + } else token = skipSingleLineComment(); } @@ -411,6 +422,8 @@ void Scanner::scanToken(bool _skipDocumentationComments) while (token == Token::WHITESPACE); m_next_token.location.end = getSourcePos(); m_next_token.token = token; + + return found_doc_comment; } bool Scanner::scanEscape() diff --git a/libsolidity/Scanner.h b/libsolidity/Scanner.h index 402f1aea8..23007fe12 100644 --- a/libsolidity/Scanner.h +++ b/libsolidity/Scanner.h @@ -111,31 +111,34 @@ public: }; Scanner() { reset(CharStream()); } - explicit Scanner(CharStream const& _source, bool _skipDocumentationComments = true) - { - reset(_source, _skipDocumentationComments); - } + explicit Scanner(CharStream const& _source) { reset(_source); } /// Resets the scanner as if newly constructed with _input as input. - void reset(CharStream const& _source, bool _skipDocumentationComments = true); + void reset(CharStream const& _source); - /// Returns the next token and advances input. - Token::Value next(bool _skipDocumentationComments = true); + /// Returns the next token and advances input. If called from reset() + /// and ScanToken() found a documentation token then next should be called + /// with _change_skipped_comment=true + Token::Value next(bool _change_skipped_comment = false); ///@{ ///@name Information about the current token /// Returns the current token - Token::Value getCurrentToken(bool _skipDocumentationComments = true) + Token::Value getCurrentToken() { - if (!_skipDocumentationComments) - next(_skipDocumentationComments); return m_current_token.token; } Location getCurrentLocation() const { return m_current_token.location; } std::string const& getCurrentLiteral() const { return m_current_token.literal; } ///@} + ///@{ + ///@name Information about the current comment token + Location getCurrentCommentLocation() const { return m_skipped_comment.location; } + std::string const& getCurrentCommentLiteral() const { return m_skipped_comment.literal; } + ///@} + ///@{ ///@name Information about the next token @@ -154,7 +157,7 @@ public: ///@} private: - // Used for the current and look-ahead token. + // Used for the current and look-ahead token and comments struct TokenDesc { Token::Value token; @@ -166,6 +169,7 @@ private: ///@name Literal buffer support inline void startNewLiteral() { m_next_token.literal.clear(); } inline void addLiteralChar(char c) { m_next_token.literal.push_back(c); } + inline void addCommentLiteralChar(char c) { m_next_skipped_comment.literal.push_back(c); } inline void dropLiteral() { m_next_token.literal.clear(); } inline void addLiteralCharAndAdvance() { addLiteralChar(m_char); advance(); } ///@} @@ -179,8 +183,9 @@ private: bool scanHexByte(char& o_scannedByte); - /// Scans a single Solidity token. - void scanToken(bool _skipDocumentationComments = true); + /// Scans a single Solidity token. Returns true if the scanned token was + /// a skipped documentation comment. False in all other cases. + bool scanToken(); /// Skips all whitespace and @returns true if something was skipped. bool skipWhitespace(); @@ -203,6 +208,9 @@ private: int getSourcePos() { return m_source.getPos(); } bool isSourcePastEndOfInput() { return m_source.isPastEndOfInput(); } + TokenDesc m_skipped_comment; // desc for current skipped comment + TokenDesc m_next_skipped_comment; // desc for next skiped comment + TokenDesc m_current_token; // desc for current token (as returned by Next()) TokenDesc m_next_token; // desc for next token (one token look-ahead) diff --git a/test/solidityScanner.cpp b/test/solidityScanner.cpp index 28f52d3af..1a2299f37 100644 --- a/test/solidityScanner.cpp +++ b/test/solidityScanner.cpp @@ -154,35 +154,20 @@ BOOST_AUTO_TEST_CASE(ambiguities) } BOOST_AUTO_TEST_CASE(documentation_comments_parsed_begin) -{ - Scanner scanner(CharStream("/// Send $(value / 1000) chocolates to the user"), false); - BOOST_CHECK_EQUAL(scanner.getCurrentToken(), Token::COMMENT_LITERAL); - BOOST_CHECK_EQUAL(scanner.getCurrentLiteral(), " Send $(value / 1000) chocolates to the user"); -} - -BOOST_AUTO_TEST_CASE(documentation_comments_skipped_begin) { Scanner scanner(CharStream("/// Send $(value / 1000) chocolates to the user")); BOOST_CHECK_EQUAL(scanner.getCurrentToken(), Token::EOS); + BOOST_CHECK_EQUAL(scanner.getCurrentCommentLiteral(), " Send $(value / 1000) chocolates to the user"); } BOOST_AUTO_TEST_CASE(documentation_comments_parsed) -{ - Scanner scanner(CharStream("some other tokens /// Send $(value / 1000) chocolates to the user")); - BOOST_CHECK_EQUAL(scanner.getCurrentToken(), Token::IDENTIFIER); - BOOST_CHECK_EQUAL(scanner.next(), Token::IDENTIFIER); - BOOST_CHECK_EQUAL(scanner.next(false), Token::IDENTIFIER); - BOOST_CHECK_EQUAL(scanner.next(false), Token::COMMENT_LITERAL); - BOOST_CHECK_EQUAL(scanner.getCurrentLiteral(), " Send $(value / 1000) chocolates to the user"); -} - -BOOST_AUTO_TEST_CASE(documentation_comments_skipped) { Scanner scanner(CharStream("some other tokens /// Send $(value / 1000) chocolates to the user")); BOOST_CHECK_EQUAL(scanner.getCurrentToken(), Token::IDENTIFIER); BOOST_CHECK_EQUAL(scanner.next(), Token::IDENTIFIER); BOOST_CHECK_EQUAL(scanner.next(), Token::IDENTIFIER); BOOST_CHECK_EQUAL(scanner.next(), Token::EOS); + BOOST_CHECK_EQUAL(scanner.getCurrentCommentLiteral(), " Send $(value / 1000) chocolates to the user"); } BOOST_AUTO_TEST_CASE(comment_before_eos) @@ -193,9 +178,9 @@ BOOST_AUTO_TEST_CASE(comment_before_eos) BOOST_AUTO_TEST_CASE(documentation_comment_before_eos) { - Scanner scanner(CharStream("///"), false); - BOOST_CHECK_EQUAL(scanner.getCurrentToken(), Token::COMMENT_LITERAL); - BOOST_CHECK_EQUAL(scanner.getCurrentLiteral(), ""); + Scanner scanner(CharStream("///")); + BOOST_CHECK_EQUAL(scanner.getCurrentToken(), Token::EOS); + BOOST_CHECK_EQUAL(scanner.getCurrentCommentLiteral(), ""); } BOOST_AUTO_TEST_SUITE_END() From 03f8208b6b02ded3f506f559ebe6d5854cbb040a Mon Sep 17 00:00:00 2001 From: Lefteris Karapetsas Date: Thu, 20 Nov 2014 22:08:16 +0100 Subject: [PATCH 4/7] styling fixes --- libsolidity/Scanner.cpp | 22 +++++++++++----------- libsolidity/Scanner.h | 4 ++-- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/libsolidity/Scanner.cpp b/libsolidity/Scanner.cpp index cddc687a9..934b30dcf 100644 --- a/libsolidity/Scanner.cpp +++ b/libsolidity/Scanner.cpp @@ -104,12 +104,12 @@ int HexValue(char c) void Scanner::reset(CharStream const& _source) { - bool found_doc_comment; + bool foundDocComment; m_source = _source; m_char = m_source.get(); skipWhitespace(); - found_doc_comment = scanToken(); - next(found_doc_comment); + foundDocComment = scanToken(); + next(foundDocComment); } @@ -135,10 +135,10 @@ bool Scanner::scanHexByte(char& o_scannedByte) // Ensure that tokens can be stored in a byte. BOOST_STATIC_ASSERT(Token::NUM_TOKENS <= 0x100); -Token::Value Scanner::next(bool _change_skipped_comment) +Token::Value Scanner::next(bool _changeSkippedComment) { m_current_token = m_next_token; - if (scanToken() || _change_skipped_comment) + if (scanToken() || _changeSkippedComment) m_skipped_comment = m_next_skipped_comment; return m_current_token.token; } @@ -173,7 +173,7 @@ Token::Value Scanner::skipSingleLineComment() return Token::WHITESPACE; } -// For the moment this function simply consumes a single line triple slash doc comment +/// For the moment this function simply consumes a single line triple slash doc comment Token::Value Scanner::scanDocumentationComment() { LiteralScope literal(this); @@ -213,7 +213,7 @@ Token::Value Scanner::skipMultiLineComment() bool Scanner::scanToken() { - bool found_doc_comment = false; + bool foundDocComment = false; m_next_token.literal.clear(); Token::Value token; do @@ -326,7 +326,7 @@ bool Scanner::scanToken() m_next_skipped_comment.location.end = getSourcePos(); m_next_skipped_comment.token = comment; token = Token::WHITESPACE; - found_doc_comment = true; + foundDocComment = true; } else token = skipSingleLineComment(); @@ -423,7 +423,7 @@ bool Scanner::scanToken() m_next_token.location.end = getSourcePos(); m_next_token.token = token; - return found_doc_comment; + return foundDocComment; } bool Scanner::scanEscape() @@ -567,9 +567,9 @@ Token::Value Scanner::scanNumber(char _charSeen) // ---------------------------------------------------------------------------- // Keyword Matcher -#define KEYWORDS(KEYWORD_GROUP, KEYWORD) \ +#define KEYWORDS(KEYWORD_GROUP, KEYWORD) \ KEYWORD_GROUP('a') \ - KEYWORD("address", Token::ADDRESS) \ + KEYWORD("address", Token::ADDRESS) \ KEYWORD_GROUP('b') \ KEYWORD("break", Token::BREAK) \ KEYWORD("bool", Token::BOOL) \ diff --git a/libsolidity/Scanner.h b/libsolidity/Scanner.h index 23007fe12..0a6778ecd 100644 --- a/libsolidity/Scanner.h +++ b/libsolidity/Scanner.h @@ -119,7 +119,7 @@ public: /// Returns the next token and advances input. If called from reset() /// and ScanToken() found a documentation token then next should be called /// with _change_skipped_comment=true - Token::Value next(bool _change_skipped_comment = false); + Token::Value next(bool _changeSkippedComment = false); ///@{ ///@name Information about the current token @@ -157,7 +157,7 @@ public: ///@} private: - // Used for the current and look-ahead token and comments + /// Used for the current and look-ahead token and comments struct TokenDesc { Token::Value token; From 06e2c08af73628c7a614b50301904a7d681d8a75 Mon Sep 17 00:00:00 2001 From: Lefteris Karapetsas Date: Thu, 20 Nov 2014 23:18:05 +0100 Subject: [PATCH 5/7] extra comments scanning test --- libsolidity/Scanner.h | 2 +- test/solidityScanner.cpp | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/libsolidity/Scanner.h b/libsolidity/Scanner.h index 0a6778ecd..94c67840a 100644 --- a/libsolidity/Scanner.h +++ b/libsolidity/Scanner.h @@ -118,7 +118,7 @@ public: /// Returns the next token and advances input. If called from reset() /// and ScanToken() found a documentation token then next should be called - /// with _change_skipped_comment=true + /// with _changeSkippedComment=true Token::Value next(bool _changeSkippedComment = false); ///@{ diff --git a/test/solidityScanner.cpp b/test/solidityScanner.cpp index 1a2299f37..573affe6d 100644 --- a/test/solidityScanner.cpp +++ b/test/solidityScanner.cpp @@ -174,6 +174,7 @@ BOOST_AUTO_TEST_CASE(comment_before_eos) { Scanner scanner(CharStream("//")); BOOST_CHECK_EQUAL(scanner.getCurrentToken(), Token::EOS); + BOOST_CHECK_EQUAL(scanner.getCurrentCommentLiteral(), ""); } BOOST_AUTO_TEST_CASE(documentation_comment_before_eos) @@ -183,6 +184,16 @@ BOOST_AUTO_TEST_CASE(documentation_comment_before_eos) BOOST_CHECK_EQUAL(scanner.getCurrentCommentLiteral(), ""); } +BOOST_AUTO_TEST_CASE(comments_mixed_in_sequence) +{ + Scanner scanner(CharStream("hello_world ///documentation comment \n" + "//simple comment \n" + "<<")); + BOOST_CHECK_EQUAL(scanner.getCurrentToken(), Token::IDENTIFIER); + BOOST_CHECK_EQUAL(scanner.next(), Token::SHL); + BOOST_CHECK_EQUAL(scanner.getCurrentCommentLiteral(), "documentation comment "); +} + BOOST_AUTO_TEST_SUITE_END() } From babddd394ec774a3b5d7426a6469bbd836de6f9a Mon Sep 17 00:00:00 2001 From: Lefteris Karapetsas Date: Thu, 20 Nov 2014 23:56:24 +0100 Subject: [PATCH 6/7] cleaning up the external interface of Scanner::next(). No special cases --- libsolidity/Scanner.cpp | 10 +++++++--- libsolidity/Scanner.h | 6 ++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/libsolidity/Scanner.cpp b/libsolidity/Scanner.cpp index 934b30dcf..d30000999 100644 --- a/libsolidity/Scanner.cpp +++ b/libsolidity/Scanner.cpp @@ -109,7 +109,11 @@ void Scanner::reset(CharStream const& _source) m_char = m_source.get(); skipWhitespace(); foundDocComment = scanToken(); - next(foundDocComment); + + // special version of Scanner:next() taking the previous scanToken() result into account + m_current_token = m_next_token; + if (scanToken() || foundDocComment) + m_skipped_comment = m_next_skipped_comment; } @@ -135,10 +139,10 @@ bool Scanner::scanHexByte(char& o_scannedByte) // Ensure that tokens can be stored in a byte. BOOST_STATIC_ASSERT(Token::NUM_TOKENS <= 0x100); -Token::Value Scanner::next(bool _changeSkippedComment) +Token::Value Scanner::next() { m_current_token = m_next_token; - if (scanToken() || _changeSkippedComment) + if (scanToken()) m_skipped_comment = m_next_skipped_comment; return m_current_token.token; } diff --git a/libsolidity/Scanner.h b/libsolidity/Scanner.h index 94c67840a..5dfe7a33a 100644 --- a/libsolidity/Scanner.h +++ b/libsolidity/Scanner.h @@ -116,10 +116,8 @@ public: /// Resets the scanner as if newly constructed with _input as input. void reset(CharStream const& _source); - /// Returns the next token and advances input. If called from reset() - /// and ScanToken() found a documentation token then next should be called - /// with _changeSkippedComment=true - Token::Value next(bool _changeSkippedComment = false); + /// Returns the next token and advances input + Token::Value next(); ///@{ ///@name Information about the current token From 7f959f12921a7620c7b3de01da5dfbbcf0d6cb95 Mon Sep 17 00:00:00 2001 From: Lefteris Karapetsas Date: Fri, 21 Nov 2014 09:09:39 +0100 Subject: [PATCH 7/7] simplifying scanDocumentationComment() --- libsolidity/Scanner.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libsolidity/Scanner.cpp b/libsolidity/Scanner.cpp index d30000999..9382b1346 100644 --- a/libsolidity/Scanner.cpp +++ b/libsolidity/Scanner.cpp @@ -184,9 +184,8 @@ Token::Value Scanner::scanDocumentationComment() advance(); //consume the last '/' while (!isSourcePastEndOfInput() && !IsLineTerminator(m_char)) { - char c = m_char; + addCommentLiteralChar(m_char); advance(); - addCommentLiteralChar(c); } literal.Complete(); return Token::COMMENT_LITERAL;