diff --git a/libsolidity/Scanner.cpp b/libsolidity/Scanner.cpp index b13e52d7e..9382b1346 100644 --- a/libsolidity/Scanner.cpp +++ b/libsolidity/Scanner.cpp @@ -104,11 +104,16 @@ int HexValue(char c) void Scanner::reset(CharStream const& _source) { + bool foundDocComment; m_source = _source; m_char = m_source.get(); skipWhitespace(); - scanToken(); - next(); + foundDocComment = scanToken(); + + // special version of Scanner:next() taking the previous scanToken() result into account + m_current_token = m_next_token; + if (scanToken() || foundDocComment) + m_skipped_comment = m_next_skipped_comment; } @@ -137,7 +142,8 @@ BOOST_STATIC_ASSERT(Token::NUM_TOKENS <= 0x100); Token::Value Scanner::next() { m_current_token = m_next_token; - scanToken(); + if (scanToken()) + m_skipped_comment = m_next_skipped_comment; return m_current_token.token; } @@ -171,6 +177,20 @@ Token::Value Scanner::skipSingleLineComment() return Token::WHITESPACE; } +/// For the moment this function simply consumes a single line triple slash doc comment +Token::Value Scanner::scanDocumentationComment() +{ + LiteralScope literal(this); + advance(); //consume the last '/' + while (!isSourcePastEndOfInput() && !IsLineTerminator(m_char)) + { + addCommentLiteralChar(m_char); + advance(); + } + literal.Complete(); + return Token::COMMENT_LITERAL; +} + Token::Value Scanner::skipMultiLineComment() { if (asserts(m_char == '*')) @@ -194,8 +214,9 @@ Token::Value Scanner::skipMultiLineComment() return Token::ILLEGAL; } -void Scanner::scanToken() +bool Scanner::scanToken() { + bool foundDocComment = false; m_next_token.literal.clear(); Token::Value token; do @@ -297,7 +318,22 @@ void Scanner::scanToken() // / // /* /= advance(); if (m_char == '/') - token = skipSingleLineComment(); + { + if (!advance()) /* double slash comment directly before EOS */ + token = Token::WHITESPACE; + else if (m_char == '/') + { + Token::Value comment; + m_next_skipped_comment.location.start = getSourcePos(); + comment = scanDocumentationComment(); + m_next_skipped_comment.location.end = getSourcePos(); + m_next_skipped_comment.token = comment; + token = Token::WHITESPACE; + foundDocComment = true; + } + else + token = skipSingleLineComment(); + } else if (m_char == '*') token = skipMultiLineComment(); else if (m_char == '=') @@ -389,6 +425,8 @@ void Scanner::scanToken() while (token == Token::WHITESPACE); m_next_token.location.end = getSourcePos(); m_next_token.token = token; + + return foundDocComment; } bool Scanner::scanEscape() @@ -532,9 +570,9 @@ Token::Value Scanner::scanNumber(char _charSeen) // ---------------------------------------------------------------------------- // Keyword Matcher -#define KEYWORDS(KEYWORD_GROUP, KEYWORD) \ +#define KEYWORDS(KEYWORD_GROUP, KEYWORD) \ KEYWORD_GROUP('a') \ - KEYWORD("address", Token::ADDRESS) \ + KEYWORD("address", Token::ADDRESS) \ KEYWORD_GROUP('b') \ KEYWORD("break", Token::BREAK) \ KEYWORD("bool", Token::BOOL) \ diff --git a/libsolidity/Scanner.h b/libsolidity/Scanner.h index 997365f3c..5dfe7a33a 100644 --- a/libsolidity/Scanner.h +++ b/libsolidity/Scanner.h @@ -116,18 +116,27 @@ public: /// Resets the scanner as if newly constructed with _input as input. void reset(CharStream const& _source); - /// Returns the next token and advances input. + /// Returns the next token and advances input Token::Value next(); ///@{ ///@name Information about the current token /// Returns the current token - Token::Value getCurrentToken() { return m_current_token.token; } + Token::Value getCurrentToken() + { + return m_current_token.token; + } Location getCurrentLocation() const { return m_current_token.location; } std::string const& getCurrentLiteral() const { return m_current_token.literal; } ///@} + ///@{ + ///@name Information about the current comment token + Location getCurrentCommentLocation() const { return m_skipped_comment.location; } + std::string const& getCurrentCommentLiteral() const { return m_skipped_comment.literal; } + ///@} + ///@{ ///@name Information about the next token @@ -146,7 +155,7 @@ public: ///@} private: - // Used for the current and look-ahead token. + /// Used for the current and look-ahead token and comments struct TokenDesc { Token::Value token; @@ -158,6 +167,7 @@ private: ///@name Literal buffer support inline void startNewLiteral() { m_next_token.literal.clear(); } inline void addLiteralChar(char c) { m_next_token.literal.push_back(c); } + inline void addCommentLiteralChar(char c) { m_next_skipped_comment.literal.push_back(c); } inline void dropLiteral() { m_next_token.literal.clear(); } inline void addLiteralCharAndAdvance() { addLiteralChar(m_char); advance(); } ///@} @@ -171,8 +181,9 @@ private: bool scanHexByte(char& o_scannedByte); - /// Scans a single JavaScript token. - void scanToken(); + /// Scans a single Solidity token. Returns true if the scanned token was + /// a skipped documentation comment. False in all other cases. + bool scanToken(); /// Skips all whitespace and @returns true if something was skipped. bool skipWhitespace(); @@ -184,6 +195,7 @@ private: Token::Value scanIdentifierOrKeyword(); Token::Value scanString(); + Token::Value scanDocumentationComment(); /// Scans an escape-sequence which is part of a string and adds the /// decoded character to the current literal. Returns true if a pattern @@ -194,6 +206,9 @@ private: int getSourcePos() { return m_source.getPos(); } bool isSourcePastEndOfInput() { return m_source.isPastEndOfInput(); } + TokenDesc m_skipped_comment; // desc for current skipped comment + TokenDesc m_next_skipped_comment; // desc for next skiped comment + TokenDesc m_current_token; // desc for current token (as returned by Next()) TokenDesc m_next_token; // desc for next token (one token look-ahead) diff --git a/libsolidity/Token.h b/libsolidity/Token.h index 67971c3d0..524487521 100644 --- a/libsolidity/Token.h +++ b/libsolidity/Token.h @@ -281,6 +281,7 @@ namespace solidity K(FALSE_LITERAL, "false", 0) \ T(NUMBER, NULL, 0) \ T(STRING_LITERAL, NULL, 0) \ + T(COMMENT_LITERAL, NULL, 0) \ \ /* Identifiers (not keywords or future reserved words). */ \ T(IDENTIFIER, NULL, 0) \ diff --git a/test/solidityScanner.cpp b/test/solidityScanner.cpp index d714699a0..573affe6d 100644 --- a/test/solidityScanner.cpp +++ b/test/solidityScanner.cpp @@ -153,6 +153,46 @@ BOOST_AUTO_TEST_CASE(ambiguities) BOOST_CHECK_EQUAL(scanner.next(), Token::SHL); } +BOOST_AUTO_TEST_CASE(documentation_comments_parsed_begin) +{ + Scanner scanner(CharStream("/// Send $(value / 1000) chocolates to the user")); + BOOST_CHECK_EQUAL(scanner.getCurrentToken(), Token::EOS); + BOOST_CHECK_EQUAL(scanner.getCurrentCommentLiteral(), " Send $(value / 1000) chocolates to the user"); +} + +BOOST_AUTO_TEST_CASE(documentation_comments_parsed) +{ + Scanner scanner(CharStream("some other tokens /// Send $(value / 1000) chocolates to the user")); + BOOST_CHECK_EQUAL(scanner.getCurrentToken(), Token::IDENTIFIER); + BOOST_CHECK_EQUAL(scanner.next(), Token::IDENTIFIER); + BOOST_CHECK_EQUAL(scanner.next(), Token::IDENTIFIER); + BOOST_CHECK_EQUAL(scanner.next(), Token::EOS); + BOOST_CHECK_EQUAL(scanner.getCurrentCommentLiteral(), " Send $(value / 1000) chocolates to the user"); +} + +BOOST_AUTO_TEST_CASE(comment_before_eos) +{ + Scanner scanner(CharStream("//")); + BOOST_CHECK_EQUAL(scanner.getCurrentToken(), Token::EOS); + BOOST_CHECK_EQUAL(scanner.getCurrentCommentLiteral(), ""); +} + +BOOST_AUTO_TEST_CASE(documentation_comment_before_eos) +{ + Scanner scanner(CharStream("///")); + BOOST_CHECK_EQUAL(scanner.getCurrentToken(), Token::EOS); + BOOST_CHECK_EQUAL(scanner.getCurrentCommentLiteral(), ""); +} + +BOOST_AUTO_TEST_CASE(comments_mixed_in_sequence) +{ + Scanner scanner(CharStream("hello_world ///documentation comment \n" + "//simple comment \n" + "<<")); + BOOST_CHECK_EQUAL(scanner.getCurrentToken(), Token::IDENTIFIER); + BOOST_CHECK_EQUAL(scanner.next(), Token::SHL); + BOOST_CHECK_EQUAL(scanner.getCurrentCommentLiteral(), "documentation comment "); +} BOOST_AUTO_TEST_SUITE_END()