node/tools/closure_linter/closure_linter/javascripttokens.py

#!/usr/bin/env python
#
# Copyright 2008 The Closure Linter Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Classes to represent JavaScript tokens."""

__author__ = ('robbyw@google.com (Robert Walker)',
              'ajp@google.com (Andy Perelson)')

from closure_linter.common import tokens

class JavaScriptTokenType(tokens.TokenType):
  """Enumeration of JavaScript token types, and useful sets of token types."""
  NUMBER = 'number'
  START_SINGLE_LINE_COMMENT = '//'
  START_BLOCK_COMMENT = '/*'
  START_DOC_COMMENT = '/**'
  END_BLOCK_COMMENT = '*/'
  END_DOC_COMMENT = 'doc */'
  COMMENT = 'comment'
  SINGLE_QUOTE_STRING_START = "'string"
  SINGLE_QUOTE_STRING_END = "string'"
  DOUBLE_QUOTE_STRING_START = '"string'
  DOUBLE_QUOTE_STRING_END = 'string"'
  STRING_TEXT = 'string'
  START_BLOCK = '{'
  END_BLOCK = '}'
  START_PAREN = '('
  END_PAREN = ')'
  START_BRACKET = '['
  END_BRACKET = ']'
  REGEX = '/regex/'
  FUNCTION_DECLARATION = 'function(...)'
  FUNCTION_NAME = 'function functionName(...)'
  START_PARAMETERS = 'startparams('
  PARAMETERS = 'pa,ra,ms'
  END_PARAMETERS = ')endparams'
  SEMICOLON = ';'
  DOC_FLAG = '@flag'
  DOC_INLINE_FLAG = '{@flag ...}'
  DOC_START_BRACE = 'doc {'
  DOC_END_BRACE = 'doc }'
  DOC_PREFIX = 'comment prefix: * '
  SIMPLE_LVALUE = 'lvalue='
  KEYWORD = 'keyword'
  OPERATOR = 'operator'
  IDENTIFIER = 'identifier'

  STRING_TYPES = frozenset([
      SINGLE_QUOTE_STRING_START, SINGLE_QUOTE_STRING_END,
      DOUBLE_QUOTE_STRING_START, DOUBLE_QUOTE_STRING_END, STRING_TEXT])

  COMMENT_TYPES = frozenset([START_SINGLE_LINE_COMMENT, COMMENT,
      START_BLOCK_COMMENT, START_DOC_COMMENT,
      END_BLOCK_COMMENT, END_DOC_COMMENT,
      DOC_START_BRACE, DOC_END_BRACE,
      DOC_FLAG, DOC_INLINE_FLAG, DOC_PREFIX])

  FLAG_DESCRIPTION_TYPES = frozenset([
      DOC_INLINE_FLAG, COMMENT, DOC_START_BRACE, DOC_END_BRACE])

  FLAG_ENDING_TYPES = frozenset([DOC_FLAG, END_DOC_COMMENT])

  NON_CODE_TYPES = COMMENT_TYPES | frozenset([
      tokens.TokenType.WHITESPACE, tokens.TokenType.BLANK_LINE])

  UNARY_OPERATORS = ['!', 'new', 'delete', 'typeof', 'void']

  UNARY_OK_OPERATORS = ['--', '++', '-', '+'] + UNARY_OPERATORS

  UNARY_POST_OPERATORS = ['--', '++']

  # An expression ender is any token that can end an object - i.e. we could have
  # x.y or [1, 2], or (10 + 9) or {a: 10}.
  EXPRESSION_ENDER_TYPES = [tokens.TokenType.NORMAL, IDENTIFIER, NUMBER,
                            SIMPLE_LVALUE, END_BRACKET, END_PAREN, END_BLOCK,
                            SINGLE_QUOTE_STRING_END, DOUBLE_QUOTE_STRING_END]


class JavaScriptToken(tokens.Token):
  """JavaScript token subclass of Token, provides extra instance checks.

  The following token types have data in attached_object:
    - All JsDoc flags: a parser.JsDocFlag object.
  """

  def IsKeyword(self, keyword):
    """Tests if this token is the given keyword.

    Args:
      keyword: The keyword to compare to.

    Returns:
      True if this token is a keyword token with the given name.
    """
    return self.type == JavaScriptTokenType.KEYWORD and self.string == keyword

  def IsOperator(self, operator):
    """Tests if this token is the given operator.

    Args:
      operator: The operator to compare to.

    Returns:
      True if this token is a operator token with the given name.
    """
    return self.type == JavaScriptTokenType.OPERATOR and self.string == operator

  def IsAssignment(self):
    """Tests if this token is an assignment operator.

    Returns:
      True if this token is an assignment operator.
    """
    return (self.type == JavaScriptTokenType.OPERATOR and
            self.string.endswith('=') and
            self.string not in ('==', '!=', '>=', '<=', '===', '!=='))

  def IsComment(self):
    """Tests if this token is any part of a comment.

    Returns:
      True if this token is any part of a comment.
    """
    return self.type in JavaScriptTokenType.COMMENT_TYPES

  def IsCode(self):
    """Tests if this token is code, as opposed to a comment or whitespace."""
    return self.type not in JavaScriptTokenType.NON_CODE_TYPES

  def __repr__(self):
    return '<JavaScriptToken: %d, %s, "%s", %r, %r>' % (self.line_number,
                                                        self.type, self.string,
                                                        self.values,
                                                        self.metadata)