def CheckCasts(filename, clean_lines, linenum, error):
  """Various cast related checks.

  Args:
    filename: The name of the current file.
    clean_lines: A CleansedLines instance containing the file.
    linenum: The number of the line to check.
    error: The function to call with any errors found.
  """
  line = clean_lines.elided[linenum]

  # Check to see if they're using an conversion function cast.
  # I just try to capture the most common basic types, though there are more.
  # Parameterless conversion functions, such as bool(), are allowed as they are
  # probably a member operator declaration or default constructor.
  match = Search(
      r'(\bnew\s+(?:const\s+)?|\S<\s*(?:const\s+)?)?\b'
      r'(int|float|double|bool|char|int32|uint32|int64|uint64)'
      r'(\([^)].*)', line)
  expecting_function = ExpectingFunctionArgs(clean_lines, linenum)
  if match and not expecting_function:
    matched_type = match.group(2)

    # matched_new_or_template is used to silence two false positives:
    # - New operators
    # - Template arguments with function types
    #
    # For template arguments, we match on types immediately following
    # an opening bracket without any spaces.  This is a fast way to
    # silence the common case where the function type is the first
    # template argument.  False negative with less-than comparison is
    # avoided because those operators are usually followed by a space.
    #
    #   function<double(double)>   // bracket + no space = false positive
    #   value < double(42)         // bracket + space = true positive
    matched_new_or_template = match.group(1)

    # Avoid arrays by looking for brackets that come after the closing
    # parenthesis.
    if Match(r'\([^()]+\)\s*\[', match.group(3)):
      return

    # Other things to ignore:
    # - Function pointers
    # - Casts to pointer types
    # - Placement new
    # - Alias declarations
    matched_funcptr = match.group(3)
    if (matched_new_or_template is None and
        not (matched_funcptr and
             (Match(r'\((?:[^() ]+::\s*\*\s*)?[^() ]+\)\s*\(',
                    matched_funcptr) or
              matched_funcptr.startswith('(*)'))) and
        not Match(r'\s*using\s+\S+\s*=\s*' + matched_type, line) and
        not Search(r'new\(\S+\)\s*' + matched_type, line)):
      error(filename, linenum, 'readability/casting', 4,
            'Using deprecated casting style.  '
            'Use static_cast<%s>(...) instead' %
            matched_type)

  if not expecting_function:
    CheckCStyleCast(filename, clean_lines, linenum, 'static_cast',
                    r'\((int|float|double|bool|char|u?int(16|32|64))\)', error)

  # This doesn't catch all cases. Consider (const char * const)"hello".
  #
  # (char *) "foo" should always be a const_cast (reinterpret_cast won't
  # compile).
  if CheckCStyleCast(filename, clean_lines, linenum, 'const_cast',
                     r'\((char\s?\*+\s?)\)\s*"', error):
    pass
  else:
    # Check pointer casts for other than string constants
    CheckCStyleCast(filename, clean_lines, linenum, 'reinterpret_cast',
                    r'\((\w+\s?\*+\s?)\)', error)

  # In addition, we look for people taking the address of a cast.  This
  # is dangerous -- casts can assign to temporaries, so the pointer doesn't
  # point where you think.
  #
  # Some non-identifier character is required before the '&' for the
  # expression to be recognized as a cast.  These are casts:
  #   expression = &static_cast<int*>(temporary());
  #   function(&(int*)(temporary()));
  #
  # This is not a cast:
  #   reference_type&(int* function_param);
  match = Search(
      r'(?:[^\w]&\(([^)*][^)]*)\)[\w(])|'
      r'(?:[^\w]&(static|dynamic|down|reinterpret)_cast\b)', line)
  if match:
    # Try a better error message when the & is bound to something
    # dereferenced by the casted pointer, as opposed to the casted
    # pointer itself.
    parenthesis_error = False
    match = Match(r'^(.*&(?:static|dynamic|down|reinterpret)_cast\b)<', line)
    if match:
      _, y1, x1 = CloseExpression(clean_lines, linenum, len(match.group(1)))
      if x1 >= 0 and clean_lines.elided[y1][x1] == '(':
        _, y2, x2 = CloseExpression(clean_lines, y1, x1)
        if x2 >= 0:
          extended_line = clean_lines.elided[y2][x2:]
          if y2 < clean_lines.NumLines() - 1:
            extended_line += clean_lines.elided[y2 + 1]
          if Match(r'\s*(?:->|\[)', extended_line):
            parenthesis_error = True

    if parenthesis_error:
      error(filename, linenum, 'readability/casting', 4,
            ('Are you taking an address of something dereferenced '
             'from a cast?  Wrapping the dereferenced expression in '
             'parentheses will make the binding more obvious'))
    else:
      error(filename, linenum, 'runtime/casting', 4,
            ('Are you taking an address of a cast?  '
             'This is dangerous: could be a temp var.  '
             'Take the address before doing the cast, rather than after'))

#   reference_type&(int* function_param);

'Are you taking an address of a cast?  '


def CheckCStyleCast(filename, clean_lines, linenum, cast_type, pattern, error):
  """Checks for a C-style cast by looking for the pattern.

  Args:
    filename: The name of the current file.
    clean_lines: A CleansedLines instance containing the file.
    linenum: The number of the line to check.
    cast_type: The string for the C++ cast to recommend.  This is either
      reinterpret_cast, static_cast, or const_cast, depending.
    pattern: The regular expression used to find C-style casts.
    error: The function to call with any errors found.

  Returns:
    True if an error was emitted.
    False otherwise.
  """
  line = clean_lines.elided[linenum]
  match = Search(pattern, line)
  if not match:
    return False

  # Exclude lines with keywords that tend to look like casts
  context = line[0:match.start(1) - 1]
  if Match(r'.*\b(?:sizeof|alignof|alignas|[_A-Z][_A-Z0-9]*)\s*$', context):
    return False

  # Try expanding current context to see if we one level of
  # parentheses inside a macro.
  if linenum > 0:
    for i in xrange(linenum - 1, max(0, linenum - 5), -1):
      context = clean_lines.elided[i] + context
  if Match(r'.*\b[_A-Z][_A-Z0-9]*\s*\((?:\([^()]*\)|[^()])*$', context):
    return False

  # operator++(int) and operator--(int)
  if context.endswith(' operator++') or context.endswith(' operator--'):
    return False

  # A single unnamed argument for a function tends to look like old style cast.
  # If we see those, don't issue warnings for deprecated casts.
  remainder = line[match.end(0):]
  if Match(r'^\s*(?:;|const\b|throw\b|final\b|override\b|[=>{),]|->)',
           remainder):
    return False

  # At this point, all that should be left is actual casts.
  error(filename, linenum, 'readability/casting', 4,
        'Using C-style cast.  Use %s<%s>(...) instead' %
        (cast_type, match.group(1)))

  return True

Checks for a C - style cast by looking for the pattern .


def ExpectingFunctionArgs(clean_lines, linenum):
  """Checks whether where function type arguments are expected.

  Args:
    clean_lines: A CleansedLines instance containing the file.
    linenum: The number of the line to check.

  Returns:
    True if the line at 'linenum' is inside something that expects arguments
    of function types.
  """
  line = clean_lines.elided[linenum]
  return (Match(r'^\s*MOCK_(CONST_)?METHOD\d+(_T)?\(', line) or
          (linenum >= 2 and
           (Match(r'^\s*MOCK_(?:CONST_)?METHOD\d+(?:_T)?\((?:\S+,)?\s*$',
                  clean_lines.elided[linenum - 1]) or
            Match(r'^\s*MOCK_(?:CONST_)?METHOD\d+(?:_T)?\(\s*$',
                  clean_lines.elided[linenum - 2]) or
            Search(r'\bstd::m?function\s*\<\s*$',
                   clean_lines.elided[linenum - 1]))))

Checks whether where function type arguments are expected .


def FilesBelongToSameModule(filename_cc, filename_h):
  """Check if these two filenames belong to the same module.

  The concept of a 'module' here is a as follows:
  foo.h, foo-inl.h, foo.cc, foo_test.cc and foo_unittest.cc belong to the
  same 'module' if they are in the same directory.
  some/path/public/xyzzy and some/path/internal/xyzzy are also considered
  to belong to the same module here.

  If the filename_cc contains a longer path than the filename_h, for example,
  '/absolute/path/to/base/sysinfo.cc', and this file would include
  'base/sysinfo.h', this function also produces the prefix needed to open the
  header. This is used by the caller of this function to more robustly open the
  header file. We don't have access to the real include paths in this context,
  so we need this guesswork here.

  Known bugs: tools/base/bar.cc and base/bar.h belong to the same module
  according to this implementation. Because of this, this function gives
  some false positives. This should be sufficiently rare in practice.

  Args:
    filename_cc: is the path for the source (e.g. .cc) file
    filename_h: is the path for the header path

  Returns:
    Tuple with a bool and a string:
    bool: True if filename_cc and filename_h belong to the same module.
    string: the additional prefix needed to open the header file.
  """
  fileinfo_cc = FileInfo(filename_cc)
  if not fileinfo_cc.Extension().lstrip('.') in GetNonHeaderExtensions():
    return (False, '')

  fileinfo_h = FileInfo(filename_h)
  if not fileinfo_h.Extension().lstrip('.') in GetHeaderExtensions():
    return (False, '')

  filename_cc = filename_cc[:-(len(fileinfo_cc.Extension()))]
  matched_test_suffix = Search(_TEST_FILE_SUFFIX, fileinfo_cc.BaseName())
  if matched_test_suffix:
    filename_cc = filename_cc[:-len(matched_test_suffix.group(1))]

  filename_cc = filename_cc.replace('/public/', '/')
  filename_cc = filename_cc.replace('/internal/', '/')

  filename_h = filename_h[:-(len(fileinfo_h.Extension()))]
  if filename_h.endswith('-inl'):
    filename_h = filename_h[:-len('-inl')]
  filename_h = filename_h.replace('/public/', '/')
  filename_h = filename_h.replace('/internal/', '/')

  files_belong_to_same_module = filename_cc.endswith(filename_h)
  common_path = ''
  if files_belong_to_same_module:
    common_path = filename_cc[:-len(filename_h)]
  return files_belong_to_same_module, common_path

def FilesBelongToSameModule ( filename_cc , filename_h ) : fileinfo_cc = FileInfo ( filename_cc ) if not fileinfo_cc . Extension ( ) . lstrip ( '.' ) in GetNonHeaderExtensions ( ) : return ( False , '' ) fileinfo_h = FileInfo ( filename_h ) if not fileinfo_h . Extension ( ) . lstrip ( '.' ) in GetHeaderExtensions ( ) : return ( False , '' ) filename_cc = filename_cc [ : - ( len ( fileinfo_cc . Extension ( ) ) ) ] matched_test_suffix = Search ( _TEST_FILE_SUFFIX , fileinfo_cc . BaseName ( ) ) if matched_test_suffix : filename_cc = filename_cc [ : - len ( matched_test_suffix . group ( 1 ) ) ] filename_cc = filename_cc . replace ( '/public/' , '/' ) filename_cc = filename_cc . replace ( '/internal/' , '/' ) filename_h = filename_h [ : - ( len ( fileinfo_h . Extension ( ) ) ) ] if filename_h . endswith ( '-inl' ) : filename_h = filename_h [ : - len ( '-inl' ) ] filename_h = filename_h . replace ( '/public/' , '/' ) filename_h = filename_h . replace ( '/internal/' , '/' ) files_belong_to_same_module = filename_cc . endswith ( filename_h ) common_path = '' if files_belong_to_same_module : common_path = filename_cc [ : - len ( filename_h ) ] return files_belong_to_same_module , common_path

Check if these two filenames belong to the same module .


def UpdateIncludeState(filename, include_dict, io=codecs):
  """Fill up the include_dict with new includes found from the file.

  Args:
    filename: the name of the header to read.
    include_dict: a dictionary in which the headers are inserted.
    io: The io factory to use to read the file. Provided for testability.

  Returns:
    True if a header was successfully added. False otherwise.
  """
  headerfile = None
  try:
    headerfile = io.open(filename, 'r', 'utf8', 'replace')
  except IOError:
    return False
  linenum = 0
  for line in headerfile:
    linenum += 1
    clean_line = CleanseComments(line)
    match = _RE_PATTERN_INCLUDE.search(clean_line)
    if match:
      include = match.group(2)
      include_dict.setdefault(include, linenum)
  return True

def UpdateIncludeState ( filename , include_dict , io = codecs ) : headerfile = None try : headerfile = io . open ( filename , 'r' , 'utf8' , 'replace' ) except IOError : return False linenum = 0 for line in headerfile : linenum += 1 clean_line = CleanseComments ( line ) match = _RE_PATTERN_INCLUDE . search ( clean_line ) if match : include = match . group ( 2 ) include_dict . setdefault ( include , linenum ) return True

Fill up the include_dict with new includes found from the file .


def CheckMakePairUsesDeduction(filename, clean_lines, linenum, error):
  """Check that make_pair's template arguments are deduced.

  G++ 4.6 in C++11 mode fails badly if make_pair's template arguments are
  specified explicitly, and such use isn't intended in any case.

  Args:
    filename: The name of the current file.
    clean_lines: A CleansedLines instance containing the file.
    linenum: The number of the line to check.
    error: The function to call with any errors found.
  """
  line = clean_lines.elided[linenum]
  match = _RE_PATTERN_EXPLICIT_MAKEPAIR.search(line)
  if match:
    error(filename, linenum, 'build/explicit_make_pair',
          4,  # 4 = high confidence
          'For C++11-compatibility, omit template arguments from make_pair'
          ' OR use pair directly OR if appropriate, construct a pair directly')

Check that make_pair s template arguments are deduced .


def CheckRedundantVirtual(filename, clean_lines, linenum, error):
  """Check if line contains a redundant "virtual" function-specifier.

  Args:
    filename: The name of the current file.
    clean_lines: A CleansedLines instance containing the file.
    linenum: The number of the line to check.
    error: The function to call with any errors found.
  """
  # Look for "virtual" on current line.
  line = clean_lines.elided[linenum]
  virtual = Match(r'^(.*)(\bvirtual\b)(.*)$', line)
  if not virtual: return

  # Ignore "virtual" keywords that are near access-specifiers.  These
  # are only used in class base-specifier and do not apply to member
  # functions.
  if (Search(r'\b(public|protected|private)\s+$', virtual.group(1)) or
      Match(r'^\s+(public|protected|private)\b', virtual.group(3))):
    return

  # Ignore the "virtual" keyword from virtual base classes.  Usually
  # there is a column on the same line in these cases (virtual base
  # classes are rare in google3 because multiple inheritance is rare).
  if Match(r'^.*[^:]:[^:].*$', line): return

  # Look for the next opening parenthesis.  This is the start of the
  # parameter list (possibly on the next line shortly after virtual).
  # TODO(unknown): doesn't work if there are virtual functions with
  # decltype() or other things that use parentheses, but csearch suggests
  # that this is rare.
  end_col = -1
  end_line = -1
  start_col = len(virtual.group(2))
  for start_line in xrange(linenum, min(linenum + 3, clean_lines.NumLines())):
    line = clean_lines.elided[start_line][start_col:]
    parameter_list = Match(r'^([^(]*)\(', line)
    if parameter_list:
      # Match parentheses to find the end of the parameter list
      (_, end_line, end_col) = CloseExpression(
          clean_lines, start_line, start_col + len(parameter_list.group(1)))
      break
    start_col = 0

  if end_col < 0:
    return  # Couldn't find end of parameter list, give up

  # Look for "override" or "final" after the parameter list
  # (possibly on the next few lines).
  for i in xrange(end_line, min(end_line + 3, clean_lines.NumLines())):
    line = clean_lines.elided[i][end_col:]
    match = Search(r'\b(override|final)\b', line)
    if match:
      error(filename, linenum, 'readability/inheritance', 4,
            ('"virtual" is redundant since function is '
             'already declared as "%s"' % match.group(1)))

    # Set end_col to check whole lines after we are done with the
    # first line.
    end_col = 0
    if Search(r'[^\w]\s*$', line):
      break

Check if line contains a redundant virtual function - specifier .


def CheckRedundantOverrideOrFinal(filename, clean_lines, linenum, error):
  """Check if line contains a redundant "override" or "final" virt-specifier.

  Args:
    filename: The name of the current file.
    clean_lines: A CleansedLines instance containing the file.
    linenum: The number of the line to check.
    error: The function to call with any errors found.
  """
  # Look for closing parenthesis nearby.  We need one to confirm where
  # the declarator ends and where the virt-specifier starts to avoid
  # false positives.
  line = clean_lines.elided[linenum]
  declarator_end = line.rfind(')')
  if declarator_end >= 0:
    fragment = line[declarator_end:]
  else:
    if linenum > 1 and clean_lines.elided[linenum - 1].rfind(')') >= 0:
      fragment = line
    else:
      return

  # Check that at most one of "override" or "final" is present, not both
  if Search(r'\boverride\b', fragment) and Search(r'\bfinal\b', fragment):
    error(filename, linenum, 'readability/inheritance', 4,
          ('"override" is redundant since function is '
           'already declared as "final"'))

Check if line contains a redundant override or final virt - specifier .


def IsBlockInNameSpace(nesting_state, is_forward_declaration):
  """Checks that the new block is directly in a namespace.

  Args:
    nesting_state: The _NestingState object that contains info about our state.
    is_forward_declaration: If the class is a forward declared class.
  Returns:
    Whether or not the new block is directly in a namespace.
  """
  if is_forward_declaration:
    return len(nesting_state.stack) >= 1 and (
      isinstance(nesting_state.stack[-1], _NamespaceInfo))

  return (len(nesting_state.stack) > 1 and
          nesting_state.stack[-1].check_namespace_indentation and
          isinstance(nesting_state.stack[-2], _NamespaceInfo))

def IsBlockInNameSpace ( nesting_state , is_forward_declaration ) : if is_forward_declaration : return len ( nesting_state . stack ) >= 1 and ( isinstance ( nesting_state . stack [ - 1 ] , _NamespaceInfo ) ) return ( len ( nesting_state . stack ) > 1 and nesting_state . stack [ - 1 ] . check_namespace_indentation and isinstance ( nesting_state . stack [ - 2 ] , _NamespaceInfo ) )

Checks that the new block is directly in a namespace .


def ShouldCheckNamespaceIndentation(nesting_state, is_namespace_indent_item,
                                    raw_lines_no_comments, linenum):
  """This method determines if we should apply our namespace indentation check.

  Args:
    nesting_state: The current nesting state.
    is_namespace_indent_item: If we just put a new class on the stack, True.
      If the top of the stack is not a class, or we did not recently
      add the class, False.
    raw_lines_no_comments: The lines without the comments.
    linenum: The current line number we are processing.

  Returns:
    True if we should apply our namespace indentation check. Currently, it
    only works for classes and namespaces inside of a namespace.
  """

  is_forward_declaration = IsForwardClassDeclaration(raw_lines_no_comments,
                                                     linenum)

  if not (is_namespace_indent_item or is_forward_declaration):
    return False

  # If we are in a macro, we do not want to check the namespace indentation.
  if IsMacroDefinition(raw_lines_no_comments, linenum):
    return False

  return IsBlockInNameSpace(nesting_state, is_forward_declaration)

This method determines if we should apply our namespace indentation check .


def FlagCxx14Features(filename, clean_lines, linenum, error):
  """Flag those C++14 features that we restrict.

  Args:
    filename: The name of the current file.
    clean_lines: A CleansedLines instance containing the file.
    linenum: The number of the line to check.
    error: The function to call with any errors found.
  """
  line = clean_lines.elided[linenum]

  include = Match(r'\s*#\s*include\s+[<"]([^<"]+)[">]', line)

  # Flag unapproved C++14 headers.
  if include and include.group(1) in ('scoped_allocator', 'shared_mutex'):
    error(filename, linenum, 'build/c++14', 5,
          ('<%s> is an unapproved C++14 header.') % include.group(1))

def
FlagCxx14Features
(
filename
,
clean_lines
,
linenum
,
error
)
:
line
=
clean_lines
.
elided
[
linenum
]
include
=
Match
(
r'\s*#\s*include\s+[<"]([^<"]+)[">]'
,
line
)
# Flag unapproved C++14 headers.
if
include
and
include
.
group
(
1
)
in
(
'scoped_allocator'
,
'shared_mutex'
)
:
error
(
filename
,
linenum
,
'build/c++14'
,
5
,
(
'<%s> is an unapproved C++14 header.'
)
%
include
.
group
(
1
)
)

Flag those C ++ 14 features that we restrict .


def ProcessFileData(filename, file_extension, lines, error,
                    extra_check_functions=None):
  """Performs lint checks and reports any errors to the given error function.

  Args:
    filename: Filename of the file that is being processed.
    file_extension: The extension (dot not included) of the file.
    lines: An array of strings, each representing a line of the file, with the
           last element being empty if the file is terminated with a newline.
    error: A callable to which errors are reported, which takes 4 arguments:
           filename, line number, error level, and message
    extra_check_functions: An array of additional check functions that will be
                           run on each source line. Each function takes 4
                           arguments: filename, clean_lines, line, error
  """
  lines = (['// marker so line numbers and indices both start at 1'] + lines +
           ['// marker so line numbers end in a known way'])

  include_state = _IncludeState()
  function_state = _FunctionState()
  nesting_state = NestingState()

  ResetNolintSuppressions()

  CheckForCopyright(filename, lines, error)
  ProcessGlobalSuppresions(lines)
  RemoveMultiLineComments(filename, lines, error)
  clean_lines = CleansedLines(lines)

  if file_extension in GetHeaderExtensions():
    CheckForHeaderGuard(filename, clean_lines, error)

  for line in range(clean_lines.NumLines()):
    ProcessLine(filename, file_extension, clean_lines, line,
                include_state, function_state, nesting_state, error,
                extra_check_functions)
    FlagCxx11Features(filename, clean_lines, line, error)
  nesting_state.CheckCompletedBlocks(filename, error)

  CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error)

  # Check that the .cc file has included its header if it exists.
  if _IsSourceExtension(file_extension):
    CheckHeaderFileIncluded(filename, include_state, error)

  # We check here rather than inside ProcessLine so that we see raw
  # lines rather than "cleaned" lines.
  CheckForBadCharacters(filename, lines, error)

  CheckForNewlineAtEOF(filename, lines, error)

Performs lint checks and reports any errors to the given error function .


def ProcessConfigOverrides(filename):
  """ Loads the configuration files and processes the config overrides.

  Args:
    filename: The name of the file being processed by the linter.

  Returns:
    False if the current |filename| should not be processed further.
  """

  abs_filename = os.path.abspath(filename)
  cfg_filters = []
  keep_looking = True
  while keep_looking:
    abs_path, base_name = os.path.split(abs_filename)
    if not base_name:
      break  # Reached the root directory.

    cfg_file = os.path.join(abs_path, "CPPLINT.cfg")
    abs_filename = abs_path
    if not os.path.isfile(cfg_file):
      continue

    try:
      with open(cfg_file) as file_handle:
        for line in file_handle:
          line, _, _ = line.partition('#')  # Remove comments.
          if not line.strip():
            continue

          name, _, val = line.partition('=')
          name = name.strip()
          val = val.strip()
          if name == 'set noparent':
            keep_looking = False
          elif name == 'filter':
            cfg_filters.append(val)
          elif name == 'exclude_files':
            # When matching exclude_files pattern, use the base_name of
            # the current file name or the directory name we are processing.
            # For example, if we are checking for lint errors in /foo/bar/baz.cc
            # and we found the .cfg file at /foo/CPPLINT.cfg, then the config
            # file's "exclude_files" filter is meant to be checked against "bar"
            # and not "baz" nor "bar/baz.cc".
            if base_name:
              pattern = re.compile(val)
              if pattern.match(base_name):
                _cpplint_state.PrintInfo('Ignoring "%s": file excluded by '
                    '"%s". File path component "%s" matches pattern "%s"\n' %
                    (filename, cfg_file, base_name, val))
                return False
          elif name == 'linelength':
            global _line_length
            try:
                _line_length = int(val)
            except ValueError:
                _cpplint_state.PrintError('Line length must be numeric.')
          elif name == 'extensions':
              global _valid_extensions
              try:
                  extensions = [ext.strip() for ext in val.split(',')]
                  _valid_extensions = set(extensions)
              except ValueError:
                  sys.stderr.write('Extensions should be a comma-separated list of values;'
                                   'for example: extensions=hpp,cpp\n'
                                   'This could not be parsed: "%s"' % (val,))
          elif name == 'headers':
              global _header_extensions
              try:
                  extensions = [ext.strip() for ext in val.split(',')]
                  _header_extensions = set(extensions)
              except ValueError:
                  sys.stderr.write('Extensions should be a comma-separated list of values;'
                                   'for example: extensions=hpp,cpp\n'
                                   'This could not be parsed: "%s"' % (val,))
          elif name == 'root':
            global _root
            _root = val
          else:
            _cpplint_state.PrintError(
                'Invalid configuration option (%s) in file %s\n' %
                (name, cfg_file))

    except IOError:
      _cpplint_state.PrintError(
          "Skipping config file '%s': Can't open for reading\n" % cfg_file)
      keep_looking = False

  # Apply all the accumulated filters in reverse order (top-level directory
  # config options having the least priority).
  for cfg_filter in reversed(cfg_filters):
     _AddFilters(cfg_filter)

  return True

Loads the configuration files and processes the config overrides .


def ProcessFile(filename, vlevel, extra_check_functions=None):
  """Does google-lint on a single file.

  Args:
    filename: The name of the file to parse.

    vlevel: The level of errors to report.  Every error of confidence
    >= verbose_level will be reported.  0 is a good default.

    extra_check_functions: An array of additional check functions that will be
                           run on each source line. Each function takes 4
                           arguments: filename, clean_lines, line, error
  """

  _SetVerboseLevel(vlevel)
  _BackupFilters()

  if not ProcessConfigOverrides(filename):
    _RestoreFilters()
    return

  lf_lines = []
  crlf_lines = []
  try:
    # Support the UNIX convention of using "-" for stdin.  Note that
    # we are not opening the file with universal newline support
    # (which codecs doesn't support anyway), so the resulting lines do
    # contain trailing '\r' characters if we are reading a file that
    # has CRLF endings.
    # If after the split a trailing '\r' is present, it is removed
    # below.
    if filename == '-':
      lines = codecs.StreamReaderWriter(sys.stdin,
                                        codecs.getreader('utf8'),
                                        codecs.getwriter('utf8'),
                                        'replace').read().split('\n')
    else:
      lines = codecs.open(filename, 'r', 'utf8', 'replace').read().split('\n')

    # Remove trailing '\r'.
    # The -1 accounts for the extra trailing blank line we get from split()
    for linenum in range(len(lines) - 1):
      if lines[linenum].endswith('\r'):
        lines[linenum] = lines[linenum].rstrip('\r')
        crlf_lines.append(linenum + 1)
      else:
        lf_lines.append(linenum + 1)

  except IOError:
    _cpplint_state.PrintError(
        "Skipping input '%s': Can't open for reading\n" % filename)
    _RestoreFilters()
    return

  # Note, if no dot is found, this will give the entire filename as the ext.
  file_extension = filename[filename.rfind('.') + 1:]

  # When reading from stdin, the extension is unknown, so no cpplint tests
  # should rely on the extension.
  if filename != '-' and file_extension not in GetAllExtensions():
    # bazel 0.5.1> uses four distinct generated files that gives a warning
    # we suppress the warning for these files
    bazel_gen_files = set([
        "external/local_config_cc/libtool",
        "external/local_config_cc/make_hashed_objlist.py",
        "external/local_config_cc/wrapped_ar",
        "external/local_config_cc/wrapped_clang",
        "external/local_config_cc/xcrunwrapper.sh",
    ])
    if not filename in bazel_gen_files:
       _cpplint_state.PrintError('Ignoring %s; not a valid file name '
                                 '(%s)\n' % (filename, ', '.join(GetAllExtensions())))
  else:
    ProcessFileData(filename, file_extension, lines, Error,
                    extra_check_functions)

    # If end-of-line sequences are a mix of LF and CR-LF, issue
    # warnings on the lines with CR.
    #
    # Don't issue any warnings if all lines are uniformly LF or CR-LF,
    # since critique can handle these just fine, and the style guide
    # doesn't dictate a particular end of line sequence.
    #
    # We can't depend on os.linesep to determine what the desired
    # end-of-line sequence should be, since that will return the
    # server-side end-of-line sequence.
    if lf_lines and crlf_lines:
      # Warn on every line with CR.  An alternative approach might be to
      # check whether the file is mostly CRLF or just LF, and warn on the
      # minority, we bias toward LF here since most tools prefer LF.
      for linenum in crlf_lines:
        Error(filename, linenum, 'whitespace/newline', 1,
              'Unexpected \\r (^M) found; better to use only \\n')

  _RestoreFilters()


def PrintCategories():
  """Prints a list of all the error-categories used by error messages.

  These are the categories used to filter messages via --filter.
  """
  sys.stderr.write(''.join('  %s\n' % cat for cat in _ERROR_CATEGORIES))
  sys.exit(0)

def
PrintCategories
(
)
:
sys
.
stderr
.
write
(
''
.
join
(
'  %s\n'
%
cat
for
cat
in
_ERROR_CATEGORIES
)
)
sys
.
exit
(
0
)

Prints a list of all the error - categories used by error messages .


def ParseArguments(args):
  """Parses the command line arguments.

  This may set the output format and verbosity level as side-effects.

  Args:
    args: The command line arguments:

  Returns:
    The list of filenames to lint.
  """
  try:
    (opts, filenames) = getopt.getopt(args, '', ['help', 'output=', 'verbose=',
                                                 'counting=',
                                                 'filter=',
                                                 'root=',
                                                 'repository=',
                                                 'linelength=',
                                                 'extensions=',
                                                 'exclude=',
                                                 'headers=',
                                                 'quiet',
                                                 'recursive'])
  except getopt.GetoptError:
    PrintUsage('Invalid arguments.')

  verbosity = _VerboseLevel()
  output_format = _OutputFormat()
  filters = ''
  counting_style = ''
  recursive = False

  for (opt, val) in opts:
    if opt == '--help':
      PrintUsage(None)
    elif opt == '--output':
      if val not in ('emacs', 'vs7', 'eclipse', 'junit'):
        PrintUsage('The only allowed output formats are emacs, vs7, eclipse '
                   'and junit.')
      output_format = val
    elif opt == '--verbose':
      verbosity = int(val)
    elif opt == '--filter':
      filters = val
      if not filters:
        PrintCategories()
    elif opt == '--counting':
      if val not in ('total', 'toplevel', 'detailed'):
        PrintUsage('Valid counting options are total, toplevel, and detailed')
      counting_style = val
    elif opt == '--root':
      global _root
      _root = val
    elif opt == '--repository':
      global _repository
      _repository = val
    elif opt == '--linelength':
      global _line_length
      try:
        _line_length = int(val)
      except ValueError:
        PrintUsage('Line length must be digits.')
    elif opt == '--exclude':
      global _excludes
      if not _excludes:
        _excludes = set()
      _excludes.update(glob.glob(val))
    elif opt == '--extensions':
      global _valid_extensions
      try:
        _valid_extensions = set(val.split(','))
      except ValueError:
          PrintUsage('Extensions must be comma seperated list.')
    elif opt == '--headers':
      global _header_extensions
      try:
          _header_extensions = set(val.split(','))
      except ValueError:
        PrintUsage('Extensions must be comma seperated list.')
    elif opt == '--recursive':
      recursive = True
    elif opt == '--quiet':
      global _quiet
      _quiet = True

  if not filenames:
    PrintUsage('No files were specified.')

  if recursive:
    filenames = _ExpandDirectories(filenames)

  if _excludes:
    filenames = _FilterExcludedFiles(filenames)

  _SetOutputFormat(output_format)
  _SetVerboseLevel(verbosity)
  _SetFilters(filters)
  _SetCountingStyle(counting_style)

  return filenames


def _ExpandDirectories(filenames):
  """Searches a list of filenames and replaces directories in the list with
  all files descending from those directories. Files with extensions not in
  the valid extensions list are excluded.

  Args:
    filenames: A list of files or directories

  Returns:
    A list of all files that are members of filenames or descended from a
    directory in filenames
  """
  expanded = set()
  for filename in filenames:
      if not os.path.isdir(filename):
        expanded.add(filename)
        continue

      for root, _, files in os.walk(filename):
        for loopfile in files:
          fullname = os.path.join(root, loopfile)
          if fullname.startswith('.' + os.path.sep):
            fullname = fullname[len('.' + os.path.sep):]
          expanded.add(fullname)

  filtered = []
  for filename in expanded:
      if os.path.splitext(filename)[1][1:] in GetAllExtensions():
          filtered.append(filename)

  return filtered

def _ExpandDirectories ( filenames ) : expanded = set ( ) for filename in filenames : if not os . path . isdir ( filename ) : expanded . add ( filename ) continue for root , _ , files in os . walk ( filename ) : for loopfile in files : fullname = os . path . join ( root , loopfile ) if fullname . startswith ( '.' + os . path . sep ) : fullname = fullname [ len ( '.' + os . path . sep ) : ] expanded . add ( fullname ) filtered = [ ] for filename in expanded : if os . path . splitext ( filename ) [ 1 ] [ 1 : ] in GetAllExtensions ( ) : filtered . append ( filename ) return filtered

Searches a list of filenames and replaces directories in the list with all files descending from those directories . Files with extensions not in the valid extensions list are excluded .


def FindHeader(self, header):
    """Check if a header has already been included.

    Args:
      header: header to check.
    Returns:
      Line number of previous occurrence, or -1 if the header has not
      been seen before.
    """
    for section_list in self.include_list:
      for f in section_list:
        if f[0] == header:
          return f[1]
    return -1

def FindHeader ( self , header ) : for section_list in self . include_list : for f in section_list : if f [ 0 ] == header : return f [ 1 ] return - 1

Check if a header has already been included .


def ResetSection(self, directive):
    """Reset section checking for preprocessor directive.

    Args:
      directive: preprocessor directive (e.g. "if", "else").
    """
    # The name of the current section.
    self._section = self._INITIAL_SECTION
    # The path of last found header.
    self._last_header = ''

    # Update list of includes.  Note that we never pop from the
    # include list.
    if directive in ('if', 'ifdef', 'ifndef'):
      self.include_list.append([])
    elif directive in ('else', 'elif'):
      self.include_list[-1] = []

Reset section checking for preprocessor directive .


def IsInAlphabeticalOrder(self, clean_lines, linenum, header_path):
    """Check if a header is in alphabetical order with the previous header.

    Args:
      clean_lines: A CleansedLines instance containing the file.
      linenum: The number of the line to check.
      header_path: Canonicalized header to be checked.

    Returns:
      Returns true if the header is in alphabetical order.
    """
    # If previous section is different from current section, _last_header will
    # be reset to empty string, so it's always less than current header.
    #
    # If previous line was a blank line, assume that the headers are
    # intentionally sorted the way they are.
    if (self._last_header > header_path and
        Match(r'^\s*#\s*include\b', clean_lines.elided[linenum - 1])):
      return False
    return True

Check if a header is in alphabetical order with the previous header .


def CheckNextIncludeOrder(self, header_type):
    """Returns a non-empty error message if the next header is out of order.

    This function also updates the internal state to be ready to check
    the next include.

    Args:
      header_type: One of the _XXX_HEADER constants defined above.

    Returns:
      The empty string if the header is in the right order, or an
      error message describing what's wrong.

    """
    error_message = ('Found %s after %s' %
                     (self._TYPE_NAMES[header_type],
                      self._SECTION_NAMES[self._section]))

    last_section = self._section

    if header_type == _C_SYS_HEADER:
      if self._section <= self._C_SECTION:
        self._section = self._C_SECTION
      else:
        self._last_header = ''
        return error_message
    elif header_type == _CPP_SYS_HEADER:
      if self._section <= self._CPP_SECTION:
        self._section = self._CPP_SECTION
      else:
        self._last_header = ''
        return error_message
    elif header_type == _LIKELY_MY_HEADER:
      if self._section <= self._MY_H_SECTION:
        self._section = self._MY_H_SECTION
      else:
        self._section = self._OTHER_H_SECTION
    elif header_type == _POSSIBLE_MY_HEADER:
      if self._section <= self._MY_H_SECTION:
        self._section = self._MY_H_SECTION
      else:
        # This will always be the fallback because we're not sure
        # enough that the header is associated with this file.
        self._section = self._OTHER_H_SECTION
    else:
      assert header_type == _OTHER_HEADER
      self._section = self._OTHER_H_SECTION

    if last_section != self._section:
      self._last_header = ''

    return ''

Returns a non - empty error message if the next header is out of order .


def SetVerboseLevel(self, level):
    """Sets the module's verbosity, and returns the previous setting."""
    last_verbose_level = self.verbose_level
    self.verbose_level = level
    return last_verbose_level

def SetVerboseLevel ( self , level ) : last_verbose_level = self . verbose_level self . verbose_level = level return last_verbose_level

Sets the module s verbosity and returns the previous setting .


def AddFilters(self, filters):
    """ Adds more filters to the existing list of error-message filters. """
    for filt in filters.split(','):
      clean_filt = filt.strip()
      if clean_filt:
        self.filters.append(clean_filt)
    for filt in self.filters:
      if not (filt.startswith('+') or filt.startswith('-')):
        raise ValueError('Every filter in --filters must start with + or -'
                         ' (%s does not)' % filt)

Adds more filters to the existing list of error - message filters .


def IncrementErrorCount(self, category):
    """Bumps the module's error statistic."""
    self.error_count += 1
    if self.counting in ('toplevel', 'detailed'):
      if self.counting != 'detailed':
        category = category.split('/')[0]
      if category not in self.errors_by_category:
        self.errors_by_category[category] = 0
      self.errors_by_category[category] += 1

def IncrementErrorCount ( self , category ) : self . error_count += 1 if self . counting in ( 'toplevel' , 'detailed' ) : if self . counting != 'detailed' : category = category . split ( '/' ) [ 0 ] if category not in self . errors_by_category : self . errors_by_category [ category ] = 0 self . errors_by_category [ category ] += 1


def PrintErrorCounts(self):
    """Print a summary of errors by category, and the total."""
    for category, count in sorted(iteritems(self.errors_by_category)):
      self.PrintInfo('Category \'%s\' errors found: %d\n' %
                       (category, count))
    if self.error_count > 0:
      self.PrintInfo('Total errors found: %d\n' % self.error_count)

def
PrintErrorCounts
(
self
)
:
for
category
,
count
in
sorted
(
iteritems
(
self
.
errors_by_category
)
)
:
self
.
PrintInfo
(
'Category \'%s\' errors found: %d\n'
%
(
category
,
count
)
)
if
self
.
error_count
>
0
:
self
.
PrintInfo
(
'Total errors found: %d\n'
%
self
.
error_count
)

Print a summary of errors by category and the total .


def Begin(self, function_name):
    """Start analyzing function body.

    Args:
      function_name: The name of the function being tracked.
    """
    self.in_a_function = True
    self.lines_in_function = 0
    self.current_function = function_name

def Begin ( self , function_name ) : self . in_a_function = True self . lines_in_function = 0 self . current_function = function_name


def RepositoryName(self):
    r"""FullName after removing the local path to the repository.

    If we have a real absolute path name here we can try to do something smart:
    detecting the root of the checkout and truncating /path/to/checkout from
    the name so that we get header guards that don't include things like
    "C:\Documents and Settings\..." or "/home/username/..." in them and thus
    people on different computers who have checked the source out to different
    locations won't see bogus errors.
    """
    fullname = self.FullName()

    if os.path.exists(fullname):
      project_dir = os.path.dirname(fullname)

      # If the user specified a repository path, it exists, and the file is
      # contained in it, use the specified repository path
      if _repository:
        repo = FileInfo(_repository).FullName()
        root_dir = project_dir
        while os.path.exists(root_dir):
          # allow case insensitive compare on Windows
          if os.path.normcase(root_dir) == os.path.normcase(repo):
            return os.path.relpath(fullname, root_dir).replace('\\', '/')
          one_up_dir = os.path.dirname(root_dir)
          if one_up_dir == root_dir:
            break
          root_dir = one_up_dir

      if os.path.exists(os.path.join(project_dir, ".svn")):
        # If there's a .svn file in the current directory, we recursively look
        # up the directory tree for the top of the SVN checkout
        root_dir = project_dir
        one_up_dir = os.path.dirname(root_dir)
        while os.path.exists(os.path.join(one_up_dir, ".svn")):
          root_dir = os.path.dirname(root_dir)
          one_up_dir = os.path.dirname(one_up_dir)

        prefix = os.path.commonprefix([root_dir, project_dir])
        return fullname[len(prefix) + 1:]

      # Not SVN <= 1.6? Try to find a git, hg, or svn top level directory by
      # searching up from the current path.
      root_dir = current_dir = os.path.dirname(fullname)
      while current_dir != os.path.dirname(current_dir):
        if (os.path.exists(os.path.join(current_dir, ".git")) or
            os.path.exists(os.path.join(current_dir, ".hg")) or
            os.path.exists(os.path.join(current_dir, ".svn"))):
          root_dir = current_dir
        current_dir = os.path.dirname(current_dir)

      if (os.path.exists(os.path.join(root_dir, ".git")) or
          os.path.exists(os.path.join(root_dir, ".hg")) or
          os.path.exists(os.path.join(root_dir, ".svn"))):
        prefix = os.path.commonprefix([root_dir, project_dir])
        return fullname[len(prefix) + 1:]

    # Don't know what to do; header guard warnings may be wrong...
    return fullname

r FullName after removing the local path to the repository .


def Split(self):
    """Splits the file into the directory, basename, and extension.

    For 'chrome/browser/browser.cc', Split() would
    return ('chrome/browser', 'browser', '.cc')

    Returns:
      A tuple of (directory, basename, extension).
    """

    googlename = self.RepositoryName()
    project, rest = os.path.split(googlename)
    return (project,) + os.path.splitext(rest)

def Split ( self ) : googlename = self . RepositoryName ( ) project , rest = os . path . split ( googlename ) return ( project , ) + os . path . splitext ( rest )

Splits the file into the directory basename and extension .


def _CollapseStrings(elided):
    """Collapses strings and chars on a line to simple "" or '' blocks.

    We nix strings first so we're not fooled by text like '"http://"'

    Args:
      elided: The line being processed.

    Returns:
      The line with collapsed strings.
    """
    if _RE_PATTERN_INCLUDE.match(elided):
      return elided

    # Remove escaped characters first to make quote/single quote collapsing
    # basic.  Things that look like escaped characters shouldn't occur
    # outside of strings and chars.
    elided = _RE_PATTERN_CLEANSE_LINE_ESCAPES.sub('', elided)

    # Replace quoted strings and digit separators.  Both single quotes
    # and double quotes are processed in the same loop, otherwise
    # nested quotes wouldn't work.
    collapsed = ''
    while True:
      # Find the first quote character
      match = Match(r'^([^\'"]*)([\'"])(.*)$', elided)
      if not match:
        collapsed += elided
        break
      head, quote, tail = match.groups()

      if quote == '"':
        # Collapse double quoted strings
        second_quote = tail.find('"')
        if second_quote >= 0:
          collapsed += head + '""'
          elided = tail[second_quote + 1:]
        else:
          # Unmatched double quote, don't bother processing the rest
          # of the line since this is probably a multiline string.
          collapsed += elided
          break
      else:
        # Found single quote, check nearby text to eliminate digit separators.
        #
        # There is no special handling for floating point here, because
        # the integer/fractional/exponent parts would all be parsed
        # correctly as long as there are digits on both sides of the
        # separator.  So we are fine as long as we don't see something
        # like "0.'3" (gcc 4.9.0 will not allow this literal).
        if Search(r'\b(?:0[bBxX]?|[1-9])[0-9a-fA-F]*$', head):
          match_literal = Match(r'^((?:\'?[0-9a-zA-Z_])*)(.*)$', "'" + tail)
          collapsed += head + match_literal.group(1).replace("'", '')
          elided = match_literal.group(2)
        else:
          second_quote = tail.find('\'')
          if second_quote >= 0:
            collapsed += head + "''"
            elided = tail[second_quote + 1:]
          else:
            # Unmatched single quote
            collapsed += elided
            break

    return collapsed

Collapses strings and chars on a line to simple or blocks .


def CheckEnd(self, filename, clean_lines, linenum, error):
    """Check end of namespace comments."""
    line = clean_lines.raw_lines[linenum]

    # Check how many lines is enclosed in this namespace.  Don't issue
    # warning for missing namespace comments if there aren't enough
    # lines.  However, do apply checks if there is already an end of
    # namespace comment and it's incorrect.
    #
    # TODO(unknown): We always want to check end of namespace comments
    # if a namespace is large, but sometimes we also want to apply the
    # check if a short namespace contained nontrivial things (something
    # other than forward declarations).  There is currently no logic on
    # deciding what these nontrivial things are, so this check is
    # triggered by namespace size only, which works most of the time.
    if (linenum - self.starting_linenum < 10
        and not Match(r'^\s*};*\s*(//|/\*).*\bnamespace\b', line)):
      return

    # Look for matching comment at end of namespace.
    #
    # Note that we accept C style "/* */" comments for terminating
    # namespaces, so that code that terminate namespaces inside
    # preprocessor macros can be cpplint clean.
    #
    # We also accept stuff like "// end of namespace <name>." with the
    # period at the end.
    #
    # Besides these, we don't accept anything else, otherwise we might
    # get false negatives when existing comment is a substring of the
    # expected namespace.
    if self.name:
      # Named namespace
      if not Match((r'^\s*};*\s*(//|/\*).*\bnamespace\s+' +
                    re.escape(self.name) + r'[\*/\.\\\s]*$'),
                   line):
        error(filename, linenum, 'readability/namespace', 5,
              'Namespace should be terminated with "// namespace %s"' %
              self.name)
    else:
      # Anonymous namespace
      if not Match(r'^\s*};*\s*(//|/\*).*\bnamespace[\*/\.\\\s]*$', line):
        # If "// namespace anonymous" or "// anonymous namespace (more text)",
        # mention "// anonymous namespace" as an acceptable form
        if Match(r'^\s*}.*\b(namespace anonymous|anonymous namespace)\b', line):
          error(filename, linenum, 'readability/namespace', 5,
                'Anonymous namespace should be terminated with "// namespace"'
                ' or "// anonymous namespace"')
        else:
          error(filename, linenum, 'readability/namespace', 5,
                'Anonymous namespace should be terminated with "// namespace"')


def InTemplateArgumentList(self, clean_lines, linenum, pos):
    """Check if current position is inside template argument list.

    Args:
      clean_lines: A CleansedLines instance containing the file.
      linenum: The number of the line to check.
      pos: position just after the suspected template argument.
    Returns:
      True if (linenum, pos) is inside template arguments.
    """
    while linenum < clean_lines.NumLines():
      # Find the earliest character that might indicate a template argument
      line = clean_lines.elided[linenum]
      match = Match(r'^[^{};=\[\]\.<>]*(.)', line[pos:])
      if not match:
        linenum += 1
        pos = 0
        continue
      token = match.group(1)
      pos += len(match.group(0))

      # These things do not look like template argument list:
      #   class Suspect {
      #   class Suspect x; }
      if token in ('{', '}', ';'): return False

      # These things look like template argument list:
      #   template <class Suspect>
      #   template <class Suspect = default_value>
      #   template <class Suspect[]>
      #   template <class Suspect...>
      if token in ('>', '=', '[', ']', '.'): return True

      # Check if token is an unmatched '<'.
      # If not, move on to the next character.
      if token != '<':
        pos += 1
        if pos >= len(line):
          linenum += 1
          pos = 0
        continue

      # We can't be sure if we just find a single '<', and need to
      # find the matching '>'.
      (_, end_line, end_pos) = CloseExpression(clean_lines, linenum, pos - 1)
      if end_pos < 0:
        # Not sure if template argument list or syntax error in file
        return False
      linenum = end_line
      pos = end_pos
    return False

Check if current position is inside template argument list .


def UpdatePreprocessor(self, line):
    """Update preprocessor stack.

    We need to handle preprocessors due to classes like this:
      #ifdef SWIG
      struct ResultDetailsPageElementExtensionPoint {
      #else
      struct ResultDetailsPageElementExtensionPoint : public Extension {
      #endif

    We make the following assumptions (good enough for most files):
    - Preprocessor condition evaluates to true from #if up to first
      #else/#elif/#endif.

    - Preprocessor condition evaluates to false from #else/#elif up
      to #endif.  We still perform lint checks on these lines, but
      these do not affect nesting stack.

    Args:
      line: current line to check.
    """
    if Match(r'^\s*#\s*(if|ifdef|ifndef)\b', line):
      # Beginning of #if block, save the nesting stack here.  The saved
      # stack will allow us to restore the parsing state in the #else case.
      self.pp_stack.append(_PreprocessorInfo(copy.deepcopy(self.stack)))
    elif Match(r'^\s*#\s*(else|elif)\b', line):
      # Beginning of #else block
      if self.pp_stack:
        if not self.pp_stack[-1].seen_else:
          # This is the first #else or #elif block.  Remember the
          # whole nesting stack up to this point.  This is what we
          # keep after the #endif.
          self.pp_stack[-1].seen_else = True
          self.pp_stack[-1].stack_before_else = copy.deepcopy(self.stack)

        # Restore the stack to how it was before the #if
        self.stack = copy.deepcopy(self.pp_stack[-1].stack_before_if)
      else:
        # TODO(unknown): unexpected #else, issue warning?
        pass
    elif Match(r'^\s*#\s*endif\b', line):
      # End of #if or #else blocks.
      if self.pp_stack:
        # If we saw an #else, we will need to restore the nesting
        # stack to its former state before the #else, otherwise we
        # will just continue from where we left off.
        if self.pp_stack[-1].seen_else:
          # Here we can just use a shallow copy since we are the last
          # reference to it.
          self.stack = self.pp_stack[-1].stack_before_else
        # Drop the corresponding #if
        self.pp_stack.pop()
      else:
        # TODO(unknown): unexpected #endif, issue warning?
        pass


def InnermostClass(self):
    """Get class info on the top of the stack.

    Returns:
      A _ClassInfo object if we are inside a class, or None otherwise.
    """
    for i in range(len(self.stack), 0, -1):
      classinfo = self.stack[i - 1]
      if isinstance(classinfo, _ClassInfo):
        return classinfo
    return None

def InnermostClass ( self ) : for i in range ( len ( self . stack ) , 0 , - 1 ) : classinfo = self . stack [ i - 1 ] if isinstance ( classinfo , _ClassInfo ) : return classinfo return None

Get class info on the top of the stack .


def CheckCompletedBlocks(self, filename, error):
    """Checks that all classes and namespaces have been completely parsed.

    Call this when all lines in a file have been processed.
    Args:
      filename: The name of the current file.
      error: The function to call with any errors found.
    """
    # Note: This test can result in false positives if #ifdef constructs
    # get in the way of brace matching. See the testBuildClass test in
    # cpplint_unittest.py for an example of this.
    for obj in self.stack:
      if isinstance(obj, _ClassInfo):
        error(filename, obj.starting_linenum, 'build/class', 5,
              'Failed to find complete declaration of class %s' %
              obj.name)
      elif isinstance(obj, _NamespaceInfo):
        error(filename, obj.starting_linenum, 'build/namespaces', 5,
              'Failed to find complete declaration of namespace %s' %
              obj.name)

Checks that all classes and namespaces have been completely parsed .


def map(self, map_function):
    """Return a new Streamlet by applying map_function to each element of this Streamlet.
    """
    from heronpy.streamlet.impl.mapbolt import MapStreamlet
    map_streamlet = MapStreamlet(map_function, self)
    self._add_child(map_streamlet)
    return map_streamlet

def map ( self , map_function ) : from heronpy . streamlet . impl . mapbolt import MapStreamlet map_streamlet = MapStreamlet ( map_function , self ) self . _add_child ( map_streamlet ) return map_streamlet

Return a new Streamlet by applying map_function to each element of this Streamlet .


def flat_map(self, flatmap_function):
    """Return a new Streamlet by applying map_function to each element of this Streamlet
       and flattening the result
    """
    from heronpy.streamlet.impl.flatmapbolt import FlatMapStreamlet
    fm_streamlet = FlatMapStreamlet(flatmap_function, self)
    self._add_child(fm_streamlet)
    return fm_streamlet

def flat_map ( self , flatmap_function ) : from heronpy . streamlet . impl . flatmapbolt import FlatMapStreamlet fm_streamlet = FlatMapStreamlet ( flatmap_function , self ) self . _add_child ( fm_streamlet ) return fm_streamlet


def filter(self, filter_function):
    """Return a new Streamlet containing only the elements that satisfy filter_function
    """
    from heronpy.streamlet.impl.filterbolt import FilterStreamlet
    filter_streamlet = FilterStreamlet(filter_function, self)
    self._add_child(filter_streamlet)
    return filter_streamlet

def filter ( self , filter_function ) : from heronpy . streamlet . impl . filterbolt import FilterStreamlet filter_streamlet = FilterStreamlet ( filter_function , self ) self . _add_child ( filter_streamlet ) return filter_streamlet


def clone(self, num_clones):
    """Return num_clones number of streamlets each containing all elements
    of the current streamlet
    """
    retval = []
    for i in range(num_clones):
      retval.append(self.repartition(self.get_num_partitions()))
    return retval

def clone ( self , num_clones ) : retval = [ ] for i in range ( num_clones ) : retval . append ( self . repartition ( self . get_num_partitions ( ) ) ) return retval


def reduce_by_window(self, window_config, reduce_function):
    """Return a new Streamlet in which each element of this Streamlet are collected
      over a window defined by window_config and then reduced using the reduce_function
      reduce_function takes two element at one time and reduces them to one element that
      is used in the subsequent operations.
    """
    from heronpy.streamlet.impl.reducebywindowbolt import ReduceByWindowStreamlet
    reduce_streamlet = ReduceByWindowStreamlet(window_config, reduce_function, self)
    self._add_child(reduce_streamlet)
    return reduce_streamlet

def reduce_by_window ( self , window_config , reduce_function ) : from heronpy . streamlet . impl . reducebywindowbolt import ReduceByWindowStreamlet reduce_streamlet = ReduceByWindowStreamlet ( window_config , reduce_function , self ) self . _add_child ( reduce_streamlet ) return reduce_streamlet

Return a new Streamlet in which each element of this Streamlet are collected over a window defined by window_config and then reduced using the reduce_function reduce_function takes two element at one time and reduces them to one element that is used in the subsequent operations .


def union(self, other_streamlet):
    """Returns a new Streamlet that consists of elements of both this and other_streamlet
    """
    from heronpy.streamlet.impl.unionbolt import UnionStreamlet
    union_streamlet = UnionStreamlet(self, other_streamlet)
    self._add_child(union_streamlet)
    other_streamlet._add_child(union_streamlet)
    return union_streamlet

def union ( self , other_streamlet ) : from heronpy . streamlet . impl . unionbolt import UnionStreamlet union_streamlet = UnionStreamlet ( self , other_streamlet ) self . _add_child ( union_streamlet ) other_streamlet . _add_child ( union_streamlet ) return union_streamlet


def log(self):
    """Logs all elements of this streamlet. This returns nothing
    """
    from heronpy.streamlet.impl.logbolt import LogStreamlet
    log_streamlet = LogStreamlet(self)
    self._add_child(log_streamlet)
    return

def log ( self ) : from heronpy . streamlet . impl . logbolt import LogStreamlet log_streamlet = LogStreamlet ( self ) self . _add_child ( log_streamlet ) return

Logs all elements of this streamlet . This returns nothing


def consume(self, consume_function):
    """Calls consume_function for each element of this streamlet. This function returns nothing
    """
    from heronpy.streamlet.impl.consumebolt import ConsumeStreamlet
    consume_streamlet = ConsumeStreamlet(consume_function, self)
    self._add_child(consume_streamlet)
    return

def consume ( self , consume_function ) : from heronpy . streamlet . impl . consumebolt import ConsumeStreamlet consume_streamlet = ConsumeStreamlet ( consume_function , self ) self . _add_child ( consume_streamlet ) return

Calls consume_function for each element of this streamlet . This function returns nothing


def join(self, join_streamlet, window_config, join_function):
    """Return a new Streamlet by joining join_streamlet with this streamlet
    """
    from heronpy.streamlet.impl.joinbolt import JoinStreamlet, JoinBolt
    join_streamlet_result = JoinStreamlet(JoinBolt.INNER, window_config,
                                          join_function, self, join_streamlet)
    self._add_child(join_streamlet_result)
    join_streamlet._add_child(join_streamlet_result)
    return join_streamlet_result

def join ( self , join_streamlet , window_config , join_function ) : from heronpy . streamlet . impl . joinbolt import JoinStreamlet , JoinBolt join_streamlet_result = JoinStreamlet ( JoinBolt . INNER , window_config , join_function , self , join_streamlet ) self . _add_child ( join_streamlet_result ) join_streamlet . _add_child ( join_streamlet_result ) return join_streamlet_result


def outer_right_join(self, join_streamlet, window_config, join_function):
    """Return a new Streamlet by outer right join_streamlet with this streamlet
    """
    from heronpy.streamlet.impl.joinbolt import JoinStreamlet, JoinBolt
    join_streamlet_result = JoinStreamlet(JoinBolt.OUTER_RIGHT, window_config,
                                          join_function, self, join_streamlet)
    self._add_child(join_streamlet_result)
    join_streamlet._add_child(join_streamlet_result)
    return join_streamlet_result

def outer_right_join ( self , join_streamlet , window_config , join_function ) : from heronpy . streamlet . impl . joinbolt import JoinStreamlet , JoinBolt join_streamlet_result = JoinStreamlet ( JoinBolt . OUTER_RIGHT , window_config , join_function , self , join_streamlet ) self . _add_child ( join_streamlet_result ) join_streamlet . _add_child ( join_streamlet_result ) return join_streamlet_result


def outer_left_join(self, join_streamlet, window_config, join_function):
    """Return a new Streamlet by left join_streamlet with this streamlet
    """
    from heronpy.streamlet.impl.joinbolt import JoinStreamlet, JoinBolt
    join_streamlet_result = JoinStreamlet(JoinBolt.OUTER_LEFT, window_config,
                                          join_function, self, join_streamlet)
    self._add_child(join_streamlet_result)
    join_streamlet._add_child(join_streamlet_result)
    return join_streamlet_result

def outer_left_join ( self , join_streamlet , window_config , join_function ) : from heronpy . streamlet . impl . joinbolt import JoinStreamlet , JoinBolt join_streamlet_result = JoinStreamlet ( JoinBolt . OUTER_LEFT , window_config , join_function , self , join_streamlet ) self . _add_child ( join_streamlet_result ) join_streamlet . _add_child ( join_streamlet_result ) return join_streamlet_result


def outer_join(self, join_streamlet, window_config, join_function):
    """Return a new Streamlet by outer join_streamlet with this streamlet
    """
    from heronpy.streamlet.impl.joinbolt import JoinStreamlet, JoinBolt

    join_streamlet_result = JoinStreamlet(JoinBolt.OUTER, window_config,
                                          join_function, self, join_streamlet)
    self._add_child(join_streamlet_result)
    join_streamlet._add_child(join_streamlet_result)
    return join_streamlet_result

def outer_join ( self , join_streamlet , window_config , join_function ) : from heronpy . streamlet . impl . joinbolt import JoinStreamlet , JoinBolt join_streamlet_result = JoinStreamlet ( JoinBolt . OUTER , window_config , join_function , self , join_streamlet ) self . _add_child ( join_streamlet_result ) join_streamlet . _add_child ( join_streamlet_result ) return join_streamlet_result


def extract_common_args(command, parser, cl_args):
  """ extract common args """
  try:
    # do not pop like cli because ``topologies`` subcommand still needs it
    cluster_role_env = cl_args['cluster/[role]/[env]']
    config_path = cl_args['config_path']
  except KeyError:
    # if some of the arguments are not found, print error and exit
    subparser = config.get_subparser(parser, command)
    print(subparser.format_help())
    return dict()
  cluster = config.get_heron_cluster(cluster_role_env)
  config_path = config.get_heron_cluster_conf_dir(cluster, config_path)

  new_cl_args = dict()
  try:
    cluster_tuple = config.parse_cluster_role_env(cluster_role_env, config_path)
    new_cl_args['cluster'] = cluster_tuple[0]
    new_cl_args['role'] = cluster_tuple[1]
    new_cl_args['environ'] = cluster_tuple[2]
    new_cl_args['config_path'] = config_path
  except Exception as e:
    Log.error("Unable to get valid topology location: %s", str(e))
    return dict()

  cl_args.update(new_cl_args)
  return cl_args


def expand_args(command):
    """Parses command strings and returns a Popen-ready list."""

    # Prepare arguments.
    if isinstance(command, (str, unicode)):
        splitter = shlex.shlex(command.encode('utf-8'))
        splitter.whitespace = '|'
        splitter.whitespace_split = True
        command = []

        while True:
            token = splitter.get_token()
            if token:
                command.append(token)
            else:
                break

        command = list(map(shlex.split, command))

    return command

def
expand_args
(
command
)
:
# Prepare arguments.
if
isinstance
(
command
,
(
str
,
unicode
)
)
:
splitter
=
shlex
.
shlex
(
command
.
encode
(
'utf-8'
)
)
splitter
.
whitespace
=
'|'
splitter
.
whitespace_split
=
True
command
=
[
]
while
True
:
token
=
splitter
.
get_token
(
)
if
token
:
command
.
append
(
token
)
else
:
break
command
=
list
(
map
(
shlex
.
split
,
command
)
)
return
command

Parses command strings and returns a Popen - ready list .


def run(command, data=None, timeout=None, kill_timeout=None, env=None, cwd=None):
    """Executes a given commmand and returns Response.

    Blocks until process is complete, or timeout is reached.
    """

    command = expand_args(command)

    history = []
    for c in command:

        if len(history):
            # due to broken pipe problems pass only first 10 KiB
            data = history[-1].std_out[0:10*1024]

        cmd = Command(c)
        try:
            out, err = cmd.run(data, timeout, kill_timeout, env, cwd)
            status_code = cmd.returncode
        except OSError as e:
            out, err = '', u"\n".join([e.strerror, traceback.format_exc()])
            status_code = 127

        r = Response(process=cmd)

        r.command = c
        r.std_out = out
        r.std_err = err
        r.status_code = status_code

        history.append(r)

    r = history.pop()
    r.history = history

    return r

Executes a given commmand and returns Response .


def connect(command, data=None, env=None, cwd=None):
    """Spawns a new process from the given command."""

    # TODO: support piped commands
    command_str = expand_args(command).pop()
    environ = dict(os.environ)
    environ.update(env or {})

    process = subprocess.Popen(command_str,
        universal_newlines=True,
        shell=False,
        env=environ,
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        bufsize=0,
        cwd=cwd,
    )

    return ConnectedCommand(process=process)

def
connect
(
command
,
data
=
,
env
=
,
cwd
=
)
:
# TODO: support piped commands
command_str
=
expand_args
(
command
)
.
pop
(
)
environ
=
dict
(
os
.
environ
)
environ
.
update
(
env
or
{
}
)
process
=
subprocess
.
Popen
(
command_str
,
universal_newlines
=
True
,
shell
=
False
,
env
=
environ
,
stdin
=
subprocess
.
PIPE
,
stdout
=
subprocess
.
PIPE
,
stderr
=
subprocess
.
PIPE
,
bufsize
=
0
,
cwd
=
cwd
,
)
return
ConnectedCommand
(
process
=
process
)

Spawns a new process from the given command .


def send(self, str, end='\n'):
        """Sends a line to std_in."""
        return self._process.stdin.write(str+end)

def send ( self , str , end = '\n' ) : return self . _process . stdin . write ( str + end )


def Js(val, Clamped=False):
    '''Converts Py type to PyJs type'''
    if isinstance(val, PyJs):
        return val
    elif val is None:
        return undefined
    elif isinstance(val, basestring):
        return PyJsString(val, StringPrototype)
    elif isinstance(val, bool):
        return true if val else false
    elif isinstance(val, float) or isinstance(val, int) or isinstance(
            val, long) or (NUMPY_AVAILABLE and isinstance(
                val,
                (numpy.int8, numpy.uint8, numpy.int16, numpy.uint16,
                 numpy.int32, numpy.uint32, numpy.float32, numpy.float64))):
        # This is supposed to speed things up. may not be the case
        if val in NUM_BANK:
            return NUM_BANK[val]
        return PyJsNumber(float(val), NumberPrototype)
    elif isinstance(val, FunctionType):
        return PyJsFunction(val, FunctionPrototype)
    #elif isinstance(val, ModuleType):
    #    mod = {}
    #    for name in dir(val):
    #        value = getattr(val, name)
    #        if isinstance(value, ModuleType):
    #            continue  # prevent recursive module conversion
    #        try:
    #            jsval = HJs(value)
    #        except RuntimeError:
    #            print 'Could not convert %s to PyJs object!' % name
    #            continue
    #        mod[name] = jsval
    #    return Js(mod)
    #elif isintance(val, ClassType):

    elif isinstance(val, dict):  # convert to object
        temp = PyJsObject({}, ObjectPrototype)
        for k, v in six.iteritems(val):
            temp.put(Js(k), Js(v))
        return temp
    elif isinstance(val, (list, tuple)):  #Convert to array
        return PyJsArray(val, ArrayPrototype)
    # convert to typedarray
    elif isinstance(val, JsObjectWrapper):
        return val.__dict__['_obj']
    elif NUMPY_AVAILABLE and isinstance(val, numpy.ndarray):
        if val.dtype == numpy.int8:
            return PyJsInt8Array(val, Int8ArrayPrototype)
        elif val.dtype == numpy.uint8 and not Clamped:
            return PyJsUint8Array(val, Uint8ArrayPrototype)
        elif val.dtype == numpy.uint8 and Clamped:
            return PyJsUint8ClampedArray(val, Uint8ClampedArrayPrototype)
        elif val.dtype == numpy.int16:
            return PyJsInt16Array(val, Int16ArrayPrototype)
        elif val.dtype == numpy.uint16:
            return PyJsUint16Array(val, Uint16ArrayPrototype)

        elif val.dtype == numpy.int32:
            return PyJsInt32Array(val, Int32ArrayPrototype)
        elif val.dtype == numpy.uint32:
            return PyJsUint16Array(val, Uint32ArrayPrototype)

        elif val.dtype == numpy.float32:
            return PyJsFloat32Array(val, Float32ArrayPrototype)
        elif val.dtype == numpy.float64:
            return PyJsFloat64Array(val, Float64ArrayPrototype)
    else:  # try to convert to js object
        return py_wrap(val)

#        if isinstance(value, ModuleType):


def _set_name(self, name):
        '''name is py type'''
        if self.own.get('name'):
            self.func_name = name
            self.own['name']['value'] = Js(name)

def _set_name ( self , name ) : if self . own . get ( 'name' ) : self . func_name = name self . own [ 'name' ] [ 'value' ] = Js ( name )


def ConstructArray(self, py_arr):
        ''' note py_arr elems are NOT converted to PyJs types!'''
        arr = self.NewArray(len(py_arr))
        arr._init(py_arr)
        return arr

def ConstructArray ( self , py_arr ) : arr = self . NewArray ( len ( py_arr ) ) arr . _init ( py_arr ) return arr


def ConstructObject(self, py_obj):
        ''' note py_obj items are NOT converted to PyJs types! '''
        obj = self.NewObject()
        for k, v in py_obj.items():
            obj.put(unicode(k), v)
        return obj

def ConstructObject ( self , py_obj ) : obj = self . NewObject ( ) for k , v in py_obj . items ( ) : obj . put ( unicode ( k ) , v ) return obj


def emit(self, op_code, *args):
        ''' Adds op_code with specified args to tape '''
        self.tape.append(OP_CODES[op_code](*args))

def emit ( self , op_code , * args ) : self . tape . append ( OP_CODES [ op_code ] ( * args ) )


def compile(self, start_loc=0):
        ''' Records locations of labels and compiles the code '''
        self.label_locs = {} if self.label_locs is None else self.label_locs
        loc = start_loc
        while loc < len(self.tape):
            if type(self.tape[loc]) == LABEL:
                self.label_locs[self.tape[loc].num] = loc
                del self.tape[loc]
                continue
            loc += 1
        self.compiled = True

def compile ( self , start_loc = 0 ) : self . label_locs = { } if self . label_locs is None else self . label_locs loc = start_loc while loc < len ( self . tape ) : if type ( self . tape [ loc ] ) == LABEL : self . label_locs [ self . tape [ loc ] . num ] = loc del self . tape [ loc ] continue loc += 1 self . compiled = True


def pad(num, n=2, sign=False):
    '''returns n digit string representation of the num'''
    s = unicode(abs(num))
    if len(s) < n:
        s = '0' * (n - len(s)) + s
    if not sign:
        return s
    if num >= 0:
        return '+' + s
    else:
        return '-' + s

def pad ( num , n = 2 , sign = False ) : s = unicode ( abs ( num ) ) if len ( s ) < n : s = '0' * ( n - len ( s ) ) + s if not sign : return s if num >= 0 : return '+' + s else : return '-' + s


def replacement_template(rep, source, span, npar):
    """Takes the replacement template and some info about the match and returns filled template
       """
    n = 0
    res = ''
    while n < len(rep) - 1:
        char = rep[n]
        if char == '$':
            if rep[n + 1] == '$':
                res += '$'
                n += 2
                continue
            elif rep[n + 1] == '`':
                # replace with string that is BEFORE match
                res += source[:span[0]]
                n += 2
                continue
            elif rep[n + 1] == '\'':
                # replace with string that is AFTER match
                res += source[span[1]:]
                n += 2
                continue
            elif rep[n + 1] in DIGS:
                dig = rep[n + 1]
                if n + 2 < len(rep) and rep[n + 2] in DIGS:
                    dig += rep[n + 2]
                num = int(dig)
                # we will not do any replacements if we dont have this npar or dig is 0
                if not num or num > len(npar):
                    res += '$' + dig
                else:
                    # None - undefined has to be replaced with ''
                    res += npar[num - 1] if npar[num - 1] else ''
                n += 1 + len(dig)
                continue
        res += char
        n += 1
    if n < len(rep):
        res += rep[-1]
    return res


def fix_js_args(func):
    '''Use this function when unsure whether func takes this and arguments as its last 2 args.
       It will append 2 args if it does not.'''
    fcode = six.get_function_code(func)
    fargs = fcode.co_varnames[fcode.co_argcount - 2:fcode.co_argcount]
    if fargs == ('this', 'arguments') or fargs == ('arguments', 'var'):
        return func
    code = append_arguments(six.get_function_code(func), ('this', 'arguments'))

    return types.FunctionType(
        code,
        six.get_function_globals(func),
        func.__name__,
        closure=six.get_function_closure(func))

def fix_js_args ( func ) : fcode = six . get_function_code ( func ) fargs = fcode . co_varnames [ fcode . co_argcount - 2 : fcode . co_argcount ] if fargs == ( 'this' , 'arguments' ) or fargs == ( 'arguments' , 'var' ) : return func code = append_arguments ( six . get_function_code ( func ) , ( 'this' , 'arguments' ) ) return types . FunctionType ( code , six . get_function_globals ( func ) , func . __name__ , closure = six . get_function_closure ( func ) )

Use this function when unsure whether func takes this and arguments as its last 2 args . It will append 2 args if it does not .


def emit(self, what, *args):
        ''' what can be either name of the op, or node, or a list of statements.'''
        if isinstance(what, basestring):
            return self.exe.emit(what, *args)
        elif isinstance(what, list):
            self._emit_statement_list(what)
        else:
            return getattr(self, what['type'])(**what)

def emit ( self , what , * args ) : if isinstance ( what , basestring ) : return self . exe . emit ( what , * args ) elif isinstance ( what , list ) : self . _emit_statement_list ( what ) else : return getattr ( self , what [ 'type' ] ) ( * * what )

what can be either name of the op or node or a list of statements .


def trans(ele, standard=False):
    """Translates esprima syntax tree to python by delegating to appropriate translating node"""
    try:
        node = globals().get(ele['type'])
        if not node:
            raise NotImplementedError('%s is not supported!' % ele['type'])
        if standard:
            node = node.__dict__[
                'standard'] if 'standard' in node.__dict__ else node
        return node(**ele)
    except:
        #print ele
        raise

def
trans
(
ele
,
standard
=
False
)
:
try
:
node
=
globals
(
)
.
get
(
ele
[
'type'
]
)
if
not
node
:
raise
NotImplementedError
(
'%s is not supported!'
%
ele
[
'type'
]
)
if
standard
:
node
=
node
.
__dict__
[
'standard'
]
if
'standard'
in
node
.
__dict__
else
node
return
node
(
*
*
ele
)
except
:
#print ele
raise


def limited(func):
    '''Decorator limiting resulting line length in order to avoid python parser stack overflow -
      If expression longer than LINE_LEN_LIMIT characters then it will be moved to upper line
     USE ONLY ON EXPRESSIONS!!! '''

    def f(standard=False, **args):
        insert_pos = len(
            inline_stack.names
        )  # in case line is longer than limit we will have to insert the lval at current position
        # this is because calling func will change inline_stack.
        # we cant use inline_stack.require here because we dont know whether line overflows yet
        res = func(**args)
        if len(res) > LINE_LEN_LIMIT:
            name = inline_stack.require('LONG')
            inline_stack.names.pop()
            inline_stack.names.insert(insert_pos, name)
            res = 'def %s(var=var):\n    return %s\n' % (name, res)
            inline_stack.define(name, res)
            return name + '()'
        else:
            return res

    f.__dict__['standard'] = func
    return f


def is_lval(t):
    """Does not chceck whether t is not resticted or internal"""
    if not t:
        return False
    i = iter(t)
    if i.next() not in IDENTIFIER_START:
        return False
    return all(e in IDENTIFIER_PART for e in i)

def is_lval ( t ) : if not t : return False i = iter ( t ) if i . next ( ) not in IDENTIFIER_START : return False return all ( e in IDENTIFIER_PART for e in i )


def translate_file(input_path, output_path):
    '''
    Translates input JS file to python and saves the it to the output path.
    It appends some convenience code at the end so that it is easy to import JS objects.

    For example we have a file 'example.js' with:   var a = function(x) {return x}
    translate_file('example.js', 'example.py')

    Now example.py can be easily importend and used:
    >>> from example import example
    >>> example.a(30)
    30
    '''
    js = get_file_contents(input_path)

    py_code = translate_js(js)
    lib_name = os.path.basename(output_path).split('.')[0]
    head = '__all__ = [%s]\n\n# Don\'t look below, you will not understand this Python code :) I don\'t.\n\n' % repr(
        lib_name)
    tail = '\n\n# Add lib to the module scope\n%s = var.to_python()' % lib_name
    out = head + py_code + tail
    write_file_contents(output_path, out)

Translates input JS file to python and saves the it to the output path . It appends some convenience code at the end so that it is easy to import JS objects .


def execute(self, js=None, use_compilation_plan=False):
        """executes javascript js in current context

        During initial execute() the converted js is cached for re-use. That means next time you
        run the same javascript snippet you save many instructions needed to parse and convert the
        js code to python code.

        This cache causes minor overhead (a cache dicts is updated) but the Js=>Py conversion process
        is typically expensive compared to actually running the generated python code.

        Note that the cache is just a dict, it has no expiration or cleanup so when running this
        in automated situations with vast amounts of snippets it might increase memory usage.
        """
        try:
            cache = self.__dict__['cache']
        except KeyError:
            cache = self.__dict__['cache'] = {}
        hashkey = hashlib.md5(js.encode('utf-8')).digest()
        try:
            compiled = cache[hashkey]
        except KeyError:
            code = translate_js(
                js, '', use_compilation_plan=use_compilation_plan)
            compiled = cache[hashkey] = compile(code, '<EvalJS snippet>',
                                                'exec')
        exec (compiled, self._context)

def
execute
(
self
,
js
=
,
use_compilation_plan
=
False
)
:
try
:
cache
=
self
.
__dict__
[
'cache'
]
except
KeyError
:
cache
=
self
.
__dict__
[
'cache'
]
=
{
}
hashkey
=
hashlib
.
md5
(
js
.
encode
(
'utf-8'
)
)
.
digest
(
)
try
:
compiled
=
cache
[
hashkey
]
except
KeyError
:
code
=
translate_js
(
js
,
''
,
use_compilation_plan
=
use_compilation_plan
)
compiled
=
cache
[
hashkey
]
=
compile
(
code
,
'<EvalJS snippet>'
,
'exec'
)
exec
(
compiled
,
self
.
_context
)


def eval(self, expression, use_compilation_plan=False):
        """evaluates expression in current context and returns its value"""
        code = 'PyJsEvalResult = eval(%s)' % json.dumps(expression)
        self.execute(code, use_compilation_plan=use_compilation_plan)
        return self['PyJsEvalResult']

def
eval
(
self
,
expression
,
use_compilation_plan
=
False
)
:
code
=
'PyJsEvalResult = eval(%s)'
%
json
.
dumps
(
expression
)
self
.
execute
(
code
,
use_compilation_plan
=
use_compilation_plan
)
return
self
[
'PyJsEvalResult'
]


def except_token(source, start, token, throw=True):
    """Token can be only a single char. Returns position after token if found. Otherwise raises syntax error if throw
    otherwise returns None"""
    start = pass_white(source, start)
    if start < len(source) and source[start] == token:
        return start + 1
    if throw:
        raise SyntaxError('Missing token. Expected %s' % token)
    return None

def
except_token
(
source
,
start
,
token
,
throw
=
True
)
:
start
=
pass_white
(
source
,
start
)
if
start
<
len
(
source
)
and
source
[
start
]
==
token
:
return
start
+
1
if
throw
:
raise
SyntaxError
(
'Missing token. Expected %s'
%
token
)
return

Token can be only a single char . Returns position after token if found . Otherwise raises syntax error if throw otherwise returns None


def parse_identifier(source, start, throw=True):
    """passes white space from start and returns first identifier,
       if identifier invalid and throw raises SyntaxError otherwise returns None"""
    start = pass_white(source, start)
    end = start
    if not end < len(source):
        if throw:
            raise SyntaxError('Missing identifier!')
        return None
    if source[end] not in IDENTIFIER_START:
        if throw:
            raise SyntaxError('Invalid identifier start: "%s"' % source[end])
        return None
    end += 1
    while end < len(source) and source[end] in IDENTIFIER_PART:
        end += 1
    if not is_valid_lval(source[start:end]):
        if throw:
            raise SyntaxError(
                'Invalid identifier name: "%s"' % source[start:end])
        return None
    return source[start:end], end

def
parse_identifier
(
source
,
start
,
throw
=
True
)
:
start
=
pass_white
(
source
,
start
)
end
=
start
if
not
end
<
len
(
source
)
:
if
throw
:
raise
SyntaxError
(
'Missing identifier!'
)
return
if
source
[
end
]
not
in
IDENTIFIER_START
:
if
throw
:
raise
SyntaxError
(
'Invalid identifier start: "%s"'
%
source
[
end
]
)
return
end
+=
1
while
end
<
len
(
source
)
and
source
[
end
]
in
IDENTIFIER_PART
:
end
+=
1
if
not
is_valid_lval
(
source
[
start
:
end
]
)
:
if
throw
:
raise
SyntaxError
(
'Invalid identifier name: "%s"'
%
source
[
start
:
end
]
)
return
return
source
[
start
:
end
]
,
end

passes white space from start and returns first identifier if identifier invalid and throw raises SyntaxError otherwise returns None


def to_arr(this):
    """Returns Python array from Js array"""
    return [this.get(str(e)) for e in xrange(len(this))]

def to_arr ( this ) : return [ this . get ( str ( e ) ) for e in xrange ( len ( this ) ) ]


def do_statement(source, start):
    """returns none if not found other functions that begin with 'do_' raise
    also this do_ type function passes white space"""
    start = pass_white(source, start)
    # start is the fist position after initial start that is not a white space or \n
    if not start < len(source):  #if finished parsing return None
        return None, start
    if any(startswith_keyword(source[start:], e) for e in {'case', 'default'}):
        return None, start
    rest = source[start:]
    for key, meth in KEYWORD_METHODS.iteritems(
    ):  # check for statements that are uniquely defined by their keywords
        if rest.startswith(key):
            # has to startwith this keyword and the next letter after keyword must be either EOF or not in IDENTIFIER_PART
            if len(key) == len(rest) or rest[len(key)] not in IDENTIFIER_PART:
                return meth(source, start)
    if rest[0] == '{':  #Block
        return do_block(source, start)
    # Now only label and expression left
    cand = parse_identifier(source, start, False)
    if cand is not None:  # it can mean that its a label
        label, cand_start = cand
        cand_start = pass_white(source, cand_start)
        if source[cand_start] == ':':
            return do_label(source, start)
    return do_expression(source, start)

returns none if not found other functions that begin with do_ raise also this do_ type function passes white space


def match(self, string, pos):
        '''string is of course a py string'''
        return self.pat.match(string, int(pos))

def match ( self , string , pos ) : return self . pat . match ( string , int ( pos ) )


def call(self, this, args=()):
        ''' Dont use this method from inside bytecode to call other bytecode. '''
        if self.is_native:
            _args = SpaceTuple(
                args
            )  # we have to do that unfortunately to pass all the necessary info to the funcs
            _args.space = self.space
            return self.code(
                this, _args
            )  # must return valid js object - undefined, null, float, unicode, bool, or PyJs
        else:
            return self.space.exe._call(self, this,
                                        args)

Dont use this method from inside bytecode to call other bytecode .


def is_empty_object(n, last):
    """n may be the inside of block or object"""
    if n.strip():
        return False
    # seems to be but can be empty code
    last = last.strip()
    markers = {
        ')',
        ';',
    }
    if not last or last[-1] in markers:
        return False
    return True

def
is_empty_object
(
n
,
last
)
:
if
n
.
strip
(
)
:
return
False
# seems to be but can be empty code
last
=
last
.
strip
(
)
markers
=
{
')'
,
';'
,
}
if
not
last
or
last
[
1
]
in
markers
:
return
False
return
True


def is_object(n, last):
    """n may be the inside of block or object.
       last is the code before object"""
    if is_empty_object(n, last):
        return True
    if not n.strip():
        return False
    #Object contains lines of code so it cant be an object
    if len(argsplit(n, ';')) > 1:
        return False
    cands = argsplit(n, ',')
    if not cands[-1].strip():
        return True  # {xxxx,} empty after last , it must be an object
    for cand in cands:
        cand = cand.strip()
        # separate each candidate element at : in dict and check whether they are correct...
        kv = argsplit(cand, ':')
        if len(
                kv
        ) > 2:  # set the len of kv to 2 because of this stupid : expression
            kv = kv[0], ':'.join(kv[1:])

        if len(kv) == 2:
            # key value pair, check whether not label or ?:
            k, v = kv
            if not is_lval(k.strip()):
                return False
            v = v.strip()
            if v.startswith('function'):
                continue
            #will fail on label... {xxx: while {}}
            if v[0] == '{':  # value cant be a code block
                return False
            for e in KEYWORD_METHODS:
                # if v starts with any statement then return false
                if v.startswith(e) and len(e) < len(v) and v[len(
                        e)] not in IDENTIFIER_PART:
                    return False
        elif not (cand.startswith('set ') or cand.startswith('get ')):
            return False
    return True

n may be the inside of block or object . last is the code before object


def remove_objects(code, count=1):
    """ This function replaces objects with OBJECTS_LVALS, returns new code, replacement dict and count.
        count arg is the number that should be added to the LVAL of the first replaced object
    """
    replacements = {}  #replacement dict
    br = bracket_split(code, ['{}', '[]'])
    res = ''
    last = ''
    for e in br:
        #test whether e is an object
        if e[0] == '{':
            n, temp_rep, cand_count = remove_objects(e[1:-1], count)
            # if e was not an object then n should not contain any :
            if is_object(n, last):
                #e was an object
                res += ' ' + OBJECT_LVAL % count
                replacements[OBJECT_LVAL % count] = e
                count += 1
            else:
                # e was just a code block but could contain objects inside
                res += '{%s}' % n
                count = cand_count
                replacements.update(temp_rep)
        elif e[0] == '[':
            if is_array(last):
                res += e  # will be translated later
            else:  # prop get
                n, rep, count = remove_objects(e[1:-1], count)
                res += '[%s]' % n
                replacements.update(rep)
        else:  # e does not contain any objects
            res += e
        last = e  #needed to test for this stipid empty object
    return res, replacements, count

This function replaces objects with OBJECTS_LVALS returns new code replacement dict and count . count arg is the number that should be added to the LVAL of the first replaced object


def _ensure_regexp(source, n):  #<- this function has to be improved
    '''returns True if regexp starts at n else returns False
      checks whether it is not a division '''
    markers = '(+~"\'=[%:?!*^|&-,;/\\'
    k = 0
    while True:
        k += 1
        if n - k < 0:
            return True
        char = source[n - k]
        if char in markers:
            return True
        if char != ' ' and char != '\n':
            break
    return False

def
_ensure_regexp
(
source
,
n
)
:
#<- this function has to be improved
markers
=
'(+~"\'=[%:?!*^|&-,;/\\'
k
=
0
while
True
:
k
+=
1
if
n
k
<
0
:
return
True
char
=
source
[
n
k
]
if
char
in
markers
:
return
True
if
char
!=
' '
and
char
!=
'\n'
:
break
return
False


def parse_num(source, start, charset):
    """Returns a first index>=start of chat not in charset"""
    while start < len(source) and source[start] in charset:
        start += 1
    return start

def parse_num ( source , start , charset ) : while start < len ( source ) and source [ start ] in charset : start += 1 return start

Returns a first index > = start of chat not in charset


def parse_exponent(source, start):
    """returns end of exponential, raises SyntaxError if failed"""
    if not source[start] in {'e', 'E'}:
        if source[start] in IDENTIFIER_PART:
            raise SyntaxError('Invalid number literal!')
        return start
    start += 1
    if source[start] in {'-', '+'}:
        start += 1
    FOUND = False
    # we need at least one dig after exponent
    while source[start] in NUMS:
        FOUND = True
        start += 1
    if not FOUND or source[start] in IDENTIFIER_PART:
        raise SyntaxError('Invalid number literal!')
    return start

returns end of exponential raises SyntaxError if failed


def unify_string_literals(js_string):
    """this function parses the string just like javascript
       for example literal '\d' in JavaScript would be interpreted
       as 'd' - backslash would be ignored and in Pyhon this
       would be interpreted as '\\d' This function fixes this problem."""
    n = 0
    res = ''
    limit = len(js_string)
    while n < limit:
        char = js_string[n]
        if char == '\\':
            new, n = do_escape(js_string, n)
            res += new
        else:
            res += char
            n += 1
    return res

def unify_string_literals ( js_string ) : n = 0 res = '' limit = len ( js_string ) while n < limit : char = js_string [ n ] if char == '\\' : new , n = do_escape ( js_string , n ) res += new else : res += char n += 1 return res

this function parses the string just like javascript for example literal \ d in JavaScript would be interpreted as d - backslash would be ignored and in Pyhon this would be interpreted as \\ d This function fixes this problem .


def in_op(self, other):
    '''checks if self is in other'''
    if not is_object(other):
        raise MakeError(
            'TypeError',
            "You can\'t use 'in' operator to search in non-objects")
    return other.has_property(to_string(self))


def maybe_download_and_extract():
  """Download and extract processed data and embeddings."""
  dest_directory = '.'
  filename = DATA_URL.split('/')[-1]
  filepath = os.path.join(dest_directory, filename)
  if not os.path.exists(filepath):
    def _progress(count, block_size, total_size):
      sys.stdout.write('\r>> Downloading %s %.1f%%' % (filename,
          float(count * block_size) / float(total_size) * 100.0))
      sys.stdout.flush()
    filepath, _ = urllib.request.urlretrieve(DATA_URL, filepath, _progress)
    print()
    statinfo = os.stat(filepath)
    print('Successfully downloaded', filename, statinfo.st_size, 'bytes.')
  extracted_dir_path = os.path.join(dest_directory, 'trees')
  if not os.path.exists(extracted_dir_path):
    zip_ref = zipfile.ZipFile(filepath, 'r')
    zip_ref.extractall(dest_directory)
    zip_ref.close()

def
maybe_download_and_extract
(
)
:
dest_directory
=
'.'
filename
=
DATA_URL
.
split
(
'/'
)
[
1
]
filepath
=
os
.
path
.
join
(
dest_directory
,
filename
)
if
not
os
.
path
.
exists
(
filepath
)
:
def
_progress
(
count
,
block_size
,
total_size
)
:
sys
.
stdout
.
write
(
'\r>> Downloading %s %.1f%%'
%
(
filename
,
float
(
count
*
block_size
)
/
float
(
total_size
)
*
100.0
)
)
sys
.
stdout
.
flush
(
)
filepath
,
_
=
urllib
.
request
.
urlretrieve
(
DATA_URL
,
filepath
,
_progress
)
print
(
)
statinfo
=
os
.
stat
(
filepath
)
print
(
'Successfully downloaded'
,
filename
,
statinfo
.
st_size
,
'bytes.'
)
extracted_dir_path
=
os
.
path
.
join
(
dest_directory
,
'trees'
)
if
not
os
.
path
.
exists
(
extracted_dir_path
)
:
zip_ref
=
zipfile
.
ZipFile
(
filepath
,
'r'
)
zip_ref
.
extractall
(
dest_directory
)
zip_ref
.
close
(
)

Download and extract processed data and embeddings .


def add_inputs(self, xs):
        """
        returns the list of states obtained by adding the given inputs
        to the current state, one by one.
        """
        states = []
        cur = self
        for x in xs:
            cur = cur.add_input(x)
            states.append(cur)
        return states

def add_inputs ( self , xs ) : states = [ ] cur = self for x in xs : cur = cur . add_input ( x ) states . append ( cur ) return states

returns the list of states obtained by adding the given inputs to the current state one by one .