# Copyright 2017 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""gcloud CLI tree generators for non-gcloud CLIs.

A CLI tree for a supported command is generated by using the root command plus
`help` or `--help` arguments to do a DFS traversal. Each node is generated
from a man-ish style runtime document.

Supported CLI commands have their own runtime help document quirks, so each is
handled by an ad-hoc parser. The parsers rely on consistency within commands
and between command releases.

The CLI tree for an unsupported command is generated from the output of
`man the-command` and contains only the command root node.
"""

import abc
import json
import os
import re
import subprocess
import sys
import textwrap

from googlecloudsdk.calliope import cli_tree
from googlecloudsdk.core import exceptions
from googlecloudsdk.core import log
from googlecloudsdk.core.console import progress_tracker
from googlecloudsdk.core.resource import resource_printer
from googlecloudsdk.core.util import encoding
from googlecloudsdk.core.util import files
from googlecloudsdk.core.util import text as text_utils

import httplib2
from typing import Any, Dict  # pylint: disable=unused-import


class NoCliTreeGeneratorForCommand(exceptions.Error):
  """Command does not have a CLI tree generator."""


class NoManPageTextForCommand(exceptions.Error):
  """Could not get man page text for command."""


# TODO(b/69033748): disable until issues resolved
def _DisableLongRunningCliTreeGeneration(command):
  """Returns True if long running CLI tree generation is disabled."""
  # Only a few command generators are long running.
  if command not in ('bq', 'gsutil', 'kubectl'):
    return False
  # Only generate these CLI trees on the fly for explicit requests.
  # It can take ~1minute, not good, especially at the interactive prompt.
  if 'update-cli-trees' in sys.argv or '--update-cli-trees' in sys.argv:
    return False
  # It's a long running generator, not explicitly requested -- disable.
  return True


def _NormalizeSpace(text):
  """Returns text dedented and multiple non-indent spaces replaced by one."""
  return re.sub('([^ ])   *', r'\1 ', textwrap.dedent(text)).strip('\n')


def _Flag(name, description='', value=None, default=None, type_='string',
          category='', is_global=False, is_required=False, nargs=None):
  """Initializes and returns a flag dict node."""
  return {
      cli_tree.LOOKUP_ATTR: {},
      cli_tree.LOOKUP_CATEGORY: category,
      cli_tree.LOOKUP_DEFAULT: default,
      cli_tree.LOOKUP_DESCRIPTION: _NormalizeSpace(description),
      cli_tree.LOOKUP_GROUP: '',
      cli_tree.LOOKUP_IS_GLOBAL: is_global,
      cli_tree.LOOKUP_IS_HIDDEN: False,
      cli_tree.LOOKUP_IS_REQUIRED: is_required,
      cli_tree.LOOKUP_NAME: name,
      cli_tree.LOOKUP_NARGS: nargs or ('0' if type_ == 'bool' else '1'),
      cli_tree.LOOKUP_VALUE: value,
      cli_tree.LOOKUP_TYPE: type_,
  }


def _Positional(name, description='', default=None, nargs='0'):
  """Initializes and returns a positional dict node."""
  return {
      cli_tree.LOOKUP_DEFAULT: default,
      cli_tree.LOOKUP_DESCRIPTION: _NormalizeSpace(description),
      cli_tree.LOOKUP_NAME: name,
      cli_tree.LOOKUP_NARGS: nargs,
  }


def _Command(path):
  # type: (str) -> Dict[str, Any]
  """Initializes and returns a command/group dict node."""
  return {
      cli_tree.LOOKUP_CAPSULE: '',
      cli_tree.LOOKUP_COMMANDS: {},
      cli_tree.LOOKUP_FLAGS: {},
      cli_tree.LOOKUP_GROUPS: {},
      cli_tree.LOOKUP_IS_GROUP: False,
      cli_tree.LOOKUP_IS_HIDDEN: False,
      cli_tree.LOOKUP_PATH: path,
      cli_tree.LOOKUP_POSITIONALS: [],
      cli_tree.LOOKUP_RELEASE: 'GA',
      cli_tree.LOOKUP_SECTIONS: {},
  }


class CliTreeGenerator(object):
  """Base CLI tree generator."""

  __metaclass__ = abc.ABCMeta

  _FAILURES = None

  @classmethod
  def MemoizeFailures(cls, enable):
    """Memoizes failed attempts and doesn't repeat them if enable is True."""
    cls._FAILURES = set() if enable else None

  @classmethod
  def AlreadyFailed(cls, command):
    """Returns True if man page request for command already failed."""
    return command in cls._FAILURES if cls._FAILURES else False

  @classmethod
  def AddFailure(cls, command):
    """Add command to the set of failed man generations."""
    if cls._FAILURES is not None:
      cls._FAILURES.add(command)

  def __init__(self, command):
    self.command = command
    self._cli_version = None  # For memoizing GetVersion()

  def Run(self, cmd):
    """Runs cmd and returns the output as a string."""
    return subprocess.check_output(cmd)

  def GetVersion(self):
    """Returns the CLI_VERSION string."""
    if not self._cli_version:
      self._cli_version = self.Run([self.command, 'version']).split()[-1]
    return self._cli_version

  @abc.abstractmethod
  def Generate(self):
    """Generates and returns the CLI tree dict."""
    return None

  def FindTreeFile(self, directories):
    """Returns (path,f) open for read for the first CLI tree in directories."""
    for directory in directories or [cli_tree.CliTreeConfigDir()]:
      path = os.path.join(directory or '.', self.command) + '.json'
      try:
        return path, open(path, 'r')
      except IOError:
        pass
    return path, None  # pytype: disable=name-error

  def IsUpToDate(self, tree, verbose=False):
    """Returns a bool tuple (readonly, up_to_date)."""

    actual_cli_version = tree.get(cli_tree.LOOKUP_CLI_VERSION)
    readonly = actual_cli_version == cli_tree.CLI_VERSION_READONLY

    # Check the schema version.
    actual_tree_version = tree.get(cli_tree.LOOKUP_VERSION)
    if actual_tree_version != cli_tree.VERSION:
      return readonly, False

    # Check the CLI version.
    expected_cli_version = self.GetVersion()
    if readonly:
      # READONLY trees are always up to date.
      pass
    elif expected_cli_version == cli_tree.CLI_VERSION_UNKNOWN:
      # Don't know how to regenerate but we have one in hand -- accept it.
      pass
    elif actual_cli_version != expected_cli_version:
      return readonly, False

    # Schema and CLI versions are up to date.
    if verbose:
      log.status.Print(u'[{}] CLI tree version [{}] is up to date.'.format(
          self.command, actual_cli_version))
    return readonly, True

  def LoadOrGenerate(self, directories=None, force=False,
                     ignore_out_of_date=False, verbose=False,
                     warn_on_exceptions=False):
    """Loads the CLI tree or generates it if necessary, and returns the tree."""
    path, f = self.FindTreeFile(directories)
    if not f:
      # TODO(b/69033748): disable until issues resolved
      if _DisableLongRunningCliTreeGeneration(self.command):
        return None
    else:
      up_to_date = False
      with f:
        try:
          tree = json.load(f)
        except ValueError:
          # Corrupt JSON -- could have been interrupted.
          tree = None
        if tree:
          readonly, up_to_date = self.IsUpToDate(tree, verbose=verbose)
      if readonly:
        return tree
      elif up_to_date:
        if not force:
          return tree
      elif ignore_out_of_date:
        return None

    def _Generate():
      """Helper that generates a CLI tree and writes it to a JSON file."""
      tree = self.Generate()
      if tree:
        try:
          f = open(path, 'w')
        except EnvironmentError as e:
          # CLI data config dir may not be initialized yet.
          directory, _ = os.path.split(path)
          try:
            files.MakeDir(directory)
            f = open(path, 'w')
          except (EnvironmentError, files.Error):
            if not warn_on_exceptions:
              raise
            log.warn(str(e))
            return None
        with f:
          resource_printer.Print(tree, print_format='json', out=f)
      return tree

    # At this point:
    #   (1) the tree is not found or is out of date
    #   (2) the tree is not readonly
    #   (3) we have a generator for the tree

    if not verbose:
      return _Generate()
    with progress_tracker.ProgressTracker(
        u'{} the [{}] CLI tree'.format(
            'Updating' if f else 'Generating', self.command)):
      return _Generate()


class _BqCollector(object):
  """bq help document section collector."""

  def __init__(self, text):
    self.text = text.split('\n')
    self.heading = 'DESCRIPTION'
    self.lookahead = None
    self.ignore_trailer = False

  def Collect(self, strip_headings=False):
    """Returns the heading and content lines from text."""
    content = []
    if self.lookahead:
      if not strip_headings:
        content.append(self.lookahead)
      self.lookahead = None
    heading = self.heading
    self.heading = None
    while self.text:
      line = self.text.pop(0)
      if line.startswith(' ') or not strip_headings and not self.ignore_trailer:
        content.append(line.rstrip())
    while content and not content[0]:
      content.pop(0)
    while content and not content[-1]:
      content.pop()
    self.ignore_trailer = True
    return heading, content


class BqCliTreeGenerator(CliTreeGenerator):
  """bq CLI tree generator."""

  def Run(self, cmd):
    """Runs cmd and returns the output as a string."""
    try:
      output = subprocess.check_output(cmd)
    except subprocess.CalledProcessError as e:
      # bq exit code is 1 for help and --help. How do you know if help failed?
      if e.returncode != 1:
        raise
      output = e.output
    return output.replace('bq.py', 'bq')

  def AddFlags(self, command, content, is_global=False):
    """Adds flags in content lines to command."""
    while content:
      line = content.pop(0)
      name, description = line.strip().split(':', 1)
      paragraph = [description.strip()]
      default = ''
      while content and not content[0].startswith('  --'):
        line = content.pop(0).strip()
        if line.startswith('(default: '):
          default = line[10:-1]
        else:
          paragraph.append(line)
      description = ' '.join(paragraph).strip()
      if name.startswith('--[no]'):
        name = '--' + name[6:]
        type_ = 'bool'
        value = ''
      else:
        value = 'VALUE'
        type_ = 'string'
      command[cli_tree.LOOKUP_FLAGS][name] = _Flag(
          name=name,
          description=description,
          type_=type_,
          value=value,
          default=default,
          is_required=False,
          is_global=is_global,
      )

  def SubTree(self, path):
    """Generates and returns the CLI subtree rooted at path."""
    command = _Command(path)
    command[cli_tree.LOOKUP_IS_GROUP] = True
    text = self.Run([path[0], 'help'] + path[1:])

    # `bq help` lists help for all commands. Command flags are "defined"
    # by example. We don't attempt to suss that out.
    content = text.split('\n')
    while content:
      line = content.pop(0)
      if not line or not line[0].islower():
        continue
      name, text = line.split(' ', 1)
      description = [text.strip()]
      examples = []
      arguments = []
      paragraph = description
      while content and (not content[0] or not content[0][0].islower()):
        line = content.pop(0).strip()
        if line == 'Arguments:':
          paragraph = arguments
        elif line == 'Examples:':
          paragraph = examples
        else:
          paragraph.append(line)
      subcommand = _Command(path + [name])
      command[cli_tree.LOOKUP_COMMANDS][name] = subcommand  # pytype: disable=attribute-error
      if description:
        subcommand[cli_tree.LOOKUP_SECTIONS]['DESCRIPTION'] = '\n'.join(
            description)  # pytype: disable=attribute-error
      if examples:
        subcommand[cli_tree.LOOKUP_SECTIONS]['EXAMPLES'] = '\n'.join(
            examples)  # pytype: disable=attribute-error

    return command

  def Generate(self):
    """Generates and returns the CLI tree rooted at self.command."""

    # Construct the tree minus the global flags.
    tree = self.SubTree([self.command])

    # Add the global flags to the root.
    text = self.Run([self.command, '--help'])
    collector = _BqCollector(text)
    _, content = collector.Collect(strip_headings=True)
    self.AddFlags(tree, content, is_global=True)

    # Finally add the version stamps.
    tree[cli_tree.LOOKUP_CLI_VERSION] = self.GetVersion()
    tree[cli_tree.LOOKUP_VERSION] = cli_tree.VERSION

    return tree


class _GsutilCollector(object):
  """gsutil help document section collector."""

  UNKNOWN, ROOT, MAN, TOPIC = range(4)

  def __init__(self, text):
    self.text = text.split('\n')
    self.heading = 'CAPSULE'
    self.page_type = self.UNKNOWN

  def Collect(self, strip_headings=False):
    """Returns the heading and content lines from text."""
    content = []
    heading = self.heading
    self.heading = None
    while self.text:
      line = self.text.pop(0)
      if self.page_type == self.UNKNOWN:
        # The first heading distinguishes the document page type.
        if line.startswith('Usage:'):
          self.page_type = self.ROOT
          continue
        elif line == 'NAME':
          self.page_type = self.MAN
          heading = 'CAPSULE'
          continue
        elif not line.startswith(' '):
          continue
      elif self.page_type == self.ROOT:
        # The root help page.
        if line == 'Available commands:':
          heading = 'COMMANDS'
          continue
        elif line == 'Additional help topics:':
          self.heading = 'TOPICS'
          break
        elif not line.startswith(' '):
          continue
      elif self.page_type == self.MAN:
        # A command/subcommand man style page.
        if line == 'OVERVIEW':
          self.page_type = self.TOPIC
          self.heading = 'DESCRIPTION'
          break
        elif line == 'SYNOPSIS':
          self.heading = line
          break
        elif line.endswith('OPTIONS'):
          self.heading = 'FLAGS'
          break
        elif line and line[0].isupper():
          self.heading = line.split(' ', 1)[-1]
          break
      elif self.page_type == self.TOPIC:
        # A topic man style page.
        if line and line[0].isupper():
          self.heading = line
          break
      if line.startswith(' ') or not strip_headings:
        content.append(line.rstrip())
    while content and not content[0]:
      content.pop(0)
    while content and not content[-1]:
      content.pop()
    return heading, content


class GsutilCliTreeGenerator(CliTreeGenerator):
  """gsutil CLI tree generator."""

  def __init__(self, *args, **kwargs):
    super(GsutilCliTreeGenerator, self).__init__(*args, **kwargs)
    self.topics = []

  def Run(self, cmd):
    """Runs the command in cmd and returns the output as a string."""
    try:
      output = subprocess.check_output(cmd)
    except subprocess.CalledProcessError as e:
      # gsutil exit code is 1 for --help depending on the context.
      if e.returncode != 1:
        raise
      output = e.output
    return output

  def AddFlags(self, command, content, is_global=False):
    """Adds flags in content lines to command."""

    def _Add(name, description):
      value = ''
      type_ = 'bool'
      default = ''
      command[cli_tree.LOOKUP_FLAGS][name] = _Flag(
          name=name,
          description=description,
          type_=type_,
          value=value,
          default=default,
          is_required=False,
          is_global=is_global,
      )

    parse = re.compile(' *((-[^ ]*,)* *(-[^ ]*) *)(.*)')
    name = None
    description = []
    for line in content:
      if line.startswith('  -'):
        if name:
          _Add(name, '\n'.join(description))
        match = parse.match(line)
        name = match.group(3)
        description = [match.group(4).rstrip()]
      elif len(line) > 16:
        description.append(line[16:].rstrip())
    if name:
      _Add(name, '\n'.join(description))

  def SubTree(self, path):
    """Generates and returns the CLI subtree rooted at path."""
    command = _Command(path)
    is_help_command = len(path) > 1 and path[1] == 'help'
    if is_help_command:
      cmd = path
    else:
      cmd = path + ['--help']
    text = self.Run(cmd)
    collector = _GsutilCollector(text)

    while True:
      heading, content = collector.Collect()
      if not heading:
        break
      elif heading == 'CAPSULE':
        if content:
          command[cli_tree.LOOKUP_CAPSULE] = content[0].split('-', 1)[1].strip()
      elif heading == 'COMMANDS':
        if is_help_command:
          continue
        for line in content:
          try:
            name = line.split()[0]
          except IndexError:
            continue
          if name == 'update':
            continue
          command[cli_tree.LOOKUP_IS_GROUP] = True
          command[cli_tree.LOOKUP_COMMANDS][name] = self.SubTree(path + [name])
      elif heading == 'FLAGS':
        self.AddFlags(command, content)
      elif heading == 'SYNOPSIS':
        commands = []
        for line in content:
          if not line:
            break
          cmd = line.split()
          if len(cmd) <= len(path):
            continue
          if cmd[:len(path)] == path:
            name = cmd[len(path)]
            if name[0].islower() and name not in ('off', 'on', 'false', 'true'):
              commands.append(name)
        if len(commands) > 1:
          command[cli_tree.LOOKUP_IS_GROUP] = True
          for name in commands:
            command[cli_tree.LOOKUP_COMMANDS][name] = self.SubTree(
                path + [name])
      elif heading == 'TOPICS':
        for line in content:
          try:
            self.topics.append(line.split()[0])
          except IndexError:
            continue
      elif heading.isupper():
        if heading.lower() == path[-1]:
          heading = 'DESCRIPTION'
        command[cli_tree.LOOKUP_SECTIONS][heading] = '\n'.join(
            [line[2:] for line in content])

    return command

  def Generate(self):
    """Generates and returns the CLI tree rooted at self.command."""
    tree = self.SubTree([self.command])

    # Add the global flags to the root.
    text = self.Run([self.command, 'help', 'options'])
    collector = _GsutilCollector(text)
    while True:
      heading, content = collector.Collect()
      if not heading:
        break
      if heading == 'FLAGS':
        self.AddFlags(tree, content, is_global=True)

    # Add the help topics.
    help_command = tree[cli_tree.LOOKUP_COMMANDS]['help']
    help_command[cli_tree.LOOKUP_IS_GROUP] = True
    for topic in self.topics:
      help_command[cli_tree.LOOKUP_COMMANDS][topic] = self.SubTree(
          help_command[cli_tree.LOOKUP_PATH] + [topic])

    # Finally add the version stamps.
    tree[cli_tree.LOOKUP_CLI_VERSION] = self.GetVersion()
    tree[cli_tree.LOOKUP_VERSION] = cli_tree.VERSION

    return tree


class _KubectlCollector(object):
  """Kubectl help document section collector."""

  def __init__(self, text):
    self.text = text.split('\n')
    self.heading = 'DESCRIPTION'
    self.lookahead = None
    self.ignore_trailer = False

  def Collect(self, strip_headings=False):
    """Returns the heading and content lines from text."""
    content = []
    if self.lookahead:
      if not strip_headings:
        content.append(self.lookahead)
      self.lookahead = None
    heading = self.heading
    self.heading = None
    while self.text:
      line = self.text.pop(0)
      usage = 'Usage:'
      if line.startswith(usage):
        line = line[len(usage):].strip()
        if line:
          self.lookahead = line
        self.heading = 'USAGE'
        break
      if line.endswith(':'):
        if 'Commands' in line:
          self.heading = 'COMMANDS'
          break
        if 'Examples' in line:
          self.heading = 'EXAMPLES'
          break
        if 'Options' in line:
          self.heading = 'FLAGS'
          break
      if line.startswith(' ') or not strip_headings and not self.ignore_trailer:
        content.append(line.rstrip())
    while content and not content[0]:
      content.pop(0)
    while content and not content[-1]:
      content.pop()
    self.ignore_trailer = True
    return heading, content


class KubectlCliTreeGenerator(CliTreeGenerator):
  """kubectl CLI tree generator."""

  def AddFlags(self, command, content, is_global=False):
    """Adds flags in content lines to command."""
    for line in content:
      flags, description = line.strip().split(':', 1)
      flag = flags.split(', ')[-1]
      name, value = flag.split('=')
      if value in ('true', 'false'):
        value = ''
        type_ = 'bool'
      else:
        value = 'VALUE'
        type_ = 'string'
      default = ''
      command[cli_tree.LOOKUP_FLAGS][name] = _Flag(
          name=name,
          description=description,
          type_=type_,
          value=value,
          default=default,
          is_required=False,
          is_global=is_global,
      )

  def SubTree(self, path):
    """Generates and returns the CLI subtree rooted at path."""
    command = _Command(path)
    text = self.Run(path + ['--help'])
    collector = _KubectlCollector(text)

    while True:
      heading, content = collector.Collect()
      if not heading:
        break
      elif heading == 'COMMANDS':
        for line in content:
          try:
            name = line.split()[0]
          except IndexError:
            continue
          command[cli_tree.LOOKUP_IS_GROUP] = True
          command[cli_tree.LOOKUP_COMMANDS][name] = self.SubTree(path + [name])
      elif heading in ('DESCRIPTION', 'EXAMPLES'):
        command[cli_tree.LOOKUP_SECTIONS][heading] = '\n'.join(content)  # pytype: disable=attribute-error
      elif heading == 'FLAGS':
        self.AddFlags(command, content)
    return command

  def GetVersion(self):
    """Returns the CLI_VERSION string."""
    if not self._cli_version:
      verbose_version = self.Run([self.command, 'version', '--client'])
      match = re.search('GitVersion:"([^"]*)"', verbose_version)
      self._cli_version = match.group(1)
    return self._cli_version

  def Generate(self):
    """Generates and returns the CLI tree rooted at self.command."""

    # Construct the tree minus the global flags.
    tree = self.SubTree([self.command])

    # Add the global flags to the root.
    text = self.Run([self.command, 'options'])
    collector = _KubectlCollector(text)
    _, content = collector.Collect(strip_headings=True)
    content.append('  --help=true: List detailed command help.')
    self.AddFlags(tree, content, is_global=True)

    # Finally add the version stamps.
    tree[cli_tree.LOOKUP_CLI_VERSION] = self.GetVersion()
    tree[cli_tree.LOOKUP_VERSION] = cli_tree.VERSION

    return tree


class _ManPageCollector(object):
  """man page help document section collector base class.

  Attributes:
    command: The man page command name.
    content_indent: A string of space characters representing the indent of
      the first line of content for any section.
    heading: The heading for the next call to Collect().
    text: The list of man page lines.
    version: The collector CLI_VERSION string.
  """

  __metaclass__ = abc.ABCMeta

  def __init__(self, command):
    self.command = command
    self.content_indent = None
    self.heading = None
    self.text = self.GetManPageText().split('\n')  # pytype: disable=none-attr

  @classmethod
  @abc.abstractmethod
  def GetVersion(cls):
    """Returns the CLI_VERSION string."""
    return None

  @abc.abstractmethod
  def _GetRawManPageText(self):
    """Returns the raw man page text."""
    return None

  @abc.abstractmethod
  def GetManPageText(self):
    """Returns the preprocessed man page text."""
    return None

  def Collect(self):
    """Returns the heading and content lines from text."""
    content = []
    heading = self.heading
    self.heading = None
    while self.text:
      line = self.text.pop(0)
      if not heading:
        # No NAME no man page.
        if line == 'NAME':
          heading = line
        continue
      elif not line:
        pass
      elif line[0] == ' ':
        if not self.content_indent:
          self.content_indent = re.sub('[^ ].*', '', line)
        if len(line) > len(self.content_indent):
          indented_char = line[len(self.content_indent)]
          if not line.startswith(self.content_indent):
            # Subsection heading or category.
            line = '### ' + line.strip()
          elif heading == 'DESCRIPTION' and indented_char == '-':
            # Some man pages, like GNU ls(1), inline flags in DESCRIPTION.
            self.text.insert(0, line)
            self.heading = 'FLAGS'
            break
          elif heading == 'FLAGS' and indented_char not in (' ', '-'):
            self.text.insert(0, line)
            self.heading = 'DESCRIPTION'
            break
      elif line in ('SYNOPSIS', 'DESCRIPTION', 'EXIT STATUS', 'SEE ALSO'):
        self.heading = line
        break
      elif 'FLAGS' in line or 'OPTIONS' in line:
        self.heading = 'FLAGS'
        break
      elif line and line[0].isupper():
        self.heading = line.split(' ', 1)[-1]
        break
      content.append(line.rstrip())
    while content and not content[0]:
      content.pop(0)
    while content and not content[-1]:
      content.pop()
    return heading, content


class _ManCommandCollector(_ManPageCollector):
  """man command help document section collector."""

  _CLI_VERSION = 'man-0.1'

  @classmethod
  def GetVersion(cls):
    return cls._CLI_VERSION

  def _GetRawManPageText(self):
    """Returns the raw man page text."""
    try:
      with open(os.devnull, 'w') as f:
        return subprocess.check_output(['man', self.command], stderr=f)
    except (OSError, subprocess.CalledProcessError):
      raise NoManPageTextForCommand(
          'Cannot get man(1) command man page text for [{}].'.format(
              self.command))

  def GetManPageText(self):
    """Returns the preprocessed man page text."""
    text = self._GetRawManPageText()
    return re.sub(
        '.\b', '', re.sub(
            u'(\u2010|\\u2010)\n *', '', encoding.Decode(text)))


class _ManUrlCollector(_ManPageCollector):
  """man URL help document section collector."""

  _CLI_VERSION = 'man7.org-0.1'

  @classmethod
  def GetVersion(cls):
    return cls._CLI_VERSION

  def _GetRawManPageText(self):
    """Returns the raw man page text."""
    url = 'http://man7.org/linux/man-pages/man1/{}.1.html'.format(self.command)
    response, content = httplib2.Http().request(url)
    if response.status != 200:
      raise NoManPageTextForCommand(
          'Cannot get URL man page text for [{}].'.format(self.command))
    return content.decode('utf-8')

  def GetManPageText(self):
    """Returns the text man page for self.command from a URL."""
    text = self._GetRawManPageText()

    # A little sed(1) to clean up the html header/trailer and anchors.
    # A mispplaced .* or (...) group match could affect timing. Keep track
    # of that if you change any of the pattern,replacement tuples.
    for pattern, replacement in (
        ('<span class="footline">.*', ''),
        ('<h2><a id="([^"]*)"[^\n]*\n', '\\1\n'),
        ('<b>( +)', '\\1*'),
        ('( +)</b>', '*\\1'),
        ('<i>( +)', '\\1_'),
        ('( +)</i>', '_\\1'),
        ('</?b>', '*'),
        ('</?i>', '_'),
        ('</pre>', ''),
        ('<a href="([^"]*)">([^\n]*)</a>', '[\\1](\\2)'),
        ('&amp;', '\\&'),
        ('&gt;', '>'),
        ('&lt;', '<'),
        ('&#39;', "'"),
    ):
      text = re.sub(pattern, replacement, text, flags=re.DOTALL)

    # ... and some non-regular edits to finish up.
    lines = []
    top = 'NAME'
    flags = False
    paragraph = False
    for line in text.split('\n'):
      if top and line == 'NAME':
        # Ignore all lines before the NAME section.
        top = None
        lines = []
      if line.startswith('       *-'):  # NOTICE: NOT a regex!
        # Drop font embellishment markdown from flag definition list lines.
        flags = True
        if paragraph:
          # Blank line was really a paragraph.
          paragraph = False
          lines.append('')
        if '  ' in line[7:]:
          head, tail = line[7:].split('  ', 1)
          head = re.sub('\\*', '', head)
          line = '  '.join(['     ', head, tail])
        else:
          line = re.sub('\\*', '', line)
      elif flags:
        if not line:
          paragraph = True
          continue
        elif not line.startswith('       '):
          flags = False
          paragraph = False
        elif paragraph:
          if not line[0].lower():
            # A real paragraph
            lines.append('+')
          paragraph = False
      lines.append(line)
    return '\n'.join(lines)


class ManPageCliTreeGenerator(CliTreeGenerator):
  """man page CLI tree generator."""

  @classmethod
  def _GetManPageCollectorType(cls):
    """Returns the man page collector type."""
    if files.FindExecutableOnPath('man'):
      return _ManCommandCollector
    return _ManUrlCollector

  def __init__(self, command):
    super(ManPageCliTreeGenerator, self).__init__(command)
    self.collector_type = self._GetManPageCollectorType()

  def GetVersion(self):
    """Returns the CLI_VERSION string."""
    if not self.collector_type:
      return cli_tree.CLI_VERSION_UNKNOWN
    return self.collector_type.GetVersion()

  def AddFlags(self, command, content, is_global=False):
    """Adds flags in content lines to command."""

    def _NameTypeValueNargs(name, type_=None, value=None, nargs=None):
      """Returns the (name, type, value-metavar, nargs) for flag name."""
      if name.startswith('--'):
        # --foo
        if '=' in name:
          name, value = name.split('=', 1)
          if name[-1] == '[':
            name = name[:-1]
            if value.endswith(']'):
              value = value[:-1]
            nargs = '?'
          else:
            nargs = '1'
          type_ = 'string'
      elif len(name) > 2:
        # -b VALUE
        value = name[2:]
        if value[0].isspace():
          value = value[1:]
        if value.startswith('['):
          value = value[1:]
          if value.endswith(']'):
            value = value[:-1]
          nargs = '?'
        else:
          nargs = '1'
        name = name[:2]
        type_ = 'string'
      if type_ is None or value is None or nargs is None:
        type_ = 'bool'
        value = ''
        nargs = '0'
      return (name, type_, value, nargs)

    def _Add(name, description, category, type_, value, nargs):
      """Adds a flag."""
      name, type_, value, nargs = _NameTypeValueNargs(name, type_, value, nargs)
      default = ''
      command[cli_tree.LOOKUP_FLAGS][name] = _Flag(
          name=name,
          description='\n'.join(description),
          type_=type_,
          value=value,
          nargs=nargs,
          category=category,
          default=default,
          is_required=False,
          is_global=is_global,
      )

    def _AddNames(names, description, category):
      """Add a flag name list."""
      if names:
        _, type_, value, nargs = _NameTypeValueNargs(names[-1])
        for name in names:
          _Add(name, description, category, type_, value, nargs)

    names = []
    description = []
    category = ''
    for line in content:
      if line.lstrip().startswith('-'):
        _AddNames(names, description, category)
        line = line.lstrip()
        names = line.strip().replace(', -', ', --').split(', -')
        if ' ' in names[-1]:
          names[-1], text = names[-1].split(' ', 1)
          description = [text.strip()]
        else:
          description = []
      elif line.startswith('### '):
        category = line[4:]
      else:
        description.append(line)
    _AddNames(names, description, category)

  def _Generate(self, path, collector):
    """Generates and returns the CLI subtree rooted at path."""
    command = _Command(path)

    while True:
      heading, content = collector.Collect()
      if not heading:
        break
      elif heading == 'NAME':
        if content:
          command[cli_tree.LOOKUP_CAPSULE] = re.sub(
              '.* -+  *', '', content[0]).strip()
      elif heading == 'FLAGS':
        self.AddFlags(command, content)
      elif heading not in ('BUGS', 'COLOPHON', 'COMPATIBILITY', 'HISTORY',
                           'STANDARDS', 'SYNOPSIS'):
        blocks = []
        begin = 0
        end = 0
        while end < len(content):
          if content[end].startswith('###'):
            if begin < end:
              blocks.append(_NormalizeSpace('\n'.join(content[begin:end])))
            blocks.append(content[end])
            begin = end + 1
          end += 1
        if begin < end:
          blocks.append(_NormalizeSpace('\n'.join(content[begin:end])))
        text = '\n'.join(blocks)
        if heading in command[cli_tree.LOOKUP_SECTIONS]:
          command[cli_tree.LOOKUP_SECTIONS][heading] += '\n\n' + text  # pytype: disable=attribute-error
        else:
          command[cli_tree.LOOKUP_SECTIONS][heading] = text  # pytype: disable=attribute-error
    return command

  def Generate(self):
    """Generates and returns the CLI tree rooted at self.command."""
    if not self.collector_type:
      return None
    collector = self.collector_type(self.command)
    if not collector:
      return None
    tree = self._Generate([self.command], collector)
    if not tree:
      return None

    # Add the version stamps.
    tree[cli_tree.LOOKUP_CLI_VERSION] = self.GetVersion()
    tree[cli_tree.LOOKUP_VERSION] = cli_tree.VERSION

    return tree


GENERATORS = {
    'bq': BqCliTreeGenerator,
    'gsutil': GsutilCliTreeGenerator,
    'kubectl': KubectlCliTreeGenerator,
}


def LoadOrGenerate(command, directories=None, force=False,
                   ignore_out_of_date=False, verbose=False,
                   warn_on_exceptions=False):
  """Returns the CLI tree for command, generating it if it does not exist.

  Args:
    command: The CLI root command name.
    directories: The list of directories containing the CLI tree JSON files.
      If None then the default installation directories are used.
    force: Update all exitsing trees by forcing them to be out of date if True.
    ignore_out_of_date: Ignore out of date trees instead of regenerating.
    verbose: Display a status line for up to date CLI trees if True.
    warn_on_exceptions: Emits warning messages in lieu of generator exceptions.
      Used during installation.

  Returns:
    The CLI tree for command or None if command not found or there is no
      generator.
  """
  command_dir, command_name = os.path.split(command)

  # Don't repeat failed attempts.
  if CliTreeGenerator.AlreadyFailed(command_name):
    if verbose:
      log.status.Print(u'No CLI tree generator for [{}].'.format(command))
    return None

  def _LoadOrGenerate():
    """Helper."""

    # The command must exist.
    if (command_dir and not os.path.exists(command) or
        not command_dir and not files.FindExecutableOnPath(command_name)):
      if verbose:
        log.status.Print(u'Command [{}] not found.'.format(command))
      return None

    # Instantiate the appropriate generator.
    try:
      generator = GENERATORS[command_name](command_name)
    except KeyError:
      generator = ManPageCliTreeGenerator(command_name)
    if not generator:
      return None

    # Load or (re)generate the CLI tree if possible.
    try:
      return generator.LoadOrGenerate(
          directories=directories,
          force=force,
          ignore_out_of_date=ignore_out_of_date,
          verbose=verbose,
          warn_on_exceptions=warn_on_exceptions)
    except NoManPageTextForCommand:
      pass

    return None

  tree = _LoadOrGenerate()
  if not tree:
    CliTreeGenerator.AddFailure(command_name)
  return tree


def _GetDirectories(directory=None, warn_on_exceptions=False):
  """Returns the list of directories to search for CLI trees.

  Args:
    directory: The directory containing the CLI tree JSON files. If None
      then the default installation and config directories are used.
    warn_on_exceptions: Emits warning messages in lieu of exceptions.
  """
  # Initialize the list of directories to search for CLI tree files. The default
  # CLI tree is only searched for and generated in directories[0]. Other
  # existing trees are updated in the directory in which they were found. New
  # trees are generated in directories[-1].
  directories = []
  if directory:
    directories.append(directory)
  else:
    try:
      directories.append(cli_tree.CliTreeDir())
    except cli_tree.SdkRootNotFoundError as e:
      if not warn_on_exceptions:
        raise
      log.warn(str(e))
    directories.append(cli_tree.CliTreeConfigDir())
  return directories


def UpdateCliTrees(cli=None, commands=None, directory=None,
                   force=False, verbose=False, warn_on_exceptions=False):
  """(re)generates the CLI trees in directory if non-existent or out ot date.

  This function uses the progress tracker because some of the updates can
  take ~minutes.

  Args:
    cli: The default CLI. If not None then the default CLI is also updated.
    commands: Update only the commands in this list.
    directory: The directory containing the CLI tree JSON files. If None
      then the default installation directories are used.
    force: Update all exitsing trees by forcing them to be out of date if True.
    verbose: Display a status line for up to date CLI trees if True.
    warn_on_exceptions: Emits warning messages in lieu of exceptions. Used
      during installation.

  Raises:
    NoCliTreeGeneratorForCommand: A command in commands is not supported
      (doesn't have a generator).
  """
  directories = _GetDirectories(
      directory=directory, warn_on_exceptions=warn_on_exceptions)
  if not commands:
    commands = set([cli_tree.DEFAULT_CLI_NAME] + GENERATORS.keys())

  failed = []
  for command in sorted(commands):
    if command != cli_tree.DEFAULT_CLI_NAME:
      tree = LoadOrGenerate(command,
                            directories=directories,
                            force=force,
                            verbose=verbose,
                            warn_on_exceptions=warn_on_exceptions)
      if not tree:
        failed.append(command)
    elif cli:
      cli_tree.Load(cli=cli,
                    path=cli_tree.CliTreePath(directory=directories[0]),
                    force=force, verbose=verbose)
  if failed:
    message = 'No CLI tree {} for [{}].'.format(
        text_utils.Pluralize(len(failed), 'generator'),
        ', '.join(sorted(failed)))
    if not warn_on_exceptions:
      raise NoCliTreeGeneratorForCommand(message)
    log.warn(message)


def LoadAll(directory=None, ignore_out_of_date=False, root=None,
            warn_on_exceptions=True):
  """Loads all CLI trees in directory and adds them to tree.

  Args:
    directory: The config directory containing the CLI tree modules.
    ignore_out_of_date: Ignore out of date trees instead of regenerating.
    root: dict, The CLI root to update. A new root is created if None.
    warn_on_exceptions: Warn on exceptions instead of raising if True.

  Raises:
    CliTreeVersionError: loaded tree version mismatch
    ImportModuleError: import errors

  Returns:
    The CLI tree.
  """
  # Create the root node if needed.
  if root is None:
    root = cli_tree.Node(description='The CLI tree root.')

  # Load the default CLI if available.
  if cli_tree.DEFAULT_CLI_NAME not in root[cli_tree.LOOKUP_COMMANDS]:
    try:
      root[cli_tree.LOOKUP_COMMANDS][cli_tree.DEFAULT_CLI_NAME] = (
          cli_tree.Load())
    except cli_tree.CliTreeLoadError:
      pass

  # Load extra CLIs by searching directories in order. .json files are treated
  # as CLI modules/data, where the file base name is the name of the CLI root
  # command.
  directories = _GetDirectories(
      directory=directory, warn_on_exceptions=warn_on_exceptions)

  loaded = {cli_tree.DEFAULT_CLI_NAME, '__init__'}  # Already loaded this above.
  for directory in directories:
    if not directory or not os.path.exists(directory):
      continue
    for (dirpath, _, filenames) in os.walk(directory):
      for filename in sorted(filenames):  # For stability across runs.
        command, extension = os.path.splitext(filename)
        if extension != '.json':
          continue
        if command in loaded:
          # Already loaded. Earlier directory hits take precedence.
          continue
        loaded.add(command)
        if command == cli_tree.DEFAULT_CLI_NAME:
          tree = cli_tree.Load(os.path.join(dirpath, filename))
        else:
          tree = LoadOrGenerate(command,
                                directories=[dirpath],
                                ignore_out_of_date=ignore_out_of_date,
                                warn_on_exceptions=warn_on_exceptions)
        if tree:
          root[cli_tree.LOOKUP_COMMANDS][command] = tree
      # Don't search subdirectories.
      break

  return root

# Copyright 2017 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""gcloud CLI tree generators for non-gcloud CLIs.

A CLI tree for a supported command is generated by using the root command plus
`help` or `--help` arguments to do a DFS traversal. Each node is generated
from a man-ish style runtime document.

Supported CLI commands have their own runtime help document quirks, so each is
handled by an ad-hoc parser. The parsers rely on consistency within commands
and between command releases.

The CLI tree for an unsupported command is generated from the output of
`man the-command` and contains only the command root node.
"""

import abc
import json
import os
import re
import subprocess
import sys
import textwrap

from googlecloudsdk.calliope import cli_tree
from googlecloudsdk.core import exceptions
from googlecloudsdk.core import log
from googlecloudsdk.core.console import progress_tracker
from googlecloudsdk.core.resource import resource_printer
from googlecloudsdk.core.util import encoding
from googlecloudsdk.core.util import files
from googlecloudsdk.core.util import text as text_utils

import httplib2
from typing import Any, Dict  # pylint: disable=unused-import


class NoCliTreeGeneratorForCommand(exceptions.Error):
  """Command does not have a CLI tree generator."""


class NoManPageTextForCommand(exceptions.Error):
  """Could not get man page text for command."""


# TODO(b/69033748): disable until issues resolved
def _DisableLongRunningCliTreeGeneration(command):
  """Returns True if long running CLI tree generation is disabled."""
  # Only a few command generators are long running.
  if command not in ('bq', 'gsutil', 'kubectl'):
    return False
  # Only generate these CLI trees on the fly for explicit requests.
  # It can take ~1minute, not good, especially at the interactive prompt.
  if 'update-cli-trees' in sys.argv or '--update-cli-trees' in sys.argv:
    return False
  # It's a long running generator, not explicitly requested -- disable.
  return True


def _NormalizeSpace(text):
  """Returns text dedented and multiple non-indent spaces replaced by one."""
  return re.sub('([^ ])   *', r'\1 ', textwrap.dedent(text)).strip('\n')


def _Flag(name, description='', value=None, default=None, type_='string',
          category='', is_global=False, is_required=False, nargs=None):
  """Initializes and returns a flag dict node."""
  return {
      cli_tree.LOOKUP_ATTR: {},
      cli_tree.LOOKUP_CATEGORY: category,
      cli_tree.LOOKUP_DEFAULT: default,
      cli_tree.LOOKUP_DESCRIPTION: _NormalizeSpace(description),
      cli_tree.LOOKUP_GROUP: '',
      cli_tree.LOOKUP_IS_GLOBAL: is_global,
      cli_tree.LOOKUP_IS_HIDDEN: False,
      cli_tree.LOOKUP_IS_REQUIRED: is_required,
      cli_tree.LOOKUP_NAME: name,
      cli_tree.LOOKUP_NARGS: nargs or ('0' if type_ == 'bool' else '1'),
      cli_tree.LOOKUP_VALUE: value,
      cli_tree.LOOKUP_TYPE: type_,
  }


def _Positional(name, description='', default=None, nargs='0'):
  """Initializes and returns a positional dict node."""
  return {
      cli_tree.LOOKUP_DEFAULT: default,
      cli_tree.LOOKUP_DESCRIPTION: _NormalizeSpace(description),
      cli_tree.LOOKUP_NAME: name,
      cli_tree.LOOKUP_NARGS: nargs,
  }


def _Command(path):
  # type: (str) -> Dict[str, Any]
  """Initializes and returns a command/group dict node."""
  return {
      cli_tree.LOOKUP_CAPSULE: '',
      cli_tree.LOOKUP_COMMANDS: {},
      cli_tree.LOOKUP_FLAGS: {},
      cli_tree.LOOKUP_GROUPS: {},
      cli_tree.LOOKUP_IS_GROUP: False,
      cli_tree.LOOKUP_IS_HIDDEN: False,
      cli_tree.LOOKUP_PATH: path,
      cli_tree.LOOKUP_POSITIONALS: [],
      cli_tree.LOOKUP_RELEASE: 'GA',
      cli_tree.LOOKUP_SECTIONS: {},
  }


class CliTreeGenerator(object):
  """Base CLI tree generator."""

  __metaclass__ = abc.ABCMeta

  _FAILURES = None

  @classmethod
  def MemoizeFailures(cls, enable):
    """Memoizes failed attempts and doesn't repeat them if enable is True."""
    cls._FAILURES = set() if enable else None

  @classmethod
  def AlreadyFailed(cls, command):
    """Returns True if man page request for command already failed."""
    return command in cls._FAILURES if cls._FAILURES else False

  @classmethod
  def AddFailure(cls, command):
    """Add command to the set of failed man generations."""
    if cls._FAILURES is not None:
      cls._FAILURES.add(command)

  def __init__(self, command):
    self.command = command
    self._cli_version = None  # For memoizing GetVersion()

  def Run(self, cmd):
    """Runs cmd and returns the output as a string."""
    return subprocess.check_output(cmd)

  def GetVersion(self):
    """Returns the CLI_VERSION string."""
    if not self._cli_version:
      self._cli_version = self.Run([self.command, 'version']).split()[-1]
    return self._cli_version

  @abc.abstractmethod
  def Generate(self):
    """Generates and returns the CLI tree dict."""
    return None

  def FindTreeFile(self, directories):
    """Returns (path,f) open for read for the first CLI tree in directories."""
    for directory in directories or [cli_tree.CliTreeConfigDir()]:
      path = os.path.join(directory or '.', self.command) + '.json'
      try:
        return path, open(path, 'r')
      except IOError:
        pass
    return path, None  # pytype: disable=name-error

  def IsUpToDate(self, tree, verbose=False):
    """Returns a bool tuple (readonly, up_to_date)."""

    actual_cli_version = tree.get(cli_tree.LOOKUP_CLI_VERSION)
    readonly = actual_cli_version == cli_tree.CLI_VERSION_READONLY

    # Check the schema version.
    actual_tree_version = tree.get(cli_tree.LOOKUP_VERSION)
    if actual_tree_version != cli_tree.VERSION:
      return readonly, False

    # Check the CLI version.
    expected_cli_version = self.GetVersion()
    if readonly:
      # READONLY trees are always up to date.
      pass
    elif expected_cli_version == cli_tree.CLI_VERSION_UNKNOWN:
      # Don't know how to regenerate but we have one in hand -- accept it.
      pass
    elif actual_cli_version != expected_cli_version:
      return readonly, False

    # Schema and CLI versions are up to date.
    if verbose:
      log.status.Print(u'[{}] CLI tree version [{}] is up to date.'.format(
          self.command, actual_cli_version))
    return readonly, True

  def LoadOrGenerate(self, directories=None, force=False,
                     ignore_out_of_date=False, verbose=False,
                     warn_on_exceptions=False):
    """Loads the CLI tree or generates it if necessary, and returns the tree."""
    path, f = self.FindTreeFile(directories)
    if not f:
      # TODO(b/69033748): disable until issues resolved
      if _DisableLongRunningCliTreeGeneration(self.command):
        return None
    else:
      up_to_date = False
      with f:
        try:
          tree = json.load(f)
        except ValueError:
          # Corrupt JSON -- could have been interrupted.
          tree = None
        if tree:
          readonly, up_to_date = self.IsUpToDate(tree, verbose=verbose)
      if readonly:
        return tree
      elif up_to_date:
        if not force:
          return tree
      elif ignore_out_of_date:
        return None

    def _Generate():
      """Helper that generates a CLI tree and writes it to a JSON file."""
      tree = self.Generate()
      if tree:
        try:
          f = open(path, 'w')
        except EnvironmentError as e:
          # CLI data config dir may not be initialized yet.
          directory, _ = os.path.split(path)
          try:
            files.MakeDir(directory)
            f = open(path, 'w')
          except (EnvironmentError, files.Error):
            if not warn_on_exceptions:
              raise
            log.warn(str(e))
            return None
        with f:
          resource_printer.Print(tree, print_format='json', out=f)
      return tree

    # At this point:
    #   (1) the tree is not found or is out of date
    #   (2) the tree is not readonly
    #   (3) we have a generator for the tree

    if not verbose:
      return _Generate()
    with progress_tracker.ProgressTracker(
        u'{} the [{}] CLI tree'.format(
            'Updating' if f else 'Generating', self.command)):
      return _Generate()


class _BqCollector(object):
  """bq help document section collector."""

  def __init__(self, text):
    self.text = text.split('\n')
    self.heading = 'DESCRIPTION'
    self.lookahead = None
    self.ignore_trailer = False

  def Collect(self, strip_headings=False):
    """Returns the heading and content lines from text."""
    content = []
    if self.lookahead:
      if not strip_headings:
        content.append(self.lookahead)
      self.lookahead = None
    heading = self.heading
    self.heading = None
    while self.text:
      line = self.text.pop(0)
      if line.startswith(' ') or not strip_headings and not self.ignore_trailer:
        content.append(line.rstrip())
    while content and not content[0]:
      content.pop(0)
    while content and not content[-1]:
      content.pop()
    self.ignore_trailer = True
    return heading, content


class BqCliTreeGenerator(CliTreeGenerator):
  """bq CLI tree generator."""

  def Run(self, cmd):
    """Runs cmd and returns the output as a string."""
    try:
      output = subprocess.check_output(cmd)
    except subprocess.CalledProcessError as e:
      # bq exit code is 1 for help and --help. How do you know if help failed?
      if e.returncode != 1:
        raise
      output = e.output
    return output.replace('bq.py', 'bq')

  def AddFlags(self, command, content, is_global=False):
    """Adds flags in content lines to command."""
    while content:
      line = content.pop(0)
      name, description = line.strip().split(':', 1)
      paragraph = [description.strip()]
      default = ''
      while content and not content[0].startswith('  --'):
        line = content.pop(0).strip()
        if line.startswith('(default: '):
          default = line[10:-1]
        else:
          paragraph.append(line)
      description = ' '.join(paragraph).strip()
      if name.startswith('--[no]'):
        name = '--' + name[6:]
        type_ = 'bool'
        value = ''
      else:
        value = 'VALUE'
        type_ = 'string'
      command[cli_tree.LOOKUP_FLAGS][name] = _Flag(
          name=name,
          description=description,
          type_=type_,
          value=value,
          default=default,
          is_required=False,
          is_global=is_global,
      )

  def SubTree(self, path):
    """Generates and returns the CLI subtree rooted at path."""
    command = _Command(path)
    command[cli_tree.LOOKUP_IS_GROUP] = True
    text = self.Run([path[0], 'help'] + path[1:])

    # `bq help` lists help for all commands. Command flags are "defined"
    # by example. We don't attempt to suss that out.
    content = text.split('\n')
    while content:
      line = content.pop(0)
      if not line or not line[0].islower():
        continue
      name, text = line.split(' ', 1)
      description = [text.strip()]
      examples = []
      arguments = []
      paragraph = description
      while content and (not content[0] or not content[0][0].islower()):
        line = content.pop(0).strip()
        if line == 'Arguments:':
          paragraph = arguments
        elif line == 'Examples:':
          paragraph = examples
        else:
          paragraph.append(line)
      subcommand = _Command(path + [name])
      command[cli_tree.LOOKUP_COMMANDS][name] = subcommand  # pytype: disable=attribute-error
      if description:
        subcommand[cli_tree.LOOKUP_SECTIONS]['DESCRIPTION'] = '\n'.join(
            description)  # pytype: disable=attribute-error
      if examples:
        subcommand[cli_tree.LOOKUP_SECTIONS]['EXAMPLES'] = '\n'.join(
            examples)  # pytype: disable=attribute-error

    return command

  def Generate(self):
    """Generates and returns the CLI tree rooted at self.command."""

    # Construct the tree minus the global flags.
    tree = self.SubTree([self.command])

    # Add the global flags to the root.
    text = self.Run([self.command, '--help'])
    collector = _BqCollector(text)
    _, content = collector.Collect(strip_headings=True)
    self.AddFlags(tree, content, is_global=True)

    # Finally add the version stamps.
    tree[cli_tree.LOOKUP_CLI_VERSION] = self.GetVersion()
    tree[cli_tree.LOOKUP_VERSION] = cli_tree.VERSION

    return tree


class _GsutilCollector(object):
  """gsutil help document section collector."""

  UNKNOWN, ROOT, MAN, TOPIC = range(4)

  def __init__(self, text):
    self.text = text.split('\n')
    self.heading = 'CAPSULE'
    self.page_type = self.UNKNOWN

  def Collect(self, strip_headings=False):
    """Returns the heading and content lines from text."""
    content = []
    heading = self.heading
    self.heading = None
    while self.text:
      line = self.text.pop(0)
      if self.page_type == self.UNKNOWN:
        # The first heading distinguishes the document page type.
        if line.startswith('Usage:'):
          self.page_type = self.ROOT
          continue
        elif line == 'NAME':
          self.page_type = self.MAN
          heading = 'CAPSULE'
          continue
        elif not line.startswith(' '):
          continue
      elif self.page_type == self.ROOT:
        # The root help page.
        if line == 'Available commands:':
          heading = 'COMMANDS'
          continue
        elif line == 'Additional help topics:':
          self.heading = 'TOPICS'
          break
        elif not line.startswith(' '):
          continue
      elif self.page_type == self.MAN:
        # A command/subcommand man style page.
        if line == 'OVERVIEW':
          self.page_type = self.TOPIC
          self.heading = 'DESCRIPTION'
          break
        elif line == 'SYNOPSIS':
          self.heading = line
          break
        elif line.endswith('OPTIONS'):
          self.heading = 'FLAGS'
          break
        elif line and line[0].isupper():
          self.heading = line.split(' ', 1)[-1]
          break
      elif self.page_type == self.TOPIC:
        # A topic man style page.
        if line and line[0].isupper():
          self.heading = line
          break
      if line.startswith(' ') or not strip_headings:
        content.append(line.rstrip())
    while content and not content[0]:
      content.pop(0)
    while content and not content[-1]:
      content.pop()
    return heading, content


class GsutilCliTreeGenerator(CliTreeGenerator):
  """gsutil CLI tree generator."""

  def __init__(self, *args, **kwargs):
    super(GsutilCliTreeGenerator, self).__init__(*args, **kwargs)
    self.topics = []

  def Run(self, cmd):
    """Runs the command in cmd and returns the output as a string."""
    try:
      output = subprocess.check_output(cmd)
    except subprocess.CalledProcessError as e:
      # gsutil exit code is 1 for --help depending on the context.
      if e.returncode != 1:
        raise
      output = e.output
    return output

  def AddFlags(self, command, content, is_global=False):
    """Adds flags in content lines to command."""

    def _Add(name, description):
      value = ''
      type_ = 'bool'
      default = ''
      command[cli_tree.LOOKUP_FLAGS][name] = _Flag(
          name=name,
          description=description,
          type_=type_,
          value=value,
          default=default,
          is_required=False,
          is_global=is_global,
      )

    parse = re.compile(' *((-[^ ]*,)* *(-[^ ]*) *)(.*)')
    name = None
    description = []
    for line in content:
      if line.startswith('  -'):
        if name:
          _Add(name, '\n'.join(description))
        match = parse.match(line)
        name = match.group(3)
        description = [match.group(4).rstrip()]
      elif len(line) > 16:
        description.append(line[16:].rstrip())
    if name:
      _Add(name, '\n'.join(description))

  def SubTree(self, path):
    """Generates and returns the CLI subtree rooted at path."""
    command = _Command(path)
    is_help_command = len(path) > 1 and path[1] == 'help'
    if is_help_command:
      cmd = path
    else:
      cmd = path + ['--help']
    text = self.Run(cmd)
    collector = _GsutilCollector(text)

    while True:
      heading, content = collector.Collect()
      if not heading:
        break
      elif heading == 'CAPSULE':
        if content:
          command[cli_tree.LOOKUP_CAPSULE] = content[0].split('-', 1)[1].strip()
      elif heading == 'COMMANDS':
        if is_help_command:
          continue
        for line in content:
          try:
            name = line.split()[0]
          except IndexError:
            continue
          if name == 'update':
            continue
          command[cli_tree.LOOKUP_IS_GROUP] = True
          command[cli_tree.LOOKUP_COMMANDS][name] = self.SubTree(path + [name])
      elif heading == 'FLAGS':
        self.AddFlags(command, content)
      elif heading == 'SYNOPSIS':
        commands = []
        for line in content:
          if not line:
            break
          cmd = line.split()
          if len(cmd) <= len(path):
            continue
          if cmd[:len(path)] == path:
            name = cmd[len(path)]
            if name[0].islower() and name not in ('off', 'on', 'false', 'true'):
              commands.append(name)
        if len(commands) > 1:
          command[cli_tree.LOOKUP_IS_GROUP] = True
          for name in commands:
            command[cli_tree.LOOKUP_COMMANDS][name] = self.SubTree(
                path + [name])
      elif heading == 'TOPICS':
        for line in content:
          try:
            self.topics.append(line.split()[0])
          except IndexError:
            continue
      elif heading.isupper():
        if heading.lower() == path[-1]:
          heading = 'DESCRIPTION'
        command[cli_tree.LOOKUP_SECTIONS][heading] = '\n'.join(
            [line[2:] for line in content])

    return command

  def Generate(self):
    """Generates and returns the CLI tree rooted at self.command."""
    tree = self.SubTree([self.command])

    # Add the global flags to the root.
    text = self.Run([self.command, 'help', 'options'])
    collector = _GsutilCollector(text)
    while True:
      heading, content = collector.Collect()
      if not heading:
        break
      if heading == 'FLAGS':
        self.AddFlags(tree, content, is_global=True)

    # Add the help topics.
    help_command = tree[cli_tree.LOOKUP_COMMANDS]['help']
    help_command[cli_tree.LOOKUP_IS_GROUP] = True
    for topic in self.topics:
      help_command[cli_tree.LOOKUP_COMMANDS][topic] = self.SubTree(
          help_command[cli_tree.LOOKUP_PATH] + [topic])

    # Finally add the version stamps.
    tree[cli_tree.LOOKUP_CLI_VERSION] = self.GetVersion()
    tree[cli_tree.LOOKUP_VERSION] = cli_tree.VERSION

    return tree


class _KubectlCollector(object):
  """Kubectl help document section collector."""

  def __init__(self, text):
    self.text = text.split('\n')
    self.heading = 'DESCRIPTION'
    self.lookahead = None
    self.ignore_trailer = False

  def Collect(self, strip_headings=False):
    """Returns the heading and content lines from text."""
    content = []
    if self.lookahead:
      if not strip_headings:
        content.append(self.lookahead)
      self.lookahead = None
    heading = self.heading
    self.heading = None
    while self.text:
      line = self.text.pop(0)
      usage = 'Usage:'
      if line.startswith(usage):
        line = line[len(usage):].strip()
        if line:
          self.lookahead = line
        self.heading = 'USAGE'
        break
      if line.endswith(':'):
        if 'Commands' in line:
          self.heading = 'COMMANDS'
          break
        if 'Examples' in line:
          self.heading = 'EXAMPLES'
          break
        if 'Options' in line:
          self.heading = 'FLAGS'
          break
      if line.startswith(' ') or not strip_headings and not self.ignore_trailer:
        content.append(line.rstrip())
    while content and not content[0]:
      content.pop(0)
    while content and not content[-1]:
      content.pop()
    self.ignore_trailer = True
    return heading, content


class KubectlCliTreeGenerator(CliTreeGenerator):
  """kubectl CLI tree generator."""

  def AddFlags(self, command, content, is_global=False):
    """Adds flags in content lines to command."""
    for line in content:
      flags, description = line.strip().split(':', 1)
      flag = flags.split(', ')[-1]
      name, value = flag.split('=')
      if value in ('true', 'false'):
        value = ''
        type_ = 'bool'
      else:
        value = 'VALUE'
        type_ = 'string'
      default = ''
      command[cli_tree.LOOKUP_FLAGS][name] = _Flag(
          name=name,
          description=description,
          type_=type_,
          value=value,
          default=default,
          is_required=False,
          is_global=is_global,
      )

  def SubTree(self, path):
    """Generates and returns the CLI subtree rooted at path."""
    command = _Command(path)
    text = self.Run(path + ['--help'])
    collector = _KubectlCollector(text)

    while True:
      heading, content = collector.Collect()
      if not heading:
        break
      elif heading == 'COMMANDS':
        for line in content:
          try:
            name = line.split()[0]
          except IndexError:
            continue
          command[cli_tree.LOOKUP_IS_GROUP] = True
          command[cli_tree.LOOKUP_COMMANDS][name] = self.SubTree(path + [name])
      elif heading in ('DESCRIPTION', 'EXAMPLES'):
        command[cli_tree.LOOKUP_SECTIONS][heading] = '\n'.join(content)  # pytype: disable=attribute-error
      elif heading == 'FLAGS':
        self.AddFlags(command, content)
    return command

  def GetVersion(self):
    """Returns the CLI_VERSION string."""
    if not self._cli_version:
      verbose_version = self.Run([self.command, 'version', '--client'])
      match = re.search('GitVersion:"([^"]*)"', verbose_version)
      self._cli_version = match.group(1)
    return self._cli_version

  def Generate(self):
    """Generates and returns the CLI tree rooted at self.command."""

    # Construct the tree minus the global flags.
    tree = self.SubTree([self.command])

    # Add the global flags to the root.
    text = self.Run([self.command, 'options'])
    collector = _KubectlCollector(text)
    _, content = collector.Collect(strip_headings=True)
    content.append('  --help=true: List detailed command help.')
    self.AddFlags(tree, content, is_global=True)

    # Finally add the version stamps.
    tree[cli_tree.LOOKUP_CLI_VERSION] = self.GetVersion()
    tree[cli_tree.LOOKUP_VERSION] = cli_tree.VERSION

    return tree


class _ManPageCollector(object):
  """man page help document section collector base class.

  Attributes:
    command: The man page command name.
    content_indent: A string of space characters representing the indent of
      the first line of content for any section.
    heading: The heading for the next call to Collect().
    text: The list of man page lines.
    version: The collector CLI_VERSION string.
  """

  __metaclass__ = abc.ABCMeta

  def __init__(self, command):
    self.command = command
    self.content_indent = None
    self.heading = None
    self.text = self.GetManPageText().split('\n')  # pytype: disable=none-attr

  @classmethod
  @abc.abstractmethod
  def GetVersion(cls):
    """Returns the CLI_VERSION string."""
    return None

  @abc.abstractmethod
  def _GetRawManPageText(self):
    """Returns the raw man page text."""
    return None

  @abc.abstractmethod
  def GetManPageText(self):
    """Returns the preprocessed man page text."""
    return None

  def Collect(self):
    """Returns the heading and content lines from text."""
    content = []
    heading = self.heading
    self.heading = None
    while self.text:
      line = self.text.pop(0)
      if not heading:
        # No NAME no man page.
        if line == 'NAME':
          heading = line
        continue
      elif not line:
        pass
      elif line[0] == ' ':
        if not self.content_indent:
          self.content_indent = re.sub('[^ ].*', '', line)
        if len(line) > len(self.content_indent):
          indented_char = line[len(self.content_indent)]
          if not line.startswith(self.content_indent):
            # Subsection heading or category.
            line = '### ' + line.strip()
          elif heading == 'DESCRIPTION' and indented_char == '-':
            # Some man pages, like GNU ls(1), inline flags in DESCRIPTION.
            self.text.insert(0, line)
            self.heading = 'FLAGS'
            break
          elif heading == 'FLAGS' and indented_char not in (' ', '-'):
            self.text.insert(0, line)
            self.heading = 'DESCRIPTION'
            break
      elif line in ('SYNOPSIS', 'DESCRIPTION', 'EXIT STATUS', 'SEE ALSO'):
        self.heading = line
        break
      elif 'FLAGS' in line or 'OPTIONS' in line:
        self.heading = 'FLAGS'
        break
      elif line and line[0].isupper():
        self.heading = line.split(' ', 1)[-1]
        break
      content.append(line.rstrip())
    while content and not content[0]:
      content.pop(0)
    while content and not content[-1]:
      content.pop()
    return heading, content


class _ManCommandCollector(_ManPageCollector):
  """man command help document section collector."""

  _CLI_VERSION = 'man-0.1'

  @classmethod
  def GetVersion(cls):
    return cls._CLI_VERSION

  def _GetRawManPageText(self):
    """Returns the raw man page text."""
    try:
      with open(os.devnull, 'w') as f:
        return subprocess.check_output(['man', self.command], stderr=f)
    except (OSError, subprocess.CalledProcessError):
      raise NoManPageTextForCommand(
          'Cannot get man(1) command man page text for [{}].'.format(
              self.command))

  def GetManPageText(self):
    """Returns the preprocessed man page text."""
    text = self._GetRawManPageText()
    return re.sub(
        '.\b', '', re.sub(
            u'(\u2010|\\u2010)\n *', '', encoding.Decode(text)))


class _ManUrlCollector(_ManPageCollector):
  """man URL help document section collector."""

  _CLI_VERSION = 'man7.org-0.1'

  @classmethod
  def GetVersion(cls):
    return cls._CLI_VERSION

  def _GetRawManPageText(self):
    """Returns the raw man page text."""
    url = 'http://man7.org/linux/man-pages/man1/{}.1.html'.format(self.command)
    response, content = httplib2.Http().request(url)
    if response.status != 200:
      raise NoManPageTextForCommand(
          'Cannot get URL man page text for [{}].'.format(self.command))
    return content.decode('utf-8')

  def GetManPageText(self):
    """Returns the text man page for self.command from a URL."""
    text = self._GetRawManPageText()

    # A little sed(1) to clean up the html header/trailer and anchors.
    # A mispplaced .* or (...) group match could affect timing. Keep track
    # of that if you change any of the pattern,replacement tuples.
    for pattern, replacement in (
        ('<span class="footline">.*', ''),
        ('<h2><a id="([^"]*)"[^\n]*\n', '\\1\n'),
        ('<b>( +)', '\\1*'),
        ('( +)</b>', '*\\1'),
        ('<i>( +)', '\\1_'),
        ('( +)</i>', '_\\1'),
        ('</?b>', '*'),
        ('</?i>', '_'),
        ('</pre>', ''),
        ('<a href="([^"]*)">([^\n]*)</a>', '[\\1](\\2)'),
        ('&amp;', '\\&'),
        ('&gt;', '>'),
        ('&lt;', '<'),
        ('&#39;', "'"),
    ):
      text = re.sub(pattern, replacement, text, flags=re.DOTALL)

    # ... and some non-regular edits to finish up.
    lines = []
    top = 'NAME'
    flags = False
    paragraph = False
    for line in text.split('\n'):
      if top and line == 'NAME':
        # Ignore all lines before the NAME section.
        top = None
        lines = []
      if line.startswith('       *-'):  # NOTICE: NOT a regex!
        # Drop font embellishment markdown from flag definition list lines.
        flags = True
        if paragraph:
          # Blank line was really a paragraph.
          paragraph = False
          lines.append('')
        if '  ' in line[7:]:
          head, tail = line[7:].split('  ', 1)
          head = re.sub('\\*', '', head)
          line = '  '.join(['     ', head, tail])
        else:
          line = re.sub('\\*', '', line)
      elif flags:
        if not line:
          paragraph = True
          continue
        elif not line.startswith('       '):
          flags = False
          paragraph = False
        elif paragraph:
          if not line[0].lower():
            # A real paragraph
            lines.append('+')
          paragraph = False
      lines.append(line)
    return '\n'.join(lines)


class ManPageCliTreeGenerator(CliTreeGenerator):
  """man page CLI tree generator."""

  @classmethod
  def _GetManPageCollectorType(cls):
    """Returns the man page collector type."""
    if files.FindExecutableOnPath('man'):
      return _ManCommandCollector
    return _ManUrlCollector

  def __init__(self, command):
    super(ManPageCliTreeGenerator, self).__init__(command)
    self.collector_type = self._GetManPageCollectorType()

  def GetVersion(self):
    """Returns the CLI_VERSION string."""
    if not self.collector_type:
      return cli_tree.CLI_VERSION_UNKNOWN
    return self.collector_type.GetVersion()

  def AddFlags(self, command, content, is_global=False):
    """Adds flags in content lines to command."""

    def _NameTypeValueNargs(name, type_=None, value=None, nargs=None):
      """Returns the (name, type, value-metavar, nargs) for flag name."""
      if name.startswith('--'):
        # --foo
        if '=' in name:
          name, value = name.split('=', 1)
          if name[-1] == '[':
            name = name[:-1]
            if value.endswith(']'):
              value = value[:-1]
            nargs = '?'
          else:
            nargs = '1'
          type_ = 'string'
      elif len(name) > 2:
        # -b VALUE
        value = name[2:]
        if value[0].isspace():
          value = value[1:]
        if value.startswith('['):
          value = value[1:]
          if value.endswith(']'):
            value = value[:-1]
          nargs = '?'
        else:
          nargs = '1'
        name = name[:2]
        type_ = 'string'
      if type_ is None or value is None or nargs is None:
        type_ = 'bool'
        value = ''
        nargs = '0'
      return (name, type_, value, nargs)

    def _Add(name, description, category, type_, value, nargs):
      """Adds a flag."""
      name, type_, value, nargs = _NameTypeValueNargs(name, type_, value, nargs)
      default = ''
      command[cli_tree.LOOKUP_FLAGS][name] = _Flag(
          name=name,
          description='\n'.join(description),
          type_=type_,
          value=value,
          nargs=nargs,
          category=category,
          default=default,
          is_required=False,
          is_global=is_global,
      )

    def _AddNames(names, description, category):
      """Add a flag name list."""
      if names:
        _, type_, value, nargs = _NameTypeValueNargs(names[-1])
        for name in names:
          _Add(name, description, category, type_, value, nargs)

    names = []
    description = []
    category = ''
    for line in content:
      if line.lstrip().startswith('-'):
        _AddNames(names, description, category)
        line = line.lstrip()
        names = line.strip().replace(', -', ', --').split(', -')
        if ' ' in names[-1]:
          names[-1], text = names[-1].split(' ', 1)
          description = [text.strip()]
        else:
          description = []
      elif line.startswith('### '):
        category = line[4:]
      else:
        description.append(line)
    _AddNames(names, description, category)

  def _Generate(self, path, collector):
    """Generates and returns the CLI subtree rooted at path."""
    command = _Command(path)

    while True:
      heading, content = collector.Collect()
      if not heading:
        break
      elif heading == 'NAME':
        if content:
          command[cli_tree.LOOKUP_CAPSULE] = re.sub(
              '.* -+  *', '', content[0]).strip()
      elif heading == 'FLAGS':
        self.AddFlags(command, content)
      elif heading not in ('BUGS', 'COLOPHON', 'COMPATIBILITY', 'HISTORY',
                           'STANDARDS', 'SYNOPSIS'):
        blocks = []
        begin = 0
        end = 0
        while end < len(content):
          if content[end].startswith('###'):
            if begin < end:
              blocks.append(_NormalizeSpace('\n'.join(content[begin:end])))
            blocks.append(content[end])
            begin = end + 1
          end += 1
        if begin < end:
          blocks.append(_NormalizeSpace('\n'.join(content[begin:end])))
        text = '\n'.join(blocks)
        if heading in command[cli_tree.LOOKUP_SECTIONS]:
          command[cli_tree.LOOKUP_SECTIONS][heading] += '\n\n' + text  # pytype: disable=attribute-error
        else:
          command[cli_tree.LOOKUP_SECTIONS][heading] = text  # pytype: disable=attribute-error
    return command

  def Generate(self):
    """Generates and returns the CLI tree rooted at self.command."""
    if not self.collector_type:
      return None
    collector = self.collector_type(self.command)
    if not collector:
      return None
    tree = self._Generate([self.command], collector)
    if not tree:
      return None

    # Add the version stamps.
    tree[cli_tree.LOOKUP_CLI_VERSION] = self.GetVersion()
    tree[cli_tree.LOOKUP_VERSION] = cli_tree.VERSION

    return tree


GENERATORS = {
    'bq': BqCliTreeGenerator,
    'gsutil': GsutilCliTreeGenerator,
    'kubectl': KubectlCliTreeGenerator,
}


def LoadOrGenerate(command, directories=None, force=False,
                   ignore_out_of_date=False, verbose=False,
                   warn_on_exceptions=False):
  """Returns the CLI tree for command, generating it if it does not exist.

  Args:
    command: The CLI root command name.
    directories: The list of directories containing the CLI tree JSON files.
      If None then the default installation directories are used.
    force: Update all exitsing trees by forcing them to be out of date if True.
    ignore_out_of_date: Ignore out of date trees instead of regenerating.
    verbose: Display a status line for up to date CLI trees if True.
    warn_on_exceptions: Emits warning messages in lieu of generator exceptions.
      Used during installation.

  Returns:
    The CLI tree for command or None if command not found or there is no
      generator.
  """
  command_dir, command_name = os.path.split(command)

  # Don't repeat failed attempts.
  if CliTreeGenerator.AlreadyFailed(command_name):
    if verbose:
      log.status.Print(u'No CLI tree generator for [{}].'.format(command))
    return None

  def _LoadOrGenerate():
    """Helper."""

    # The command must exist.
    if (command_dir and not os.path.exists(command) or
        not command_dir and not files.FindExecutableOnPath(command_name)):
      if verbose:
        log.status.Print(u'Command [{}] not found.'.format(command))
      return None

    # Instantiate the appropriate generator.
    try:
      generator = GENERATORS[command_name](command_name)
    except KeyError:
      generator = ManPageCliTreeGenerator(command_name)
    if not generator:
      return None

    # Load or (re)generate the CLI tree if possible.
    try:
      return generator.LoadOrGenerate(
          directories=directories,
          force=force,
          ignore_out_of_date=ignore_out_of_date,
          verbose=verbose,
          warn_on_exceptions=warn_on_exceptions)
    except NoManPageTextForCommand:
      pass

    return None

  tree = _LoadOrGenerate()
  if not tree:
    CliTreeGenerator.AddFailure(command_name)
  return tree


def _GetDirectories(directory=None, warn_on_exceptions=False):
  """Returns the list of directories to search for CLI trees.

  Args:
    directory: The directory containing the CLI tree JSON files. If None
      then the default installation and config directories are used.
    warn_on_exceptions: Emits warning messages in lieu of exceptions.
  """
  # Initialize the list of directories to search for CLI tree files. The default
  # CLI tree is only searched for and generated in directories[0]. Other
  # existing trees are updated in the directory in which they were found. New
  # trees are generated in directories[-1].
  directories = []
  if directory:
    directories.append(directory)
  else:
    try:
      directories.append(cli_tree.CliTreeDir())
    except cli_tree.SdkRootNotFoundError as e:
      if not warn_on_exceptions:
        raise
      log.warn(str(e))
    directories.append(cli_tree.CliTreeConfigDir())
  return directories


def UpdateCliTrees(cli=None, commands=None, directory=None,
                   force=False, verbose=False, warn_on_exceptions=False):
  """(re)generates the CLI trees in directory if non-existent or out ot date.

  This function uses the progress tracker because some of the updates can
  take ~minutes.

  Args:
    cli: The default CLI. If not None then the default CLI is also updated.
    commands: Update only the commands in this list.
    directory: The directory containing the CLI tree JSON files. If None
      then the default installation directories are used.
    force: Update all exitsing trees by forcing them to be out of date if True.
    verbose: Display a status line for up to date CLI trees if True.
    warn_on_exceptions: Emits warning messages in lieu of exceptions. Used
      during installation.

  Raises:
    NoCliTreeGeneratorForCommand: A command in commands is not supported
      (doesn't have a generator).
  """
  directories = _GetDirectories(
      directory=directory, warn_on_exceptions=warn_on_exceptions)
  if not commands:
    commands = set([cli_tree.DEFAULT_CLI_NAME] + GENERATORS.keys())

  failed = []
  for command in sorted(commands):
    if command != cli_tree.DEFAULT_CLI_NAME:
      tree = LoadOrGenerate(command,
                            directories=directories,
                            force=force,
                            verbose=verbose,
                            warn_on_exceptions=warn_on_exceptions)
      if not tree:
        failed.append(command)
    elif cli:
      cli_tree.Load(cli=cli,
                    path=cli_tree.CliTreePath(directory=directories[0]),
                    force=force, verbose=verbose)
  if failed:
    message = 'No CLI tree {} for [{}].'.format(
        text_utils.Pluralize(len(failed), 'generator'),
        ', '.join(sorted(failed)))
    if not warn_on_exceptions:
      raise NoCliTreeGeneratorForCommand(message)
    log.warn(message)


def LoadAll(directory=None, ignore_out_of_date=False, root=None,
            warn_on_exceptions=True):
  """Loads all CLI trees in directory and adds them to tree.

  Args:
    directory: The config directory containing the CLI tree modules.
    ignore_out_of_date: Ignore out of date trees instead of regenerating.
    root: dict, The CLI root to update. A new root is created if None.
    warn_on_exceptions: Warn on exceptions instead of raising if True.

  Raises:
    CliTreeVersionError: loaded tree version mismatch
    ImportModuleError: import errors

  Returns:
    The CLI tree.
  """
  # Create the root node if needed.
  if root is None:
    root = cli_tree.Node(description='The CLI tree root.')

  # Load the default CLI if available.
  if cli_tree.DEFAULT_CLI_NAME not in root[cli_tree.LOOKUP_COMMANDS]:
    try:
      root[cli_tree.LOOKUP_COMMANDS][cli_tree.DEFAULT_CLI_NAME] = (
          cli_tree.Load())
    except cli_tree.CliTreeLoadError:
      pass

  # Load extra CLIs by searching directories in order. .json files are treated
  # as CLI modules/data, where the file base name is the name of the CLI root
  # command.
  directories = _GetDirectories(
      directory=directory, warn_on_exceptions=warn_on_exceptions)

  loaded = {cli_tree.DEFAULT_CLI_NAME, '__init__'}  # Already loaded this above.
  for directory in directories:
    if not directory or not os.path.exists(directory):
      continue
    for (dirpath, _, filenames) in os.walk(directory):
      for filename in sorted(filenames):  # For stability across runs.
        command, extension = os.path.splitext(filename)
        if extension != '.json':
          continue
        if command in loaded:
          # Already loaded. Earlier directory hits take precedence.
          continue
        loaded.add(command)
        if command == cli_tree.DEFAULT_CLI_NAME:
          tree = cli_tree.Load(os.path.join(dirpath, filename))
        else:
          tree = LoadOrGenerate(command,
                                directories=[dirpath],
                                ignore_out_of_date=ignore_out_of_date,
                                warn_on_exceptions=warn_on_exceptions)
        if tree:
          root[cli_tree.LOOKUP_COMMANDS][command] = tree
      # Don't search subdirectories.
      break

  return root

# Copyright 2017 Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #    http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. gcloud CLI tree generators for non-gcloud CLIs.  A CLI tree for a supported command is generated by using the root command plus `help` or `--help` arguments to do a DFS traversal. Each node is generated from a man-ish style runtime document.  Supported CLI commands have their own runtime help document quirks, so each is handled by an ad-hoc parser. The parsers rely on consistency within commands and between command releases.  The CLI tree for an unsupported command is generated from the output of `man the-command` and contains only the command root node. # pylint: disable=unused-import Command does not have a CLI tree generator. Could not get man page text for command. # TODO(b/69033748): disable until issues resolved Returns True if long running CLI tree generation is disabled. # Only a few command generators are long running. # Only generate these CLI trees on the fly for explicit requests. # It can take ~1minute, not good, especially at the interactive prompt. # It's a long running generator, not explicitly requested -- disable. Returns text dedented and multiple non-indent spaces replaced by one. Initializes and returns a flag dict node. Initializes and returns a positional dict node. # type: (str) -> Dict[str, Any] Initializes and returns a command/group dict node. Base CLI tree generator. Memoizes failed attempts and doesn't repeat them if enable is True. Returns True if man page request for command already failed. Add command to the set of failed man generations. # For memoizing GetVersion() Runs cmd and returns the output as a string. Returns the CLI_VERSION string. Generates and returns the CLI tree dict. Returns (path,f) open for read for the first CLI tree in directories. # pytype: disable=name-error Returns a bool tuple (readonly, up_to_date). # Check the schema version. # Check the CLI version. # READONLY trees are always up to date. # Don't know how to regenerate but we have one in hand -- accept it. # Schema and CLI versions are up to date. Loads the CLI tree or generates it if necessary, and returns the tree. # TODO(b/69033748): disable until issues resolved # Corrupt JSON -- could have been interrupted. Helper that generates a CLI tree and writes it to a JSON file. # CLI data config dir may not be initialized yet. # At this point: #   (1) the tree is not found or is out of date #   (2) the tree is not readonly #   (3) we have a generator for the tree bq help document section collector. Returns the heading and content lines from text. bq CLI tree generator. Runs cmd and returns the output as a string. # bq exit code is 1 for help and --help. How do you know if help failed? Adds flags in content lines to command. Generates and returns the CLI subtree rooted at path. # `bq help` lists help for all commands. Command flags are "defined" # by example. We don't attempt to suss that out. # pytype: disable=attribute-error # pytype: disable=attribute-error # pytype: disable=attribute-error Generates and returns the CLI tree rooted at self.command. # Construct the tree minus the global flags. # Add the global flags to the root. # Finally add the version stamps. gsutil help document section collector. Returns the heading and content lines from text. # The first heading distinguishes the document page type. # The root help page. # A command/subcommand man style page. # A topic man style page. gsutil CLI tree generator. Runs the command in cmd and returns the output as a string. # gsutil exit code is 1 for --help depending on the context. Adds flags in content lines to command. Generates and returns the CLI subtree rooted at path. Generates and returns the CLI tree rooted at self.command. # Add the global flags to the root. # Add the help topics. # Finally add the version stamps. Kubectl help document section collector. Returns the heading and content lines from text. kubectl CLI tree generator. Adds flags in content lines to command. Generates and returns the CLI subtree rooted at path. # pytype: disable=attribute-error Returns the CLI_VERSION string. Generates and returns the CLI tree rooted at self.command. # Construct the tree minus the global flags. # Add the global flags to the root. # Finally add the version stamps. man page help document section collector base class.    Attributes:     command: The man page command name.     content_indent: A string of space characters representing the indent of       the first line of content for any section.     heading: The heading for the next call to Collect().     text: The list of man page lines.     version: The collector CLI_VERSION string. # pytype: disable=none-attr Returns the CLI_VERSION string. Returns the raw man page text. Returns the preprocessed man page text. Returns the heading and content lines from text. # No NAME no man page. # Subsection heading or category. ## ' + line.strip() # Some man pages, like GNU ls(1), inline flags in DESCRIPTION. man command help document section collector. Returns the raw man page text. Returns the preprocessed man page text. man URL help document section collector. Returns the raw man page text. Returns the text man page for self.command from a URL. # A little sed(1) to clean up the html header/trailer and anchors. # A mispplaced .* or (...) group match could affect timing. Keep track # of that if you change any of the pattern,replacement tuples. #39;', "'"), # ... and some non-regular edits to finish up. # Ignore all lines before the NAME section. # NOTICE: NOT a regex! # Drop font embellishment markdown from flag definition list lines. # Blank line was really a paragraph. # A real paragraph man page CLI tree generator. Returns the man page collector type. Returns the CLI_VERSION string. Adds flags in content lines to command. Returns the (name, type, value-metavar, nargs) for flag name. # --foo # -b VALUE Adds a flag. Add a flag name list. ## '): Generates and returns the CLI subtree rooted at path. ##'): # pytype: disable=attribute-error # pytype: disable=attribute-error Generates and returns the CLI tree rooted at self.command. # Add the version stamps. Returns the CLI tree for command, generating it if it does not exist.    Args:     command: The CLI root command name.     directories: The list of directories containing the CLI tree JSON files.       If None then the default installation directories are used.     force: Update all exitsing trees by forcing them to be out of date if True.     ignore_out_of_date: Ignore out of date trees instead of regenerating.     verbose: Display a status line for up to date CLI trees if True.     warn_on_exceptions: Emits warning messages in lieu of generator exceptions.       Used during installation.    Returns:     The CLI tree for command or None if command not found or there is no       generator. # Don't repeat failed attempts. Helper. # The command must exist. # Instantiate the appropriate generator. # Load or (re)generate the CLI tree if possible. Returns the list of directories to search for CLI trees.    Args:     directory: The directory containing the CLI tree JSON files. If None       then the default installation and config directories are used.     warn_on_exceptions: Emits warning messages in lieu of exceptions. # Initialize the list of directories to search for CLI tree files. The default # CLI tree is only searched for and generated in directories[0]. Other # existing trees are updated in the directory in which they were found. New # trees are generated in directories[-1]. (re)generates the CLI trees in directory if non-existent or out ot date.    This function uses the progress tracker because some of the updates can   take ~minutes.    Args:     cli: The default CLI. If not None then the default CLI is also updated.     commands: Update only the commands in this list.     directory: The directory containing the CLI tree JSON files. If None       then the default installation directories are used.     force: Update all exitsing trees by forcing them to be out of date if True.     verbose: Display a status line for up to date CLI trees if True.     warn_on_exceptions: Emits warning messages in lieu of exceptions. Used       during installation.    Raises:     NoCliTreeGeneratorForCommand: A command in commands is not supported       (doesn't have a generator). Loads all CLI trees in directory and adds them to tree.    Args:     directory: The config directory containing the CLI tree modules.     ignore_out_of_date: Ignore out of date trees instead of regenerating.     root: dict, The CLI root to update. A new root is created if None.     warn_on_exceptions: Warn on exceptions instead of raising if True.    Raises:     CliTreeVersionError: loaded tree version mismatch     ImportModuleError: import errors    Returns:     The CLI tree. # Create the root node if needed. # Load the default CLI if available. # Load extra CLIs by searching directories in order. .json files are treated # as CLI modules/data, where the file base name is the name of the CLI root # command. # Already loaded this above. # For stability across runs. # Already loaded. Earlier directory hits take precedence. # Don't search subdirectories.