def parse_query_string(self, query):
    """Returns a parse tree for the query, each of the node is a
    subclass of Operator. This is both a lexical as well as syntax analyzer step."""
    if not query:
      return None
    # Just braces do not matter
    if query[0] == '(':
      index = self.find_closing_braces(query)
      # This must be the last index, since this was an NOP starting brace
      if index != len(query) - 1:
        raise Exception("Invalid syntax")
      else:
        return self.parse_query_string(query[1:-1])
    start_index = query.find("(")
    # There must be a ( in the query
    if start_index < 0:
      # Otherwise it must be a constant
      try:
        constant = float(query)
        return constant
      except ValueError:
        raise Exception("Invalid syntax")
    token = query[:start_index]
    if token not in self.operators:
      raise Exception("Invalid token: " + token)

    # Get sub components
    rest_of_the_query = query[start_index:]
    braces_end_index = self.find_closing_braces(rest_of_the_query)
    if braces_end_index != len(rest_of_the_query) - 1:
      raise Exception("Invalid syntax")
    parts = self.get_sub_parts(rest_of_the_query[1:-1])

    # parts are simple strings in this case
    if token == "TS":
      # This will raise exception if parts are not syntactically correct
      return self.operators[token](parts)

    children = []
    for part in parts:
      children.append(self.parse_query_string(part))

    # Make a node for the current token
    node = self.operators[token](children)
    return node

Returns a parse tree for the query each of the node is a subclass of Operator . This is both a lexical as well as syntax analyzer step .


def process_incoming_tuples(self):
    """Should be called when tuple was buffered into in_stream

    This method is equivalent to ``addBoltTasks()`` but
    is designed for event-driven single-thread bolt.
    """
    # back-pressure
    if self.output_helper.is_out_queue_available():
      self._read_tuples_and_execute()
      self.output_helper.send_out_tuples()
    else:
      # update outqueue full count
      self.bolt_metrics.update_out_queue_full_count()

def
process_incoming_tuples
(
self
)
:
# back-pressure
if
self
.
output_helper
.
is_out_queue_available
(
)
:
self
.
_read_tuples_and_execute
(
)
self
.
output_helper
.
send_out_tuples
(
)
else
:
# update outqueue full count
self
.
bolt_metrics
.
update_out_queue_full_count
(
)


def ack(self, tup):
    """Indicate that processing of a Tuple has succeeded

    It is compatible with StreamParse API.
    """
    if not isinstance(tup, HeronTuple):
      Log.error("Only HeronTuple type is supported in ack()")
      return

    if self.acking_enabled:
      ack_tuple = tuple_pb2.AckTuple()
      ack_tuple.ackedtuple = int(tup.id)

      tuple_size_in_bytes = 0
      for rt in tup.roots:
        to_add = ack_tuple.roots.add()
        to_add.CopyFrom(rt)
        tuple_size_in_bytes += rt.ByteSize()
      super(BoltInstance, self).admit_control_tuple(ack_tuple, tuple_size_in_bytes, True)

    process_latency_ns = (time.time() - tup.creation_time) * system_constants.SEC_TO_NS
    self.pplan_helper.context.invoke_hook_bolt_ack(tup, process_latency_ns)
    self.bolt_metrics.acked_tuple(tup.stream, tup.component, process_latency_ns)


def fail(self, tup):
    """Indicate that processing of a Tuple has failed

    It is compatible with StreamParse API.
    """
    if not isinstance(tup, HeronTuple):
      Log.error("Only HeronTuple type is supported in fail()")
      return

    if self.acking_enabled:
      fail_tuple = tuple_pb2.AckTuple()
      fail_tuple.ackedtuple = int(tup.id)

      tuple_size_in_bytes = 0
      for rt in tup.roots:
        to_add = fail_tuple.roots.add()
        to_add.CopyFrom(rt)
        tuple_size_in_bytes += rt.ByteSize()
      super(BoltInstance, self).admit_control_tuple(fail_tuple, tuple_size_in_bytes, False)

    fail_latency_ns = (time.time() - tup.creation_time) * system_constants.SEC_TO_NS
    self.pplan_helper.context.invoke_hook_bolt_fail(tup, fail_latency_ns)
    self.bolt_metrics.failed_tuple(tup.stream, tup.component, fail_latency_ns)


def template_slave_hcl(cl_args, masters):
  '''
  Template slave config file
  '''
  slave_config_template = "%s/standalone/templates/slave.template.hcl" % cl_args["config_path"]
  slave_config_actual = "%s/standalone/resources/slave.hcl" % cl_args["config_path"]
  masters_in_quotes = ['"%s"' % master for master in masters]
  template_file(slave_config_template, slave_config_actual,
                {"<nomad_masters:master_port>": ", ".join(masters_in_quotes)})


def template_scheduler_yaml(cl_args, masters):
  '''
  Template scheduler.yaml
  '''
  single_master = masters[0]
  scheduler_config_actual = "%s/standalone/scheduler.yaml" % cl_args["config_path"]

  scheduler_config_template = "%s/standalone/templates/scheduler.template.yaml" \
                              % cl_args["config_path"]
  template_file(scheduler_config_template, scheduler_config_actual,
                {"<scheduler_uri>": "http://%s:4646" % single_master})


def template_uploader_yaml(cl_args, masters):
  '''
  Tempate uploader.yaml
  '''
  single_master = masters[0]
  uploader_config_template = "%s/standalone/templates/uploader.template.yaml" \
                             % cl_args["config_path"]
  uploader_config_actual = "%s/standalone/uploader.yaml" % cl_args["config_path"]

  template_file(uploader_config_template, uploader_config_actual,
                {"<http_uploader_uri>": "http://%s:9000/api/v1/file/upload" % single_master})


def template_apiserver_hcl(cl_args, masters, zookeepers):
  """
  template apiserver.hcl
  """
  single_master = masters[0]
  apiserver_config_template = "%s/standalone/templates/apiserver.template.hcl" \
                              % cl_args["config_path"]
  apiserver_config_actual = "%s/standalone/resources/apiserver.hcl" % cl_args["config_path"]

  replacements = {
      "<heron_apiserver_hostname>": '"%s"' % get_hostname(single_master, cl_args),
      "<heron_apiserver_executable>": '"%s/heron-apiserver"'
                                      % config.get_heron_bin_dir()
                                      if is_self(single_master)
                                      else '"%s/.heron/bin/heron-apiserver"'
                                      % get_remote_home(single_master, cl_args),
      "<zookeeper_host:zookeeper_port>": ",".join(
          ['%s' % zk if ":" in zk else '%s:2181' % zk for zk in zookeepers]),
      "<scheduler_uri>": "http://%s:4646" % single_master
  }

  template_file(apiserver_config_template, apiserver_config_actual, replacements)


def template_statemgr_yaml(cl_args, zookeepers):
  '''
  Template statemgr.yaml
  '''
  statemgr_config_file_template = "%s/standalone/templates/statemgr.template.yaml" \
                                  % cl_args["config_path"]
  statemgr_config_file_actual = "%s/standalone/statemgr.yaml" % cl_args["config_path"]

  template_file(statemgr_config_file_template, statemgr_config_file_actual,
                {"<zookeeper_host:zookeeper_port>": ",".join(
                    ['"%s"' % zk if ":" in zk else '"%s:2181"' % zk for zk in zookeepers])})


def template_heron_tools_hcl(cl_args, masters, zookeepers):
  '''
  template heron tools
  '''
  heron_tools_hcl_template = "%s/standalone/templates/heron_tools.template.hcl" \
                             % cl_args["config_path"]
  heron_tools_hcl_actual = "%s/standalone/resources/heron_tools.hcl" \
                             % cl_args["config_path"]

  single_master = masters[0]
  template_file(heron_tools_hcl_template, heron_tools_hcl_actual,
                {
                    "<zookeeper_host:zookeeper_port>": ",".join(
                        ['%s' % zk if ":" in zk else '%s:2181' % zk for zk in zookeepers]),
                    "<heron_tracker_executable>": '"%s/heron-tracker"' % config.get_heron_bin_dir(),
                    "<heron_tools_hostname>": '"%s"' % get_hostname(single_master, cl_args),
                    "<heron_ui_executable>": '"%s/heron-ui"' % config.get_heron_bin_dir()
                })


def print_cluster_info(cl_args):
  '''
  get cluster info for standalone cluster
  '''
  parsed_roles = read_and_parse_roles(cl_args)
  masters = list(parsed_roles[Role.MASTERS])
  slaves = list(parsed_roles[Role.SLAVES])
  zookeepers = list(parsed_roles[Role.ZOOKEEPERS])
  cluster = list(parsed_roles[Role.CLUSTER])

  # OrderedDicts are used here so that the key order can be
  # specified directly
  info = OrderedDict()
  info['numNodes'] = len(cluster)
  info['nodes'] = cluster
  roles = OrderedDict()
  roles['masters'] = masters
  roles['slaves'] = slaves
  roles['zookeepers'] = zookeepers
  urls = OrderedDict()
  urls['serviceUrl'] = get_service_url(cl_args)
  urls['heronUi'] = get_heron_ui_url(cl_args)
  urls['heronTracker'] = get_heron_tracker_url(cl_args)
  info['roles'] = roles
  info['urls'] = urls

  print json.dumps(info, indent=2)


def add_additional_args(parsers):
  '''
  add additional parameters to parser
  '''
  for parser in parsers:
    cli_args.add_verbose(parser)
    cli_args.add_config(parser)
    parser.add_argument(
        '--heron-dir',
        default=config.get_heron_dir(),
        help='Path to Heron home directory')

def
add_additional_args
(
parsers
)
:
for
parser
in
parsers
:
cli_args
.
add_verbose
(
parser
)
cli_args
.
add_config
(
parser
)
parser
.
add_argument
(
'--heron-dir'
,
default
=
config
.
get_heron_dir
(
)
,
help
=
'Path to Heron home directory'
)


def stop_cluster(cl_args):
  '''
  teardown the cluster
  '''
  Log.info("Terminating cluster...")

  roles = read_and_parse_roles(cl_args)
  masters = roles[Role.MASTERS]
  slaves = roles[Role.SLAVES]
  dist_nodes = masters.union(slaves)

  # stop all jobs
  if masters:
    try:
      single_master = list(masters)[0]
      jobs = get_jobs(cl_args, single_master)
      for job in jobs:
        job_id = job["ID"]
        Log.info("Terminating job %s" % job_id)
        delete_job(cl_args, job_id, single_master)
    except:
      Log.debug("Error stopping jobs")
      Log.debug(sys.exc_info()[0])

  for node in dist_nodes:
    Log.info("Terminating processes on %s" % node)
    if not is_self(node):
      cmd = "ps aux | grep heron-nomad | awk '{print \$2}' " \
            "| xargs kill"
      cmd = ssh_remote_execute(cmd, node, cl_args)
    else:
      cmd = "ps aux | grep heron-nomad | awk '{print $2}' " \
            "| xargs kill"
    Log.debug(cmd)
    pid = subprocess.Popen(cmd,
                           shell=True,
                           stdout=subprocess.PIPE,
                           stderr=subprocess.PIPE)

    return_code = pid.wait()
    output = pid.communicate()
    Log.debug("return code: %s output: %s" % (return_code, output))

    Log.info("Cleaning up directories on %s" % node)
    cmd = "rm -rf /tmp/slave ; rm -rf /tmp/master"
    if not is_self(node):
      cmd = ssh_remote_execute(cmd, node, cl_args)
    Log.debug(cmd)
    pid = subprocess.Popen(cmd,
                           shell=True,
                           stdout=subprocess.PIPE,
                           stderr=subprocess.PIPE)

    return_code = pid.wait()
    output = pid.communicate()
    Log.debug("return code: %s output: %s" % (return_code, output))


def start_cluster(cl_args):
  '''
  Start a Heron standalone cluster
  '''
  roles = read_and_parse_roles(cl_args)
  masters = roles[Role.MASTERS]
  slaves = roles[Role.SLAVES]
  zookeepers = roles[Role.ZOOKEEPERS]
  Log.info("Roles:")
  Log.info(" - Master Servers: %s" % list(masters))
  Log.info(" - Slave Servers: %s" % list(slaves))
  Log.info(" - Zookeeper Servers: %s" % list(zookeepers))
  if not masters:
    Log.error("No master servers specified!")
    sys.exit(-1)
  if not slaves:
    Log.error("No slave servers specified!")
    sys.exit(-1)
  if not zookeepers:
    Log.error("No zookeeper servers specified!")
    sys.exit(-1)
  # make sure configs are templated
  update_config_files(cl_args)

  dist_nodes = list(masters.union(slaves))
  # if just local deployment
  if not (len(dist_nodes) == 1 and is_self(dist_nodes[0])):
    distribute_package(roles, cl_args)
  start_master_nodes(masters, cl_args)
  start_slave_nodes(slaves, cl_args)
  start_api_server(masters, cl_args)
  start_heron_tools(masters, cl_args)
  Log.info("Heron standalone cluster complete!")

def
start_cluster
(
cl_args
)
:
roles
=
read_and_parse_roles
(
cl_args
)
masters
=
roles
[
Role
.
MASTERS
]
slaves
=
roles
[
Role
.
SLAVES
]
zookeepers
=
roles
[
Role
.
ZOOKEEPERS
]
Log
.
info
(
"Roles:"
)
Log
.
info
(
" - Master Servers: %s"
%
list
(
masters
)
)
Log
.
info
(
" - Slave Servers: %s"
%
list
(
slaves
)
)
Log
.
info
(
" - Zookeeper Servers: %s"
%
list
(
zookeepers
)
)
if
not
masters
:
Log
.
error
(
"No master servers specified!"
)
sys
.
exit
(
1
)
if
not
slaves
:
Log
.
error
(
"No slave servers specified!"
)
sys
.
exit
(
1
)
if
not
zookeepers
:
Log
.
error
(
"No zookeeper servers specified!"
)
sys
.
exit
(
1
)
# make sure configs are templated
update_config_files
(
cl_args
)
dist_nodes
=
list
(
masters
.
union
(
slaves
)
)
# if just local deployment
if
not
(
len
(
dist_nodes
)
==
1
and
is_self
(
dist_nodes
[
0
]
)
)
:
distribute_package
(
roles
,
cl_args
)
start_master_nodes
(
masters
,
cl_args
)
start_slave_nodes
(
slaves
,
cl_args
)
start_api_server
(
masters
,
cl_args
)
start_heron_tools
(
masters
,
cl_args
)
Log
.
info
(
"Heron standalone cluster complete!"
)


def start_heron_tools(masters, cl_args):
  '''
  Start Heron tracker and UI
  '''
  single_master = list(masters)[0]
  wait_for_master_to_start(single_master)

  cmd = "%s run %s >> /tmp/heron_tools_start.log 2>&1 &" \
        % (get_nomad_path(cl_args), get_heron_tools_job_file(cl_args))
  Log.info("Starting Heron Tools on %s" % single_master)

  if not is_self(single_master):
    cmd = ssh_remote_execute(cmd, single_master, cl_args)
  Log.debug(cmd)
  pid = subprocess.Popen(cmd,
                         shell=True,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)

  return_code = pid.wait()
  output = pid.communicate()
  Log.debug("return code: %s output: %s" % (return_code, output))
  if return_code != 0:
    Log.error("Failed to start Heron Tools on %s with error:\n%s" % (single_master, output[1]))
    sys.exit(-1)

  wait_for_job_to_start(single_master, "heron-tools")
  Log.info("Done starting Heron Tools")


def distribute_package(roles, cl_args):
  '''
  distribute Heron packages to all nodes
  '''
  Log.info("Distributing heron package to nodes (this might take a while)...")
  masters = roles[Role.MASTERS]
  slaves = roles[Role.SLAVES]

  tar_file = tempfile.NamedTemporaryFile(suffix=".tmp").name
  Log.debug("TAR file %s to %s" % (cl_args["heron_dir"], tar_file))
  make_tarfile(tar_file, cl_args["heron_dir"])
  dist_nodes = masters.union(slaves)

  scp_package(tar_file, dist_nodes, cl_args)


def wait_for_master_to_start(single_master):
  '''
  Wait for a nomad master to start
  '''
  i = 0
  while True:
    try:
      r = requests.get("http://%s:4646/v1/status/leader" % single_master)
      if r.status_code == 200:
        break
    except:
      Log.debug(sys.exc_info()[0])
      Log.info("Waiting for cluster to come up... %s" % i)
      time.sleep(1)
      if i > 10:
        Log.error("Failed to start Nomad Cluster!")
        sys.exit(-1)
    i = i + 1

def
wait_for_master_to_start
(
single_master
)
:
i
=
0
while
True
:
try
:
r
=
requests
.
get
(
"http://%s:4646/v1/status/leader"
%
single_master
)
if
r
.
status_code
==
200
:
break
except
:
Log
.
debug
(
sys
.
exc_info
(
)
[
0
]
)
Log
.
info
(
"Waiting for cluster to come up... %s"
%
i
)
time
.
sleep
(
1
)
if
i
>
10
:
Log
.
error
(
"Failed to start Nomad Cluster!"
)
sys
.
exit
(
1
)
i
=
i
+
1


def wait_for_job_to_start(single_master, job):
  '''
  Wait for a Nomad job to start
  '''
  i = 0
  while True:
    try:
      r = requests.get("http://%s:4646/v1/job/%s" % (single_master, job))
      if r.status_code == 200 and r.json()["Status"] == "running":
        break
      else:
        raise RuntimeError()
    except:
      Log.debug(sys.exc_info()[0])
      Log.info("Waiting for %s to come up... %s" % (job, i))
      time.sleep(1)
      if i > 20:
        Log.error("Failed to start Nomad Cluster!")
        sys.exit(-1)
    i = i + 1

def
wait_for_job_to_start
(
single_master
,
job
)
:
i
=
0
while
True
:
try
:
r
=
requests
.
get
(
"http://%s:4646/v1/job/%s"
%
(
single_master
,
job
)
)
if
r
.
status_code
==
200
and
r
.
json
(
)
[
"Status"
]
==
"running"
:
break
else
:
raise
RuntimeError
(
)
except
:
Log
.
debug
(
sys
.
exc_info
(
)
[
0
]
)
Log
.
info
(
"Waiting for %s to come up... %s"
%
(
job
,
i
)
)
time
.
sleep
(
1
)
if
i
>
20
:
Log
.
error
(
"Failed to start Nomad Cluster!"
)
sys
.
exit
(
1
)
i
=
i
+
1


def scp_package(package_file, destinations, cl_args):
  '''
  scp and extract package
  '''
  pids = []
  for dest in destinations:
    if is_self(dest):
      continue
    Log.info("Server: %s" % dest)
    file_path = "/tmp/heron.tar.gz"
    dest_file_path = "%s:%s" % (dest, file_path)

    remote_cmd = "rm -rf ~/.heron && mkdir ~/.heron " \
                 "&& tar -xzvf %s -C ~/.heron --strip-components 1" % (file_path)
    cmd = '%s && %s' \
          % (scp_cmd(package_file, dest_file_path, cl_args),
             ssh_remote_execute(remote_cmd, dest, cl_args))
    Log.debug(cmd)
    pid = subprocess.Popen(cmd,
                           shell=True,
                           stdout=subprocess.PIPE,
                           stderr=subprocess.PIPE)
    pids.append({"pid": pid, "dest": dest})

  errors = []
  for entry in pids:
    pid = entry["pid"]
    return_code = pid.wait()
    output = pid.communicate()
    Log.debug("return code: %s output: %s" % (return_code, output))
    if return_code != 0:
      errors.append("Failed to scp package to %s with error:\n%s" % (entry["dest"], output[1]))

  if errors:
    for error in errors:
      Log.error(error)
    sys.exit(-1)

  Log.info("Done distributing packages")


def make_tarfile(output_filename, source_dir):
  '''
  Tar a directory
  '''
  with tarfile.open(output_filename, "w:gz") as tar:
    tar.add(source_dir, arcname=os.path.basename(source_dir))

def make_tarfile ( output_filename , source_dir ) : with tarfile . open ( output_filename , "w:gz" ) as tar : tar . add ( source_dir , arcname = os . path . basename ( source_dir ) )


def start_master_nodes(masters, cl_args):
  '''
  Start master nodes
  '''
  pids = []
  for master in masters:
    Log.info("Starting master on %s" % master)
    cmd = "%s agent -config %s >> /tmp/nomad_server_log 2>&1 &" \
          % (get_nomad_path(cl_args), get_nomad_master_config_file(cl_args))
    if not is_self(master):
      cmd = ssh_remote_execute(cmd, master, cl_args)
    Log.debug(cmd)
    pid = subprocess.Popen(cmd,
                           shell=True,
                           stdout=subprocess.PIPE,
                           stderr=subprocess.PIPE)
    pids.append({"pid": pid, "dest": master})

  errors = []
  for entry in pids:
    pid = entry["pid"]
    return_code = pid.wait()
    output = pid.communicate()
    Log.debug("return code: %s output: %s" % (return_code, output))
    if return_code != 0:
      errors.append("Failed to start master on %s with error:\n%s" % (entry["dest"], output[1]))

  if errors:
    for error in errors:
      Log.error(error)
    sys.exit(-1)

  Log.info("Done starting masters")


def start_slave_nodes(slaves, cl_args):
  '''
  Star slave nodes
  '''
  pids = []
  for slave in slaves:
    Log.info("Starting slave on %s" % slave)
    cmd = "%s agent -config %s >> /tmp/nomad_client.log 2>&1 &" \
          % (get_nomad_path(cl_args), get_nomad_slave_config_file(cl_args))
    if not is_self(slave):
      cmd = ssh_remote_execute(cmd, slave, cl_args)
    Log.debug(cmd)
    pid = subprocess.Popen(cmd,
                           shell=True,
                           stdout=subprocess.PIPE,
                           stderr=subprocess.PIPE)
    pids.append({"pid": pid, "dest": slave})

  errors = []
  for entry in pids:
    pid = entry["pid"]
    return_code = pid.wait()
    output = pid.communicate()
    Log.debug("return code: %s output: %s" % (return_code, output))
    if return_code != 0:
      errors.append("Failed to start slave on %s with error:\n%s" % (entry["dest"], output[1]))

  if errors:
    for error in errors:
      Log.error(error)
    sys.exit(-1)

  Log.info("Done starting slaves")


def read_and_parse_roles(cl_args):
  '''
  read config files to get roles
  '''
  roles = dict()

  with open(get_inventory_file(cl_args), 'r') as stream:
    try:
      roles = yaml.load(stream)
    except yaml.YAMLError as exc:
      Log.error("Error parsing inventory file: %s" % exc)
      sys.exit(-1)

  if Role.ZOOKEEPERS not in roles or not roles[Role.ZOOKEEPERS]:
    Log.error("Zookeeper servers node defined!")
    sys.exit(-1)

  if Role.CLUSTER not in roles or not roles[Role.CLUSTER]:
    Log.error("Heron cluster nodes defined!")
    sys.exit(-1)

  # Set roles
  roles[Role.MASTERS] = set([roles[Role.CLUSTER][0]])
  roles[Role.SLAVES] = set(roles[Role.CLUSTER])
  roles[Role.ZOOKEEPERS] = set(roles[Role.ZOOKEEPERS])
  roles[Role.CLUSTER] = set(roles[Role.CLUSTER])

  return roles

def
read_and_parse_roles
(
cl_args
)
:
roles
=
dict
(
)
with
open
(
get_inventory_file
(
cl_args
)
,
'r'
)
as
stream
:
try
:
roles
=
yaml
.
load
(
stream
)
except
yaml
.
YAMLError
as
exc
:
Log
.
error
(
"Error parsing inventory file: %s"
%
exc
)
sys
.
exit
(
1
)
if
Role
.
ZOOKEEPERS
not
in
roles
or
not
roles
[
Role
.
ZOOKEEPERS
]
:
Log
.
error
(
"Zookeeper servers node defined!"
)
sys
.
exit
(
1
)
if
Role
.
CLUSTER
not
in
roles
or
not
roles
[
Role
.
CLUSTER
]
:
Log
.
error
(
"Heron cluster nodes defined!"
)
sys
.
exit
(
1
)
# Set roles
roles
[
Role
.
MASTERS
]
=
set
(
[
roles
[
Role
.
CLUSTER
]
[
0
]
]
)
roles
[
Role
.
SLAVES
]
=
set
(
roles
[
Role
.
CLUSTER
]
)
roles
[
Role
.
ZOOKEEPERS
]
=
set
(
roles
[
Role
.
ZOOKEEPERS
]
)
roles
[
Role
.
CLUSTER
]
=
set
(
roles
[
Role
.
CLUSTER
]
)
return
roles


def get_remote_home(host, cl_args):
  '''
  get home directory of remote host
  '''
  cmd = "echo ~"
  if not is_self(host):
    cmd = ssh_remote_execute(cmd, host, cl_args)
  pid = subprocess.Popen(cmd,
                         shell=True,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)
  return_code = pid.wait()
  output = pid.communicate()

  if return_code != 0:
    Log.error("Failed to get home path for remote host %s with output:\n%s" % (host, output))
    sys.exit(-1)
  return output[0].strip("\n")


def get_hostname(ip_addr, cl_args):
  '''
  get host name of remote host
  '''
  if is_self(ip_addr):
    return get_self_hostname()
  cmd = "hostname"
  ssh_cmd = ssh_remote_execute(cmd, ip_addr, cl_args)
  pid = subprocess.Popen(ssh_cmd,
                         shell=True,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE)
  return_code = pid.wait()
  output = pid.communicate()

  if return_code != 0:
    Log.error("Failed to get hostname for remote host %s with output:\n%s" % (ip_addr, output))
    sys.exit(-1)
  return output[0].strip("\n")


def is_self(addr):
  '''
  check if this host is this addr
  '''
  ips = []
  for i in netifaces.interfaces():
    entry = netifaces.ifaddresses(i)
    if netifaces.AF_INET in entry:
      for ipv4 in entry[netifaces.AF_INET]:
        if "addr" in ipv4:
          ips.append(ipv4["addr"])
  return addr in ips or addr == get_self_hostname()

def is_self ( addr ) : ips = [ ] for i in netifaces . interfaces ( ) : entry = netifaces . ifaddresses ( i ) if netifaces . AF_INET in entry : for ipv4 in entry [ netifaces . AF_INET ] : if "addr" in ipv4 : ips . append ( ipv4 [ "addr" ] ) return addr in ips or addr == get_self_hostname ( )


def log(self, message, level=None):
    """Log message, optionally providing a logging level

    It is compatible with StreamParse API.

    :type message: str
    :param message: the log message to send
    :type level: str
    :param level: the logging level,
                  one of: trace (=debug), debug, info, warn or error (default: info)
    """
    if level is None:
      _log_level = logging.INFO
    else:
      if level == "trace" or level == "debug":
        _log_level = logging.DEBUG
      elif level == "info":
        _log_level = logging.INFO
      elif level == "warn":
        _log_level = logging.WARNING
      elif level == "error":
        _log_level = logging.ERROR
      else:
        raise ValueError("%s is not supported as logging level" % str(level))

    self.logger.log(_log_level, message)

def
log
(
self
,
message
,
level
=
)
:
if
level
is
:
_log_level
=
logging
.
INFO
else
:
if
level
==
"trace"
or
level
==
"debug"
:
_log_level
=
logging
.
DEBUG
elif
level
==
"info"
:
_log_level
=
logging
.
INFO
elif
level
==
"warn"
:
_log_level
=
logging
.
WARNING
elif
level
==
"error"
:
_log_level
=
logging
.
ERROR
else
:
raise
ValueError
(
"%s is not supported as logging level"
%
str
(
level
)
)
self
.
logger
.
log
(
_log_level
,
message
)

Log message optionally providing a logging level


def dereference_symlinks(src):
    """
    Resolve all symbolic references that `src` points to.  Note that this
    is different than `os.path.realpath` as path components leading up to
    the final location may still be symbolic links.
    """
    while os.path.islink(src):
        src = os.path.join(os.path.dirname(src), os.readlink(src))

    return src

def dereference_symlinks ( src ) : while os . path . islink ( src ) : src = os . path . join ( os . path . dirname ( src ) , os . readlink ( src ) ) return src

Resolve all symbolic references that src points to . Note that this is different than os . path . realpath as path components leading up to the final location may still be symbolic links .


def to_table(result):
  ''' normalize raw result to table '''
  max_count = 20
  table, count = [], 0
  for role, envs_topos in result.items():
    for env, topos in envs_topos.items():
      for topo in topos:
        count += 1
        if count > max_count:
          continue
        else:
          table.append([role, env, topo])
  header = ['role', 'env', 'topology']
  rest_count = 0 if count <= max_count else count - max_count
  return table, header, rest_count

def to_table ( result ) : max_count = 20 table , count = [ ] , 0 for role , envs_topos in result . items ( ) : for env , topos in envs_topos . items ( ) : for topo in topos : count += 1 if count > max_count : continue else : table . append ( [ role , env , topo ] ) header = [ 'role' , 'env' , 'topology' ] rest_count = 0 if count <= max_count else count - max_count return table , header , rest_count


def add_context(self, err_context, succ_context=None):
    """ Prepend msg to add some context information

    :param pmsg: context info
    :return: None
    """
    self.err_context = err_context
    self.succ_context = succ_context

def add_context ( self , err_context , succ_context = None ) : self . err_context = err_context self . succ_context = succ_context


def is_host_port_reachable(self):
    """
    Returns true if the host is reachable. In some cases, it may not be reachable a tunnel
    must be used.
    """
    for hostport in self.hostportlist:
      try:
        socket.create_connection(hostport, StateManager.TIMEOUT_SECONDS)
        return True
      except:
        LOG.info("StateManager %s Unable to connect to host: %s port %i"
                 % (self.name, hostport[0], hostport[1]))
        continue
    return False

Returns true if the host is reachable . In some cases it may not be reachable a tunnel must be used .


def pick_unused_port(self):
    """ Pick an unused port. There is a slight chance that this wont work. """
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.bind(('127.0.0.1', 0))
    _, port = s.getsockname()
    s.close()
    return port

def pick_unused_port ( self ) : s = socket . socket ( socket . AF_INET , socket . SOCK_STREAM ) s . bind ( ( '127.0.0.1' , 0 ) ) _ , port = s . getsockname ( ) s . close ( ) return port

Pick an unused port . There is a slight chance that this wont work .


def establish_ssh_tunnel(self):
    """
    Establish an ssh tunnel for each local host and port
    that can be used to communicate with the state host.
    """
    localportlist = []
    for (host, port) in self.hostportlist:
      localport = self.pick_unused_port()
      self.tunnel.append(subprocess.Popen(
          ('ssh', self.tunnelhost, '-NL127.0.0.1:%d:%s:%d' % (localport, host, port))))
      localportlist.append(('127.0.0.1', localport))
    return localportlist

def establish_ssh_tunnel ( self ) : localportlist = [ ] for ( host , port ) in self . hostportlist : localport = self . pick_unused_port ( ) self . tunnel . append ( subprocess . Popen ( ( 'ssh' , self . tunnelhost , '-NL127.0.0.1:%d:%s:%d' % ( localport , host , port ) ) ) ) localportlist . append ( ( '127.0.0.1' , localport ) ) return localportlist

Establish an ssh tunnel for each local host and port that can be used to communicate with the state host .


def monitor(self):
    """
    Monitor the rootpath and call the callback
    corresponding to the change.
    This monitoring happens periodically. This function
    is called in a seperate thread from the main thread,
    because it sleeps for the intervals between each poll.
    """

    def trigger_watches_based_on_files(watchers, path, directory, ProtoClass):
      """
      For all the topologies in the watchers, check if the data
      in directory has changed. Trigger the callback if it has.
      """
      for topology, callbacks in watchers.items():
        file_path = os.path.join(path, topology)
        data = ""
        if os.path.exists(file_path):
          with open(os.path.join(path, topology)) as f:
            data = f.read()
        if topology not in directory or data != directory[topology]:
          proto_object = ProtoClass()
          proto_object.ParseFromString(data)
          for callback in callbacks:
            callback(proto_object)
          directory[topology] = data

    while not self.monitoring_thread_stop_signal:
      topologies_path = self.get_topologies_path()

      topologies = []
      if os.path.isdir(topologies_path):
        topologies = list(filter(
            lambda f: os.path.isfile(os.path.join(topologies_path, f)),
            os.listdir(topologies_path)))
      if set(topologies) != set(self.topologies_directory):
        for callback in self.topologies_watchers:
          callback(topologies)
      self.topologies_directory = topologies

      trigger_watches_based_on_files(
          self.topology_watchers, topologies_path, self.topologies_directory, Topology)

      # Get the directory name for execution state
      execution_state_path = os.path.dirname(self.get_execution_state_path(""))
      trigger_watches_based_on_files(
          self.execution_state_watchers, execution_state_path,
          self.execution_state_directory, ExecutionState)

      # Get the directory name for packing_plan
      packing_plan_path = os.path.dirname(self.get_packing_plan_path(""))
      trigger_watches_based_on_files(
          self.packing_plan_watchers, packing_plan_path, self.packing_plan_directory, PackingPlan)

      # Get the directory name for pplan
      pplan_path = os.path.dirname(self.get_pplan_path(""))
      trigger_watches_based_on_files(
          self.pplan_watchers, pplan_path,
          self.pplan_directory, PhysicalPlan)

      # Get the directory name for tmaster
      tmaster_path = os.path.dirname(self.get_tmaster_path(""))
      trigger_watches_based_on_files(
          self.tmaster_watchers, tmaster_path,
          self.tmaster_directory, TMasterLocation)

      # Get the directory name for scheduler location
      scheduler_location_path = os.path.dirname(self.get_scheduler_location_path(""))
      trigger_watches_based_on_files(
          self.scheduler_location_watchers, scheduler_location_path,
          self.scheduler_location_directory, SchedulerLocation)

      # Sleep for some time
      self.event.wait(timeout=5)

Monitor the rootpath and call the callback corresponding to the change . This monitoring happens periodically . This function is called in a seperate thread from the main thread because it sleeps for the intervals between each poll .


def get_pplan(self, topologyName, callback=None):
    """
    Get physical plan of a topology
    """
    if callback:
      self.pplan_watchers[topologyName].append(callback)
    else:
      pplan_path = self.get_pplan_path(topologyName)
      with open(pplan_path) as f:
        data = f.read()
        pplan = PhysicalPlan()
        pplan.ParseFromString(data)
        return pplan

def get_pplan ( self , topologyName , callback = None ) : if callback : self . pplan_watchers [ topologyName ] . append ( callback ) else : pplan_path = self . get_pplan_path ( topologyName ) with open ( pplan_path ) as f : data = f . read ( ) pplan = PhysicalPlan ( ) pplan . ParseFromString ( data ) return pplan


def get_execution_state(self, topologyName, callback=None):
    """
    Get execution state
    """
    if callback:
      self.execution_state_watchers[topologyName].append(callback)
    else:
      execution_state_path = self.get_execution_state_path(topologyName)
      with open(execution_state_path) as f:
        data = f.read()
        executionState = ExecutionState()
        executionState.ParseFromString(data)
        return executionState

def get_execution_state ( self , topologyName , callback = None ) : if callback : self . execution_state_watchers [ topologyName ] . append ( callback ) else : execution_state_path = self . get_execution_state_path ( topologyName ) with open ( execution_state_path ) as f : data = f . read ( ) executionState = ExecutionState ( ) executionState . ParseFromString ( data ) return executionState


def get_scheduler_location(self, topologyName, callback=None):
    """
    Get scheduler location
    """
    if callback:
      self.scheduler_location_watchers[topologyName].append(callback)
    else:
      scheduler_location_path = self.get_scheduler_location_path(topologyName)
      with open(scheduler_location_path) as f:
        data = f.read()
        scheduler_location = SchedulerLocation()
        scheduler_location.ParseFromString(data)
        return scheduler_location

def get_scheduler_location ( self , topologyName , callback = None ) : if callback : self . scheduler_location_watchers [ topologyName ] . append ( callback ) else : scheduler_location_path = self . get_scheduler_location_path ( topologyName ) with open ( scheduler_location_path ) as f : data = f . read ( ) scheduler_location = SchedulerLocation ( ) scheduler_location . ParseFromString ( data ) return scheduler_location


def create_socket_options():
  """Creates SocketOptions object from a given sys_config dict"""
  sys_config = system_config.get_sys_config()
  opt_list = [const.INSTANCE_NETWORK_WRITE_BATCH_SIZE_BYTES,
              const.INSTANCE_NETWORK_WRITE_BATCH_TIME_MS,
              const.INSTANCE_NETWORK_READ_BATCH_SIZE_BYTES,
              const.INSTANCE_NETWORK_READ_BATCH_TIME_MS,
              const.INSTANCE_NETWORK_OPTIONS_SOCKET_RECEIVED_BUFFER_SIZE_BYTES,
              const.INSTANCE_NETWORK_OPTIONS_SOCKET_SEND_BUFFER_SIZE_BYTES]

  Log.debug("In create_socket_options()")
  try:
    value_lst = [int(sys_config[opt]) for opt in opt_list]
    sock_opt = SocketOptions(*value_lst)
    return sock_opt
  except ValueError as e:
    # couldn't convert to int
    raise ValueError("Invalid value in sys_config: %s" % str(e))
  except KeyError as e:
    # option key was not found
    raise KeyError("Incomplete sys_config: %s" % str(e))


def init_topology(mcs, classname, class_dict):
    """Initializes a topology protobuf"""
    if classname == 'Topology':
      # Base class can't initialize protobuf
      return
    heron_options = TopologyType.get_heron_options_from_env()
    initial_state = heron_options.get("cmdline.topology.initial.state", "RUNNING")
    tmp_directory = heron_options.get("cmdline.topologydefn.tmpdirectory")
    if tmp_directory is None:
      raise RuntimeError("Topology definition temp directory not specified")

    topology_name = heron_options.get("cmdline.topology.name", classname)
    topology_id = topology_name + str(uuid.uuid4())

    # create protobuf
    topology = topology_pb2.Topology()
    topology.id = topology_id
    topology.name = topology_name
    topology.state = topology_pb2.TopologyState.Value(initial_state)
    topology.topology_config.CopyFrom(TopologyType.get_topology_config_protobuf(class_dict))

    TopologyType.add_bolts_and_spouts(topology, class_dict)

    class_dict['topology_name'] = topology_name
    class_dict['topology_id'] = topology_id
    class_dict['protobuf_topology'] = topology
    class_dict['topologydefn_tmpdir'] = tmp_directory
    class_dict['heron_runtime_options'] = heron_options


def get_heron_options_from_env():
    """Retrieves heron options from the `HERON_OPTIONS` environment variable.

    Heron options have the following format:

        cmdline.topologydefn.tmpdirectory=/var/folders/tmpdir
        cmdline.topology.initial.state=PAUSED

    In this case, the returned map will contain:

        #!json
        {
          "cmdline.topologydefn.tmpdirectory": "/var/folders/tmpdir",
          "cmdline.topology.initial.state": "PAUSED"
        }

    Currently supports the following options natively:

    - `cmdline.topologydefn.tmpdirectory`: (required) the directory to which this
    topology's defn file is written
    - `cmdline.topology.initial.state`: (default: "RUNNING") the initial state of the topology
    - `cmdline.topology.name`: (default: class name) topology name on deployment

    Returns: map mapping from key to value
    """
    heron_options_raw = os.environ.get("HERON_OPTIONS")
    if heron_options_raw is None:
      raise RuntimeError("HERON_OPTIONS environment variable not found")

    options = {}
    for option_line in heron_options_raw.replace("%%%%", " ").split(','):
      key, sep, value = option_line.partition("=")
      if sep:
        options[key] = value
      else:
        raise ValueError("Invalid HERON_OPTIONS part %r" % option_line)
    return options

Retrieves heron options from the HERON_OPTIONS environment variable .


def add_spec(self, *specs):
    """Add specs to the topology

    :type specs: HeronComponentSpec
    :param specs: specs to add to the topology
    """
    for spec in specs:
      if not isinstance(spec, HeronComponentSpec):
        raise TypeError("Argument to add_spec needs to be HeronComponentSpec, given: %s"
                        % str(spec))
      if spec.name is None:
        raise ValueError("TopologyBuilder cannot take a spec without name")
      if spec.name == "config":
        raise ValueError("config is a reserved name")
      if spec.name in self._specs:
        raise ValueError("Attempting to add duplicate spec name: %r %r" % (spec.name, spec))

      self._specs[spec.name] = spec


def add_spout(self, name, spout_cls, par, config=None, optional_outputs=None):
    """Add a spout to the topology"""
    spout_spec = spout_cls.spec(name=name, par=par, config=config,
                                optional_outputs=optional_outputs)
    self.add_spec(spout_spec)
    return spout_spec

def add_spout ( self , name , spout_cls , par , config = None , optional_outputs = None ) : spout_spec = spout_cls . spec ( name = name , par = par , config = config , optional_outputs = optional_outputs ) self . add_spec ( spout_spec ) return spout_spec


def add_bolt(self, name, bolt_cls, par, inputs, config=None, optional_outputs=None):
    """Add a bolt to the topology"""
    bolt_spec = bolt_cls.spec(name=name, par=par, inputs=inputs, config=config,
                              optional_outputs=optional_outputs)
    self.add_spec(bolt_spec)
    return bolt_spec

def add_bolt ( self , name , bolt_cls , par , inputs , config = None , optional_outputs = None ) : bolt_spec = bolt_cls . spec ( name = name , par = par , inputs = inputs , config = config , optional_outputs = optional_outputs ) self . add_spec ( bolt_spec ) return bolt_spec


def set_config(self, config):
    """Set topology-wide configuration to the topology

    :type config: dict
    :param config: topology-wide config
    """
    if not isinstance(config, dict):
      raise TypeError("Argument to set_config needs to be dict, given: %s" % str(config))
    self._topology_config = config

Set topology - wide configuration to the topology


def build_and_submit(self):
    """Builds the topology and submits to the destination"""
    class_dict = self._construct_topo_class_dict()
    topo_cls = TopologyType(self.topology_name, (Topology,), class_dict)
    topo_cls.write()

def build_and_submit ( self ) : class_dict = self . _construct_topo_class_dict ( ) topo_cls = TopologyType ( self . topology_name , ( Topology , ) , class_dict ) topo_cls . write ( )


def queries_map():
  """map from query parameter to query name"""
  qs = _all_metric_queries()
  return dict(zip(qs[0], qs[1]) + zip(qs[2], qs[3]))

def queries_map ( ) : qs = _all_metric_queries ( ) return dict ( zip ( qs [ 0 ] , qs [ 1 ] ) + zip ( qs [ 2 ] , qs [ 3 ] ) )


def get_clusters():
  """Synced API call to get all cluster names"""
  instance = tornado.ioloop.IOLoop.instance()
  # pylint: disable=unnecessary-lambda
  try:
    return instance.run_sync(lambda: API.get_clusters())
  except Exception:
    Log.debug(traceback.format_exc())
    raise

def
get_clusters
(
)
:
instance
=
tornado
.
ioloop
.
IOLoop
.
instance
(
)
# pylint: disable=unnecessary-lambda
try
:
return
instance
.
run_sync
(
lambda
:
API
.
get_clusters
(
)
)
except
Exception
:
Log
.
debug
(
traceback
.
format_exc
(
)
)
raise


def get_logical_plan(cluster, env, topology, role):
  """Synced API call to get logical plans"""
  instance = tornado.ioloop.IOLoop.instance()
  try:
    return instance.run_sync(lambda: API.get_logical_plan(cluster, env, topology, role))
  except Exception:
    Log.debug(traceback.format_exc())
    raise

def get_logical_plan ( cluster , env , topology , role ) : instance = tornado . ioloop . IOLoop . instance ( ) try : return instance . run_sync ( lambda : API . get_logical_plan ( cluster , env , topology , role ) ) except Exception : Log . debug ( traceback . format_exc ( ) ) raise


def get_topology_info(*args):
  """Synced API call to get topology information"""
  instance = tornado.ioloop.IOLoop.instance()
  try:
    return instance.run_sync(lambda: API.get_topology_info(*args))
  except Exception:
    Log.debug(traceback.format_exc())
    raise

def get_topology_info ( * args ) : instance = tornado . ioloop . IOLoop . instance ( ) try : return instance . run_sync ( lambda : API . get_topology_info ( * args ) ) except Exception : Log . debug ( traceback . format_exc ( ) ) raise


def get_component_metrics(component, cluster, env, topology, role):
  """Synced API call to get component metrics"""
  all_queries = metric_queries()
  try:
    result = get_topology_metrics(cluster, env, topology, component, [],
                                  all_queries, [0, -1], role)
    return result["metrics"]
  except Exception:
    Log.debug(traceback.format_exc())
    raise

def get_component_metrics ( component , cluster , env , topology , role ) : all_queries = metric_queries ( ) try : result = get_topology_metrics ( cluster , env , topology , component , [ ] , all_queries , [ 0 , - 1 ] , role ) return result [ "metrics" ] except Exception : Log . debug ( traceback . format_exc ( ) ) raise


def configure(level=logging.INFO, logfile=None):
  """ Configure logger which dumps log on terminal

  :param level: logging level: info, warning, verbose...
  :type level: logging level
  :param logfile: log file name, default to None
  :type logfile: string
  :return: None
  :rtype: None
  """

  # Remove all the existing StreamHandlers to avoid duplicate
  for handler in Log.handlers:
    if isinstance(handler, logging.StreamHandler):
      Log.handlers.remove(handler)

  Log.setLevel(level)

  # if logfile is specified, FileHandler is used
  if logfile is not None:
    log_format = "[%(asctime)s] [%(levelname)s]: %(message)s"
    formatter = logging.Formatter(fmt=log_format, datefmt=date_format)
    file_handler = logging.FileHandler(logfile)
    file_handler.setFormatter(formatter)
    Log.addHandler(file_handler)
  # otherwise, use StreamHandler to output to stream (stdout, stderr...)
  else:
    log_format = "[%(asctime)s] %(log_color)s[%(levelname)s]%(reset)s: %(message)s"
    # pylint: disable=redefined-variable-type
    formatter = colorlog.ColoredFormatter(fmt=log_format, datefmt=date_format)
    stream_handler = logging.StreamHandler()
    stream_handler.setFormatter(formatter)
    Log.addHandler(stream_handler)


def init_rotating_logger(level, logfile, max_files, max_bytes):
  """Initializes a rotating logger

  It also makes sure that any StreamHandler is removed, so as to avoid stdout/stderr
  constipation issues
  """
  logging.basicConfig()

  root_logger = logging.getLogger()
  log_format = "[%(asctime)s] [%(levelname)s] %(filename)s: %(message)s"

  root_logger.setLevel(level)
  handler = RotatingFileHandler(logfile, maxBytes=max_bytes, backupCount=max_files)
  handler.setFormatter(logging.Formatter(fmt=log_format, datefmt=date_format))
  root_logger.addHandler(handler)

  for handler in root_logger.handlers:
    root_logger.debug("Associated handlers - " + str(handler))
    if isinstance(handler, logging.StreamHandler):
      root_logger.debug("Removing StreamHandler: " + str(handler))
      root_logger.handlers.remove(handler)


def set_logging_level(cl_args):
  """simply set verbose level based on command-line args

  :param cl_args: CLI arguments
  :type cl_args: dict
  :return: None
  :rtype: None
  """
  if 'verbose' in cl_args and cl_args['verbose']:
    configure(logging.DEBUG)
  else:
    configure(logging.INFO)

def set_logging_level ( cl_args ) : if 'verbose' in cl_args and cl_args [ 'verbose' ] : configure ( logging . DEBUG ) else : configure ( logging . INFO )

simply set verbose level based on command - line args


def _get_spout(self):
    """Returns Spout protobuf message"""
    spout = topology_pb2.Spout()
    spout.comp.CopyFrom(self._get_base_component())

    # Add output streams
    self._add_out_streams(spout)
    return spout

def
_get_spout
(
self
)
:
spout
=
topology_pb2
.
Spout
(
)
spout
.
comp
.
CopyFrom
(
self
.
_get_base_component
(
)
)
# Add output streams
self
.
_add_out_streams
(
spout
)
return
spout


def _get_bolt(self):
    """Returns Bolt protobuf message"""
    bolt = topology_pb2.Bolt()
    bolt.comp.CopyFrom(self._get_base_component())

    # Add streams
    self._add_in_streams(bolt)
    self._add_out_streams(bolt)
    return bolt

def
_get_bolt
(
self
)
:
bolt
=
topology_pb2
.
Bolt
(
)
bolt
.
comp
.
CopyFrom
(
self
.
_get_base_component
(
)
)
# Add streams
self
.
_add_in_streams
(
bolt
)
self
.
_add_out_streams
(
bolt
)
return
bolt


def _get_base_component(self):
    """Returns Component protobuf message"""
    comp = topology_pb2.Component()
    comp.name = self.name
    comp.spec = topology_pb2.ComponentObjectSpec.Value("PYTHON_CLASS_NAME")
    comp.class_name = self.python_class_path
    comp.config.CopyFrom(self._get_comp_config())
    return comp

def _get_base_component ( self ) : comp = topology_pb2 . Component ( ) comp . name = self . name comp . spec = topology_pb2 . ComponentObjectSpec . Value ( "PYTHON_CLASS_NAME" ) comp . class_name = self . python_class_path comp . config . CopyFrom ( self . _get_comp_config ( ) ) return comp


def _get_comp_config(self):
    """Returns component-specific Config protobuf message

    It first adds ``topology.component.parallelism``, and is overriden by
    a user-defined component-specific configuration, specified by spec().
    """
    proto_config = topology_pb2.Config()

    # first add parallelism
    key = proto_config.kvs.add()
    key.key = TOPOLOGY_COMPONENT_PARALLELISM
    key.value = str(self.parallelism)
    key.type = topology_pb2.ConfigValueType.Value("STRING_VALUE")

    # iterate through self.custom_config
    if self.custom_config is not None:
      sanitized = self._sanitize_config(self.custom_config)
      for key, value in sanitized.items():
        if isinstance(value, str):
          kvs = proto_config.kvs.add()
          kvs.key = key
          kvs.value = value
          kvs.type = topology_pb2.ConfigValueType.Value("STRING_VALUE")
        else:
          # need to serialize
          kvs = proto_config.kvs.add()
          kvs.key = key
          kvs.serialized_value = default_serializer.serialize(value)
          kvs.type = topology_pb2.ConfigValueType.Value("PYTHON_SERIALIZED_VALUE")

    return proto_config

def
_get_comp_config
(
self
)
:
proto_config
=
topology_pb2
.
Config
(
)
# first add parallelism
key
=
proto_config
.
kvs
.
add
(
)
key
.
key
=
TOPOLOGY_COMPONENT_PARALLELISM
key
.
value
=
str
(
self
.
parallelism
)
key
.
type
=
topology_pb2
.
ConfigValueType
.
Value
(
"STRING_VALUE"
)
# iterate through self.custom_config
if
self
.
custom_config
is
not
:
sanitized
=
self
.
_sanitize_config
(
self
.
custom_config
)
for
key
,
value
in
sanitized
.
items
(
)
:
if
isinstance
(
value
,
str
)
:
kvs
=
proto_config
.
kvs
.
add
(
)
kvs
.
key
=
key
kvs
.
value
=
value
kvs
.
type
=
topology_pb2
.
ConfigValueType
.
Value
(
"STRING_VALUE"
)
else
:
# need to serialize
kvs
=
proto_config
.
kvs
.
add
(
)
kvs
.
key
=
key
kvs
.
serialized_value
=
default_serializer
.
serialize
(
value
)
kvs
.
type
=
topology_pb2
.
ConfigValueType
.
Value
(
"PYTHON_SERIALIZED_VALUE"
)
return
proto_config

Returns component - specific Config protobuf message


def _add_in_streams(self, bolt):
    """Adds inputs to a given protobuf Bolt message"""
    if self.inputs is None:
      return
    # sanitize inputs and get a map <GlobalStreamId -> Grouping>
    input_dict = self._sanitize_inputs()

    for global_streamid, gtype in input_dict.items():
      in_stream = bolt.inputs.add()
      in_stream.stream.CopyFrom(self._get_stream_id(global_streamid.component_id,
                                                    global_streamid.stream_id))
      if isinstance(gtype, Grouping.FIELDS):
        # it's a field grouping
        in_stream.gtype = gtype.gtype
        in_stream.grouping_fields.CopyFrom(self._get_stream_schema(gtype.fields))
      elif isinstance(gtype, Grouping.CUSTOM):
        # it's a custom grouping
        in_stream.gtype = gtype.gtype
        in_stream.custom_grouping_object = gtype.python_serialized
        in_stream.type = topology_pb2.CustomGroupingObjectType.Value("PYTHON_OBJECT")
      else:
        in_stream.gtype = gtype


def _add_out_streams(self, spbl):
    """Adds outputs to a given protobuf Bolt or Spout message"""
    if self.outputs is None:
      return

    # sanitize outputs and get a map <stream_id -> out fields>
    output_map = self._sanitize_outputs()

    for stream_id, out_fields in output_map.items():
      out_stream = spbl.outputs.add()
      out_stream.stream.CopyFrom(self._get_stream_id(self.name, stream_id))
      out_stream.schema.CopyFrom(self._get_stream_schema(out_fields))


def get_out_streamids(self):
    """Returns a set of output stream ids registered for this component"""
    if self.outputs is None:
      return set()

    if not isinstance(self.outputs, (list, tuple)):
      raise TypeError("Argument to outputs must be either list or tuple, given: %s"
                      % str(type(self.outputs)))
    ret_lst = []
    for output in self.outputs:
      if not isinstance(output, (str, Stream)):
        raise TypeError("Outputs must be a list of strings or Streams, given: %s" % str(output))
      ret_lst.append(Stream.DEFAULT_STREAM_ID if isinstance(output, str) else output.stream_id)
    return set(ret_lst)


def _get_stream_id(comp_name, stream_id):
    """Returns a StreamId protobuf message"""
    proto_stream_id = topology_pb2.StreamId()
    proto_stream_id.id = stream_id
    proto_stream_id.component_name = comp_name
    return proto_stream_id


def _get_stream_schema(fields):
    """Returns a StreamSchema protobuf message"""
    stream_schema = topology_pb2.StreamSchema()
    for field in fields:
      key = stream_schema.keys.add()
      key.key = field
      key.type = topology_pb2.Type.Value("OBJECT")

    return stream_schema

def _get_stream_schema ( fields ) : stream_schema = topology_pb2 . StreamSchema ( ) for field in fields : key = stream_schema . keys . add ( ) key . key = field key . type = topology_pb2 . Type . Value ( "OBJECT" ) return stream_schema


def component_id(self):
    """Returns component_id of this GlobalStreamId

    Note that if HeronComponentSpec is specified as componentId and its name is not yet
    available (i.e. when ``name`` argument was not given in ``spec()`` method in Bolt or Spout),
    this property returns a message with uuid. However, this is provided only for safety
    with __eq__(), __str__(), and __hash__() methods, and not meant to be called explicitly
    before TopologyType class finally sets the name attribute of HeronComponentSpec.
    """
    if isinstance(self._component_id, HeronComponentSpec):
      if self._component_id.name is None:
        # HeronComponentSpec instance's name attribute might not be available until
        # TopologyType metaclass finally sets it. This statement is to support __eq__(),
        # __hash__() and __str__() methods with safety, as raising Exception is not
        # appropriate this case.
        return "<No name available for HeronComponentSpec yet, uuid: %s>" % self._component_id.uuid
      return self._component_id.name
    elif isinstance(self._component_id, str):
      return self._component_id
    else:
      raise ValueError("Component Id for this GlobalStreamId is not properly set: <%s:%s>"
                       % (str(type(self._component_id)), str(self._component_id)))


def register_metric(self, name, metric, time_bucket_in_sec):
    """Registers a new metric to this context"""
    collector = self.get_metrics_collector()
    collector.register_metric(name, metric, time_bucket_in_sec)

def register_metric ( self , name , metric , time_bucket_in_sec ) : collector = self . get_metrics_collector ( ) collector . register_metric ( name , metric , time_bucket_in_sec )


def get_sources(self, component_id):
    """Returns the declared inputs to specified component

    :return: map <streamId namedtuple (same structure as protobuf msg) -> gtype>, or
             None if not found
    """
    # this is necessary because protobuf message is not hashable
    StreamId = namedtuple('StreamId', 'id, component_name')
    if component_id in self.inputs:
      ret = {}
      for istream in self.inputs.get(component_id):
        key = StreamId(id=istream.stream.id, component_name=istream.stream.component_name)
        ret[key] = istream.gtype
      return ret
    else:
      return None


def get_component_tasks(self, component_id):
    """Returns the task ids allocated for the given component id"""
    ret = []
    for task_id, comp_id in self.task_to_component_map.items():
      if comp_id == component_id:
        ret.append(task_id)
    return ret

def get_component_tasks ( self , component_id ) : ret = [ ] for task_id , comp_id in self . task_to_component_map . items ( ) : if comp_id == component_id : ret . append ( task_id ) return ret


def add_task_hook(self, task_hook):
    """Registers a specified task hook to this context

    :type task_hook: heron.instance.src.python.utils.topology.ITaskHook
    :param task_hook: Implementation of ITaskHook
    """
    if not isinstance(task_hook, ITaskHook):
      raise TypeError("In add_task_hook(): attempt to add non ITaskHook instance, given: %s"
                      % str(type(task_hook)))
    self.task_hooks.append(task_hook)


def get_metrics_collector(self):
    """Returns this context's metrics collector"""
    if self.metrics_collector is None or not isinstance(self.metrics_collector, MetricsCollector):
      raise RuntimeError("Metrics collector is not registered in this context")
    return self.metrics_collector

Returns this context s metrics collector


def invoke_hook_spout_ack(self, message_id, complete_latency_ns):
    """invoke task hooks for every time spout acks a tuple

    :type message_id: str
    :param message_id: message id to which an acked tuple was anchored
    :type complete_latency_ns: float
    :param complete_latency_ns: complete latency in nano seconds
    """
    if len(self.task_hooks) > 0:
      spout_ack_info = SpoutAckInfo(message_id=message_id,
                                    spout_task_id=self.get_task_id(),
                                    complete_latency_ms=complete_latency_ns *
                                    system_constants.NS_TO_MS)
      for task_hook in self.task_hooks:
        task_hook.spout_ack(spout_ack_info)

def invoke_hook_spout_ack ( self , message_id , complete_latency_ns ) : if len ( self . task_hooks ) > 0 : spout_ack_info = SpoutAckInfo ( message_id = message_id , spout_task_id = self . get_task_id ( ) , complete_latency_ms = complete_latency_ns * system_constants . NS_TO_MS ) for task_hook in self . task_hooks : task_hook . spout_ack ( spout_ack_info )


def invoke_hook_spout_fail(self, message_id, fail_latency_ns):
    """invoke task hooks for every time spout fails a tuple

    :type message_id: str
    :param message_id: message id to which a failed tuple was anchored
    :type fail_latency_ns: float
    :param fail_latency_ns: fail latency in nano seconds
    """
    if len(self.task_hooks) > 0:
      spout_fail_info = SpoutFailInfo(message_id=message_id,
                                      spout_task_id=self.get_task_id(),
                                      fail_latency_ms=fail_latency_ns * system_constants.NS_TO_MS)
      for task_hook in self.task_hooks:
        task_hook.spout_fail(spout_fail_info)

def invoke_hook_spout_fail ( self , message_id , fail_latency_ns ) : if len ( self . task_hooks ) > 0 : spout_fail_info = SpoutFailInfo ( message_id = message_id , spout_task_id = self . get_task_id ( ) , fail_latency_ms = fail_latency_ns * system_constants . NS_TO_MS ) for task_hook in self . task_hooks : task_hook . spout_fail ( spout_fail_info )


def invoke_hook_bolt_execute(self, heron_tuple, execute_latency_ns):
    """invoke task hooks for every time bolt processes a tuple

    :type heron_tuple: HeronTuple
    :param heron_tuple: tuple that is executed
    :type execute_latency_ns: float
    :param execute_latency_ns: execute latency in nano seconds
    """
    if len(self.task_hooks) > 0:
      bolt_execute_info = \
        BoltExecuteInfo(heron_tuple=heron_tuple,
                        executing_task_id=self.get_task_id(),
                        execute_latency_ms=execute_latency_ns * system_constants.NS_TO_MS)
      for task_hook in self.task_hooks:
        task_hook.bolt_execute(bolt_execute_info)

def invoke_hook_bolt_execute ( self , heron_tuple , execute_latency_ns ) : if len ( self . task_hooks ) > 0 : bolt_execute_info = BoltExecuteInfo ( heron_tuple = heron_tuple , executing_task_id = self . get_task_id ( ) , execute_latency_ms = execute_latency_ns * system_constants . NS_TO_MS ) for task_hook in self . task_hooks : task_hook . bolt_execute ( bolt_execute_info )


def invoke_hook_bolt_ack(self, heron_tuple, process_latency_ns):
    """invoke task hooks for every time bolt acks a tuple

    :type heron_tuple: HeronTuple
    :param heron_tuple: tuple that is acked
    :type process_latency_ns: float
    :param process_latency_ns: process latency in nano seconds
    """
    if len(self.task_hooks) > 0:
      bolt_ack_info = BoltAckInfo(heron_tuple=heron_tuple,
                                  acking_task_id=self.get_task_id(),
                                  process_latency_ms=process_latency_ns * system_constants.NS_TO_MS)
      for task_hook in self.task_hooks:
        task_hook.bolt_ack(bolt_ack_info)

def invoke_hook_bolt_ack ( self , heron_tuple , process_latency_ns ) : if len ( self . task_hooks ) > 0 : bolt_ack_info = BoltAckInfo ( heron_tuple = heron_tuple , acking_task_id = self . get_task_id ( ) , process_latency_ms = process_latency_ns * system_constants . NS_TO_MS ) for task_hook in self . task_hooks : task_hook . bolt_ack ( bolt_ack_info )


def invoke_hook_bolt_fail(self, heron_tuple, fail_latency_ns):
    """invoke task hooks for every time bolt fails a tuple

    :type heron_tuple: HeronTuple
    :param heron_tuple: tuple that is failed
    :type fail_latency_ns: float
    :param fail_latency_ns: fail latency in nano seconds
    """
    if len(self.task_hooks) > 0:
      bolt_fail_info = BoltFailInfo(heron_tuple=heron_tuple,
                                    failing_task_id=self.get_task_id(),
                                    fail_latency_ms=fail_latency_ns * system_constants.NS_TO_MS)
      for task_hook in self.task_hooks:
        task_hook.bolt_fail(bolt_fail_info)

def invoke_hook_bolt_fail ( self , heron_tuple , fail_latency_ns ) : if len ( self . task_hooks ) > 0 : bolt_fail_info = BoltFailInfo ( heron_tuple = heron_tuple , failing_task_id = self . get_task_id ( ) , fail_latency_ms = fail_latency_ns * system_constants . NS_TO_MS ) for task_hook in self . task_hooks : task_hook . bolt_fail ( bolt_fail_info )


def submit_fatjar(cl_args, unknown_args, tmp_dir):
  '''
  We use the packer to make a package for the jar and dump it
  to a well-known location. We then run the main method of class
  with the specified arguments. We pass arguments as an environment variable HERON_OPTIONS.

  This will run the jar file with the topology_class_name. The submitter
  inside will write out the topology defn file to a location that
  we specify. Then we write the topology defn file to a well known
  location. We then write to appropriate places in zookeeper
  and launch the scheduler jobs
  :param cl_args:
  :param unknown_args:
  :param tmp_dir:
  :return:
  '''
  # execute main of the topology to create the topology definition
  topology_file = cl_args['topology-file-name']

  main_class = cl_args['topology-class-name']

  res = execute.heron_class(
      class_name=main_class,
      lib_jars=config.get_heron_libs(jars.topology_jars()),
      extra_jars=[topology_file],
      args=tuple(unknown_args),
      java_defines=cl_args['topology_main_jvm_property'])

  result.render(res)

  if not result.is_successful(res):
    err_context = ("Failed to create topology definition " \
      "file when executing class '%s' of file '%s'") % (main_class, topology_file)
    res.add_context(err_context)
    return res

  results = launch_topologies(cl_args, topology_file, tmp_dir)

  return results

We use the packer to make a package for the jar and dump it to a well - known location . We then run the main method of class with the specified arguments . We pass arguments as an environment variable HERON_OPTIONS .


def submit_tar(cl_args, unknown_args, tmp_dir):
  '''
  Extract and execute the java files inside the tar and then add topology
  definition file created by running submitTopology

  We use the packer to make a package for the tar and dump it
  to a well-known location. We then run the main method of class
  with the specified arguments. We pass arguments as an environment variable HERON_OPTIONS.
  This will run the jar file with the topology class name.

  The submitter inside will write out the topology defn file to a location
  that we specify. Then we write the topology defn file to a well known
  packer location. We then write to appropriate places in zookeeper
  and launch the aurora jobs
  :param cl_args:
  :param unknown_args:
  :param tmp_dir:
  :return:
  '''
  # execute main of the topology to create the topology definition
  topology_file = cl_args['topology-file-name']
  java_defines = cl_args['topology_main_jvm_property']
  main_class = cl_args['topology-class-name']
  res = execute.heron_tar(
      main_class,
      topology_file,
      tuple(unknown_args),
      tmp_dir,
      java_defines)

  result.render(res)

  if not result.is_successful(res):
    err_context = ("Failed to create topology definition " \
      "file when executing class '%s' of file '%s'") % (main_class, topology_file)
    res.add_context(err_context)
    return res

  return launch_topologies(cl_args, topology_file, tmp_dir)


def setup(self, context):
    """Implements TextFile Generator's setup method"""
    myindex = context.get_partition_index()
    self._files_to_consume = self._files[myindex::context.get_num_partitions()]
    self.logger.info("TextFileSpout files to consume %s" % self._files_to_consume)
    self._lines_to_consume = self._get_next_lines()
    self._emit_count = 0

def
setup
(
self
,
context
)
:
myindex
=
context
.
get_partition_index
(
)
self
.
_files_to_consume
=
self
.
_files
[
myindex
:
:
context
.
get_num_partitions
(
)
]
self
.
logger
.
info
(
"TextFileSpout files to consume %s"
%
self
.
_files_to_consume
)
self
.
_lines_to_consume
=
self
.
_get_next_lines
(
)
self
.
_emit_count
=
0

Implements TextFile Generator s setup method


def add_verbose(parser):
  """ add optional verbose argument"""
  parser.add_argument(
      '--verbose',
      metavar='(a boolean; default: "false")',
      type=bool,
      default=False)
  return parser

def
add_verbose
(
parser
)
:
parser
.
add_argument
(
'--verbose'
,
metavar
=
'(a boolean; default: "false")'
,
type
=
bool
,
default
=
False
)
return
parser


def add_tracker_url(parser):
  """ add optional tracker_url argument """
  parser.add_argument(
      '--tracker_url',
      metavar='(tracker url; default: "' + DEFAULT_TRACKER_URL + '")',
      type=str, default=DEFAULT_TRACKER_URL)
  return parser

def
add_tracker_url
(
parser
)
:
parser
.
add_argument
(
'--tracker_url'
,
metavar
=
'(tracker url; default: "'
+
DEFAULT_TRACKER_URL
+
'")'
,
type
=
str
,
default
=
DEFAULT_TRACKER_URL
)
return
parser


def hex_escape(bin_str):
  """
  Hex encode a binary string
  """
  printable = string.ascii_letters + string.digits + string.punctuation + ' '
  return ''.join(ch if ch in printable else r'0x{0:02x}'.format(ord(ch)) for ch in bin_str)

def
hex_escape
(
bin_str
)
:
printable
=
string
.
ascii_letters
+
string
.
digits
+
string
.
punctuation
+
' '
return
''
.
join
(
ch
if
ch
in
printable
else
r'0x{0:02x}'
.
format
(
ord
(
ch
)
)
for
ch
in
bin_str
)


def make_shell_endpoint(topologyInfo, instance_id):
  """
  Makes the http endpoint for the heron shell
  if shell port is present, otherwise returns None.
  """
  # Format: container_<id>_<instance_id>
  pplan = topologyInfo["physical_plan"]
  stmgrId = pplan["instances"][instance_id]["stmgrId"]
  host = pplan["stmgrs"][stmgrId]["host"]
  shell_port = pplan["stmgrs"][stmgrId]["shell_port"]
  return "http://%s:%d" % (host, shell_port)

def
make_shell_endpoint
(
topologyInfo
,
instance_id
)
:
# Format: container_<id>_<instance_id>
pplan
=
topologyInfo
[
"physical_plan"
]
stmgrId
=
pplan
[
"instances"
]
[
instance_id
]
[
"stmgrId"
]
host
=
pplan
[
"stmgrs"
]
[
stmgrId
]
[
"host"
]
shell_port
=
pplan
[
"stmgrs"
]
[
stmgrId
]
[
"shell_port"
]
return
"http://%s:%d"
%
(
host
,
shell_port
)

Makes the http endpoint for the heron shell if shell port is present otherwise returns None .


def make_shell_logfiles_url(host, shell_port, _, instance_id=None):
  """
  Make the url for log-files in heron-shell
  from the info stored in stmgr.
  If no instance_id is provided, the link will
  be to the dir for the whole container.
  If shell port is not present, it returns None.
  """
  if not shell_port:
    return None
  if not instance_id:
    return "http://%s:%d/browse/log-files" % (host, shell_port)
  else:
    return "http://%s:%d/file/log-files/%s.log.0" % (host, shell_port, instance_id)

def make_shell_logfiles_url ( host , shell_port , _ , instance_id = None ) : if not shell_port : return None if not instance_id : return "http://%s:%d/browse/log-files" % ( host , shell_port ) else : return "http://%s:%d/file/log-files/%s.log.0" % ( host , shell_port , instance_id )

Make the url for log - files in heron - shell from the info stored in stmgr . If no instance_id is provided the link will be to the dir for the whole container . If shell port is not present it returns None .


def make_shell_logfile_data_url(host, shell_port, instance_id, offset, length):
  """
  Make the url for log-file data in heron-shell
  from the info stored in stmgr.
  """
  return "http://%s:%d/filedata/log-files/%s.log.0?offset=%s&length=%s" % \
    (host, shell_port, instance_id, offset, length)

Make the url for log - file data in heron - shell from the info stored in stmgr .


def create_packet(reqid, message):
    """Creates Outgoing Packet from a given reqid and message

    :param reqid: REQID object
    :param message: protocol buffer object
    """
    assert message.IsInitialized()
    packet = ''

    # calculate the totla size of the packet incl. header
    typename = message.DESCRIPTOR.full_name

    datasize = HeronProtocol.get_size_to_pack_string(typename) + \
               REQID.REQID_SIZE + HeronProtocol.get_size_to_pack_message(message)

    # first write out how much data is there as the header
    packet += HeronProtocol.pack_int(datasize)

    # next write the type string
    packet += HeronProtocol.pack_int(len(typename))
    packet += typename

    # reqid
    packet += reqid.pack()

    # add the proto
    packet += HeronProtocol.pack_int(message.ByteSize())
    packet += message.SerializeToString()
    return OutgoingPacket(packet)


def send(self, dispatcher):
    """Sends this outgoing packet to dispatcher's socket"""
    if self.sent_complete:
      return

    sent = dispatcher.send(self.to_send)
    self.to_send = self.to_send[sent:]

def send ( self , dispatcher ) : if self . sent_complete : return sent = dispatcher . send ( self . to_send ) self . to_send = self . to_send [ sent : ]

Sends this outgoing packet to dispatcher s socket


def create_packet(header, data):
    """Creates an IncomingPacket object from header and data

    This method is for testing purposes
    """
    packet = IncomingPacket()
    packet.header = header
    packet.data = data

    if len(header) == HeronProtocol.HEADER_SIZE:
      packet.is_header_read = True
      if len(data) == packet.get_datasize():
        packet.is_complete = True

    return packet

def create_packet ( header , data ) : packet = IncomingPacket ( ) packet . header = header packet . data = data if len ( header ) == HeronProtocol . HEADER_SIZE : packet . is_header_read = True if len ( data ) == packet . get_datasize ( ) : packet . is_complete = True return packet


def read(self, dispatcher):
    """Reads incoming data from asyncore.dispatcher"""
    try:
      if not self.is_header_read:
        # try reading header
        to_read = HeronProtocol.HEADER_SIZE - len(self.header)
        self.header += dispatcher.recv(to_read)
        if len(self.header) == HeronProtocol.HEADER_SIZE:
          self.is_header_read = True
        else:
          Log.debug("Header read incomplete; read %d bytes of header" % len(self.header))
          return

      if self.is_header_read and not self.is_complete:
        # try reading data
        to_read = self.get_datasize() - len(self.data)
        self.data += dispatcher.recv(to_read)
        if len(self.data) == self.get_datasize():
          self.is_complete = True
    except socket.error as e:
      if e.errno == socket.errno.EAGAIN or e.errno == socket.errno.EWOULDBLOCK:
        # Try again later -> call continue_read later
        Log.debug("Try again error")
      else:
        # Fatal error
        Log.debug("Fatal error when reading IncomingPacket")
        raise RuntimeError("Fatal error occured in IncomingPacket.read()")

Reads incoming data from asyncore . dispatcher


def generate():
    """Generates a random REQID for request"""
    data_bytes = bytearray(random.getrandbits(8) for i in range(REQID.REQID_SIZE))
    return REQID(data_bytes)

def generate ( ) : data_bytes = bytearray ( random . getrandbits ( 8 ) for i in range ( REQID . REQID_SIZE ) ) return REQID ( data_bytes )


def yaml_config_reader(config_path):
  """Reads yaml config file and returns auto-typed config_dict"""
  if not config_path.endswith(".yaml"):
    raise ValueError("Config file not yaml")

  with open(config_path, 'r') as f:
    config = yaml.load(f)

  return config

def
yaml_config_reader
(
config_path
)
:
if
not
config_path
.
endswith
(
".yaml"
)
:
raise
ValueError
(
"Config file not yaml"
)
with
open
(
config_path
,
'r'
)
as
f
:
config
=
yaml
.
load
(
f
)
return
config

Reads yaml config file and returns auto - typed config_dict


def send_buffered_messages(self):
    """Send messages in out_stream to the Stream Manager"""
    while not self.out_stream.is_empty() and self._stmgr_client.is_registered:
      tuple_set = self.out_stream.poll()
      if isinstance(tuple_set, tuple_pb2.HeronTupleSet):
        tuple_set.src_task_id = self.my_pplan_helper.my_task_id
        self.gateway_metrics.update_sent_packet(tuple_set.ByteSize())
      self._stmgr_client.send_message(tuple_set)

def send_buffered_messages ( self ) : while not self . out_stream . is_empty ( ) and self . _stmgr_client . is_registered : tuple_set = self . out_stream . poll ( ) if isinstance ( tuple_set , tuple_pb2 . HeronTupleSet ) : tuple_set . src_task_id = self . my_pplan_helper . my_task_id self . gateway_metrics . update_sent_packet ( tuple_set . ByteSize ( ) ) self . _stmgr_client . send_message ( tuple_set )


def _handle_state_change_msg(self, new_helper):
    """Called when state change is commanded by stream manager"""
    assert self.my_pplan_helper is not None
    assert self.my_instance is not None and self.my_instance.py_class is not None

    if self.my_pplan_helper.get_topology_state() != new_helper.get_topology_state():
      # handle state change
      # update the pplan_helper
      self.my_pplan_helper = new_helper
      if new_helper.is_topology_running():
        if not self.is_instance_started:
          self.start_instance_if_possible()
        self.my_instance.py_class.invoke_activate()
      elif new_helper.is_topology_paused():
        self.my_instance.py_class.invoke_deactivate()
      else:
        raise RuntimeError("Unexpected TopologyState update: %s" % new_helper.get_topology_state())
    else:
      Log.info("Topology state remains the same.")

def
_handle_state_change_msg
(
self
,
new_helper
)
:
assert
self
.
my_pplan_helper
is
not
assert
self
.
my_instance
is
not
and
self
.
my_instance
.
py_class
is
not
if
self
.
my_pplan_helper
.
get_topology_state
(
)
!=
new_helper
.
get_topology_state
(
)
:
# handle state change
# update the pplan_helper
self
.
my_pplan_helper
=
new_helper
if
new_helper
.
is_topology_running
(
)
:
if
not
self
.
is_instance_started
:
self
.
start_instance_if_possible
(
)
self
.
my_instance
.
py_class
.
invoke_activate
(
)
elif
new_helper
.
is_topology_paused
(
)
:
self
.
my_instance
.
py_class
.
invoke_deactivate
(
)
else
:
raise
RuntimeError
(
"Unexpected TopologyState update: %s"
%
new_helper
.
get_topology_state
(
)
)
else
:
Log
.
info
(
"Topology state remains the same."
)


def check_output_schema(self, stream_id, tup):
    """Checks if a given stream_id and tuple matches with the output schema

    :type stream_id: str
    :param stream_id: stream id into which tuple is sent
    :type tup: list
    :param tup: tuple that is going to be sent
    """
    # do some checking to make sure that the number of fields match what's expected
    size = self._output_schema.get(stream_id, None)
    if size is None:
      raise RuntimeError("%s emitting to stream %s but was not declared in output fields"
                         % (self.my_component_name, stream_id))
    elif size != len(tup):
      raise RuntimeError("Number of fields emitted in stream %s does not match what's expected. "
                         "Expected: %s, Observed: %s" % (stream_id, size, len(tup)))


def get_topology_config(self):
    """Returns the topology config"""
    if self.pplan.topology.HasField("topology_config"):
      return self._get_dict_from_config(self.pplan.topology.topology_config)
    else:
      return {}

def get_topology_config ( self ) : if self . pplan . topology . HasField ( "topology_config" ) : return self . _get_dict_from_config ( self . pplan . topology . topology_config ) else : return { }


def set_topology_context(self, metrics_collector):
    """Sets a new topology context"""
    Log.debug("Setting topology context")
    cluster_config = self.get_topology_config()
    cluster_config.update(self._get_dict_from_config(self.my_component.config))
    task_to_component_map = self._get_task_to_comp_map()
    self.context = TopologyContextImpl(cluster_config, self.pplan.topology, task_to_component_map,
                                       self.my_task_id, metrics_collector,
                                       self.topology_pex_abs_path)

def
set_topology_context
(
self
,
metrics_collector
)
:
Log
.
debug
(
"Setting topology context"
)
cluster_config
=
self
.
get_topology_config
(
)
cluster_config
.
update
(
self
.
_get_dict_from_config
(
self
.
my_component
.
config
)
)
task_to_component_map
=
self
.
_get_task_to_comp_map
(
)
self
.
context
=
TopologyContextImpl
(
cluster_config
,
self
.
pplan
.
topology
,
task_to_component_map
,
self
.
my_task_id
,
metrics_collector
,
self
.
topology_pex_abs_path
)


def _get_dict_from_config(topology_config):
    """Converts Config protobuf message to python dictionary

    Values are converted according to the rules below:

    - Number string (e.g. "12" or "1.2") is appropriately converted to ``int`` or ``float``
    - Boolean string ("true", "True", "false" or "False") is converted to built-in boolean type
      (i.e. ``True`` or ``False``)
    - Normal string is inserted to dict as is
    - Serialized value is deserialized and inserted as a corresponding Python object
    """
    config = {}
    for kv in topology_config.kvs:
      if kv.HasField("value"):
        assert kv.type == topology_pb2.ConfigValueType.Value("STRING_VALUE")
        # value is string
        if PhysicalPlanHelper._is_number(kv.value):
          config[kv.key] = PhysicalPlanHelper._get_number(kv.value)
        elif kv.value.lower() in ("true", "false"):
          config[kv.key] = True if kv.value.lower() == "true" else False
        else:
          config[kv.key] = kv.value
      elif kv.HasField("serialized_value") and \
        kv.type == topology_pb2.ConfigValueType.Value("PYTHON_SERIALIZED_VALUE"):
        # deserialize that
        config[kv.key] = default_serializer.deserialize(kv.serialized_value)
      else:
        assert kv.HasField("type")
        Log.error("Unsupported config <key:value> found: %s, with type: %s"
                  % (str(kv), str(kv.type)))
        continue

    return config


def _setup_custom_grouping(self, topology):
    """Checks whether there are any bolts that consume any of my streams using custom grouping"""
    for i in range(len(topology.bolts)):
      for in_stream in topology.bolts[i].inputs:
        if in_stream.stream.component_name == self.my_component_name and \
          in_stream.gtype == topology_pb2.Grouping.Value("CUSTOM"):
          # this bolt takes my output in custom grouping manner
          if in_stream.type == topology_pb2.CustomGroupingObjectType.Value("PYTHON_OBJECT"):
            custom_grouping_obj = default_serializer.deserialize(in_stream.custom_grouping_object)
            if isinstance(custom_grouping_obj, str):
              pex_loader.load_pex(self.topology_pex_abs_path)
              grouping_cls = \
                pex_loader.import_and_get_class(self.topology_pex_abs_path, custom_grouping_obj)
              custom_grouping_obj = grouping_cls()
            assert isinstance(custom_grouping_obj, ICustomGrouping)
            self.custom_grouper.add(in_stream.stream.id,
                                    self._get_taskids_for_component(topology.bolts[i].comp.name),
                                    custom_grouping_obj,
                                    self.my_component_name)

          elif in_stream.type == topology_pb2.CustomGroupingObjectType.Value("JAVA_OBJECT"):
            raise NotImplementedError("Java-serialized custom grouping is not yet supported "
                                      "for python topology")
          else:
            raise ValueError("Unrecognized custom grouping type found: %s" % str(in_stream.type))


def add(self, stream_id, task_ids, grouping, source_comp_name):
    """Adds the target component

    :type stream_id: str
    :param stream_id: stream id into which tuples are emitted
    :type task_ids: list of str
    :param task_ids: list of task ids to which tuples are emitted
    :type grouping: ICustomStreamGrouping object
    :param grouping: custom grouping to use
    :type source_comp_name: str
    :param source_comp_name: source component name
    """
    if stream_id not in self.targets:
      self.targets[stream_id] = []
    self.targets[stream_id].append(Target(task_ids, grouping, source_comp_name))

def add ( self , stream_id , task_ids , grouping , source_comp_name ) : if stream_id not in self . targets : self . targets [ stream_id ] = [ ] self . targets [ stream_id ] . append ( Target ( task_ids , grouping , source_comp_name ) )


def prepare(self, context):
    """Prepares the custom grouping for this component"""
    for stream_id, targets in self.targets.items():
      for target in targets:
        target.prepare(context, stream_id)

def prepare ( self , context ) : for stream_id , targets in self . targets . items ( ) : for target in targets : target . prepare ( context , stream_id )


def choose_tasks(self, stream_id, values):
    """Choose tasks for a given stream_id and values and Returns a list of target tasks"""
    if stream_id not in self.targets:
      return []

    ret = []
    for target in self.targets[stream_id]:
      ret.extend(target.choose_tasks(values))
    return ret

def choose_tasks ( self , stream_id , values ) : if stream_id not in self . targets : return [ ] ret = [ ] for target in self . targets [ stream_id ] : ret . extend ( target . choose_tasks ( values ) ) return ret


def format_mode(sres):
  """
  Format a line in the directory list based on the file's type and other attributes.
  """
  mode = sres.st_mode

  root = (mode & 0o700) >> 6
  group = (mode & 0o070) >> 3
  user = (mode & 0o7)

  def stat_type(md):
    ''' stat type'''
    if stat.S_ISDIR(md):
      return 'd'
    elif stat.S_ISSOCK(md):
      return 's'
    else:
      return '-'

  def triple(md):
    ''' triple '''
    return '%c%c%c' % (
        'r' if md & 0b100 else '-',
        'w' if md & 0b010 else '-',
        'x' if md & 0b001 else '-')

  return ''.join([stat_type(mode), triple(root), triple(group), triple(user)])

def
format_mode
(
sres
)
:
mode
=
sres
.
st_mode
root
=
(
mode
&
0o700
)
>>
6
group
=
(
mode
&
0o070
)
>>
3
user
=
(
mode
&
0o7
)
def
stat_type
(
md
)
:
''' stat type'''
if
stat
.
S_ISDIR
(
md
)
:
return
'd'
elif
stat
.
S_ISSOCK
(
md
)
:
return
's'
else
:
return
'-'
def
triple
(
md
)
:
''' triple '''
return
'%c%c%c'
%
(
'r'
if
md
&
0b100
else
'-'
,
'w'
if
md
&
0b010
else
'-'
,
'x'
if
md
&
0b001
else
'-'
)
return
''
.
join
(
[
stat_type
(
mode
)
,
triple
(
root
)
,
triple
(
group
)
,
triple
(
user
)
]
)

Format a line in the directory list based on the file s type and other attributes .


def format_mtime(mtime):
  """
  Format the date associated with a file to be displayed in directory listing.
  """
  now = datetime.now()
  dt = datetime.fromtimestamp(mtime)
  return '%s %2d %5s' % (
      dt.strftime('%b'), dt.day,
      dt.year if dt.year != now.year else dt.strftime('%H:%M'))

def
format_mtime
(
mtime
)
:
now
=
datetime
.
now
(
)
dt
=
datetime
.
fromtimestamp
(
mtime
)
return
'%s %2d %5s'
%
(
dt
.
strftime
(
'%b'
)
,
dt
.
day
,
dt
.
year
if
dt
.
year
!=
now
.
year
else
dt
.
strftime
(
'%H:%M'
)
)

Format the date associated with a file to be displayed in directory listing .


def format_prefix(filename, sres):
  """
  Prefix to a filename in the directory listing. This is to make the
  listing similar to an output of "ls -alh".
  """
  try:
    pwent = pwd.getpwuid(sres.st_uid)
    user = pwent.pw_name
  except KeyError:
    user = sres.st_uid

  try:
    grent = grp.getgrgid(sres.st_gid)
    group = grent.gr_name
  except KeyError:
    group = sres.st_gid

  return '%s %3d %10s %10s %10d %s' % (
      format_mode(sres),
      sres.st_nlink,
      user,
      group,
      sres.st_size,
      format_mtime(sres.st_mtime),
  )

def
format_prefix
(
filename
,
sres
)
:
try
:
pwent
=
pwd
.
getpwuid
(
sres
.
st_uid
)
user
=
pwent
.
pw_name
except
KeyError
:
user
=
sres
.
st_uid
try
:
grent
=
grp
.
getgrgid
(
sres
.
st_gid
)
group
=
grent
.
gr_name
except
KeyError
:
group
=
sres
.
st_gid
return
'%s %3d %10s %10s %10d %s'
%
(
format_mode
(
sres
)
,
sres
.
st_nlink
,
user
,
group
,
sres
.
st_size
,
format_mtime
(
sres
.
st_mtime
)
,
)

Prefix to a filename in the directory listing . This is to make the listing similar to an output of ls - alh .


def read_chunk(filename, offset=-1, length=-1, escape_data=False):
  """
  Read a chunk of a file from an offset upto the length.
  """
  try:
    length = int(length)
    offset = int(offset)
  except ValueError:
    return {}

  if not os.path.isfile(filename):
    return {}

  try:
    fstat = os.stat(filename)
  except Exception:
    return {}

  if offset == -1:
    offset = fstat.st_size

  if length == -1:
    length = fstat.st_size - offset

  with open(filename, "r") as fp:
    fp.seek(offset)
    try:
      data = fp.read(length)
    except IOError:
      return {}

  if data:
    data = _escape_data(data) if escape_data else data
    return dict(offset=offset, length=len(data), data=data)

  return dict(offset=offset, length=0)

def read_chunk ( filename , offset = - 1 , length = - 1 , escape_data = False ) : try : length = int ( length ) offset = int ( offset ) except ValueError : return { } if not os . path . isfile ( filename ) : return { } try : fstat = os . stat ( filename ) except Exception : return { } if offset == - 1 : offset = fstat . st_size if length == - 1 : length = fstat . st_size - offset with open ( filename , "r" ) as fp : fp . seek ( offset ) try : data = fp . read ( length ) except IOError : return { } if data : data = _escape_data ( data ) if escape_data else data return dict ( offset = offset , length = len ( data ) , data = data ) return dict ( offset = offset , length = 0 )

Read a chunk of a file from an offset upto the length .


def pipe(prev_proc, to_cmd):
  """
  Pipes output of prev_proc into to_cmd.
  Returns piped process
  """
  stdin = None if prev_proc is None else prev_proc.stdout
  process = subprocess.Popen(to_cmd,
                             stdout=subprocess.PIPE,
                             stdin=stdin)
  if prev_proc is not None:
    prev_proc.stdout.close() # Allow prev_proc to receive a SIGPIPE
  return process

def
pipe
(
prev_proc
,
to_cmd
)
:
stdin
=
if
prev_proc
is
else
prev_proc
.
stdout
process
=
subprocess
.
Popen
(
to_cmd
,
stdout
=
subprocess
.
PIPE
,
stdin
=
stdin
)
if
prev_proc
is
not
:
prev_proc
.
stdout
.
close
(
)
# Allow prev_proc to receive a SIGPIPE
return
process

Pipes output of prev_proc into to_cmd . Returns piped process


def str_cmd(cmd, cwd, env):
  """
  Runs the command and returns its stdout and stderr.
  """
  process = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE, cwd=cwd, env=env)
  stdout_builder, stderr_builder = proc.async_stdout_stderr_builder(process)
  process.wait()
  stdout, stderr = stdout_builder.result(), stderr_builder.result()
  return {'command': ' '.join(cmd), 'stderr': stderr, 'stdout': stdout}

def
str_cmd
(
cmd
,
cwd
,
env
)
:
process
=
subprocess
.
Popen
(
cmd
,
stdout
=
subprocess
.
PIPE
,
stderr
=
subprocess
.
PIPE
,
cwd
=
cwd
,
env
=
env
)
stdout_builder
,
stderr_builder
=
proc
.
async_stdout_stderr_builder
(
process
)
process
.
wait
(
)
stdout
,
stderr
=
stdout_builder
.
result
(
)
,
stderr_builder
.
result
(
)
return
{
'command'
:
' '
.
join
(
cmd
)
,
'stderr'
:
stderr
,
'stdout'
:
stdout
}

Runs the command and returns its stdout and stderr .


def chain(cmd_list):
  """
  Feed output of one command to the next and return final output
  Returns string output of chained application of commands.
  """
  command = ' | '.join(map(lambda x: ' '.join(x), cmd_list))
  chained_proc = functools.reduce(pipe, [None] + cmd_list)
  stdout_builder = proc.async_stdout_builder(chained_proc)
  chained_proc.wait()
  return {
      'command': command,
      'stdout': stdout_builder.result()
  }

def
chain
(
cmd_list
)
:
command
=
' | '
.
join
(
map
(
lambda
x
:
' '
.
join
(
x
)
,
cmd_list
)
)
chained_proc
=
functools
.
reduce
(
pipe
,
[
]
+
cmd_list
)
stdout_builder
=
proc
.
async_stdout_builder
(
chained_proc
)
chained_proc
.
wait
(
)
return
{
'command'
:
command
,
'stdout'
:
stdout_builder
.
result
(
)
}

Feed output of one command to the next and return final output Returns string output of chained application of commands .


def parse_topo_loc(cl_args):
  """ parse topology location """
  try:
    topo_loc = cl_args['cluster/[role]/[env]'].split('/')
    topo_name = cl_args['topology-name']
    topo_loc.append(topo_name)
    if len(topo_loc) != 4:
      raise
    return topo_loc
  except Exception:
    Log.error('Invalid topology location')
    raise

def
parse_topo_loc
(
cl_args
)
:
try
:
topo_loc
=
cl_args
[
'cluster/[role]/[env]'
]
.
split
(
'/'
)
topo_name
=
cl_args
[
'topology-name'
]
topo_loc
.
append
(
topo_name
)
if
len
(
topo_loc
)
!=
4
:
raise
return
topo_loc
except
Exception
:
Log
.
error
(
'Invalid topology location'
)
raise


def to_table(metrics):
  """ normalize raw metrics API result to table """
  all_queries = tracker_access.metric_queries()
  m = tracker_access.queries_map()
  names = metrics.values()[0].keys()
  stats = []
  for n in names:
    info = [n]
    for field in all_queries:
      try:
        info.append(str(metrics[field][n]))
      except KeyError:
        pass
    stats.append(info)
  header = ['container id'] + [m[k] for k in all_queries if k in metrics.keys()]
  return stats, header

def
to_table
(
metrics
)
:
all_queries
=
tracker_access
.
metric_queries
(
)
m
=
tracker_access
.
queries_map
(
)
names
=
metrics
.
values
(
)
[
0
]
.
keys
(
)
stats
=
[
]
for
n
in
names
:
info
=
[
n
]
for
field
in
all_queries
:
try
:
info
.
append
(
str
(
metrics
[
field
]
[
n
]
)
)
except
KeyError
:
pass
stats
.
append
(
info
)
header
=
[
'container id'
]
+
[
m
[
k
]
for
k
in
all_queries
if
k
in
metrics
.
keys
(
)
]
return
stats
,
header


def run_metrics(command, parser, cl_args, unknown_args):
  """ run metrics subcommand """
  cluster, role, env = cl_args['cluster'], cl_args['role'], cl_args['environ']
  topology = cl_args['topology-name']
  try:
    result = tracker_access.get_topology_info(cluster, env, topology, role)
    spouts = result['physical_plan']['spouts'].keys()
    bolts = result['physical_plan']['bolts'].keys()
    components = spouts + bolts
    cname = cl_args['component']
    if cname:
      if cname in components:
        components = [cname]
      else:
        Log.error('Unknown component: \'%s\'' % cname)
        raise
  except Exception:
    Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"])
    return False
  cresult = []
  for comp in components:
    try:
      metrics = tracker_access.get_component_metrics(comp, cluster, env, topology, role)
    except:
      Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"])
      return False
    stat, header = to_table(metrics)
    cresult.append((comp, stat, header))
  for i, (comp, stat, header) in enumerate(cresult):
    if i != 0:
      print('')
    print('\'%s\' metrics:' % comp)
    print(tabulate(stat, headers=header))
  return True

def
run_metrics
(
command
,
parser
,
cl_args
,
unknown_args
)
:
cluster
,
role
,
env
=
cl_args
[
'cluster'
]
,
cl_args
[
'role'
]
,
cl_args
[
'environ'
]
topology
=
cl_args
[
'topology-name'
]
try
:
result
=
tracker_access
.
get_topology_info
(
cluster
,
env
,
topology
,
role
)
spouts
=
result
[
'physical_plan'
]
[
'spouts'
]
.
keys
(
)
bolts
=
result
[
'physical_plan'
]
[
'bolts'
]
.
keys
(
)
components
=
spouts
+
bolts
cname
=
cl_args
[
'component'
]
if
cname
:
if
cname
in
components
:
components
=
[
cname
]
else
:
Log
.
error
(
'Unknown component: \'%s\''
%
cname
)
raise
except
Exception
:
Log
.
error
(
"Fail to connect to tracker: \'%s\'"
,
cl_args
[
"tracker_url"
]
)
return
False
cresult
=
[
]
for
comp
in
components
:
try
:
metrics
=
tracker_access
.
get_component_metrics
(
comp
,
cluster
,
env
,
topology
,
role
)
except
:
Log
.
error
(
"Fail to connect to tracker: \'%s\'"
,
cl_args
[
"tracker_url"
]
)
return
False
stat
,
header
=
to_table
(
metrics
)
cresult
.
append
(
(
comp
,
stat
,
header
)
)
for
i
,
(
comp
,
stat
,
header
)
in
enumerate
(
cresult
)
:
if
i
!=
0
:
print
(
''
)
print
(
'\'%s\' metrics:'
%
comp
)
print
(
tabulate
(
stat
,
headers
=
header
)
)
return
True


def run_bolts(command, parser, cl_args, unknown_args):
  """ run bolts subcommand """
  cluster, role, env = cl_args['cluster'], cl_args['role'], cl_args['environ']
  topology = cl_args['topology-name']
  try:
    result = tracker_access.get_topology_info(cluster, env, topology, role)
    bolts = result['physical_plan']['bolts'].keys()
    bolt_name = cl_args['bolt']
    if bolt_name:
      if bolt_name in bolts:
        bolts = [bolt_name]
      else:
        Log.error('Unknown bolt: \'%s\'' % bolt_name)
        raise
  except Exception:
    Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"])
    return False
  bolts_result = []
  for bolt in bolts:
    try:
      metrics = tracker_access.get_component_metrics(bolt, cluster, env, topology, role)
      stat, header = to_table(metrics)
      bolts_result.append((bolt, stat, header))
    except Exception:
      Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"])
      return False
  for i, (bolt, stat, header) in enumerate(bolts_result):
    if i != 0:
      print('')
    print('\'%s\' metrics:' % bolt)
    print(tabulate(stat, headers=header))
  return True

def
run_bolts
(
command
,
parser
,
cl_args
,
unknown_args
)
:
cluster
,
role
,
env
=
cl_args
[
'cluster'
]
,
cl_args
[
'role'
]
,
cl_args
[
'environ'
]
topology
=
cl_args
[
'topology-name'
]
try
:
result
=
tracker_access
.
get_topology_info
(
cluster
,
env
,
topology
,
role
)
bolts
=
result
[
'physical_plan'
]
[
'bolts'
]
.
keys
(
)
bolt_name
=
cl_args
[
'bolt'
]
if
bolt_name
:
if
bolt_name
in
bolts
:
bolts
=
[
bolt_name
]
else
:
Log
.
error
(
'Unknown bolt: \'%s\''
%
bolt_name
)
raise
except
Exception
:
Log
.
error
(
"Fail to connect to tracker: \'%s\'"
,
cl_args
[
"tracker_url"
]
)
return
False
bolts_result
=
[
]
for
bolt
in
bolts
:
try
:
metrics
=
tracker_access
.
get_component_metrics
(
bolt
,
cluster
,
env
,
topology
,
role
)
stat
,
header
=
to_table
(
metrics
)
bolts_result
.
append
(
(
bolt
,
stat
,
header
)
)
except
Exception
:
Log
.
error
(
"Fail to connect to tracker: \'%s\'"
,
cl_args
[
"tracker_url"
]
)
return
False
for
i
,
(
bolt
,
stat
,
header
)
in
enumerate
(
bolts_result
)
:
if
i
!=
0
:
print
(
''
)
print
(
'\'%s\' metrics:'
%
bolt
)
print
(
tabulate
(
stat
,
headers
=
header
)
)
return
True


def run_containers(command, parser, cl_args, unknown_args):
  """ run containers subcommand """
  cluster, role, env = cl_args['cluster'], cl_args['role'], cl_args['environ']
  topology = cl_args['topology-name']
  container_id = cl_args['id']
  try:
    result = tracker_access.get_topology_info(cluster, env, topology, role)
  except:
    Log.error("Fail to connect to tracker: \'%s\'", cl_args["tracker_url"])
    return False
  containers = result['physical_plan']['stmgrs']
  all_bolts, all_spouts = set(), set()
  for _, bolts in result['physical_plan']['bolts'].items():
    all_bolts = all_bolts | set(bolts)
  for _, spouts in result['physical_plan']['spouts'].items():
    all_spouts = all_spouts | set(spouts)
  stmgrs = containers.keys()
  stmgrs.sort()
  if container_id is not None:
    try:
      normalized_cid = container_id - 1
      if normalized_cid < 0:
        raise
      stmgrs = [stmgrs[normalized_cid]]
    except:
      Log.error('Invalid container id: %d' % container_id)
      return False
  table = []
  for sid, name in enumerate(stmgrs):
    cid = sid + 1
    host = containers[name]["host"]
    port = containers[name]["port"]
    pid = containers[name]["pid"]
    instances = containers[name]["instance_ids"]
    bolt_nums = len([instance for instance in instances if instance in all_bolts])
    spout_nums = len([instance for instance in instances if instance in all_spouts])
    table.append([cid, host, port, pid, bolt_nums, spout_nums, len(instances)])
  headers = ["container", "host", "port", "pid", "#bolt", "#spout", "#instance"]
  sys.stdout.flush()
  print(tabulate(table, headers=headers))
  return True

def
run_containers
(
command
,
parser
,
cl_args
,
unknown_args
)
:
cluster
,
role
,
env
=
cl_args
[
'cluster'
]
,
cl_args
[
'role'
]
,
cl_args
[
'environ'
]
topology
=
cl_args
[
'topology-name'
]
container_id
=
cl_args
[
'id'
]
try
:
result
=
tracker_access
.
get_topology_info
(
cluster
,
env
,
topology
,
role
)
except
:
Log
.
error
(
"Fail to connect to tracker: \'%s\'"
,
cl_args
[
"tracker_url"
]
)
return
False
containers
=
result
[
'physical_plan'
]
[
'stmgrs'
]
all_bolts
,
all_spouts
=
set
(
)
,
set
(
)
for
_
,
bolts
in
result
[
'physical_plan'
]
[
'bolts'
]
.
items
(
)
:
all_bolts
=
all_bolts
|
set
(
bolts
)
for
_
,
spouts
in
result
[
'physical_plan'
]
[
'spouts'
]
.
items
(
)
:
all_spouts
=
all_spouts
|
set
(
spouts
)
stmgrs
=
containers
.
keys
(
)
stmgrs
.
sort
(
)
if
container_id
is
not
:
try
:
normalized_cid
=
container_id
1
if
normalized_cid
<
0
:
raise
stmgrs
=
[
stmgrs
[
normalized_cid
]
]
except
:
Log
.
error
(
'Invalid container id: %d'
%
container_id
)
return
False
table
=
[
]
for
sid
,
name
in
enumerate
(
stmgrs
)
:
cid
=
sid
+
1
host
=
containers
[
name
]
[
"host"
]
port
=
containers
[
name
]
[
"port"
]
pid
=
containers
[
name
]
[
"pid"
]
instances
=
containers
[
name
]
[
"instance_ids"
]
bolt_nums
=
len
(
[
instance
for
instance
in
instances
if
instance
in
all_bolts
]
)
spout_nums
=
len
(
[
instance
for
instance
in
instances
if
instance
in
all_spouts
]
)
table
.
append
(
[
cid
,
host
,
port
,
pid
,
bolt_nums
,
spout_nums
,
len
(
instances
)
]
)
headers
=
[
"container"
,
"host"
,
"port"
,
"pid"
,
"#bolt"
,
"#spout"
,
"#instance"
]
sys
.
stdout
.
flush
(
)
print
(
tabulate
(
table
,
headers
=
headers
)
)
return
True