#!/usr/bin/env python
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# This script is to be called by Cloudera Manager to collect Breakpad minidump files up to
# a specified date/time. A compressed tarball is created in the user specified location.
# We try to fit as many files as possible into the tarball until a size limit is reached.
# Example invokation by CM to:
#     ./collect_minidumps.py --conf_dir=/var/run/.../5555-impala-STATESTORE/impala-conf \
#       --role_name=statestored --max_output_size=50000000 --end_time=1463033495000 \
#       --output_file_path=/tmp/minidump_package.tar.gz

import os
import re
import sys
import tarfile

from contextlib import closing
from optparse import OptionParser

class FileArchiver(object):
  '''This is a generic class that makes a tarball out of files in the source_dir.  We
  assume that source_dir contains only files. The resulting file will be compressed with
  gzip and placed into output_file_path. If a file with that name already exists, it will
  be deleted and re-created. Max_result_size is the maximum allowed size of the resulting
  tarball. If all files in the source_dir can't fit into the allowed size, most recent
  files will be preferred. start_time and end_time paramenters (in milliseconds UTC) allow
  us to specify an interval of time for which to consider the files.
  '''

  def __init__(self,
      source_dir,
      output_file_path,
      max_output_size,
      start_time=None,
      end_time=None):
    self.source_dir = source_dir
    self.max_output_size = max_output_size
    self.start_time = start_time
    self.end_time = end_time
    self.output_file_path = output_file_path
    # Maps the number of files in the tarball to the resulting size (in bytes).
    self.resulting_sizes = {}
    self.file_list = []

  def _remove_output_file(self):
    try:
      os.remove(self.output_file_path)
    except OSError:
      pass

  def _tar_files(self, num_files=None):
    '''Make a tarball with num_files most recent files in the file_list. Record the
    resulting size into resulting_sizes map and return it.
    '''
    num_files = num_files or len(self.file_list)
    self._remove_output_file()
    if num_files == 0:
      size = 0
    else:
      with closing(tarfile.open(self.output_file_path, mode='w:gz')) as out:
        for i in xrange(num_files):
          out.add(self.file_list[i])
      size = os.stat(self.output_file_path).st_size
    self.resulting_sizes[num_files] = size
    return size

  def _compute_file_list(self):
    '''Computes a sorted list of eligible files in the source directory by filtering out
    files with modified date not in the desired time range. Directories and other
    non-files are ignored.
    '''
    file_list = []
    for f in os.listdir(self.source_dir):
      full_path = os.path.join(self.source_dir, f)
      if not os.path.isfile(full_path):
        continue
      # st_mtime is in seconds UTC, so we need to multiply by 1000 to get milliseconds.
      time_modified = os.stat(full_path).st_mtime * 1000
      if self.start_time and self.start_time > time_modified:
        continue
      if self.end_time and self.end_time < time_modified:
        continue
      file_list.append(full_path)
    self.file_list = sorted(file_list, key=lambda f: os.stat(f).st_mtime, reverse=True)

  def _binary_search(self):
    '''Calculates the maximum number of files that can be collected, such that the tarball
    size is less than max_output_size.
    '''
    min_num = 0
    max_num = len(self.file_list)
    while max_num - min_num > 1:
      mid = (min_num + max_num) // 2
      if self._tar_files(mid) <= self.max_output_size:
        min_num = mid
      else:
        max_num = mid
    return min_num

  def make_tarball(self):
    '''Make a tarball with the maximum number of files such that the size of the tarball
    is less than or equal to max_output_size. Returns a pair (status (int), message
    (str)). status represents the result of the operation and follows the unix convention
    where 0 equals success. message provides additional information. A status of 1 is
    returned if source_dir is not empty and no files were able to fit into the tarball.
    '''
    self._compute_file_list()
    if len(self.file_list) == 0:
      status = 0
      msg = 'No files found in "{0}".'
      return status, msg.format(self.source_dir)
    output_size = self._tar_files()
    if output_size <= self.max_output_size:
      status = 0
      msg = 'Success, archived all {0} files in "{1}".'
      return status, msg.format(len(self.file_list), self.source_dir)
    else:
      max_num_files = self._binary_search()
      if max_num_files == 0:
        self._remove_output_file()
        status = 1
        msg = ('Unable to archive any files in "{0}". '
            'Increase max_output_size to at least {1} bytes.')
        # If max_num_files is 0, we are guaranteed that the binary search tried making a
        # tarball with 1 file.
        return status, msg.format(self.source_dir, self.resulting_sizes[1])
      else:
        self._tar_files(max_num_files)
        status = 0
        msg = 'Success. Archived {0} out of {1} files in "{2}".'
        return status, msg.format(max_num_files, len(self.file_list), self.source_dir)

def get_config_parameter_value(conf_dir, role_name, config_parameter_name):
  '''Extract a single config parameter from the configuration file of a particular
  daemon.
  '''
  ROLE_FLAGFILE_MAP = {
      'impalad': 'impalad_flags',
      'statestored': 'state_store_flags',
      'catalogd': 'catalogserver_flags'}
  config_parameter_value = None
  try:
    file_path = os.path.join(conf_dir, ROLE_FLAGFILE_MAP[role_name])
    with open(file_path, 'r') as f:
      for line in f:
        m = re.match('-{0}=(.*)'.format(config_parameter_name), line)
        if m:
          config_parameter_value = m.group(1)
  except IOError as e:
    print >> sys.stderr, 'Error: Unable to open "{0}".'.format(file_path)
    sys.exit(1)
  return config_parameter_value

def get_minidump_dir(conf_dir, role_name):
  '''Extracts the minidump directory path for a given role from the configuration file.
  The directory defaults to 'minidumps', relative paths are prepended with log_dir, which
  defaults to '/tmp'.
  '''
  minidump_path = get_config_parameter_value(
    conf_dir, role_name, 'minidump_path') or 'minidumps'
  if not os.path.isabs(minidump_path):
    log_dir = get_config_parameter_value(conf_dir, role_name, 'log_dir') or '/tmp'
    minidump_path = os.path.join(log_dir, minidump_path)
  result = os.path.join(minidump_path, role_name)
  if not os.path.isdir(result):
    msg = 'Error: minidump directory does not exist.'
    print >> sys.stderr, msg
    sys.exit(1)
  return result

def main():
  parser = OptionParser()
  parser.add_option('--conf_dir',
      help='Directory in which to look for the config file with startup flags')
  parser.add_option('--role_name', type='choice',
      choices=['impalad', 'statestored', 'catalogd'], default='impalad',
      help='For which role to collect the minidumps.')
  parser.add_option('--max_output_size', default=40*1024*1024, type='int',
      help='The maximum file size of the result tarball to be written given in bytes. '
           'If the total size exceeds this value, most recent files will be preferred')
  parser.add_option('--start_time', default=None, type='int',
      help='Interval start time (in epoch milliseconds UTC).')
  parser.add_option('--end_time', default=None, type='int',
      help='Interval end time, until when to collect the minidump files '
           '(in epoch milliseconds UTC).')
  parser.add_option('--output_file_path', help='The full path of the output file.')
  options, args = parser.parse_args()
  if not options.conf_dir:
    msg = 'Error: conf_dir is not specified.'
    print >> sys.stderr, msg
    sys.exit(1)
  if not options.output_file_path:
    msg = 'Error: output_file_path is not specified.'
    print >> sys.stderr, msg
    sys.exit(1)

  minidump_dir = get_minidump_dir(options.conf_dir, options.role_name)
  file_archiver = FileArchiver(source_dir=minidump_dir,
      max_output_size=options.max_output_size,
      start_time=options.start_time,
      end_time=options.end_time,
      output_file_path=options.output_file_path)
  status, msg = file_archiver.make_tarball()
  print >> sys.stderr, msg
  sys.exit(status)

if __name__ == '__main__':
  main()

#!/usr/bin/env python
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# This script is to be called by Cloudera Manager to collect Breakpad minidump files up to
# a specified date/time. A compressed tarball is created in the user specified location.
# We try to fit as many files as possible into the tarball until a size limit is reached.
# Example invokation by CM to:
#     ./collect_minidumps.py --conf_dir=/var/run/.../5555-impala-STATESTORE/impala-conf \
#       --role_name=statestored --max_output_size=50000000 --end_time=1463033495000 \
#       --output_file_path=/tmp/minidump_package.tar.gz

import os
import re
import sys
import tarfile

from contextlib import closing
from optparse import OptionParser

class FileArchiver(object):
  '''This is a generic class that makes a tarball out of files in the source_dir.  We
  assume that source_dir contains only files. The resulting file will be compressed with
  gzip and placed into output_file_path. If a file with that name already exists, it will
  be deleted and re-created. Max_result_size is the maximum allowed size of the resulting
  tarball. If all files in the source_dir can't fit into the allowed size, most recent
  files will be preferred. start_time and end_time paramenters (in milliseconds UTC) allow
  us to specify an interval of time for which to consider the files.
  '''

  def __init__(self,
      source_dir,
      output_file_path,
      max_output_size,
      start_time=None,
      end_time=None):
    self.source_dir = source_dir
    self.max_output_size = max_output_size
    self.start_time = start_time
    self.end_time = end_time
    self.output_file_path = output_file_path
    # Maps the number of files in the tarball to the resulting size (in bytes).
    self.resulting_sizes = {}
    self.file_list = []

  def _remove_output_file(self):
    try:
      os.remove(self.output_file_path)
    except OSError:
      pass

  def _tar_files(self, num_files=None):
    '''Make a tarball with num_files most recent files in the file_list. Record the
    resulting size into resulting_sizes map and return it.
    '''
    num_files = num_files or len(self.file_list)
    self._remove_output_file()
    if num_files == 0:
      size = 0
    else:
      with closing(tarfile.open(self.output_file_path, mode='w:gz')) as out:
        for i in xrange(num_files):
          out.add(self.file_list[i])
      size = os.stat(self.output_file_path).st_size
    self.resulting_sizes[num_files] = size
    return size

  def _compute_file_list(self):
    '''Computes a sorted list of eligible files in the source directory by filtering out
    files with modified date not in the desired time range. Directories and other
    non-files are ignored.
    '''
    file_list = []
    for f in os.listdir(self.source_dir):
      full_path = os.path.join(self.source_dir, f)
      if not os.path.isfile(full_path):
        continue
      # st_mtime is in seconds UTC, so we need to multiply by 1000 to get milliseconds.
      time_modified = os.stat(full_path).st_mtime * 1000
      if self.start_time and self.start_time > time_modified:
        continue
      if self.end_time and self.end_time < time_modified:
        continue
      file_list.append(full_path)
    self.file_list = sorted(file_list, key=lambda f: os.stat(f).st_mtime, reverse=True)

  def _binary_search(self):
    '''Calculates the maximum number of files that can be collected, such that the tarball
    size is less than max_output_size.
    '''
    min_num = 0
    max_num = len(self.file_list)
    while max_num - min_num > 1:
      mid = (min_num + max_num) // 2
      if self._tar_files(mid) <= self.max_output_size:
        min_num = mid
      else:
        max_num = mid
    return min_num

  def make_tarball(self):
    '''Make a tarball with the maximum number of files such that the size of the tarball
    is less than or equal to max_output_size. Returns a pair (status (int), message
    (str)). status represents the result of the operation and follows the unix convention
    where 0 equals success. message provides additional information. A status of 1 is
    returned if source_dir is not empty and no files were able to fit into the tarball.
    '''
    self._compute_file_list()
    if len(self.file_list) == 0:
      status = 0
      msg = 'No files found in "{0}".'
      return status, msg.format(self.source_dir)
    output_size = self._tar_files()
    if output_size <= self.max_output_size:
      status = 0
      msg = 'Success, archived all {0} files in "{1}".'
      return status, msg.format(len(self.file_list), self.source_dir)
    else:
      max_num_files = self._binary_search()
      if max_num_files == 0:
        self._remove_output_file()
        status = 1
        msg = ('Unable to archive any files in "{0}". '
            'Increase max_output_size to at least {1} bytes.')
        # If max_num_files is 0, we are guaranteed that the binary search tried making a
        # tarball with 1 file.
        return status, msg.format(self.source_dir, self.resulting_sizes[1])
      else:
        self._tar_files(max_num_files)
        status = 0
        msg = 'Success. Archived {0} out of {1} files in "{2}".'
        return status, msg.format(max_num_files, len(self.file_list), self.source_dir)

def get_config_parameter_value(conf_dir, role_name, config_parameter_name):
  '''Extract a single config parameter from the configuration file of a particular
  daemon.
  '''
  ROLE_FLAGFILE_MAP = {
      'impalad': 'impalad_flags',
      'statestored': 'state_store_flags',
      'catalogd': 'catalogserver_flags'}
  config_parameter_value = None
  try:
    file_path = os.path.join(conf_dir, ROLE_FLAGFILE_MAP[role_name])
    with open(file_path, 'r') as f:
      for line in f:
        m = re.match('-{0}=(.*)'.format(config_parameter_name), line)
        if m:
          config_parameter_value = m.group(1)
  except IOError as e:
    print >> sys.stderr, 'Error: Unable to open "{0}".'.format(file_path)
    sys.exit(1)
  return config_parameter_value

def get_minidump_dir(conf_dir, role_name):
  '''Extracts the minidump directory path for a given role from the configuration file.
  The directory defaults to 'minidumps', relative paths are prepended with log_dir, which
  defaults to '/tmp'.
  '''
  minidump_path = get_config_parameter_value(
    conf_dir, role_name, 'minidump_path') or 'minidumps'
  if not os.path.isabs(minidump_path):
    log_dir = get_config_parameter_value(conf_dir, role_name, 'log_dir') or '/tmp'
    minidump_path = os.path.join(log_dir, minidump_path)
  result = os.path.join(minidump_path, role_name)
  if not os.path.isdir(result):
    msg = 'Error: minidump directory does not exist.'
    print >> sys.stderr, msg
    sys.exit(1)
  return result

def main():
  parser = OptionParser()
  parser.add_option('--conf_dir',
      help='Directory in which to look for the config file with startup flags')
  parser.add_option('--role_name', type='choice',
      choices=['impalad', 'statestored', 'catalogd'], default='impalad',
      help='For which role to collect the minidumps.')
  parser.add_option('--max_output_size', default=40*1024*1024, type='int',
      help='The maximum file size of the result tarball to be written given in bytes. '
           'If the total size exceeds this value, most recent files will be preferred')
  parser.add_option('--start_time', default=None, type='int',
      help='Interval start time (in epoch milliseconds UTC).')
  parser.add_option('--end_time', default=None, type='int',
      help='Interval end time, until when to collect the minidump files '
           '(in epoch milliseconds UTC).')
  parser.add_option('--output_file_path', help='The full path of the output file.')
  options, args = parser.parse_args()
  if not options.conf_dir:
    msg = 'Error: conf_dir is not specified.'
    print >> sys.stderr, msg
    sys.exit(1)
  if not options.output_file_path:
    msg = 'Error: output_file_path is not specified.'
    print >> sys.stderr, msg
    sys.exit(1)

  minidump_dir = get_minidump_dir(options.conf_dir, options.role_name)
  file_archiver = FileArchiver(source_dir=minidump_dir,
      max_output_size=options.max_output_size,
      start_time=options.start_time,
      end_time=options.end_time,
      output_file_path=options.output_file_path)
  status, msg = file_archiver.make_tarball()
  print >> sys.stderr, msg
  sys.exit(status)

if __name__ == '__main__':
  main()

#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements.  See the NOTICE file # distributed with this work for additional information # regarding copyright ownership.  The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License.  You may obtain a copy of the License at # #   http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied.  See the License for the # specific language governing permissions and limitations # under the License. # This script is to be called by Cloudera Manager to collect Breakpad minidump files up to # a specified date/time. A compressed tarball is created in the user specified location. # We try to fit as many files as possible into the tarball until a size limit is reached. # Example invokation by CM to: #     ./collect_minidumps.py --conf_dir=/var/run/.../5555-impala-STATESTORE/impala-conf \ #       --role_name=statestored --max_output_size=50000000 --end_time=1463033495000 \ #       --output_file_path=/tmp/minidump_package.tar.gz This is a generic class that makes a tarball out of files in the source_dir.  We   assume that source_dir contains only files. The resulting file will be compressed with   gzip and placed into output_file_path. If a file with that name already exists, it will   be deleted and re-created. Max_result_size is the maximum allowed size of the resulting   tarball. If all files in the source_dir can't fit into the allowed size, most recent   files will be preferred. start_time and end_time paramenters (in milliseconds UTC) allow   us to specify an interval of time for which to consider the files. # Maps the number of files in the tarball to the resulting size (in bytes). Make a tarball with num_files most recent files in the file_list. Record the     resulting size into resulting_sizes map and return it. Computes a sorted list of eligible files in the source directory by filtering out     files with modified date not in the desired time range. Directories and other     non-files are ignored. # st_mtime is in seconds UTC, so we need to multiply by 1000 to get milliseconds. Calculates the maximum number of files that can be collected, such that the tarball     size is less than max_output_size. Make a tarball with the maximum number of files such that the size of the tarball     is less than or equal to max_output_size. Returns a pair (status (int), message     (str)). status represents the result of the operation and follows the unix convention     where 0 equals success. message provides additional information. A status of 1 is     returned if source_dir is not empty and no files were able to fit into the tarball. # If max_num_files is 0, we are guaranteed that the binary search tried making a # tarball with 1 file. Extract a single config parameter from the configuration file of a particular   daemon. Extracts the minidump directory path for a given role from the configuration file.   The directory defaults to 'minidumps', relative paths are prepended with log_dir, which   defaults to '/tmp'.