original_docstring: Takes raw tracelog dict and computes disk stats
docstring: Takes raw tracelog dict and computes disk stats
docstring_tokens: Takes raw tracelog dict and computes disk stats
code: def _assemble_io_stats(self):
          """Takes raw tracelog dict and computes disk stats"""
  
          for v in self.tracelog['trace_io'].values():
  
              # some of the traces triplets don't contain 'fio'
              #   if so, don't bother looking at this...
              if not 'fio' in v:
                  continue
                          
              # the disk data was read/written to
              try:
                  disk = int(v['fio']['disknum'])
                  v['iosize'] = v['fio']['nSectors'] * self._SECTOR_SIZE
              except TypeError as te:
                  print "Type error: {0}".format(te)
                  print v
  
              # some of the client logs don't have QIO/SIO for certain things
              #   like log writes, so check for those...
              if 'sio' in v and \
                      'fio' in v:
                  v['iotime'] = v['fio']['tracetime'] - \
                                  v['sio']['tracetime']
                  self.tracelog['trace_io']['disks'][disk]['iotimes'].setdefault(disk, []).append(
                          float(v['iotime']))
  
              # figure out how long the IO was queued...
              if 'qio' in v and \
                      'sio' in v:
                  io_time_in_queue = v['sio']['tracetime'] - \
                                     v['qio']['tracetime']
                  v['time_in_queue'] = io_time_in_queue
  
                  # get the start time in epoch, since we now have the finish time (epoch)
                  #   and the duration of the IO
                  io_start_time = v['sio']['tracetime'] + self.trace_start_epoch
                  io_queued_time = v['qio']['tracetime'] + self.trace_start_epoch
                  v['sio']['start_time'] = io_start_time
                  v['qio']['queued_time'] = io_queued_time
  
              # increment the disks bucket per FIO
              try:
                  self.tracelog['trace_io']['disks'][disk]['iosizes'].setdefault(
                          disk, []).append(int(v['iosize']))
              except TypeError as te:
                  print "Type error: {0}".format(te)
                  continue
  
          total_bytes = 0
          total_iops = 0
          avg_io_tm_data = []
          avg_io_tm_meta = []
          avg_long_io_tm_data = []
          avg_long_io_tm_meta = []
          avg_bucket_data = []
          avg_bucket_meta = []
  
  
          # calculate the average time per iop per disk and average io size
          for k, v in self.tracelog['trace_io']['disks'].iteritems():
  
              try:
                  num_iops = len(v['iosizes'][k])
                  total_io_bytes = sum(v['iosizes'][k])
                  total_io_time = sum(v['iotimes'][k])
                  average_io_size = total_io_bytes / len(v['iosizes'][k])
                  average_io_time = total_io_time / len(v['iotimes'][k])
  
                  v['stats']['avg_io_tm'] = average_io_time
                  v['stats']['avg_io_sz'] = average_io_size
                  v['stats']['total_bytes_io'] = total_io_bytes
                  v['stats']['total_time_io'] = total_io_time
                  v['stats']['longest_io'] = max(v['iotimes'][k])
                  v['stats']['num_iops'] = num_iops
              except ZeroDivisionError as zde:
                  #print "Zero Division error: {0}".format(zde)
                  # these only happen when the "write logData" operation
                  #   the problem is that the PID changes on the FIO of a triplet,
                  #   so the unique ID "disknum:diskaddr:pid" changes
                  continue
              except ValueError as ve:
                  print "Value error: {0}".format(ve)
                  continue
  
              if v['stats']['avg_io_sz'] > 1048576: 
                  avg_io_tm_data.append(v['stats']['avg_io_tm'])
                  avg_long_io_tm_data.append(v['stats']['longest_io'])
                  avg_bucket_data.append(v['stats']['avg_io_tm'])
              else:   # metadata disks most likely, fix later...
                  avg_io_tm_meta.append(v['stats']['avg_io_tm'])
                  avg_long_io_tm_meta.append(v['stats']['longest_io'])
                  avg_bucket_meta.append(v['stats']['avg_io_tm'])
  
              # gather some totals
              total_bytes += v['stats']['total_bytes_io']
              total_iops += v['stats']['num_iops']
  
          # compute the standard deviations and then set those k, v pairs
          std_dev_avg_io_tm_data, data_var = stddev(avg_io_tm_data)[0:2]
          std_dev_avg_io_tm_meta, meta_var = stddev(avg_io_tm_meta)[0:2]
          avg_io_tm_data = sum(avg_bucket_data) / len(avg_bucket_data)
          avg_io_tm_meta = sum(avg_bucket_meta) / len(avg_bucket_meta)
  
          self.tracelog['trace_io']['stats']['avg_io_tm_data'] = avg_io_tm_data
          self.tracelog['trace_io']['stats']['avg_io_tm_meta'] = avg_io_tm_meta
          self.tracelog['trace_io']['stats']['stddev_io_data'] = std_dev_avg_io_tm_data
          self.tracelog['trace_io']['stats']['stddev_io_meta'] = std_dev_avg_io_tm_meta
          self.tracelog['trace_io']['stats']['stddev_io_data_var'] = data_var
          self.tracelog['trace_io']['stats']['stddev_io_meta_var'] = meta_var
          self.tracelog['trace_io']['stats']['total_bytes'] = total_bytes
          self.tracelog['trace_io']['stats']['total_iops'] = total_iops
  
          return
code_tokens: def _assemble_io_stats ( self ) : for v in self . tracelog [ 'trace_io' ] . values ( ) : if not 'fio' in v : continue try : disk = int ( v [ 'fio' ] [ 'disknum' ] ) v [ 'iosize' ] = v [ 'fio' ] [ 'nSectors' ] * self . _SECTOR_SIZE except TypeError as te : print "Type error: {0}" . format ( te ) print v if 'sio' in v and 'fio' in v : v [ 'iotime' ] = v [ 'fio' ] [ 'tracetime' ] - v [ 'sio' ] [ 'tracetime' ] self . tracelog [ 'trace_io' ] [ 'disks' ] [ disk ] [ 'iotimes' ] . setdefault ( disk , [ ] ) . append ( float ( v [ 'iotime' ] ) ) if 'qio' in v and 'sio' in v : io_time_in_queue = v [ 'sio' ] [ 'tracetime' ] - v [ 'qio' ] [ 'tracetime' ] v [ 'time_in_queue' ] = io_time_in_queue io_start_time = v [ 'sio' ] [ 'tracetime' ] + self . trace_start_epoch io_queued_time = v [ 'qio' ] [ 'tracetime' ] + self . trace_start_epoch v [ 'sio' ] [ 'start_time' ] = io_start_time v [ 'qio' ] [ 'queued_time' ] = io_queued_time try : self . tracelog [ 'trace_io' ] [ 'disks' ] [ disk ] [ 'iosizes' ] . setdefault ( disk , [ ] ) . append ( int ( v [ 'iosize' ] ) ) except TypeError as te : print "Type error: {0}" . format ( te ) continue total_bytes = 0 total_iops = 0 avg_io_tm_data = [ ] avg_io_tm_meta = [ ] avg_long_io_tm_data = [ ] avg_long_io_tm_meta = [ ] avg_bucket_data = [ ] avg_bucket_meta = [ ] for k , v in self . tracelog [ 'trace_io' ] [ 'disks' ] . iteritems ( ) : try : num_iops = len ( v [ 'iosizes' ] [ k ] ) total_io_bytes = sum ( v [ 'iosizes' ] [ k ] ) total_io_time = sum ( v [ 'iotimes' ] [ k ] ) average_io_size = total_io_bytes / len ( v [ 'iosizes' ] [ k ] ) average_io_time = total_io_time / len ( v [ 'iotimes' ] [ k ] ) v [ 'stats' ] [ 'avg_io_tm' ] = average_io_time v [ 'stats' ] [ 'avg_io_sz' ] = average_io_size v [ 'stats' ] [ 'total_bytes_io' ] = total_io_bytes v [ 'stats' ] [ 'total_time_io' ] = total_io_time v [ 'stats' ] [ 'longest_io' ] = max ( v [ 'iotimes' ] [ k ] ) v [ 'stats' ] [ 'num_iops' ] = num_iops except ZeroDivisionError as zde : continue except ValueError as ve : print "Value error: {0}" . format ( ve ) continue if v [ 'stats' ] [ 'avg_io_sz' ] > 1048576 : avg_io_tm_data . append ( v [ 'stats' ] [ 'avg_io_tm' ] ) avg_long_io_tm_data . append ( v [ 'stats' ] [ 'longest_io' ] ) avg_bucket_data . append ( v [ 'stats' ] [ 'avg_io_tm' ] ) else : avg_io_tm_meta . append ( v [ 'stats' ] [ 'avg_io_tm' ] ) avg_long_io_tm_meta . append ( v [ 'stats' ] [ 'longest_io' ] ) avg_bucket_meta . append ( v [ 'stats' ] [ 'avg_io_tm' ] ) total_bytes += v [ 'stats' ] [ 'total_bytes_io' ] total_iops += v [ 'stats' ] [ 'num_iops' ] std_dev_avg_io_tm_data , data_var = stddev ( avg_io_tm_data ) [ 0 : 2 ] std_dev_avg_io_tm_meta , meta_var = stddev ( avg_io_tm_meta ) [ 0 : 2 ] avg_io_tm_data = sum ( avg_bucket_data ) / len ( avg_bucket_data ) avg_io_tm_meta = sum ( avg_bucket_meta ) / len ( avg_bucket_meta ) self . tracelog [ 'trace_io' ] [ 'stats' ] [ 'avg_io_tm_data' ] = avg_io_tm_data self . tracelog [ 'trace_io' ] [ 'stats' ] [ 'avg_io_tm_meta' ] = avg_io_tm_meta self . tracelog [ 'trace_io' ] [ 'stats' ] [ 'stddev_io_data' ] = std_dev_avg_io_tm_data self . tracelog [ 'trace_io' ] [ 'stats' ] [ 'stddev_io_meta' ] = std_dev_avg_io_tm_meta self . tracelog [ 'trace_io' ] [ 'stats' ] [ 'stddev_io_data_var' ] = data_var self . tracelog [ 'trace_io' ] [ 'stats' ] [ 'stddev_io_meta_var' ] = meta_var self . tracelog [ 'trace_io' ] [ 'stats' ] [ 'total_bytes' ] = total_bytes self . tracelog [ 'trace_io' ] [ 'stats' ] [ 'total_iops' ] = total_iops return
short_docstring: Takes raw tracelog dict and computes disk stats
short_docstring_tokens: Takes raw tracelog dict and computes disk stats
comment: """Takes raw tracelog dict and computes disk stats"""$SEP$# some of the traces triplets don't contain 'fio'$SEP$#   if so, don't bother looking at this...$SEP$# the disk data was read/written to$SEP$# some of the client logs don't have QIO/SIO for certain things$SEP$#   like log writes, so check for those...$SEP$# figure out how long the IO was queued...$SEP$# get the start time in epoch, since we now have the finish time (epoch)$SEP$#   and the duration of the IO$SEP$# increment the disks bucket per FIO$SEP$# calculate the average time per iop per disk and average io size$SEP$#print "Zero Division error: {0}".format(zde)$SEP$# these only happen when the "write logData" operation$SEP$#   the problem is that the PID changes on the FIO of a triplet,$SEP$#   so the unique ID "disknum:diskaddr:pid" changes$SEP$# metadata disks most likely, fix later...$SEP$# gather some totals$SEP$# compute the standard deviations and then set those k, v pairs