#!/usr/bin/env python
# /ncluster/runs/one.01/info.log
# min:  1714.27, median:  1719.90, mean:  1724.42
# /ncluster/runs/two.01/info.log
# min:  1770.60, median:  1785.12, mean:  1791.83
# /ncluster/runs/four.02/info.log
# min:  1758.16, median:  1769.33, mean:  1786.09
# /ncluster/runs/eight.02/info.log
# min:  1951.20, median:  2027.51, mean:  2041.98
# /ncluster/runs/sixteen.01/info.log
# min:  1021.19, median:  2081.99, mean:  2083.05


import os
import re
import numpy as np

import subprocess

def ossystem(cmd):
    """Like os.system, but returns output of command as string."""
    p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
                         stderr=subprocess.STDOUT)
    (stdout, stderr) = p.communicate()
    return stdout.decode('ascii')


def getlogs(fn):
    return ossystem(f'ncluster cat {fn}').split('\n')


def get_times(fn):
    time_list = []
    for line in getlogs(fn):
        matches = re.findall('.+ms/batch ([0-9.]+).*', line)
        if matches:
            time_list.append(float(matches[0]))
    time_list = np.array(time_list)
    min_time = np.min(time_list)
    median = np.median(time_list)
    print(fn)
    print(f"min: {min_time:8.2f}, median: {median:8.2f}, mean: {np.mean(time_list):8.2f}")


# This analyzes training throughput of Transformer-XL 300M parameter model
# training on multiple p3dn instances


# 8 GPU throughput
get_times('/ncluster/runs/one.01/info.log')
    
# 16 GPU throughput
get_times('/ncluster/runs/two.01/info.log')
    
# 32 GPU throughput
get_times('/ncluster/runs/four.02/info.log')

# 64 GPU throughput
get_times('/ncluster/runs/eight.02/info.log')

# 128 GPU throughput
get_times('/ncluster/runs/sixteen.01/info.log')
    

#!/usr/bin/env python
# /ncluster/runs/one.01/info.log
# min:  1714.27, median:  1719.90, mean:  1724.42
# /ncluster/runs/two.01/info.log
# min:  1770.60, median:  1785.12, mean:  1791.83
# /ncluster/runs/four.02/info.log
# min:  1758.16, median:  1769.33, mean:  1786.09
# /ncluster/runs/eight.02/info.log
# min:  1951.20, median:  2027.51, mean:  2041.98
# /ncluster/runs/sixteen.01/info.log
# min:  1021.19, median:  2081.99, mean:  2083.05


import os
import re
import numpy as np

import subprocess

def ossystem(cmd):
    """Like os.system, but returns output of command as string."""
    p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
                         stderr=subprocess.STDOUT)
    (stdout, stderr) = p.communicate()
    return stdout.decode('ascii')


def getlogs(fn):
    return ossystem(f'ncluster cat {fn}').split('\n')


def get_times(fn):
    time_list = []
    for line in getlogs(fn):
        matches = re.findall('.+ms/batch ([0-9.]+).*', line)
        if matches:
            time_list.append(float(matches[0]))
    time_list = np.array(time_list)
    min_time = np.min(time_list)
    median = np.median(time_list)
    print(fn)
    print(f"min: {min_time:8.2f}, median: {median:8.2f}, mean: {np.mean(time_list):8.2f}")


# This analyzes training throughput of Transformer-XL 300M parameter model
# training on multiple p3dn instances


# 8 GPU throughput
get_times('/ncluster/runs/one.01/info.log')
    
# 16 GPU throughput
get_times('/ncluster/runs/two.01/info.log')
    
# 32 GPU throughput
get_times('/ncluster/runs/four.02/info.log')

# 64 GPU throughput
get_times('/ncluster/runs/eight.02/info.log')

# 128 GPU throughput
get_times('/ncluster/runs/sixteen.01/info.log')
    

#!/usr/bin/env python # /ncluster/runs/one.01/info.log # min:  1714.27, median:  1719.90, mean:  1724.42 # /ncluster/runs/two.01/info.log # min:  1770.60, median:  1785.12, mean:  1791.83 # /ncluster/runs/four.02/info.log # min:  1758.16, median:  1769.33, mean:  1786.09 # /ncluster/runs/eight.02/info.log # min:  1951.20, median:  2027.51, mean:  2041.98 # /ncluster/runs/sixteen.01/info.log # min:  1021.19, median:  2081.99, mean:  2083.05 Like os.system, but returns output of command as string. # This analyzes training throughput of Transformer-XL 300M parameter model # training on multiple p3dn instances # 8 GPU throughput # 16 GPU throughput # 32 GPU throughput # 64 GPU throughput # 128 GPU throughput