<filename>weightgrad/cifar10_train_sync_updated_2_with_gps.py<gh_stars>0
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import pickle as pickle
import socket
import time
from datetime import datetime

import tensorflow as tf

import bingrad_common_updated
import cifar10

# import bingrad_common

TCP_IP = '127.0.0.1'
# TCP_IP = '192.168.3.11'
TCP_PORT = 5014

port = 0
port_main = 0
s = 0

FLAGS = tf.app.flags.FLAGS

tf.app.flags.DEFINE_string('train_dir', 'data/cifar10_train',
                           """Directory where to write event logs """
                           """and checkpoint.""")
tf.app.flags.DEFINE_integer('max_steps', 500000,
                            """Number of batches to run.""")
tf.app.flags.DEFINE_boolean('log_device_placement', False,
                            """Whether to log device placement.""")
tf.app.flags.DEFINE_integer('log_frequency', 10,
                            """How often to log results to the console.""")
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.30)


def safe_recv(size, server_socket):
    data = ""
    temp = ""
    data = bytearray()
    recv_size = 0
    while 1:
        try:
            # print("~~~~~~~~~INSIDE safe_recv")
            temp = server_socket.recv(size - len(data))
            data.extend(temp)
            recv_size = len(data)
            if recv_size >= size:
                break
        except:
            print("Error")
    data = bytes(data)
    # print(type(data))
    return data


def get_placeholder_for_ternarized_gradient():
    placeholder_list = [tf.placeholder('float', shape=(1201,)), tf.placeholder('float', shape=(17,)),
                        tf.placeholder('float', shape=(25601,)), tf.placeholder('float', shape=(17,)),
                        tf.placeholder('float', shape=(221185,)), tf.placeholder('float', shape=(97,)),
                        tf.placeholder('float', shape=(18433,)), tf.placeholder('float', shape=(49,)),
                        tf.placeholder('float', shape=(481,)), tf.placeholder('float', shape=(3,))]
    return placeholder_list


def build_grad_shape():
    return [tf.TensorShape([5, 5, 3, 64]), tf.TensorShape([64, ]), tf.TensorShape([5, 5, 64, 64]), tf.TensorShape([64, ]),
            tf.TensorShape([2304, 384]), tf.TensorShape([384, ]), tf.TensorShape([384, 192]), tf.TensorShape([192, ]),
            tf.TensorShape([192, 10]), tf.TensorShape([10, ])]


def train():
    """Train CIFAR-10 for a number of steps."""

    g1 = tf.Graph()
    with g1.as_default():
        # global_step = tf.contrib.framework.get_or_create_global_step()
        global_step = tf.Variable(-1, name='global_step', trainable=False, dtype=tf.int32)
        increment_global_step_op = tf.assign(global_step, global_step + 1)

        # Get images and labels for CIFAR-10.
        images, labels = cifar10.distorted_inputs1()

        # Build a Graph that computes the logits predictions from the
        # inference model.
        logits = cifar10.inference(images)

        loss = cifar10.loss(logits, labels)
        grads = cifar10.train_part1(loss, global_step)
        print("GGGGGGGGGGGGGGGGGGGGGGGradients")

        only_gradients = [g for g, _ in grads]
        only_gradients = bingrad_common_updated.clip_gradients_by_stddev_2(only_gradients)
        # print(only_gradients)
        # ternarized_gradient, grad_shape = bingrad_common_updated.encode_to_ternary_gradients(grads, get_shape=True)
        # scalers = bingrad_common_updated.gradient_binarizing_scalers(grads, 0)
        # mean_scalers = bingrad_common_updated.max_scalers([scalers])
        scalers_2 = bingrad_common_updated.gradient_binarizing_scalers_2(only_gradients, 0)
        mean_scalers_2 = bingrad_common_updated.max_scalers_2([scalers_2])
        ternarized_gradient_2 = bingrad_common_updated.stochastical_binarize_gradients_2(only_gradients, mean_scalers_2[0])
        ternarized_gradient_2, grad_shape_2 = bingrad_common_updated.encode_to_ternary_gradients_2(ternarized_gradient_2,
                                                                                                   get_shape=True)
        # print("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
        aaa = tf.TensorShape([5, 5, 3, 64])
        # print(aaa)

        ternarized_placeholder = get_placeholder_for_ternarized_gradient()
        scaler_placeholder = tf.placeholder('float', shape=(10,))
        global_weights = bingrad_common_updated.decode_from_ternary_gradients_2(
            ternarized_placeholder, scaler_placeholder, build_grad_shape())

        class _LoggerHook(tf.train.SessionRunHook):
            """Logs loss and runtime."""

            def begin(self):
                self._step = -1
                self._start_time = time.time()

            def before_run(self, run_context):
                self._step += 1
                return tf.train.SessionRunArgs(loss)  # Asks for loss value.

            def after_run(self, run_context, run_values):
                if self._step % FLAGS.log_frequency == 0:
                    current_time = time.time()
                    duration = current_time - self._start_time
                    self._start_time = current_time

                    loss_value = run_values.results
                    examples_per_sec = FLAGS.log_frequency * FLAGS.batch_size / duration
                    sec_per_batch = float(duration / FLAGS.log_frequency)

                    format_str = ('%s: step %d, loss = %.2f (%.1f examples/sec; %.3f '
                                  'sec/batch)')
                    print(format_str % (datetime.now(), self._step, loss_value,
                                        examples_per_sec, sec_per_batch))

        with tf.train.MonitoredTrainingSession(
                checkpoint_dir=FLAGS.train_dir,
                hooks=[tf.train.StopAtStepHook(last_step=FLAGS.max_steps),
                       tf.train.NanTensorHook(loss),
                       _LoggerHook()],
                config=tf.ConfigProto(
                    log_device_placement=FLAGS.log_device_placement, gpu_options=gpu_options)) as mon_sess:
            # log_device_placement=FLAGS.log_device_placement)) as mon_sess:
            global port
            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            s.connect((TCP_IP, port_main))
            # receiving the variable values
            # print("Connected::::::::::::::")
            # recv_size = safe_recv(8, s)
            recv_size = safe_recv(17, s)
            # print("recv_size:::{}, {}".format(recv_size, type(recv_size)))
            # recv_size = bytes(recv_size,'utf-8')
            recv_size = pickle.loads(recv_size)
            print("recv_size in number:::{}".format(recv_size))
            recv_data = safe_recv(recv_size, s)
            var_vals = pickle.loads(recv_data)
            s.close()
            feed_dict = {}
            i = 0
            for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES):
                feed_dict[v] = var_vals[i]
                i = i + 1
            print("Received variable values from ps")
            # Opening the socket and connecting to server
            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            print(port)
            s.connect((TCP_IP, port))
            while not mon_sess.should_stop():
                gradients, step_val = mon_sess.run([only_gradients, increment_global_step_op], feed_dict=feed_dict)

                ternarized_gradient_value_2, ternarized_mean_scalers_2 = mon_sess.run([ternarized_gradient_2, mean_scalers_2],
                                                                                  feed_dict=feed_dict)
                loss_val = mon_sess.run(loss, feed_dict=feed_dict)

                send_data_tern = pickle.dumps(ternarized_gradient_value_2, pickle.HIGHEST_PROTOCOL)
                to_send_size_tern = len(send_data_tern)
                send_size_tern = pickle.dumps(to_send_size_tern, pickle.HIGHEST_PROTOCOL)
                s.sendall(send_size_tern)
                s.sendall(send_data_tern)

                send_scaler = pickle.dumps(ternarized_mean_scalers_2, pickle.HIGHEST_PROTOCOL)
                to_send_size_scaler = len(send_scaler)
                send_size_scaler = pickle.dumps(to_send_size_scaler, pickle.HIGHEST_PROTOCOL)
                s.sendall(send_size_scaler)
                s.sendall(send_scaler)
                loss_val = pickle.dumps(loss_val, pickle.HIGHEST_PROTOCOL)
                s.sendall(loss_val)
                recv_size = safe_recv(17, s)
                recv_size = pickle.loads(recv_size)
                recv_data = safe_recv(recv_size, s)
                var_vals = pickle.loads(recv_data)

                feed_dict = {}
                i = 0
                for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES):
                    feed_dict[v] = var_vals[i]
                    i = i + 1
            s.close()


def main(argv=None):  # pylint: disable=unused-argument
    global port
    global port_main
    global s
    # if (len(sys.argv) != 3):
    #     print("<port> <worker-id> required")
    #     sys.exit()
    # port = int(sys.argv[1]) + int(sys.argv[2])
    # port_main = int(sys.argv[1])
    port_main = 17000
    port = 17001
    print("Connecting to port ", port)
    FLAGS.data_dir = "data"
    FLAGS.train_dir = "data/train"
    cifar10.maybe_download_and_extract()
    if tf.gfile.Exists(FLAGS.train_dir):
        tf.gfile.DeleteRecursively(FLAGS.train_dir)
    tf.gfile.MakeDirs(FLAGS.train_dir)
    total_start_time = time.time()
    train()
    print("--- %s seconds ---" % (time.time() - total_start_time))


if __name__ == '__main__':
    tf.app.run()
<filename>weightgrad/cifar10_train_sync_updated_2_with_gps.py<gh_stars>0
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import pickle as pickle
import socket
import time
from datetime import datetime

import tensorflow as tf

import bingrad_common_updated
import cifar10

# import bingrad_common

TCP_IP = '127.0.0.1'
# TCP_IP = '192.168.3.11'
TCP_PORT = 5014

port = 0
port_main = 0
s = 0

FLAGS = tf.app.flags.FLAGS

tf.app.flags.DEFINE_string('train_dir', 'data/cifar10_train',
                           """Directory where to write event logs """
                           """and checkpoint.""")
tf.app.flags.DEFINE_integer('max_steps', 500000,
                            """Number of batches to run.""")
tf.app.flags.DEFINE_boolean('log_device_placement', False,
                            """Whether to log device placement.""")
tf.app.flags.DEFINE_integer('log_frequency', 10,
                            """How often to log results to the console.""")
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.30)


def safe_recv(size, server_socket):
    data = ""
    temp = ""
    data = bytearray()
    recv_size = 0
    while 1:
        try:
            # print("~~~~~~~~~INSIDE safe_recv")
            temp = server_socket.recv(size - len(data))
            data.extend(temp)
            recv_size = len(data)
            if recv_size >= size:
                break
        except:
            print("Error")
    data = bytes(data)
    # print(type(data))
    return data


def get_placeholder_for_ternarized_gradient():
    placeholder_list = [tf.placeholder('float', shape=(1201,)), tf.placeholder('float', shape=(17,)),
                        tf.placeholder('float', shape=(25601,)), tf.placeholder('float', shape=(17,)),
                        tf.placeholder('float', shape=(221185,)), tf.placeholder('float', shape=(97,)),
                        tf.placeholder('float', shape=(18433,)), tf.placeholder('float', shape=(49,)),
                        tf.placeholder('float', shape=(481,)), tf.placeholder('float', shape=(3,))]
    return placeholder_list


def build_grad_shape():
    return [tf.TensorShape([5, 5, 3, 64]), tf.TensorShape([64, ]), tf.TensorShape([5, 5, 64, 64]), tf.TensorShape([64, ]),
            tf.TensorShape([2304, 384]), tf.TensorShape([384, ]), tf.TensorShape([384, 192]), tf.TensorShape([192, ]),
            tf.TensorShape([192, 10]), tf.TensorShape([10, ])]


def train():
    """Train CIFAR-10 for a number of steps."""

    g1 = tf.Graph()
    with g1.as_default():
        # global_step = tf.contrib.framework.get_or_create_global_step()
        global_step = tf.Variable(-1, name='global_step', trainable=False, dtype=tf.int32)
        increment_global_step_op = tf.assign(global_step, global_step + 1)

        # Get images and labels for CIFAR-10.
        images, labels = cifar10.distorted_inputs1()

        # Build a Graph that computes the logits predictions from the
        # inference model.
        logits = cifar10.inference(images)

        loss = cifar10.loss(logits, labels)
        grads = cifar10.train_part1(loss, global_step)
        print("GGGGGGGGGGGGGGGGGGGGGGGradients")

        only_gradients = [g for g, _ in grads]
        only_gradients = bingrad_common_updated.clip_gradients_by_stddev_2(only_gradients)
        # print(only_gradients)
        # ternarized_gradient, grad_shape = bingrad_common_updated.encode_to_ternary_gradients(grads, get_shape=True)
        # scalers = bingrad_common_updated.gradient_binarizing_scalers(grads, 0)
        # mean_scalers = bingrad_common_updated.max_scalers([scalers])
        scalers_2 = bingrad_common_updated.gradient_binarizing_scalers_2(only_gradients, 0)
        mean_scalers_2 = bingrad_common_updated.max_scalers_2([scalers_2])
        ternarized_gradient_2 = bingrad_common_updated.stochastical_binarize_gradients_2(only_gradients, mean_scalers_2[0])
        ternarized_gradient_2, grad_shape_2 = bingrad_common_updated.encode_to_ternary_gradients_2(ternarized_gradient_2,
                                                                                                   get_shape=True)
        # print("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")
        aaa = tf.TensorShape([5, 5, 3, 64])
        # print(aaa)

        ternarized_placeholder = get_placeholder_for_ternarized_gradient()
        scaler_placeholder = tf.placeholder('float', shape=(10,))
        global_weights = bingrad_common_updated.decode_from_ternary_gradients_2(
            ternarized_placeholder, scaler_placeholder, build_grad_shape())

        class _LoggerHook(tf.train.SessionRunHook):
            """Logs loss and runtime."""

            def begin(self):
                self._step = -1
                self._start_time = time.time()

            def before_run(self, run_context):
                self._step += 1
                return tf.train.SessionRunArgs(loss)  # Asks for loss value.

            def after_run(self, run_context, run_values):
                if self._step % FLAGS.log_frequency == 0:
                    current_time = time.time()
                    duration = current_time - self._start_time
                    self._start_time = current_time

                    loss_value = run_values.results
                    examples_per_sec = FLAGS.log_frequency * FLAGS.batch_size / duration
                    sec_per_batch = float(duration / FLAGS.log_frequency)

                    format_str = ('%s: step %d, loss = %.2f (%.1f examples/sec; %.3f '
                                  'sec/batch)')
                    print(format_str % (datetime.now(), self._step, loss_value,
                                        examples_per_sec, sec_per_batch))

        with tf.train.MonitoredTrainingSession(
                checkpoint_dir=FLAGS.train_dir,
                hooks=[tf.train.StopAtStepHook(last_step=FLAGS.max_steps),
                       tf.train.NanTensorHook(loss),
                       _LoggerHook()],
                config=tf.ConfigProto(
                    log_device_placement=FLAGS.log_device_placement, gpu_options=gpu_options)) as mon_sess:
            # log_device_placement=FLAGS.log_device_placement)) as mon_sess:
            global port
            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            s.connect((TCP_IP, port_main))
            # receiving the variable values
            # print("Connected::::::::::::::")
            # recv_size = safe_recv(8, s)
            recv_size = safe_recv(17, s)
            # print("recv_size:::{}, {}".format(recv_size, type(recv_size)))
            # recv_size = bytes(recv_size,'utf-8')
            recv_size = pickle.loads(recv_size)
            print("recv_size in number:::{}".format(recv_size))
            recv_data = safe_recv(recv_size, s)
            var_vals = pickle.loads(recv_data)
            s.close()
            feed_dict = {}
            i = 0
            for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES):
                feed_dict[v] = var_vals[i]
                i = i + 1
            print("Received variable values from ps")
            # Opening the socket and connecting to server
            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            print(port)
            s.connect((TCP_IP, port))
            while not mon_sess.should_stop():
                gradients, step_val = mon_sess.run([only_gradients, increment_global_step_op], feed_dict=feed_dict)

                ternarized_gradient_value_2, ternarized_mean_scalers_2 = mon_sess.run([ternarized_gradient_2, mean_scalers_2],
                                                                                  feed_dict=feed_dict)
                loss_val = mon_sess.run(loss, feed_dict=feed_dict)

                send_data_tern = pickle.dumps(ternarized_gradient_value_2, pickle.HIGHEST_PROTOCOL)
                to_send_size_tern = len(send_data_tern)
                send_size_tern = pickle.dumps(to_send_size_tern, pickle.HIGHEST_PROTOCOL)
                s.sendall(send_size_tern)
                s.sendall(send_data_tern)

                send_scaler = pickle.dumps(ternarized_mean_scalers_2, pickle.HIGHEST_PROTOCOL)
                to_send_size_scaler = len(send_scaler)
                send_size_scaler = pickle.dumps(to_send_size_scaler, pickle.HIGHEST_PROTOCOL)
                s.sendall(send_size_scaler)
                s.sendall(send_scaler)
                loss_val = pickle.dumps(loss_val, pickle.HIGHEST_PROTOCOL)
                s.sendall(loss_val)
                recv_size = safe_recv(17, s)
                recv_size = pickle.loads(recv_size)
                recv_data = safe_recv(recv_size, s)
                var_vals = pickle.loads(recv_data)

                feed_dict = {}
                i = 0
                for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES):
                    feed_dict[v] = var_vals[i]
                    i = i + 1
            s.close()


def main(argv=None):  # pylint: disable=unused-argument
    global port
    global port_main
    global s
    # if (len(sys.argv) != 3):
    #     print("<port> <worker-id> required")
    #     sys.exit()
    # port = int(sys.argv[1]) + int(sys.argv[2])
    # port_main = int(sys.argv[1])
    port_main = 17000
    port = 17001
    print("Connecting to port ", port)
    FLAGS.data_dir = "data"
    FLAGS.train_dir = "data/train"
    cifar10.maybe_download_and_extract()
    if tf.gfile.Exists(FLAGS.train_dir):
        tf.gfile.DeleteRecursively(FLAGS.train_dir)
    tf.gfile.MakeDirs(FLAGS.train_dir)
    total_start_time = time.time()
    train()
    print("--- %s seconds ---" % (time.time() - total_start_time))


if __name__ == '__main__':
    tf.app.run()
# import bingrad_common # TCP_IP = '192.168.3.11' Directory where to write event logs and checkpoint. Number of batches to run. Whether to log device placement. How often to log results to the console. # print("~~~~~~~~~INSIDE safe_recv") # print(type(data)) Train CIFAR-10 for a number of steps. # global_step = tf.contrib.framework.get_or_create_global_step() # Get images and labels for CIFAR-10. # Build a Graph that computes the logits predictions from the # inference model. # print(only_gradients) # ternarized_gradient, grad_shape = bingrad_common_updated.encode_to_ternary_gradients(grads, get_shape=True) # scalers = bingrad_common_updated.gradient_binarizing_scalers(grads, 0) # mean_scalers = bingrad_common_updated.max_scalers([scalers]) # print("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA") # print(aaa) Logs loss and runtime. # Asks for loss value. # log_device_placement=FLAGS.log_device_placement)) as mon_sess: # receiving the variable values # print("Connected::::::::::::::") # recv_size = safe_recv(8, s) # print("recv_size:::{}, {}".format(recv_size, type(recv_size))) # recv_size = bytes(recv_size,'utf-8') # Opening the socket and connecting to server # pylint: disable=unused-argument # if (len(sys.argv) != 3): #     print("<port> <worker-id> required") #     sys.exit() # port = int(sys.argv[1]) + int(sys.argv[2]) # port_main = int(sys.argv[1])