{"action":"created","comment":{"url":"https://api.github.com/repos/tensorflow/cleverhans/pulls/comments/216538086","pull_request_review_id":154040865,"id":216538086,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDIxNjUzODA4Ng==","diff_hunk":"@@ -0,0 +1,338 @@\n+\"\"\"\n+Functionality for evaluating expressions across entire datasets.\n+Includes multi-GPU support for fast evaluation.\n+\"\"\"\n+\n+from distutils.version import LooseVersion\n+import numpy as np\n+from six.moves import range\n+import tensorflow as tf\n+\n+from cleverhans.utils_tf import infer_devices\n+\n+\n+def accuracy(sess, model, x, y, batch_size=None, devices=None, feed=None):\n+    \"\"\"\n+    Compute the accuracy of a TF model on some data\n+    :param sess: TF session to use when training the graph\n+    :param model: cleverhans.model.Model instance\n+    :param x: numpy array containing input examples (e.g. MNIST().x_test )\n+    :param y: numpy array containing example labels (e.g. MNIST().y_test )\n+    :batch_size: Number of examples to use in a single evaluation batch.\n+        If not specified, this function will use a reasonable guess and\n+        may run out of memory.\n+        When choosing the batch size, keep in mind that the batch will\n+        be divided up evenly among available devices. If you can fit 128\n+        examples in memory on one GPU and you have 8 GPUs, you probably\n+        want to use a batch size of 1024 (unless a different batch size\n+        runs faster with the ops you are using, etc.)\n+    :param feed: An optional dictionary that is appended to the feeding\n+             dictionary before the session runs. Can be used to feed\n+             the learning phase of a Keras model for instance.\n+    :return: a float with the accuracy value\n+    \"\"\"\n+\n+    factory = _CorrectFactory(model)\n+\n+    correct, = batch_eval_multi_worker(sess, factory, [x, y],\n+                                       batch_size=batch_size, devices=devices,\n+                                       feed=feed)\n+\n+    return correct.mean()\n+\n+\n+# Cache for storing output of `batch_eval_multi_worker`'s calls to\n+# `graph_factory`, to avoid making the tf graph too big\n+_batch_eval_multi_worker_cache = {}\n+\n+\n+def batch_eval_multi_worker(sess, graph_factory, numpy_inputs, batch_size=None,\n+                            devices=None, feed=None):\n+    \"\"\"\n+    Generic computation engine for evaluating an expression across a whole\n+    dataset, divided into batches.\n+\n+    This function assumes that the work can be parallelized with one worker\n+    device handling one batch of data. If you need multiple devices per\n+    batch, use `batch_eval`.\n+\n+    The tensorflow graph for multiple workers is large, so the first few\n+    runs of the graph will be very slow. If you expect to run the graph\n+    few times (few calls to `batch_eval_multi_worker` that each run few\n+    batches) the startup cost might dominate the runtime, and it might be\n+    preferable to use the single worker `batch_eval` just because its\n+    startup cost will be lower.\n+\n+    :param sess: tensorflow Session\n+    :param graph_factory: callable\n+        When called, returns (tf_inputs, tf_outputs) where:\n+            tf_inputs is a list of placeholders to feed from the dataset\n+            tf_outputs is a list of tf tensors to calculate\n+        Example: tf_inputs is [x, y] placeholders, tf_outputs is [accuracy].\n+        This factory must make new tensors when called, rather than, e.g.\n+        handing out a reference to existing tensors.\n+        This factory must make exactly equivalent expressions every time\n+        it is called, otherwise the results of `batch_eval` will vary\n+        depending on how work is distributed to devices.\n+        This factory must respect \"with tf.device()\" context managers\n+        that are active when it is called, otherwise work will not be\n+        distributed to devices correctly.\n+    :param numpy_inputs:\n+        A list of numpy arrays defining the dataset to be evaluated.\n+        The list should have the same length as tf_inputs.\n+        Each array should have the same number of examples (shape[0]).\n+        Example: numpy_inputs is [MNIST().x_test, MNIST().y_test]\n+    :batch_size: Number of examples to use in a single evaluation batch.\n+        If not specified, this function will use a reasonable guess and\n+        may run out of memory.\n+        When choosing the batch size, keep in mind that the batch will\n+        be divided up evenly among available devices. If you can fit 128\n+        examples in memory on one GPU and you have 8 GPUs, you probably\n+        want to use a batch size of 1024 (unless a different batch size\n+        runs faster with the ops you are using, etc.)\n+    :param devices: List of devices to run on. If unspecified, uses all\n+        available GPUs if any GPUS are available, otherwise uses CPUs.\n+    :param feed: An optional dictionary that is appended to the feeding\n+             dictionary before the session runs. Can be used to feed\n+             the learning phase of a Keras model for instance.\n+    \"\"\"\n+    global _batch_eval_multi_worker_cache\n+\n+    devices = infer_devices(devices)\n+\n+    if batch_size is None:\n+        # For big models this might result in OOM and then the user\n+        # should just specify batch_size\n+        batch_size = len(devices) * DEFAULT_EXAMPLES_PER_DEVICE\n+\n+    n = len(numpy_inputs)\n+    assert n > 0\n+    m = numpy_inputs[0].shape[0]\n+    for i in range(1, n):\n+        assert numpy_inputs[i].shape[0] == m\n+    out = []\n+\n+    replicated_tf_inputs = []\n+    replicated_tf_outputs = []\n+    p = None\n+\n+    num_devices = len(devices)\n+    assert batch_size % num_devices == 0\n+    device_batch_size = batch_size // num_devices\n+\n+    cache_key = (graph_factory, tuple(devices))\n+    if cache_key in _batch_eval_multi_worker_cache:\n+        # Retrieve graph for multi-GPU inference from cache.\n+        # This avoids adding tf ops to the graph\n+        packed = _batch_eval_multi_worker_cache[cache_key]\n+        replicated_tf_inputs, replicated_tf_outputs = packed\n+        p = len(replicated_tf_outputs[0])\n+        assert p > 0\n+    else:\n+        # This graph has not been built before.\n+        # Build it now.\n+\n+        for device in devices:\n+            with tf.device(device):\n+                tf_inputs, tf_outputs = graph_factory()\n+                assert len(tf_inputs) == n\n+                if p is None:\n+                    p = len(tf_outputs)\n+                    assert p > 0\n+                else:\n+                    assert len(tf_outputs) == p\n+                replicated_tf_inputs.append(tf_inputs)\n+                replicated_tf_outputs.append(tf_outputs)\n+        del tf_inputs\n+        del tf_outputs\n+        # Store the result in the cache\n+        packed = replicated_tf_inputs, replicated_tf_outputs\n+        _batch_eval_multi_worker_cache[cache_key] = packed\n+    for _ in range(p):\n+        out.append([])\n+    flat_tf_outputs = []\n+    for output in range(p):\n+        for dev_idx in range(num_devices):\n+            flat_tf_outputs.append(replicated_tf_outputs[dev_idx][output])\n+\n+    # pad data to have # examples be multiple of batch size\n+    # we discard the excess later\n+    num_batches = int(np.ceil(float(m) / batch_size))\n+    needed_m = num_batches * batch_size\n+    excess = needed_m - m\n+    if excess > m:\n+        raise NotImplementedError(\"Your batch size is bigger than the\"\n+                                  \" dataset, this function is probably\"\n+                                  \" overkill.\")\n+\n+    def pad(array):\n+        if excess > 0:\n+            array = np.concatenate((array, array[:excess]), axis=0)\n+        return array\n+    numpy_inputs = [pad(numpy_input) for numpy_input in numpy_inputs]\n+    orig_m = m\n+    m = needed_m\n+\n+    for start in range(0, m, batch_size):\n+        batch = start // batch_size\n+        if batch % 100 == 0 and batch > 0:\n+            _logger.debug(\"Batch \" + str(batch))\n+\n+        # Compute batch start and end indices\n+        end = start + batch_size\n+        numpy_input_batches = [numpy_input[start:end]\n+                               for numpy_input in numpy_inputs]\n+        feed_dict = {}\n+        for dev_idx, tf_inputs in enumerate(replicated_tf_inputs):\n+            for tf_input, numpy_input in zip(tf_inputs, numpy_input_batches):\n+                dev_start = dev_idx * device_batch_size\n+                dev_end = (dev_idx + 1) * device_batch_size\n+                value = numpy_input[dev_start:dev_end]\n+                assert value.shape[0] == device_batch_size\n+                feed_dict[tf_input] = value\n+        if feed is not None:\n+            feed_dict.update(feed)\n+        flat_output_batches = sess.run(flat_tf_outputs, feed_dict=feed_dict)\n+        for e in flat_output_batches:\n+            assert e.shape[0] == device_batch_size, e.shape\n+\n+        output_batches = []\n+        for output in xrange(p):\n+            o_start = output * num_devices\n+            o_end = (output + 1) * num_devices\n+            device_values = flat_output_batches[o_start:o_end]\n+            assert len(device_values) == num_devices\n+            output_batches.append(device_values)\n+\n+        for out_elem, device_values in zip(out, output_batches):\n+            assert len(device_values) == num_devices, (len(device_values),\n+                                                       num_devices)\n+            for device_value in device_values:\n+                assert device_value.shape[0] == device_batch_size\n+            out_elem.extend(device_values)\n+\n+    out = [np.concatenate(x, axis=0) for x in out]\n+    for e in out:\n+        assert e.shape[0] == m, e.shape\n+\n+    # Trim off the examples we used to pad up to batch size\n+    out = [e[:orig_m] for e in out]\n+    assert len(out) == p, (len(out), p)\n+\n+    return out\n+\n+\n+def batch_eval(sess, tf_inputs, tf_outputs, numpy_inputs, batch_size=None,\n+               feed=None,\n+               args=None):\n+    \"\"\"\n+    A helper function that computes a tensor on numpy inputs by batches.\n+    This version uses exactly the tensorflow graph constructed by the\n+    caller, so the caller can place specific ops on specific devices\n+    to implement model parallelism.\n+    Most users probably prefer `batch_eval_multi_worker` which maps\n+    a single-device expression to multiple devices in order to evaluate\n+    faster by parallelizing across data.\n+\n+    :param sess: tf Session to use\n+    :param tf_inputs: list of tf Placeholders to feed from the dataset\n+    :param tf_outputs: list of tf tensors to calculate\n+    :param numpy_inputs: list of numpy arrays defining the dataset\n+    :param batch_size: int, batch size to use for evaluation\n+        If not specified, this function will try to guess the batch size,\n+        but might get an out of memory error or run the model with an\n+        unsupported batch size, etc.\n+    :param feed: An optional dictionary that is appended to the feeding\n+             dictionary before the session runs. Can be used to feed\n+             the learning phase of a Keras model for instance.\n+    :param args: dict or argparse `Namespace` object.\n+                Deprecated and included only for backwards compatibility.\n+                 Should contain `batch_size`\n+    \"\"\"\n+\n+    if args is not None:\n+        warnings.warn(\"`args` is deprecated and will be removde on or \"\n+                      \"after 2019-03-09. Pass `batch_size` directly.\")\n+        if \"batch_size\" in args:\n+            assert batch_size is None\n+            batch_size = args[\"batch_size\"]\n+\n+    if batch_size is None:\n+        batch_size = DEFAULT_EXAMPLES_PER_DEVICE\n+\n+    n = len(numpy_inputs)\n+    assert n > 0\n+    assert n == len(tf_inputs)\n+    m = numpy_inputs[0].shape[0]\n+    for i in range(1, n):\n+        assert numpy_inputs[i].shape[0] == m\n+    out = []\n+    for _ in tf_outputs:\n+        out.append([])\n+    with sess.as_default():","path":"cleverhans/evaluation.py","position":null,"original_position":272,"commit_id":"d941d348faf76d4d529595cc3926b59106ef5fcc","original_commit_id":"3512bc344997108043ff373532610cfa93c16b0c","user":{"login":"goodfeli","id":387866,"node_id":"MDQ6VXNlcjM4Nzg2Ng==","avatar_url":"https://avatars3.githubusercontent.com/u/387866?v=4","gravatar_id":"","url":"https://api.github.com/users/goodfeli","html_url":"https://github.com/goodfeli","followers_url":"https://api.github.com/users/goodfeli/followers","following_url":"https://api.github.com/users/goodfeli/following{/other_user}","gists_url":"https://api.github.com/users/goodfeli/gists{/gist_id}","starred_url":"https://api.github.com/users/goodfeli/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/goodfeli/subscriptions","organizations_url":"https://api.github.com/users/goodfeli/orgs","repos_url":"https://api.github.com/users/goodfeli/repos","events_url":"https://api.github.com/users/goodfeli/events{/privacy}","received_events_url":"https://api.github.com/users/goodfeli/received_events","type":"User","site_admin":false},"body":"Thanks, I removed it.","created_at":"2018-09-11T03:47:18Z","updated_at":"2018-09-11T03:47:18Z","html_url":"https://github.com/tensorflow/cleverhans/pull/551#discussion_r216538086","pull_request_url":"https://api.github.com/repos/tensorflow/cleverhans/pulls/551","author_association":"COLLABORATOR","_links":{"self":{"href":"https://api.github.com/repos/tensorflow/cleverhans/pulls/comments/216538086"},"html":{"href":"https://github.com/tensorflow/cleverhans/pull/551#discussion_r216538086"},"pull_request":{"href":"https://api.github.com/repos/tensorflow/cleverhans/pulls/551"}},"in_reply_to_id":216470052},"pull_request":{"url":"https://api.github.com/repos/tensorflow/cleverhans/pulls/551","id":214171773,"node_id":"MDExOlB1bGxSZXF1ZXN0MjE0MTcxNzcz","html_url":"https://github.com/tensorflow/cleverhans/pull/551","diff_url":"https://github.com/tensorflow/cleverhans/pull/551.diff","patch_url":"https://github.com/tensorflow/cleverhans/pull/551.patch","issue_url":"https://api.github.com/repos/tensorflow/cleverhans/issues/551","number":551,"state":"open","locked":false,"title":"multi-device evaluation","user":{"login":"goodfeli","id":387866,"node_id":"MDQ6VXNlcjM4Nzg2Ng==","avatar_url":"https://avatars3.githubusercontent.com/u/387866?v=4","gravatar_id":"","url":"https://api.github.com/users/goodfeli","html_url":"https://github.com/goodfeli","followers_url":"https://api.github.com/users/goodfeli/followers","following_url":"https://api.github.com/users/goodfeli/following{/other_user}","gists_url":"https://api.github.com/users/goodfeli/gists{/gist_id}","starred_url":"https://api.github.com/users/goodfeli/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/goodfeli/subscriptions","organizations_url":"https://api.github.com/users/goodfeli/orgs","repos_url":"https://api.github.com/users/goodfeli/repos","events_url":"https://api.github.com/users/goodfeli/events{/privacy}","received_events_url":"https://api.github.com/users/goodfeli/received_events","type":"User","site_admin":false},"body":"","created_at":"2018-09-09T23:43:22Z","updated_at":"2018-09-11T03:47:18Z","closed_at":null,"merged_at":null,"merge_commit_sha":"1944d307709225884e0e90210da6d6dc72d92342","assignee":null,"assignees":[],"requested_reviewers":[],"requested_teams":[],"labels":[],"milestone":null,"commits_url":"https://api.github.com/repos/tensorflow/cleverhans/pulls/551/commits","review_comments_url":"https://api.github.com/repos/tensorflow/cleverhans/pulls/551/comments","review_comment_url":"https://api.github.com/repos/tensorflow/cleverhans/pulls/comments{/number}","comments_url":"https://api.github.com/repos/tensorflow/cleverhans/issues/551/comments","statuses_url":"https://api.github.com/repos/tensorflow/cleverhans/statuses/d941d348faf76d4d529595cc3926b59106ef5fcc","head":{"label":"goodfeli:multigpu","ref":"multigpu","sha":"d941d348faf76d4d529595cc3926b59106ef5fcc","user":{"login":"goodfeli","id":387866,"node_id":"MDQ6VXNlcjM4Nzg2Ng==","avatar_url":"https://avatars3.githubusercontent.com/u/387866?v=4","gravatar_id":"","url":"https://api.github.com/users/goodfeli","html_url":"https://github.com/goodfeli","followers_url":"https://api.github.com/users/goodfeli/followers","following_url":"https://api.github.com/users/goodfeli/following{/other_user}","gists_url":"https://api.github.com/users/goodfeli/gists{/gist_id}","starred_url":"https://api.github.com/users/goodfeli/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/goodfeli/subscriptions","organizations_url":"https://api.github.com/users/goodfeli/orgs","repos_url":"https://api.github.com/users/goodfeli/repos","events_url":"https://api.github.com/users/goodfeli/events{/privacy}","received_events_url":"https://api.github.com/users/goodfeli/received_events","type":"User","site_admin":false},"repo":{"id":93456271,"node_id":"MDEwOlJlcG9zaXRvcnk5MzQ1NjI3MQ==","name":"cleverhans","full_name":"goodfeli/cleverhans","owner":{"login":"goodfeli","id":387866,"node_id":"MDQ6VXNlcjM4Nzg2Ng==","avatar_url":"https://avatars3.githubusercontent.com/u/387866?v=4","gravatar_id":"","url":"https://api.github.com/users/goodfeli","html_url":"https://github.com/goodfeli","followers_url":"https://api.github.com/users/goodfeli/followers","following_url":"https://api.github.com/users/goodfeli/following{/other_user}","gists_url":"https://api.github.com/users/goodfeli/gists{/gist_id}","starred_url":"https://api.github.com/users/goodfeli/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/goodfeli/subscriptions","organizations_url":"https://api.github.com/users/goodfeli/orgs","repos_url":"https://api.github.com/users/goodfeli/repos","events_url":"https://api.github.com/users/goodfeli/events{/privacy}","received_events_url":"https://api.github.com/users/goodfeli/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/goodfeli/cleverhans","description":"A library for benchmarking vulnerability to adversarial examples","fork":true,"url":"https://api.github.com/repos/goodfeli/cleverhans","forks_url":"https://api.github.com/repos/goodfeli/cleverhans/forks","keys_url":"https://api.github.com/repos/goodfeli/cleverhans/keys{/key_id}","collaborators_url":"https://api.github.com/repos/goodfeli/cleverhans/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/goodfeli/cleverhans/teams","hooks_url":"https://api.github.com/repos/goodfeli/cleverhans/hooks","issue_events_url":"https://api.github.com/repos/goodfeli/cleverhans/issues/events{/number}","events_url":"https://api.github.com/repos/goodfeli/cleverhans/events","assignees_url":"https://api.github.com/repos/goodfeli/cleverhans/assignees{/user}","branches_url":"https://api.github.com/repos/goodfeli/cleverhans/branches{/branch}","tags_url":"https://api.github.com/repos/goodfeli/cleverhans/tags","blobs_url":"https://api.github.com/repos/goodfeli/cleverhans/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/goodfeli/cleverhans/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/goodfeli/cleverhans/git/refs{/sha}","trees_url":"https://api.github.com/repos/goodfeli/cleverhans/git/trees{/sha}","statuses_url":"https://api.github.com/repos/goodfeli/cleverhans/statuses/{sha}","languages_url":"https://api.github.com/repos/goodfeli/cleverhans/languages","stargazers_url":"https://api.github.com/repos/goodfeli/cleverhans/stargazers","contributors_url":"https://api.github.com/repos/goodfeli/cleverhans/contributors","subscribers_url":"https://api.github.com/repos/goodfeli/cleverhans/subscribers","subscription_url":"https://api.github.com/repos/goodfeli/cleverhans/subscription","commits_url":"https://api.github.com/repos/goodfeli/cleverhans/commits{/sha}","git_commits_url":"https://api.github.com/repos/goodfeli/cleverhans/git/commits{/sha}","comments_url":"https://api.github.com/repos/goodfeli/cleverhans/comments{/number}","issue_comment_url":"https://api.github.com/repos/goodfeli/cleverhans/issues/comments{/number}","contents_url":"https://api.github.com/repos/goodfeli/cleverhans/contents/{+path}","compare_url":"https://api.github.com/repos/goodfeli/cleverhans/compare/{base}...{head}","merges_url":"https://api.github.com/repos/goodfeli/cleverhans/merges","archive_url":"https://api.github.com/repos/goodfeli/cleverhans/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/goodfeli/cleverhans/downloads","issues_url":"https://api.github.com/repos/goodfeli/cleverhans/issues{/number}","pulls_url":"https://api.github.com/repos/goodfeli/cleverhans/pulls{/number}","milestones_url":"https://api.github.com/repos/goodfeli/cleverhans/milestones{/number}","notifications_url":"https://api.github.com/repos/goodfeli/cleverhans/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/goodfeli/cleverhans/labels{/name}","releases_url":"https://api.github.com/repos/goodfeli/cleverhans/releases{/id}","deployments_url":"https://api.github.com/repos/goodfeli/cleverhans/deployments","created_at":"2017-06-05T23:34:54Z","updated_at":"2018-04-26T07:28:45Z","pushed_at":"2018-09-11T03:47:11Z","git_url":"git://github.com/goodfeli/cleverhans.git","ssh_url":"git@github.com:goodfeli/cleverhans.git","clone_url":"https://github.com/goodfeli/cleverhans.git","svn_url":"https://github.com/goodfeli/cleverhans","homepage":null,"size":4211,"stargazers_count":2,"watchers_count":2,"language":"Python","has_issues":false,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":4,"mirror_url":null,"archived":false,"open_issues_count":0,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit","node_id":"MDc6TGljZW5zZTEz"},"forks":4,"open_issues":0,"watchers":2,"default_branch":"master"}},"base":{"label":"tensorflow:master","ref":"master","sha":"e5e42e03a9fa7d55fd98951146ee085715febfd4","user":{"login":"tensorflow","id":15658638,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE1NjU4NjM4","avatar_url":"https://avatars1.githubusercontent.com/u/15658638?v=4","gravatar_id":"","url":"https://api.github.com/users/tensorflow","html_url":"https://github.com/tensorflow","followers_url":"https://api.github.com/users/tensorflow/followers","following_url":"https://api.github.com/users/tensorflow/following{/other_user}","gists_url":"https://api.github.com/users/tensorflow/gists{/gist_id}","starred_url":"https://api.github.com/users/tensorflow/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tensorflow/subscriptions","organizations_url":"https://api.github.com/users/tensorflow/orgs","repos_url":"https://api.github.com/users/tensorflow/repos","events_url":"https://api.github.com/users/tensorflow/events{/privacy}","received_events_url":"https://api.github.com/users/tensorflow/received_events","type":"Organization","site_admin":false},"repo":{"id":68254498,"node_id":"MDEwOlJlcG9zaXRvcnk2ODI1NDQ5OA==","name":"cleverhans","full_name":"tensorflow/cleverhans","owner":{"login":"tensorflow","id":15658638,"node_id":"MDEyOk9yZ2FuaXphdGlvbjE1NjU4NjM4","avatar_url":"https://avatars1.githubusercontent.com/u/15658638?v=4","gravatar_id":"","url":"https://api.github.com/users/tensorflow","html_url":"https://github.com/tensorflow","followers_url":"https://api.github.com/users/tensorflow/followers","following_url":"https://api.github.com/users/tensorflow/following{/other_user}","gists_url":"https://api.github.com/users/tensorflow/gists{/gist_id}","starred_url":"https://api.github.com/users/tensorflow/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tensorflow/subscriptions","organizations_url":"https://api.github.com/users/tensorflow/orgs","repos_url":"https://api.github.com/users/tensorflow/repos","events_url":"https://api.github.com/users/tensorflow/events{/privacy}","received_events_url":"https://api.github.com/users/tensorflow/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/tensorflow/cleverhans","description":"An adversarial example library for constructing attacks, building defenses, and benchmarking both","fork":false,"url":"https://api.github.com/repos/tensorflow/cleverhans","forks_url":"https://api.github.com/repos/tensorflow/cleverhans/forks","keys_url":"https://api.github.com/repos/tensorflow/cleverhans/keys{/key_id}","collaborators_url":"https://api.github.com/repos/tensorflow/cleverhans/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/tensorflow/cleverhans/teams","hooks_url":"https://api.github.com/repos/tensorflow/cleverhans/hooks","issue_events_url":"https://api.github.com/repos/tensorflow/cleverhans/issues/events{/number}","events_url":"https://api.github.com/repos/tensorflow/cleverhans/events","assignees_url":"https://api.github.com/repos/tensorflow/cleverhans/assignees{/user}","branches_url":"https://api.github.com/repos/tensorflow/cleverhans/branches{/branch}","tags_url":"https://api.github.com/repos/tensorflow/cleverhans/tags","blobs_url":"https://api.github.com/repos/tensorflow/cleverhans/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/tensorflow/cleverhans/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/tensorflow/cleverhans/git/refs{/sha}","trees_url":"https://api.github.com/repos/tensorflow/cleverhans/git/trees{/sha}","statuses_url":"https://api.github.com/repos/tensorflow/cleverhans/statuses/{sha}","languages_url":"https://api.github.com/repos/tensorflow/cleverhans/languages","stargazers_url":"https://api.github.com/repos/tensorflow/cleverhans/stargazers","contributors_url":"https://api.github.com/repos/tensorflow/cleverhans/contributors","subscribers_url":"https://api.github.com/repos/tensorflow/cleverhans/subscribers","subscription_url":"https://api.github.com/repos/tensorflow/cleverhans/subscription","commits_url":"https://api.github.com/repos/tensorflow/cleverhans/commits{/sha}","git_commits_url":"https://api.github.com/repos/tensorflow/cleverhans/git/commits{/sha}","comments_url":"https://api.github.com/repos/tensorflow/cleverhans/comments{/number}","issue_comment_url":"https://api.github.com/repos/tensorflow/cleverhans/issues/comments{/number}","contents_url":"https://api.github.com/repos/tensorflow/cleverhans/contents/{+path}","compare_url":"https://api.github.com/repos/tensorflow/cleverhans/compare/{base}...{head}","merges_url":"https://api.github.com/repos/tensorflow/cleverhans/merges","archive_url":"https://api.github.com/repos/tensorflow/cleverhans/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/tensorflow/cleverhans/downloads","issues_url":"https://api.github.com/repos/tensorflow/cleverhans/issues{/number}","pulls_url":"https://api.github.com/repos/tensorflow/cleverhans/pulls{/number}","milestones_url":"https://api.github.com/repos/tensorflow/cleverhans/milestones{/number}","notifications_url":"https://api.github.com/repos/tensorflow/cleverhans/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/tensorflow/cleverhans/labels{/name}","releases_url":"https://api.github.com/repos/tensorflow/cleverhans/releases{/id}","deployments_url":"https://api.github.com/repos/tensorflow/cleverhans/deployments","created_at":"2016-09-15T00:28:04Z","updated_at":"2018-09-11T00:51:41Z","pushed_at":"2018-09-11T03:47:13Z","git_url":"git://github.com/tensorflow/cleverhans.git","ssh_url":"git@github.com:tensorflow/cleverhans.git","clone_url":"https://github.com/tensorflow/cleverhans.git","svn_url":"https://github.com/tensorflow/cleverhans","homepage":"","size":4184,"stargazers_count":2946,"watchers_count":2946,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":643,"mirror_url":null,"archived":false,"open_issues_count":27,"license":{"key":"mit","name":"MIT License","spdx_id":"MIT","url":"https://api.github.com/licenses/mit","node_id":"MDc6TGljZW5zZTEz"},"forks":643,"open_issues":27,"watchers":2946,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/tensorflow/cleverhans/pulls/551"},"html":{"href":"https://github.com/tensorflow/cleverhans/pull/551"},"issue":{"href":"https://api.github.com/repos/tensorflow/cleverhans/issues/551"},"comments":{"href":"https://api.github.com/repos/tensorflow/cleverhans/issues/551/comments"},"review_comments":{"href":"https://api.github.com/repos/tensorflow/cleverhans/pulls/551/comments"},"review_comment":{"href":"https://api.github.com/repos/tensorflow/cleverhans/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/tensorflow/cleverhans/pulls/551/commits"},"statuses":{"href":"https://api.github.com/repos/tensorflow/cleverhans/statuses/d941d348faf76d4d529595cc3926b59106ef5fcc"}},"author_association":"COLLABORATOR"}}
{"actor":{"display_login":"goodfeli"}}