{"action":"created","comment":{"url":"https://api.github.com/repos/awslabs/sockeye/pulls/comments/182793807","pull_request_review_id":113661421,"id":182793807,"diff_hunk":"@@ -0,0 +1,474 @@\n+# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.\n+#\n+# Licensed under the Apache License, Version 2.0 (the \"License\"). You may not\n+# use this file except in compliance with the License. A copy of the License\n+# is located at\n+#\n+#     http://aws.amazon.com/apache2.0/\n+#\n+# or in the \"license\" file accompanying this file. This file is distributed on\n+# an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either\n+# express or implied. See the License for the specific language governing\n+# permissions and limitations under the License.\n+\n+\"\"\"\n+Implement scoring of existing translations. Scores are obtained by forwarding\n+source sentences through a trained model and computing the probability assigned\n+to an existing target sentence.\n+\"\"\"\n+import os\n+import logging\n+import argparse\n+from typing import Optional, List, Tuple, cast, DefaultDict\n+\n+import mxnet as mx\n+import numpy as np\n+\n+from . import config\n+from . import constants as C\n+from . import data_io\n+from . import vocab\n+from . import model\n+from . import utils\n+from . import loss\n+\n+from sockeye.utils import check_condition\n+\n+logger = logging.getLogger(__name__)\n+\n+\n+class ScoringModel(model.SockeyeModel):\n+    \"\"\"\n+    Defines a model to score input/output data.\n+\n+    :param context: The context(s) that MXNet will be run in (GPU(s)/CPU).\n+    :param data_iter: The iterator over the data set.\n+    :param config: Configuration object holding details about the model.\n+    :param checkpoint: Checkpoint to load. If None, finds best parameters in\n+    model_folder.\n+    :param bucketing: If True bucketing will be used, if False the computation\n+    graph will always be unrolled to the full length.\n+    \"\"\"\n+\n+    def __init__(self,\n+                 model_folder: str,\n+                 context: List[mx.context.Context],\n+                 data_iter: data_io.BaseParallelSampleIter,\n+                 config: model.ModelConfig,\n+                 checkpoint: Optional[int] = None,\n+                 bucketing: bool = None) -> None:\n+        super().__init__(config)\n+        self.context = context\n+        self.bucketing = bucketing\n+        self.module = self._build_module(data_iter)\n+\n+        self.config = config\n+        self.batch_size = data_iter.batch_size\n+        self.fname_params = os.path.join(model_folder, C.PARAMS_NAME % checkpoint if checkpoint else C.PARAMS_BEST_NAME)\n+\n+        self.module.bind(data_shapes=data_iter.provide_data, label_shapes=data_iter.provide_label, for_training=False, force_rebind=True, grad_req='null')\n+\n+        self.load_params_from_file(self.fname_params)\n+        self.module.init_params(arg_params=self.params, allow_missing=False)\n+\n+    def _build_module(self, data_iter: data_io.BaseParallelSampleIter):\n+        \"\"\"\n+        Initializes model components, creates training symbol and module, and binds it.\n+\n+        :param data_iter: Iterator that provides data and labels.\n+        \"\"\"\n+\n+        source = mx.sym.Variable(C.SOURCE_NAME)\n+        source_words = source.split(num_outputs=self.config.config_embed_source.num_factors,\n+                                    axis=2, squeeze_axis=True)[0]\n+        source_length = utils.compute_lengths(source_words)\n+        target = mx.sym.Variable(C.TARGET_NAME)\n+        target_length = utils.compute_lengths(target)\n+        labels = mx.sym.reshape(data=mx.sym.Variable(C.TARGET_LABEL_NAME), shape=(-1,))\n+\n+        model_loss = loss.get_loss(self.config.config_loss)\n+\n+        data_names = [x[0] for x in data_iter.provide_data]\n+        label_names = [x[0] for x in data_iter.provide_label]\n+\n+        def sym_gen(seq_lens):\n+            \"\"\"\n+            Returns a (grouped) loss symbol given source & target input lengths.\n+            Also returns data and label names for the BucketingModule.\n+            \"\"\"\n+            source_seq_len, target_seq_len = seq_lens\n+\n+            # source embedding\n+            (source_embed,\n+             source_embed_length,\n+             source_embed_seq_len) = self.embedding_source.encode(source, source_length, source_seq_len)\n+\n+            # target embedding\n+            (target_embed,\n+             target_embed_length,\n+             target_embed_seq_len) = self.embedding_target.encode(target, target_length, target_seq_len)\n+\n+            # encoder\n+            # source_encoded: (batch_size, source_encoded_length, encoder_depth)\n+            (source_encoded,\n+             source_encoded_length,\n+             source_encoded_seq_len) = self.encoder.encode(source_embed,\n+                                                           source_embed_length,\n+                                                           source_embed_seq_len)\n+\n+            # decoder\n+            # target_decoded: (batch-size, target_len, decoder_depth)\n+            target_decoded = self.decoder.decode_sequence(source_encoded, source_encoded_length, source_encoded_seq_len,\n+                                                          target_embed, target_embed_length, target_embed_seq_len)\n+\n+            # target_decoded: (batch_size * target_seq_len, rnn_num_hidden)\n+            target_decoded = mx.sym.reshape(data=target_decoded, shape=(-3, 0))\n+\n+            # output layer\n+            logits = self.output_layer(target_decoded)\n+\n+            probs = model_loss.get_loss(logits, labels)\n+            return mx.symbol.Group(probs), data_names, label_names\n+\n+        if self.bucketing:\n+            logger.info(\"Using bucketing. Default max_seq_len=%s\", data_iter.default_bucket_key)\n+            return mx.mod.BucketingModule(sym_gen=sym_gen,\n+                                          logger=logger,\n+                                          default_bucket_key=data_iter.default_bucket_key,\n+                                          context=self.context)\n+        else:\n+            logger.info(\"No bucketing. Unrolled to (%d,%d)\",\n+                        self.config.config_data.max_seq_len_source, self.config.config_data.max_seq_len_target)\n+            symbol, _, __ = sym_gen(data_iter.buckets[0])\n+            return mx.mod.Module(symbol=symbol,\n+                                 data_names=data_names,\n+                                 label_names=label_names,\n+                                 logger=logger,\n+                                 context=self.context)\n+\n+\n+class ScoringOutput:\n+    \"\"\"\n+    Wraps the output of scoring.\n+\n+    :param sentence_id: Id of input sentence.\n+    :param source_tokens: Tokens of the source sentence.\n+    :param target_tokens: Tokens of the target sentence.\n+    :param scores: Score obtained by this sentence pair, an average score\n+    for an ensemble of models.\n+    \"\"\"\n+\n+    __slots__ = ('sentence_id', 'source_tokens', 'target_tokens', 'score')\n+\n+    def __init__(self,\n+                 sentence_id: int,\n+                 source_tokens: Optional[vocab.Tokens],\n+                 target_tokens: Optional[vocab.Tokens],\n+                 score: float) -> None:\n+        self.sentence_id = sentence_id\n+        self.source_tokens = source_tokens\n+        self.target_tokens = target_tokens\n+        self.score = score\n+\n+    def __str__(self):\n+        \"\"\"\n+        Returns a string representation of a scoring output.\n+        \"\"\"\n+        return 'ScoringOutput(%d, %s, %s, %f)' % (self.sentence_id, self.source_tokens, self.target_tokens, self.score)\n+\n+\n+ScoredBatch = List[Tuple[int, float]]\n+ScoredSamples = List[ScoredBatch]\n+\n+\n+class Scorer:\n+    \"\"\"\n+    Scorer uses one or several models to score pairs of input strings.\n+\n+    :param batch_size: Batch size.\n+    :param context: The context(s) that MXNet will be run in (GPU(s)/CPU).\n+    :param no_bucketing: If False bucketing will be used, if True the\n+    computation graph will always be unrolled to the full length.\n+    :param normalize: If True, normalize scores by the length of the target\n+    sequence.\n+    \"\"\"\n+    def __init__(self,\n+                 batch_size: int,\n+                 context: List[mx.context.Context],\n+                 no_bucketing: bool = False,\n+                 normalize: Optional[bool] = True) -> None:\n+\n+        self.batch_size = batch_size\n+        self.context = context\n+        self.no_bucketing = no_bucketing\n+        self.normalize = normalize\n+\n+    @staticmethod\n+    def _linear_interpolation(predictions):\n+        return np.mean(predictions, axis=0)\n+\n+    def _score_batch(self,\n+                     model: ScoringModel,\n+                     batch: mx.io.DataBatch,\n+                     batch_index: Tuple[int, int],\n+                     mapid: np.ndarray\n+                     ) -> ScoredBatch:\n+        \"\"\"\n+        Scores a batch of examples, returns a list of tuples (sentence_id, score).\n+        :param model: The model used for scoring, an instance of ScoringModel.\n+        :param batch: A batch of data that is forwarded through the model.\n+        :param batch_index: An identifier for the specific batch.\n+        :param mapid: Nested dictionary mapping  positions in buckets to the\n+        original ordering in the input.\n+        :return: A ScoredBatch which is a list of tuples (sentence_id, score).\n+        \"\"\"\n+        scored_batch = []\n+\n+        model.module.forward(batch, is_train=False)\n+        # outputs[0]: (batch_size * t_len, t_vocab)\n+        outputs = model.module.get_outputs()\n+\n+        # split output array into probs per batch\n+        sample_length = int(len(outputs[0]) / self.batch_size)\n+\n+        # probs: (batch_size, t_len, t_vocab)\n+        probs = outputs[0].reshape((self.batch_size, sample_length, -1))\n+        probs = probs.as_in_context(self.context)\n+\n+        # get bucket index of batch\n+        (bucket_index, offset) = batch_index\n+\n+        for sample_number, sample_probs in enumerate(probs):\n+\n+            mapsample_number = sample_number + offset\n+\n+            try:\n+                sentence_id = mapid[bucket_index][mapsample_number]\n+            except IndexError:\n+                # this sample is a replica to fill up the batch\n+                continue\n+\n+            labels = batch.label[0][sample_number].as_in_context(self.context)\n+\n+            scores = mx.nd.pick(sample_probs, labels)\n+            # remove scores for padding symbols\n+            scores = mx.nd.take(scores, labels != 0)\n+            log_probs = - mx.nd.log(scores)\n+            log_probs = log_probs.flatten().asnumpy()\n+\n+            # mask INF and NAN values\n+            score = np.ma.masked_invalid(log_probs).sum()\n+            if self.normalize:\n+                score = np.mean(log_probs)\n+\n+            scored_batch.append((sentence_id, score))\n+\n+        return scored_batch\n+\n+    def _score_single_model(self,\n+                            model: ScoringModel,\n+                            data_iter: data_io.BaseParallelSampleIter,\n+                            mapid: np.ndarray) -> ScoredSamples:\n+        \"\"\"\n+        Scores all batches with a single model. Returns a list of\n+        scored samples.\n+\n+        :param model: The model used for scoring, an instance of ScoringModel.\n+        :param data_iter: Iterator that returns batches of data.\n+        :param mapid: Nested dictionary mapping positions in buckets to the\n+        original ordering in the input.\n+        :return: ScoredSamples, which is a list of scored batches.\n+        \"\"\"\n+        scored_samples = []\n+\n+        for i, batch in enumerate(data_iter):\n+\n+                batch_index = data_iter.batch_indices[i]\n+                scored_batch = self._score_batch(model=model,\n+                                                 batch=batch,\n+                                                 batch_index=batch_index,\n+                                                 mapid=mapid)\n+                scored_samples.extend(scored_batch)\n+\n+        return sorted(scored_samples)\n+\n+    def score(self,\n+              models: List[ScoringModel],\n+              data_iters: List[data_io.BaseParallelSampleIter],\n+              mapids: List[np.ndarray]\n+              ) -> List[ScoringOutput]:\n+        \"\"\"\n+        Scores all examples returned by an iterator, once for each model.\n+        Returns a list of ScoringOutputs.\n+\n+        :param models: The models used for scoring, instances of ScoringModel.\n+        :param data_iters: Iterators that return batches of data.\n+        :param mapid: Nested dictionaries mapping  positions in buckets to the\n+        original ordering in the input.\n+        :return: Returns a list of ScoringOutputs.\n+        \"\"\"\n+        scored_samples = []\n+\n+        for model, data_iter, mapid in zip(models, data_iters, mapids):\n+            single_model_scored_samples = self._score_single_model(model=model,\n+                                                                   data_iter=data_iter,\n+                                                                   mapid=mapid)\n+            scored_samples.append(single_model_scored_samples)\n+\n+        return self._make_outputs(scored_samples)\n+\n+    def _make_outputs(self,\n+                      scored_samples: ScoredSamples) -> List[ScoringOutput]:\n+        \"\"\"\n+        Generates a list of ScoringOutputs from scorer output.\n+\n+        :param scored_samples: A list of scored batches.\n+        :return: A list of ScoringOutputs that can be handled by an output\n+        handler.\n+        \"\"\"\n+        scored_outputs = []\n+\n+        for sample in zip(*scored_samples):\n+            scores = [score for (sentence_id, score) in sample]\n+            ensemble_score = self._linear_interpolation(scores)\n+\n+            sentence_id = sample[0][0]\n+\n+            scored_output = ScoringOutput(sentence_id=sentence_id,\n+                                          source_tokens=[],\n+                                          target_tokens=[],\n+                                          score=ensemble_score)\n+            scored_outputs.append(scored_output)\n+\n+        return scored_outputs\n+\n+\n+def load_models(context: mx.context.Context,\n+                batch_size: int,\n+                model_folders: List[str],\n+                data_iters: List[data_io.BaseParallelSampleIter],\n+                configs: List[data_io.DataConfig],\n+                checkpoints: Optional[List[int]] = None,\n+                bucketing: bool = False) -> List[ScoringModel]:\n+\n+    \"\"\"\n+    Loads a list of models for scoring.\n+\n+    :param context: MXNet context to bind modules to.\n+    :param batch_size: Batch size.\n+    :param model_folders: List of model folders to load models from.\n+    :param data_iter: List of iterators over the data set (one per model, can use different vocabularies).\n+    :param configs: List of configuration objects holding details about the models (one per model).\n+    :param checkpoints: List of checkpoints to use for each model in model_folders. Use None to load best checkpoint.\n+    :param bucketing: If True bucketing will be used, if False the computation\n+    graph will always be unrolled to the full length.\n+    :return: List of models.\n+    \"\"\"\n+\n+    models = []\n+    if checkpoints is None:\n+        checkpoints = [None] * len(model_folders)\n+\n+    for model_folder, checkpoint, config, data_iter in zip(model_folders, checkpoints, configs, data_iters):\n+\n+        model = ScoringModel(model_folder=model_folder,\n+                             context=context,\n+                             data_iter=data_iter,\n+                             config=config,\n+                             checkpoint=checkpoint,\n+                             bucketing=bucketing)\n+        models.append(model)\n+\n+    return models\n+\n+\n+def create_data_iter_and_vocab(args: argparse.Namespace,\n+                               max_seq_len_source: int,\n+                               max_seq_len_target: int,\n+                               model_dir: str)-> Tuple['data_io.BaseParallelSampleIter', 'data_io.DataConfig']:\n+    \"\"\"\n+    Create the data iterator.\n+\n+    :param args: Arguments as returned by argparse.\n+    :param max_seq_len_source: Max length input.\n+    :param max_seq_len_target: Max length output.\n+    :model_dir: model folder to load vocabularies and config from.\n+    :return: The data iterator.\n+    \"\"\"\n+    batch_num_devices = 1 if args.use_cpu else sum(-di if di < 0 else 1 for di in args.device_ids)\n+    batch_by_words = args.batch_type == C.BATCH_TYPE_WORD\n+\n+    source_vocabs = vocab.load_source_vocabs(model_dir)\n+    target_vocab = vocab.vocab_from_json(os.path.join(model_dir, C.VOCAB_TRG_NAME))\n+\n+    # Recover the vocabulary path from the data info file:\n+    data_info = cast(data_io.DataInfo, config.Config.load(os.path.join(model_dir, C.DATA_INFO)))\n+\n+    # get max_seq_len_source and max_seq_len_target from config,\n+    # warn if smaller than given values\n+    config_ = model.SockeyeModel.load_config(os.path.join(model_dir, C.CONFIG_NAME))\n+    if max_seq_len_source > config_.config_data.max_seq_len_source:\n+            logger.warning(\"Source sentence of length %d in test set exceeds maximum source sentence length in config of %d\",max_seq_len_source, config_.config_data.max_seq_len_source)\n+    if max_seq_len_target > config_.config_data.max_seq_len_target:\n+            logger.warning(\"Target sentence of length %d in test set exceeds maximum target sentence length in config of %d\",max_seq_len_target, config_.config_data.max_seq_len_target)\n+\n+    check_condition(len(args.source_factors) == len(args.source_factors_num_embed),\n+                        \"Number of source factor data (%d) differs from provided source factor dimensions (%d)\" % (len(args.source_factors), len(args.source_factors_num_embed)))\n+\n+    sources = [args.source] + args.source_factors\n+    sources = [str(os.path.abspath(source)) for source in sources]\n+\n+    # print(\"sources {} source vocabs {}\".format(sources, source_vocabs))\n+    sources_sentences = [data_io.SequenceReader(source, vocab, add_bos=False) for source, vocab in zip(sources, source_vocabs)]\n+    target_sentences = data_io.SequenceReader(args.target, target_vocab, add_bos=True, limit=None)\n+\n+    # Pass 1: get target/source length ratios.\n+    length_statistics = data_io.analyze_sequence_lengths(sources, args.target, source_vocabs, target_vocab, max_seq_len_source, max_seq_len_target)\n+\n+    # define buckets\n+    bucketing = not args.no_bucketing\n+    buckets = data_io.define_parallel_buckets(max_seq_len_source, max_seq_len_target, args.bucket_width, length_statistics.length_ratio_mean) if bucketing else [(max_seq_len_source, max_seq_len_target)]\n+\n+    # get iterator\n+    # Pass 2: Get data statistics\n+    data_statistics = data_io.get_data_statistics(sources_sentences,\n+                                                  target_sentences,\n+                                                  buckets,\n+                                                  length_statistics.length_ratio_mean,\n+                                                  length_statistics.length_ratio_std,\n+                                                  source_vocabs,\n+                                                  target_vocab)\n+\n+    bucket_batch_sizes = data_io.define_bucket_batch_sizes(buckets,\n+                                                           args.batch_size,\n+                                                           batch_by_words,\n+                                                           batch_num_devices,\n+                                                           data_statistics.average_len_target_per_bucket)\n+\n+    data_statistics.log(bucket_batch_sizes)\n+\n+    data_loader = data_io.RawParallelDatasetLoader(buckets=buckets,\n+                                                   eos_id=target_vocab[C.EOS_SYMBOL],\n+                                                   pad_id=C.PAD_ID,\n+                                                   skip_if_no_bucket=False)\n+\n+    parallel_data = data_loader.load(sources_sentences, target_sentences,\n+                                     data_statistics.num_sents_per_bucket).fill_up(bucket_batch_sizes, args.fill_up)\n+    map_buckets2sentence_ids = data_loader.map_buckets2sentence_ids\n+\n+    data_iter = data_io.ParallelSampleIter(parallel_data,\n+                                           buckets,\n+                                           args.batch_size,\n+                                           bucket_batch_sizes,\n+                                           shuffle=False)\n+\n+    return data_iter, config_, map_buckets2sentence_ids\n+\n+\n+def get_max_source_and_target(args: argparse.Namespace) -> Tuple[int, int]:\n+    source_lines = utils.smart_open(args.source).readlines()\n+    target_lines = utils.smart_open(args.target).readlines()\n+    max_len_source = max([len(line.rstrip().split()) for line in source_lines])","path":"sockeye/scoring.py","position":471,"original_position":471,"commit_id":"bd94d1d55f380c1c2b272232fd6fa75ac5fc3fdb","original_commit_id":"bd94d1d55f380c1c2b272232fd6fa75ac5fc3fdb","user":{"login":"tdomhan","id":363737,"avatar_url":"https://avatars1.githubusercontent.com/u/363737?v=4","gravatar_id":"","url":"https://api.github.com/users/tdomhan","html_url":"https://github.com/tdomhan","followers_url":"https://api.github.com/users/tdomhan/followers","following_url":"https://api.github.com/users/tdomhan/following{/other_user}","gists_url":"https://api.github.com/users/tdomhan/gists{/gist_id}","starred_url":"https://api.github.com/users/tdomhan/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tdomhan/subscriptions","organizations_url":"https://api.github.com/users/tdomhan/orgs","repos_url":"https://api.github.com/users/tdomhan/repos","events_url":"https://api.github.com/users/tdomhan/events{/privacy}","received_events_url":"https://api.github.com/users/tdomhan/received_events","type":"User","site_admin":false},"body":"you could use `data_io.read_content` here and maybe a generator instead of constructing the full list:\r\n```\r\nmax([len(sent) for sent in data_io.read_content(source))\r\n```","created_at":"2018-04-19T15:45:19Z","updated_at":"2018-04-19T16:00:29Z","html_url":"https://github.com/awslabs/sockeye/pull/341#discussion_r182793807","pull_request_url":"https://api.github.com/repos/awslabs/sockeye/pulls/341","author_association":"CONTRIBUTOR","_links":{"self":{"href":"https://api.github.com/repos/awslabs/sockeye/pulls/comments/182793807"},"html":{"href":"https://github.com/awslabs/sockeye/pull/341#discussion_r182793807"},"pull_request":{"href":"https://api.github.com/repos/awslabs/sockeye/pulls/341"}}},"pull_request":{"url":"https://api.github.com/repos/awslabs/sockeye/pulls/341","id":178003485,"html_url":"https://github.com/awslabs/sockeye/pull/341","diff_url":"https://github.com/awslabs/sockeye/pull/341.diff","patch_url":"https://github.com/awslabs/sockeye/pull/341.patch","issue_url":"https://api.github.com/repos/awslabs/sockeye/issues/341","number":341,"state":"open","locked":false,"title":"[WIP] Scoring of existing translations","user":{"login":"bricksdont","id":12611348,"avatar_url":"https://avatars2.githubusercontent.com/u/12611348?v=4","gravatar_id":"","url":"https://api.github.com/users/bricksdont","html_url":"https://github.com/bricksdont","followers_url":"https://api.github.com/users/bricksdont/followers","following_url":"https://api.github.com/users/bricksdont/following{/other_user}","gists_url":"https://api.github.com/users/bricksdont/gists{/gist_id}","starred_url":"https://api.github.com/users/bricksdont/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bricksdont/subscriptions","organizations_url":"https://api.github.com/users/bricksdont/orgs","repos_url":"https://api.github.com/users/bricksdont/repos","events_url":"https://api.github.com/users/bricksdont/events{/privacy}","received_events_url":"https://api.github.com/users/bricksdont/received_events","type":"User","site_admin":false},"body":"Adding classes and cli for scoring existing translations. Work in progress, your feedback is very welcome.\r\n\r\n#### Pull Request Checklist ##\r\n- [ ] Changes are complete (if posting work-in-progress code, prefix your pull request title with '[WIP]'\r\nuntil you can check this box.\r\n- [ ] Unit tests pass (`pytest`)\r\n- [ ] Were system tests modified? If so did you run these at least 5 times to account for the variation across runs?\r\n- [ ] System tests pass (`pytest test/system`)\r\n- [ ] Passed code style checking (`./style-check.sh`)\r\n- [ ] You have considered writing a test\r\n- [ ] Updated major/minor version in `sockeye/__init__.py`. Major version bump if this is a backwards incompatible change.\r\n- [ ] Updated CHANGELOG.md\r\n\r\n\r\nBy submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.\r\n\r\n","created_at":"2018-03-28T11:07:26Z","updated_at":"2018-04-19T16:00:30Z","closed_at":null,"merged_at":null,"merge_commit_sha":"03cbd211c2bca7aaa577b8a908eb1962b8fee6d1","assignee":null,"assignees":[],"requested_reviewers":[{"login":"mjpost","id":455256,"avatar_url":"https://avatars0.githubusercontent.com/u/455256?v=4","gravatar_id":"","url":"https://api.github.com/users/mjpost","html_url":"https://github.com/mjpost","followers_url":"https://api.github.com/users/mjpost/followers","following_url":"https://api.github.com/users/mjpost/following{/other_user}","gists_url":"https://api.github.com/users/mjpost/gists{/gist_id}","starred_url":"https://api.github.com/users/mjpost/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mjpost/subscriptions","organizations_url":"https://api.github.com/users/mjpost/orgs","repos_url":"https://api.github.com/users/mjpost/repos","events_url":"https://api.github.com/users/mjpost/events{/privacy}","received_events_url":"https://api.github.com/users/mjpost/received_events","type":"User","site_admin":false},{"login":"mjdenkowski","id":636653,"avatar_url":"https://avatars3.githubusercontent.com/u/636653?v=4","gravatar_id":"","url":"https://api.github.com/users/mjdenkowski","html_url":"https://github.com/mjdenkowski","followers_url":"https://api.github.com/users/mjdenkowski/followers","following_url":"https://api.github.com/users/mjdenkowski/following{/other_user}","gists_url":"https://api.github.com/users/mjdenkowski/gists{/gist_id}","starred_url":"https://api.github.com/users/mjdenkowski/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mjdenkowski/subscriptions","organizations_url":"https://api.github.com/users/mjdenkowski/orgs","repos_url":"https://api.github.com/users/mjdenkowski/repos","events_url":"https://api.github.com/users/mjdenkowski/events{/privacy}","received_events_url":"https://api.github.com/users/mjdenkowski/received_events","type":"User","site_admin":false},{"login":"davvil","id":683589,"avatar_url":"https://avatars0.githubusercontent.com/u/683589?v=4","gravatar_id":"","url":"https://api.github.com/users/davvil","html_url":"https://github.com/davvil","followers_url":"https://api.github.com/users/davvil/followers","following_url":"https://api.github.com/users/davvil/following{/other_user}","gists_url":"https://api.github.com/users/davvil/gists{/gist_id}","starred_url":"https://api.github.com/users/davvil/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/davvil/subscriptions","organizations_url":"https://api.github.com/users/davvil/orgs","repos_url":"https://api.github.com/users/davvil/repos","events_url":"https://api.github.com/users/davvil/events{/privacy}","received_events_url":"https://api.github.com/users/davvil/received_events","type":"User","site_admin":false}],"requested_teams":[],"labels":[{"id":837948970,"url":"https://api.github.com/repos/awslabs/sockeye/labels/feature","name":"feature","color":"0e8a16","default":false}],"milestone":null,"commits_url":"https://api.github.com/repos/awslabs/sockeye/pulls/341/commits","review_comments_url":"https://api.github.com/repos/awslabs/sockeye/pulls/341/comments","review_comment_url":"https://api.github.com/repos/awslabs/sockeye/pulls/comments{/number}","comments_url":"https://api.github.com/repos/awslabs/sockeye/issues/341/comments","statuses_url":"https://api.github.com/repos/awslabs/sockeye/statuses/bd94d1d55f380c1c2b272232fd6fa75ac5fc3fdb","head":{"label":"ZurichNLP:scoring","ref":"scoring","sha":"bd94d1d55f380c1c2b272232fd6fa75ac5fc3fdb","user":{"login":"ZurichNLP","id":35448498,"avatar_url":"https://avatars3.githubusercontent.com/u/35448498?v=4","gravatar_id":"","url":"https://api.github.com/users/ZurichNLP","html_url":"https://github.com/ZurichNLP","followers_url":"https://api.github.com/users/ZurichNLP/followers","following_url":"https://api.github.com/users/ZurichNLP/following{/other_user}","gists_url":"https://api.github.com/users/ZurichNLP/gists{/gist_id}","starred_url":"https://api.github.com/users/ZurichNLP/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ZurichNLP/subscriptions","organizations_url":"https://api.github.com/users/ZurichNLP/orgs","repos_url":"https://api.github.com/users/ZurichNLP/repos","events_url":"https://api.github.com/users/ZurichNLP/events{/privacy}","received_events_url":"https://api.github.com/users/ZurichNLP/received_events","type":"Organization","site_admin":false},"repo":{"id":119822925,"name":"sockeye","full_name":"ZurichNLP/sockeye","owner":{"login":"ZurichNLP","id":35448498,"avatar_url":"https://avatars3.githubusercontent.com/u/35448498?v=4","gravatar_id":"","url":"https://api.github.com/users/ZurichNLP","html_url":"https://github.com/ZurichNLP","followers_url":"https://api.github.com/users/ZurichNLP/followers","following_url":"https://api.github.com/users/ZurichNLP/following{/other_user}","gists_url":"https://api.github.com/users/ZurichNLP/gists{/gist_id}","starred_url":"https://api.github.com/users/ZurichNLP/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ZurichNLP/subscriptions","organizations_url":"https://api.github.com/users/ZurichNLP/orgs","repos_url":"https://api.github.com/users/ZurichNLP/repos","events_url":"https://api.github.com/users/ZurichNLP/events{/privacy}","received_events_url":"https://api.github.com/users/ZurichNLP/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/ZurichNLP/sockeye","description":"Sequence-to-sequence framework with a focus on Neural Machine Translation based on Apache MXNet","fork":true,"url":"https://api.github.com/repos/ZurichNLP/sockeye","forks_url":"https://api.github.com/repos/ZurichNLP/sockeye/forks","keys_url":"https://api.github.com/repos/ZurichNLP/sockeye/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ZurichNLP/sockeye/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ZurichNLP/sockeye/teams","hooks_url":"https://api.github.com/repos/ZurichNLP/sockeye/hooks","issue_events_url":"https://api.github.com/repos/ZurichNLP/sockeye/issues/events{/number}","events_url":"https://api.github.com/repos/ZurichNLP/sockeye/events","assignees_url":"https://api.github.com/repos/ZurichNLP/sockeye/assignees{/user}","branches_url":"https://api.github.com/repos/ZurichNLP/sockeye/branches{/branch}","tags_url":"https://api.github.com/repos/ZurichNLP/sockeye/tags","blobs_url":"https://api.github.com/repos/ZurichNLP/sockeye/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ZurichNLP/sockeye/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ZurichNLP/sockeye/git/refs{/sha}","trees_url":"https://api.github.com/repos/ZurichNLP/sockeye/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ZurichNLP/sockeye/statuses/{sha}","languages_url":"https://api.github.com/repos/ZurichNLP/sockeye/languages","stargazers_url":"https://api.github.com/repos/ZurichNLP/sockeye/stargazers","contributors_url":"https://api.github.com/repos/ZurichNLP/sockeye/contributors","subscribers_url":"https://api.github.com/repos/ZurichNLP/sockeye/subscribers","subscription_url":"https://api.github.com/repos/ZurichNLP/sockeye/subscription","commits_url":"https://api.github.com/repos/ZurichNLP/sockeye/commits{/sha}","git_commits_url":"https://api.github.com/repos/ZurichNLP/sockeye/git/commits{/sha}","comments_url":"https://api.github.com/repos/ZurichNLP/sockeye/comments{/number}","issue_comment_url":"https://api.github.com/repos/ZurichNLP/sockeye/issues/comments{/number}","contents_url":"https://api.github.com/repos/ZurichNLP/sockeye/contents/{+path}","compare_url":"https://api.github.com/repos/ZurichNLP/sockeye/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ZurichNLP/sockeye/merges","archive_url":"https://api.github.com/repos/ZurichNLP/sockeye/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ZurichNLP/sockeye/downloads","issues_url":"https://api.github.com/repos/ZurichNLP/sockeye/issues{/number}","pulls_url":"https://api.github.com/repos/ZurichNLP/sockeye/pulls{/number}","milestones_url":"https://api.github.com/repos/ZurichNLP/sockeye/milestones{/number}","notifications_url":"https://api.github.com/repos/ZurichNLP/sockeye/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ZurichNLP/sockeye/labels{/name}","releases_url":"https://api.github.com/repos/ZurichNLP/sockeye/releases{/id}","deployments_url":"https://api.github.com/repos/ZurichNLP/sockeye/deployments","created_at":"2018-02-01T10:50:51Z","updated_at":"2018-03-27T16:51:45Z","pushed_at":"2018-04-12T15:36:00Z","git_url":"git://github.com/ZurichNLP/sockeye.git","ssh_url":"git@github.com:ZurichNLP/sockeye.git","clone_url":"https://github.com/ZurichNLP/sockeye.git","svn_url":"https://github.com/ZurichNLP/sockeye","homepage":"","size":5498,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"archived":false,"open_issues_count":0,"license":null,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"awslabs:master","ref":"master","sha":"ad74b9cb9b72b6aa39d5eb2eec04f99690807290","user":{"login":"awslabs","id":3299148,"avatar_url":"https://avatars0.githubusercontent.com/u/3299148?v=4","gravatar_id":"","url":"https://api.github.com/users/awslabs","html_url":"https://github.com/awslabs","followers_url":"https://api.github.com/users/awslabs/followers","following_url":"https://api.github.com/users/awslabs/following{/other_user}","gists_url":"https://api.github.com/users/awslabs/gists{/gist_id}","starred_url":"https://api.github.com/users/awslabs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/awslabs/subscriptions","organizations_url":"https://api.github.com/users/awslabs/orgs","repos_url":"https://api.github.com/users/awslabs/repos","events_url":"https://api.github.com/users/awslabs/events{/privacy}","received_events_url":"https://api.github.com/users/awslabs/received_events","type":"Organization","site_admin":false},"repo":{"id":93721432,"name":"sockeye","full_name":"awslabs/sockeye","owner":{"login":"awslabs","id":3299148,"avatar_url":"https://avatars0.githubusercontent.com/u/3299148?v=4","gravatar_id":"","url":"https://api.github.com/users/awslabs","html_url":"https://github.com/awslabs","followers_url":"https://api.github.com/users/awslabs/followers","following_url":"https://api.github.com/users/awslabs/following{/other_user}","gists_url":"https://api.github.com/users/awslabs/gists{/gist_id}","starred_url":"https://api.github.com/users/awslabs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/awslabs/subscriptions","organizations_url":"https://api.github.com/users/awslabs/orgs","repos_url":"https://api.github.com/users/awslabs/repos","events_url":"https://api.github.com/users/awslabs/events{/privacy}","received_events_url":"https://api.github.com/users/awslabs/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/awslabs/sockeye","description":"Sequence-to-sequence framework with a focus on Neural Machine Translation based on Apache MXNet","fork":false,"url":"https://api.github.com/repos/awslabs/sockeye","forks_url":"https://api.github.com/repos/awslabs/sockeye/forks","keys_url":"https://api.github.com/repos/awslabs/sockeye/keys{/key_id}","collaborators_url":"https://api.github.com/repos/awslabs/sockeye/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/awslabs/sockeye/teams","hooks_url":"https://api.github.com/repos/awslabs/sockeye/hooks","issue_events_url":"https://api.github.com/repos/awslabs/sockeye/issues/events{/number}","events_url":"https://api.github.com/repos/awslabs/sockeye/events","assignees_url":"https://api.github.com/repos/awslabs/sockeye/assignees{/user}","branches_url":"https://api.github.com/repos/awslabs/sockeye/branches{/branch}","tags_url":"https://api.github.com/repos/awslabs/sockeye/tags","blobs_url":"https://api.github.com/repos/awslabs/sockeye/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/awslabs/sockeye/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/awslabs/sockeye/git/refs{/sha}","trees_url":"https://api.github.com/repos/awslabs/sockeye/git/trees{/sha}","statuses_url":"https://api.github.com/repos/awslabs/sockeye/statuses/{sha}","languages_url":"https://api.github.com/repos/awslabs/sockeye/languages","stargazers_url":"https://api.github.com/repos/awslabs/sockeye/stargazers","contributors_url":"https://api.github.com/repos/awslabs/sockeye/contributors","subscribers_url":"https://api.github.com/repos/awslabs/sockeye/subscribers","subscription_url":"https://api.github.com/repos/awslabs/sockeye/subscription","commits_url":"https://api.github.com/repos/awslabs/sockeye/commits{/sha}","git_commits_url":"https://api.github.com/repos/awslabs/sockeye/git/commits{/sha}","comments_url":"https://api.github.com/repos/awslabs/sockeye/comments{/number}","issue_comment_url":"https://api.github.com/repos/awslabs/sockeye/issues/comments{/number}","contents_url":"https://api.github.com/repos/awslabs/sockeye/contents/{+path}","compare_url":"https://api.github.com/repos/awslabs/sockeye/compare/{base}...{head}","merges_url":"https://api.github.com/repos/awslabs/sockeye/merges","archive_url":"https://api.github.com/repos/awslabs/sockeye/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/awslabs/sockeye/downloads","issues_url":"https://api.github.com/repos/awslabs/sockeye/issues{/number}","pulls_url":"https://api.github.com/repos/awslabs/sockeye/pulls{/number}","milestones_url":"https://api.github.com/repos/awslabs/sockeye/milestones{/number}","notifications_url":"https://api.github.com/repos/awslabs/sockeye/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/awslabs/sockeye/labels{/name}","releases_url":"https://api.github.com/repos/awslabs/sockeye/releases{/id}","deployments_url":"https://api.github.com/repos/awslabs/sockeye/deployments","created_at":"2017-06-08T07:44:30Z","updated_at":"2018-04-19T15:37:01Z","pushed_at":"2018-04-19T15:30:31Z","git_url":"git://github.com/awslabs/sockeye.git","ssh_url":"git@github.com:awslabs/sockeye.git","clone_url":"https://github.com/awslabs/sockeye.git","svn_url":"https://github.com/awslabs/sockeye","homepage":"","size":5469,"stargazers_count":473,"watchers_count":473,"language":"Python","has_issues":true,"has_projects":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":134,"mirror_url":null,"archived":false,"open_issues_count":13,"license":{"key":"apache-2.0","name":"Apache License 2.0","spdx_id":"Apache-2.0","url":"https://api.github.com/licenses/apache-2.0"},"forks":134,"open_issues":13,"watchers":473,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/awslabs/sockeye/pulls/341"},"html":{"href":"https://github.com/awslabs/sockeye/pull/341"},"issue":{"href":"https://api.github.com/repos/awslabs/sockeye/issues/341"},"comments":{"href":"https://api.github.com/repos/awslabs/sockeye/issues/341/comments"},"review_comments":{"href":"https://api.github.com/repos/awslabs/sockeye/pulls/341/comments"},"review_comment":{"href":"https://api.github.com/repos/awslabs/sockeye/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/awslabs/sockeye/pulls/341/commits"},"statuses":{"href":"https://api.github.com/repos/awslabs/sockeye/statuses/bd94d1d55f380c1c2b272232fd6fa75ac5fc3fdb"}},"author_association":"CONTRIBUTOR"}}
{"actor":{"display_login":"tdomhan"}}