title: Reusing Parameters from a layer inside a subnetwork to a layer outside of that subnetwork
body: <p>In my network structure I have a layer of class "rec" named as "output". Within the "unit" of that layer I have several layers, one of the being 'pivot_target_embed_raw'. 
  The layer 'pivot_target_embed_raw' will be loaded from another checkpoint. I now want to use the parameters of 'pivot_target_embed_raw' for my 'source_embed_raw' layer too, which is outside of the unit of 'output' and instead a layer in my network with the same 'network depth' as 'output'. 
  In my config I now tried 2 things, both resulting in different errors: 
  1. For parameter <code>'reuse_params': {'map': {'W': {'reuse_layer': 'pivot_target_embed_raw'}, 'b': None}}</code> resulting in the following error (posting part of the error because I think the simple problem is here the way pivot_target_embed_raw is called so most likely look at 2.)</p>
  
  <pre><code>  File "/u/hilmes/returnn/TFNetworkLayer.py", line 448, in transform_config_dict
      line: for src_name in src_names
      locals:
        src_name = &lt;not found&gt;
        src_names = &lt;local&gt; ['source_embed_raw'], _[0]: {len = 16}
    File "/u/hilmes/returnn/TFNetworkLayer.py", line 449, in &lt;listcomp&gt;
      line: d["sources"] = [
              get_layer(src_name)
              for src_name in src_names
              if not src_name == "none"]
      locals:
        d = &lt;not found&gt;
        get_layer = &lt;local&gt; &lt;function TFNetwork.construct_layer.&lt;locals&gt;.get_layer at 0x7f781e7a6d90&gt;
        src_name = &lt;local&gt; 'source_embed_raw', len = 16
        src_names = &lt;not found&gt;
    File "/u/hilmes/returnn/TFNetwork.py", line 607, in get_layer
      line: return self.construct_layer(net_dict=net_dict, name=src_name)  # set get_layer to wrap construct_layer
      locals:
        self = &lt;local&gt; &lt;TFNetwork 'root' train=&lt;tf.Tensor 'globals/train_flag:0' shape=() dtype=bool&gt;&gt;
        self.construct_layer = &lt;local&gt; &lt;bound method TFNetwork.construct_layer of &lt;TFNetwork 'root' train=&lt;tf.Tensor 'globals/train_flag:0' shape=() dtype=bool&gt;&gt;&gt;
        net_dict = &lt;local&gt; {'dec_03_att_key0': {'from': ['encoder'], 'class': 'linear', 'with_bias': False, 'n_out': 512, 'activation': None, 'forward_weights_init': "variance_scaling_initializer(mode='fan_in', distribution='uniform', scale=1.0)"}, 'enc_06_self_att_lin': {'from': ['enc_06_self_att_att'], 'class': 'linear',..., len = 98
        name = &lt;not found&gt;
        src_name = &lt;local&gt; 'source_embed_raw', len = 16
    File "/u/hilmes/returnn/TFNetwork.py", line 652, in construct_layer
      line: layer_class.transform_config_dict(layer_desc, network=self, get_layer=get_layer)
      locals:
        layer_class = &lt;local&gt; &lt;class 'TFNetworkLayer.LinearLayer'&gt;
        layer_class.transform_config_dict = &lt;local&gt; &lt;bound method LayerBase.transform_config_dict of &lt;class 'TFNetworkLayer.LinearLayer'&gt;&gt;
        layer_desc = &lt;local&gt; {'reuse_params': {'map': {'W': {'reuse_layer': 'pivot_target_embed_raw'}, 'b': None}}, 'with_bias': False, 'n_out': 512, 'sources': [&lt;SourceLayer 'data:data' out_type=Data(shape=(None,), dtype='int32', sparse=True, dim=35356, batch_shape_meta=[B,T|'time:var:extern_data:data'])&gt;], 'activation': None}
        network = &lt;not found&gt;
        self = &lt;local&gt; &lt;TFNetwork 'root' train=&lt;tf.Tensor 'globals/train_flag:0' shape=() dtype=bool&gt;&gt;
        get_layer = &lt;local&gt; &lt;function TFNetwork.construct_layer.&lt;locals&gt;.get_layer at 0x7f781e7a6ea0&gt;
    File "/u/hilmes/returnn/TFNetworkLayer.py", line 456, in transform_config_dict
      line: d["reuse_params"] = ReuseParams.from_config_dict(d["reuse_params"], network=network, get_layer=get_layer)
      locals:
        d = &lt;local&gt; {'reuse_params': {'map': {'W': {'reuse_layer': 'pivot_target_embed_raw'}, 'b': None}}, 'with_bias': False, 'n_out': 512, 'sources': [&lt;SourceLayer 'data:data' out_type=Data(shape=(None,), dtype='int32', sparse=True, dim=35356, batch_shape_meta=[B,T|'time:var:extern_data:data'])&gt;], 'activation': None}
        ReuseParams = &lt;global&gt; &lt;class 'TFNetworkLayer.ReuseParams'&gt;
        ReuseParams.from_config_dict = &lt;global&gt; &lt;bound method ReuseParams.from_config_dict of &lt;class 'TFNetworkLayer.ReuseParams'&gt;&gt;
        network = &lt;local&gt; &lt;TFNetwork 'root' train=&lt;tf.Tensor 'globals/train_flag:0' shape=() dtype=bool&gt;&gt;
        get_layer = &lt;local&gt; &lt;function TFNetwork.construct_layer.&lt;locals&gt;.get_layer at 0x7f781e7a6ea0&gt;
    File "/u/hilmes/returnn/TFNetworkLayer.py", line 1386, in from_config_dict
      line: value["reuse_layer"] = optional_get_layer(value["reuse_layer"])
      locals:
        value = &lt;local&gt; {'reuse_layer': 'pivot_target_embed_raw'}
        optional_get_layer = &lt;local&gt; &lt;function ReuseParams.from_config_dict.&lt;locals&gt;.optional_get_layer at 0x7f781e7a6f28&gt;
    File "/u/hilmes/returnn/TFNetworkLayer.py", line 1362, in optional_get_layer
      line: return get_layer(layer_name)
      locals:
        get_layer = &lt;local&gt; &lt;function TFNetwork.construct_layer.&lt;locals&gt;.get_layer at 0x7f781e7a6ea0&gt;
        layer_name = &lt;local&gt; 'pivot_target_embed_raw', len = 22
    File "/u/hilmes/returnn/TFNetwork.py", line 607, in get_layer
      line: return self.construct_layer(net_dict=net_dict, name=src_name)  # set get_layer to wrap construct_layer
      locals:
        self = &lt;local&gt; &lt;TFNetwork 'root' train=&lt;tf.Tensor 'globals/train_flag:0' shape=() dtype=bool&gt;&gt;
        self.construct_layer = &lt;local&gt; &lt;bound method TFNetwork.construct_layer of &lt;TFNetwork 'root' train=&lt;tf.Tensor 'globals/train_flag:0' shape=() dtype=bool&gt;&gt;&gt;
        net_dict = &lt;local&gt; {'dec_03_att_key0': {'from': ['encoder'], 'class': 'linear', 'with_bias': False, 'n_out': 512, 'activation': None, 'forward_weights_init': "variance_scaling_initializer(mode='fan_in', distribution='uniform', scale=1.0)"}, 'enc_06_self_att_lin': {'from': ['enc_06_self_att_att'], 'class': 'linear',..., len = 98
        name = &lt;not found&gt;
        src_name = &lt;local&gt; 'pivot_target_embed_raw', len = 22
    File "/u/hilmes/returnn/TFNetwork.py", line 643, in construct_layer
      line: raise LayerNotFound("layer %r not found in %r" % (name, self))
      locals:
        LayerNotFound = &lt;global&gt; &lt;class 'TFNetwork.LayerNotFound'&gt;
        name = &lt;local&gt; 'pivot_target_embed_raw', len = 22
        self = &lt;local&gt; &lt;TFNetwork 'root' train=&lt;tf.Tensor 'globals/train_flag:0' shape=() dtype=bool&gt;&gt;
  LayerNotFound: layer 'pivot_target_embed_raw' not found in &lt;TFNetwork 'root' train=&lt;tf.Tensor 'globals/train_flag:0' shape=() dtype=bool&gt;&gt;
  </code></pre>
  
  <p>In the second try I changed the code to <code>'reuse_params': {'map': {'W': {'reuse_layer': 'output/pivot_target_embed_raw'}, 'b': None}}</code>
  Again I get a really long stacktrace beginning with:</p>
  
  <pre><code>ReuseParams: layer 'output/pivot_target_embed_raw' does not exist yet and there is a dependency loop, thus creating it on dummy inputs now
  Exception creating layer root/'source_embed_raw' of class LinearLayer with opts:
  {'activation': None,
   'n_out': 512,
   'name': 'source_embed_raw',
   'network': &lt;TFNetwork 'root' train=&lt;tf.Tensor 'globals/train_flag:0' shape=() dtype=bool&gt;&gt;,
   'output': Data(name='source_embed_raw_output', shape=(None, 512), batch_shape_meta=[B,T|'time:var:extern_data:data',F|512]),
   'reuse_params': &lt;TFNetworkLayer.ReuseParams object at 0x7fcb3e959ac8&gt;,
   'sources': [&lt;SourceLayer 'data:data' out_type=Data(shape=(None,), dtype='int32', sparse=True, dim=35356, batch_shape_meta=[B,T|'time:var:extern_data:data'])&gt;],
   'with_bias': False}
  EXCEPTION
  layer root/'source_embed_raw' output: Data(name='source_embed_raw_output', shape=(None, 512), batch_shape_meta=[B,T|'time:var:extern_data:data',F|512])
  ReuseParams: layer 'output/pivot_target_embed_raw' does not exist yet and there is a dependency loop, thus creating it on dummy inputs now
  Exception creating layer root/'source_embed_raw' of class LinearLayer with opts:
  {'activation': None,
   'n_out': 512,
   'name': 'source_embed_raw',
   'network': &lt;TFNetwork 'root' train=&lt;tf.Tensor 'globals/train_flag:0' shape=() dtype=bool&gt;&gt;,
   'output': Data(name='source_embed_raw_output', shape=(None, 512), batch_shape_meta=[B,T|'time:var:extern_data:data',F|512]),
   'reuse_params': &lt;TFNetworkLayer.ReuseParams object at 0x7fcb3e60e7f0&gt;,
   'sources': [&lt;SourceLayer 'data:data' out_type=Data(shape=(None,), dtype='int32', sparse=True, dim=35356, batch_shape_meta=[B,T|'time:var:extern_data:data'])&gt;],
   'with_bias': False}
  Traceback (most recent call last):
  </code></pre>
  
  <p>and ending in:</p>
  
  <pre><code>  File "/u/hilmes/opt/returnn/lib/python3.5/site-packages/tensorflow/python/ops/variable_scope.py", line 1220, in get_variable
      line: return var_store.get_variable(
                full_name,
                shape=shape,
                dtype=dtype,
                initializer=initializer,
                regularizer=regularizer,
                reuse=reuse,
                trainable=trainable,
                collections=collections,
                caching_device=caching_device,
                partitioner=partitioner,
                validate_shape=validate_shape,
                use_resource=use_resource,
                custom_getter=custom_getter,
                constraint=constraint,
                synchronization=synchronization,
                aggregation=aggregation)
      locals:
        var_store = &lt;local&gt; &lt;tensorflow.python.ops.variable_scope._VariableStore object at 0x7fca58cac198&gt;
        var_store.get_variable = &lt;local&gt; &lt;bound method _VariableStore.get_variable of &lt;tensorflow.python.ops.variable_scope._VariableStore object at 0x7fca58cac198&gt;&gt;
        full_name = &lt;local&gt; 'source_embed_raw/W', len = 18
        shape = &lt;local&gt; (35356, 512)
        dtype = &lt;local&gt; tf.float32
        initializer = &lt;local&gt; &lt;tensorflow.python.ops.init_ops.GlorotUniform object at 0x7fcb3e96a7b8&gt;
        regularizer = &lt;local&gt; None
        reuse = &lt;local&gt; &lt;_ReuseMode.AUTO_REUSE: 1&gt;
        trainable = &lt;local&gt; None
        collections = &lt;local&gt; None
        caching_device = &lt;local&gt; None
        partitioner = &lt;local&gt; None
        validate_shape = &lt;local&gt; True
        use_resource = &lt;local&gt; None
        custom_getter = &lt;local&gt; &lt;function ReuseParams.get_variable_scope.&lt;locals&gt;._variable_custom_getter at 0x7fcb3e9616a8&gt;
        constraint = &lt;local&gt; None
        synchronization = &lt;local&gt; &lt;VariableSynchronization.AUTO: 0&gt;
        aggregation = &lt;local&gt; &lt;VariableAggregation.NONE: 0&gt;
    File "/u/hilmes/opt/returnn/lib/python3.5/site-packages/tensorflow/python/ops/variable_scope.py", line 530, in get_variable
      line: return custom_getter(**custom_getter_kwargs)
      locals:
        custom_getter = &lt;local&gt; &lt;function ReuseParams.get_variable_scope.&lt;locals&gt;._variable_custom_getter at 0x7fcb3e9616a8&gt;
        custom_getter_kwargs = &lt;local&gt; {'use_resource': None, 'caching_device': None, 'collections': None, 'shape': (35356, 512), 'initializer': &lt;tensorflow.python.ops.init_ops.GlorotUn
  iform object at 0x7fcb3e96a7b8&gt;, 'name': 'source_embed_raw/W', 'synchronization': &lt;VariableSynchronization.AUTO: 0&gt;, 'validate_shape': True, 'getter': ..., len = 16
    File "/u/hilmes/returnn/TFNetworkLayer.py", line 1537, in _variable_custom_getter
      line: return self.variable_custom_getter(base_layer=base_layer, **kwargs_)
      locals:
        self = &lt;local&gt; &lt;TFNetworkLayer.ReuseParams object at 0x7fcb3e959ac8&gt;
        self.variable_custom_getter = &lt;local&gt; &lt;bound method ReuseParams.variable_custom_getter of &lt;TFNetworkLayer.ReuseParams object at 0x7fcb3e959ac8&gt;&gt;
        base_layer = &lt;local&gt; &lt;LinearLayer 'source_embed_raw' out_type=Data(shape=(None, 512), batch_shape_meta=[B,T|'time:var:extern_data:data',F|512])&gt;
        kwargs_ = &lt;local&gt; {'aggregation': &lt;VariableAggregation.NONE: 0&gt;, 'partitioner': None, 'caching_device': None, 'use_resource': None, 'getter': &lt;function _VariableStore.get_variable.&lt;locals&gt;._true_getter at 0x7fcb3e961730&gt;, 'name': 'source_embed_raw/W', 'synchronization': &lt;VariableSynchronization.AUTO: 0&gt;, 'validate..., len = 16
    File "/u/hilmes/returnn/TFNetworkLayer.py", line 1575, in variable_custom_getter
      line: return self.param_map[param_name].variable_custom_getter(
              getter=getter, name=name, base_layer=base_layer, **kwargs)
      locals:
        self = &lt;local&gt; &lt;TFNetworkLayer.ReuseParams object at 0x7fcb3e959ac8&gt;
        self.param_map = &lt;local&gt; {'W': &lt;TFNetworkLayer.ReuseParams object at 0x7fcb3e959c18&gt;, 'b': &lt;TFNetworkLayer.ReuseParams object at 0x7fcb3e959dd8&gt;}
        param_name = &lt;local&gt; 'W'
        variable_custom_getter = &lt;not found&gt;
        getter = &lt;local&gt; &lt;function _VariableStore.get_variable.&lt;locals&gt;._true_getter at 0x7fcb3e961730&gt;
        name = &lt;local&gt; 'source_embed_raw/W', len = 18
        base_layer = &lt;local&gt; &lt;LinearLayer 'source_embed_raw' out_type=Data(shape=(None, 512), batch_shape_meta=[B,T|'time:var:extern_data:data',F|512])&gt;
        kwargs = &lt;local&gt; {'partitioner': None, 'caching_device': None, 'use_resource': None, 'dtype': tf.float32, 'synchronization': &lt;VariableSynchronization.AUTO: 0&gt;, 'validate_shape': True, 'initializer': &lt;tensorflow.python.ops.init_ops.GlorotUniform object at 0x7fcb3e96a7b8&gt;, 'regularizer': None, 'constraint': None, '..., len = 14
    File "/u/hilmes/returnn/TFNetworkLayer.py", line 1576, in variable_custom_getter
      line: if self.reuse_layer:
      locals:
        self = &lt;local&gt; &lt;TFNetworkLayer.ReuseParams object at 0x7fcb3e959c18&gt;
        self.reuse_layer = &lt;local&gt; !KeyError: 'output/pivot_target_embed_raw'
    File "/u/hilmes/returnn/TFNetworkLayer.py", line 1495, in reuse_layer
      line: self._reuse_layer = self._reuse_layer.get_layer()
      locals:
        self = &lt;local&gt; &lt;TFNetworkLayer.ReuseParams object at 0x7fcb3e959c18&gt;
        self._reuse_layer = &lt;local&gt; &lt;TFNetworkLayer.ReuseParams.LazyLayerResolver object at 0x7fcb3e959b38&gt;
        self._reuse_layer.get_layer = &lt;local&gt; &lt;bound method ReuseParams.LazyLayerResolver.get_layer of &lt;TFNetworkLayer.ReuseParams.LazyLayerResolver object at 0x7fcb3e959b38&gt;&gt;
    File "/u/hilmes/returnn/TFNetworkLayer.py", line 1424, in get_layer
      line: return self.create_dummy_layer(dep_loop_exception=exc)
      locals:
        self = &lt;local&gt; &lt;TFNetworkLayer.ReuseParams.LazyLayerResolver object at 0x7fcb3e959b38&gt;
        self.create_dummy_layer = &lt;local&gt; &lt;bound method ReuseParams.LazyLayerResolver.create_dummy_layer of &lt;TFNetworkLayer.ReuseParams.LazyLayerResolver object at 0x7fcb3e959b38&gt;&gt;
        dep_loop_exception = &lt;not found&gt;
        exc = &lt;not found&gt;
    File "/u/hilmes/returnn/TFNetworkLayer.py", line 1467, in create_dummy_layer
      line: layer_desc = dep_loop_exception.net_dict[self.layer_name].copy()
      locals:
        layer_desc = &lt;not found&gt;
        dep_loop_exception = &lt;local&gt; NetworkConstructionDependencyLoopException("Error: There is a dependency loop on layer 'output'.\nConstruction stack (most recent first):\n  source_embed_weighted\n  source_embed_with_pos\n  source_embed\n  enc_01_self_att_out\n  enc_01_ff_out\n  enc_01\n  enc_02_self_att_out\n  enc_02_ff_out\n  ...
        dep_loop_exception.net_dict = &lt;local&gt; {'enc_06_self_att_laynorm': {'class': 'layer_norm', 'from': ['enc_05']}, 'source_embed_weighted': {'class': 'eval', 'from': ['source_embed_raw'], 'eval': 'source(0) * 22.627417'}, 'enc_01_ff_drop': {'dropout': 0.1, 'class': 'dropout', 'from': ['enc_01_ff_conv2']}, 'enc_05_ff_drop': {'dropout': 0...., len = 98
        self = &lt;local&gt; &lt;TFNetworkLayer.ReuseParams.LazyLayerResolver object at 0x7fcb3e959b38&gt;
        self.layer_name = &lt;local&gt; 'output/pivot_target_embed_raw', len = 29
        copy = &lt;not found&gt;
  KeyError: 'output/pivot_target_embed_raw'
  </code></pre>
  
  <p>Is it possible that the function <code>create_dummy_layer</code> is not able to handle the layer being part of a subnetwork or am I using reuse_parameters wrong?</p>
  
  <p>EDIT: Shrunk version of the config: </p>
  
  <pre><code>network = { 'dec_01_att_key': {'axis': 'F', 'class': 'split_dims', 'dims': (8, 64), 'from': ['dec_01_att_key0']},
    'dec_01_att_key0': { 'activation': None,
                         'class': 'linear',
                         'forward_weights_init': "variance_scaling_initializer(mode='fan_in', distribution='uniform', scale=1.0)",
                         'from': ['encoder'],
                         'n_out': 512,
                         'with_bias': False},
    'dec_01_att_value': {'axis': 'F', 'class': 'split_dims', 'dims': (8, 64), 'from': ['dec_01_att_value0']},
    'dec_01_att_value0': { 'activation': None,
                           'class': 'linear',
                           'forward_weights_init': "variance_scaling_initializer(mode='fan_in', distribution='uniform', scale=1.0)",
                           'from': ['encoder'],
                           'n_out': 512,
                           'with_bias': False},
    'decision': {'class': 'decide', 'from': ['output'], 'loss': 'edit_distance', 'loss_opts': {}, 'target': 'classes'},
    'enc_01': {'class': 'copy', 'from': ['enc_01_ff_out']},
    'enc_01_ff_conv1': { 'activation': 'relu',
                         'class': 'linear',
                         'forward_weights_init': "variance_scaling_initializer(mode='fan_in', distribution='uniform', scale=1.0)",
                         'from': ['enc_01_ff_laynorm'],
                         'n_out': 2048,
                         'with_bias': True},
    'enc_01_ff_conv2': { 'activation': None,
                         'class': 'linear',
                         'dropout': 0.1,
                         'forward_weights_init': "variance_scaling_initializer(mode='fan_in', distribution='uniform', scale=1.0)",
                         'from': ['enc_01_ff_conv1'],
                         'n_out': 512,
                         'with_bias': True},
    'enc_01_ff_drop': {'class': 'dropout', 'dropout': 0.1, 'from': ['enc_01_ff_conv2']},
    'enc_01_ff_laynorm': {'class': 'layer_norm', 'from': ['enc_01_self_att_out']},
    'enc_01_ff_out': {'class': 'combine', 'from': ['enc_01_self_att_out', 'enc_01_ff_drop'], 'kind': 'add', 'n_out': 512},
    'enc_01_self_att_att': { 'attention_dropout': 0.1,
                             'attention_left_only': False,
                             'class': 'self_attention',
                             'forward_weights_init': "variance_scaling_initializer(mode='fan_in', distribution='uniform', scale=1.0)",
                             'from': ['enc_01_self_att_laynorm'],
                             'n_out': 512,
                             'num_heads': 8,
                             'total_key_dim': 512},
    'enc_01_self_att_drop': {'class': 'dropout', 'dropout': 0.1, 'from': ['enc_01_self_att_lin']},
    'enc_01_self_att_laynorm': {'class': 'layer_norm', 'from': ['source_embed']},
    'enc_01_self_att_lin': { 'activation': None,
                             'class': 'linear',
                             'forward_weights_init': "variance_scaling_initializer(mode='fan_in', distribution='uniform', scale=1.0)",
                             'from': ['enc_01_self_att_att'],
                             'n_out': 512,
                             'with_bias': False},
    'enc_01_self_att_out': {'class': 'combine', 'from': ['source_embed', 'enc_01_self_att_drop'], 'kind': 'add', 'n_out': 512},
    'encoder': {'class': 'layer_norm', 'from': ['enc_01']},
    'output': { 'class': 'rec',
                'from': [],
                'max_seq_len': "max_len_from('base:encoder') * 3",
                'target': 'classes',
                'unit': { 'dec_01': {'class': 'copy', 'from': ['dec_01_ff_out']},
                          'dec_01_att0': {'base': 'base:dec_01_att_value', 'class': 'generic_attention', 'weights': 'dec_01_att_weights_drop'},
                          'dec_01_att_att': {'axes': 'static', 'class': 'merge_dims', 'from': ['dec_01_att0']},
                          'dec_01_att_drop': {'class': 'dropout', 'dropout': 0.1, 'from': ['dec_01_att_lin']},
                          'dec_01_att_energy': { 'class': 'dot',
                                                 'from': ['base:dec_01_att_key', 'dec_01_att_query'],
                                                 'red1': -1,
                                                 'red2': -1,
                                                 'var1': 'T',
                                                 'var2': 'T?'},
                          'dec_01_att_laynorm': {'class': 'layer_norm', 'from': ['dec_01_self_att_out']},
                          'dec_01_att_lin': { 'activation': None,
                                              'class': 'linear',
                                              'forward_weights_init': "variance_scaling_initializer(mode='fan_in', distribution='uniform', scale=1.0)",
                                              'from': ['dec_01_att_att'],
                                              'n_out': 512,
                                              'with_bias': False},
                          'dec_01_att_out': {'class': 'combine', 'from': ['dec_01_self_att_out', 'dec_01_att_drop'], 'kind': 'add', 'n_out': 512},
                          'dec_01_att_query': {'axis': 'F', 'class': 'split_dims', 'dims': (8, 64), 'from': ['dec_01_att_query0']},
                          'dec_01_att_query0': { 'activation': None,
                                                 'class': 'linear',
                                                 'forward_weights_init': "variance_scaling_initializer(mode='fan_in', distribution='uniform', "
                                                                         'scale=1.0)',
                                                 'from': ['dec_01_att_laynorm'],
                                                 'n_out': 512,
                                                 'with_bias': False},
                          'dec_01_att_weights': {'class': 'softmax_over_spatial', 'energy_factor': 0.125, 'from': ['dec_01_att_energy']},
                          'dec_01_att_weights_drop': { 'class': 'dropout',
                                                       'dropout': 0.1,
                                                       'dropout_noise_shape': {'*': None},
                                                       'from': ['dec_01_att_weights']},
                          'dec_01_ff_conv1': { 'activation': 'relu',
                                               'class': 'linear',
                                               'forward_weights_init': "variance_scaling_initializer(mode='fan_in', distribution='uniform', scale=1.0)",
                                               'from': ['dec_01_ff_laynorm'],
                                               'n_out': 2048,
                                               'with_bias': True},
                          'dec_01_ff_conv2': { 'activation': None,
                                               'class': 'linear',
                                               'dropout': 0.1,
                                               'forward_weights_init': "variance_scaling_initializer(mode='fan_in', distribution='uniform', scale=1.0)",
                                               'from': ['dec_01_ff_conv1'],
                                               'n_out': 512,
                                               'with_bias': True},
                          'dec_01_ff_drop': {'class': 'dropout', 'dropout': 0.1, 'from': ['dec_01_ff_conv2']},
                          'dec_01_ff_laynorm': {'class': 'layer_norm', 'from': ['dec_01_att_out']},
                          'dec_01_ff_out': {'class': 'combine', 'from': ['dec_01_att_out', 'dec_01_ff_drop'], 'kind': 'add', 'n_out': 512},
                          'dec_01_self_att_att': { 'attention_dropout': 0.1,
                                                   'attention_left_only': True,
                                                   'class': 'self_attention',
                                                   'forward_weights_init': "variance_scaling_initializer(mode='fan_in', distribution='uniform', "
                                                                           'scale=1.0)',
                                                   'from': ['dec_01_self_att_laynorm'],
                                                   'n_out': 512,
                                                   'num_heads': 8,
                                                   'total_key_dim': 512},
                          'dec_01_self_att_drop': {'class': 'dropout', 'dropout': 0.1, 'from': ['dec_01_self_att_lin']},
                          'dec_01_self_att_laynorm': {'class': 'layer_norm', 'from': ['target_embed']},
                          'dec_01_self_att_lin': { 'activation': None,
                                                   'class': 'linear',
                                                   'forward_weights_init': "variance_scaling_initializer(mode='fan_in', distribution='uniform', "
                                                                           'scale=1.0)',
                                                   'from': ['dec_01_self_att_att'],
                                                   'n_out': 512,
                                                   'with_bias': False},
                          'dec_01_self_att_out': {'class': 'combine', 'from': ['target_embed', 'dec_01_self_att_drop'], 'kind': 'add', 'n_out': 512},
                          'decoder': {'class': 'layer_norm', 'from': ['dec_01']},
                          'end': {'class': 'compare', 'from': ['output'], 'value': 0},
                          'output': {'beam_size': 12, 'class': 'choice', 'from': ['output_prob'], 'initial_output': 0, 'target': 'classes'},
                          'output_prob': { 'class': 'softmax',
                                           'dropout': 0.0,
                                           'forward_weights_init': "variance_scaling_initializer(mode='fan_in', distribution='uniform', scale=1.0)",
                                           'from': ['decoder'],
                                           'loss': 'ce',
                                           'loss_opts': {'use_normalized_loss': True},
                                           'reuse_params': {'map': {'W': {'custom': None, 'reuse_layer': 'target_embed_raw'}, 'b': None}},
                                           'target': 'classes',
                                           'with_bias': True},
                          'target_embed': {'class': 'dropout', 'dropout': 0.1, 'from': ['target_embed_with_pos']},
                          'target_embed_raw': { 'activation': None,
                                                'class': 'linear',
                                                'forward_weights_init': "variance_scaling_initializer(mode='fan_in', distribution='uniform', "
                                                                        'scale=1.0)',
                                                'from': ['prev:output'],
                                                'n_out': 512,
                                                'with_bias': False},
                          'target_embed_weighted': {'class': 'eval', 'eval': 'source(0) * 22.627417', 'from': ['target_embed_raw'], 'trainable': False},
                          'target_embed_with_pos': { 'add_to_input': True,
                                                     'class': 'positional_encoding',
                                                     'from': ['target_embed_weighted']}},
                          'pivot_target_embed_raw': { 'activation': None,
                                                'class': 'linear',
                                                'forward_weights_init': "variance_scaling_initializer(mode='fan_in', distribution='uniform', "
                                                                        'scale=1.0)',
                                                #'from': ['prev:output'],
                                                'n_out': 512,
                                                'trainable': False,
                                                'with_bias': False}
                          },
    'source_embed': {'class': 'dropout', 'dropout': 0.1, 'from': ['source_embed_with_pos']},
    'source_embed_raw': { 'activation': None,
                                'class': 'linear',
                                #'forward_weights_init': "variance_scaling_initializer(mode='fan_in', distribution='uniform', scale=1.0)",
                                'from': ['data:data'],
                                'n_out': 512,
                                'with_bias': False,
                                #'reuse_params': {'map': {'W': {'reuse_layer': 'pivot_source_embed_raw'}, 'b': None}},
                                'reuse_params': {'map': {'W': {'reuse_layer': 'output/pivot_target_embed_raw'}, 'b': None}}
                        },
    'source_embed_weighted': {'class': 'eval', 'eval': 'source(0) * 22.627417', 'from': ['source_embed_raw']},
    'source_embed_with_pos': {'add_to_input': True, 'class': 'positional_encoding', 'from': ['source_embed_weighted']}}
  
  pivot_file = [Pathplaceholder] 
  pivot_prefix = 'pivot_'
  preload_from_files = {}
  if not task == "search":
      preload_from_files = {
      "pivot" : {"filename": pivot_file, "prefix": pivot_prefix, "init_for_train": True},
      }
  </code></pre>