<p dir="auto"><strong>System information</strong></p>
<ul dir="auto">
<li>I have written custom code.</li>
<li>Linux Ubuntu 19.04, Debian 9.9</li>
<li>TensorFlow installed from binary</li>
<li>TensorFlow version 2.0.0-b1, 1.14, 1.15.0</li>
<li>Python version: 3.7.1, 3.7.3</li>
<li>CUDA/cuDNN version: 10.0, 10.1</li>
<li>GPU model and memory: Tesla T4, Tesla P4, P100, does not exist on k80</li>
</ul>
<p dir="auto"><strong>Describe the current behavior</strong></p>
<p dir="auto">The result on the GPU is significantly different from CPU. It is correct on CPU, but wrong on GPU. The difference on GPU is much bigger than zero.</p>
<p dir="auto"><strong>Describe the expected behavior</strong></p>
<p dir="auto">The print out of the difference should be zero.</p>
<p dir="auto"><strong>Code to reproduce the issue</strong></p>
<p dir="auto"><strong>tf 2.0.0-b1</strong></p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="
import tensorflow as tf
import numpy as np



@tf.function
def sample_y_nn_tf(w, X_sample, config):

        layers = config['layers']
        current_w_index = 0
        h = X_sample
        w_shape = tf.shape(input=w)

        l = layers[0]
        w_current = tf.slice(w, begin=[0,0,0,0],size=[w_shape[0], w_shape[1], w_shape[2], l[0] * l[1]])
        w_current = tf.reshape(w_current, [config['batch_size'], w_shape[1], w_shape[2]] + list(l))
        b_current = tf.slice(w, begin=[0,0,0,l[0] * l[1]-1],size=[w_shape[0], w_shape[1], w_shape[2], l[1]])
        h = tf.einsum('lm,ikjmn-&gt;likjn', h, w_current) + b_current
        current_w_index = current_w_index + (l[0] + 1) * l[1]

        for l in layers[1:]:
            h = tf.nn.elu(h)
            w_current = tf.slice(w, begin=[0,0,0,current_w_index],size=[w_shape[0], w_shape[1], w_shape[2], l[0] * l[1]])
            w_current = tf.reshape(w_current, [config['batch_size'],w_shape[1], w_shape[2]]+list(l))
            b_current = tf.slice(w, begin=[0, 0, 0,current_w_index + l[0] * l[1] - 1], size=[w_shape[0], w_shape[1], w_shape[2], l[1]])
            h = tf.einsum('likjm,ikjmn-&gt;likjn',h,w_current)+b_current
            current_w_index = current_w_index+(l[0]+1)*l[1]

        h = tf.squeeze(h, axis=4)

        return h

config = {'batch_size': 8, 'layers': [[2,2],[2,1]]}

np.random.seed(111)

w = np.random.normal(size=(config['batch_size'],64,512,13)).astype(dtype=np.float32)
x_sample = np.random.normal(size=(100,2)).astype(dtype=np.float32)

permutation = tf.random.shuffle(tf.range(config['batch_size'], dtype=tf.int32), seed=111)

print('permutation', permutation)


with tf.device(&quot;/cpu:0&quot;):
    means = sample_y_nn_tf(w=w, X_sample=x_sample, config=config)
    means_p = sample_y_nn_tf( w = tf.gather(w, permutation, axis=0), X_sample=x_sample, config=config)
    print('difference on cpu', np.sum(np.abs(tf.gather(means,permutation, axis=1)-means_p)))

with tf.device(&quot;/gpu:0&quot;):
    means = sample_y_nn_tf(w=w, X_sample=x_sample, config=config)
    means_p = sample_y_nn_tf( w = tf.gather(w, permutation, axis=0), X_sample=x_sample, config=config)
    print('difference on gpu', np.sum(np.abs(tf.gather(means,permutation, axis=1)-means_p)))

"><pre class="notranslate"><code class="notranslate">
import tensorflow as tf
import numpy as np



@tf.function
def sample_y_nn_tf(w, X_sample, config):

        layers = config['layers']
        current_w_index = 0
        h = X_sample
        w_shape = tf.shape(input=w)

        l = layers[0]
        w_current = tf.slice(w, begin=[0,0,0,0],size=[w_shape[0], w_shape[1], w_shape[2], l[0] * l[1]])
        w_current = tf.reshape(w_current, [config['batch_size'], w_shape[1], w_shape[2]] + list(l))
        b_current = tf.slice(w, begin=[0,0,0,l[0] * l[1]-1],size=[w_shape[0], w_shape[1], w_shape[2], l[1]])
        h = tf.einsum('lm,ikjmn-&gt;likjn', h, w_current) + b_current
        current_w_index = current_w_index + (l[0] + 1) * l[1]

        for l in layers[1:]:
            h = tf.nn.elu(h)
            w_current = tf.slice(w, begin=[0,0,0,current_w_index],size=[w_shape[0], w_shape[1], w_shape[2], l[0] * l[1]])
            w_current = tf.reshape(w_current, [config['batch_size'],w_shape[1], w_shape[2]]+list(l))
            b_current = tf.slice(w, begin=[0, 0, 0,current_w_index + l[0] * l[1] - 1], size=[w_shape[0], w_shape[1], w_shape[2], l[1]])
            h = tf.einsum('likjm,ikjmn-&gt;likjn',h,w_current)+b_current
            current_w_index = current_w_index+(l[0]+1)*l[1]

        h = tf.squeeze(h, axis=4)

        return h

config = {'batch_size': 8, 'layers': [[2,2],[2,1]]}

np.random.seed(111)

w = np.random.normal(size=(config['batch_size'],64,512,13)).astype(dtype=np.float32)
x_sample = np.random.normal(size=(100,2)).astype(dtype=np.float32)

permutation = tf.random.shuffle(tf.range(config['batch_size'], dtype=tf.int32), seed=111)

print('permutation', permutation)


with tf.device("/cpu:0"):
    means = sample_y_nn_tf(w=w, X_sample=x_sample, config=config)
    means_p = sample_y_nn_tf( w = tf.gather(w, permutation, axis=0), X_sample=x_sample, config=config)
    print('difference on cpu', np.sum(np.abs(tf.gather(means,permutation, axis=1)-means_p)))

with tf.device("/gpu:0"):
    means = sample_y_nn_tf(w=w, X_sample=x_sample, config=config)
    means_p = sample_y_nn_tf( w = tf.gather(w, permutation, axis=0), X_sample=x_sample, config=config)
    print('difference on gpu', np.sum(np.abs(tf.gather(means,permutation, axis=1)-means_p)))


</code></pre></div>
<p dir="auto">Output:</p>
<blockquote>
<p dir="auto">difference on cpu 0.0<br>
difference on gpu 37953784.0</p>
</blockquote>
<p dir="auto"><strong>tf.1.14</strong></p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="from tensorflow.python.framework.versions import VERSION
import tensorflow as tf
import numpy as np
import os

print(tf.__version__)

@tf.function
def sample_y_nn_tf(w, X_sample, config):
    
        layers = config['layers']
        current_w_index = 0
        h = X_sample
        w_shape = tf.shape(input=w)
        
        l = layers[0]
        w_current = tf.slice(w, begin=[0,0,0,0],size=[w_shape[0], w_shape[1], w_shape[2], l[0] * l[1]])
        w_current = tf.reshape(w_current, [config['batch_size'], w_shape[1], w_shape[2]] + list(l))
        b_current = tf.slice(w, begin=[0,0,0,l[0] * l[1]-1],size=[w_shape[0], w_shape[1], w_shape[2], l[1]])
        h = tf.einsum('lm,ikjmn-&gt;likjn', h, w_current) + b_current
        current_w_index = current_w_index + (l[0] + 1) * l[1]
        
        for l in layers[1:]:
            h = tf.nn.elu(h)
            w_current = tf.slice(w, begin=[0,0,0,current_w_index],size=[w_shape[0], w_shape[1], w_shape[2], l[0] * l[1]])
            w_current = tf.reshape(w_current, [config['batch_size'],w_shape[1], w_shape[2]]+list(l))
            b_current = tf.slice(w, begin=[0, 0, 0,current_w_index + l[0] * l[1] - 1], size=[w_shape[0], w_shape[1], w_shape[2], l[1]])
            h = tf.einsum('likjm,ikjmn-&gt;likjn',h,w_current)+b_current
            current_w_index = current_w_index+(l[0]+1)*l[1]
    
        h = tf.squeeze(h, axis=4)

        return h

config = {'batch_size': 8, 'layers': [[2,2],[2,1]]}

np.random.seed(111)

w = np.random.normal(size=(config['batch_size'],64,512,13)).astype(dtype=np.float32)
precision = np.exp(np.random.normal(size=(config['batch_size'],64,512))).astype(dtype=np.float32)
x_sample = np.random.normal(size=(100,2)).astype(dtype=np.float32)

permutation = tf.random.shuffle(tf.range(config['batch_size'], dtype=tf.int32), seed=111)

conf = tf.ConfigProto()
conf.gpu_options.allow_growth = True
sess = tf.Session(config=conf)


print('permutation', sess.run(permutation))


with tf.device(&quot;/cpu:0&quot;):
    means = sample_y_nn_tf(w=w, X_sample=x_sample, config=config)
    means_p = sample_y_nn_tf( w = tf.gather(w, permutation, axis=0), X_sample=x_sample, config=config)
    net_cpu = tf.gather(means,permutation, axis=1)-means_p

with tf.device(&quot;/device:GPU:0&quot;):
    means = sample_y_nn_tf(w=w, X_sample=x_sample, config=config)
    means_p = sample_y_nn_tf( w = tf.gather(w, permutation, axis=0), X_sample=x_sample, config=config)
    net_gpu = tf.gather(means,permutation, axis=1)-means_p

net_cpu_result = sess.run(net_cpu)
net_gpu_result = sess.run(net_gpu)
print(&quot;tensorflow version: {}&quot;.format(VERSION))
print('difference on cpu', np.sum(np.abs(net_cpu_result)))
print('difference on gpu', np.sum(np.abs(net_gpu_result)))
"><pre class="notranslate"><code class="notranslate">from tensorflow.python.framework.versions import VERSION
import tensorflow as tf
import numpy as np
import os

print(tf.__version__)

@tf.function
def sample_y_nn_tf(w, X_sample, config):
    
        layers = config['layers']
        current_w_index = 0
        h = X_sample
        w_shape = tf.shape(input=w)
        
        l = layers[0]
        w_current = tf.slice(w, begin=[0,0,0,0],size=[w_shape[0], w_shape[1], w_shape[2], l[0] * l[1]])
        w_current = tf.reshape(w_current, [config['batch_size'], w_shape[1], w_shape[2]] + list(l))
        b_current = tf.slice(w, begin=[0,0,0,l[0] * l[1]-1],size=[w_shape[0], w_shape[1], w_shape[2], l[1]])
        h = tf.einsum('lm,ikjmn-&gt;likjn', h, w_current) + b_current
        current_w_index = current_w_index + (l[0] + 1) * l[1]
        
        for l in layers[1:]:
            h = tf.nn.elu(h)
            w_current = tf.slice(w, begin=[0,0,0,current_w_index],size=[w_shape[0], w_shape[1], w_shape[2], l[0] * l[1]])
            w_current = tf.reshape(w_current, [config['batch_size'],w_shape[1], w_shape[2]]+list(l))
            b_current = tf.slice(w, begin=[0, 0, 0,current_w_index + l[0] * l[1] - 1], size=[w_shape[0], w_shape[1], w_shape[2], l[1]])
            h = tf.einsum('likjm,ikjmn-&gt;likjn',h,w_current)+b_current
            current_w_index = current_w_index+(l[0]+1)*l[1]
    
        h = tf.squeeze(h, axis=4)

        return h

config = {'batch_size': 8, 'layers': [[2,2],[2,1]]}

np.random.seed(111)

w = np.random.normal(size=(config['batch_size'],64,512,13)).astype(dtype=np.float32)
precision = np.exp(np.random.normal(size=(config['batch_size'],64,512))).astype(dtype=np.float32)
x_sample = np.random.normal(size=(100,2)).astype(dtype=np.float32)

permutation = tf.random.shuffle(tf.range(config['batch_size'], dtype=tf.int32), seed=111)

conf = tf.ConfigProto()
conf.gpu_options.allow_growth = True
sess = tf.Session(config=conf)


print('permutation', sess.run(permutation))


with tf.device("/cpu:0"):
    means = sample_y_nn_tf(w=w, X_sample=x_sample, config=config)
    means_p = sample_y_nn_tf( w = tf.gather(w, permutation, axis=0), X_sample=x_sample, config=config)
    net_cpu = tf.gather(means,permutation, axis=1)-means_p

with tf.device("/device:GPU:0"):
    means = sample_y_nn_tf(w=w, X_sample=x_sample, config=config)
    means_p = sample_y_nn_tf( w = tf.gather(w, permutation, axis=0), X_sample=x_sample, config=config)
    net_gpu = tf.gather(means,permutation, axis=1)-means_p

net_cpu_result = sess.run(net_cpu)
net_gpu_result = sess.run(net_gpu)
print("tensorflow version: {}".format(VERSION))
print('difference on cpu', np.sum(np.abs(net_cpu_result)))
print('difference on gpu', np.sum(np.abs(net_gpu_result)))

</code></pre></div>
<p dir="auto">Output:</p>
<blockquote>
<p dir="auto">difference on cpu 0.0<br>
difference on gpu 39042744.0</p>
</blockquote>
<p dir="auto"><strong>Other info / logs</strong></p>
<p dir="auto">I did not check the tf2 version on K80, P100.<br>
The bug probably persists when using Pythonic slicing operations.</p>
<p dir="auto"><strong>System information</strong></p>
<ul dir="auto">
<li>Have I written custom code (as opposed to using a stock example script provided in TensorFlow): Yes</li>
<li>OS Platform and Distribution (e.g., Linux Ubuntu 16.04): Linux Ubuntu 18.04</li>
<li>Mobile device (e.g. iPhone 8, Pixel 2, Samsung Galaxy) if the issue happens on mobile device:</li>
<li>TensorFlow installed from (source or binary): binary</li>
<li>TensorFlow version (use command below): v1.14.0-rc1-22-gaf24dc91b5 1.14.0</li>
<li>Python version: 3.6.7</li>
<li>GCC/Compiler version (if compiling from source): 8.2.0</li>
<li>CUDA/cuDNN version: cuda-10.0</li>
<li>GPU model and memory:</li>
</ul>
<p dir="auto"><strong>Describe the current behavior</strong><br>
tf.matmul on tensorflow-gpu gave wrong results. Here is a simplified version of the code.</p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt

r = [[ 1.0, 0.0],    [0.0, 1.0]]

x, y = np.meshgrid(list(range(400)), list(range(400)))
coords = np.stack([x,y],-1).reshape((400,400,2,1))
coords = tf.convert_to_tensor(coords,dtype=tf.float32)

r1 = tf.constant(r)

newCoords = tf.matmul(r1, coords)

sess = tf.Session()
ret = sess.run(newCoords,feed_dict={r1:r})

plt.matshow(ret[:,:,0,0])
plt.show()"><pre class="notranslate"><code class="notranslate">import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt

r = [[ 1.0, 0.0],    [0.0, 1.0]]

x, y = np.meshgrid(list(range(400)), list(range(400)))
coords = np.stack([x,y],-1).reshape((400,400,2,1))
coords = tf.convert_to_tensor(coords,dtype=tf.float32)

r1 = tf.constant(r)

newCoords = tf.matmul(r1, coords)

sess = tf.Session()
ret = sess.run(newCoords,feed_dict={r1:r})

plt.matshow(ret[:,:,0,0])
plt.show()
</code></pre></div>
<p dir="auto">When I ran it on my tensorflow-gpu, here is the result:<br>
<a target="_blank" rel="noopener noreferrer nofollow" href="https://gist.githubusercontent.com/sWizad/3a25d6559ea3308e5cc2731519635c32/raw/2479341ed43fb8e69c94eda3a1362d5cbde7d2d7/Figure_1.png"><img src="https://gist.githubusercontent.com/sWizad/3a25d6559ea3308e5cc2731519635c32/raw/2479341ed43fb8e69c94eda3a1362d5cbde7d2d7/Figure_1.png" alt="bug" style="max-width: 100%;"></a><br>
Looks like it stops computing halfway through and gave the rest 0 as a result.</p>
<p dir="auto"><strong>Describe the expected behavior</strong><br>
Here is the result with CPU:<br>
<a target="_blank" rel="noopener noreferrer nofollow" href="https://user-images.githubusercontent.com/45821224/62137176-08631480-b2d5-11e9-8b8e-6348e8b206e7.png"><img src="https://user-images.githubusercontent.com/45821224/62137176-08631480-b2d5-11e9-8b8e-6348e8b206e7.png" alt="CPU" style="max-width: 100%;"></a></p>
<hr>
<p dir="auto">Below is my old post. Initially, I thought the problem was related to TFRecord, but seems like this problem occurs without even using tfrecord too. For completeness, I keep the old example code with tfrecord.</p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt

def parser(serialized_example):
      fs = tf.io.parse_single_example(
          serialized_example,
          features={ &quot;r&quot;: tf.FixedLenFeature([4], tf.float32) })
      fs[&quot;r&quot;] = tf.reshape(fs[&quot;r&quot;], [2, 2])
      return fs

r = [[ 1.0, 0.0],[0.0, 1.0]]

with tf.io.TFRecordWriter(&quot;cc.test&quot;) as tfrecord_writer:
    feature = {&quot;r&quot;: tf.train.Feature(float_list=tf.train.FloatList(value=np.array(r).flatten() ))}
    example = tf.train.Example(features=tf.train.Features(feature=feature))
    tfrecord_writer.write(example.SerializeToString())
dataset = tf.data.TFRecordDataset([&quot;cc.test&quot;])
dataset = dataset.map(parser).repeat().make_one_shot_iterator()
features = dataset.get_next()

x, y = tf.meshgrid(list(range(400)), list(range(400)))
coords = tf.stack([x, y], -1)     #(h,w,2)
coords = tf.expand_dims(tf.cast(coords,tf.float32),-1) #(h,w,2,1)

r1 = features[&quot;r&quot;]
r2 = tf.constant(r)

newCoords = tf.matmul(r1, coords)

sess = tf.Session()
ret = sess.run(newCoords[:,:,0,0])
plt.matshow(ret)
plt.show()"><pre class="notranslate"><code class="notranslate">import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt

def parser(serialized_example):
      fs = tf.io.parse_single_example(
          serialized_example,
          features={ "r": tf.FixedLenFeature([4], tf.float32) })
      fs["r"] = tf.reshape(fs["r"], [2, 2])
      return fs

r = [[ 1.0, 0.0],[0.0, 1.0]]

with tf.io.TFRecordWriter("cc.test") as tfrecord_writer:
    feature = {"r": tf.train.Feature(float_list=tf.train.FloatList(value=np.array(r).flatten() ))}
    example = tf.train.Example(features=tf.train.Features(feature=feature))
    tfrecord_writer.write(example.SerializeToString())
dataset = tf.data.TFRecordDataset(["cc.test"])
dataset = dataset.map(parser).repeat().make_one_shot_iterator()
features = dataset.get_next()

x, y = tf.meshgrid(list(range(400)), list(range(400)))
coords = tf.stack([x, y], -1)     #(h,w,2)
coords = tf.expand_dims(tf.cast(coords,tf.float32),-1) #(h,w,2,1)

r1 = features["r"]
r2 = tf.constant(r)

newCoords = tf.matmul(r1, coords)

sess = tf.Session()
ret = sess.run(newCoords[:,:,0,0])
plt.matshow(ret)
plt.show()
</code></pre></div>
<p dir="auto">The code will create "cc.test" file to save the variable <code class="notranslate">r</code> and load it as <code class="notranslate">r1</code>. Then matmul <code class="notranslate">r1</code> with some big varibles.</p>