Tensorflow 멀티 GPU MNIST 분류기 : 저 정확도

Question

Tensorflow에 여러 GPU MNIST 분류기가 붙어 있습니다. 코드는 오류없이 실행되지만 정확도는 매우 낮습니다 (30 %). 나는 Tensorflow를 처음 사용하므로 어디에 문제가 있는지 모릅니다. GPU : 2x GTX 1080 Ti.

여러 GPU에 대한 자습서가 여러 개 있지만 코드를 따르기가 어렵습니다. 이런 이유로 저는 처음부터 MNIST CNN 분류기를 개발하려고합니다. 내가 눈치

from __future__ import print_function 
from tensorflow.examples.tutorials.mnist import input_data 
import tensorflow as tf 
import datetime 

def average_gradients(tower_grads): 
    average_grads = [] 
    for grad_and_vars in zip(*tower_grads): 
    # Note that each grad_and_vars looks like the following: 
    # ((grad0_gpu0, var0_gpu0), ... , (grad0_gpuN, var0_gpuN)) 
    grads = [] 
    for g, _ in grad_and_vars: 
     # Add 0 dimension to the gradients to represent the tower. 
     expanded_g = tf.expand_dims(g, 0) 

     # Append on a 'tower' dimension which we will average over below. 
     grads.append(expanded_g) 

    # Average over the 'tower' dimension. 
    grad = tf.concat(axis=0, values=grads) 
    grad = tf.reduce_mean(grad, 0) 

    # Keep in mind that the Variables are redundant because they are shared 
    # across towers. So .. we will just return the first tower's pointer to 
    # the Variable. 
    v = grad_and_vars[0][1] 
    grad_and_var = (grad, v) 
    average_grads.append(grad_and_var) 
    return average_grads 

with tf.device('/cpu:0'): 
    x = tf.placeholder(tf.float32, [None, 784], name='x') 
    x_img=tf.reshape(x, [-1, 28, 28, 1]) 
    x_dict={} 
    x_dict['x0'],x_dict['x1'] = tf.split(x_img,2) 

    y_dict={} 
    y = tf.placeholder(tf.float32, [None, 10], name='y') 
    y_dict['y0'],y_dict['y1'] = tf.split(y,2) 

    opt=tf.train.GradientDescentOptimizer(0.01) 


    keep_prob = tf.placeholder(tf.float32) 

    w0=tf.get_variable('w0',initializer=tf.truncated_normal([5, 5,1,32], stddev=0.1)) 
    b0=tf.get_variable('b0',initializer=tf.zeros([32])) 

    w1=tf.get_variable('w1',initializer=tf.truncated_normal([5,5,32,64], stddev=0.1)) 
    b1=tf.get_variable('b1',initializer=tf.zeros([64])) 

    w2=tf.get_variable('w2',initializer=tf.truncated_normal([7*7*64,1024], stddev=0.1)) 
    b2=tf.get_variable('b2',initializer=tf.zeros([1024])) 

    w3=tf.get_variable('w3',initializer=tf.truncated_normal([1024,10], stddev=0.1)) 
    b3=tf.get_variable('b3',initializer=tf.zeros([10])) 


    grads=[] 



def conv2d(xx, W): 
    return tf.nn.conv2d(xx, W, strides=[1, 1, 1, 1], padding='SAME') 

def max_pool_2x2(xx): 
    return tf.nn.max_pool(xx, ksize=[1, 2, 2, 1],strides=[1, 2, 2, 1], padding='SAME') 


def model_forward(xx): 
    h_conv1=tf.nn.relu(conv2d(xx,w0)+b0); 
    h_pool1=max_pool_2x2(h_conv1) 

    h_conv2=tf.nn.relu(conv2d(h_pool1,w1)+b1); 
    h_pool2=max_pool_2x2(h_conv2) 

    h_pool2_flat = tf.reshape(h_pool2, [-1, 7*7*64]) 

    h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat,w2)+b2) 

    h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob) 

    y = tf.nn.sigmoid(tf.matmul(h_fc1_drop,w3)+b3) 
    return y 


for i in range(0,2): 
    with tf.device(('/gpu:{0}').format(i)): 
     with tf.variable_scope(('scope_gpu_{0}').format(i)): 
      yy=model_forward(x_dict[('x{0}').format(i)]) 
      cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_dict[('y{0}').format(i)] * tf.log(yy), reduction_indices=[1])) 
      grads.append(opt.compute_gradients(cross_entropy,tf.trainable_variables())) 


with tf.device('/cpu:0'): 
    grad = average_gradients(grads) 
    train_step = opt.apply_gradients(grad) 
    yy=model_forward(x_dict['x0']) 
    correct_prediction = tf.equal(tf.argmax(yy, 1), tf.argmax(y_dict['y0'], 1)) 
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name='accuracy') 


def main(): 

    mnist = input_data.read_data_sets("MNIST_data/", one_hot=True) 

    with tf.Session(config=tf.ConfigProto(log_device_placement=True)) as sess: 

     sess.run(tf.global_variables_initializer()) 
     writer = tf.summary.FileWriter('C:\\tmp\\test\\', graph=tf.get_default_graph()) 

     t1_1 = datetime.datetime.now() 
     for step in range(0,10000): 
      batch_x, batch_y = mnist.train.next_batch(100) 
      sess.run(train_step, feed_dict={x: batch_x, y: batch_y, keep_prob: 0.5}) 

      if (step % 200) == 0: 
       print(step, sess.run(accuracy, feed_dict={x: mnist.test.images, y: mnist.test.labels, keep_prob: 1})) 
     t2_1 = datetime.datetime.now() 

    print("Computation time: " + str(t2_1-t1_1)) 



if __name__ == "__main__": 
    main() 

Answer 1

문제점 :

• 크로스 엔트로피 손실은 잘못된 것입니다 (한마디로 당신은 진 교차 엔트로피를 계산하고, 자세한 내용은 this question 참조).
• 나는 수동 그라디언트 계산을 tf.train.AdamOptimizer으로 내렸다.
• x의 입력 분할을 삭제했습니다 (tensorflow에서 분산 계산을 수행하는 올바른 방법이 아닙니다).

결과 모델은 하나의 GPU에서도 99 %의 정확도를 쉽게 얻습니다.

from tensorflow.examples.tutorials.mnist import input_data 
import tensorflow as tf 
import datetime 

x = tf.placeholder(tf.float32, [None, 784], name='x') 
x_img = tf.reshape(x, [-1, 28, 28, 1]) 
y = tf.placeholder(tf.float32, [None, 10], name='y') 
keep_prob = tf.placeholder(tf.float32) 

stddev = 0.1 
w0 = tf.get_variable('w0', initializer=tf.truncated_normal([5, 5, 1, 32], stddev=stddev)) 
b0 = tf.get_variable('b0', initializer=tf.zeros([32])) 

w1 = tf.get_variable('w1', initializer=tf.truncated_normal([5, 5, 32, 64], stddev=stddev)) 
b1 = tf.get_variable('b1', initializer=tf.zeros([64])) 

w2 = tf.get_variable('w2', initializer=tf.truncated_normal([7 * 7 * 64, 1024], stddev=stddev)) 
b2 = tf.get_variable('b2', initializer=tf.zeros([1024])) 

w3 = tf.get_variable('w3', initializer=tf.truncated_normal([1024, 10], stddev=stddev)) 
b3 = tf.get_variable('b3', initializer=tf.zeros([10])) 

def conv2d(xx, W): 
    return tf.nn.conv2d(xx, W, strides=[1, 1, 1, 1], padding='SAME') 

def max_pool_2x2(xx): 
    return tf.nn.max_pool(xx, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') 

def model_forward(xx): 
    h_conv1 = tf.nn.relu(conv2d(xx, w0) + b0) 
    h_pool1 = max_pool_2x2(h_conv1) 

    h_conv2 = tf.nn.relu(conv2d(h_pool1, w1) + b1) 
    h_pool2 = max_pool_2x2(h_conv2) 

    h_pool2_flat = tf.reshape(h_pool2, [-1, 7 * 7 * 64]) 
    h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, w2) + b2) 
    h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob) 
    y = tf.matmul(h_fc1_drop, w3) + b3 
    return y 

yy = model_forward(x_img) 
loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=yy, labels=y)) 
train_step = tf.train.AdamOptimizer().minimize(loss) 
correct_prediction = tf.equal(tf.argmax(yy, 1), tf.argmax(y, 1)) 
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name='accuracy') 

def main(): 
    mnist = input_data.read_data_sets("/home/maxim/p/data/mnist-tf", one_hot=True) 
    with tf.Session(config=tf.ConfigProto(log_device_placement=True)) as sess: 
    sess.run(tf.global_variables_initializer()) 

    t1_1 = datetime.datetime.now() 
    for step in range(0, 10000): 
     batch_x, batch_y = mnist.train.next_batch(100) 
     sess.run(train_step, feed_dict={x: batch_x, y: batch_y, keep_prob: 0.5}) 

     if (step % 200) == 0: 
     print(step, sess.run(accuracy, feed_dict={x: mnist.test.images, y: mnist.test.labels, keep_prob: 1})) 

    t2_1 = datetime.datetime.now() 
    print("Computation time: " + str(t2_1 - t1_1)) 

if __name__ == "__main__": 
    main() 

지금, 당신이 정말로 그것을 원한다면, 당신은 (거기에 대한 a great post이지만, 때로는 인해 호스팅 문제로 제대로 렌더링되지 않음) GPU 파워를 활용하는 데이터 모델의 병렬 처리를 할 수 있습니다.