Tensorflow에서 Async n-step DQNetwork 업데이트를 위해 그라디언트를 축적하고 적용하는 방법은 무엇입니까?

Question

Asynchronous Methods for Deep Reinforcement Learning을 구현 하려는데 단계 중 하나에 다른 단계에서 그라디언트를 축적 한 다음 적용해야합니다. tensorflow에서 이것을 달성하는 가장 좋은 방법은 무엇입니까? 나는 그라디언트를 축적하기에 이르렀고, 이것을 달성하는 가장 빠른 방법은 없다고 생각합니다. (tensorflow에서부터 파이썬으로 그리고 뒤로 돌아 가기). 모든 제안을 환영합니다. 이것은 장난감 NN의 코드입니다. 그것은 단지 내가 사용하고자하는 연산을 실행하는 것을 모델화하거나 계산하지 않습니다.

import tensorflow as tf 

from model import * 


graph = tf.Graph() 

with graph.as_default(): 

    state = tf.placeholder(tf.float32, shape=[None, 80,80,1]) 

    with tf.variable_scope('layer1'): 
     W = weight_variable([8, 8, 1, 32]) 
     variable_summaries(W, "layer1/W") 
     b = bias_variable([32]) 
     variable_summaries(b, "layer1/b") 
     h = conv2d(state, W, 4) + b 
     activation = tf.nn.relu(h) 
     pool1 = max_pool_2x2(activation) 

    print(pool1.get_shape()) 
    pool1 = tf.reshape(pool1, [-1, 3200]) 

    with tf.variable_scope('readout'): 
     W = weight_variable([3200, 3]) 
     b = bias_variable([3]) 
     logits = tf.matmul(pool1, W) + b 
     variable_summaries(h, "y") 

    action_indexes = tf.placeholder(tf.int32, shape=[None], name="action_indexes") 

    loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, action_indexes) 

    starter_learning_rate = 1e-6 

    global_step = tf.Variable(0, trainable=False) 

    # decay every 1000 steps with a base of 0.96: 
    learning_rate = tf.train.exponential_decay(starter_learning_rate, 
     global_step, 
     10000, 0.96, staircase=True) 

    optimizer = tf.train.RMSPropOptimizer(learning_rate) 

    gradients_and_variables = optimizer.compute_gradients(loss, tf.trainable_variables()) 

    discounted_values = tf.placeholder(tf.float32, shape=[None, 1]) 

with tf.Session(graph=graph) as s: 

    for v in tf.trainable_variables(): 
     print(v.name, v.dtype, v.get_shape()) 

    s.run(tf.initialize_all_variables()) 

    feed_dict= { 
     state : np.zeros([1, 80, 80, 1]), 
     action_indexes: [1], 
    } 


    var_to_grad = dict((var.name, grad) for grad, var in gradients_and_variables) 
    keys = sorted(var_to_grad.keys()) 
    print(keys) 

    name_to_var = dict((var.name, var) for _, var in gradients_and_variables) 

    for i in range(10): 

     gradients = s.run([ var_to_grad[k] for k in keys], feed_dict=feed_dict) 

     for k,v in zip(keys, gradients): 
      var_to_grad[k] += v 

    for k in keys: 
     print(var_to_grad[k]) 

    s.run(optimizer.apply_gradients((g, name_to_var[v]) for v,g in var_to_grad.iteritems()), feed_dict=feed_dict) 

---

@yaroslave 제안 후

업데이트 코드 : 당신은 정말 수동으로 그라디언트를 축적 할 필요가 없습니다

import tensorflow as tf 

from model import * 


graph = tf.Graph() 

with graph.as_default(): 

    minibatch = 32 
    state = tf.placeholder(tf.float32, shape=[minibatch, 80,80,1], name="input") 

    with tf.variable_scope('layer1'): 
     W = weight_variable([8, 8, 1, 32]) 
     variable_summaries(W, "layer1/W") 
     b = bias_variable([32]) 
     variable_summaries(b, "layer1/b") 
     h = conv2d(state, W, 4) + b 
     activation = tf.nn.relu(h) 
     pool1 = max_pool_2x2(activation) 

    print(pool1.get_shape()) 
    pool1 = tf.reshape(pool1, [-1, 3200]) 

    with tf.variable_scope('readout'): 
     W = weight_variable([3200, 3]) 
     b = bias_variable([3]) 
     logits = tf.matmul(pool1, W) + b 
     variable_summaries(h, "y") 

    action_indexes = tf.placeholder(tf.int32, shape=[minibatch], name="action_indexes") 

    loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, action_indexes) 

    starter_learning_rate = 1e-6 

    global_step = tf.Variable(0, trainable=False) 

    # decay every 1000 steps with a base of 0.96: 
    learning_rate = tf.train.exponential_decay(starter_learning_rate, 
     global_step, 
     10000, 0.96, staircase=True) 

    optimizer = tf.train.RMSPropOptimizer(learning_rate) 

    trainable_variables = tf.trainable_variables() 
    varname_to_var = dict((v.name, v) for v in trainable_variables) 
    keys = sorted(varname_to_var.keys()) 

    gradients_and_variables = optimizer.compute_gradients(loss, [ varname_to_var[k] for k in keys]) 

    var_to_grad = dict((var.name, grad) for grad, var in gradients_and_variables) 

    name_to_var = dict((var.name, var) for _, var in gradients_and_variables) 

    # save the gradients in memory 
    var_to_ref_grad = {} 
    for k in keys: 
     grad = var_to_grad[k] 
     print(k, grad.get_shape()) 
     ref = tf.Variable(tf.zeros_like(grad)) 
     ref = ref.assign_add(grad) 
     var_to_ref_grad[k] = ref 

    discounted_values = tf.placeholder(tf.float32, shape=[None, 1], name='discounted_values') 

    # control when to apply gradients 
    compute_gradients_flag = tf.placeholder(tf.int32, name="compute_gradients") 
    def fn1(): 
     var_grad_list = [] 
     for k in keys: 
      grad = var_to_ref_grad[k] 
      var = varname_to_var[k] 
      var_grad_list.append((grad,var)) 

     optimizer.apply_gradients(var_grad_list) 
     return tf.no_op() 

    fn2 = lambda : tf.no_op() 

    last_op = tf.cond(tf.equal(compute_gradients_flag, 1), fn1, fn2) 

with tf.Session(graph=graph) as s: 

    feed_dict= { 
     state : np.zeros([minibatch, 80, 80, 1]), 
     action_indexes: [1], 
     compute_gradients_flag: False, 
    } 

    s.run(tf.initialize_all_variables()) 

    for i in range(10): 

     # accumulate gradients 
     s.run(last_op, feed_dict=feed_dict) 

Answer 1

. 롤아웃 업데이트를 배치로 적용하여 Tensorflow를 축적 할 수 있습니다. 이 같은

s_list = list_of_states_visited 
a_list = list_of_actions_taken 
R_list = list_of_value_targets 

sess.run(local_net.update, feed_dict={ 
    local_net.input: s_list, 
    local_net.a: a_list, 
    local_net.R: R_list 
}) 

Answer 2

뭔가 축적 된 그라디언트, 그라디언트를 축적 축적 된 그라디언트를 재설정하고, 적용하기위한 작전을 만들 작동 할 수 있습니다 (안된!) :

def build_gradient_accumulators(optimizer, gradients_and_variables): 
    accum_grads_and_vars = [] 
    accumulators = [] 
    resetters = [] 

    for grad, var in gradients_and_variables: 
     accum = tf.Variable(tf.zeros_like(grad)) 
     accum = accum.assign_add(grad) 
     accumulators.append(accum) 
     accum_grads_and_vars.append((accum, var)) 
     resetters.append(tf.assign(accum, tf.zeros_like(accum))) 

    reset_op = tf.group(*resetters) 
    accum_op = tf.group(*accumulators) 
    apply_op = optimizer.apply_gradients(accum_grads_and_vars) 
    return reset_op, accum_op, apply_op