from pandas_datareader import dataimport matplotlib.pyplot as pltimport pandas as pdimport datetime as dtimport urllib.request, jsonimport osimport numpy as np# This code has been tested with TensorFlow 1.6import tensorflow as tffrom tensorflow.examples.tutorials.mnist import input_data
defaccuracy(predictions,labels):''' Accuracy of a given set of predictions of size (N x n_classes) and labels of size (N x n_classes) '''return np.sum(np.argmax(predictions,axis=1)==np.argmax(labels,axis=1))*100.0/labels.shape[0]
4.1 定义输入输出和权重
batch_size =100layer_ids = ['hidden1','hidden2','hidden3','hidden4','hidden5','out']layer_sizes = [784,500,400,300,200,100,10]tf.reset_default_graph()# Inputs and Labelstrain_inputs = tf.placeholder(tf.float32, shape=[batch_size, layer_sizes[0]], name='train_inputs')train_labels = tf.placeholder(tf.float32, shape=[batch_size, layer_sizes[-1]], name='train_labels')# Weight and Bias definitionsfor idx, lid inenumerate(layer_ids):with tf.variable_scope(lid): w = tf.get_variable('weights',shape=[layer_sizes[idx], layer_sizes[idx+1]], initializer=tf.truncated_normal_initializer(stddev=0.05)) b = tf.get_variable('bias',shape= [layer_sizes[idx+1]], initializer=tf.random_uniform_initializer(-0.1,0.1))
# Name scope allows you to group various summaries together# Summaries having the same name_scope will be displayed on the same rowwith tf.name_scope('performance'):# Summaries need to be displayed# Whenever you need to record the loss, feed the mean loss to this placeholder tf_loss_ph = tf.placeholder(tf.float32,shape=None,name='loss_summary')# Create a scalar summary object for the loss so it can be displayed tf_loss_summary = tf.summary.scalar('loss', tf_loss_ph)# Whenever you need to record the loss, feed the mean test accuracy to this placeholder tf_accuracy_ph = tf.placeholder(tf.float32,shape=None, name='accuracy_summary')# Create a scalar summary object for the accuracy so it can be displayed tf_accuracy_summary = tf.summary.scalar('accuracy', tf_accuracy_ph)# Gradient norm summaryfor g,v in grads_and_vars:if'hidden5'in v.name and'weights'in v.name:with tf.name_scope('gradients'): tf_last_grad_norm = tf.sqrt(tf.reduce_mean(g**2)) tf_gradnorm_summary = tf.summary.scalar('grad_norm', tf_last_grad_norm)break# Merge all summaries togetherperformance_summaries = tf.summary.merge([tf_loss_summary,tf_accuracy_summary])
image_size =28n_channels =1n_classes =10n_train =55000n_valid =5000n_test =10000n_epochs =25config = tf.ConfigProto(allow_soft_placement=True)config.gpu_options.allow_growth =Trueconfig.gpu_options.per_process_gpu_memory_fraction =0.9# making sure Tensorflow doesn't overflow the GPUsession = tf.InteractiveSession(config=config)ifnot os.path.exists('summaries'): os.mkdir('summaries')ifnot os.path.exists(os.path.join('summaries','first')): os.mkdir(os.path.join('summaries','first'))summ_writer = tf.summary.FileWriter(os.path.join('summaries','first'), session.graph) # create summary_writer and add graph in it
tf.global_variables_initializer().run()accuracy_per_epoch = []mnist_data = input_data.read_data_sets('MNIST_data', one_hot=True)for epoch inrange(n_epochs): loss_per_epoch = []for i inrange(n_train//batch_size):# =================================== Training for one step ======================================== batch = mnist_data.train.next_batch(batch_size)# Get one batch of training dataif i ==0:# Only for the first epoch, get the summary data# Otherwise, it can clutter the visualization l,_,gn_summ = session.run([tf_loss,tf_loss_minimize,tf_gradnorm_summary], feed_dict={train_inputs: batch[0].reshape(batch_size,image_size*image_size), train_labels: batch[1], tf_learning_rate: 0.0001}) summ_writer.add_summary(gn_summ, epoch)else:# Optimize with training data l,_ = session.run([tf_loss,tf_loss_minimize], feed_dict={train_inputs: batch[0].reshape(batch_size,image_size*image_size), train_labels: batch[1], tf_learning_rate: 0.0001}) loss_per_epoch.append(l)print('Average loss in epoch %d: %.5f'%(epoch,np.mean(loss_per_epoch))) avg_loss = np.mean(loss_per_epoch)# ====================== Calculate the Validation Accuracy ========================== valid_accuracy_per_epoch = []for i inrange(n_valid//batch_size): valid_images,valid_labels = mnist_data.validation.next_batch(batch_size) valid_batch_predictions = session.run( tf_predictions,feed_dict={train_inputs: valid_images.reshape(batch_size,image_size*image_size)}) valid_accuracy_per_epoch.append(accuracy(valid_batch_predictions,valid_labels)) mean_v_acc = np.mean(valid_accuracy_per_epoch)print('\tAverage Valid Accuracy in epoch %d: %.5f'%(epoch,np.mean(valid_accuracy_per_epoch)))# ===================== Calculate the Test Accuracy =============================== accuracy_per_epoch = []for i inrange(n_test//batch_size): test_images, test_labels = mnist_data.test.next_batch(batch_size) test_batch_predictions = session.run( tf_predictions,feed_dict={train_inputs: test_images.reshape(batch_size,image_size*image_size)} ) accuracy_per_epoch.append(accuracy(test_batch_predictions,test_labels))print('\tAverage Test Accuracy in epoch %d: %.5f\n'%(epoch,np.mean(accuracy_per_epoch))) avg_test_accuracy = np.mean(accuracy_per_epoch)# Execute the summaries defined above summ = session.run(performance_summaries, feed_dict={tf_loss_ph:avg_loss, tf_accuracy_ph:avg_test_accuracy})# Write the obtained summaries to the file, so it can be displayed in the TensorBoard summ_writer.add_summary(summ, epoch)session.close()
# Summaries need to be displayed# Create a summary for each weight bias in each layerall_summaries = []for lid in layer_ids:with tf.name_scope(lid+'_hist'):with tf.variable_scope(lid,reuse=True): w,b = tf.get_variable('weights'), tf.get_variable('bias')# Create a scalar summary object for the loss so it can be displayed tf_w_hist = tf.summary.histogram('weights_hist', tf.reshape(w,[-1])) tf_b_hist = tf.summary.histogram('bias_hist', b) all_summaries.extend([tf_w_hist, tf_b_hist])# Merge all parameter histogram summaries togethertf_param_summaries = tf.summary.merge(all_summaries)
5.2 训练神经网络
这一步和之前训练的过程几乎一样了,只是多了一个tf_param_summaries
image_size =28n_channels =1n_classes =10n_train =55000n_valid =5000n_test =10000n_epochs =25config = tf.ConfigProto(allow_soft_placement=True)config.gpu_options.allow_growth =Trueconfig.gpu_options.per_process_gpu_memory_fraction =0.9# making sure Tensorflow doesn't overflow the GPUsession = tf.InteractiveSession(config=config)ifnot os.path.exists('summaries'): os.mkdir('summaries')ifnot os.path.exists(os.path.join('summaries','third')): os.mkdir(os.path.join('summaries','third'))summ_writer_3 = tf.summary.FileWriter(os.path.join('summaries','third'), session.graph)tf.global_variables_initializer().run()accuracy_per_epoch = []mnist_data = input_data.read_data_sets('MNIST_data', one_hot=True)for epoch inrange(n_epochs): loss_per_epoch = []for i inrange(n_train//batch_size):# =================================== Training for one step ======================================== batch = mnist_data.train.next_batch(batch_size)# Get one batch of training dataif i ==0:# Only for the first epoch, get the summary data# Otherwise, it can clutter the visualization l,_,gn_summ, wb_summ = session.run([tf_loss,tf_loss_minimize,tf_gradnorm_summary, tf_param_summaries], feed_dict={train_inputs: batch[0].reshape(batch_size,image_size*image_size), train_labels: batch[1], tf_learning_rate: 0.00001}) summ_writer_3.add_summary(gn_summ, epoch) summ_writer_3.add_summary(wb_summ, epoch)else:# Optimize with training data l,_ = session.run([tf_loss,tf_loss_minimize], feed_dict={train_inputs: batch[0].reshape(batch_size,image_size*image_size), train_labels: batch[1], tf_learning_rate: 0.01}) loss_per_epoch.append(l)print('Average loss in epoch %d: %.5f'%(epoch,np.mean(loss_per_epoch))) avg_loss = np.mean(loss_per_epoch)# ====================== Calculate the Validation Accuracy ========================== valid_accuracy_per_epoch = []for i inrange(n_valid//batch_size): valid_images,valid_labels = mnist_data.validation.next_batch(batch_size) valid_batch_predictions = session.run( tf_predictions,feed_dict={train_inputs: valid_images.reshape(batch_size,image_size*image_size)}) valid_accuracy_per_epoch.append(accuracy(valid_batch_predictions,valid_labels)) mean_v_acc = np.mean(valid_accuracy_per_epoch)print('\tAverage Valid Accuracy in epoch %d: %.5f'%(epoch,np.mean(valid_accuracy_per_epoch)))# ===================== Calculate the Test Accuracy =============================== accuracy_per_epoch = []for i inrange(n_test//batch_size): test_images, test_labels = mnist_data.test.next_batch(batch_size) test_batch_predictions = session.run( tf_predictions,feed_dict={train_inputs: test_images.reshape(batch_size,image_size*image_size)} ) accuracy_per_epoch.append(accuracy(test_batch_predictions,test_labels))print('\tAverage Test Accuracy in epoch %d: %.5f\n'%(epoch,np.mean(accuracy_per_epoch))) avg_test_accuracy = np.mean(accuracy_per_epoch)# Execute the summaries defined above summ = session.run(performance_summaries, feed_dict={tf_loss_ph:avg_loss, tf_accuracy_ph:avg_test_accuracy})# Write the obtained summaries to the file, so they can be displayed summ_writer_3.add_summary(summ, epoch)session.close()