'''Load MNIST data and print some information'''
data = input_data.read_data_sets("MNIST_data", one_hot = True)
print("Size of:")
print("\t training-set:\t\t{}".format(len(data.train.labels)))
print("\t test-set:\t\t\t{}".format(len(data.test.labels)))
print("\t validation-set:\t{}".format(len(data.validation.labels)))
print(data.test.labels[0:5])
data.test.cls = np.array([label.argmax() for label in data.test.labels])   # get the actual value
print(data.test.cls[0:5])

'''define a funciton to plot 9 images'''
def plot_images(images, cls_true, cls_pred = None):
    '''
    @parameter images:   the images info
    @parameter cls_true: the true value of image
    @parameter cls_pred: the prediction value, default is None
    '''
    assert len(images) == len(cls_true) == 9  # only show 9 images
    fig, axes = plt.subplots(nrows=3, ncols=3)
    for i, ax in enumerate(axes.flat):
        ax.imshow(images[i].reshape(img_shape), cmap="binary")  # binary means black_white image
        # show the true and pred values
        if cls_pred is None:
            xlabel = "True: {0}".format(cls_true[i])
        else:
            xlabel = "True: {0},Pred: {1}".format(cls_true[i],cls_pred[i])
        ax.set_xlabel(xlabel)
        ax.set_xticks([])  # remove the ticks
        ax.set_yticks([])
    plt.show()

'''show 9 images'''
images = data.test.images[0:9]
cls_true = data.test.cls[0:9]
plot_images(images, cls_true)

'''define the placeholder'''
X = tf.placeholder(tf.float32, [None, img_size_flat])    # None means the arbitrary number of labels, the features size is img_size_flat 
y_true = tf.placeholder(tf.float32, [None, num_classes]) # output size is num_classes
y_true_cls = tf.placeholder(tf.int64, [None])

'''define weights and biases'''
weights = tf.Variable(tf.zeros([img_size_flat, num_classes]))  # img_size_flat*num_classes
biases = tf.Variable(tf.zeros([num_classes]))

'''define the model'''
logits = tf.matmul(X,weights) + biases 
y_pred = tf.nn.softmax(logits)
y_pred_cls = tf.argmax(y_pred, dimension=1)
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=y_true, 
                                                       logits=logits)
cost = tf.reduce_mean(cross_entropy)
'''define the optimizer'''
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.5).minimize(cost)

'''define the accuracy'''
correct_prediction = tf.equal(y_pred_cls, y_true_cls)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

'''run the datagraph and use batch gradient descent'''
session = tf.Session()
session.run(tf.global_variables_initializer())
batch_size = 100

'''define a function to run the optimizer'''
def optimize(num_iterations):
    '''
    @parameter num_iterations: the traning times
    '''
    for i in range(num_iterations):
        x_batch, y_true_batch = data.train.next_batch(batch_size)
        feed_dict_train = {X: x_batch,y_true: y_true_batch}
        session.run(optimizer, feed_dict=feed_dict_train)

feed_dict_test = {X: data.test.images, 
                  y_true: data.test.labels, 
                  y_true_cls: data.test.cls}        
'''define a function to print the accuracy'''    
def print_accuracy():
    acc = session.run(accuracy, feed_dict=feed_dict_test)
    print("Accuracy on test-set:{0:.1%}".format(acc))

'''define a function to plot the error prediciton'''    
def plot_example_errors():
    correct, cls_pred = session.run([correct_prediction, y_pred_cls], feed_dict=feed_dict_test) 
    incorrect = (correct == False)
    images = data.test.images[incorrect]  # get the prediction error images
    cls_pred = cls_pred[incorrect]        # get prediction value
    cls_true = data.test.cls[incorrect]   # get true value
    plot_images(images[0:9], cls_true[0:9], cls_pred[0:9])

'''define a fucntion to plot weights'''
def plot_weights():
    w = session.run(weights)
    w_min = np.min(w)
    w_max = np.max(w)
    fig, axes = plt.subplots(3, 4)
    fig.subplots_adjust(0.3, 0.3)
    for i, ax in enumerate(axes.flat):
        if i<10:
            image = w[:,i].reshape(img_shape)
            ax.set_xlabel("Weights: {0}".format(i))
            ax.imshow(image, vmin=w_min,vmax=w_max,cmap="seismic")
        ax.set_xticks([])
        ax.set_yticks([])
    plt.show()

'''define a function to printand plot the confusion matrix using scikit-learn.'''   
def print_confusion_martix():
    cls_true = data.test.cls  # test set actual value 
    cls_pred = session.run(y_pred_cls, feed_dict=feed_dict_test)  # test set predict value
    cm = confusion_matrix(y_true=cls_true,y_pred=cls_pred)        # use sklearn confusion_matrix
    print(cm)
    plt.imshow(cm, interpolation='nearest',cmap=plt.cm.Blues) # Plot the confusion matrix as an image.
    plt.tight_layout()
    plt.colorbar()
    tick_marks = np.arange(num_classes)
    tick_marks = np.arange(num_classes)
    plt.xticks(tick_marks, range(num_classes))
    plt.yticks(tick_marks, range(num_classes))
    plt.xlabel('Predicted')
    plt.ylabel('True')    
    plt.show()

'''define cnn description'''
filter_size1 = 5     # the first conv filter size is 5x5 
num_filters1 = 32    # there are 32 filters
filter_size2 = 5     # the second conv filter size
num_filters2 = 64    # there are 64 filters
fc_size = 1024       # fully-connected layer

'''define a function to intialize weights'''
def initialize_weights(shape):
    '''
    @param shape：the shape of weights
    '''
    return tf.Variable(tf.truncated_normal(shape=shape, stddev=0.1))
'''define a function to intialize biases'''
def initialize_biases(length):
    '''
    @param length: the length of biases, which is a vector
    '''
    return tf.Variable(tf.constant(0.1,shape=[length]))

'''define a function to do conv and pooling if used'''
def conv_layer(input, 
               num_input_channels,
               filter_size,
               num_output_filters,
               use_pooling=True):
    '''
    @param input: the input of previous layer's output
    @param num_input_channels: input channels
    @param filter_size: the weights filter size
    @param num_output_filters: the output number channels
    @param use_pooling: if use pooling operation
    '''
    shape = [filter_size, filter_size, num_input_channels, num_output_filters]
    weights = initialize_weights(shape=shape)
    biases = initialize_biases(length=num_output_filters)   # one for each filter
    layer = tf.nn.conv2d(input=input, filter=weights, strides=[1,1,1,1], padding='SAME')
    layer += biases
    if use_pooling:
        layer = tf.nn.max_pool(value=layer,
                               ksize=[1,2,2,1],
                               strides=[1,2,2,1],
                               padding="SAME")   # the kernel function size is 2x2,so the ksize=[1,2,2,1]
    layer = tf.nn.relu(layer)
    return layer, weights

'''define a function to flat conv layer'''
def flatten_layer(layer):
    '''
    @param layer: the conv layer
    '''
    layer_shape = layer.get_shape() # get the shape of the layer(layer_shape == [num_images, img_height, img_width, num_channels])
    num_features = layer_shape[1:4].num_elements()  # [1:4] means the last three demension, namely the flatten size
    layer_flat = tf.reshape(layer, [-1, num_features])   # reshape to flat,-1 means don't care about the number of images
    return layer_flat, num_features

'''define a function to do fully-connected'''
def fc_layer(input, num_inputs, num_outputs, use_relu=True):
    '''
    @param input: the input
    @param num_inputs: the input size
    @param num_outputs: the output size
    @param use_relu: if use relu activation function
    '''
    weights = initialize_weights(shape=[num_inputs, num_outputs])
    biases = initialize_biases(num_outputs)
    layer = tf.matmul(input, weights) + biases
    if use_relu:
        layer = tf.nn.relu(layer)
    return layer

'''define the placeholder'''
X = tf.placeholder(tf.float32, shape=[None, img_flat_size], name="X")
X_image = tf.reshape(X, shape=[-1, img_size, img_size, num_channels])  # reshape to the image shape
y_true = tf.placeholder(tf.float32, [None, num_classes], name="y_true")
y_true_cls = tf.argmax(y_true, axis=1)
keep_prob = tf.placeholder(tf.float32)  # drop out placeholder

'''define the cnn model'''
layer_conv1, weights_conv1 = conv_layer(input=X_image, num_input_channels=num_channels, 
                                       filter_size=filter_size1, 
                                       num_output_filters=num_filters1,
                                       use_pooling=True)
print("conv1:",layer_conv1)
layer_conv2, weights_conv2 = conv_layer(input=layer_conv1, num_input_channels=num_filters1, 
                                        filter_size=filter_size2,
                                        num_output_filters=num_filters2,
                                        use_pooling=True)
print("conv2:",layer_conv2)
layer_flat, num_features = flatten_layer(layer_conv2) # the num_feature is 7x7x36=1764
print("flatten layer:", layer_flat)  
layer_fc1 = fc_layer(layer_flat, num_features, fc_size, use_relu=True)
print("fully-connected layer1:", layer_fc1)
layer_drop_out = tf.nn.dropout(layer_fc1, keep_prob)   # dropout operation
layer_fc2 = fc_layer(layer_drop_out, fc_size, num_classes,use_relu=False)
print("fully-connected layer2:", layer_fc2)
y_pred = tf.nn.softmax(layer_fc2)
y_pred_cls = tf.argmax(y_pred, axis=1)
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=y_true, 
                                                       logits=layer_fc2)
cost = tf.reduce_mean(cross_entropy)
optimizer = tf.train.AdamOptimizer(learning_rate=1e-3).minimize(cost)  # use AdamOptimizer优化

'''define accuracy'''
correct_prediction = tf.equal(y_true_cls, y_pred_cls)
accuracy = tf.reduce_mean(tf.cast(correct_prediction,dtype=tf.float32))

'''define a function to run train the model with bgd'''
total_iterations = 0  # record the total iterations
def optimize(num_iterations):
    '''
    @param num_iterations: the total interations of train batch_size operation
    '''
    global total_iterations
    start_time = time.time()
    for i in range(total_iterations,total_iterations + num_iterations):
        x_batch, y_batch = data.train.next_batch(batch_size)
        feed_dict = {X: x_batch, y_true: y_batch, keep_prob: 0.5}
        session.run(optimizer, feed_dict=feed_dict)
        if i % 10 == 0:
            acc = session.run(accuracy, feed_dict=feed_dict)
            msg = "Optimization Iteration: {0:>6}, Training Accuracy: {1:>6.1%}"    # {:>6}means the fixed width,{1:>6.1%}means the fixed width is 6 and keep 1 decimal place         
            print(msg.format(i + 1, acc))
    total_iterations += num_iterations
    end_time = time.time()
    time_dif = end_time-start_time
    print("time usage:"+str(timedelta(seconds=int(round(time_dif)))))

Optimization Iteration:    651, Training Accuracy:  99.0%
Optimization Iteration:    661, Training Accuracy:  99.0%
Optimization Iteration:    671, Training Accuracy:  99.0%
Optimization Iteration:    681, Training Accuracy:  99.0%
Optimization Iteration:    691, Training Accuracy:  99.0%
Optimization Iteration:    701, Training Accuracy:  99.0%
Optimization Iteration:    711, Training Accuracy:  99.0%
Optimization Iteration:    721, Training Accuracy:  99.0%
Optimization Iteration:    731, Training Accuracy:  99.0%
Optimization Iteration:    741, Training Accuracy: 100.0%
Optimization Iteration:    751, Training Accuracy:  99.0%
Optimization Iteration:    761, Training Accuracy:  99.0%
Optimization Iteration:    771, Training Accuracy:  97.0%
Optimization Iteration:    781, Training Accuracy:  96.0%
Optimization Iteration:    791, Training Accuracy:  98.0%
Optimization Iteration:    801, Training Accuracy: 100.0%
Optimization Iteration:    811, Training Accuracy: 100.0%
Optimization Iteration:    821, Training Accuracy:  97.0%
Optimization Iteration:    831, Training Accuracy:  98.0%
Optimization Iteration:    841, Training Accuracy:  99.0%
Optimization Iteration:    851, Training Accuracy:  99.0%
Optimization Iteration:    861, Training Accuracy:  99.0%
Optimization Iteration:    871, Training Accuracy:  96.0%
Optimization Iteration:    881, Training Accuracy:  99.0%
Optimization Iteration:    891, Training Accuracy:  99.0%
Optimization Iteration:    901, Training Accuracy:  98.0%
Optimization Iteration:    911, Training Accuracy:  99.0%
Optimization Iteration:    921, Training Accuracy:  99.0%
Optimization Iteration:    931, Training Accuracy:  99.0%
Optimization Iteration:    941, Training Accuracy:  98.0%
Optimization Iteration:    951, Training Accuracy: 100.0%
Optimization Iteration:    961, Training Accuracy:  99.0%
Optimization Iteration:    971, Training Accuracy:  98.0%
Optimization Iteration:    981, Training Accuracy:  99.0%
Optimization Iteration:    991, Training Accuracy: 100.0%
time usage:0:07:07

batch_size_test = 256
def print_test_accuracy(print_error=False,print_confusion_matrix=False):
    '''
    @param print_error: whether plot the error images
    @param print_confusion_matrix: whether plot the confusion_matrix
    '''
    num_test = len(data.test.images)   
    cls_pred = np.zeros(shape=num_test, dtype=np.int)  # declare the cls_pred
    i = 0
    #predict the test set using batch_size
    while i < num_test:
        j = min(i + batch_size_test, num_test)
        images = data.test.images[i:j,:]
        labels = data.test.labels[i:j,:]
        feed_dict = {X:images,y_true:labels,keep_prob:0.5}
        cls_pred[i:j] = session.run(y_pred_cls,feed_dict=feed_dict)
        i = j
    cls_true = data.test.cls
    correct = (cls_true == cls_pred)
    correct_sum = correct.sum()   # correct predictions
    acc = float(correct_sum)/num_test
    msg = "Accuracy on Test-Set: {0:.1%} ({1} / {2})"
    print(msg.format(acc, correct_sum, num_test))    
    if print_error:
        plot_error_pred(cls_pred,correct)
    if print_confusion_matrix:
        plot_confusin_martrix(cls_pred)

'''define a function to plot conv weights'''
def plot_conv_weights(weights,input_channel=0):
    '''
    @param weights: the conv filter weights, for example: the weights_conv1 and weights_conv2, which are 4 dimension [filter_size, filter_size, num_input_channels, num_output_filters]
    @param input_channel: the input_channels
    '''
    w = session.run(weights)
    w_min = np.min(w)
    w_max = np.max(w)
    num_filters = w.shape[3]   # get the number of filters
    num_grids = math.ceil(math.sqrt(num_filters))
    fig, axes = plt.subplots(num_grids, num_grids)
    for i, ax in enumerate(axes.flat):
        if i < num_filters:
            img = w[:,:,input_channel,i]   # the ith weight
            ax.imshow(img,vmin=w_min,vmax=w_max,interpolation="nearest",cmap='seismic')
        ax.set_xticks([])
        ax.set_yticks([])
    plt.show()

'''define a function to plot conv output layer'''
def plot_conv_layer(layer, image):
    '''
    @param layer: the conv layer, which is also a image after conv
    @param image: the image info
    '''
    feed_dict = {X:[image]}
    values = session.run(layer, feed_dict=feed_dict)
    num_filters = values.shape[3]   # get the number of filters
    num_grids = math.ceil(math.sqrt(num_filters))
    fig, axes = plt.subplots(num_grids,num_grids)
    for i, ax in enumerate(axes.flat):
        if i < num_filters:
            img = values[0,:,:,i]
            ax.imshow(img, interpolation="nearest",cmap="binary")
        ax.set_xticks([])
        ax.set_yticks([])
    plt.show()

'''declare the placeholder'''
X = tf.placeholder(tf.float32, [None, img_flat_size], name="X")
X_img = tf.reshape(X, shape=[-1,img_size,img_size, num_channels])
y_true = tf.placeholder(tf.float32, shape=[None, num_classes], name="y_true")
y_true_cls = tf.argmax(y_true,1)

'''define the cnn model with prettytensor'''
x_pretty = pt.wrap(X_img)
with pt.defaults_scope():   # or pt.defaults_scope(activation_fn=tf.nn.relu) if just use one activation function
    y_pred, loss = x_pretty.\
        conv2d(kernel=5, depth=16, activation_fn=tf.nn.relu, name="conv_layer1").\
        max_pool(kernel=2, stride=2).\
        conv2d(kernel=5, depth=36, activation_fn=tf.nn.relu, name="conv_layer2").\
        max_pool(kernel=2, stride=2).\
        flatten().\
        fully_connected(size=128, activation_fn=tf.nn.relu, name="fc_layer1").\
        softmax_classifier(num_classes=num_classes, labels=y_true)

'''define a function to get weights'''
def get_weights_variable(layer_name):
    with tf.variable_scope(layer_name, reuse=True):
        variable = tf.get_variable("weights")
    return variable
conv1_weights = get_weights_variable("conv_layer1")
conv2_weights = get_weights_variable("conv_layer2")

'''define optimizer to train'''
optimizer = tf.train.AdamOptimizer().minimize(loss)
y_pred_cls = tf.argmax(y_pred,1)
correct_prediction = tf.equal(y_pred_cls, y_true_cls)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
session = tf.Session()
session.run(tf.global_variables_initializer())

'''define a Saver to save the network'''
saver = tf.train.Saver()
save_dir = "checkpoints/"
if not os.path.exists(save_dir):
    os.makedirs(save_dir)
save_path = os.path.join(save_dir, 'best_validation')

saver.save(sess=session, save_path=save_path)

'''declear the train info'''
train_batch_size = 64
best_validation_accuracy = 0.0
last_improvement = 0
require_improvement_iterations = 1000
total_iterations = 0
'''define a function to optimize the optimizer'''
def optimize(num_iterations):
    global total_iterations
    global best_validation_accuracy
    global last_improvement
    start_time = time.time()
    for i in range(num_iterations):
        total_iterations += 1
        X_batch, y_true_batch = data.train.next_batch(train_batch_size)
        feed_dict_train = {X: X_batch,
                     y_true: y_true_batch}
        session.run(optimizer, feed_dict=feed_dict_train)
        if (total_iterations%100 == 0) or (i == num_iterations-1):
            acc_train = session.run(accuracy, feed_dict=feed_dict_train)
            acc_validation, _ = validation_accuracy()
            if acc_validation > best_validation_accuracy:
                best_validation_accuracy = acc_validation
                last_improvement = total_iterations
                saver.save(sess=session, save_path=save_path)
                improved_str = "*"
            else:
                improved_str = ""
            msg = "Iter: {0:>6}, Train_batch accuracy:{1:>6.1%}, validation acc:{2:>6.1%} {3}"
            print(msg.format(i+1, acc_train, acc_validation, improved_str))
        if total_iterations-last_improvement > require_improvement_iterations:
            print('No improvement found in a while, stop running')
            break
    end_time = time.time()
    time_diff = end_time-start_time
    print("Time usage:" + str(timedelta(seconds=int(round(time_diff)))))

Iter:   5100, Train_batch accuracy:100.0%, validation acc: 98.8% *
Iter:   5200, Train_batch accuracy:100.0%, validation acc: 98.3% 
Iter:   5300, Train_batch accuracy:100.0%, validation acc: 98.7% 
Iter:   5400, Train_batch accuracy: 98.4%, validation acc: 98.6% 
Iter:   5500, Train_batch accuracy: 98.4%, validation acc: 98.6% 
Iter:   5600, Train_batch accuracy:100.0%, validation acc: 98.7% 
Iter:   5700, Train_batch accuracy: 96.9%, validation acc: 98.9% *
Iter:   5800, Train_batch accuracy:100.0%, validation acc: 98.6% 
Iter:   5900, Train_batch accuracy:100.0%, validation acc: 98.6% 
Iter:   6000, Train_batch accuracy: 98.4%, validation acc: 98.7% 
Iter:   6100, Train_batch accuracy:100.0%, validation acc: 98.7% 
Iter:   6200, Train_batch accuracy:100.0%, validation acc: 98.7% 
Iter:   6300, Train_batch accuracy: 98.4%, validation acc: 98.8% 
Iter:   6400, Train_batch accuracy: 98.4%, validation acc: 98.8% 
Iter:   6500, Train_batch accuracy:100.0%, validation acc: 98.7% 
Iter:   6600, Train_batch accuracy:100.0%, validation acc: 98.7% 
Iter:   6700, Train_batch accuracy:100.0%, validation acc: 98.8% 
No improvement found in a while, stop running
Time usage:0:18:43

'''define a function to predict using batch'''
batch_size_predict = 256
def predict_cls(images, labels, cls_true):
    num_images = len(images)
    cls_pred = np.zeros(shape=num_images, dtype=np.int)
    i = 0
    while i < num_images:
        j = min(i+batch_size_predict, num_images)
        feed_dict = {X: images[i:j,:],
                     y_true: labels[i:j,:]}
        cls_pred[i:j] = session.run(y_pred_cls, feed_dict=feed_dict)
        i = j
    correct = (cls_true==cls_pred)
    return correct, cls_pred

def predict_cls_test():
    return predict_cls(data.test.images, data.test.labels, data.test.cls)

def predict_cls_validation():
    return predict_cls(data.validation.images, data.validation.labels, data.validation.cls)

'''calculate the acc'''
def cls_accuracy(correct):
    correct_sum = correct.sum()
    acc = float(correct_sum)/len(correct)
    return acc, correct_sum
'''define a function to calculate the validation acc'''
def validation_accuracy():
    correct, _ = predict_cls_validation()
    return cls_accuracy(correct)

'''define a function to calculate test acc'''
def print_test_accuracy(show_example_errors=False,
                        show_confusion_matrix=False):
    correct, cls_pred = predict_cls_test()
    acc, num_correct = cls_accuracy(correct)
    num_images = len(correct)
    msg = "Accuracy on Test-Set: {0:.1%} ({1} / {2})"
    print(msg.format(acc, num_correct, num_images))

    # Plot some examples of mis-classifications, if desired.
    if show_example_errors:
        print("Example errors:")
        plot_example_errors(cls_pred=cls_pred, correct=correct)

    # Plot the confusion matrix, if desired.
    if show_confusion_matrix:
        print("Confusion Matrix:")
        plot_confusion_matrix(cls_pred=cls_pred) 

'''将training set和validation set合并，并重新划分'''
combine_images = np.concatenate([data.train.images, data.validation.images], axis=0)
combine_labels = np.concatenate([data.train.labels, data.validation.labels], axis=0)
print("合并后图片：", combine_images.shape)
print("合并后label：", combine_labels.shape)
combined_size = combine_labels.shape[0]
train_size = int(0.8*combined_size)
validation_size = combined_size - train_size
'''函数：将合并后的重新随机划分'''
def random_training_set():
    idx = np.random.permutation(combined_size)   # 将0-combined_size数字随机排列
    idx_train = idx[0:train_size]
    idx_validation = idx[train_size:]
    x_train = combine_images[idx_train, :]
    y_train = combine_labels[idx_train, :]
    
    x_validation = combine_images[idx_validation, :]
    y_validation = combine_images[idx_validation, :]
    return x_train, y_train, x_validation, y_validation

def ensemble_predictions():
    pred_labels = []
    test_accuracies = []
    validation_accuracies = []
    for i in range(num_networks):
        saver.restore(sess=session, save_path=get_save_path(i))
        test_acc = test_accuracy()
        test_accuracies.append(test_acc)
        validation_acc = validation_accuracy()
        validation_accuracies.append(validation_acc)
        msg = "网络：{0}，验证集：{1:.4f}，测试集{2:.4f}"
        print(msg.format(i, validation_acc, test_acc))
        pred = predict_labels(data.test.images)
        pred_labels.append(pred)
    return np.array(pred_labels),\
           np.array(test_accuracies),\
           np.array(validation_accuracies)

ensemble_correct = (ensemble_cls_pred == data.test.cls)
ensemble_incorrect = np.logical_not(ensemble_correct)
print(test_accuracies)
best_net = np.argmax(test_accuracies)
print(best_net)
print(test_accuracies[best_net])
best_net_pred_labels = pred_labels[best_net, :, :]
best_net_cls_pred = np.argmax(best_net_pred_labels, axis=1)
best_net_correct = (best_net_cls_pred == data.test.cls)
best_net_incorrect = np.logical_not(best_net_correct)
print("融合后预测对的：", np.sum(ensemble_correct))
print("单个最好模型预测对的", np.sum(best_net_correct))
ensemble_better = np.logical_and(best_net_incorrect, ensemble_correct)  # 融合之后好于单个的个数
print(ensemble_better.sum())
best_net_better = np.logical_and(best_net_correct, ensemble_incorrect)  # 单个好于融合之后的个数
print(best_net_better.sum())

import cifar10
from cifar10 import img_size, num_channels, num_classes

'''下载cifar10数据集, 大概163M'''
cifar10.maybe_download_and_extract()
'''加载数据集'''
images_train, cls_train, labels_train = cifar10.load_training_data()
images_test,  cls_test,  labels_test  = cifar10.load_test_data()

'''打印一些信息'''
class_names = cifar10.load_class_names()
print(class_names)
print("Size of:")
print("training set:\t\t{}".format(len(images_train)))
print("test set:\t\t\t{}".format(len(images_test)))

'''显示9张图片函数'''
def plot_images(images, cls_true, cls_pred=None, smooth=True):   # smooth是否平滑显示
    assert len(images) == len(cls_true) == 9
    fig, axes = plt.subplots(3,3)
    
    for i, ax in enumerate(axes.flat):
        if smooth:
            interpolation = 'spline16'
        else:
            interpolation = 'nearest'
        ax.imshow(images[i, :, :, :], interpolation=interpolation)
        cls_true_name = class_names[cls_true[i]]
        if cls_pred is None:
            xlabel = "True:{0}".format(cls_true_name)
        else:
            cls_pred_name = class_names[cls_pred[i]]
            xlabel = "True:{0}, Pred:{1}".format(cls_true_name, cls_pred_name)
        ax.set_xlabel(xlabel)
        ax.set_xticks([])
        ax.set_yticks([])
    plt.show()

X = tf.placeholder(tf.float32, shape=[None, img_size, img_size, num_channels], name="X")
y_true = tf.placeholder(tf.float32, shape=[None, num_classes], name="y")
y_true_cls = tf.argmax(y_true, axis=1)

'''单个图片预处理, 测试集只需要裁剪就行了'''
def pre_process_image(image, training):
    if training:
        image = tf.random_crop(image, size=[img_size_cropped, img_size_cropped, num_channels])  # 裁剪
        image = tf.image.random_flip_left_right(image)                  # 左右翻转
        image = tf.image.random_hue(image, max_delta=0.05)              # 色调调整
        image = tf.image.random_brightness(image, max_delta=0.2)        # 曝光
        image = tf.image.random_saturation(image, lower=0.0, upper=2.0) # 饱和度
        '''上面的调整可能pixel值超过[0, 1], 所以约束一下'''        
        image = tf.minimum(image, 1.0)
        image = tf.maximum(image, 0.0)
    else:
        image = tf.image.resize_image_with_crop_or_pad(image, target_height=img_size_cropped, 
                                              target_width=img_size_cropped)
    return image

'''调用上面的函数，处理多个图片images'''
def pre_process(images, training):
    images = tf.map_fn(lambda image: pre_process_image(image, training), images)   # tf.map_fn()使用lambda函数
    return images

'''定义主网络函数'''
def main_network(images, training):
    x_pretty = pt.wrap(images)
    if training:
        phase = pt.Phase.train
    else:
        phase = pt.Phase.infer
    with pt.defaults_scope(activation_fn=tf.nn.relu, phase=phase):
        y_pred, loss = x_pretty.\
        conv2d(kernel=5, depth=64, name="layer_conv1", batch_normalize=True).\
        max_pool(kernel=2, stride=2).\
        conv2d(kernel=5, depth=64, name="layer_conv2").\
        max_pool(kernel=2, stride=2).\
        flatten().\
        fully_connected(size=256, name="layer_fc1").\
        fully_connected(size=128, name="layer_fc2").\
        softmax_classifier(num_classes, labels=y_true)
    return y_pred, loss

'''创建所有网络, 包含预处理和主网络，'''
def create_network(training):
    # 使用variable_scope可以重复使用定义的变量，训练时创建新的，测试时重复使用
    with tf.variable_scope("network", reuse=not training):
        images = X
        images = pre_process(images=images, training=training)
        y_pred, loss = main_network(images=images, training=training)
    return y_pred, loss

'''训练阶段网络创建'''
global_step = tf.Variable(initial_value=0, 
                          name="global_step",
                          trainable=False) # trainable 在训练阶段不会改变
_, loss = create_network(training=True)
optimizer = tf.train.AdamOptimizer(learning_rate=0.0001).minimize(loss, global_step)

'''测试阶段网络创建'''
y_pred, _ = create_network(training=False)
y_pred_cls = tf.argmax(y_pred, dimension=1)
correct_prediction = tf.equal(y_pred_cls, y_true_cls)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

def get_weights_variable(layer_name):
    with tf.variable_scope("network/" + layer_name, reuse=True):
        variable = tf.get_variable("weights")
    return variable 
weights_conv1 = get_weights_variable("layer_conv1")
weights_conv2 = get_weights_variable("layer_conv2")

def get_layer_output(layer_name):
    tensor_name = "network/" + layer_name + "/Relu:0"
    tensor = tf.get_default_graph().get_tensor_by_name(tensor_name)
    return tensor
output_conv1 = get_layer_output("layer_conv1")
output_conv2 = get_layer_output("layer_conv2")

'''执行tensorflow graph'''
session = tf.Session()
save_dir = "checkpoints/"
if not os.path.exists(save_dir):
    os.makedirs(save_dir)
save_path = os.path.join(save_dir, 'cifat10_cnn')

'''尝试存储最新的checkpoint, 可能会失败，比如第一次运行checkpoint不存在等'''
try:
    print("开始存储最新的存储...")
    last_chk_path = tf.train.latest_checkpoint(save_dir)
    saver.restore(session, save_path=last_chk_path)
    print("存储点来自：", last_chk_path)
except:
    print("存储错误, 初始化变量")
    session.run(tf.global_variables_initializer())

'''SGD'''
train_batch_size = 64
def random_batch():
    num_images = len(images_train)
    idx = np.random.choice(num_images, size=train_batch_size, replace=False)
    x_batch = images_train[idx, :, :, :]
    y_batch = labels_train[idx, :]
    return x_batch, y_batch

def optimize(num_iterations):
    start_time = time.time()
    for i in range(num_iterations):
        x_batch, y_batch = random_batch()
        feed_dict_train = {X: x_batch, y_true: y_batch}
        i_global, _ = session.run([global_step, optimizer], feed_dict=feed_dict_train)
        if (i_global%100==0) or (i == num_iterations-1):
            batch_acc = session.run(accuracy, feed_dict=feed_dict_train)
            msg = "global step: {0:>6}, training batch accuracy: {1:>6.1%}"
            print(msg.format(i_global, batch_acc))
        if(i_global%1000==0) or (i==num_iterations-1):
            saver.save(session, save_path=save_path,
                       global_step=global_step)
            print("保存checkpoint")
    end_time = time.time()
    time_diff = end_time-start_time
    print("耗时：", str(timedelta(seconds=int(round(time_diff)))))

import numpy as np
import tensorflow as tf
from matplotlib import pyplot as plt
import inception # 第三方类加载inception model
import os

'''下载和加载inception model'''
inception.maybe_download()
model = inception.Inception()

'''预测和显示图片'''
def classify(image_path):
    plt.imshow(plt.imread(image_path))
    plt.show()
    pred = model.classify(image_path=image_path)
    model.print_scores(pred=pred, k=10, only_first_name=True)

'''显示处理后图片的样式'''
def plot_resized_image(image_path):
    resized_image = model.get_resized_image(image_path)
    plt.imshow(resized_image, interpolation='nearest')
    plt.show()
plot_resized_image(image_path)

import numpy as np
import tensorflow as tf
import prettytensor as pt
from matplotlib import pyplot as plt
import time
from datetime import timedelta
import os
import inception   # 第三方下载inception model的代码
from inception import transfer_values_cache  # cache
import cifar10     # 也是第三方的库，下载cifar-10数据集
from cifar10 import num_classes

'''下载cifar-10数据集'''
cifar10.maybe_download_and_extract()
class_names = cifar10.load_class_names()
print("所有类别是：",class_names)
'''训练和测试集'''
images_train, cls_train, labels_train = cifar10.load_training_data()
images_test,  cls_test,  labels_test  = cifar10.load_test_data()

'''下载inception model'''
inception.maybe_download()
model = inception.Inception()

'''训练和测试的cache的路径'''
file_path_cache_train = os.path.join(cifar10.data_path, 'inception_cifar10_train.pkl')
file_path_cache_test = os.path.join(cifar10.data_path, 'inception_cifar10_test.pkl')

print('处理训练集上的transfer-values.......... ')
image_scaled = images_train * 255.0  # cifar-10的pixel是0-1的, shape=(50000, 32, 32, 3)
transfer_values_train = transfer_values_cache(cache_path=file_path_cache_train,
                                              images=image_scaled, 
                                              model=model)  # shape=(50000, 2048)
print('处理测试集上的transfer-values.......... ')
images_scaled = images_test * 255.0
transfer_values_test = transfer_values_cache(cache_path=file_path_cache_test,
                                             model=model,
                                             images=images_scaled)
print("transfer_values_train: ",transfer_values_train.shape)
print("transfer_values_test: ",transfer_values_test.shape)

'''显示transfer values'''
def plot_transfer_values(i):
    print("输入图片：")
    plt.imshow(images_test[i], interpolation='nearest')
    plt.show()
    print('transfer values --> 此图片在inception model上')
    img = transfer_values_test[i]
    img = img.reshape((32, 64))
    plt.imshow(img, interpolation='nearest', cmap='Reds')
    plt.show()
plot_transfer_values(16)

'''使用PCA分析transfer values'''
from sklearn.decomposition import PCA
pca = PCA(n_components=2)
transfer_values = transfer_values_train[0:3000]  # 取3000个，大的话计算量太大
cls = cls_train[0:3000]
print(transfer_values.shape)
transfer_values_reduced = pca.fit_transform(transfer_values)
print(transfer_values_reduced.shape)

## 显示降维后的transfer values
def plot_scatter(values, cls):
    from matplotlib import cm as cm
    cmap = cm.rainbow(np.linspace(0.0, 1.0, num_classes))
    colors = cmap[cls]
    x = values[:, 0]
    y = values[:, 1]
    plt.scatter(x, y, color=colors)
    plt.show()
plot_scatter(transfer_values_reduced, cls)

from sklearn.manifold import TSNE
pca = PCA(n_components=50)
transfer_values_50d = pca.fit_transform(transfer_values)
tsne = TSNE(n_components=2)
transfer_values_reduced = tsne.fit_transform(transfer_values_50d)
print("最终降维后：", transfer_values_reduced.shape)
plot_scatter(transfer_values_reduced, cls)

'''创建网络'''
transfer_len = model.transfer_len   # 获取transfer values的大小，这里是2048
x = tf.placeholder(tf.float32, shape=[None, transfer_len], name="x")
y_true = tf.placeholder(tf.float32, shape=[None, num_classes], name="y")
y_true_cls = tf.argmax(y_true, axis=1)
x_pretty = pt.wrap(x)
with pt.defaults_scope(activation_fn=tf.nn.relu):
    y_pred, loss = x_pretty.\
        fully_connected(1024, name="layer_fc1").\
        softmax_classifier(num_classes, labels=y_true)

'''优化器'''
global_step = tf.Variable(initial_value=0, name="global_step", trainable=False)
optimizer = tf.train.AdamOptimizer(0.0001).minimize(loss, global_step)

'''accuracy'''
y_pred_cls = tf.argmax(y_pred, axis=1)
correct_prediction = tf.equal(y_pred_cls, y_true_cls)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

'''SGD 训练'''
session = tf.Session()
session.run(tf.initialize_all_variables())
train_batch_size = 64
def random_batch():
    num_images = len(images_train)
    idx = np.random.choice(num_images, 
                           size=train_batch_size,
                           replace=False)
    x_batch = transfer_values_train[idx]
    y_batch = labels_train[idx]
    return x_batch, y_batch
def optimize(num_iterations):
    start_time = time.time()
    for i in range(num_iterations):
        x_batch, y_true_batch = random_batch()
        feed_dict_train = {x: x_batch,
                           y_true: y_true_batch}
        i_global, _ = session.run([global_step, optimizer], feed_dict=feed_dict_train)
        if (i_global % 100 == 0) or (i==num_iterations-1):
            batch_acc = session.run(accuracy, feed_dict=feed_dict_train)
            msg = "Global Step: {0:>6}, Training Batch Accuracy: {1:>6.1%}"
            print(msg.format(i_global, batch_acc))            
    end_time = time.time()
    time_diff = end_time - start_time
    print("耗时：", str(timedelta(seconds=int(round(time_diff)))))

'''batch 预测'''
batch_size = 256
def predict_cls(transfer_values, labels, cls_true):
    num_images = len(images_test)
    cls_pred = np.zeros(shape=num_images, dtype=np.int)
    i = 0
    while i < num_images:
        j = min(i + batch_size, num_images)
        feed_dict = {x: transfer_values[i:j],
                     y_true: labels[i:j]}
        cls_pred[i:j] = session.run(y_pred_cls, feed_dict=feed_dict)
        i = j
    correct = (cls_true == cls_pred)
    return correct, cls_pred

print("tensorflow版本", tf.__version__)
'''读取数据'''
mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
print("size of")
print('--training set:\t\t{}'.format(len(mnist.train.labels)))
print('--test set:\t\t\t{}'.format(len(mnist.test.labels)))
print('--validation set:\t{}'.format(len(mnist.validation.labels)))
'''定义超参数'''
learning_rate = 0.001
batch_size = 128
n_inputs = 28
n_steps = 28
state_size = 128
n_classes = 10

'''定义placehoder和初始化weights和biases'''
x = tf.placeholder(tf.float32, [batch_size, n_steps, n_inputs], name='x')
y = tf.placeholder(tf.float32, [batch_size, n_classes], name='y')
weights = {
    # (28, 128)
    #'in': tf.Variable(initial_value=tf.random_normal([n_inputs, state_size])),
    # (128, 10)
    'out': tf.Variable(tf.random_normal(shape=[state_size, n_classes], mean=0.0, stddev=1.0, 
                                       dtype=tf.float32, 
                                       seed=None, 
                                       name=None))
}
biases = {
    # (128, )
    #'in': tf.Variable(initial_value=tf.constant(0.1,shape=[state_size,]), trainable=True, collections=None, 
                     #validate_shape=True, 
                     #caching_device=None, name=None, 
                     #variable_def=None, dtype=None, 
                     #expected_shape=None, 
                     #import_scope=None),
    # (10, )
    'out': tf.Variable(initial_value=tf.constant(0.1, shape=[n_classes, ]), trainable=True, collections=None, 
                      validate_shape=True, 
                      caching_device=None, name=None, 
                      variable_def=None, dtype=None, 
                      expected_shape=None, 
                      import_scope=None)
}

'''定义RNN 结构'''
def RNN(X, weights, biases):
    '''这里输入X 不用再做权重的运算，cell中会自动运算（_linear函数）, 做了运算也没有实际意义，因为LSTM的cell输入的流向有多个'''
    # 原始的 X 是 3 维数据, 我们需要把它变成 2 维数据才能使用 weights 的矩阵乘法
    # X ==> (128 batch_size * 28 steps, 28 inputs)
    #X = tf.reshape(X, [-1, n_inputs])
    #X_in = tf.matmul(X, weights['in']) + biases['in']
    #  再换回3维
    # X_in ==> (128 batches, 28 steps, 128 hidden)
    #X_in = tf.reshape(X_in, shape=[-1, n_steps, state_size])
    '''cell中的计算方式1'''
    cell = tf.nn.rnn_cell.BasicLSTMCell(num_units=state_size)
    init_state = cell.zero_state(batch_size, dtype=tf.float32)
    rnn_outputs, final_state = tf.nn.dynamic_rnn(cell=cell,
                                                 inputs=X,
                                                 initial_state=init_state,
                                                 time_major=False)
    results = tf.matmul(final_state[1], weights['out']) + biases['out']
    return results

prediction = RNN(x, weights, biases)
losses = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=prediction,
                                                                labels=y))
train_step = tf.train.AdamOptimizer(learning_rate).minimize(losses)
prediction_cls = tf.argmax(prediction, axis=1)
correct_pred = tf.equal(prediction_cls, tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))

def optimize(n_epochs):
    '''训练RNN'''
    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        for i in range(n_epochs):
            batch_x, batch_y = mnist.train.next_batch(batch_size)
            batch_x = batch_x.reshape([batch_size, n_steps, n_inputs])
            feed_dict = {x: batch_x, y: batch_y}
            sess.run(train_step, feed_dict=feed_dict)
            if i % 50 == 0:
                print("epoch: {0}, accuracy:{1}".format(i, sess.run(accuracy, feed_dict=feed_dict)))

epoch: 0, accuracy:0.1796875
epoch: 50, accuracy:0.7109375
epoch: 100, accuracy:0.828125
epoch: 150, accuracy:0.8359375
epoch: 200, accuracy:0.8984375
epoch: 250, accuracy:0.9296875
epoch: 300, accuracy:0.9375
epoch: 350, accuracy:0.921875
epoch: 400, accuracy:0.9609375
epoch: 450, accuracy:0.953125
epoch: 500, accuracy:0.921875
epoch: 550, accuracy:0.9296875
epoch: 600, accuracy:0.9609375
epoch: 650, accuracy:0.9375
epoch: 700, accuracy:0.9765625
epoch: 750, accuracy:0.96875
epoch: 800, accuracy:0.9375
epoch: 850, accuracy:0.9296875
epoch: 900, accuracy:0.9609375
epoch: 950, accuracy:0.96875