pyspark 和tensorflow训练模型

TensorFlow：

# TensorFlow and tf.keras
import tensorflow as tf
from tensorflow import keras

# Helper libraries
import numpy as np
import matplotlib.pyplot as plt

print(tf.__version__)

fashion_mnist = keras.datasets.fashion_mnist

(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()

class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',
               'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']


plt.figure()
plt.imshow(train_images[0])
plt.colorbar()
plt.grid(False)
plt.show()

train_images = train_images / 255.0

test_images = test_images / 255.0

plt.figure(figsize=(10,10))
for i in range(25):
plt.subplot(5,5,i+1)
    plt.xticks([])
    plt.yticks([])
    plt.grid(False)
    plt.imshow(train_images[i], cmap=plt.cm.binary)
    plt.xlabel(class_names[train_labels[i]])
plt.show()

model = keras.Sequential([
    keras.layers.Flatten(input_shape=(28, 28)),
    keras.layers.Dense(128, activation='relu'),
    keras.layers.Dense(10)
])

model.compile(optimizer='adam',
              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
              metrics=['accuracy'])

model.fit(train_images, train_labels, epochs=10)

test_loss, test_acc = model.evaluate(test_images,  test_labels, verbose=2)

print('\nTest accuracy:', test_acc)
probability_model = tf.keras.Sequential([model,
                                         tf.keras.layers.Softmax()])

predictions = probability_model.predict(test_images)
def plot_image(i, predictions_array, true_label, img):
  predictions_array, true_label, img = predictions_array, true_label[i], img[i]
  plt.grid(False)
  plt.xticks([])
  plt.yticks([])

  plt.imshow(img, cmap=plt.cm.binary)

  predicted_label = np.argmax(predictions_array)
  if predicted_label == true_label:
    color = 'blue'
  else:
    color = 'red'

  plt.xlabel("{} {:2.0f}% ({})".format(class_names[predicted_label],
                                100*np.max(predictions_array),
                                class_names[true_label]),
                                color=color)

def plot_value_array(i, predictions_array, true_label):
  predictions_array, true_label = predictions_array, true_label[i]
  plt.grid(False)
  plt.xticks(range(10))
  plt.yticks([])
  thisplot = plt.bar(range(10), predictions_array, color="#777777")
  plt.ylim([0, 1])
  predicted_label = np.argmax(predictions_array)

  thisplot[predicted_label].set_color('red')
  thisplot[true_label].set_color('blue')

i = 0
plt.figure(figsize=(6,3))
plt.subplot(1,2,1)
plot_image(i, predictions[i], test_labels, test_images)
plt.subplot(1,2,2)
plot_value_array(i, predictions[i],  test_labels)
plt.show()

i = 12
plt.figure(figsize=(6,3))
plt.subplot(1,2,1)
plot_image(i, predictions[i], test_labels, test_images)
plt.subplot(1,2,2)
plot_value_array(i, predictions[i],  test_labels)
plt.show()

num_rows = 5
num_cols = 3
num_images = num_rows*num_cols
plt.figure(figsize=(2*2*num_cols, 2*num_rows))
for i in range(num_images):
  plt.subplot(num_rows, 2*num_cols, 2*i+1)
  plot_image(i, predictions[i], test_labels, test_images)
  plt.subplot(num_rows, 2*num_cols, 2*i+2)
  plot_value_array(i, predictions[i], test_labels)
plt.tight_layout()
plt.show()

　　

pytorch:

import torch
import torch.nn as nn
import torch.nn.functional as F


class Net(nn.Module):

    def __init__(self):
        super(Net, self).__init__()
        # 1 input image channel, 6 output channels, 5x5 square convolution
        # kernel
        self.conv1 = nn.Conv2d(1, 6, 5)
        self.conv2 = nn.Conv2d(6, 16, 5)
        # an affine operation: y = Wx + b
        self.fc1 = nn.Linear(16 * 5 * 5, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 10)

    def forward(self, x):
        # Max pooling over a (2, 2) window
        x = F.max_pool2d(F.relu(self.conv1(x)), (2, 2))
        # If the size is a square you can only specify a single number
        x = F.max_pool2d(F.relu(self.conv2(x)), 2)
        x = x.view(-1, self.num_flat_features(x))
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x

    def num_flat_features(self, x):
        size = x.size()[1:]  # all dimensions except the batch dimension
        num_features = 1
        for s in size:
            num_features *= s
        return num_features


net = Net()
print(net)

 

 

params = list(net.parameters())
print(len(params))
print(params[0].size())  # conv1's .weight

 

 

input = torch.randn(1, 1, 32, 32)
out = net(input)
print(out)

 

 

net.zero_grad()
out.backward(torch.randn(1, 10))

output = net(input)
target = torch.randn(10)  # a dummy target, for example
target = target.view(1, -1)  # make it the same shape as output
criterion = nn.MSELoss()

loss = criterion(output, target)
print(loss)

 

 

print(loss.grad_fn)  # MSELoss
print(loss.grad_fn.next_functions[0][0])  # Linear
print(loss.grad_fn.next_functions[0][0].next_functions[0][0])  # ReLU

 

 

net.zero_grad()     # zeroes the gradient buffers of all parameters

print('conv1.bias.grad before backward')
print(net.conv1.bias.grad)

loss.backward()

print('conv1.bias.grad after backward')
print(net.conv1.bias.grad)

 

 

learning_rate = 0.01
for f in net.parameters():
    f.data.sub_(f.grad.data * learning_rate)

import torch.optim as optim

# create your optimizer
optimizer = optim.SGD(net.parameters(), lr=0.01)

# in your training loop:
optimizer.zero_grad()   # zero the gradient buffers
output = net(input)
loss = criterion(output, target)
loss.backward()
optimizer.step()    # Does the update

CatAndDog:

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers, regularizers
import numpy as np
import os
import cv2
import matplotlib.pyplot as plt


os.environ["CUDA_VISIBLE_DEVICES"] = "1"

resize = 224
path ="train/"

def load_data():
    imgs = os.listdir(path)
    num = len(imgs)
    train_data = np.empty((5000, resize, resize, 3), dtype="int32")
    train_label = np.empty((5000, ), dtype="int32")
    test_data = np.empty((5000, resize, resize, 3), dtype="int32")
    test_label = np.empty((5000, ), dtype="int32")
    for i in range(5000):
        if i % 2:
            train_data[i] = cv2.resize(cv2.imread(path+'/'+ 'dog.' + str(i) + '.jpg'), (resize, resize))
            train_label[i] = 1
        else:
            train_data[i] = cv2.resize(cv2.imread(path+'/' + 'cat.' + str(i) + '.jpg'), (resize, resize))
            train_label[i] = 0
    for i in range(5000, 10000):
        if i % 2:
            test_data[i-5000] = cv2.resize(cv2.imread(path+'/' + 'dog.' + str(i) + '.jpg'), (resize, resize))
            test_label[i-5000] = 1
        else:
            test_data[i-5000] = cv2.resize(cv2.imread(path+'/' + 'cat.' + str(i) + '.jpg'), (resize, resize))
            test_label[i-5000] = 0
    return train_data, train_label, test_data, test_label

def vgg16():
    weight_decay = 0.0005
    nb_epoch = 100
    batch_size = 32
    
    # layer1
    model = keras.Sequential()
    model.add(layers.Conv2D(64, (3, 3), padding='same',
                     input_shape=(224, 224, 3), kernel_regularizer=regularizers.l2(weight_decay)))
    model.add(layers.Activation('relu'))
    model.add(layers.BatchNormalization())
    model.add(layers.Dropout(0.3))
    # layer2
    model.add(layers.Conv2D(64, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
    model.add(layers.Activation('relu'))
    model.add(layers.BatchNormalization())
    model.add(layers.MaxPooling2D(pool_size=(2, 2)))
    # layer3
    model.add(layers.Conv2D(128, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
    model.add(layers.Activation('relu'))
    model.add(layers.BatchNormalization())
    model.add(layers.Dropout(0.4))
    # layer4
    model.add(layers.Conv2D(128, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
    model.add(layers.Activation('relu'))
    model.add(layers.BatchNormalization())
    model.add(layers.MaxPooling2D(pool_size=(2, 2)))
    # layer5
    model.add(layers.Conv2D(256, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
    model.add(layers.Activation('relu'))
    model.add(layers.BatchNormalization())
    model.add(layers.Dropout(0.4))
    # layer6
    model.add(layers.Conv2D(256, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
    model.add(layers.Activation('relu'))
    model.add(layers.BatchNormalization())
    model.add(layers.Dropout(0.4))
    # layer7
    model.add(layers.Conv2D(256, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
    model.add(layers.Activation('relu'))
    model.add(layers.BatchNormalization())
    model.add(layers.MaxPooling2D(pool_size=(2, 2)))
    # layer8
    model.add(layers.Conv2D(512, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
    model.add(layers.Activation('relu'))
    model.add(layers.BatchNormalization())
    model.add(layers.Dropout(0.4))
    # layer9
    model.add(layers.Conv2D(512, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
    model.add(layers.Activation('relu'))
    model.add(layers.BatchNormalization())
    model.add(layers.Dropout(0.4))
    # layer10
    model.add(layers.Conv2D(512, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
    model.add(layers.Activation('relu'))
    model.add(layers.BatchNormalization())
    model.add(layers.MaxPooling2D(pool_size=(2, 2)))
    # layer11
    model.add(layers.Conv2D(512, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
    model.add(layers.Activation('relu'))
    model.add(layers.BatchNormalization())
    model.add(layers.Dropout(0.4))
    # layer12
    model.add(layers.Conv2D(512, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
    model.add(layers.Activation('relu'))
    model.add(layers.BatchNormalization())
    model.add(layers.Dropout(0.4))
    # layer13
    model.add(layers.Conv2D(512, (3, 3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
    model.add(layers.Activation('relu'))
    model.add(layers.BatchNormalization())
    model.add(layers.MaxPooling2D(pool_size=(2, 2)))
    model.add(layers.Dropout(0.5))
    # layer14
    model.add(layers.Flatten())
    model.add(layers.Dense(512, kernel_regularizer=regularizers.l2(weight_decay)))
    model.add(layers.Activation('relu'))
    model.add(layers.BatchNormalization())
    # layer15
    model.add(layers.Dense(512, kernel_regularizer=regularizers.l2(weight_decay)))
    model.add(layers.Activation('relu'))
    model.add(layers.BatchNormalization())
    # layer16
    model.add(layers.Dropout(0.5))
    model.add(layers.Dense(2))
    model.add(layers.Activation('softmax'))

    return model

#if __name__ == '__main__':
train_data, train_label, test_data, test_label = load_data()
train_data = train_data.astype('float32')
test_data = test_data.astype('float32')
train_label = keras.utils.to_categorical(train_label, 2)
test_label = keras.utils.to_categorical(test_label, 2)

#定义训练方法,超参数设置
model = vgg16()
sgd = tf.keras.optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) #设置优化器为SGD
model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])
history = model.fit(train_data, train_label,
              batch_size=20,
              epochs=10,
              validation_split=0.2,  #把训练集中的五分之一作为验证集
              shuffle=True)
scores = model.evaluate(test_data,test_label,verbose=1)
print(scores)
model.save('model/vgg16dogcat.h5')

 

 

acc = history.history['accuracy']  # 获取训练集准确性数据
val_acc = history.history['val_accuracy']  # 获取验证集准确性数据
loss = history.history['loss']  # 获取训练集错误值数据
val_loss = history.history['val_loss']  # 获取验证集错误值数据
epochs = range(1, len(acc) + 1)
plt.plot(epochs, acc, 'bo', label='Trainning acc')  # 以epochs为横坐标，以训练集准确性为纵坐标
plt.plot(epochs, val_acc, 'b', label='Vaildation acc')  # 以epochs为横坐标，以验证集准确性为纵坐标
plt.legend()  # 绘制图例，即标明图中的线段代表何种含义
 
plt.show()

 

 

 

 BP:

 

import math
import numpy as np
import csv
from sklearn import preprocessing
import xlsxwriter
import matplotlib.pyplot as plt

#首先一开始权重w和偏置b是不知道的，所以先要随机生成一个
def initial_bp_neural_network(n_inputs, n_hidden_layers, n_outputs):
    # hidden_layers 是一个列表，表示第n层隐含层神经元有几个
    global hidden_layers_weights
    global hidden_layers_bias
    hidden_layers_weights = []
    hidden_layers_bias = []
    # 一定要有隐含层, np.ones((行,列))；每个神经元对应着一个偏置
    if len(n_hidden_layers) > 0:
        hidden_layers_weights.append(np.random.randn(n_hidden_layers[0], n_inputs) * 0.01)
        #print("1:",hidden_layers_weights)
        for i in range(len(n_hidden_layers) - 1):
            hidden_layers_weights.append(np.random.rand(n_hidden_layers[i + 1], n_hidden_layers[i]) * 0.01)
            #print("i:",n_hidden_layers[i + 1])
            #print("2:",hidden_layers_weights)
        for i in n_hidden_layers:
            hidden_layers_bias.append(np.zeros(i))
            #print("3:",hidden_layers_bias)
        hidden_layers_weights.append(np.random.rand(n_outputs, n_hidden_layers[len(n_hidden_layers) - 1]) * 0.01)
        #print("4:",hidden_layers_weights)
        hidden_layers_bias.append(np.zeros(n_outputs))
        #print("5:",hidden_layers_bias)
    return hidden_layers_weights #最后合并在一起

# import tensorflow.compat.v1 as tf
# def leaky_relu(x, leak=0.2, name="leaky_relu"):
#      with tf.variable_scope(name):#tf.variable_scope()用来指定变量的作用域
#          f1 = 0.5 * (1 + leak)
#          f2 = 0.5 * (1 - leak)
#          return f1 * x + f2 * abs(x)
# def logistic(z,name="logistic"):
#     with tf.variable_scope(name):
#         return 1 / (1 + np.exp(-z))

#向前传播
def forward_propagate(inputs):
    global hidden_layers_weights
    global hidden_layers_bias
    global train_outputs
    global train_inputs
    train_inputs = []
    train_outputs = []
    function_vector = np.vectorize(Leaky_ReLu)
    #np.vecotrize(pyfunc)定义一个向量化的函数（pyfunc），使得数组可以作为输入, 并返回数组作为输出。
    for i in range(len(hidden_layers_weights)):
        outputs = np.array(hidden_layers_weights[i]).dot(inputs)
        outputs = np.array(outputs) + np.array(hidden_layers_bias[i])
        outputs = np.array(batch_normalization(outputs))
        train_inputs.append(outputs)
        outputs = function_vector(outputs)
        # 把每一层的输出都记录进去
        train_outputs.append(outputs)
        inputs = outputs.copy()
    return train_outputs

#反向传播
def backward_error_propagate(outputs):
    global error
    # error 是倒着来存放的
    error = []
    function_vector = np.vectorize(Leaky_ReLu_derivative)
    # train_outputs 包含了每一层的输出，最后一个才是最终的结果
    print("len(train_outputs):",len(train_outputs))
    print("len(train_inputs):",len(train_inputs))
    for i in range(len(train_outputs)):
        if i == 0:
            # 一个是对均方误差求导数，一个是对激活函数求导数
            # error.append(error_function_derivative(outputs, train_outputs[len(train_outputs) - 1]) *
            #              function_vector(np.array(train_inputs[len(train_inputs) - 1])))
            error.append(error_function_derivative(outputs, soft_max()) *
                         function_vector(np.array(train_inputs[len(train_inputs) - 1])))
        else:
            # 激活函数求导，直接点乘
            # print(function_vector(np.array(train_inputs[len(train_inputs) - 1 - i])))
            # 一个神经元对应的权重的误差求和
            error.append(function_vector(np.array(train_inputs[len(train_inputs) - 1 - i])) *
                         hidden_layers_weights[len(train_outputs) - i].T.dot(error[i - 1]))

for i in range(2):
    print(i)

 

 

#更新
def update(inputs):
    global hidden_layers_weights
    global hidden_layers_bias
    global error
    global train_inputs
    # 更新权重
    for i in range(len(hidden_layers_weights)):
        if i == 0:
            hidden_layers_weights[i] = hidden_layers_weights[i] - learning_rate * (
                    error[len(error) - i - 1].reshape(len(error[len(error) - i - 1]), 1) *
                    np.array(inputs).reshape(1, len(inputs)) + np.array(hidden_layers_weights[i]))
        else:
            hidden_layers_weights[i] = hidden_layers_weights[i] - learning_rate * (
                    error[len(error) - i - 1].reshape(len(error[len(error) - i - 1]), 1) *
                    np.array(train_outputs[i - 1]).reshape(1, len(train_outputs[i - 1])) +
                    np.array(hidden_layers_weights[i]))
            # print('error:')
            # print(learning_rate * (
            #         error[len(error) - i - 1].reshape(len(error[len(error) - i - 1]), 1) *
            #         np.array(train_inputs[i - 1]).reshape(1, len(train_inputs[i - 1])) + hidden_layers_weights[i]))
            # print('hidden_layer_weight:')
            # print(hidden_layers_weights[i])
    for i in range(len(hidden_layers_bias)):
        hidden_layers_bias[i] = hidden_layers_bias[i] - learning_rate * (error[len(error) - i - 1])

# 交叉熵函数作为损失函数
def error_function(actual_outputs, predict_outputs):
    function_vector = np.vectorize(math.log)
    return -sum(actual_outputs * function_vector(predict_outputs))


# 交叉熵函数的导数,针对输出层所有神经元
# 这个顺序一定不能颠倒
def error_function_derivative(actual_outputs, predict_outputs):
    return predict_outputs - actual_outputs


# 激活函数 Leaky_ReLu
def Leaky_ReLu(x):
    if x > 0:
        return x
    return 0.01 * x


# 激活函数Leaky_ReLu函数的导数
def Leaky_ReLu_derivative(x):
    if x > 0:
        return 1
    return 0.01


# 预测判断,判断是否达到收敛或者是和预期结果一样,精度可以自己调试
def judge_predict():
    for i in range(len(train_outputs[len(train_outputs) - 1])):
        if abs(train_outputs[len(train_outputs) - 1][i] - test_outputs[i]) > 0.00001:
            return 0
    return 1


# 分类判断,输出精确度
def judge_classification(label):
    max_value = max(soft_max())
    max_index = soft_max().index(max_value)
    if max_index + 1 == label:
        # return 1
        return label
    return 0

# 将输出的数值转换为概率
def soft_max():
    return [math.exp(train_outputs[len(train_outputs) - 1][0]) / (math.exp(train_outputs[len(train_outputs) - 1][0]) +
                                                                  math.exp(train_outputs[len(train_outputs) - 1][1]) +
                                                                  math.exp(train_outputs[len(train_outputs) - 1][2])),
            math.exp(train_outputs[len(train_outputs) - 1][1]) / (math.exp(train_outputs[len(train_outputs) - 1][0]) +
                                                                  math.exp(train_outputs[len(train_outputs) - 1][1]) +
                                                                  math.exp(train_outputs[len(train_outputs) - 1][2])),
            math.exp(train_outputs[len(train_outputs) - 1][2]) / (math.exp(train_outputs[len(train_outputs) - 1][0]) +
                                                                  math.exp(train_outputs[len(train_outputs) - 1][1]) +
                                                                  math.exp(train_outputs[len(train_outputs) - 1][2]))]

with open(r'C:\Users\Administrator\Desktop\Datasets-master\wheat-seeds.csv', 'r') as file:
    reader = csv.reader(file)
    print(reader)
    column = [row for row in reader]
    data = np.array(column).astype(float)
    print(data)

 

 

# 训练网络
def train_network(epoch):
    global data
    global test_inputs
    global test_outputs
    with open(r'C:\Users\Administrator\Desktop\Datasets-master\wheat-seeds.csv', 'r') as file:
        reader = csv.reader(file)
        column = [row for row in reader]
        # 乱序排列
        data = np.array(column).astype(float)

    # batch_size
    batch_size = 30
    iteration = 7

    #
    workbook = xlsxwriter.Workbook('data.xlsx')
    worksheet = workbook.add_worksheet()

    for _epoch in range(epoch):
        # 获取乱序, 让每次迭代都不一样
        if _epoch != 1:
            np.random.shuffle(data)

        # 获取输入
        test_inputs = [row[0:7] for row in data]
        test_inputs = min_max_scaler.fit_transform(test_inputs)

        # 获取输出
        test_outputs = [row[7] for row in data]
        # 训练一个batch就是一次iteration
        for i in range(iteration):
            n1 = 0
            n2 = 0
            n3 = 0
            # 一次迭代更新网络的参数
            # len(test_outputs)
            for j in range(batch_size):
                forward_propagate(test_inputs[j + i * batch_size])
                # 使用独热编码
                if test_outputs[j + i * batch_size] == 1:
                    backward_error_propagate(np.array([1, 0, 0]))
                elif test_outputs[j + i * batch_size] == 2:
                    backward_error_propagate(np.array([0, 1, 0]))
                else:
                    backward_error_propagate(np.array([0, 0, 1]))
                update(test_inputs[j + i * batch_size])
            # print("迭代完成")
            # 使用训练好的网络来得到测试的结果

        print('--------开始测试集的工作--------')
        # for j in range(len(test_inputs)):
        #     forward_propagate(test_inputs[j])
        #     print(soft_max())
        for j in range(len(test_outputs)):
            forward_propagate(test_inputs[j])
            # print(judge_classification(test_outputs[j]))
            if judge_classification(test_outputs[j]) == 1:
                # n = n + 1
                n1 = n1 + 1
            elif judge_classification(test_outputs[j]) == 2:
                n2 = n2 + 1
            elif judge_classification(test_outputs[j]) == 3:
                n3 = n3 + 1
        # acc = n / len(test_outputs)
        acc1 = n1 / 70
        acc2 = n2 / 70
        acc3 = n3 / 70
        worksheet.write(_epoch, 0, acc1)
        worksheet.write(_epoch, 1, acc2)
        worksheet.write(_epoch, 2, acc3)
        print('第' + str(_epoch) + "次迭代")
        print('准确率为:')
        print(acc1)
        print(acc2)
        print(acc3)
    workbook.close()

def batch_normalization(outputs):
    output = []
    # 求均值
    arr_mean = np.mean(outputs)
    # 求方差
    arr_var = np.var(outputs)
    for i in outputs:
        output.append((i-arr_mean)/(math.pow(arr_var, 0.5)))
    return output

if __name__ == '__main__':
    learning_rate = 0.0005
    hidden_layers_weights = []
    hidden_layers_bias = []
    train_outputs = []
    train_inputs = []
    # 误差项
    error = []
    test_inputs = []
    test_outputs = []
    min_max_scaler = preprocessing.MinMaxScaler()
    data = []

    # 这是一个三分类的问题，一共有三个标签
    initial_bp_neural_network(7, [10], 3)
    train_network(100)

 

 

 ...