n

dnn.model.py

@@ -0,0 +1,84 @@
+from keras.models import Sequential
+from keras.layers import LSTM, Dense, Dropout
+import numpy as np
+import re
+from keras.utils import to_categorical
+
+
+data_path = "/var/dataset/2020-6-29.dat"
+test_path = "/var/dataset/2020-6-27.dat"
+
+data_dim = 22
+timesteps = 60
+num_classes = 2
+
+start_from = 19
+end_from = 19
+
+
+def getData(path):
+
+    data = []
+    for line in open(path):
+        tmp = re.findall(r"\d+\.?\d*",line)
+        if int(tmp[20]) >= start_from:
+            if int(tmp[20]) <= end_from:
+                data.append(tmp)
+
+
+    x = []
+    y = []
+
+    for item in data:
+        x.append(item[:data_dim])
+        y.append(item[-4:])
+
+
+    x_data = []
+    y_data = []
+
+
+    for index in range(len(x) - timesteps - 2):
+        tmp_sec_block = timesteps
+        tmp_array = []
+        while tmp_sec_block > 0:
+            tmp_array += x[index + tmp_sec_block - 1]
+            tmp_sec_block = tmp_sec_block - 1
+        x_data.append(tmp_array)
+
+
+
+
+
+    for index in range(len(y) - timesteps - 2):
+        tmp = y[index + timesteps - 1]
+        tmp = list(map(int, tmp))
+        y_data.append(tmp[3])
+
+    return np.array(x_data), np.array(y_data)
+
+
+
+# Generate dummy training data
+x_train, y_train = getData(data_path)
+x_test, y_test = getData(test_path)
+
+model = Sequential()
+model.add(Dense(64, input_dim=data_dim*timesteps, activation='relu'))
+model.add(Dropout(0.5))
+model.add(Dense(64, activation='relu'))
+model.add(Dropout(0.5))
+model.add(Dense(64, activation='relu'))
+model.add(Dropout(0.5))
+model.add(Dense(1, activation='sigmoid'))
+
+model.compile(loss='binary_crossentropy',
+        optimizer='rmsprop',
+        metrics=['accuracy'])
+model.save('./dnn.h5')
+
+
+model.fit(x_train, y_train,
+    epochs=50,
+    batch_size=128, validation_data=(x_test, y_test))
+#score = model.evaluate(x_test, y_test, batch_size=128)

dnn.py

@@ -0,0 +1,86 @@
+from keras.models import Sequential
+from keras.layers import LSTM, Dense, Dropout
+import numpy as np
+import re
+from keras.utils import to_categorical
+from keras.models import load_model
+
+data_path = "/var/dataset/2020-6-29.dat"
+test_path = "/var/dataset/2020-6-27.dat"
+
+data_dim = 22
+timesteps = 60
+num_classes = 2
+
+start_from = 18
+end_from = 23
+
+
+def getData(path):
+
+    data = []
+    for line in open(path):
+        tmp = re.findall(r"\d+\.?\d*",line)
+        if int(tmp[20]) >= start_from:
+            if int(tmp[20]) <= end_from:
+                data.append(tmp)
+
+
+    x = []
+    y = []
+
+    for item in data:
+        x.append(item[:data_dim])
+        y.append(item[-4:])
+
+
+    x_data = []
+    y_data = []
+
+
+    for index in range(len(x) - timesteps - 2):
+        tmp_sec_block = timesteps
+        tmp_array = []
+        while tmp_sec_block > 0:
+            tmp_array += x[index + tmp_sec_block - 1]
+            tmp_sec_block = tmp_sec_block - 1
+        x_data.append(tmp_array)
+
+
+
+
+
+    for index in range(len(y) - timesteps - 2):
+        tmp = y[index + timesteps - 1]
+        tmp = list(map(int, tmp))
+        y_data.append(tmp[3])
+
+    return np.array(x_data), np.array(y_data)
+
+
+
+
+# Generate dummy training data
+x_t0, y_t0 = getData('/var/dataset/2020-6-27.dat')
+x_t1, y_t1 = getData('/var/dataset/2020-6-28.dat')
+x_t2, y_t2 = getData('/var/dataset/2020-6-29.dat')
+x_t3, y_t3 = getData('/var/dataset/2020-6-30.dat')
+x_test, y_test = getData('/var/dataset/2020-7-1.dat')
+
+x_t = [x_t0, x_t1, x_t2, x_t3]
+y_t = [y_t0, y_t1, y_t2, y_t3]
+
+model = load_model('./dnn.h5')
+
+
+
+for i in range(4):
+
+    model.fit(x_t[i], y_t[i],
+        epochs=200,
+        batch_size=128, validation_data=(x_test, y_test))
+
+    model.save('./dnn.h5')
+
+
+#score = model.evaluate(x_test, y_test, batch_size=128)

index.py

@@ -0,0 +1,110 @@
+import tensorflow as tf
+import numpy as np
+import json
+import os
+import re
+
+
+model_path = "./model_/600"
+data_path = "/var/dataset/2020-6-27.dat"
+
+
+sec_block = 600
+cell = 40
+batch_size = 2
+
+steps = 60*60*24*3
+show_loss_steps = 60*60 
+
+# 添加神经层封装函数
+def add_layer(inputs, in_size, out_size, activation_function=None):
+    Weights = tf.Variable(tf.random_normal([in_size, out_size]))
+    biases = tf.Variable(tf.zeros([1, out_size]) + 0.1)
+    Wx_plus_b = tf.matmul(inputs, Weights) + biases
+    if activation_function is None:
+        outputs = Wx_plus_b
+    else:
+        outputs = activation_function(Wx_plus_b)
+    return outputs
+
+
+
+data = []
+for line in open(data_path):
+    data.append(re.findall(r"\d+\.?\d*",line))
+
+
+x = []
+y = []
+
+for item in data:
+    x.append(item[:22])
+    y.append(item[-4:])
+
+
+x_data = []
+y_data = []
+
+for index in range(len(x) - sec_block - 2):
+    tmp_sec_block = sec_block
+    tmp_array = []
+    while tmp_sec_block > 0:
+        tmp_array += x[index + tmp_sec_block - 1]
+        tmp_sec_block = tmp_sec_block - 1
+    x_data.append(tmp_array)
+
+
+
+for index in range(len(y) - sec_block - 2):
+    y_data.append(y[index + sec_block - 1])
+
+
+
+
+ # 构建所需的数据
+
+
+    # 定义占位符输入变量
+xs = tf.placeholder(tf.float32, [None, 22*sec_block])
+ys = tf.placeholder(tf.float32, [None, 4])
+
+    ############### 搭建网络 ###############
+
+    # 输入层1个，隐藏层10个，激励函数relu
+l1 = add_layer(xs, 22*sec_block, cell, activation_function=tf.nn.relu)
+
+    # 输输入层10个，输出层1个，无激励函数
+prediction = add_layer(l1, cell, 4, activation_function=None)
+
+    # 计算预测值prediction和真实值的误差，对二者差的平方求和再取平均。
+loss = tf.reduce_mean(tf.reduce_sum(tf.square(ys - prediction),
+            reduction_indices=[1]))
+
+    # 接下来，是很关键的一步，如何让机器学习提升它的准确率。
+    # tf.train.GradientDescentOptimizer()中的值通常都小于1，
+    # 这里取的是0.1，代表以0.1的效率来最小化误差loss。
+train_step = tf.train.GradientDescentOptimizer(0.1).minimize(loss)
+
+# 初始化
+init = tf.global_variables_initializer()
+sess = tf.Session()
+saver = tf.train.Saver()
+sess.run(init)
+
+#saver.restore(sess, model_path)
+
+    # 比如这里，我们让机器学习1000次。机器学习的内容是train_step,
+    # 用 Session 来 run 每一次 training 的数据，逐步提升神经网络的预测准确性。
+    # (注意：当运算要用到placeholder时，就需要feed_dict这个字典来指定输入。)
+for i in range(steps):
+    # training
+    start = (i * batch_size) % len(x_data)
+    end = start + batch_size
+    sess.run(train_step, feed_dict={xs: x_data[start:end], ys: y_data[start:end]})
+    if i % show_loss_steps == 0:
+        # 每50步我们输出一下机器学习的误差。
+        print(sess.run(loss, feed_dict={xs: x_data, ys: y_data}))
+
+saver.save(sess, model_path)
+print('---------')
+#print(sess.run(prediction, feed_dict={xs: x_data[0]}))

liv.py

@@ -0,0 +1,144 @@
+#-*- coding:utf-8 -*-
+import tensorflow as tf
+from tensorflow.examples.tutorials.mnist import input_data
+import re
+import random
+
+#mnist = input_data.read_data_sets("MNIST_data/")
+
+model_path = "./model_liv/180"
+data_path = "/var/dataset/2020-6-27.dat"
+test_path = "/var/dataset/2020-6-29.dat"
+
+
+
+# 训练参数
+n_epoches = 50
+batch_size = 150
+Learning_rate = 0.001
+# 网络参数，把28x28的图片数据拆成28行的时序数据喂进RNN
+n_inputs = 22
+n_steps = 180 # 追溯秒数
+n_hiddens = 150
+n_outputs = 2  # 10分类
+
+start_from = 18
+end_from = 22
+
+
+
+# 输入tensors
+X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])
+y = tf.placeholder(tf.int32, [None])
+
+# 构建RNN结构
+basic_cell = tf.contrib.rnn.BasicLSTMCell(num_units=n_hiddens, state_is_tuple=True)
+#basic_cell = tf.contrib.rnn.BasicRNNCell(num_units=n_hiddens)
+#basic_cell = tf.nn.rnn_cell.BasicRNNCell(num_units=n_hiddens)  # 另一种创建基本单元的方式
+outputs, states = tf.nn.dynamic_rnn(basic_cell, X, dtype=tf.float32)
+
+# 前向传播，定义损失函数、优化器
+logits = tf.layers.dense(states[-1], n_outputs)  # 与states tensor连接的全连接层，LSTM时为states[-1]，即h张量
+cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)
+loss = tf.reduce_mean(cross_entropy)
+#loss = tf.reduce_mean(tf.reduce_sum(tf.square(y - prediction)
+
+optimizer = tf.train.AdamOptimizer(learning_rate=Learning_rate)
+prediction = tf.nn.in_top_k(logits, y, 1)
+train_op = optimizer.minimize(loss)
+
+accuracy = tf.reduce_mean(tf.cast(prediction, tf.float32))  # cast函数将tensor转换为指定类型
+
+# 从MNIST中读取数据
+#X_test = mnist.test.images.reshape([-1, n_steps, n_inputs])
+#y_test = mnist.test.labels
+
+
+
+
+def getData(path):
+
+    data = []
+    for line in open(path):
+        tmp = re.findall(r"\d+\.?\d*",line)
+        if int(tmp[20]) >= start_from:
+            if int(tmp[20]) <= end_from:
+                data.append(tmp)
+
+
+    x = []
+    y = []
+
+    for item in data:
+        x.append(item[:22])
+        y.append(item[-4:])
+
+
+    x_data = []
+    y_data = []
+
+
+    for index in range(len(x) - n_steps - 2):
+        tmp_sec_block = n_steps
+        tmp_array = []
+        while tmp_sec_block > 0:
+            tmp_array.append(x[index + tmp_sec_block - 1])
+            tmp_sec_block = tmp_sec_block - 1
+        x_data.append(tmp_array)
+
+
+
+    for index in range(len(y) - n_steps - 2):
+        tmp = y[index + n_steps - 1]
+        tmp = list(map(int, tmp))
+        y_data.append(tmp[3])
+
+    return x_data, y_data
+
+
+
+
+
+
+# 训练阶段
+init = tf.global_variables_initializer()
+saver = tf.train.Saver()
+loss_list = []
+accuracy_list = []
+
+
+with tf.Session() as sess:
+    sess.run(init)
+    
+    saver.restore(sess, model_path)
+    x_b, y_b = getData(data_path)
+    x_t, y_t = getData(test_path)
+    #print(len(x_b),'      ', len(y_b),'         ', len(x_t),'            ', len(y_t))
+    n_batches = len(x_b) // batch_size  # 整除返回整数部分
+    n_test = len(x_t) // batch_size
+    # print("Batch_number: {}".format(n_batches))
+    for epoch in range(n_epoches):
+        a_b = 0
+        a_t = 0
+        for iteration in range(min(n_batches, n_test)):
+            #X_batch, y_batch = mnist.train.next_batch(batch_size)
+            #X_batch = X_batch.reshape([-1, n_steps, n_inputs])
+            X_batch = x_b[iteration * batch_size : (iteration + 1) * batch_size] 
+            y_batch = y_b[iteration * batch_size : (iteration + 1) * batch_size] 
+            X_test = x_t[iteration * batch_size : (iteration + 1) * batch_size] 
+            y_test = y_t[iteration * batch_size : (iteration + 1) * batch_size] 
+            sess.run(train_op, feed_dict={X: X_batch, y: y_batch})
+            loss_train = loss.eval(feed_dict={X: X_batch, y: y_batch})
+            loss_list.append(loss_train)
+            acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})
+            acc_test = accuracy.eval(feed_dict={X: X_test, y: y_test})
+            accuracy_list.append(acc_test)
+            a_b = a_b + acc_train
+            a_t = a_t + acc_test
+            #print(epoch, '-', X_batch[0][0][20], '  ', "Train accuracy: {:.3f}".format(acc_train), "Test accuracy: {:.3f}".format(acc_test))
+
+        print(epoch, '-', X_batch[0][0][20], '  ', "Train accuracy: {:.3f}".format(a_b / min(n_batches, n_test)), "Test accuracy: {:.3f}".format(a_t / min(n_batches, n_test)))
+        
+        saver.save(sess, model_path)
+
+

model/checkpoint

@@ -0,0 +1,2 @@
+model_checkpoint_path: "180"
+all_model_checkpoint_paths: "180"

model_liv/checkpoint

@@ -0,0 +1,2 @@
+model_checkpoint_path: "180"
+all_model_checkpoint_paths: "180"

rnn.py

@@ -0,0 +1,145 @@
+#-*- coding:utf-8 -*-
+import tensorflow as tf
+from tensorflow.examples.tutorials.mnist import input_data
+import re
+import random
+
+#mnist = input_data.read_data_sets("MNIST_data/")
+
+model_path = "./model/180"
+data_path = "/var/dataset/2020-6-29.dat"
+test_path = "/var/dataset/2020-6-27.dat"
+
+
+
+# 训练参数
+n_epoches = 50
+batch_size = 150
+Learning_rate = 0.001
+# 网络参数，把28x28的图片数据拆成28行的时序数据喂进RNN
+n_inputs = 22
+n_steps = 180 # 追溯秒数
+n_hiddens = 150
+n_outputs = 16  # 10分类
+
+start_from = 17
+end_from = 23
+
+
+
+# 输入tensors
+X = tf.placeholder(tf.float32, [None, n_steps, n_inputs])
+y = tf.placeholder(tf.int32, [None])
+
+# 构建RNN结构
+basic_cell = tf.contrib.rnn.BasicLSTMCell(num_units=n_hiddens, state_is_tuple=True)
+#basic_cell = tf.contrib.rnn.BasicRNNCell(num_units=n_hiddens)
+#basic_cell = tf.nn.rnn_cell.BasicRNNCell(num_units=n_hiddens)  # 另一种创建基本单元的方式
+outputs, states = tf.nn.dynamic_rnn(basic_cell, X, dtype=tf.float32)
+
+# 前向传播，定义损失函数、优化器
+logits = tf.layers.dense(states[-1], n_outputs)  # 与states tensor连接的全连接层，LSTM时为states[-1]，即h张量
+cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=logits)
+loss = tf.reduce_mean(cross_entropy)
+#loss = tf.reduce_mean(tf.reduce_sum(tf.square(y - prediction)
+
+optimizer = tf.train.AdamOptimizer(learning_rate=Learning_rate)
+prediction = tf.nn.in_top_k(logits, y, 1)
+train_op = optimizer.minimize(loss)
+
+accuracy = tf.reduce_mean(tf.cast(prediction, tf.float32))  # cast函数将tensor转换为指定类型
+
+# 从MNIST中读取数据
+#X_test = mnist.test.images.reshape([-1, n_steps, n_inputs])
+#y_test = mnist.test.labels
+
+
+
+
+def getData(path):
+
+    data = []
+    for line in open(path):
+        tmp = re.findall(r"\d+\.?\d*",line)
+        if int(tmp[20]) >= start_from:
+            if int(tmp[20]) <= end_from:
+                data.append(tmp)
+
+
+    x = []
+    y = []
+
+    for item in data:
+        x.append(item[:22])
+        y.append(item[-4:])
+
+
+    x_data = []
+    y_data = []
+
+
+    for index in range(len(x) - n_steps - 2):
+        tmp_sec_block = n_steps
+        tmp_array = []
+        while tmp_sec_block > 0:
+            tmp_array.append(x[index + tmp_sec_block - 1])
+            tmp_sec_block = tmp_sec_block - 1
+        x_data.append(tmp_array)
+
+
+
+    for index in range(len(y) - n_steps - 2):
+        tmp = y[index + n_steps - 1]
+        tmp = list(map(int, tmp))
+        y_data.append(tmp[0] * 1 + tmp[1] * 2 + tmp[2] * 4 + tmp[3] * 8)
+
+
+    return x_data, y_data
+
+
+
+
+
+
+# 训练阶段
+init = tf.global_variables_initializer()
+saver = tf.train.Saver()
+loss_list = []
+accuracy_list = []
+
+
+with tf.Session() as sess:
+    sess.run(init)
+    
+    saver.restore(sess, model_path)
+    x_b, y_b = getData(data_path)
+    x_t, y_t = getData(test_path)
+    #print(len(x_b),'      ', len(y_b),'         ', len(x_t),'            ', len(y_t))
+    n_batches = len(x_b) // batch_size  # 整除返回整数部分
+    n_test = len(x_t) // batch_size
+    # print("Batch_number: {}".format(n_batches))
+    for epoch in range(n_epoches):
+        a_b = 0
+        a_t = 0
+        for iteration in range(min(n_batches, n_test)):
+            #X_batch, y_batch = mnist.train.next_batch(batch_size)
+            #X_batch = X_batch.reshape([-1, n_steps, n_inputs])
+            X_batch = x_b[iteration * batch_size : (iteration + 1) * batch_size] 
+            y_batch = y_b[iteration * batch_size : (iteration + 1) * batch_size] 
+            X_test = x_t[iteration * batch_size : (iteration + 1) * batch_size] 
+            y_test = y_t[iteration * batch_size : (iteration + 1) * batch_size] 
+            sess.run(train_op, feed_dict={X: X_batch, y: y_batch})
+            loss_train = loss.eval(feed_dict={X: X_batch, y: y_batch})
+            loss_list.append(loss_train)
+            acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch})
+            acc_test = accuracy.eval(feed_dict={X: X_test, y: y_test})
+            accuracy_list.append(acc_test)
+            a_b = a_b + acc_train
+            a_t = a_t + acc_test
+            #print(epoch, '-', X_batch[0][0][20], '  ', "Train accuracy: {:.3f}".format(acc_train), "Test accuracy: {:.3f}".format(acc_test))
+
+        print(epoch, '-', X_batch[0][0][20], '  ', "Train accuracy: {:.3f}".format(a_b / min(n_batches, n_test)), "Test accuracy: {:.3f}".format(a_t / min(n_batches, n_test)))
+        
+        saver.save(sess, model_path)
+
+

rnn_keras.py

@@ -0,0 +1,99 @@
+from keras.models import Sequential
+from keras.layers import LSTM, Dense
+import numpy as np
+import re
+from keras.utils import to_categorical
+
+
+data_path = "/var/dataset/2020-6-29.dat"
+test_path = "/var/dataset/2020-6-27.dat"
+
+data_dim = 20
+timesteps = 8
+num_classes = 2
+
+start_from = 19
+end_from = 19
+
+
+def getData(path):
+
+    data = []
+    for line in open(path):
+        tmp = re.findall(r"\d+\.?\d*",line)
+        if int(tmp[20]) >= start_from:
+            if int(tmp[20]) <= end_from:
+                data.append(tmp)
+
+
+    x = []
+    y = []
+
+    for item in data:
+        x.append(item[:20])
+        y.append(item[-4:])
+
+
+    x_data = []
+    y_data = []
+
+
+    for index in range(len(x) - timesteps - 2):
+        tmp_sec_block = timesteps
+        tmp_array = []
+        while tmp_sec_block > 0:
+            tmp_array.append(x[index + tmp_sec_block - 1])
+            tmp_sec_block = tmp_sec_block - 1
+        x_data.append(tmp_array)
+
+
+
+    for index in range(len(y) - timesteps - 2):
+        tmp = y[index + timesteps - 1]
+        tmp = list(map(int, tmp))
+        y_data.append(tmp[3])
+
+    return np.array(x_data), np.array(y_data)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+# expected input data shape: (batch_size, timesteps, data_dim)
+model = Sequential()
+model.add(LSTM(32, return_sequences=True,
+input_shape=(timesteps, data_dim)))  # returns a sequence of vectors of dimension 32
+model.add(LSTM(32, return_sequences=True))  # returns a sequence of vectors of dimension 32
+model.add(LSTM(32))  # return a single vector of dimension 32
+model.add(Dense(1, activation='softmax'))
+
+model.compile(loss='hinge',#'categorical_crossentropy',
+        optimizer='rmsprop',
+        metrics=['accuracy'])
+
+# Generate dummy training data
+x_train, y_train = getData(data_path)
+
+print(x_train[0])
+print(y_train[0])
+
+# Generate dummy validation data
+#x_val = np.random.random((100, timesteps, data_dim))
+#y_val = np.random.random((100, num_classes))
+x_val, y_val = getData(test_path)
+
+model.fit(x_train, y_train,
+        batch_size=64, epochs=5,
+        validation_data=(x_val, y_val))
+
+
+