# 数据分析的核心库
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
# 时间序列处理
from datetime import datetime
from dateutil.parser import parse as dt_parse
# 正则库
import re
# os库
from os import listdir
# NLTK自然语言处理库
import nltk
from nltk.corpus import stopwords
# seaborn成对图矩阵生成
from seaborn import pairplot
# sklearn库的归一化、训练集测试集划分
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split
# Keras LSTM
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM, Dense, Dropout
# sklearn MSE
from sklearn.metrics import mean_squared_error

# 设置pandas的最大显示行数、列数和输出宽度
pd.set_option('display.max_rows', 6)
pd.set_option('display.max_columns', 999)
pd.set_option('display.max_colwidth', 50)

sharePrices = pd.read_csv('股票价格.csv')
sharePrices

sharePricesAAPL = sharePrices[sharePrices['TICKER']==CORPORATIONABBR]

sharePricesAAPL.drop(['PERMNO', 'COMNAM', 'TICKER'], axis=1, inplace=True)

sharePricesAAPL.info()

sharePricesAAPL['date'].is_unique

# date列转化为datetime类
sharePricesAAPL['date'] = sharePricesAAPL['date'].apply(lambda dt: datetime.strptime(str(dt), '%Y%m%d'))
# 设date列为索引
sharePricesAAPL.set_index('date', inplace=True)
# 按date升序排列
sharePricesAAPL.sort_values(by='date', inplace=True, ascending=True)

sharePricesAAPL.isnull().mean()

# rename
AAPL_newIndex = {'BIDLO': 'low',
                 'ASKHI': 'high',
                 'PRC': 'close',
                 'VOL': 'vol',
                 'OPENPRC': 'open'}
sharePricesAAPL.rename(columns=AAPL_newIndex, inplace=True)
# reindex
AAPL_newColOrder = ['open', 'high', 'low', 'vol', 'close']
sharePricesAAPL = sharePricesAAPL.reindex(columns=AAPL_newColOrder)

sharePricesAAPL.describe()

sharePricesAAPL.to_csv('补充数据1925102007/AAPL股票价格.csv')

forum = pd.read_csv('论坛.csv')
forum.dropna(inplace=True)

forum_regExp = re.compile(r'seekingalpha\.com/symbol/([A-Z]+)')
def forumAbbr(link):
    # 成功查找公司简称则返回简称，否则以缺失值填补
    res = forum_regExp.search(link)
    return np.NAN if res is None else res.group(1)
forum['字段4_链接'] = forum['字段4_链接'].apply(forumAbbr)

# 索引过滤
forum = forum[forum['字段4_链接']==CORPORATIONABBR]
# 降维处理
forum.drop(['字段1', '字段4_链接'], axis=1, inplace=True)
# 索引重构
AAPL_newIndex_forum = {'字段2': 'date', '字段3': 'remark'}
forum.rename(columns=AAPL_newIndex_forum, inplace=True)
# 时间序列
forum['date'] = forum['date'].apply(lambda dt: datetime.strptime(str(dt), '%d-%b-%y'))

forum_regExp_linkFilter = re.compile(r'(http|https):\/\/[\w\-_]+(\.[\w\-_]+)+([\w\-\.,@?^=%&:/~\+#]*[\w\-\@?^=%&/~\+#])?')
forum['remark'] = forum['remark'].apply(lambda x: forum_regExp_linkFilter.sub('', x))
forum

# 数据储存
forum.to_csv('补充数据1925102007/AAPL论坛.csv', index=False)

allTitles = pd.read_excel('全标题.xlsx')
# 索引过滤
allTitles = allTitles[allTitles['ABOUT']==CORPORATIONABBR]
# 降维
allTitles.drop(['字段1_链接_链接',
                'ABOUT',
                'AUTHOR',
                'COMMENTS',
                'Unnamed: 6'], axis=1, inplace=True)
# 索引重构
AAPL_newIndex_allTitles = {'标题': 'title', 'TIME': 'date'}
allTitles.rename(columns=AAPL_newIndex_allTitles, inplace=True)
# 时间序列处理
# 因时间日期格式非统一，故选用dateutil包对parser.parse方法识别多变时间格式
allTitles['date'] = allTitles['date'].apply(lambda dt: dt_parse(dt))
# 设date列为索引
allTitles.set_index('date', inplace=True)
# 按date升序排列
allTitles.sort_values(by='date', inplace=True, ascending=True)
# 数据储存
allTitles.to_csv('补充数据1925102007/AAPL全标题.csv')
allTitles

abstracts = pd.read_excel('摘要.xlsx')
abstracts.drop(['字段2', '字段5'], axis=1, inplace=True)
newIndex_abstracts = {'标题': 'title', '字段1': 'abstract'}
abstracts.rename(columns=newIndex_abstracts, inplace=True)

abstracts = abstracts.merge(allTitles, on=['title'], how='inner')

abstracts.to_csv('补充数据1925102007/AAPL摘要.csv', index=False)
abstracts

pd.read_excel('回帖/SA_Comment_Page131-153.xlsx')

pd.read_csv('回帖/SA_Comment_Page181-255(1).csv')

# 数据文件读取
repliesFiles = listdir('回帖')
allAALPReplies = []
newIndex_replies_csv = {'字段1': 'content', '标题': 'title'}
newIndex_replies_xlsx = {'字段': 'content', '标题1': 'title'}
# 遍历回帖目录下所有回帖数据找出和AAPL相关的回帖
for file in repliesFiles:
    path = '回帖/'+file
    if file.endswith('.csv'):
        replies = pd.read_csv(path)
        newIndex_replies = newIndex_replies_csv
    elif file.endswith('.xlsx'):
        replies = pd.read_excel(path)
        newIndex_replies = newIndex_replies_xlsx
    else:
        print('Wrong file format,', file)
        break
    # 索引重命名
    replies.rename(columns=newIndex_replies, inplace=True)
    # 缺失值填充
    replies.fillna(method='bfill', inplace=True)
    # 回帖过滤
    allAALPReplies.extend(replies.merge(allTitles, on=['title'], how='inner').values)
# 所有和AAPL文章标题所对应的回帖
allAALPReplies = pd.DataFrame(allAALPReplies, columns=['content', 'title'])
# 保存
allAALPReplies.to_csv('补充数据1925102007/AAPL回帖.csv', index=False)
# 展示
allAALPReplies

# 词库导入
wordListsPath = '补充数据1925102007/LoughranMcDonald_SentimentWordLists_2018.xlsx'
posWords = pd.read_excel(wordListsPath, header=None, sheet_name='Positive').iloc[:,0].values
negWords = pd.read_excel(wordListsPath, header=None, sheet_name='Negative').iloc[:,0].values

# 添加停用词
extraStopwords = ['!', ',' ,'.' ,'?' ,'-s' ,'-ly' ,'</s> ', 's', 'AAPL', 'apple', '$', '%']
stopWs = stopwords.words('english') + extraStopwords

def structComment(sentence, posW, negW, stopW):
    """
    结构化句子
    :param sentence: 待结构化的评论
    :param posW: 正词性
    :param negW: 负词性
    :param stopW: 停用词
    :return: 去除停用词后的评论中posWords和negWords的占比(pos, neg)
    """
    # 分词
    tokenizer = nltk.word_tokenize(sentence)
    # 停用词过滤
    tokenizer = [w.upper() for w in tokenizer if w.lower() not in stopW]
    # 正词提取
    posWs = [w for w in tokenizer if w in posW]
    # 负词提取
    negWs = [w for w in tokenizer if w in negW]
    # tokenizer长度
    len_token = len(tokenizer)
    # 句子长度为0，即分母为0时
    if len_token<=0:
        return 0, 0
    else:
        return len(posWs)/len_token, len(negWs)/len_token

def NLProcessing(fileName, colName):
    """
    自然语言处理方法：将传入的fileName(.csv)对应的数据中的colName列文本数据结构化，并保存
    :param fileName: 文件名，在文件夹 补充数据1925102007/ 下查找对应文件
    :param colName: 需要结构化的文本数据列
    :return: 新增pos和neg列的DataFrame
    """
    pathNLP = '补充数据1925102007/'+fileName+'.csv'
    data = pd.read_csv(pathNLP)
    # pos和neg结构化数据列构造
    posAndneg = [ structComment(st, posWords, negWords, stopWs) for st in data[colName].values]
    # 构造posAndneg的DataFrame
    posAndneg = pd.DataFrame(posAndneg, columns=['pos', 'neg'])
    # 轴向连接
    data = pd.concat([data, posAndneg], axis=1)
    # 删除文本数据列
    data.drop([colName], axis=1, inplace=True)
    # 保存结构化的数据
    data.to_csv(pathNLP)
    return data

# AAPL论坛.csv
forum = NLProcessing('AAPL论坛', 'remark')
# AAPL摘要.csv
abstracts = NLProcessing('AAPL摘要', 'abstract')
# AAPL回帖.csv
allAALPReplies = NLProcessing('AAPL回帖', 'content')

# 轴相连接abstracts和allAALPReplies
allEssaysComment = pd.concat([abstracts,allAALPReplies], ignore_index=True)
# 联表
allEssaysComment = allTitles.merge(allEssaysComment, how='outer', on='title')
# 删除缺失行
allEssaysComment.dropna(inplace=True)
# 删除title列
allEssaysComment.drop('title', axis=1, inplace=True)
# 和forum情感数据进行轴向连接
allEssaysComment = pd.concat([allEssaysComment,forum], ignore_index=True)
# 删除pos和neg均为0的无用数据行
allEssaysComment = allEssaysComment[(allEssaysComment['pos']+allEssaysComment['neg'])>0]

# 设date为时间序列索引
allEssaysComment['date'] = pd.to_datetime(allEssaysComment['date'])
allEssaysComment.set_index('date', inplace=True)
# 按"工作日"重采样，求pos和neg的均值，不存在的天以0填充
allEssaysComment = allEssaysComment.resample('B').mean()
allEssaysComment.fillna(0, inplace=True)
# 储存
allEssaysComment.to_csv('补充数据1925102007/allPosAndNeg.csv')
# 展示
allEssaysComment

# 文件读取
sharePricesAAPL = pd.read_csv('补充数据1925102007/AAPL股票价格.csv')
allPosAndNeg = pd.read_csv('补充数据1925102007/allPosAndNeg.csv')
# 合并
sharePricesAAPLwithEmotion = sharePricesAAPL.merge(allPosAndNeg, how='inner', on='date')
# 序列化时间索引date
sharePricesAAPLwithEmotion['date'] = pd.DatetimeIndex(sharePricesAAPLwithEmotion['date'])
sharePricesAAPLwithEmotion.set_index('date', inplace=True)
# reindex
AAPL_newColOrder_emotionPrices = ['open', 
                                  'high', 
                                  'low', 
                                  'vol', 
                                  'pos', 
                                  'neg', 
                                  'close']
sharePricesAAPLwithEmotion = sharePricesAAPLwithEmotion.reindex(columns=AAPL_newColOrder_emotionPrices)
# 保存
sharePricesAAPLwithEmotion.to_csv('补充数据1925102007/AAPL股票价格融合情感.csv')

# Parameters:
# data: pandas.DataFrame [Tidy (long-form) dataframe where each column is a variable and each row is an observation.]
# diag_kind: {‘auto’, ‘hist’, ‘kde’, None} [Kind of plot for the diagonal subplots.]
# kind: {‘scatter’, ‘kde’, ‘hist’, ‘reg’} [Kind of plot to make.]
fig1 = pairplot(sharePricesAAPLwithEmotion[['vol', 'pos', 'neg']], diag_kind='hist', kind='reg')
# save the fig1 to 补充数据1925102007/
fig1.savefig('补充数据1925102007/fig1_a_Grid_of_Axes.png')

def series_to_supervised(data, n_in=1):
    # 默认参数
    n_out=1
    dropnan=True
    # 对该函数进行微调，注意data为以close列（需要预测的列）结尾的DataFrame时间序列股票数据
    n_vars = 1 if type(data) is list else data.shape[1]
    df = pd.DataFrame(data)
    cols, names = list(), list()
    # input sequence (t-n, ... t-1)
    for i in range(n_in, 0, -1):
        cols.append(df.shift(i))
        names += [('var%d(t-%d)' % (j+1, i)) for j in range(n_vars)]
    # forecast sequence (t, t+1, ... t+n)
    for i in range(0, n_out):
        cols.append(df.shift(-i))
        if i == 0:
            names += [('var%d(t)' % (j+1)) for j in range(n_vars)]
        else:
            names += [('var%d(t+%d)' % (j+1, i)) for j in range(n_vars)]
    # put it all together
    agg = pd.concat(cols, axis=1)
    agg.columns = names
    # 删除无关的次态[t]列，只留下需要预测的close[t]列和上一时刻状态特征[t-1]列
    agg.drop(agg.columns[[x for x in range(data.shape[1], 2*data.shape[1]-1)]], axis=1, inplace=True)
    # drop rows with NaN values
    if dropnan:
        agg.dropna(inplace=True)
    return agg

# 读取数据
sharePricesAAPLwithEmotion = pd.read_csv('补充数据1925102007/AAPL股票价格融合情感.csv', parse_dates=['date'], index_col='date').values
# 生成归一化容器
# feature_range参数沿用默认(0,1)
scaler = MinMaxScaler()
# 训练模型
scaler = scaler.fit(sharePricesAAPLwithEmotion)
# 归一化
sharePricesAAPLwithEmotion = scaler.fit_transform(sharePricesAAPLwithEmotion)
# 部分结果展示
sharePricesAAPLwithEmotion[:5,:]

# 使用series_to_supervised函数构建有监督数据集
sharePricesAAPLwithEmotion = series_to_supervised(sharePricesAAPLwithEmotion)
sharePricesAAPLwithEmotion

# 必须规定ndarray的dtype为float32（默认float64），否则后续输入LSTM模型报错
sharePricesAAPLwithEmotion = sharePricesAAPLwithEmotion.values.astype(np.float32)
# 训练集:验证集=7:3
X_train, X_test, y_train, y_test = train_test_split(sharePricesAAPLwithEmotion[:,:-1], sharePricesAAPLwithEmotion[:,-1], test_size=0.3, shuffle=False)

# reshape input to be 3D [samples, timesteps, features]
X_train = X_train.reshape((X_train.shape[0], 1, X_train.shape[1]))
X_test = X_test.reshape((X_test.shape[0], 1, X_test.shape[1]))

def LSTMModelGenerate(Xtrain, Xtest, ytrain, ytest):
    """
    LSTM模型搭建函数
    :param Xtrain: 训练集属性
    :param Xtest: 测试集属性
    :param ytrain: 训练集标签
    :param ytest: 测试集标签
    :return: history,model（模型编译记录和模型）
    """
    # 搭建LSTM模型
    _model = Sequential()
    _model.add(LSTM(64, input_shape=(Xtrain.shape[1], Xtrain.shape[2])))
    _model.add(Dropout(0.4))
    _model.add(Dense(1, activation='relu'))
    # 模型编译
    _model.compile(loss='mse', optimizer='adam')
    # 模型训练
    _history = _model.fit(Xtrain, ytrain, epochs=50, batch_size=30, validation_data=(Xtest, ytest), shuffle=False, verbose=0)
    return _history,_model
  
history, model = LSTMModelGenerate(X_train, X_test, y_train, y_test)

def drawLossGraph(_history, title, num):
    """
    损失图绘制，寻找最优epochs
    :param _history: 训练历史
    :param title: 图表标题
    :param num: 图表编号
    :return: 无
    """
    plt.plot(_history.history['loss'], color='g', label='train')
    plt.plot(_history.history['val_loss'], color='r', label='test')
    plt.title('Fig'+num+'. '+title)
    plt.xlabel('epochs')
    plt.ylabel('loss')
    plt.legend()
    # 保存于 补充数据1925102007/
    savingPath = '补充数据1925102007/fig'+num+'_'+title.replace(' ', '_')+'.png'
    plt.savefig(savingPath, dpi=400, bbox_inches='tight')
    # 展示
    plt.show()
    
drawLossGraph(history, title='LSTM Loss Graph for Stock Prices with Emotions', num='2')

# 因为只要对结果列进行反归一化操作，
# 故不用inverse_transform函数，
# 这里自定义对某列的反归一化函数 inverse_transform_col
def inverse_transform_col(_scaler, y, n_col):
    """
    对某个列进行反归一化处理的函数
    :param _scaler: sklearn归一化模型
    :param y: 需要反归一化的数据列
    :param n_col: y在归一化时所属的列编号
    :return: y的反归一化结果
    """
    y = y.copy()
    y -= _scaler.min_[n_col]
    y /= _scaler.scale_[n_col]
    return y

# 模型预测结果绘图函数
def predictGraph(yTrain, yPredict, yTest, timelabels, title, num):
    """
    预测结果图像绘制函数
    :param yTrain: 训练集结果
    :param yPredict: 验证集的预测结果
    :param yTest: 验证集的真实结果
    :param timelabels: x轴刻度标签
    :param title: 图表标题
    :param num: 图标编号
    :return: 无
    """
    len_yTrain = yTrain.shape[0]
    len_y = len_yTrain+yPredict.shape[0]
    # 真实曲线绘制
    plt.plot(np.concatenate([yTrain,yTest]), color='r', label='sample')
    # 预测曲线绘制
    plt.plot([x for x in range(len_yTrain,len_y)],yPredict, color='g', label='predict')
    # 标题和轴标签
    plt.title('Fig'+num+'. '+title)
    plt.xlabel('date')
    plt.ylabel('close')
    plt.legend()
    # 刻度和刻度标签
    xticks = [0,len_yTrain,len_y-1]
    xtick_labels = [timelabels[x] for x in xticks]
    plt.xticks(ticks=xticks, labels=xtick_labels, rotation=30)
    # 保存于 补充数据1925102007/
    savingPath = '补充数据1925102007/fig'+num+'_'+title.replace(' ', '_')+'.png'
    plt.savefig(savingPath, dpi=400, bbox_inches='tight')
    # 展示
    plt.show()

# 由X_test前日股票指标预测当天股票close值
# 注：predict生成的array需降维成 shape=(n_samples, )
y_predict = model.predict(X_test)[:,0]

# 反归一化
# 重新读取 AAPL股票价格融合情感.csv
sharePricesAAPLwithEmotion = pd.read_csv('补充数据1925102007/AAPL股票价格融合情感.csv')
col_n = sharePricesAAPLwithEmotion.shape[1]-2
# 预测结果反归一化
inv_yPredict = inverse_transform_col(scaler, y_predict, col_n)
# 真实结果反归一化
inv_yTest = inverse_transform_col(scaler, y_test, col_n)
# 训练集结果反归一化（以绘制完整图像）
inv_yTrain = inverse_transform_col(scaler, y_train, col_n)
# 绘图
predictGraph(inv_yTrain, inv_yPredict, inv_yTest, timelabels=sharePricesAAPLwithEmotion['date'].values, title='Prediction Graph of Stock Prices with Emotions', num='3')

# sklearn.metrics.mean_squared_error(y_true, y_pred)
mse = mean_squared_error(inv_yTest, inv_yPredict)
print('带有情感特征的股票数据预测结果的均方误差（MSE）为 ', mse)

def formatData(sharePricesData):
    """
    模式化样本数据的函数
    :param sharePricesData: 样本数据的DataFrame
    :return: X_train, X_test, y_train, y_test, scaler
    """
    # 归一化
    _scaler = MinMaxScaler()
    _scaler = _scaler.fit(sharePricesData)
    sharePricesData = _scaler.fit_transform(sharePricesData)
    # 构建有监督数据集
    sharePricesData = series_to_supervised(sharePricesData)
    # dtype为float32
    sharePricesData = sharePricesData.values.astype(np.float32)
    # 训练集和验证集的划分
    _X_train, _X_test, _y_train, _y_test = train_test_split(sharePricesData[:,:-1], sharePricesData[:,-1], test_size=0.3, shuffle=False)
    # reshape input
    _X_train = _X_train.reshape((_X_train.shape[0], 1, _X_train.shape[1]))
    _X_test = _X_test.reshape((_X_test.shape[0], 1, _X_test.shape[1]))
    return _X_train, _X_test, _y_train, _y_test, _scaler

def invTransformMulti(_scaler, _y_predict, _y_test, _y_train, _col_n):
    # 批量反归一化
    _inv_yPredict = inverse_transform_col(_scaler, _y_predict, _col_n)
    _inv_yTest = inverse_transform_col(_scaler, _y_test, _col_n)
    _inv_yTrain = inverse_transform_col(_scaler, _y_train, _col_n)
    return _inv_yPredict, _inv_yTest, _inv_yTrain

# 读取数据
sharePricesAAPL = pd.read_csv('补充数据1925102007/AAPL股票价格.csv', parse_dates=['date'], index_col='date').values
# 标准化数据输入
X_train, X_test, y_train, y_test, scaler = formatData(sharePricesAAPL)
# 建模
history, model = LSTMModelGenerate(X_train, X_test, y_train, y_test)

# 损失函数绘图
drawLossGraph(history, title='LSTM Loss Graph for Stock Prices without Emotions', num='4')

# 预测
y_predict = model.predict(X_test)[:,0]
# 反归一化
sharePricesAAPL = pd.read_csv('补充数据1925102007/AAPL股票价格.csv')
col_n = sharePricesAAPL.shape[1]-2
inv_yPredict, inv_yTest, inv_yTrain = invTransformMulti(scaler, y_predict, y_test, y_train, col_n)
# 绘图
predictGraph(inv_yTrain, inv_yPredict, inv_yTest, timelabels=sharePricesAAPL['date'].values, title='Prediction Graph of Stock Prices without Emotions', num='5')

# 均方误差
mse = mean_squared_error(inv_yTest, inv_yPredict)
print('无情感特征的纯技术指标股票数据预测结果的均方误差（MSE）为 ', mse)

# 读取数据
allYearAAPL = pd.read_csv('补充数据1925102007/AAPLHistoricalData_5years.csv', parse_dates=['Date'], index_col='Date')
# 时间序列索引切片
allYearAAPL = allYearAAPL['2018-12-31':'2018-01-01']
# 排序
allYearAAPL.sort_index(inplace=True)
# 展示
allYearAAPL

# pandas字符串切割、Series类型修改（去除$）
allYearAAPL[['Close/Last', 'Open', 'High', 'Low']] = allYearAAPL[['Close/Last', 'Open', 'High', 'Low']].apply(lambda x: (x.str[1:]).astype(np.float32))
# reindex
allAAPL_newColOrder = ['Open', 'High', 'Low', 'Volume', 'Close/Last']
allYearAAPL = allYearAAPL.reindex(columns=allAAPL_newColOrder)
# 保存为AAPL2018allYearData.csv
allYearAAPL.to_csv('补充数据1925102007/AAPL2018allYearData.csv')
# 展示
allYearAAPL

# 标准化数据输入
X_train, X_test, y_train, y_test, scaler = formatData(allYearAAPL)
# 建模
history, model = LSTMModelGenerate(X_train, X_test, y_train, y_test)
# 损失函数绘图
drawLossGraph(history, title='LSTM Loss Graph for 2018 All Year AAPL Stock Prices', num='6')

# 预测
y_predict = model.predict(X_test)[:,0]
# 反归一化
allYearAAPL = pd.read_csv('补充数据1925102007/AAPL2018allYearData.csv')
col_n = allYearAAPL.shape[1]-2
inv_yPredict, inv_yTest, inv_yTrain = invTransformMulti(scaler, y_predict, y_test, y_train, col_n)
# 绘图
predictGraph(inv_yTrain, inv_yPredict, inv_yTest, timelabels=allYearAAPL['Date'].values, title='Prediction Graph of 2018 All Year AAPL Stock Prices', num='7')

# 均方误差
mse = mean_squared_error(inv_yTest, inv_yPredict)
print('2018全年纯技术指标AAPL股票数据预测结果的均方误差（MSE）为 ', mse)

# 文件读取
allYearAAPL_withEmos = pd.read_csv('补充数据1925102007/AAPL2018allYearData.csv')
allPosAndNeg = pd.read_csv('补充数据1925102007/allPosAndNeg.csv')
# 合并
allYearAAPL_withEmos = allYearAAPL_withEmos.merge(allPosAndNeg, how='inner', left_on='Date', right_on='date').drop('date', axis=1)
# 序列化时间索引date
allYearAAPL_withEmos['Date'] = pd.DatetimeIndex(allYearAAPL_withEmos['Date'])
allYearAAPL_withEmos.set_index('Date', inplace=True)
# reindex
allYearAAPLwithEmos_newColOrder = ['Open',
                                   'High',
                                   'Low',
                                   'Volume',
                                   'pos',
                                   'neg',
                                   'Close/Last']
allYearAAPL_withEmos = allYearAAPL_withEmos.reindex(columns=allYearAAPLwithEmos_newColOrder)
# 保存
allYearAAPL_withEmos.to_csv('补充数据1925102007/AAPL2018allYearData_withEmos.csv')
# 展示
allYearAAPL_withEmos

# 标准化数据输入
X_train, X_test, y_train, y_test, scaler = formatData(allYearAAPL_withEmos)
# 建模
history, model = LSTMModelGenerate(X_train, X_test, y_train, y_test)
# 损失函数绘图
drawLossGraph(history, title='LSTM Loss Graph for 2018 All Year AAPL Stock Prices with Emotions', num='8')

# 预测
y_predict = model.predict(X_test)[:,0]
# 反归一化
allYearAAPL_withEmos = pd.read_csv('补充数据1925102007/AAPL2018allYearData_withEmos.csv')
col_n = allYearAAPL_withEmos.shape[1]-2
inv_yPredict, inv_yTest, inv_yTrain = invTransformMulti(scaler, y_predict, y_test, y_train, col_n)
# 绘图
predictGraph(inv_yTrain, inv_yPredict, inv_yTest, timelabels=allYearAAPL_withEmos['Date'].values, title='Prediction Graph of 2018 All Year AAPL Stock Prices with Emotions', num='9')

# 均方误差
mse = mean_squared_error(inv_yTest, inv_yPredict)
print('2018全年含情感特征的AAPL股票数据预测结果的均方误差（MSE）为 ', mse)