Python如何使用神经网络进行简单文本分类

import pandas as pd
import numpy as np
import pickle
from keras.preprocessing.text import Tokenizer
from keras.models import Sequential
from keras.layers import Activation, Dense, Dropout
from sklearn.preprocessing import LabelBinarizer
import sklearn.datasets as skds
from pathlib import Path

# 为了复现性

np.random.seed(1237)
  
label_index = files_train.target
label_names = files_train.target_names
labelled_files = files_train.filenames
 
data_tags = ["filename","category","news"]
data_list = []
 
# 读取文件中的数据并将其添加到列表


 
data = pd.DataFrame.from_records(data_list, columns=data_tags)

# 让我们以80％的数据作为训练，剩下的20％作为测试。


train_size = int(len(data) * .8)
 
train_posts = data['news'][:train_size]
train_tags = data['category'][:train_size]
train_files_names = data['filename'][:train_size]
 
test_posts = data['news'][train_size:]
test_tags = data['category'][train_size:]
test_files_names = data['filename'][train_size:]

# 20个新闻组


num_labels = 20
vocab_size = 15000
batch_size = 100
 
# 用Vocab Size定义Tokenizer


tokenizer = Tokenizer(num_words=vocab_size)
tokenizer.fit_on_texts(train_posts) 

model = Sequential()

100/8145 [..............................] - ETA: 31s - loss: 1.0746e-04 - acc: 1.0000
200/8145 [..............................] - ETA: 31s - loss: 0.0186 - acc: 0.9950    
300/8145 [>.............................] - ETA: 35s - loss: 0.0125 - acc: 0.9967
400/8145 [>.............................] - ETA: 32s - loss: 0.0094 - acc: 0.9975
500/8145 [>.............................] - ETA: 30s - loss: 0.0153 - acc: 0.9960
...
7900/8145 [============================>.] - ETA: 0s - loss: 0.1256 - acc: 0.9854
8000/8145 [============================>.] - ETA: 0s - loss: 0.1261 - acc: 0.9855
8100/8145 [============================>.] - ETA: 0s - loss: 0.1285 - acc: 0.9854
8145/8145 [==============================] - 29s 4ms/step - loss: 0.1293 - acc: 0.9854 - val_loss: 1.0597 - val_acc: 0.8742
 
Test accuracy: 0.8767123321648251

for i in range(10):
    prediction = model.predict(np.array([x_test[i]]))
    predicted_label = text_labels[np.argmax(prediction[0])]
    print(test_files_names.iloc[i])
    print('Actual label:' + test_tags.iloc[i])
    print("Predicted label: " + predicted_label)

# 创建一个HDF5文件'my_model.h5'


model.model.save('my_model.h5')
 
# 保存令牌生成器，即词汇表


with open('tokenizer.pickle', 'wb') as handle:
    pickle.dump(tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)

encoder.classes_ #标签二值化

 labels = np.array(['alt.atheism', 'comp.graphics', 'comp.os.ms-windows.misc',
'comp.sys.ibm.pc.hardware', 'comp.sys.mac.hardware', 'comp.windows.x',
'misc.forsale', 'rec.autos', 'rec.motorcycles', 'rec.sport.baseball',
'rec.sport.hockey', 'sci.crypt', 'sci.electronics', 'sci.med', 'sci.space',
'soc.religion.christian', 'talk.politics.guns', 'talk.politics.mideast',
'talk.politics.misc', 'talk.religion.misc'])
 ...
for x_t in x_tokenized:
    prediction = model.predict(np.array([x_t]))
    predicted_label = labels[np.argmax(prediction[0])]
    print("File ->", test_files[i], "Predicted label: " + predicted_label)
    i += 1

File -> C:DL20news-bydate20news-bydate-testcomp.graphics38758 Predicted label: comp.graphics
File -> C:DL20news-bydate20news-bydate-testmisc.forsale76115 Predicted label: misc.forsale
File -> C:DL20news-bydate20news-bydate-testsoc.religion.christian21329 Predicted label: soc.religion.christian