IMDB影评情感分析项目
imdb数据集下载地址: http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz
import os
import torch
from torch import nn
import dltoolsdef read_imdb(data_dir, is_train):data, labels = [], []for label in ('pos', 'neg'):folder_name = os.path.join(data_dir, 'train' if is_train else 'test', label)for file in os.listdir(folder_name): #os.listdir()相当于形成文件夹列表，可以遍历路径下的每一个文件with open(os.path.join(folder_name, file), 'rb') as f:#文本没有分行，就一次性读取review = f.read().decode('utf-8').replace('\n', '')data.append(review)labels.append(1 if label == 'pos' else 0) #三目运算符return data, labelsdata_dir = r'E:\ALOT\10_deep_learning\data\aclImdb'
train_data = read_imdb(data_dir, is_train=True)
print('训练集数目：', len(train_data[0]))训练集数目： 25000#列表和元祖没有shape
for x, y in zip(train_data[0][:3], train_data[1][:3]):print('标签：', y, 'review:', x[:60])标签： 1 review: Bromwell High is a cartoon comedy. It ran at the same time a 标签： 1 review: Homelessness (or Houselessness as George Carlin stated) has 标签： 1 review: Brilliant over-acting by Lesley Ann Warren. Best dramatic ho
#分词
train_tokens = dltools.tokenize(train_data[0], token='word')
train_tokens #是一个二维列表， 一个文本文件一个（内置）列表#构建词汇表vocab
vocab = dltools.Vocab(train_tokens, min_freq=5, reserved_tokens=['<pad>'])len(vocab)49347
#绘制直方图查看每条评论的字数分布
dltools.set_figsize()
dltools.plt.xlabel('# tokens per review')
dltools.plt.ylabel('count')
dltools.plt.hist([len(line) for line in train_tokens], bins=range(0,1000, 50)) #分成50份