84、循环神经网络实现语言模型

‘‘‘
Created on 2017年5月13日

@author: weizhen
‘‘‘
import numpy as np
import tensorflow as tf
import ptb_iterator as reader
from tensorflow.contrib import rnn 

DATA_PATH = "/path/to/ptb/data"  # 数据存放的路径
HIDDEN_SIZE = 200  # 隐藏层的规模
NUM_LAYERS = 2  # 深层循环神经网络中LSTM结构的层数
VOCAB_SIZE = 10000  # 词典规模，加上语句结束标识符和稀有单词标识符总共一万个单词

LEARNING_RATE = 1.0  # 学习速率
TRAIN_BATCH_SIZE = 20  # 训练数据batch的大小
TRAIN_NUM_STEP = 35  # 训练数据截断长度

# 在测试时不需要使用截断，所以可以将测试数据看成一个超长的序列
EVAL_BATCH_SIZE = 1  # 测试数据batch的大小
EVAL_NUM_STEP = 1  # 测试数据截断长度
NUM_EPOCH = 2  # 使用训练数据的轮数
KEEP_PROB = 0.5  # 节点不被dropout的概率
MAX_GRAD_NORM = 5  # 用于控制梯度膨胀的参数

def LstmCell(is_training):
    lstm_cell = rnn.BasicLSTMCell(HIDDEN_SIZE,reuse=tf.get_variable_scope().reuse)
    if is_training:
            lstm_cell = rnn.DropoutWrapper(lstm_cell, output_keep_prob=KEEP_PROB)
    return lstm_cell
# 通过一个PTBModel类来描述模型，这样方便维护循环神经网络中的状态
class PTBModel(object):
    def __init__(self, is_training, batch_size, num_steps):
        # 记录使用的batch大小和截断长度
        self.batch_size = batch_size
        self.num_steps = num_steps

        # 定义输入层。可以看到输入层的维度为batch_size*num_steps,这和
        # ptb_iterator函数输出的训练数据batch是一致的
        self.input_data = tf.placeholder(tf.int32, [batch_size, num_steps])

        # 定义预期输出，它的维度和ptb_iterator函数输出的正确答案维度也是一样的
        self.targets = tf.placeholder(tf.int32, [batch_size, num_steps])

        # 定义使用LSTM结构为循环体结构且使用dropout的深层循环神经网络
#         lstm_cell = rnn.BasicLSTMCell(HIDDEN_SIZE)
#         if is_training:
#             lstm_cell = rnn.DropoutWrapper(lstm_cell, output_keep_prob=KEEP_PROB)
        cell = rnn.MultiRNNCell([LstmCell(is_training) for _ in range(NUM_LAYERS)])

        # 初始化最初的状态，也就是全零的向量
        self.initial_state = cell.zero_state(batch_size, tf.float32)

        # 将单词ID转换成为单词向量。因为总共有VOCAB_SIZE个单词，每个单词向量的维度为HIDDEN_SIZE
        # 所以embedding参数的维度为VOCAB_SIZE*HIDDEN_SIZE
        embedding = tf.get_variable("embedding", [VOCAB_SIZE, HIDDEN_SIZE])

        # 将原本batch_size*num_steps个单词ID转化为单词向量，转换后的输入层维度为batch_size*num_steps*HIDDEN_SIZE
        inputs = tf.nn.embedding_lookup(embedding, self.input_data)

        # 只在训练时使用dropout
        if is_training:
            inputs = tf.nn.dropout(inputs, KEEP_PROB)

        # 定义输出列表，在这里先将不同时刻LSTM结构的输出收集起来，再通过一个全连接层得到最终的输出
        outputs = []
        # state存储不同batch种LSTM的状态，将其初始化为0
        state = self.initial_state
        with tf.variable_scope("RNN"):
            for time_step in range(num_steps):
                if time_step > 0: tf.get_variable_scope().reuse_variables()
                # 从输入数据中获取当前时刻的输入并传入LSTM结构

                cell_output, state = cell(inputs[:, time_step, :], state)
                #cell_output, state = tf.nn.dynamic_rnn(cell,inputs[:, time_step, :], state,time_major=False)
                # 当前输出加入输出队列
                outputs.append(cell_output)

        # 把输出队列展开成[batch*hidden_size*num_steps]的形状，然后再
        # reshape成[batch*numsteps,hidden_size]的形状
        output = tf.reshape(tf.concat(outputs, 1), [-1, HIDDEN_SIZE])

        # 将从LSTM中得到的输出再经过一个全连接层得到最后的预测结果，最终的预测结果在每一个时刻上都是一个长度为VOCAB_Size的数组，
        # 经过softmax层之后表示下一个位置是不同单词的概率
        weight = tf.get_variable("weight", [HIDDEN_SIZE, VOCAB_SIZE])
        bias = tf.get_variable("bias", [VOCAB_SIZE])
        logits = tf.matmul(output, weight) + bias

        # 定义交叉熵损失函数。TensorFlow提供了sequence_loss_by_example函数来计算一个序列的交叉熵的和
        loss = tf.contrib.legacy_seq2seq.sequence_loss_by_example(
            [logits],  # 预测的结果
            [tf.reshape(self.targets, [-1])],  # 期待的正确答案，这里讲[batch_size,num_steps]二维数组压缩成一维数组
            [tf.ones([batch_size * num_steps], dtype=tf.float32)]  # 损失的权重，在这里所有的权重都为1，也就是说不同batch和不同时刻的重要程度是一样的
            )

        # 计算得到每个batch的平均损失
        self.cost = tf.reduce_sum(loss) / batch_size
        self.final_state = state

        # 只在训练模型时定义方向传播操作
        if not is_training:
            return
        trainable_variables = tf.trainable_variables()
        # 通过clip_by_global_norm函数控制梯度的大小，避免梯度膨胀的问题
        grads, _ = tf.clip_by_global_norm(tf.gradients(self.cost, trainable_variables), MAX_GRAD_NORM)

        # 定义优化方法
        optimizer = tf.train.GradientDescentOptimizer(LEARNING_RATE)
        # 定义训练步骤
        self.train_op = optimizer.apply_gradients(zip(grads, trainable_variables))

# 使用给定的模型model在数据data上运行train_op并返回在全部数据上的perplexity值
def run_epoch(session, model, data, train_op, output_log):
    # 计算perplexity的辅助变量。
    total_costs = 0.0
    iters = 0
    state = session.run(model.initial_state)
    # 训练一个epoch。
    for step,(x,y) in enumerate(reader.ptb_iterator(data,model.batch_size,model.num_steps)):
        #在当前batch上运行train_op并计算损失值。交叉熵损失函数计算的就是下一个单词为给定单词的概率
        cost,state,_ = session.run([model.cost,model.final_state,train_op],
                                   {model.input_data:x,model.targets:y,
                                    model.initial_state:state})
        #将不同时刻，不同batch的概率加起来就可以得到第二个perplexity公司等号右边的部分，
        #再将这个和做指数运算就可以得到perplexity值
        total_costs+=cost
        iters+=model.num_steps

        #只有在训练时输出日志
        if output_log and step % 100 == 0:
            print("after % step ,perplexity is %.3f" %(step,np.exp(total_costs/iters)))

        #返回给定模型在给定数据上的perplexity值
        return np.exp(total_costs/iters)

def main(_):
    # 获取原始数据
    train_data, valid_data, test_data, _ = reader.ptb_raw_data(DATA_PATH)

    # 计算一个epoch需要训练的次数
    #train_data_len = len(train_data)
    #train_batch_len = train_data_len  # # TRAIN_BATCH_SIZE
    #train_epoch_size = (train_batch_len - 1)  # # TRAIN_NUM_STEP

    #valid_data_len = len(valid_data)
    #valid_batch_len = valid_data_len  # # EVAL_BATCH_SIZE
    #valid_epoch_size = (valid_batch_len - 1)  # # EVAL_NUM_STEP

    #test_data_len = len(test_data)
    #test_batch_len = test_data_len  # # EVAL_BATCH_SIZE
    #test_epoch_size = (test_batch_len - 1)  # # EVAL_NUM_STEP

    # 定义初始化函数
    initializer = tf.random_uniform_initializer(-0.05, 0.05)

    with tf.variable_scope("language_model", reuse=None, initializer=initializer):
        train_model = PTBModel(True, TRAIN_BATCH_SIZE, TRAIN_NUM_STEP)

    # 定义评测用的循环神经网络模型
    with tf.variable_scope("language_model", reuse=True, initializer=initializer):
        eval_model = PTBModel(False, EVAL_BATCH_SIZE, EVAL_NUM_STEP)

    with tf.Session() as session:
        tf.global_variables_initializer().run()

        # 使用训练数据训练模型
        for i in range(NUM_EPOCH):
            print("In iteration:%d" % (i + 1))
            # 在所有训练数据上训练循环神经网络模型
            run_epoch(session, train_model, train_data, train_model.train_op, True)

            # 使用验证数据评测模型效果
            valid_perplexity = run_epoch(session, eval_model, valid_data, tf.no_op(), False)
            print("Epoch: %d Validation Perplexity : %.3f" % (i + 1, valid_perplexity))

        # 最后使用测试数据测试模型效果
        test_perplexity = run_epoch(session, eval_model, test_data, tf.no_op(), False)
        print("Test Perplexity:%.3f" % test_perplexity)

if __name__ == "__main__":
    tf.app.run()

不过感觉很奇怪，就训练了两轮就结束了

2017-05-21 10:47:16.695751: W c:\tf_jenkins\home\workspace\release-win\device\cpu\os\windows\tensorflow\core\platform\cpu_feature_guard.cc:45] The TensorFlow library wasn‘t compiled to use SSE instructions, but these are available on your machine and could speed up CPU computations.
2017-05-21 10:47:16.696456: W c:\tf_jenkins\home\workspace\release-win\device\cpu\os\windows\tensorflow\core\platform\cpu_feature_guard.cc:45] The TensorFlow library wasn‘t compiled to use SSE2 instructions, but these are available on your machine and could speed up CPU computations.
2017-05-21 10:47:16.696955: W c:\tf_jenkins\home\workspace\release-win\device\cpu\os\windows\tensorflow\core\platform\cpu_feature_guard.cc:45] The TensorFlow library wasn‘t compiled to use SSE3 instructions, but these are available on your machine and could speed up CPU computations.
2017-05-21 10:47:16.697919: W c:\tf_jenkins\home\workspace\release-win\device\cpu\os\windows\tensorflow\core\platform\cpu_feature_guard.cc:45] The TensorFlow library wasn‘t compiled to use SSE4.1 instructions, but these are available on your machine and could speed up CPU computations.
2017-05-21 10:47:16.698685: W c:\tf_jenkins\home\workspace\release-win\device\cpu\os\windows\tensorflow\core\platform\cpu_feature_guard.cc:45] The TensorFlow library wasn‘t compiled to use SSE4.2 instructions, but these are available on your machine and could speed up CPU computations.
2017-05-21 10:47:16.699159: W c:\tf_jenkins\home\workspace\release-win\device\cpu\os\windows\tensorflow\core\platform\cpu_feature_guard.cc:45] The TensorFlow library wasn‘t compiled to use AVX instructions, but these are available on your machine and could speed up CPU computations.
2017-05-21 10:47:16.699770: W c:\tf_jenkins\home\workspace\release-win\device\cpu\os\windows\tensorflow\core\platform\cpu_feature_guard.cc:45] The TensorFlow library wasn‘t compiled to use AVX2 instructions, but these are available on your machine and could speed up CPU computations.
2017-05-21 10:47:16.700265: W c:\tf_jenkins\home\workspace\release-win\device\cpu\os\windows\tensorflow\core\platform\cpu_feature_guard.cc:45] The TensorFlow library wasn‘t compiled to use FMA instructions, but these are available on your machine and could speed up CPU computations.
In iteration:1
after 0tep ,perplexity is 9962.385
Epoch: 1 Validation Perplexity : 2167.129
In iteration:2
after 0tep ,perplexity is 5994.104
Epoch: 2 Validation Perplexity : 417.495
Test Perplexity:418.547

下面是用到的解析ptb数据的工具类ptb_reader

# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Utilities for parsing PTB text files."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import collections
import os

import numpy as np
import tensorflow as tf

def _read_words(filename):
  with tf.gfile.GFile(filename, "r") as f:
    return f.read().replace("\n", "<eos>").split() #读取文件， 将换行符替换为 <eos>， 然后将文件按空格分割。 返回一个 1-D list

def _build_vocab(filename):  #用于建立字典
  data = _read_words(filename)
  counter = collections.Counter(data) #输出一个字典： key是word， value是这个word出现的次数
  count_pairs = sorted(counter.items(), key=lambda x: (-x[1], x[0]))
#counter.items() 会返回一个tuple列表， tuple是(key, value), 按 value的降序，key的升序排列
  words, _ = list(zip(*count_pairs)) #感觉这个像unzip 就是把key放在一个tuple里，value放在一个tuple里
  word_to_id = dict(zip(words, range(len(words))))#对每个word进行编号， 按照之前words输出的顺序（value降序，key升序）
  return word_to_id  #返回dict， key：word， value：id

def _file_to_word_ids(filename, word_to_id): #将file表示为word_id的形式
  data = _read_words(filename)
  return [word_to_id[word] for word in data]

def ptb_raw_data(data_path=None):
  """Load PTB raw data from data directory "data_path".
  Reads PTB text files, converts strings to integer ids,
  and performs mini-batching of the inputs.
  The PTB dataset comes from Tomas Mikolov‘s webpage:
  http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz
  Args:
    data_path: string path to the directory where simple-examples.tgz has
      been extracted.
  Returns:
    tuple (train_data, valid_data, test_data, vocabulary)
    where each of the data objects can be passed to PTBIterator.
  """

  train_path = os.path.join(data_path, "ptb.train.txt")
  valid_path = os.path.join(data_path, "ptb.valid.txt")
  test_path = os.path.join(data_path, "ptb.test.txt")

  word_to_id = _build_vocab(train_path) #使用训练集确定word id
  train_data = _file_to_word_ids(train_path, word_to_id)
  valid_data = _file_to_word_ids(valid_path, word_to_id)
  test_data = _file_to_word_ids(test_path, word_to_id)
  vocabulary = len(word_to_id)#字典的大小
  return train_data, valid_data, test_data, vocabulary

def ptb_iterator(raw_data, batch_size, num_steps):
  """Iterate on the raw PTB data.
  This generates batch_size pointers into the raw PTB data, and allows
  minibatch iteration along these pointers.
  Args:
    raw_data: one of the raw data outputs from ptb_raw_data.
    batch_size: int, the batch size.
    num_steps: int, the number of unrolls.
  Yields:
    Pairs of the batched data, each a matrix of shape [batch_size, num_steps].
    The second element of the tuple is the same data time-shifted to the
    right by one.
  Raises:
    ValueError: if batch_size or num_steps are too high.
  """
  raw_data = np.array(raw_data, dtype=np.int32)#raw data : train_data | vali_data | test data

  data_len = len(raw_data) #how many words in the data_set
  batch_len = data_len // batch_size
  data = np.zeros([batch_size, batch_len], dtype=np.int32)#batch_len 就是几个word的意思
  for i in range(batch_size):
    data[i] = raw_data[batch_len * i:batch_len * (i + 1)]

  epoch_size = (batch_len - 1) // num_steps

  if epoch_size == 0:
    raise ValueError("epoch_size == 0, decrease batch_size or num_steps")

  for i in range(epoch_size):
    x = data[:, i*num_steps:(i+1)*num_steps]
    y = data[:, i*num_steps+1:(i+1)*num_steps+1]
  yield (x, y)

ptb数据集放置在C盘的根目录下

时间： 2024-08-10 19:18:16

84、循环神经网络实现语言模型的相关文章

循环神经网络-语言模型

在构建语言模型中,我们需要理解n元模型以及网络架构. 一. n元语法 n元语法通过马尔可夫假设简化模型,马尔科夫假设是指一个词的出现只与前面n个词相关,即n阶马尔可夫链(Markov chain of order n). 来看以下几个例子,下面分别是1元,2元,3元语法模型的结果. $P\left(w_{1}, w_{2}, w_{3}, w_{4}\right)=P\left(w_{1}\right) P\left(w_{2}\right) P\left(w_{3}\right) P\left

DataWhale 动手学深度学习PyTorch版-task3+4+5：文本预处理；语言模型；循环神经网络基础

课程引用自伯禹平台:https://www.boyuai.com/elites/course/cZu18YmweLv10OeV <动手学深度学习>官方网址:http://zh.gluon.ai/ ——面向中文读者的能运行.可讨论的深度学习教科书. 第二次打卡: Task03: 过拟合.欠拟合及其解决方案:梯度消失.梯度爆炸:循环神经网络进阶 Task04:机器翻译及相关技术:注意力机制与Seq2seq模型:Transformer Task05:卷积神经网络基础:leNet:卷积神经网络进阶 有

基于PaddlePaddle框架利用RNN（循环神经网络）生成古诗句

基于PaddlePaddle框架利用RNN(循环神经网络)生成古诗句 在本项目中,将使用PaddlePaddle实现循环神经网络模型(即RNN模型,以下循环神经网络都称作RNN),并实现基于RNN语言模型进行诗句的生成. 本项目利用全唐诗数据集对RNN语言模型进行训练,能够实现根据输入的前缀诗句,自动生成后续诗句. 本实验所用全唐诗数据集下载地址:https://pan.baidu.com/s/1OgIdxjO2jh5KC8XzG-j8ZQ 1.背景知识 RNN是一个序列模型,基本思路是:在时刻

循环神经网络RNN公式推导走读

0语言模型-N-Gram 语言模型就是给定句子前面部分,预测后面缺失部分 eg．我昨天上学迟到了,老师批评了____. N-Gram模型: ,对一句话切词 我 昨天 上学 迟到 了 ,老师 批评 了 ____. 2-N-Gram 会在语料库中找 了 后面最可能的词: 3-N-Gram 会在预料库中找 批评了 后面最可能的词: 4-N-Gram 的内存耗费就非常巨大了(语料库中保存所有的四个词的预料组合). 1.1单向循环神经网络 一个单隐层结构示意图: 参数:输入到隐层的权重U.隐层到输出的权重

TensorFlow框架(6)之RNN循环神经网络详解

1. RNN循环神经网络 1.1 结构 循环神经网络(recurrent neural network,RNN)源自于1982年由Saratha Sathasivam 提出的霍普菲尔德网络.RNN的主要用途是处理和预测序列数据.全连接的前馈神经网络和卷积神经网络模型中,网络结构都是从输入层到隐藏层再到输出层,层与层之间是全连接或部分连接的,但每层之间的节点是无连接的. 图 11 RNN-rolled 如图 11所示是一个典型的循环神经网络.对于循环神经网络,一个非常重要的概念就是时刻.循环神经网

theano学习指南--词向量的循环神经网络(翻译)

欢迎fork我的github:https://github.com/zhaoyu611/DeepLearningTutorialForChinese 最近在学习Git,所以正好趁这个机会,把学习到的知识实践一下~ 看完DeepLearning的原理,有了大体的了解,但是对于theano的代码,还是自己撸一遍印象更深 所以照着deeplearning.net上的代码,重新写了一遍,注释部分是原文翻译和自己的理解. 感兴趣的小伙伴可以一起完成这个工作哦~ 有问题欢迎联系我 Email: [email

《转》循环神经网络(RNN, Recurrent Neural Networks)学习笔记：基础理论

转自 http://blog.csdn.net/xingzhedai/article/details/53144126 更多参考:http://blog.csdn.net/mafeiyu80/article/details/51446558 http://blog.csdn.net/caimouse/article/details/70225998 http://kubicode.me/2017/05/15/Deep%20Learning/Understanding-about-RNN/ RNN

Recurrent Neural Network(循环神经网络)

Reference:   Alex Graves的[Supervised Sequence Labelling with RecurrentNeural Networks] Alex是RNN最著名变种,LSTM发明者Jürgen Schmidhuber的高徒,现加入University of Toronto,拜师Hinton. 统计语言模型与序列学习 1.1 基于频数统计的语言模型 NLP领域最著名的语言模型莫过于N-Gram. 它基于马尔可夫假设,当然,这是一个2-Gram(Bi-Gram)模

学习笔记TF057:TensorFlow MNIST，卷积神经网络、循环神经网络、无监督学习

MNIST 卷积神经网络.https://github.com/nlintz/TensorFlow-Tutorials/blob/master/05_convolutional_net.py .TensorFlow搭建卷积神经网络(CNN)模型,训练MNIST数据集. 构建模型. 定义输入数据,预处理数据.读取数据MNIST,得到训练集图片.标记矩阵,测试集图片标记矩阵.trX.trY.teX.teY 数据矩阵表现.trX.teX形状变为[-1,28,28,1],-1 不考虑输入图片数量,28x