1

2

3

4

5

6

7

8

9

10

11

class RNNNumpy:

def __init__(self, word_dim, hidden_dim=100, bptt_truncate=4):

# Assign instance variables

self.word_dim = word_dim

self.hidden_dim = hidden_dim

self.bptt_truncate = bptt_truncate

# Randomly initialize the network parameters

self.U = np.random.uniform(-np.sqrt(1./word_dim), np.sqrt(1./word_dim), (hidden_dim, word_dim))

self.V = np.random.uniform(-np.sqrt(1./hidden_dim), np.sqrt(1./hidden_dim), (word_dim, hidden_dim))

self.W = np.random.uniform(-np.sqrt(1./hidden_dim), np.sqrt(1./hidden_dim), (hidden_dim, hidden_dim))

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

def forward_propagation(self, x):

# The total number of time steps

T = len(x)

# During forward propagation we save all hidden states in s because need them later.

# We add one additional element for the initial hidden, which we set to 0

s = np.zeros((T + 1, self.hidden_dim))

s[-1] = np.zeros(self.hidden_dim)

# The outputs at each time step. Again, we save them for later.

o = np.zeros((T, self.word_dim))

# For each time step...

for t in np.arange(T):

# Note that we are indxing U by x[t]. This is the same as multiplying U with a one-hot vector.

s[t] = np.tanh(self.U[:,x[t]] + self.W.dot(s[t-1]))

o[t] = softmax(self.V.dot(s[t]))

return [o, s]

1

2

3

4

def predict(self, x):

# Perform forward propagation and return index of the highest score

o, s = self.forward_propagation(x)

return np.argmax(o, axis=1)

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

def calculate_total_loss(self, x, y):

L = 0

# For each sentence...

for i in np.arange(len(y)):

o, s = self.forward_propagation(x[i])

# We only care about our prediction of the "correct" words

correct_word_predictions = o[np.arange(len(y[i])), y[i]]

# Add to the loss based on how off we were

L += -1 * np.sum(np.log(correct_word_predictions))

return L

def calculate_loss(self, x, y):

# Divide the total loss by the number of training examples

N = np.sum((len(y_i) for y_i in y))

return self.calculate_total_loss(x,y)/N

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

def bptt(self, x, y):

T = len(y)

# Perform forward propagation

o, s = self.forward_propagation(x)

# We accumulate the gradients in these variables

dLdU = np.zeros(self.U.shape)

dLdV = np.zeros(self.V.shape)

dLdW = np.zeros(self.W.shape)

delta_o = o

delta_o[np.arange(len(y)), y] -= 1.

# For each output backwards...

for t in np.arange(T)[::-1]:

dLdV += np.outer(delta_o[t], s[t].T)

# Initial delta calculation: dL/dz

delta_t = self.V.T.dot(delta_o[t]) * (1 - (s[t] ** 2))

# Backpropagation through time (for at most self.bptt_truncate steps)

for bptt_step in np.arange(max(0, t-self.bptt_truncate), t+1)[::-1]:

# print "Backpropagation step t=%d bptt step=%d " % (t, bptt_step)

# Add to gradients at each previous step

dLdW += np.outer(delta_t, s[bptt_step-1])

dLdU[:,x[bptt_step]] += delta_t

# Update delta for next step dL/dz at t-1

delta_t = self.W.T.dot(delta_t) * (1 - s[bptt_step-1] ** 2)

return [dLdU, dLdV, dLdW]

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

def gradient_check(self, x, y, h=0.001, error_threshold=0.01):

# Calculate the gradients using backpropagation. We want to checker if these are correct.

bptt_gradients = self.bptt(x, y)

# List of all parameters we want to check.

model_parameters = [‘U‘, ‘V‘, ‘W‘]

# Gradient check for each parameter

for pidx, pname in enumerate(model_parameters):

# Get the actual parameter value from the mode, e.g. model.W

parameter = operator.attrgetter(pname)(self)

print "Performing gradient check for parameter %s with size %d." % (pname, np.prod(parameter.shape))

# Iterate over each element of the parameter matrix, e.g. (0,0), (0,1), ...

it = np.nditer(parameter, flags=[‘multi_index‘], op_flags=[‘readwrite‘])

while not it.finished:

ix = it.multi_index

# Save the original value so we can reset it later

original_value = parameter[ix]

# Estimate the gradient using (f(x+h) - f(x-h))/(2*h)

parameter[ix] = original_value + h

gradplus = self.calculate_total_loss([x],[y])

parameter[ix] = original_value - h

gradminus = self.calculate_total_loss([x],[y])

estimated_gradient = (gradplus - gradminus)/(2*h)

# Reset parameter to original value

parameter[ix] = original_value

# The gradient for this parameter calculated using backpropagation

backprop_gradient = bptt_gradients[pidx][ix]

# calculate The relative error: (|x - y|/(|x| + |y|))

relative_error = np.abs(backprop_gradient - estimated_gradient)/(np.abs(backprop_gradient) + np.abs(estimated_gradient))

# If the error is to large fail the gradient check

if relative_error > error_threshold:

print "Gradient Check ERROR: parameter=%s ix=%s" % (pname, ix)

print "+h Loss: %f" % gradplus

print "-h Loss: %f" % gradminus

print "Estimated_gradient: %f" % estimated_gradient

print "Backpropagation gradient: %f" % backprop_gradient

print "Relative Error: %f" % relative_error

return

it.iternext()

print "Gradient check for parameter %s passed." % (pname)

1

2

3

4

5

6

7

8

# Performs one step of SGD.

def numpy_sdg_step(self, x, y, learning_rate):

# Calculate the gradients

dLdU, dLdV, dLdW = self.bptt(x, y)

# Change parameters according to gradients and learning rate

self.U -= learning_rate * dLdU

self.V -= learning_rate * dLdV

self.W -= learning_rate * dLdW

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

def generate_sentence(model):

# We start the sentence with the start token

new_sentence = [word_to_index[sentence_start_token]]

# Repeat until we get an end token

while not new_sentence[-1] == word_to_index[sentence_end_token]:

next_word_probs = model.forward_propagation(new_sentence)

sampled_word = word_to_index[unknown_token]

# We don‘t want to sample unknown words

while sampled_word == word_to_index[unknown_token]:

samples = np.random.multinomial(1, next_word_probs[-1])

sampled_word = np.argmax(samples)

new_sentence.append(sampled_word)

sentence_str = [index_to_word[x] for x in new_sentence[1:-1]]

return sentence_str

num_sentences = 10

senten_min_length = 7

for i in range(num_sentences):

sent = []

# We want long sentences, not sentences with one or two words

while len(sent) < senten_min_length:

sent = generate_sentence(model)

print " ".join(sent)