在《Q-learning简明实例》中我们介绍了Q-learning算法的简单例子，从中我们可以总结出Q-learning算法的基本思想
本次选择的经验得分 = 本次选择的反馈得分 + 本次选择后场景的历史最佳经验得分
其中反馈得分是单个步骤的价值分值（固定的分值），经验得分是完成目标的学习分值（动态的分值）。
简明实例的Java实现如下
package com.coshaho.learn.qlearning;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Random;
/**
*
* QLearning.java Create on 2017年9月4日 下午10:08:49
*
* 类功能说明: QLearning简明例子实现
*
* Copyright: Copyright(c) 2013
* Company: COSHAHO
* @Version 1.0
* @Author coshaho
*/
public class QLearning
{
FeedbackMatrix R = new FeedbackMatrix();
ExperienceMatrix Q = new ExperienceMatrix();
public static void main(String[] args)
{
QLearning ql = new QLearning();
for(int i = 0; i < 500; i++)
{
Random random = new Random();
int x = random.nextInt(100) % 6;
System.out.println("第" + i + "次学习, 初始房间是" + x);
ql.learn(x);
System.out.println();
}
}
public void learn(int x)
{
do
{
// 随机选择一个联通的房间进入
int y = chooseRandomRY(x);
// 获取以进入的房间为起始点的历史最佳得分
int qy = getMaxQY(y);
// 计算此次移动的得分
int value = calculateNewQ(x, y, qy);
Q.set(x, y, value);
x = y;
}
// 走出房间则学习结束
while(5 != x);
Q.print();
}
public int chooseRandomRY(int x)
{
int[] qRow = R.getRow(x);
List<Integer> yValues = new ArrayList<Integer>();
for(int i = 0; i < qRow.length; i++)
{
if(qRow[i] >= 0)
{
yValues.add(i);
}
}
Random random = new Random();
int i = random.nextInt(yValues.size()) % yValues.size();
return yValues.get(i);
}
public int getMaxQY(int x)
{
int[] qRow = Q.getRow(x);
int length = qRow.length;
List<YAndValue> yValues = new ArrayList<YAndValue>();
for(int i = 0; i < length; i++)
{
YAndValue yv = new YAndValue(i, qRow[i]);
yValues.add(yv);
}
Collections.sort(yValues);
int num = 1;
int value = yValues.get(0).getValue();
for(int i = 1; i < length; i++)
{
if(yValues.get(i).getValue() == value)
{
num = i + 1;
}
else
{
break;
}
}
Random random = new Random();
int i = random.nextInt(num) % num;
return yValues.get(i).getY();
}
// Q(x,y) = R(x,y) + 0.8 * max(Q(y,i))
public int calculateNewQ(int x, int y, int qy)
{
return (int) (R.get(x, y) + 0.8 * Q.get(y, qy));
}
public static class YAndValue implements Comparable<YAndValue>
{
int y;
int value;
public int getY() {
return y;
}
public void setY(int y) {
this.y = y;
}
public int getValue() {
return value;
}
public void setValue(int value) {
this.value = value;
}
public YAndValue(int y, int value)
{
this.y = y;
this.value = value;
}
public int compareTo(YAndValue o)
{
return o.getValue() - this.value;
}
}
}
package com.coshaho.learn.qlearning;
/**
*
* FeedbackMatrix.java Create on 2017年9月4日 下午9:52:41
*
* 类功能说明: 反馈矩阵
*
* Copyright: Copyright(c) 2013
* Company: COSHAHO
* @Version 1.0
* @Author coshaho
*/
public class FeedbackMatrix
{
public int get(int x, int y)
{
return R[x][y];
}
public int[] getRow(int x)
{
return R[x];
}
private static int[][] R = new int[6][6];
static
{
R[0][0] = -1;
R[0][1] = -1;
R[0][2] = -1;
R[0][3] = -1;
R[0][4] = 0;
R[0][5] = -1;
R[1][0] = -1;
R[1][1] = -1;
R[1][2] = -1;
R[1][3] = 0;
R[1][4] = -1;
R[1][5] = 100;
R[2][0] = -1;
R[2][1] = -1;
R[2][2] = -1;
R[2][3] = 0;
R[2][4] = -1;
R[2][5] = -1;
R[3][0] = -1;
R[3][1] = 0;
R[3][2] = 0;
R[3][3] = -1;
R[3][4] = 0;
R[3][5] = -1;
R[4][0] = 0;
R[4][1] = -1;
R[4][2] = -1;
R[4][3] = 0;
R[4][4] = -1;
R[4][5] = 100;
R[5][0] = -1;
R[5][1] = 0;
R[5][2] = -1;
R[5][3] = -1;
R[5][4] = 0;
R[5][5] = 100;
}
}
package com.coshaho.learn.qlearning;
/**
*
* ExperienceMatrix.java Create on 2017年9月4日 下午10:03:08
*
* 类功能说明: 经验矩阵
*
* Copyright: Copyright(c) 2013
* Company: COSHAHO
* @Version 1.0
* @Author coshaho
*/
public class ExperienceMatrix
{
public int get(int x, int y)
{
return Q[x][y];
}
public int[] getRow(int x)
{
return Q[x];
}
public void set(int x, int y, int value)
{
Q[x][y] = value;
}
public void print()
{
for(int i = 0; i < 6; i++)
{
for(int j = 0; j < 6; j++)
{
String s = Q[i][j] + " ";
if(Q[i][j] < 10)
{
s = s + " ";
}
else if(Q[i][j] < 100)
{
s = s + " ";
}
System.out.print(s);
}
System.out.println();
}
}
private static int[][] Q = new int[6][6];
static
{
Q[0][0] = 0;
Q[0][1] = 0;
Q[0][2] = 0;
Q[0][3] = 0;
Q[0][4] = 0;
Q[0][5] = 0;
Q[1][0] = 0;
Q[1][1] = 0;
Q[1][2] = 0;
Q[1][3] = 0;
Q[1][4] = 0;
Q[1][5] = 0;
Q[2][0] = 0;
Q[2][1] = 0;
Q[2][2] = 0;
Q[2][3] = 0;
Q[2][4] = 0;
Q[2][5] = 0;
Q[3][0] = 0;
Q[3][1] = 0;
Q[3][2] = 0;
Q[3][3] = 0;
Q[3][4] = 0;
Q[3][5] = 0;
Q[4][0] = 0;
Q[4][1] = 0;
Q[4][2] = 0;
Q[4][3] = 0;
Q[4][4] = 0;
Q[4][5] = 0;
Q[5][0] = 0;
Q[5][1] = 0;
Q[5][2] = 0;
Q[5][3] = 0;
Q[5][4] = 0;
Q[5][5] = 0;
}
}
经过500次计算得到如下结果
第499次学习, 初始房间是1 0 0 0 0 396 0 0 0 0 316 0 496 0 0 0 316 0 0 0 396 252 0 396 0 316 0 0 316 0 496 0 396 0 0 396 496
此时，我们从任意一个房间进入，每次选取最高分值步骤移动，总可以找到最短的逃离路径。
时间： 2024-11-02 16:33:22