多元梯度下降的Java实现
作者：互联网
我正在尝试在Java中实现多变量梯度下降算法(来自AI coursera课程),但是我无法弄清楚代码中的错误在哪里.
这是以下程序的输出：
Before train: parameters := [0.0, 0.0, 0.0] -> cost function := 2.5021875E9
After first iteration: parameters := [378.5833333333333, 2.214166666666667, 50043.75000000001] -> cost function := 5.404438291015627E9
如您所见,在第一次迭代之后,这些值相去甚远.我究竟做错了什么？
这是我正在尝试实现的算法：
和代码：
import java.util.*;
public class GradientDescent {
private double[][] trainingData;
private double[] means;
private double[] scale;
private double[] parameters;
private double learningRate;
GradientDescent() {
this.learningRate = 0D;
}
public double predict(double[] inp){
double[] features = new double[inp.length + 1];
features[0] = 1;
for(int i = 0; i < inp.length; i++) {
features[i+1] = inp[i];
}
double prediction = 0;
for(int i = 0; i < parameters.length; i++) {
prediction = parameters[i] * features[i];
}
return prediction;
}
public void train(){
double[] tempParameters = new double[parameters.length];
for(int i = 0; i < parameters.length; i++) {
tempParameters[i] = parameters[i] - learningRate * partialDerivative(i);
//System.out.println(tempParameters[i] + " = " + parameters[i] + " - " + learningRate + " * " + partialDerivative(i));
}
System.out.println("Before train: parameters := " + Arrays.toString(parameters) + " -> cost function := " + costFunction());
parameters = tempParameters;
System.out.println("After first iteration: parameters := " + Arrays.toString(parameters) + " -> cost function := " + costFunction());
}
private double partialDerivative(int index) {
double sum = 0;
for(int i = 0; i < trainingData.length; i++) {
double[] input = new double[trainingData[i].length - 1];
int j = 0;
for(; j < trainingData[i].length - 1; j++) {
input[j] = trainingData[i][j];
}
sum += ((predict(input) - trainingData[i][j]) * trainingData[i][index]);
}
return (1D/trainingData.length) * sum;
}
public double[][] getTrainingData() {
return trainingData;
}
public void setTrainingData(double[][] data) {
this.trainingData = data;
this.means = new double[this.trainingData[0].length-1];
this.scale = new double[this.trainingData[0].length-1];
for(int j = 0; j < data[0].length-1; j++) {
double min = data[0][j], max = data[0][j];
double sum = 0;
for(int i = 0; i < data.length; i++) {
if(data[i][j] < min) min = data[i][j];
if(data[i][j] > max) max = data[i][j];
sum += data[i][j];
}
scale[j] = max - min;
means[j] = sum / data.length;
}
}
public double[] getParameters() {
return parameters;
}
public void setParameters(double[] parameters) {
this.parameters = parameters;
}
public double getLearningRate() {
return learningRate;
}
public void setLearningRate(double learningRate) {
this.learningRate = learningRate;
}
/** 1 m i i 2
* J(theta) = ----- * SUM( h (x ) - y )
* 2*m i=1 theta
*/
public double costFunction() {
double sum = 0;
for(int i = 0; i < trainingData.length; i++) {
double[] input = new double[trainingData[i].length - 1];
int j = 0;
for(; j < trainingData[i].length - 1; j++) {
input[j] = trainingData[i][j];
}
sum += Math.pow(predict(input) - trainingData[i][j], 2);
}
double factor = 1D/(2*trainingData.length);
return factor * sum;
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder("hypothesis: ");
int i = 0;
sb.append(parameters[i++] + " + ");
for(; i < parameters.length-1; i++) {
sb.append(parameters[i] + "*x" + i + " + ");
}
sb.append(parameters[i] + "*x" + i);
sb.append("\n Feature scale: ");
for(i = 0; i < scale.length-1; i++) {
sb.append(scale[i] + " ");
}
sb.append(scale[i]);
sb.append("\n Feature means: ");
for(i = 0; i < means.length-1; i++) {
sb.append(means[i] + " ");
}
sb.append(means[i]);
sb.append("\n Cost fuction: " + costFunction());
return sb.toString();
}
public static void main(String[] args) {
final double[][] TDATA = {
{200, 2, 20000},
{300, 2, 41000},
{400, 3, 51000},
{500, 3, 61500},
{800, 4, 41000},
{900, 5, 141000}
};
GradientDescent gd = new GradientDescent();
gd.setTrainingData(TDATA);
gd.setParameters(new double[]{0D,0D,0D});
gd.setLearningRate(0.00001);
gd.train();
//System.out.println(gd);
//System.out.println("PREDICTION: " + gd.predict(new double[]{300, 2}));
}
}
编辑：
我已经更新了代码以使其更具可读性,并尝试将其映射到使用的Douglas表示法.我认为现在效果更好,但是仍有一些我不完全了解的阴暗区域.
看来,如果我有多个参数(如下面的示例,房间数量和区域),则预测与第二个参数(在本例中为区域)密切相关,并且更改第一个参数没有太大影响(房间的数量).
以下是对{2,200}的预测：
PREDICTION: 200000.00686158828
以下是对{5,200}的预测：
PREDICTION: 200003.0068315415
如您所见,两个值之间几乎没有任何区别.
我尝试将数学转换为代码时仍然存在错误吗？
这是更新的代码：
import java.util.*;
public class GradientDescent {
private double[][] trainingData;
private double[] means;
private double[] scale;
private double[] parameters;
private double learningRate;
GradientDescent() {
this.learningRate = 0D;
}
public double predict(double[] inp) {
return predict(inp, this.parameters);
}
private double predict(double[] inp, double[] parameters){
double[] features = concatenate(new double[]{1}, inp);
double prediction = 0;
for(int j = 0; j < features.length; j++) {
prediction += parameters[j] * features[j];
}
return prediction;
}
public void train(){
readjustLearningRate();
double costFunctionDelta = Math.abs(costFunction() - costFunction(iterateGradient()));
while(costFunctionDelta > 0.0000000001) {
System.out.println("Old cost function : " + costFunction());
System.out.println("New cost function : " + costFunction(iterateGradient()));
System.out.println("Delta: " + costFunctionDelta);
parameters = iterateGradient();
costFunctionDelta = Math.abs(costFunction() - costFunction(iterateGradient()));
readjustLearningRate();
}
}
private double[] iterateGradient() {
double[] nextParameters = new double[parameters.length];
// Calculate parameters for the next iteration
for(int r = 0; r < parameters.length; r++) {
nextParameters[r] = parameters[r] - learningRate * partialDerivative(r);
}
return nextParameters;
}
private double partialDerivative(int index) {
double sum = 0;
for(int i = 0; i < trainingData.length; i++) {
int indexOfResult = trainingData[i].length - 1;
double[] input = Arrays.copyOfRange(trainingData[i], 0, indexOfResult);
sum += ((predict(input) - trainingData[i][indexOfResult]) * trainingData[i][index]);
}
return sum/trainingData.length ;
}
private void readjustLearningRate() {
while(costFunction(iterateGradient()) > costFunction()) {
// If the cost function of the new parameters is higher that the current cost function
// it means the gradient is diverging and we have to adjust the learning rate
// and recalculate new parameters
System.out.print("Learning rate: " + learningRate + " is too big, readjusted to: ");
learningRate = learningRate/2;
System.out.println(learningRate);
}
// otherwise we are taking small enough steps, we have the right learning rate
}
public double[][] getTrainingData() {
return trainingData;
}
public void setTrainingData(double[][] data) {
this.trainingData = data;
this.means = new double[this.trainingData[0].length-1];
this.scale = new double[this.trainingData[0].length-1];
for(int j = 0; j < data[0].length-1; j++) {
double min = data[0][j], max = data[0][j];
double sum = 0;
for(int i = 0; i < data.length; i++) {
if(data[i][j] < min) min = data[i][j];
if(data[i][j] > max) max = data[i][j];
sum += data[i][j];
}
scale[j] = max - min;
means[j] = sum / data.length;
}
}
public double[] getParameters() {
return parameters;
}
public void setParameters(double[] parameters) {
this.parameters = parameters;
}
public double getLearningRate() {
return learningRate;
}
public void setLearningRate(double learningRate) {
this.learningRate = learningRate;
}
/** 1 m i i 2
* J(theta) = ----- * SUM( h (x ) - y )
* 2*m i=1 theta
*/
public double costFunction() {
return costFunction(this.parameters);
}
private double costFunction(double[] parameters) {
int m = trainingData.length;
double sum = 0;
for(int i = 0; i < m; i++) {
int indexOfResult = trainingData[i].length - 1;
double[] input = Arrays.copyOfRange(trainingData[i], 0, indexOfResult);
sum += Math.pow(predict(input, parameters) - trainingData[i][indexOfResult], 2);
}
double factor = 1D/(2*m);
return factor * sum;
}
private double[] normalize(double[] input) {
double[] normalized = new double[input.length];
for(int i = 0; i < input.length; i++) {
normalized[i] = (input[i] - means[i]) / scale[i];
}
return normalized;
}
private double[] concatenate(double[] a, double[] b) {
int size = a.length + b.length;
double[] concatArray = new double[size];
int index = 0;
for(double d : a) {
concatArray[index++] = d;
}
for(double d : b) {
concatArray[index++] = d;
}
return concatArray;
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder("hypothesis: ");
int i = 0;
sb.append(parameters[i++] + " + ");
for(; i < parameters.length-1; i++) {
sb.append(parameters[i] + "*x" + i + " + ");
}
sb.append(parameters[i] + "*x" + i);
sb.append("\n Feature scale: ");
for(i = 0; i < scale.length-1; i++) {
sb.append(scale[i] + " ");
}
sb.append(scale[i]);
sb.append("\n Feature means: ");
for(i = 0; i < means.length-1; i++) {
sb.append(means[i] + " ");
}
sb.append(means[i]);
sb.append("\n Cost fuction: " + costFunction());
return sb.toString();
}
public static void main(String[] args) {
final double[][] TDATA = {
//number of rooms, area, price
{2, 200, 200000},
{3, 300, 300000},
{4, 400, 400000},
{5, 500, 500000},
{8, 800, 800000},
{9, 900, 900000}
};
GradientDescent gd = new GradientDescent();
gd.setTrainingData(TDATA);
gd.setParameters(new double[]{0D, 0D, 0D});
gd.setLearningRate(0.1);
gd.train();
System.out.println(gd);
System.out.println("PREDICTION: " + gd.predict(new double[]{3, 600}));
}
}
解决方法:
看来您的起步很合理,但是将数学转换为代码时遇到了一些问题.请参阅以下数学.
我采取了一些步骤来阐明数学和算法的收敛机制.
>为了提高可读性,而不是使用括号上标来表示行,在此符号中使用了更标准的逗号分隔下标.
>尝试对累加控制变量使用零基,以匹配Java / C索引约定,而不会在数学中引入错误. (希望正确完成.)
>对课程材料进行了各种替换.
>确定发布代码中的变量名称与数学表示形式之间的映射.
在那之后,很明显,求和循环中的错误比丢失的加号更多.偏导数似乎需要重写或进行重大修改以匹配课程概念.
注意,k ＝ 0-＞ n的内部循环在所有特征上产生点积,然后在i ＝ 0-＞ m-1的循环内应用以说明每种训练情况.
所有这些都必须包含在每个迭代r中.该外部循环的循环标准不应为某个最大r值.一旦收敛充分完成,您将需要满足一些条件.
针对注释的其他注释：
由于马丁·福勒(Martin Fowler)所说的“对称鸿沟”,很难发现代码中存在的不一致之处.在这种情况下,它介于三件事之间.
>数学表示
>演讲用语
>代码中的算法
重构成员变量并从x矩阵中破坏y向量(如下所示)可能会有助于发现不一致性.
private int countMExamples;
private int countNFeatures;
private double[][] aX;
private double[] aY;
private double[] aMeans;
private double[] aScales;
private double[] aParamsTheta;
private double learnRate;
标签：machine-learning,gradient-descent,java 来源： https://codeday.me/bug/20191112/2023512.html