逻辑斯谛回归代码实现

2014-10-28

https://github.com/yusugomori/DeepLearning包含了多种编程语言版本(C、C++、Python、Java、scala)的逻辑斯谛回归(或者“逻辑回归”, Logistic Regression)的实现。

我对python版本的代码略做整理和注释,最终内容如下:

# !/usr/bin/env python
# -*- coding: utf-8 -*-

'''
 Logistic Regression

 References :
   - Jason Rennie: Logistic Regression,
   http://qwone.com/~jason/writing/lr.pdf

   - DeepLearningTutorials
   https://github.com/lisa-lab/DeepLearningTutorials

'''

'''
该代码原始版本在:https://github.com/yusugomori/DeepLearning
'''

import sys
import numpy
numpy.seterr(all='ignore')


def sigmoid(x):
    return 1. / (1 + numpy.exp(-x))


def softmax(x):
    e = numpy.exp(x - numpy.max(x))  # prevent overflow;numpy.max(x)取矩阵中的最大值,结果是一个数字
    if e.ndim == 1:  # 一维空间,即e为向量
        return e / numpy.sum(e, axis=0)
    else:     # 二维空间
        return e / numpy.array([numpy.sum(e, axis=1)]).T  # axis -> 每一行求和


class LogisticRegression(object):
    def __init__(self, input, label, n_in, n_out):
        self.x = input
        self.y = label
        self.W = numpy.zeros((n_in, n_out))  # initialize W 0; 权重矩阵初始化为0,n_in->样本特征数,n_out->类别数。
        self.b = numpy.zeros(n_out)          # initialize bias 0; 偏置项初始化为0,

        # self.params = [self.W, self.b]

    def train(self, lr=0.1, input=None, L2_reg=0.00):

        # L2_reg,是L2正则化下的惩罚系数;正则化,即“惩罚”,数越大,惩罚力度越大,
        # http://sobuhu.com/ml/2012/12/29/normalization-regularization.html

        if input is not None:
            self.x = input

        # p_y_given_x = sigmoid(numpy.dot(self.x, self.W) + self.b)
        p_y_given_x = softmax(numpy.dot(self.x, self.W) + self.b) # 样本特征构成的矩阵×权重矩阵+偏置项
        d_y = self.y - p_y_given_x  # delta y

        self.W += lr * numpy.dot(self.x.T, d_y) - lr * L2_reg * self.W
        self.b += lr * numpy.mean(d_y, axis=0)

        # cost = self.negative_log_likelihood()
        # return cost

    def negative_log_likelihood(self):

        # 交叉熵:http://en.wikipedia.org/wiki/Cross_entropy

        # sigmoid_activation = sigmoid(numpy.dot(self.x, self.W) + self.b)
        sigmoid_activation = softmax(numpy.dot(self.x, self.W) + self.b)

        cross_entropy = - numpy.mean(
            numpy.sum(self.y * numpy.log(sigmoid_activation) +
            (1 - self.y) * numpy.log(1 - sigmoid_activation),
                      axis=1))

        return cross_entropy


    def predict(self, x):
        # return sigmoid(numpy.dot(x, self.W) + self.b)
        return softmax(numpy.dot(x, self.W) + self.b)


def test_lr(learning_rate=0.01, n_epochs=200):
    # training data
    x = numpy.array([[1,1,1,0,0,0],
                     [1,0,1,0,0,0],
                     [1,1,1,0,0,0],
                     [0,0,1,1,1,0],
                     [0,0,1,1,0,0],
                     [0,0,1,1,1,0],
                     [0,0,0,0,0,1],
                     [0,0,0,0,0,1]])
    y = numpy.array([[1, 0, 0],  # 类0
                     [1, 0, 0],  # 类0
                     [1, 0, 0],  # 类0
                     [0, 1, 0],  # 类1
                     [0, 1, 0],  # 类1
                     [0, 1, 0],  # 类1
                     [0, 0, 1],  # 类2
                     [0, 0, 1]]) # 类2


    # construct LogisticRegression
    classifier = LogisticRegression(input=x, label=y, n_in=6, n_out=3)

    # train
    for epoch in xrange(n_epochs):
        classifier.train(lr=learning_rate, L2_reg=1.)
        # cost = classifier.negative_log_likelihood()
        # print >> sys.stderr, 'Training epoch %d, cost is ' % epoch, cost
        learning_rate *= 0.95


    # test
    # x = numpy.array([[1, 1, 0, 0, 0, 0],
    #                  [0, 0, 0, 1, 1, 0],
    #                  [1, 1, 1, 1, 1, 0]])
    # print x.shape
    predict = classifier.predict(x)

    print predict
    print numpy.argmax(predict, axis=1) # 每一行最大数的位置
    print classifier.negative_log_likelihood()


if __name__ == "__main__":
    test_lr()

运行test_lr()函数的结果如下:

[[ 0.56100127  0.25894207  0.18005667]
 [ 0.48722535  0.30494234  0.20783232]
 [ 0.56100127  0.25894207  0.18005667]
 [ 0.25894207  0.56100127  0.18005667]
 [ 0.30494234  0.48722535  0.20783232]
 [ 0.25894207  0.56100127  0.18005667]
 [ 0.29463485  0.29463485  0.41073029]
 [ 0.29463485  0.29463485  0.41073029]]
[0 0 0 1 1 1 2 2]
1.26403157582
( 完 )