org.retro.neural
Class BPLayer

java.lang.Object
  org.retro.neural.BPLayer

public class BPLayer

extends java.lang.Object

Field Summary

protected SNVector	a
protected SNVector	a1
protected SNVector	b
protected java.util.Stack	bptt_ah
protected SNVector	bptt_db
protected SNVector	bptt_e1
protected SNVector	d
protected SNVector	db
protected SNVector	db1
protected java.util.Vector	dly
protected SNVector	e
protected java.util.Vector	in
protected boolean	marked
protected org.retro.neural.BPNetwork	net
protected java.util.Vector	out
protected SNMatrix	p
protected SNMatrix	sqrerrs

Constructor Summary

BPLayer(int siz)
Creates a neural network layer.

Method Summary

double	actdev(double xact) Applies first derivative of layer's activation function to activation on unit (not net input).
double	actfun(double xnet) Applies layer's activation function to net input.
protected void	actinit()
protected void	actinit(double v)
double[]	activate(BPLayer src, double[] clmp) Activates this layer by clamping activations on another and running a forward pass.
protected void	activate(BPLayer src, SNVector clmp)
protected void	activate(int patno)
void	attach(double[][] pattern) Attaches pattern to layer as input or target.
protected void	backprop(int patno)
void	batch(double eta, double mu) Runs one step of back-propagation in batch mode.
void	batch(int nep, double eta, double mu) Same as batch(nep, eta, mu, report), but with error reporting every second.
void	batch(int nep, double eta, double mu, int report) Trains all layer's in this layer's network, using back-propagation in batch mode.
protected void	bptt_delayerr()
protected void	bptt_dwadd()
protected void	bptt_dwavg(int patlen)
protected void	bptt_dwreset()
protected void	bptt_errinit()
protected void	bptt_geta()
protected void	bptt_inita()
protected void	bptt_saverr()
protected void	bptt_seta()
void	bpttPattern() Steps through one pattern for Back-Prop Thru Time.
void	bpttPattern(int n) Steps through one pattern for Back-Prop Thru Time.
void	bpttResetEta() Resets current weight- and bias-changes for Back-Prop-Through-Time.
void	bpttUpdate(double eta, double mu, int npat) Updates weights and bias on layer using the Delta Rule, for Back-Prop-Through-Time.
void	connect(BPLayer from) Makes a normal (full, Sigma) connection to this layer from another layer.
void	connect(BPLayer from1, BPLayer from2) Makes a Sigma-Pi connection to this layer from two other layers.
void	delay(BPLayer from) Same as delay(from, weight), with weight = 1.0.
void	delay(BPLayer from, double weight) Makes one-to-one time-delay connection to this layer from the specified layer, using the specified connection strength.
protected void	delayact()
double	dontCare() Returns out-of-bounds value for don't-care condition.
protected void	dwadd()
protected void	dwavg(int npat)
protected void	dwreset()
protected void	dwset()
protected boolean	emptyWeights()
protected void	errreset()
double[]	getBias() Returns the bias on this layer.
double	getMaxError() Returns error of maximum magnitude on layer over all training patterns.
double	getRMSError() Returns Root-Mean-Squared error on layer over all training patterns.
double[][]	getSquaredErrors() Returns squared errors on layer over all training patterns.
double[][]	getWeights(BPLayer from) Returns the weights on this layer from another layer (Sigma connection), as a 2D array.
double[][][]	getWeights(BPLayer from1, BPLayer from2) Returns the weights on this layer from two other layers (Sigma-Pi connection), as an array of 2D arrays.
protected boolean	hasPattern()
protected boolean	isOutput()
protected void	mark()
protected double	maxerr(int patlen)
protected void	mureset()
protected boolean	nosqrerrs()
void	online(int nep, double eta, double mu) Same as online(nep, eta, mu, report), but with error report every second.
void	online(int nep, double eta, double mu, int report) Trains all layers in this layer's network, using back-propagation in on-line mode.
protected int	patlen()
void	randomize() Same as randomize(seed), with arbitrary seed.
void	randomize(long seed) Randomizes biases and weights on this layer.
void	randomize(java.util.Random rand) Randomizes biases and weights on this layer to values normally distributed around zero.
static void	reportValue(int iter, int maxit, int report, double value, java.io.PrintStream stream) Reports a value in a friendly way.
void	resetMu() Resets previous weight- and bias- changes.
protected double	rmserr(int patlen)
void	setBias(double[] v) Sets bias on this layer to the values in vector.
void	setWeights(BPLayer from1, BPLayer from2, double[][][] w) Sets weights on this layer from two others (Sigma-Pi connection) to values in array of 2D arrays.
void	setWeights(BPLayer from, double[][] w) Sets weights on this layer from another (Sigma connection) to values in 2D array.
protected int	size()
protected void	sqrerrinit(int patlen)
double[][]	test() Tests a (trained) network layer.
double[][]	test(int n, double a) Tests a (recurrent) network layer without any input.
protected void	unmark()
protected void	update(double eta, double mu, int npat)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

a

protected SNVector a

a1

protected SNVector a1

b

protected SNVector b

db

protected SNVector db

db1

protected SNVector db1

e

protected SNVector e

d

protected SNVector d

p

protected SNMatrix p

bptt_db

protected SNVector bptt_db

bptt_e1

protected SNVector bptt_e1

bptt_ah

protected java.util.Stack bptt_ah

marked

protected boolean marked

net

protected org.retro.neural.BPNetwork net

in

protected java.util.Vector in

out

protected java.util.Vector out

dly

protected java.util.Vector dly

sqrerrs

protected SNMatrix sqrerrs

Constructor Detail

BPLayer

public BPLayer(int siz)

Creates a neural network layer. Activation function for non-input layers is the logistic sigmoid f(x) = 1/(1+exp(-x)), whose first derivative is f(x)*(1-f(x)). These defaults can be overridden by subclassing with a class that defines the actfun and actdev methods.

Parameters:

siz - number of units in the layer

Method Detail

actfun

public double actfun(double xnet)

Applies layer's activation function to net input. Override this method to get different activation functions.

Parameters:

xnet - net input to activation function

Returns:

activations

actdev

public double actdev(double xact)

Applies first derivative of layer's activation function to activation on unit (not net input). Override this method to get different activation functions.

Parameters:

xact - activation on unit

Returns:

first deriviative of activation

dontCare

public double dontCare()

Returns out-of-bounds value for don't-care condition.

Returns:

don't-care value

randomize

public void randomize(long seed)

Randomizes biases and weights on this layer. Creates random weights and biases using specified random seed, supporting reproducible experiments.

Parameters:

seed - seed for random number generator

randomize

public void randomize()

Same as randomize(seed), with arbitrary seed.

randomize

public void randomize(java.util.Random rand)

Randomizes biases and weights on this layer to values normally distributed around zero. Creates random weights and biases using specified random number generator, supporting reproducible experiments.

setWeights

public void setWeights(BPLayer from,
                       double[][] w)
                throws org.retro.neural.BPConnectException,
                       org.retro.neural.BPSizeException

Sets weights on this layer from another (Sigma connection) to values in 2D array. Array should have M rows and N columns, where M is the number of units in this layer, and N is the number in the connected-from layer.

Parameters:

from - the layer connected from

w - the matrix of weights

Throws:

BPConnectException - if there is no such conenction

BPSizeException - if array size mismatches layer sizes

setWeights

public void setWeights(BPLayer from1,
                       BPLayer from2,
                       double[][][] w)
                throws org.retro.neural.BPConnectException,
                       org.retro.neural.BPSizeException

Sets weights on this layer from two others (Sigma-Pi connection) to values in array of 2D arrays. The array should have K elements of M rows and N columns, where K is the number of units in this layer, M is the number of units in the first connected-from layer, and N is the number in the second connected-from layer.

Parameters:

from1 - layer connected from

from2 - layer connected from

w - array of weights

Throws:

BPConnectException - if there is no such conenction

BPSizeException - if array size mismatches layer sizes

setBias

public void setBias(double[] v)
             throws org.retro.neural.BPSizeException

Sets bias on this layer to the values in vector.

Parameters:

v - the vector of bias values

Throws:

BPSizeException - if vector size doesn't equal layer size

getWeights

public double[][] getWeights(BPLayer from)
                      throws org.retro.neural.BPConnectException,
                             org.retro.neural.BPInitException

Returns the weights on this layer from another layer (Sigma connection), as a 2D array. The array will have M rows and N columns, where M is the number of units in this layer, and N is the number in the connected-from layer.

Parameters:

from - the layer connected from

Returns:

connection weights from that layer to this

Throws:

BPConnectException - if there is no such conenction

BPInitException - some network weights are uninitialized

getWeights

public double[][][] getWeights(BPLayer from1,
                               BPLayer from2)
                        throws org.retro.neural.BPConnectException,
                               org.retro.neural.BPInitException

Returns the weights on this layer from two other layers (Sigma-Pi connection), as an array of 2D arrays. The array will have K elements of M rows and N columns, where K is the number of units in this layer, M is the number of units in the first connected-from layer, and N is the number in the second connected-from layer.

Parameters:

from1 - layer connected from

from2 - layer connected from

Returns:

connection weights from those layers to this

Throws:

BPConnectException - if there is no such conenction

BPInitException - some network weights are uninitialized

getBias

public double[] getBias()
                 throws org.retro.neural.BPInitException

Returns the bias on this layer.

Returns:

vector of bias values

Throws:

BPInitException - if the bias is uninitialized

getRMSError

public double getRMSError()
                   throws org.retro.neural.BPInitException,
                          org.retro.neural.BPSizeException

Returns Root-Mean-Squared error on layer over all training patterns. Network weights must be set before this method is invoked, or an exception will be raised. The sizes of the input and output patterns must be be compatible, or an exception will be raised.

Returns:

RMS error

Throws:

BPInitException - if the weights are uninitialized

BPSizeException - if number of input and output patterns differs

getMaxError

public double getMaxError()
                   throws org.retro.neural.BPInitException,
                          org.retro.neural.BPSizeException

Returns error of maximum magnitude on layer over all training patterns. Network.init must be called between network creation and this method, or an exception will be raised. The number of input and output patterns must be be equal, or an exception will be raised.

Returns:

maximum error

Throws:

BPInitException - if some network weights are uninitialized

BPSizeException - if number of input and output patterns differs

getSquaredErrors

public double[][] getSquaredErrors()
                            throws org.retro.neural.BPInitException,
                                   org.retro.neural.BPSizeException

Returns squared errors on layer over all training patterns. Network.init must be called between network creation and this method, or an exception will be raised. The number of input and output patterns must be be equal, or an exception will be raised.

Returns:

array of squared errors

Throws:

BPInitException - if some network weights are uninitialized

BPSizeException - if number of input and output patterns differs

connect

public void connect(BPLayer from)

Makes a normal (full, Sigma) connection to this layer from another layer. The activations a_j on this layer is computed as a_j = f ( &sum_ia_iw_ji + b_j ), where a_i are the activations on the other layer, w are the weights between the layers, b are the biases on this layer, and f is the layer's activation function. Weights and biases can be initialized via the randomize or setWeights and setBias methods.

Parameters:

from - layer to connect from

connect

public void connect(BPLayer from1,
                    BPLayer from2)

Makes a Sigma-Pi connection to this layer from two other layers. The activation on this layer is computed as a_k = f ( &sum_i&sum_j a_ia_j w_kji + b_k ), where a_i and a_j are the activations on the other two layers w are the weights between the layers, b are the biases on this layer, and f is the layer's activation function. Weights and biases can be initialized via the randomize or setWeights and setBias methods.

Parameters:

from1 - layer to connect from

from2 - layer to connect from

attach

public void attach(double[][] pattern)
            throws org.retro.neural.BPSizeException

Attaches pattern to layer as input or target. Width (number of columns) of pattern must match layer size. Pattern matrices should contain one pattern per row. For example, A = {{0,0}, {0,1}, {1,0}, {1,1}}, B = {{0}, {1}, {1}, {0}} are the patterns for learning B = XOR(A). Don't-care conditions are encoded as out-of-bound values. For example, B = {{0}, {1}, {1}, {2}} would encode XOR where you don't care about the output B when A = {1,1}.

Parameters:

pattern - pattern to attach

Throws:

BPSizeException - on width/size mismatch

batch

public void batch(int nep,
                  double eta,
                  double mu,
                  int report)
           throws org.retro.neural.BPInitException,
                  org.retro.neural.BPSizeException

Trains all layer's in this layer's network, using back-propagation in batch mode. This means that the errors are computed over all patterns, and then averaged to compute the weight- and bias- changes. The lengths of any patterns attached to layers and output patterns must be be compatible, or an exception will be raised.

Parameters:

nep - number of epochs for training

eta - learning rate

mu - momentum

report - number of generations between error reports

Throws:

BPInitException - if some network weights are uninitialized

BPSizeException - if lengths of patterns differ

batch

public void batch(int nep,
                  double eta,
                  double mu)
           throws org.retro.neural.BPInitException,
                  org.retro.neural.BPSizeException

Same as batch(nep, eta, mu, report), but with error reporting every second.

Parameters:

nep - number of epochs for training

eta - learning rate

mu - momentum

Throws:

BPInitException - if weights hav not been initialized

BPSizeException - if number of input and output patterns differs

batch

public void batch(double eta,
                  double mu)
           throws org.retro.neural.BPInitException,
                  org.retro.neural.BPSizeException

Runs one step of back-propagation in batch mode.

Parameters:

eta - learning rate

mu - momentum

Throws:

BPInitException - if some network weights haven't been initialized

BPSizeException - if number of input and output patterns differs

online

public void online(int nep,
                   double eta,
                   double mu,
                   int report)
            throws org.retro.neural.BPInitException,
                   org.retro.neural.BPSizeException

Trains all layers in this layer's network, using back-propagation in on-line mode. This means that the weights and biases are updated after the presentation of each pattern. The lengths of any patterns attached to layers and output patterns must be be compatible, or an exception will be raised.

Parameters:

nep - number of passes through data

eta - learning rate

mu - momentum

report - number of generations between error reports

Throws:

BPInitException - if some network weights are uninitialized

BPSizeException - if lengths of patterns differ

online

public void online(int nep,
                   double eta,
                   double mu)
            throws org.retro.neural.BPInitException,
                   org.retro.neural.BPSizeException

Same as online(nep, eta, mu, report), but with error report every second.

Parameters:

nep - number of epochs for training

eta - learning rate

mu - momentum

Throws:

BPInitException - if weights hav not been initialized

BPSizeException - if number of input and output patterns differs

test

public double[][] test()
                throws org.retro.neural.BPInitException,
                       org.retro.neural.BPSizeException

Tests a (trained) network layer.

Returns:

output patterns

Throws:

BPInitException - if some network weights are uninitialized

BPSizeException - if number of input and output patterns differs

test

public double[][] test(int n,
                       double a)
                throws org.retro.neural.BPInitException,
                       org.retro.neural.BPSizeException

Tests a (recurrent) network layer without any input.

Parameters:

n - number of steps to run

a - intial activation value

Returns:

output pattern on layer

Throws:

BPInitException - if some network weights are uninitialized

BPSizeException - if number of input and output patterns differs

delay

public void delay(BPLayer from,
                  double weight)
           throws org.retro.neural.BPSizeException

Makes one-to-one time-delay connection to this layer from the specified layer, using the specified connection strength. This means that this layer's activations will come from a scalar multiple of the activations of the other layer on the previous time step. On the first time step these activations are are defined be the average value of this layer's activation function (e.g., 0.5 for the logistic simgoid squashing function). If the layers have different sizes, and exception will be raised.

Parameters:

from - layer to delay from

weight - connection strength

Throws:

BPSizeException - if layers have different sizes

delay

public void delay(BPLayer from)

Same as delay(from, weight), with weight = 1.0.

Parameters:

from - layer to delay from

resetMu

public void resetMu()

Resets previous weight- and bias- changes. This is only necessary when running one epoch at a time.

bpttResetEta

public void bpttResetEta()

Resets current weight- and bias-changes for Back-Prop-Through-Time.

bpttUpdate

public void bpttUpdate(double eta,
                       double mu,
                       int npat)
                throws org.retro.neural.BPMomentumException

Updates weights and bias on layer using the Delta Rule, for Back-Prop-Through-Time.

Parameters:

eta - learning rate

mu - momentum

npat - total number of patterns

Throws:

BPMomentumException - if no momentum has been set on the layer

activate

public double[] activate(BPLayer src,
                         double[] clmp)

Activates this layer by clamping activations on another and running a forward pass. Clamped activations are temporary and do not affect patterns set by attach.

Parameters:

src - "source" layer to clamp

clmp - vector of clamping values

Returns:

activation of this layer

Throws:

BPConnectException - if there is no path between the layers

BPInitException - if some network weights are uninitialized

bpttPattern

public void bpttPattern(int n)
                 throws org.retro.neural.BPInitException,
                        org.retro.neural.BPSizeException,
                        java.lang.IllegalArgumentException

Steps through one pattern for Back-Prop Thru Time. Implements a restricted version of BPTT, specified by the number of ticks in the argument.

Parameters:

n - number of ticks from end of pattern

Throws:

BPInitException - if some network weights are uninitialized

BPSizeException - if lengths of patterns differ

java.lang.IllegalArgumentException - if n < 0 or n >= current pattern length

bpttPattern

public void bpttPattern()
                 throws org.retro.neural.BPInitException,
                        org.retro.neural.BPSizeException

Steps through one pattern for Back-Prop Thru Time. Implements the full BPTT algorithm on the pattern. Same as bptt(n), with n = current pattern length - 1.

Throws:

BPInitException - if some network weights are uninitialized

BPSizeException - if lengths of patterns differ

reportValue

public static void reportValue(int iter,
                               int maxit,
                               int report,
                               double value,
                               java.io.PrintStream stream)

Reports a value in a friendly way. This is useful for reporting RMS and maximum error when training one epoch at a time.

Parameters:

iter - iteration number

maxit - maximum number of iterations

report - reporting interval

value - value to report

stream - print stream that reports value

activate

protected void activate(BPLayer src,
                        SNVector clmp)
                 throws org.retro.neural.BPConnectException,
                        org.retro.neural.BPInitException

Throws:

org.retro.neural.BPConnectException

org.retro.neural.BPInitException

activate

protected void activate(int patno)

mureset

protected void mureset()

dwreset

protected void dwreset()

update

protected void update(double eta,
                      double mu,
                      int npat)
               throws org.retro.neural.BPMomentumException

Throws:

org.retro.neural.BPMomentumException

size

protected int size()

patlen

protected int patlen()

hasPattern

protected boolean hasPattern()

isOutput

protected boolean isOutput()

dwadd

protected void dwadd()

dwset

protected void dwset()

dwavg

protected void dwavg(int npat)

emptyWeights

protected boolean emptyWeights()

backprop

protected void backprop(int patno)

bptt_inita

protected void bptt_inita()

bptt_seta

protected void bptt_seta()

bptt_geta

protected void bptt_geta()

bptt_dwreset

protected void bptt_dwreset()

bptt_dwadd

protected void bptt_dwadd()

bptt_dwavg

protected void bptt_dwavg(int patlen)

bptt_delayerr

protected void bptt_delayerr()

bptt_saverr

protected void bptt_saverr()

bptt_errinit

protected void bptt_errinit()

actinit

protected void actinit()

actinit

protected void actinit(double v)

sqrerrinit

protected void sqrerrinit(int patlen)

delayact

protected void delayact()

errreset

protected void errreset()

mark

protected void mark()

unmark

protected void unmark()

rmserr

protected double rmserr(int patlen)

maxerr

protected double maxerr(int patlen)

nosqrerrs

protected boolean nosqrerrs()