/// <summary>
/// Process a certain ndarray with a certain computation handler.
/// </summary>
/// <param name="array">The ndarray to process.</param>
/// <param name="handler">The computation handler to do the processing with.</param>
/// <returns>An ndarray with the processed contents of the given array (can be the same or a new one).</returns>
internal override INDArray ProcessDirect(INDArray array, IComputationHandler handler)
{
int recordLength = (int)(array.Length / array.Shape[0]);
long[] firstBufferIndices = new long[array.Shape.Length];
long[] secondBufferIndices = new long[array.Shape.Length];
_random = new Random(31415926); // fixed rng for reproducability
for (int i = 0; i < array.Shape[0]; i++)
{
int swapIndex = _random.Next((int)array.Shape[0]);
for (int y = 0; y < recordLength; y++)
{
NDArrayUtils.GetIndices(recordLength * i + y, array.Shape, array.Strides, firstBufferIndices);
NDArrayUtils.GetIndices(recordLength * swapIndex + y, array.Shape, array.Strides, secondBufferIndices);
double firstValue = array.GetValue <double>(firstBufferIndices);
double secondValue = array.GetValue <double>(secondBufferIndices);
array.SetValue(secondValue, firstBufferIndices);
array.SetValue(firstValue, secondBufferIndices);
}
}
return(array);
}
/// <inheritdoc />
public override void Fill(INDArray filler, INDArray arrayToFill, long[] sourceBeginIndices, long[] sourceEndIndices, long[] destinationBeginIndices, long[] destinationEndIndices)
{
IDataBuffer <float> fillerData = InternaliseArray(filler).Data;
IDataBuffer <float> arrayToFillData = InternaliseArray(arrayToFill).Data;
int sourceOffset = (int)NDArrayUtils.GetFlatIndex(filler.Shape, filler.Strides, sourceBeginIndices);
int sourceLength = (int)NDArrayUtils.GetFlatIndex(filler.Shape, filler.Strides, sourceEndIndices) - sourceOffset + 1; // +1 because end is inclusive
int destinationOffset = (int)NDArrayUtils.GetFlatIndex(arrayToFill.Shape, arrayToFill.Strides, destinationBeginIndices);
int destinationLength = (int)NDArrayUtils.GetFlatIndex(arrayToFill.Shape, arrayToFill.Strides, destinationEndIndices) - destinationOffset + 1; // same here
if (sourceLength < 0)
{
throw new ArgumentOutOfRangeException($"Source begin indices must be smaller than source end indices, but source length was {sourceLength}.");
}
if (destinationLength < 0)
{
throw new ArgumentOutOfRangeException($"Destination begin indices must be smaller than destination end indices, but destination length was {destinationLength}.");
}
if (sourceLength != destinationLength)
{
throw new ArgumentException($"Source and destination indices length must batch, but source length was {sourceLength} and destination legnth was {destinationLength}.");
}
Array.Copy(fillerData.Data, sourceOffset, arrayToFillData.Data, destinationOffset, sourceLength);
}
/// <inheritdoc />
public override INumber AsNumber(INDArray array, params long[] indices)
{
ADFloat32NDArray internalArray = InternaliseArray(array);
long flatIndex = NDArrayUtils.GetFlatIndex(array.Shape, array.Strides, indices);
return(new ADFloat32Number(DNDArray.ToDNumber(internalArray.Handle, (int)flatIndex)));
}
/// <inheritdoc />
internal override INDArray ProcessDirect(INDArray array, IComputationHandler handler)
{
//BTF with single feature dimension, TODO couldn't feature dimension just be flattened and ignored?
if (array.Rank != 3)
{
throw new ArgumentException($"Cannot one-hot encode ndarrays which are not of rank 3 (BTF with single feature dimension), but given ndarray was of rank {array.Rank}.");
}
INDArray encodedArray = handler.NDArray(array.Shape[0], array.Shape[1], array.Shape[2] * _valueToIndexMapping.Count);
long[] bufferIndices = new long[3];
for (long i = 0; i < array.Shape[0] * array.Shape[1]; i++)
{
bufferIndices = NDArrayUtils.GetIndices(i, array.Shape, array.Strides, bufferIndices);
object value = array.GetValue <int>(bufferIndices);
if (!_valueToIndexMapping.ContainsKey(value))
{
throw new ArgumentException($"Cannot one-hot encode unknown value {value}, value was not registered as a possible value.");
}
bufferIndices[2] = _valueToIndexMapping[value];
encodedArray.SetValue(1, bufferIndices);
}
return(encodedArray);
}
/// <inheritdoc />
public override Dictionary <string, INDArray> ExtractDirectFrom(object readData, int numberOfRecords, IComputationHandler handler)
{
Dictionary <string, INDArray> unprocessedNamedArrays = (Dictionary <string, INDArray>)readData;
Dictionary <string, INDArray> processedNamedArrays = new Dictionary <string, INDArray>();
foreach (string sectionName in unprocessedNamedArrays.Keys)
{
INDArray processedArray = unprocessedNamedArrays[sectionName];
if (processedArray.Shape[0] != numberOfRecords)
{
long[] beginIndices = (long[])processedArray.Shape.Clone();
long[] endIndices = (long[])processedArray.Shape.Clone();
beginIndices = NDArrayUtils.GetSliceIndicesAlongDimension(0, 0, beginIndices, copyResultShape: false, sliceEndIndex: false);
endIndices = NDArrayUtils.GetSliceIndicesAlongDimension(0, Math.Min(numberOfRecords, processedArray.Shape[0]), endIndices, copyResultShape: false, sliceEndIndex: true);
processedArray = processedArray.Slice(beginIndices, endIndices);
}
if (ProcessedSectionNames == null || ProcessedSectionNames.Contains(sectionName))
{
processedArray = ProcessDirect(processedArray, handler);
}
processedNamedArrays.Add(sectionName, processedArray);
}
return(processedNamedArrays);
}
/// <summary>
/// Create an ndarray of a certain array (array will be COPIED into a data buffer) and shape.
/// Total shape length must be smaller or equal than the data array length.
/// </summary>
/// <param name="data">The data to use to fill this ndarray.</param>
/// <param name="shape">The shape.</param>
public ADNDArray(T[] data, params long[] shape)
{
if (data.Length < ArrayUtils.Product(shape))
{
throw new ArgumentException($"Data must contain the entire shape, but data length was {data.Length} and total shape length {ArrayUtils.Product(shape)} (shape = {ArrayUtils.ToString(shape)}).");
}
Initialise(NDArrayUtils.CheckShape(shape), NDArrayUtils.GetStrides(shape));
Data = new DataBuffer <T>(data, 0L, Length);
}
protected virtual void UpdateValues(DrawCanvas canvas)
{
double[] newVals = DrawCanvasValuesSingle(canvas);
lock (Values)
{
for (int i = 0; i < newVals.Length; i++)
{
Values.SetValue(newVals[i], NDArrayUtils.GetIndices(i, Values.Shape, Values.Strides));
}
}
}
/// <summary>
/// Create a ndarray of a certain buffer and shape.
/// </summary>
/// <param name="buffer">The buffer to back this ndarray.</param>
/// <param name="shape">The shape.</param>
public ADNDArray(IDataBuffer <T> buffer, long[] shape)
{
if (buffer.Length < ArrayUtils.Product(shape))
{
throw new ArgumentException($"Buffer must contain the entire shape, but buffer length was {buffer.Length} and total shape length {ArrayUtils.Product(shape)} (shape = {ArrayUtils.ToString(shape)}).");
}
Initialise(NDArrayUtils.CheckShape(shape), NDArrayUtils.GetStrides(shape));
Data = buffer;
}
/// <inheritdoc />
public virtual INDArray ReshapeSelf(params long[] newShape)
{
if (Length != ArrayUtils.Product(newShape))
{
throw new ArgumentException($"Reshaping cannot change total ndarray length ({Length}), only array shape" +
$" (attempted change from [{string.Join(", ", Shape)}] to [{string.Join(", ", newShape)}]).");
}
Reinitialise(NDArrayUtils.CheckShape(newShape), NDArrayUtils.GetStrides(newShape));
return(this);
}
public static LayerConstruct Construct(string name, string inputAlias, params long[] shape)
{
NDArrayUtils.CheckShape(shape);
LayerConstruct construct = new LayerConstruct(name, typeof(InputLayer));
construct.ExternalInputs = new[] { inputAlias };
construct.Parameters["external_input_alias"] = inputAlias;
construct.Parameters["shape"] = shape;
construct.Parameters["size"] = (int)ArrayUtils.Product(shape);
return(construct);
}
/// <inheritdoc />
public virtual INDArray Slice(long[] beginIndices, long[] endIndices)
{
long[] slicedShape = GetSlicedShape(beginIndices, endIndices);
//we want the end indices to be inclusive for easier handling
endIndices = endIndices.Select(i => i - 1).ToArray();
long absoluteBeginOffset = NDArrayUtils.GetFlatIndex(Shape, Strides, beginIndices);
long absoluteEndOffset = NDArrayUtils.GetFlatIndex(Shape, Strides, endIndices);
long length = absoluteEndOffset - absoluteBeginOffset + 1;
return(new ADNDArray <T>(new DataBuffer <T>(Data, absoluteBeginOffset, length), slicedShape));
}
/// <summary>
/// Get a slice of this ndarray of a certain region as a new ndarray with the same underlying data.
/// </summary>
/// <param name="beginIndices">The begin indices (inclusively, where the slice should begin).</param>
/// <param name="endIndices">The end indices (exclusively, where the slice should end).</param>
/// <returns></returns>
public override INDArray Slice(long[] beginIndices, long[] endIndices)
{
long[] slicedShape = GetSlicedShape(beginIndices, endIndices);
//we want the end indices to be inclusive for easier handling
endIndices = endIndices.Select(i => i - 1).ToArray();
long absoluteBeginOffset = NDArrayUtils.GetFlatIndex(Shape, Strides, beginIndices);
long absoluteEndOffset = NDArrayUtils.GetFlatIndex(Shape, Strides, endIndices);
long length = absoluteEndOffset - absoluteBeginOffset + 1;
return(new CudaFloat32NDArray(new DNDArray(new CudaSigmaDiffDataBuffer <float>(Data, absoluteBeginOffset, length, ((SigmaDiffDataBuffer <float>)Data).BackendTag, _underlyingCudaBuffer.CudaContext), slicedShape)));
}
/// <inheritdoc />
public override void Fill <T>(T[] filler, INDArray arrayToFill, long[] destinationBeginIndices, long[] destinationEndIndices)
{
IDataBuffer <float> arrayToFillData = InternaliseArray(arrayToFill).Data;
int destinationOffset = (int)NDArrayUtils.GetFlatIndex(arrayToFill.Shape, arrayToFill.Strides, destinationBeginIndices);
int destinationLength = (int)NDArrayUtils.GetFlatIndex(arrayToFill.Shape, arrayToFill.Strides, destinationEndIndices) - destinationOffset + 1; // +1 because end is inclusive
if (destinationLength < 0)
{
throw new ArgumentOutOfRangeException($"Destination begin indices must be smaller than destination end indices, but destination length was {destinationLength}.");
}
Array.Copy(filler, 0, arrayToFillData.Data, destinationOffset, destinationLength);
}
/// <summary>
/// Permute this ndarray's data according to rearranged dimensions in place.
/// Note: Arguments are not checked for correctness (and are asssumed to be correct).
/// </summary>
/// <param name="rearrangedDimensions">The re-arranged dimensions (numbered 0 to rank - 1).</param>
/// <param name="originalShape">The original shape.</param>
/// <param name="rearrangedShape">The new, re-arranged shape.</param>
/// <param name="data">The data array.</param>
/// <param name="offset">The data array offset.</param>
/// <param name="length">The data array length.</param>
internal static void _InternalPermuteSelf(T[] data, int offset, int length, int[] rearrangedDimensions, long[] originalShape, long[] rearrangedShape)
{
// TODO optimise heavily
long[] originalStrides = NDArrayUtils.GetStrides(originalShape);
long[] rearrangedStrides = NDArrayUtils.GetStrides(rearrangedShape);
long[] bufferIndices = new long[rearrangedDimensions.Length];
BitArray traversedIndicies = new BitArray(length);
for (int i = 0; i < length; i++)
{
int currentIndex = i;
T previousValue = data[offset + currentIndex];
if (traversedIndicies[i])
{
continue;
}
do
{
NDArrayUtils.GetIndices(currentIndex, originalShape, originalStrides, bufferIndices);
bufferIndices = ArrayUtils.PermuteArray(bufferIndices, rearrangedDimensions);
int swapIndex = (int)NDArrayUtils.GetFlatIndex(rearrangedShape, rearrangedStrides, bufferIndices);
T nextPreviousValue = data[offset + swapIndex];
if (swapIndex != currentIndex)
{
data[offset + swapIndex] = previousValue;
}
previousValue = nextPreviousValue;
traversedIndicies[currentIndex] = true;
currentIndex = swapIndex;
} while (i != currentIndex && !traversedIndicies[currentIndex]);
}
}
public void Initialise(INDArray array, IComputationHandler handler, Random random)
{
if (array == null)
{
throw new ArgumentNullException(nameof(array));
}
if (handler == null)
{
throw new ArgumentNullException(nameof(handler));
}
if (random == null)
{
throw new ArgumentNullException(nameof(random));
}
long[] indices = new long[array.Rank];
for (long i = 0; i < array.Length; i++)
{
indices = NDArrayUtils.GetIndices(i, array.Shape, array.Strides, indices);
array.SetValue(GetValue(indices, array.Shape, random), indices);
}
}
public override Dictionary <string, INDArray> ExtractDirectFrom(object readData, int numberOfRecords, IComputationHandler handler)
{
// read data being null means no more data could be read so we will just pass that along
if (readData == null)
{
return(null);
}
T[][] rawRecords = (T[][])readData;
int numberOfRecordsToExtract = Math.Min(rawRecords.Length, numberOfRecords);
_logger.Debug($"Extracting {numberOfRecordsToExtract} records from reader {Reader} (requested: {numberOfRecords})...");
Dictionary <string, INDArray> namedArrays = new Dictionary <string, INDArray>();
foreach (string name in _indexMappings.Keys)
{
long[][] mappings = _indexMappings[name];
long[][] perMappingShape = new long[mappings.Length / 2][];
long[] perMappingLength = new long[mappings.Length / 2];
long[] featureShape = new long[mappings[0].Length];
for (int i = 0; i < mappings.Length; i += 2)
{
int halfIndex = i / 2;
perMappingShape[halfIndex] = new long[mappings[0].Length];
for (int y = 0; y < featureShape.Length; y++)
{
perMappingShape[halfIndex][y] = mappings[i + 1][y] - mappings[i][y];
featureShape[y] += perMappingShape[halfIndex][y];
}
perMappingLength[i / 2] = ArrayUtils.Product(perMappingShape[halfIndex]);
}
long[] shape = new long[featureShape.Length + 2];
shape[0] = numberOfRecordsToExtract;
shape[1] = 1;
Array.Copy(featureShape, 0, shape, 2, featureShape.Length);
INDArray array = handler.NDArray(shape);
long[] globalBufferIndices = new long[shape.Length];
long sectionOffset = _sectionOffsets.ContainsKey(name) ? _sectionOffsets[name] : 0L;
for (int r = 0; r < numberOfRecordsToExtract; r++)
{
T[] record = rawRecords[r];
globalBufferIndices[0] = r; //BatchTimeFeatures indexing
globalBufferIndices[1] = 0;
for (int i = 0; i < mappings.Length; i += 2)
{
long[] beginShape = mappings[i];
long[] localShape = perMappingShape[i / 2];
long[] localStrides = NDArrayUtils.GetStrides(localShape);
long[] localBufferIndices = new long[mappings[i].Length];
long length = perMappingLength[i / 2];
long beginFlatIndex = ArrayUtils.Product(beginShape);
for (int y = 0; y < length; y++)
{
localBufferIndices = NDArrayUtils.GetIndices(y, localShape, localStrides, localBufferIndices);
localBufferIndices = ArrayUtils.Add(beginShape, localBufferIndices, localBufferIndices);
Array.Copy(localBufferIndices, 0, globalBufferIndices, 2, localBufferIndices.Length);
array.SetValue(record[beginFlatIndex + y + sectionOffset], globalBufferIndices);
}
}
}
namedArrays.Add(name, array);
}
_logger.Debug($"Done extracting {numberOfRecordsToExtract} records from reader {Reader} (requested: {numberOfRecords}).");
return(namedArrays);
}
/// <summary>
/// Create an ndarray of a certain shape (initialised with zeros).
/// </summary>
/// <param name="shape">The shape.</param>
public ADNDArray(params long[] shape)
{
Initialise(NDArrayUtils.CheckShape(shape), NDArrayUtils.GetStrides(shape));
Data = new DataBuffer <T>(Length);
}
/// <inheritdoc />
public TOther GetValue <TOther>(params long[] indices)
{
return(Data.GetValueAs <TOther>(NDArrayUtils.GetFlatIndex(Shape, Strides, indices)));
}
/// <inheritdoc />
public void SetValue <TOther>(TOther value, params long[] indices)
{
Data.SetValue((T)Convert.ChangeType(value, Data.Type.UnderlyingType), NDArrayUtils.GetFlatIndex(Shape, Strides, indices));
}
public CudaFloat32NDArray(CudaSigmaDiffDataBuffer <float> buffer, long[] shape) : this(new DNDArray(buffer, NDArrayUtils.CheckShape(shape)))
{
}
public ADNDFloat32Array(long backendTag, IDataBuffer <float> buffer, long[] shape) : this(new DNDArray(new SigmaDiffDataBuffer <float>(buffer, 0, buffer.Length, backendTag), NDArrayUtils.CheckShape(shape)))
{
}
public ADNDFloat32Array(long backendTag, float[] data, params long[] shape) : this(new DNDArray(new SigmaDiffDataBuffer <float>(data, backendTag), NDArrayUtils.CheckShape(shape)))
{
}
/// <summary>
/// Create a vectorised ndarray of a certain array (array will be COPIED into a data buffer).
/// </summary>
/// <param name="data">The data to use to fill this ndarray.</param>
public ADNDArray(T[] data)
{
Initialise(new long[] { 1, data.Length }, NDArrayUtils.GetStrides(1, data.Length));
Data = new DataBuffer <T>(data, 0L, data.Length);
}
public ADFloat32NDArray(long backendTag, params long[] shape) : this(new DNDArray(new SigmaDiffDataBuffer <float>(ArrayUtils.Product(shape), backendTag), NDArrayUtils.CheckShape(shape)))
{
}
/// <summary>
/// Constructs a string representing the contents of this ndarray, formatted properly and somewhat customisable.
/// </summary>
/// <returns>A fancy string representing the contents of this ndarray.</returns>
public string ToString(ToStringElement toStringElement, int dimensionNewLine = 2, bool printSeperator = true)
{
if (toStringElement == null)
{
toStringElement = element => element.ToString();
}
int rank = Rank;
int lastIndex = rank - 1;
int openBraces = 0;
long[] indices = new long[rank];
long[] shape = Shape;
long[] strides = Strides;
long length = Length;
StringBuilder builder = new StringBuilder();
builder.Append("ndarray with shape " + ArrayUtils.ToString(shape) + ": ");
if (dimensionNewLine < rank)
{
builder.Append('\n');
}
for (long i = 0; i < length; i++)
{
indices = NDArrayUtils.GetIndices(i, shape, strides, indices);
for (int y = rank - 1; y >= 0; y--)
{
if (indices[y] == 0)
{
builder.Append('[');
openBraces++;
}
else
{
break;
}
}
builder.Append(toStringElement(Data.GetValue(i)));
if (printSeperator && indices[lastIndex] < shape[lastIndex] - 1)
{
builder.Append(", ");
}
bool requestNewLine = false;
int maxRankNewLine = rank - dimensionNewLine;
for (int y = rank - 1; y >= 0; y--)
{
if (indices[y] == shape[y] - 1)
{
builder.Append(']');
openBraces--;
if (y > 0 && indices[y - 1] != shape[y - 1] - 1)
{
builder.Append(", ");
if (!requestNewLine && y < maxRankNewLine)
{
requestNewLine = true;
}
}
}
else
{
break;
}
}
if (requestNewLine)
{
builder.Append("\n ");
for (int y = 0; y < openBraces; y++)
{
builder.Append(' ');
}
}
}
return(builder.ToString());
}
/// <summary>
/// Create a vectorised ndarray of a certain buffer.
/// </summary>
/// <param name="buffer">The buffer to back this ndarray.</param>
public ADNDArray(IDataBuffer <T> buffer)
{
Initialise(new long[] { 1, (int)buffer.Length }, NDArrayUtils.GetStrides(1, (int)buffer.Length));
Data = buffer;
}