/// <summary>
/// Process a certain ndarray with a certain computation handler.
/// </summary>
/// <param name="array">The ndarray to process.</param>
/// <param name="handler">The computation handler to do the processing with.</param>
/// <returns>An ndarray with the processed contents of the given array (can be the same or a new one).</returns>
internal override INDArray ProcessDirect(INDArray array, IComputationHandler handler)
{
int recordLength = (int)(array.Length / array.Shape[0]);
long[] firstBufferIndices = new long[array.Shape.Length];
long[] secondBufferIndices = new long[array.Shape.Length];
_random = new Random(31415926); // fixed rng for reproducability
for (int i = 0; i < array.Shape[0]; i++)
{
int swapIndex = _random.Next((int)array.Shape[0]);
for (int y = 0; y < recordLength; y++)
{
NDArrayUtils.GetIndices(recordLength * i + y, array.Shape, array.Strides, firstBufferIndices);
NDArrayUtils.GetIndices(recordLength * swapIndex + y, array.Shape, array.Strides, secondBufferIndices);
double firstValue = array.GetValue <double>(firstBufferIndices);
double secondValue = array.GetValue <double>(secondBufferIndices);
array.SetValue(secondValue, firstBufferIndices);
array.SetValue(firstValue, secondBufferIndices);
}
}
return(array);
}
/// <inheritdoc />
internal override INDArray ProcessDirect(INDArray array, IComputationHandler handler)
{
//BTF with single feature dimension, TODO couldn't feature dimension just be flattened and ignored?
if (array.Rank != 3)
{
throw new ArgumentException($"Cannot one-hot encode ndarrays which are not of rank 3 (BTF with single feature dimension), but given ndarray was of rank {array.Rank}.");
}
INDArray encodedArray = handler.NDArray(array.Shape[0], array.Shape[1], array.Shape[2] * _valueToIndexMapping.Count);
long[] bufferIndices = new long[3];
for (long i = 0; i < array.Shape[0] * array.Shape[1]; i++)
{
bufferIndices = NDArrayUtils.GetIndices(i, array.Shape, array.Strides, bufferIndices);
object value = array.GetValue <int>(bufferIndices);
if (!_valueToIndexMapping.ContainsKey(value))
{
throw new ArgumentException($"Cannot one-hot encode unknown value {value}, value was not registered as a possible value.");
}
bufferIndices[2] = _valueToIndexMapping[value];
encodedArray.SetValue(1, bufferIndices);
}
return(encodedArray);
}
protected virtual void UpdateValues(DrawCanvas canvas)
{
double[] newVals = DrawCanvasValuesSingle(canvas);
lock (Values)
{
for (int i = 0; i < newVals.Length; i++)
{
Values.SetValue(newVals[i], NDArrayUtils.GetIndices(i, Values.Shape, Values.Strides));
}
}
}
/// <summary>
/// Permute this ndarray's data according to rearranged dimensions in place.
/// Note: Arguments are not checked for correctness (and are asssumed to be correct).
/// </summary>
/// <param name="rearrangedDimensions">The re-arranged dimensions (numbered 0 to rank - 1).</param>
/// <param name="originalShape">The original shape.</param>
/// <param name="rearrangedShape">The new, re-arranged shape.</param>
/// <param name="data">The data array.</param>
/// <param name="offset">The data array offset.</param>
/// <param name="length">The data array length.</param>
internal static void _InternalPermuteSelf(T[] data, int offset, int length, int[] rearrangedDimensions, long[] originalShape, long[] rearrangedShape)
{
// TODO optimise heavily
long[] originalStrides = NDArrayUtils.GetStrides(originalShape);
long[] rearrangedStrides = NDArrayUtils.GetStrides(rearrangedShape);
long[] bufferIndices = new long[rearrangedDimensions.Length];
BitArray traversedIndicies = new BitArray(length);
for (int i = 0; i < length; i++)
{
int currentIndex = i;
T previousValue = data[offset + currentIndex];
if (traversedIndicies[i])
{
continue;
}
do
{
NDArrayUtils.GetIndices(currentIndex, originalShape, originalStrides, bufferIndices);
bufferIndices = ArrayUtils.PermuteArray(bufferIndices, rearrangedDimensions);
int swapIndex = (int)NDArrayUtils.GetFlatIndex(rearrangedShape, rearrangedStrides, bufferIndices);
T nextPreviousValue = data[offset + swapIndex];
if (swapIndex != currentIndex)
{
data[offset + swapIndex] = previousValue;
}
previousValue = nextPreviousValue;
traversedIndicies[currentIndex] = true;
currentIndex = swapIndex;
} while (i != currentIndex && !traversedIndicies[currentIndex]);
}
}
public void Initialise(INDArray array, IComputationHandler handler, Random random)
{
if (array == null)
{
throw new ArgumentNullException(nameof(array));
}
if (handler == null)
{
throw new ArgumentNullException(nameof(handler));
}
if (random == null)
{
throw new ArgumentNullException(nameof(random));
}
long[] indices = new long[array.Rank];
for (long i = 0; i < array.Length; i++)
{
indices = NDArrayUtils.GetIndices(i, array.Shape, array.Strides, indices);
array.SetValue(GetValue(indices, array.Shape, random), indices);
}
}
public override Dictionary <string, INDArray> ExtractDirectFrom(object readData, int numberOfRecords, IComputationHandler handler)
{
// read data being null means no more data could be read so we will just pass that along
if (readData == null)
{
return(null);
}
T[][] rawRecords = (T[][])readData;
int numberOfRecordsToExtract = Math.Min(rawRecords.Length, numberOfRecords);
_logger.Debug($"Extracting {numberOfRecordsToExtract} records from reader {Reader} (requested: {numberOfRecords})...");
Dictionary <string, INDArray> namedArrays = new Dictionary <string, INDArray>();
foreach (string name in _indexMappings.Keys)
{
long[][] mappings = _indexMappings[name];
long[][] perMappingShape = new long[mappings.Length / 2][];
long[] perMappingLength = new long[mappings.Length / 2];
long[] featureShape = new long[mappings[0].Length];
for (int i = 0; i < mappings.Length; i += 2)
{
int halfIndex = i / 2;
perMappingShape[halfIndex] = new long[mappings[0].Length];
for (int y = 0; y < featureShape.Length; y++)
{
perMappingShape[halfIndex][y] = mappings[i + 1][y] - mappings[i][y];
featureShape[y] += perMappingShape[halfIndex][y];
}
perMappingLength[i / 2] = ArrayUtils.Product(perMappingShape[halfIndex]);
}
long[] shape = new long[featureShape.Length + 2];
shape[0] = numberOfRecordsToExtract;
shape[1] = 1;
Array.Copy(featureShape, 0, shape, 2, featureShape.Length);
INDArray array = handler.NDArray(shape);
long[] globalBufferIndices = new long[shape.Length];
long sectionOffset = _sectionOffsets.ContainsKey(name) ? _sectionOffsets[name] : 0L;
for (int r = 0; r < numberOfRecordsToExtract; r++)
{
T[] record = rawRecords[r];
globalBufferIndices[0] = r; //BatchTimeFeatures indexing
globalBufferIndices[1] = 0;
for (int i = 0; i < mappings.Length; i += 2)
{
long[] beginShape = mappings[i];
long[] localShape = perMappingShape[i / 2];
long[] localStrides = NDArrayUtils.GetStrides(localShape);
long[] localBufferIndices = new long[mappings[i].Length];
long length = perMappingLength[i / 2];
long beginFlatIndex = ArrayUtils.Product(beginShape);
for (int y = 0; y < length; y++)
{
localBufferIndices = NDArrayUtils.GetIndices(y, localShape, localStrides, localBufferIndices);
localBufferIndices = ArrayUtils.Add(beginShape, localBufferIndices, localBufferIndices);
Array.Copy(localBufferIndices, 0, globalBufferIndices, 2, localBufferIndices.Length);
array.SetValue(record[beginFlatIndex + y + sectionOffset], globalBufferIndices);
}
}
}
namedArrays.Add(name, array);
}
_logger.Debug($"Done extracting {numberOfRecordsToExtract} records from reader {Reader} (requested: {numberOfRecords}).");
return(namedArrays);
}
/// <summary>
/// Constructs a string representing the contents of this ndarray, formatted properly and somewhat customisable.
/// </summary>
/// <returns>A fancy string representing the contents of this ndarray.</returns>
public string ToString(ToStringElement toStringElement, int dimensionNewLine = 2, bool printSeperator = true)
{
if (toStringElement == null)
{
toStringElement = element => element.ToString();
}
int rank = Rank;
int lastIndex = rank - 1;
int openBraces = 0;
long[] indices = new long[rank];
long[] shape = Shape;
long[] strides = Strides;
long length = Length;
StringBuilder builder = new StringBuilder();
builder.Append("ndarray with shape " + ArrayUtils.ToString(shape) + ": ");
if (dimensionNewLine < rank)
{
builder.Append('\n');
}
for (long i = 0; i < length; i++)
{
indices = NDArrayUtils.GetIndices(i, shape, strides, indices);
for (int y = rank - 1; y >= 0; y--)
{
if (indices[y] == 0)
{
builder.Append('[');
openBraces++;
}
else
{
break;
}
}
builder.Append(toStringElement(Data.GetValue(i)));
if (printSeperator && indices[lastIndex] < shape[lastIndex] - 1)
{
builder.Append(", ");
}
bool requestNewLine = false;
int maxRankNewLine = rank - dimensionNewLine;
for (int y = rank - 1; y >= 0; y--)
{
if (indices[y] == shape[y] - 1)
{
builder.Append(']');
openBraces--;
if (y > 0 && indices[y - 1] != shape[y - 1] - 1)
{
builder.Append(", ");
if (!requestNewLine && y < maxRankNewLine)
{
requestNewLine = true;
}
}
}
else
{
break;
}
}
if (requestNewLine)
{
builder.Append("\n ");
for (int y = 0; y < openBraces; y++)
{
builder.Append(' ');
}
}
}
return(builder.ToString());
}
