/// <summary>
/// Create an ndarray of a certain array (array will be COPIED into a data buffer) and shape.
/// Total shape length must be smaller or equal than the data array length.
/// </summary>
/// <param name="data">The data to use to fill this ndarray.</param>
/// <param name="shape">The shape.</param>
public ADNDArray(T[] data, params long[] shape)
{
if (data.Length < ArrayUtils.Product(shape))
{
throw new ArgumentException($"Data must contain the entire shape, but data length was {data.Length} and total shape length {ArrayUtils.Product(shape)} (shape = {ArrayUtils.ToString(shape)}).");
}
Initialise(NDArrayUtils.CheckShape(shape), NDArrayUtils.GetStrides(shape));
Data = new DataBuffer <T>(data, 0L, Length);
}
/// <summary>
/// Create a ndarray of a certain buffer and shape.
/// </summary>
/// <param name="buffer">The buffer to back this ndarray.</param>
/// <param name="shape">The shape.</param>
public ADNDArray(IDataBuffer <T> buffer, long[] shape)
{
if (buffer.Length < ArrayUtils.Product(shape))
{
throw new ArgumentException($"Buffer must contain the entire shape, but buffer length was {buffer.Length} and total shape length {ArrayUtils.Product(shape)} (shape = {ArrayUtils.ToString(shape)}).");
}
Initialise(NDArrayUtils.CheckShape(shape), NDArrayUtils.GetStrides(shape));
Data = buffer;
}
/// <inheritdoc />
public virtual INDArray ReshapeSelf(params long[] newShape)
{
if (Length != ArrayUtils.Product(newShape))
{
throw new ArgumentException($"Reshaping cannot change total ndarray length ({Length}), only array shape" +
$" (attempted change from [{string.Join(", ", Shape)}] to [{string.Join(", ", newShape)}]).");
}
Reinitialise(NDArrayUtils.CheckShape(newShape), NDArrayUtils.GetStrides(newShape));
return(this);
}
/// <summary>
/// Permute this ndarray's data according to rearranged dimensions in place.
/// Note: Arguments are not checked for correctness (and are asssumed to be correct).
/// </summary>
/// <param name="rearrangedDimensions">The re-arranged dimensions (numbered 0 to rank - 1).</param>
/// <param name="originalShape">The original shape.</param>
/// <param name="rearrangedShape">The new, re-arranged shape.</param>
/// <param name="data">The data array.</param>
/// <param name="offset">The data array offset.</param>
/// <param name="length">The data array length.</param>
internal static void _InternalPermuteSelf(T[] data, int offset, int length, int[] rearrangedDimensions, long[] originalShape, long[] rearrangedShape)
{
// TODO optimise heavily
long[] originalStrides = NDArrayUtils.GetStrides(originalShape);
long[] rearrangedStrides = NDArrayUtils.GetStrides(rearrangedShape);
long[] bufferIndices = new long[rearrangedDimensions.Length];
BitArray traversedIndicies = new BitArray(length);
for (int i = 0; i < length; i++)
{
int currentIndex = i;
T previousValue = data[offset + currentIndex];
if (traversedIndicies[i])
{
continue;
}
do
{
NDArrayUtils.GetIndices(currentIndex, originalShape, originalStrides, bufferIndices);
bufferIndices = ArrayUtils.PermuteArray(bufferIndices, rearrangedDimensions);
int swapIndex = (int)NDArrayUtils.GetFlatIndex(rearrangedShape, rearrangedStrides, bufferIndices);
T nextPreviousValue = data[offset + swapIndex];
if (swapIndex != currentIndex)
{
data[offset + swapIndex] = previousValue;
}
previousValue = nextPreviousValue;
traversedIndicies[currentIndex] = true;
currentIndex = swapIndex;
} while (i != currentIndex && !traversedIndicies[currentIndex]);
}
}
public override Dictionary <string, INDArray> ExtractDirectFrom(object readData, int numberOfRecords, IComputationHandler handler)
{
// read data being null means no more data could be read so we will just pass that along
if (readData == null)
{
return(null);
}
T[][] rawRecords = (T[][])readData;
int numberOfRecordsToExtract = Math.Min(rawRecords.Length, numberOfRecords);
_logger.Debug($"Extracting {numberOfRecordsToExtract} records from reader {Reader} (requested: {numberOfRecords})...");
Dictionary <string, INDArray> namedArrays = new Dictionary <string, INDArray>();
foreach (string name in _indexMappings.Keys)
{
long[][] mappings = _indexMappings[name];
long[][] perMappingShape = new long[mappings.Length / 2][];
long[] perMappingLength = new long[mappings.Length / 2];
long[] featureShape = new long[mappings[0].Length];
for (int i = 0; i < mappings.Length; i += 2)
{
int halfIndex = i / 2;
perMappingShape[halfIndex] = new long[mappings[0].Length];
for (int y = 0; y < featureShape.Length; y++)
{
perMappingShape[halfIndex][y] = mappings[i + 1][y] - mappings[i][y];
featureShape[y] += perMappingShape[halfIndex][y];
}
perMappingLength[i / 2] = ArrayUtils.Product(perMappingShape[halfIndex]);
}
long[] shape = new long[featureShape.Length + 2];
shape[0] = numberOfRecordsToExtract;
shape[1] = 1;
Array.Copy(featureShape, 0, shape, 2, featureShape.Length);
INDArray array = handler.NDArray(shape);
long[] globalBufferIndices = new long[shape.Length];
long sectionOffset = _sectionOffsets.ContainsKey(name) ? _sectionOffsets[name] : 0L;
for (int r = 0; r < numberOfRecordsToExtract; r++)
{
T[] record = rawRecords[r];
globalBufferIndices[0] = r; //BatchTimeFeatures indexing
globalBufferIndices[1] = 0;
for (int i = 0; i < mappings.Length; i += 2)
{
long[] beginShape = mappings[i];
long[] localShape = perMappingShape[i / 2];
long[] localStrides = NDArrayUtils.GetStrides(localShape);
long[] localBufferIndices = new long[mappings[i].Length];
long length = perMappingLength[i / 2];
long beginFlatIndex = ArrayUtils.Product(beginShape);
for (int y = 0; y < length; y++)
{
localBufferIndices = NDArrayUtils.GetIndices(y, localShape, localStrides, localBufferIndices);
localBufferIndices = ArrayUtils.Add(beginShape, localBufferIndices, localBufferIndices);
Array.Copy(localBufferIndices, 0, globalBufferIndices, 2, localBufferIndices.Length);
array.SetValue(record[beginFlatIndex + y + sectionOffset], globalBufferIndices);
}
}
}
namedArrays.Add(name, array);
}
_logger.Debug($"Done extracting {numberOfRecordsToExtract} records from reader {Reader} (requested: {numberOfRecords}).");
return(namedArrays);
}
/// <summary>
/// Create a vectorised ndarray of a certain buffer.
/// </summary>
/// <param name="buffer">The buffer to back this ndarray.</param>
public ADNDArray(IDataBuffer <T> buffer)
{
Initialise(new long[] { 1, (int)buffer.Length }, NDArrayUtils.GetStrides(1, (int)buffer.Length));
Data = buffer;
}
/// <summary>
/// Create an ndarray of a certain shape (initialised with zeros).
/// </summary>
/// <param name="shape">The shape.</param>
public ADNDArray(params long[] shape)
{
Initialise(NDArrayUtils.CheckShape(shape), NDArrayUtils.GetStrides(shape));
Data = new DataBuffer <T>(Length);
}
/// <summary>
/// Create a vectorised ndarray of a certain array (array will be COPIED into a data buffer).
/// </summary>
/// <param name="data">The data to use to fill this ndarray.</param>
public ADNDArray(T[] data)
{
Initialise(new long[] { 1, data.Length }, NDArrayUtils.GetStrides(1, data.Length));
Data = new DataBuffer <T>(data, 0L, data.Length);
}