/// <summary> /// Process a certain ndarray with a certain computation handler. /// </summary> /// <param name="array">The ndarray to process.</param> /// <param name="handler">The computation handler to do the processing with.</param> /// <returns>An ndarray with the processed contents of the given array (can be the same or a new one).</returns> internal override INDArray ProcessDirect(INDArray array, IComputationHandler handler) { int recordLength = (int)(array.Length / array.Shape[0]); long[] firstBufferIndices = new long[array.Shape.Length]; long[] secondBufferIndices = new long[array.Shape.Length]; _random = new Random(31415926); // fixed rng for reproducability for (int i = 0; i < array.Shape[0]; i++) { int swapIndex = _random.Next((int)array.Shape[0]); for (int y = 0; y < recordLength; y++) { NDArrayUtils.GetIndices(recordLength * i + y, array.Shape, array.Strides, firstBufferIndices); NDArrayUtils.GetIndices(recordLength * swapIndex + y, array.Shape, array.Strides, secondBufferIndices); double firstValue = array.GetValue <double>(firstBufferIndices); double secondValue = array.GetValue <double>(secondBufferIndices); array.SetValue(secondValue, firstBufferIndices); array.SetValue(firstValue, secondBufferIndices); } } return(array); }
/// <inheritdoc /> internal override INDArray ProcessDirect(INDArray array, IComputationHandler handler) { //BTF with single feature dimension, TODO couldn't feature dimension just be flattened and ignored? if (array.Rank != 3) { throw new ArgumentException($"Cannot one-hot encode ndarrays which are not of rank 3 (BTF with single feature dimension), but given ndarray was of rank {array.Rank}."); } INDArray encodedArray = handler.NDArray(array.Shape[0], array.Shape[1], array.Shape[2] * _valueToIndexMapping.Count); long[] bufferIndices = new long[3]; for (long i = 0; i < array.Shape[0] * array.Shape[1]; i++) { bufferIndices = NDArrayUtils.GetIndices(i, array.Shape, array.Strides, bufferIndices); object value = array.GetValue <int>(bufferIndices); if (!_valueToIndexMapping.ContainsKey(value)) { throw new ArgumentException($"Cannot one-hot encode unknown value {value}, value was not registered as a possible value."); } bufferIndices[2] = _valueToIndexMapping[value]; encodedArray.SetValue(1, bufferIndices); } return(encodedArray); }
protected virtual void UpdateValues(DrawCanvas canvas) { double[] newVals = DrawCanvasValuesSingle(canvas); lock (Values) { for (int i = 0; i < newVals.Length; i++) { Values.SetValue(newVals[i], NDArrayUtils.GetIndices(i, Values.Shape, Values.Strides)); } } }
/// <summary> /// Permute this ndarray's data according to rearranged dimensions in place. /// Note: Arguments are not checked for correctness (and are asssumed to be correct). /// </summary> /// <param name="rearrangedDimensions">The re-arranged dimensions (numbered 0 to rank - 1).</param> /// <param name="originalShape">The original shape.</param> /// <param name="rearrangedShape">The new, re-arranged shape.</param> /// <param name="data">The data array.</param> /// <param name="offset">The data array offset.</param> /// <param name="length">The data array length.</param> internal static void _InternalPermuteSelf(T[] data, int offset, int length, int[] rearrangedDimensions, long[] originalShape, long[] rearrangedShape) { // TODO optimise heavily long[] originalStrides = NDArrayUtils.GetStrides(originalShape); long[] rearrangedStrides = NDArrayUtils.GetStrides(rearrangedShape); long[] bufferIndices = new long[rearrangedDimensions.Length]; BitArray traversedIndicies = new BitArray(length); for (int i = 0; i < length; i++) { int currentIndex = i; T previousValue = data[offset + currentIndex]; if (traversedIndicies[i]) { continue; } do { NDArrayUtils.GetIndices(currentIndex, originalShape, originalStrides, bufferIndices); bufferIndices = ArrayUtils.PermuteArray(bufferIndices, rearrangedDimensions); int swapIndex = (int)NDArrayUtils.GetFlatIndex(rearrangedShape, rearrangedStrides, bufferIndices); T nextPreviousValue = data[offset + swapIndex]; if (swapIndex != currentIndex) { data[offset + swapIndex] = previousValue; } previousValue = nextPreviousValue; traversedIndicies[currentIndex] = true; currentIndex = swapIndex; } while (i != currentIndex && !traversedIndicies[currentIndex]); } }
public void Initialise(INDArray array, IComputationHandler handler, Random random) { if (array == null) { throw new ArgumentNullException(nameof(array)); } if (handler == null) { throw new ArgumentNullException(nameof(handler)); } if (random == null) { throw new ArgumentNullException(nameof(random)); } long[] indices = new long[array.Rank]; for (long i = 0; i < array.Length; i++) { indices = NDArrayUtils.GetIndices(i, array.Shape, array.Strides, indices); array.SetValue(GetValue(indices, array.Shape, random), indices); } }
public override Dictionary <string, INDArray> ExtractDirectFrom(object readData, int numberOfRecords, IComputationHandler handler) { // read data being null means no more data could be read so we will just pass that along if (readData == null) { return(null); } T[][] rawRecords = (T[][])readData; int numberOfRecordsToExtract = Math.Min(rawRecords.Length, numberOfRecords); _logger.Debug($"Extracting {numberOfRecordsToExtract} records from reader {Reader} (requested: {numberOfRecords})..."); Dictionary <string, INDArray> namedArrays = new Dictionary <string, INDArray>(); foreach (string name in _indexMappings.Keys) { long[][] mappings = _indexMappings[name]; long[][] perMappingShape = new long[mappings.Length / 2][]; long[] perMappingLength = new long[mappings.Length / 2]; long[] featureShape = new long[mappings[0].Length]; for (int i = 0; i < mappings.Length; i += 2) { int halfIndex = i / 2; perMappingShape[halfIndex] = new long[mappings[0].Length]; for (int y = 0; y < featureShape.Length; y++) { perMappingShape[halfIndex][y] = mappings[i + 1][y] - mappings[i][y]; featureShape[y] += perMappingShape[halfIndex][y]; } perMappingLength[i / 2] = ArrayUtils.Product(perMappingShape[halfIndex]); } long[] shape = new long[featureShape.Length + 2]; shape[0] = numberOfRecordsToExtract; shape[1] = 1; Array.Copy(featureShape, 0, shape, 2, featureShape.Length); INDArray array = handler.NDArray(shape); long[] globalBufferIndices = new long[shape.Length]; long sectionOffset = _sectionOffsets.ContainsKey(name) ? _sectionOffsets[name] : 0L; for (int r = 0; r < numberOfRecordsToExtract; r++) { T[] record = rawRecords[r]; globalBufferIndices[0] = r; //BatchTimeFeatures indexing globalBufferIndices[1] = 0; for (int i = 0; i < mappings.Length; i += 2) { long[] beginShape = mappings[i]; long[] localShape = perMappingShape[i / 2]; long[] localStrides = NDArrayUtils.GetStrides(localShape); long[] localBufferIndices = new long[mappings[i].Length]; long length = perMappingLength[i / 2]; long beginFlatIndex = ArrayUtils.Product(beginShape); for (int y = 0; y < length; y++) { localBufferIndices = NDArrayUtils.GetIndices(y, localShape, localStrides, localBufferIndices); localBufferIndices = ArrayUtils.Add(beginShape, localBufferIndices, localBufferIndices); Array.Copy(localBufferIndices, 0, globalBufferIndices, 2, localBufferIndices.Length); array.SetValue(record[beginFlatIndex + y + sectionOffset], globalBufferIndices); } } } namedArrays.Add(name, array); } _logger.Debug($"Done extracting {numberOfRecordsToExtract} records from reader {Reader} (requested: {numberOfRecords})."); return(namedArrays); }
/// <summary> /// Constructs a string representing the contents of this ndarray, formatted properly and somewhat customisable. /// </summary> /// <returns>A fancy string representing the contents of this ndarray.</returns> public string ToString(ToStringElement toStringElement, int dimensionNewLine = 2, bool printSeperator = true) { if (toStringElement == null) { toStringElement = element => element.ToString(); } int rank = Rank; int lastIndex = rank - 1; int openBraces = 0; long[] indices = new long[rank]; long[] shape = Shape; long[] strides = Strides; long length = Length; StringBuilder builder = new StringBuilder(); builder.Append("ndarray with shape " + ArrayUtils.ToString(shape) + ": "); if (dimensionNewLine < rank) { builder.Append('\n'); } for (long i = 0; i < length; i++) { indices = NDArrayUtils.GetIndices(i, shape, strides, indices); for (int y = rank - 1; y >= 0; y--) { if (indices[y] == 0) { builder.Append('['); openBraces++; } else { break; } } builder.Append(toStringElement(Data.GetValue(i))); if (printSeperator && indices[lastIndex] < shape[lastIndex] - 1) { builder.Append(", "); } bool requestNewLine = false; int maxRankNewLine = rank - dimensionNewLine; for (int y = rank - 1; y >= 0; y--) { if (indices[y] == shape[y] - 1) { builder.Append(']'); openBraces--; if (y > 0 && indices[y - 1] != shape[y - 1] - 1) { builder.Append(", "); if (!requestNewLine && y < maxRankNewLine) { requestNewLine = true; } } } else { break; } } if (requestNewLine) { builder.Append("\n "); for (int y = 0; y < openBraces; y++) { builder.Append(' '); } } } return(builder.ToString()); }