/// <summary>
/// Process a certain ndarray with a certain computation handler.
/// </summary>
/// <param name="array">The ndarray to process.</param>
/// <param name="handler">The computation handler to do the processing with.</param>
/// <returns>An ndarray with the processed contents of the given array (can be the same or a new one).</returns>
internal override INDArray ProcessDirect(INDArray array, IComputationHandler handler)
{
int recordLength = (int)(array.Length / array.Shape[0]);
long[] firstBufferIndices = new long[array.Shape.Length];
long[] secondBufferIndices = new long[array.Shape.Length];
_random = new Random(31415926); // fixed rng for reproducability
for (int i = 0; i < array.Shape[0]; i++)
{
int swapIndex = _random.Next((int)array.Shape[0]);
for (int y = 0; y < recordLength; y++)
{
NDArrayUtils.GetIndices(recordLength * i + y, array.Shape, array.Strides, firstBufferIndices);
NDArrayUtils.GetIndices(recordLength * swapIndex + y, array.Shape, array.Strides, secondBufferIndices);
double firstValue = array.GetValue <double>(firstBufferIndices);
double secondValue = array.GetValue <double>(secondBufferIndices);
array.SetValue(secondValue, firstBufferIndices);
array.SetValue(firstValue, secondBufferIndices);
}
}
return(array);
}
public override Dictionary <string, INDArray> ExtractDirectFrom(object readData, int numberOfRecords, IComputationHandler handler)
{
//read data being null indicates that nothing could be read so we can't extract anything either
if (readData == null)
{
return(null);
}
string[][] lineParts = (string[][])readData;
int numberOfRecordsToExtract = Math.Min(lineParts.Length, numberOfRecords);
_logger.Debug($"Extracting {numberOfRecordsToExtract} records from reader {Reader} (requested: {numberOfRecords})...");
Dictionary <string, INDArray> namedArrays = new Dictionary <string, INDArray>();
foreach (string name in NamedColumnIndexMapping.Keys)
{
IList <int> mappings = NamedColumnIndexMapping[name];
INDArray array = handler.NDArray(numberOfRecordsToExtract, 1, mappings.Count);
TypeConverter converter = TypeDescriptor.GetConverter(typeof(double));
for (int i = 0; i < numberOfRecordsToExtract; i++)
{
for (int y = 0; y < mappings.Count; y++)
{
int column = mappings[y];
string value = lineParts[i][column];
try
{
if (_columnValueMappings.ContainsKey(column) && _columnValueMappings[column].ContainsKey(value))
{
array.SetValue(_columnValueMappings[column][value], i, 0, y);
}
else
{
//double.Parse(value, new CultureInfo("en-GB"))
array.SetValue(converter.ConvertFromString(value), i, 0, y);
}
}
catch (NotSupportedException)
{
throw new FormatException($"Cannot convert value \"{value}\" of type {value.GetType()} to double for further processing (are you missing a column value mapping?).");
}
}
}
namedArrays.Add(name, array);
}
_logger.Debug($"Done extracting {numberOfRecordsToExtract} records from reader {Reader} (requested: {numberOfRecords}).");
return(namedArrays);
}
/// <inheritdoc />
internal override INDArray ProcessDirect(INDArray array, IComputationHandler handler)
{
//BTF with single feature dimension, TODO couldn't feature dimension just be flattened and ignored?
if (array.Rank != 3)
{
throw new ArgumentException($"Cannot one-hot encode ndarrays which are not of rank 3 (BTF with single feature dimension), but given ndarray was of rank {array.Rank}.");
}
INDArray encodedArray = handler.NDArray(array.Shape[0], array.Shape[1], array.Shape[2] * _valueToIndexMapping.Count);
long[] bufferIndices = new long[3];
for (long i = 0; i < array.Shape[0] * array.Shape[1]; i++)
{
bufferIndices = NDArrayUtils.GetIndices(i, array.Shape, array.Strides, bufferIndices);
object value = array.GetValue <int>(bufferIndices);
if (!_valueToIndexMapping.ContainsKey(value))
{
throw new ArgumentException($"Cannot one-hot encode unknown value {value}, value was not registered as a possible value.");
}
bufferIndices[2] = _valueToIndexMapping[value];
encodedArray.SetValue(1, bufferIndices);
}
return(encodedArray);
}
private void InitField()
{
if (_field == null)
{
_field = new TicTacToePlayer[_buttons.GetLength(0), _buttons.GetLength(1)];
}
for (int i = 0; i < _field.GetLength(0); i++)
{
for (int j = 0; j < _field.GetLength(1); j++)
{
SetIndexFast(i, j, TicTacToePlayer.None);
}
}
for (int i = 0; i < _field.GetLength(0) * _field.GetLength(1); i++)
{
Field.SetValue(MapToInt(TicTacToePlayer.None), 0, i);
}
OnFieldChange();
}
internal override INDArray ProcessDirect(INDArray array, IComputationHandler handler)
{
double outputRange = MaxOutputValue - MinOutputValue;
for (int i = 0; i < array.Shape[0]; i++)
{
for (int j = 0; j < array.Shape[1]; j++)
{
foreach (int index in PerIndexMinMaxMappings.Keys)
{
double[] minMaxRange = PerIndexMinMaxMappings[index];
double value = array.GetValue <double>(i, j, index);
value = (value - minMaxRange[0]) / minMaxRange[2];
value = (value + MinOutputValue) * outputRange;
array.SetValue(value, i, j, index);
}
}
}
return(array);
}
public void Initialise(INDArray array, IComputationHandler handler, Random random)
{
if (array == null)
{
throw new ArgumentNullException(nameof(array));
}
if (handler == null)
{
throw new ArgumentNullException(nameof(handler));
}
if (random == null)
{
throw new ArgumentNullException(nameof(random));
}
long[] indices = new long[array.Rank];
for (long i = 0; i < array.Length; i++)
{
indices = NDArrayUtils.GetIndices(i, array.Shape, array.Strides, indices);
array.SetValue(GetValue(indices, array.Shape, random), indices);
}
}
public override Dictionary <string, INDArray> ExtractDirectFrom(object readData, int numberOfRecords, IComputationHandler handler)
{
// read data being null means no more data could be read so we will just pass that along
if (readData == null)
{
return(null);
}
T[][] rawRecords = (T[][])readData;
int numberOfRecordsToExtract = Math.Min(rawRecords.Length, numberOfRecords);
_logger.Debug($"Extracting {numberOfRecordsToExtract} records from reader {Reader} (requested: {numberOfRecords})...");
Dictionary <string, INDArray> namedArrays = new Dictionary <string, INDArray>();
foreach (string name in _indexMappings.Keys)
{
long[][] mappings = _indexMappings[name];
long[][] perMappingShape = new long[mappings.Length / 2][];
long[] perMappingLength = new long[mappings.Length / 2];
long[] featureShape = new long[mappings[0].Length];
for (int i = 0; i < mappings.Length; i += 2)
{
int halfIndex = i / 2;
perMappingShape[halfIndex] = new long[mappings[0].Length];
for (int y = 0; y < featureShape.Length; y++)
{
perMappingShape[halfIndex][y] = mappings[i + 1][y] - mappings[i][y];
featureShape[y] += perMappingShape[halfIndex][y];
}
perMappingLength[i / 2] = ArrayUtils.Product(perMappingShape[halfIndex]);
}
long[] shape = new long[featureShape.Length + 2];
shape[0] = numberOfRecordsToExtract;
shape[1] = 1;
Array.Copy(featureShape, 0, shape, 2, featureShape.Length);
INDArray array = handler.NDArray(shape);
long[] globalBufferIndices = new long[shape.Length];
long sectionOffset = _sectionOffsets.ContainsKey(name) ? _sectionOffsets[name] : 0L;
for (int r = 0; r < numberOfRecordsToExtract; r++)
{
T[] record = rawRecords[r];
globalBufferIndices[0] = r; //BatchTimeFeatures indexing
globalBufferIndices[1] = 0;
for (int i = 0; i < mappings.Length; i += 2)
{
long[] beginShape = mappings[i];
long[] localShape = perMappingShape[i / 2];
long[] localStrides = NDArrayUtils.GetStrides(localShape);
long[] localBufferIndices = new long[mappings[i].Length];
long length = perMappingLength[i / 2];
long beginFlatIndex = ArrayUtils.Product(beginShape);
for (int y = 0; y < length; y++)
{
localBufferIndices = NDArrayUtils.GetIndices(y, localShape, localStrides, localBufferIndices);
localBufferIndices = ArrayUtils.Add(beginShape, localBufferIndices, localBufferIndices);
Array.Copy(localBufferIndices, 0, globalBufferIndices, 2, localBufferIndices.Length);
array.SetValue(record[beginFlatIndex + y + sectionOffset], globalBufferIndices);
}
}
}
namedArrays.Add(name, array);
}
_logger.Debug($"Done extracting {numberOfRecordsToExtract} records from reader {Reader} (requested: {numberOfRecords}).");
return(namedArrays);
}