public static XArray RandomGamma(Shape shape, double alpha, double beta, DType dtype = DType.Float32)
{
XArray r = new XArray(shape, dtype);
NativeWrapper.TS_Random_Gamma(r.GetRef(), alpha, beta);
return(r);
}
public static XArray RandomLogNormal(Shape shape, double mean, double std_dev, DType dtype = DType.Float32)
{
XArray r = new XArray(shape, dtype);
NativeWrapper.TS_Random_LogNormal(r.GetRef(), mean, std_dev);
return(r);
}
public static XArray RandomNegativeBinomial(Shape shape, int k, double prob, DType dtype = DType.Float32)
{
XArray r = new XArray(shape, dtype);
NativeWrapper.TS_Random_NegativeBinomial(r.GetRef(), k, prob);
return(r);
}
public static XArray RandomExponential(Shape shape, double rate, DType dtype = DType.Float32)
{
XArray r = new XArray(shape, dtype);
NativeWrapper.TS_Random_Exponential(r.GetRef(), rate);
return(r);
}
public static XArray Cast(XArray a, DType dtype)
{
XArray r = new XArray(a.Shape, a.DataType);
NativeWrapper.TS_Cast(a.GetRef(), (int)dtype, r.GetRef());
return(r);
}
public static XArray RandomRandInt(Shape shape, double low, double high, DType dtype = DType.Float32)
{
XArray r = new XArray(shape, dtype);
NativeWrapper.TS_Random_RandInt(r.GetRef(), low, high);
return(r);
}
public bool[] Convert(XArray xarray, out Array result)
{
Allocator.AllocateToSize(ref _array, (xarray.Selector.IsSingleValue ? 1 : xarray.Count));
Allocator.AllocateToSize(ref _couldNotConvertArray, (xarray.Selector.IsSingleValue ? 1 : xarray.Count));
bool anyCouldNotConvert = false;
String8[] sourceArray = (String8[])xarray.Array;
if (!xarray.Selector.IsSingleValue)
{
for (int i = 0; i < xarray.Count; ++i)
{
_couldNotConvertArray[i] = !_tryConvert(sourceArray[xarray.Index(i)], out _array[i]);
if (_couldNotConvertArray[i])
{
_array[i] = _defaultValue;
anyCouldNotConvert = true;
}
}
}
else
{
_couldNotConvertArray[0] = !_tryConvert(sourceArray[xarray.Index(0)], out _array[0]);
if (_couldNotConvertArray[0])
{
_array[0] = _defaultValue;
anyCouldNotConvert = true;
}
}
result = _array;
return(anyCouldNotConvert ? _couldNotConvertArray : null);
}
public static XArray ReduceAll(XArray x, ReduceFunc op, double p1 = 0, int p2 = 0)
{
XArray r = new XArray(new Shape(1), x.DataType);
NativeWrapper.TS_ReduceAll(x.GetRef(), r.GetRef(), (int)op, p1, p2);
return(r);
}
private XArray ReadIndices(ArraySelector selector)
{
Allocator.AllocateToSize(ref _resultArray, selector.Count);
// Read all string positions
XArray positions = _positionsReader.Read(ArraySelector.All(_positionsReader.Count));
int[] positionArray = (int[])positions.Array;
// Read all raw string bytes
XArray bytes = _bytesReader.Read(ArraySelector.All(_bytesReader.Count));
byte[] textArray = (byte[])bytes.Array;
// Update the String8 array to point to them
for (int i = 0; i < selector.Count; ++i)
{
int rowIndex = selector.Index(i);
int valueStart = (rowIndex == 0 ? 0 : positionArray[rowIndex - 1]);
int valueEnd = positionArray[rowIndex];
_resultArray[i] = new String8(textArray, valueStart, valueEnd - valueStart);
}
// Cache the xarray and return it
_currentArray = XArray.All(_resultArray, selector.Count);
_currentSelector = selector;
return(_currentArray);
}
public bool[] Convert(XArray xarray, out Array result)
{
Allocator.AllocateToSize(ref _string8Array, xarray.Count);
Allocator.AllocateToSize(ref _buffer, xarray.Count * _bytesPerItem);
int bufferBytesUsed = 0;
T[] sourceArray = (T[])xarray.Array;
for (int i = 0; i < xarray.Count; ++i)
{
int index = xarray.Index(i);
if (xarray.HasNulls && xarray.NullRows[index])
{
// Always turn nulls into the default value rather than converting default of other type
_string8Array[i] = _defaultValue;
}
else
{
String8 converted = _converter(sourceArray[index], _buffer, bufferBytesUsed);
_string8Array[i] = converted;
bufferBytesUsed += converted.Length;
}
}
result = _string8Array;
return(null);
}
public void Append(XArray xarray)
{
// If we already had too many values, we're just writing them out normally
if (_dictionary == null)
{
_valueWriter.Append(xarray);
_rowCountWritten += xarray.Count;
return;
}
// Otherwise, find the index of each value added
if (_dictionary.Add(xarray, ref _currentArrayIndices))
{
// If we're still under 256 values, write the indices
_rowIndexWriter.Append(XArray.All(_currentArrayIndices, xarray.Count));
_rowCountWritten += xarray.Count;
}
else
{
// If we went over 256 values, convert to writing the values directly
Convert();
_valueWriter.Append(xarray);
_rowCountWritten += xarray.Count;
return;
}
}
public XArray Values()
{
bool[] nulls = null;
if (_nullItemIndex != -1)
{
nulls = new bool[this.Metadata.Length];
nulls[_nullItemIndex] = true;
}
int[] indicesInOrder = new int[this.Count];
for (int i = 0; i < this.Metadata.Length; ++i)
{
if (this.Metadata[i] != 0)
{
indicesInOrder[_values[i]] = i;
}
}
// Build an indexed XArray pointing to the keys in insertion order
XArray keysInOrder = XArray.All(_keys, this.Count, nulls).Reselect(ArraySelector.Map(indicesInOrder, this.Count));
// Convert it to a contiguous, 0-based XArray
T[] contiguousCopy = null;
bool[] contiguousIsNull = null;
return(keysInOrder.ToContiguous <T>(ref contiguousCopy, ref contiguousIsNull));
}
private static void RoundTrip(string columnName, int[] array, int batchSize = 128)
{
XDatabaseContext context = new XDatabaseContext();
string columnPath = Path.Combine("VariableIntegerReaderWriterTests", columnName);
string columnPrefix = Path.Combine(columnPath, "Vl");
context.StreamProvider.Delete(columnPath);
Directory.CreateDirectory(columnPath);
XArray values = XArray.All(array, array.Length);
using (IColumnWriter writer = new VariableIntegerWriter(context.StreamProvider, columnPrefix))
{
ArraySelector page = ArraySelector.All(0).NextPage(array.Length, batchSize);
while (page.Count > 0)
{
writer.Append(values.Reselect(page));
page = page.NextPage(array.Length, batchSize);
}
}
XArray returned = default(XArray);
using (IColumnReader reader = new VariableIntegerReader(context.StreamProvider, columnPrefix, CachingOption.AsConfigured))
{
returned = reader.Read(ArraySelector.All(array.Length));
}
TableTestHarness.AssertAreEqual(values, returned, array.Length);
context.StreamProvider.Delete(columnPath);
}
public static XArray RandomCauchy(Shape shape, double a, double b, DType dtype = DType.Float32)
{
XArray r = new XArray(shape, dtype);
NativeWrapper.TS_Random_Cauchy(r.GetRef(), a, b);
return(r);
}
public static XArray RandomFisherF(Shape shape, double m, double n, DType dtype = DType.Float32)
{
XArray r = new XArray(shape, dtype);
NativeWrapper.TS_Random_FisherF(r.GetRef(), m, n);
return(r);
}
private XArray Convert(XArray xarray1, XArray xarray2)
{
int count = xarray1.Count;
if (count != xarray2.Count)
{
throw new InvalidOperationException("SimpleTwoArgumentFunction must get the same number of rows from each argument.");
}
// Allocate for results
Allocator.AllocateToSize(ref _buffer, count);
Allocator.AllocateToSize(ref _isNull, count);
// Convert each non-null value
bool areAnyNull = false;
T[] array1 = (T[])xarray1.Array;
U[] array2 = (U[])xarray2.Array;
for (int i = 0; i < count; ++i)
{
int index1 = xarray1.Index(i);
int index2 = xarray2.Index(i);
bool rowIsNull = (xarray1.HasNulls && xarray1.NullRows[index1]) || (xarray2.HasNulls && xarray2.NullRows[index2]);
areAnyNull |= rowIsNull;
_isNull[i] = rowIsNull;
_buffer[i] = (rowIsNull ? default(V) : _function(array1[index1], array2[index2]));
}
return(XArray.All(_buffer, count, (areAnyNull ? _isNull : null)));
}
public BitVector TryGetValues(XArray keys, out ArraySelector rightSideSelector)
{
Allocator.AllocateToSize(ref _returnedVector, keys.Count);
Allocator.AllocateToSize(ref _returnedIndicesBuffer, keys.Count);
_returnedVector.None();
int countFound = 0;
T[] keyArray = (T[])keys.Array;
for (int i = 0; i < keys.Count; ++i)
{
int index = keys.Index(i);
int foundAtIndex;
if ((keys.HasNulls && keys.NullRows[index]) || !_dictionary.TryGetValue(keyArray[index], out foundAtIndex))
{
_returnedVector.Clear(i);
}
else
{
_returnedVector.Set(i);
_returnedIndicesBuffer[countFound++] = foundAtIndex;
}
}
// Write out the indices of the joined rows for each value found
rightSideSelector = ArraySelector.Map(_returnedIndicesBuffer, countFound);
// Return the vector of which input rows matched
return(_returnedVector);
}
public XArray Remap(XArray source, ref int[] remapArray)
{
// See if we have the remapping cached already
ArraySelector cachedMapping;
if (_cachedRemappings.TryGetValue(source.Selector, out cachedMapping))
{
return(source.Reselect(cachedMapping));
}
// Convert the BitVector to indices if we haven't yet (deferred to first column wanting values)
if (!_indicesFound)
{
_indicesFound = true;
Allocator.AllocateToSize(ref _indices, _count);
int countFound = _vector.Page(_indices, ref _nextVectorIndex, _count);
if (countFound != _count)
{
throw new InvalidOperationException($"RowRemapper found {countFound:n0} rows when {_count:n0} expected paging in Vector with {_vector.Count:n0} total matches up to index {_nextVectorIndex:n0}.");
}
}
// Remap the outer selector
XArray remapped = source.Select(ArraySelector.Map(_indices, _count), ref remapArray);
// Cache the remapping
_cachedRemappings[source.Selector] = remapped.Selector;
return(remapped);
}
public XArray Read(ArraySelector selector)
{
XArray raw = _reader.Read(selector);
XArray asInteger = (_converter == null ? raw : _converter(raw));
return(asInteger);
}
public void Add(XArray keys, int firstRowIndex)
{
T[] keyArray = (T[])keys.Array;
if (_valueCopier != null || keys.HasNulls)
{
for (int i = 0; i < keys.Count; ++i)
{
int index = keys.Index(i);
if (keys.HasNulls && keys.NullRows[index])
{
continue;
}
T key = keyArray[index];
if (_valueCopier != null)
{
key = _valueCopier.Copy(key);
}
_dictionary[key] = firstRowIndex + i;
}
}
else
{
for (int i = 0; i < keys.Count; ++i)
{
int index = keys.Index(i);
T key = keyArray[index];
_dictionary[key] = firstRowIndex + i;
}
}
}
public static XArray Reduce(XArray x, int[] axes, double p1, int p2, int p3, ReduceFunc op)
{
unsafe
{
List <long> newshape = new List <long>();
for (int i = 0; i < x.Shape.Data.Length; i++)
{
if (!axes.Contains(i))
{
newshape.Add(x.Shape[i]);
}
else
{
newshape.Add(1);
}
}
fixed(int *axes_ptr = axes)
{
XArray r = new XArray(new Shape(newshape.ToArray()), x.DataType);
NativeWrapper.TS_Reduce(x.GetRef(), axes_ptr, axes.Length, r.GetRef(), p1, p2, p3, (int)op);
return(r);
}
}
}
public void Append(XArray xarray)
{
if (_converter != null)
{
// Convert the array to the type we're writing
XArray convertedArray = _converter(xarray);
if (!convertedArray.HasNulls)
{
// If values were all in range, just write them
_writer.Append(convertedArray);
}
else
{
// If values were out of range, we have to upconvert
Upconvert(xarray);
}
}
else
{
// If we're already up to integer, just write directly
_writer.Append(xarray);
}
_rowCountWritten += xarray.Count;
}
public static XArray IsClose(XArray a, XArray b, double rtol = 1e-5, double atol = 1e-8, bool equal_nan = false)
{
XArray r = new XArray(a.Shape, a.DataType);
NativeWrapper.TS_IsClose(a.GetRef(), b.GetRef(), rtol, atol, equal_nan, r.GetRef());
return(r);
}
public static XArray AllClose(XArray a, XArray b, double rtol = 1e-5, double atol = 1e-8)
{
XArray r = new XArray(a.Shape, a.DataType);
NativeWrapper.TS_AllClose(a.GetRef(), b.GetRef(), rtol, atol, r.GetRef());
return(r);
}
public static XArray Pow(XArray x, double value)
{
XArray r = new XArray(x.Shape, x.DataType);
NativeWrapper.TS_Pow(x.GetRef(), value, r.GetRef());
return(r);
}
public static XArray Clip(XArray x, double min, double max)
{
XArray r = new XArray(x.Shape, x.DataType);
NativeWrapper.TS_Clip(x.GetRef(), min, max, r.GetRef());
return(r);
}
public static XArray BitwiseOps(XArray a, XArray b, ElementwiseFunc op)
{
XArray r = new XArray(a.Shape, a.DataType);
NativeWrapper.TS_Elementwise_2_Bitwise(a.GetRef(), b.GetRef(), r.GetRef(), (int)op);
return(r);
}
public static XArray RandomPermutation(Shape shape, DType dtype = DType.Float32)
{
XArray r = new XArray(shape, dtype);
NativeWrapper.TS_Random_Permutation(r.GetRef());
return(r);
}
public static XArray RandomChoice(Shape shape, int n, XArray weight, bool replace = true, DType dtype = DType.Float32)
{
XArray r = new XArray(shape, dtype);
NativeWrapper.TS_Random_ChoiceWithWeight(r.GetRef(), n, weight.GetRef(), replace);
return(r);
}
public static XArray RandomStudentT(Shape shape, double n, DType dtype = DType.Float32)
{
XArray r = new XArray(shape, dtype);
NativeWrapper.TS_Random_StudentT(r.GetRef(), n);
return(r);
}
protected override double OverrideProperty(XArray arr)
{
return arr.level;
}
protected override double OverrideProperty(XArray arr)
{
return arr.vars.Length;
}
