private protected unsafe TransformerEstimatorSafeHandle CreateTransformerFromEstimatorBase(DateTimeEstimator.HolidayList country)
{
bool success;
IntPtr errorHandle;
IntPtr estimator;
if (country == DateTimeEstimator.HolidayList.None)
{
success = CreateEstimatorHelper(null, null, out estimator, out errorHandle);
}
else
{
fixed(byte *dataRootDir = Encoding.UTF8.GetBytes(AppDomain.CurrentDomain.BaseDirectory + char.MinValue))
fixed(byte *countryPointer = Encoding.UTF8.GetBytes(Enum.GetName(typeof(DateTimeEstimator.HolidayList), country) + char.MinValue))
{
success = CreateEstimatorHelper(countryPointer, dataRootDir, out estimator, out errorHandle);
}
}
if (!success)
{
throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle));
}
using (var estimatorHandler = new TransformerEstimatorSafeHandle(estimator, DestroyEstimatorHelper))
{
success = CreateTransformerFromEstimatorHelper(estimatorHandler, out IntPtr transformer, out errorHandle);
if (!success)
{
throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle));
}
return(new TransformerEstimatorSafeHandle(transformer, DestroyTransformerHelper));
}
}
private protected override unsafe TransformerEstimatorSafeHandle CreateTransformerFromSavedDataHelper(byte *rawData, IntPtr dataSize)
{
var result = CreateTransformerFromSavedDataNative(rawData, dataSize, out IntPtr transformer, out IntPtr errorHandle);
if (!result)
{
throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle));
}
var handle = new TransformerEstimatorSafeHandle(transformer, DestroyTransformerNative);
// Get the result of the transform and cache it. Pass in null so we get the Mode back.
result = TransformDataNative(handle, null, out IntPtr output, out errorHandle);
if (!result)
{
throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle));
}
using (var handler = new TransformedDataSafeHandle(output, DestroyTransformedDataNative))
{
Result = PointerToString(output).AsMemory();
}
return(handle);
}
// Normal constructor.
internal TimeSeriesImputerTransformer(IHostEnvironment host, TimeSeriesImputerEstimator.Options options, IDataView input)
{
_host = host.Register(nameof(TimeSeriesImputerTransformer));
_timeSeriesColumn = options.TimeSeriesColumn;
_grainColumns = options.GrainColumns;
_imputeMode = options.ImputeMode;
_suppressTypeErrors = options.SupressTypeErrors;
IEnumerable <string> tempDataColumns;
if (options.FilterMode == TimeSeriesImputerEstimator.FilterMode.Exclude)
{
tempDataColumns = input.Schema.Where(x => !options.FilterColumns.Contains(x.Name)).Select(x => x.Name);
}
else if (options.FilterMode == TimeSeriesImputerEstimator.FilterMode.Include)
{
tempDataColumns = input.Schema.Where(x => options.FilterColumns.Contains(x.Name)).Select(x => x.Name);
}
else
{
tempDataColumns = input.Schema.Select(x => x.Name);
}
// Time series and Grain columns should never be included in the data columns
_dataColumns = tempDataColumns.Where(x => x != _timeSeriesColumn && !_grainColumns.Contains(x)).ToArray();
// 1 is for the time series column. Make one array in the correct order of all the columns.
// Order is Timeseries column, All grain columns, All data columns.
_allColumnNames = new string[1 + _grainColumns.Length + _dataColumns.Length];
_allColumnNames[0] = _timeSeriesColumn;
Array.Copy(_grainColumns, 0, _allColumnNames, 1, _grainColumns.Length);
Array.Copy(_dataColumns, 0, _allColumnNames, 1 + _grainColumns.Length, _dataColumns.Length);
TransformerHandle = CreateTransformerFromEstimator(input);
}
private protected unsafe override bool FitHelper(TransformerEstimatorSafeHandle estimator, ReadOnlyMemory <char> input, out FitResult fitResult, out IntPtr errorHandle)
{
var inputAsString = input.ToString();
fixed(byte *interopInput = string.IsNullOrEmpty(inputAsString)?null : Encoding.UTF8.GetBytes(inputAsString + char.MinValue))
{
return(FitNative(estimator, interopInput, out fitResult, out errorHandle));
}
}
internal unsafe void CreateTransformerFromSavedData(byte[] data)
{
fixed(byte *rawData = data)
{
IntPtr dataSize = new IntPtr(data.Count());
TransformerHandler = CreateTransformerFromSavedDataHelper(rawData, dataSize);
}
}
private protected override unsafe void CreateTransformerFromSavedDataHelper(byte *rawData, IntPtr dataSize)
{
fixed(byte *dataRootDir = Encoding.UTF8.GetBytes(AppDomain.CurrentDomain.BaseDirectory + char.MinValue))
{
var result = CreateTransformerFromSavedDataNative(rawData, dataSize, dataRootDir, out IntPtr transformer, out IntPtr errorHandle);
if (!result)
{
throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle));
}
_transformerHandler = new TransformerEstimatorSafeHandle(transformer, DestroyTransformerNative);
}
}
// Factory method for SignatureLoadModel.
internal TimeSeriesImputerTransformer(IHostEnvironment host, ModelLoadContext ctx)
{
_host = host.Register(nameof(TimeSeriesImputerTransformer));
_host.Check(!CommonExtensions.OsIsCentOS7(), "CentOS7 is not supported");
// *** Binary format ***
// name of time series column
// length of grain column array
// all column names in grain column array
// length of filter column array
// all column names in filter column array
// byte value of filter mode
// byte value of impute mode
// length of C++ state array
// C++ byte state array
_timeSeriesColumn = ctx.Reader.ReadString();
_grainColumns = new string[ctx.Reader.ReadInt32()];
for (int i = 0; i < _grainColumns.Length; i++)
{
_grainColumns[i] = ctx.Reader.ReadString();
}
_dataColumns = new string[ctx.Reader.ReadInt32()];
for (int i = 0; i < _dataColumns.Length; i++)
{
_dataColumns[i] = ctx.Reader.ReadString();
}
_imputeMode = (TimeSeriesImputerEstimator.ImputationStrategy)ctx.Reader.ReadByte();
_allColumnNames = new string[1 + _grainColumns.Length + _dataColumns.Length];
_allColumnNames[0] = _timeSeriesColumn;
Array.Copy(_grainColumns, 0, _allColumnNames, 1, _grainColumns.Length);
Array.Copy(_dataColumns, 0, _allColumnNames, 1 + _grainColumns.Length, _dataColumns.Length);
var nativeState = ctx.Reader.ReadByteArray();
TransformerHandle = CreateTransformerFromSavedData(nativeState);
}
private protected override unsafe TransformerEstimatorSafeHandle CreateTransformerFromSavedDataHelper(byte *rawData, IntPtr dataSize)
{
var success = CreateTransformerFromSavedDataNative(rawData, dataSize, out IntPtr transformer, out IntPtr errorHandle);
if (!success)
{
throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle));
}
var handle = new TransformerEstimatorSafeHandle(transformer, DestroyTransformerNative);
// Get the result of the transform and cache it so its cached before transform is called. Pass in float.NaN so we get the Mode back.
success = TransformDataNative(handle, float.NaN, out float output, out errorHandle);
if (!success)
{
throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle));
}
Result = output;
return(handle);
}
private static extern bool GetStateNative(TransformerEstimatorSafeHandle estimator, out TrainingState trainingState, out IntPtr errorHandle);
private unsafe TransformerEstimatorSafeHandle CreateTransformerFromEstimator(IDataView input)
{
IntPtr estimator;
IntPtr errorHandle;
bool success;
var allColumns = input.Schema.Where(x => _allColumnNames.Contains(x.Name)).Select(x => TypedColumn.CreateTypedColumn(x, _dataColumns)).ToDictionary(x => x.Column.Name);
// Create buffer to hold binary data
var columnBuffer = new byte[4096];
// Create TypeId[] for types of grain and data columns;
var dataColumnTypes = new TypeId[_dataColumns.Length];
var grainColumnTypes = new TypeId[_grainColumns.Length];
foreach (var column in _grainColumns.Select((value, index) => new { index, value }))
{
grainColumnTypes[column.index] = allColumns[column.value].GetTypeId();
}
foreach (var column in _dataColumns.Select((value, index) => new { index, value }))
{
dataColumnTypes[column.index] = allColumns[column.value].GetTypeId();
fixed(bool *suppressErrors = &_suppressTypeErrors)
fixed(TypeId * rawDataColumnTypes = dataColumnTypes)
fixed(TypeId * rawGrainColumnTypes = grainColumnTypes)
{
success = CreateEstimatorNative(rawGrainColumnTypes, new IntPtr(grainColumnTypes.Length), rawDataColumnTypes, new IntPtr(dataColumnTypes.Length), _imputeMode, suppressErrors, out estimator, out errorHandle);
}
if (!success)
{
throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle));
}
using (var estimatorHandler = new TransformerEstimatorSafeHandle(estimator, DestroyEstimatorNative))
{
var fitResult = FitResult.Continue;
while (fitResult != FitResult.Complete)
{
using (var cursor = input.GetRowCursorForAllColumns())
{
// Initialize getters for start of loop
foreach (var column in allColumns.Values)
{
column.InitializeGetter(cursor);
}
while ((fitResult == FitResult.Continue || fitResult == FitResult.ResetAndContinue) && cursor.MoveNext())
{
BuildColumnByteArray(allColumns, ref columnBuffer, out int serializedDataLength);
fixed(byte *bufferPointer = columnBuffer)
{
var binaryArchiveData = new NativeBinaryArchiveData()
{
Data = bufferPointer, DataSize = new IntPtr(serializedDataLength)
};
success = FitNative(estimatorHandler, binaryArchiveData, out fitResult, out errorHandle);
}
if (!success)
{
throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle));
}
}
success = CompleteTrainingNative(estimatorHandler, out fitResult, out errorHandle);
if (!success)
{
throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle));
}
}
}
success = CreateTransformerFromEstimatorNative(estimatorHandler, out IntPtr transformer, out errorHandle);
if (!success)
{
throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle));
}
return(new TransformerEstimatorSafeHandle(transformer, DestroyTransformerNative));
}
}
private protected override bool CompleteTrainingHelper(TransformerEstimatorSafeHandle estimator, out IntPtr errorHandle) => CompleteTrainingNative(estimator, out errorHandle);
private protected override bool FitHelper(TransformerEstimatorSafeHandle estimator, double input, out FitResult fitResult, out IntPtr errorHandle) => FitNative(estimator, input, out fitResult, out errorHandle);
private static unsafe extern bool TransformDataNative(TransformerEstimatorSafeHandle transformer, in float input, out float output, out IntPtr errorHandle);
private static unsafe extern bool FitNative(TransformerEstimatorSafeHandle estimator, byte *input, out FitResult fitResult, out IntPtr errorHandle);
private protected override bool OnDataCompletedHelper(TransformerEstimatorSafeHandle estimator, out IntPtr errorHandle) => OnDataCompletedNative(estimator, out errorHandle);
private static extern bool OnDataCompletedNative(TransformerEstimatorSafeHandle estimator, out IntPtr errorHandle);
private protected override bool GetStateHelper(TransformerEstimatorSafeHandle estimator, out TrainingState trainingState, out IntPtr errorHandle) => GetStateNative(estimator, out trainingState, out errorHandle);
private unsafe TransformerEstimatorSafeHandle CreateTransformerFromEstimator(IDataView input)
{
IntPtr estimator;
IntPtr errorHandle;
bool success;
var allColumns = input.Schema.Where(x => _allColumnNames.Contains(x.Name)).Select(x => TypedColumn.CreateTypedColumn(x, _dataColumns)).ToDictionary(x => x.Column.Name);
// Create TypeId[] for types of grain and data columns;
var dataColumnTypes = new TypeId[_dataColumns.Length];
var grainColumnTypes = new TypeId[_grainColumns.Length];
foreach (var column in _grainColumns.Select((value, index) => new { index, value }))
{
grainColumnTypes[column.index] = allColumns[column.value].GetTypeId();
}
foreach (var column in _dataColumns.Select((value, index) => new { index, value }))
{
dataColumnTypes[column.index] = allColumns[column.value].GetTypeId();
fixed(bool *suppressErrors = &_suppressTypeErrors)
fixed(TypeId * rawDataColumnTypes = dataColumnTypes)
fixed(TypeId * rawGrainColumnTypes = grainColumnTypes)
{
success = CreateEstimatorNative(rawGrainColumnTypes, new IntPtr(grainColumnTypes.Length), rawDataColumnTypes, new IntPtr(dataColumnTypes.Length), _imputeMode, suppressErrors, out estimator, out errorHandle);
}
if (!success)
{
throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle));
}
using (var estimatorHandle = new TransformerEstimatorSafeHandle(estimator, DestroyEstimatorNative))
{
TrainingState trainingState;
FitResult fitResult;
// Create buffer to hold binary data
var memoryStream = new MemoryStream(4096);
var binaryWriter = new BinaryWriter(memoryStream, Encoding.UTF8);
// Can't use a using with this because it potentially needs to be reset. Manually disposing as needed.
var cursor = input.GetRowCursorForAllColumns();
// Initialize getters
foreach (var column in allColumns.Values)
{
column.InitializeGetter(cursor);
}
// Start the loop with the cursor in a valid state already.
var valid = cursor.MoveNext();
// Make sure its not an empty data frame
Debug.Assert(valid);
while (true)
{
// Get the state of the native estimator.
success = GetStateNative(estimatorHandle, out trainingState, out errorHandle);
if (!success)
{
throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle));
}
// If we are no longer training then exit loop.
if (trainingState != TrainingState.Training)
{
break;
}
// Build byte array to send column data to native featurizer
BuildColumnByteArray(allColumns, ref binaryWriter);
// Fit the estimator
fixed(byte *bufferPointer = memoryStream.GetBuffer())
{
var binaryArchiveData = new NativeBinaryArchiveData()
{
Data = bufferPointer, DataSize = new IntPtr(memoryStream.Position)
};
success = FitNative(estimatorHandle, binaryArchiveData, out fitResult, out errorHandle);
}
// Reset memory stream to 0
memoryStream.Position = 0;
if (!success)
{
throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle));
}
// If we need to reset the data to the beginning.
if (fitResult == FitResult.ResetAndContinue)
{
ResetCursor(input, ref cursor, allColumns);
}
// If we are at the end of the data.
if (!cursor.MoveNext())
{
// If we get here fitResult should never be ResetAndContinue
Debug.Assert(fitResult != FitResult.ResetAndContinue);
OnDataCompletedNative(estimatorHandle, out errorHandle);
if (!success)
{
throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle));
}
ResetCursor(input, ref cursor, allColumns);
}
}
// When done training complete the estimator.
success = CompleteTrainingNative(estimatorHandle, out errorHandle);
if (!success)
{
throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle));
}
// Create the native transformer from the estimator;
success = CreateTransformerFromEstimatorNative(estimatorHandle, out IntPtr transformer, out errorHandle);
if (!success)
{
throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle));
}
// Manually dispose of the IEnumerator since we don't have a using statement;
cursor.Dispose();
return(new TransformerEstimatorSafeHandle(transformer, DestroyTransformerNative));
}
}
private protected abstract bool CompleteTrainingHelper(TransformerEstimatorSafeHandle estimator, out IntPtr errorHandle);
private static extern bool TransformDataNativeX64(TransformerEstimatorSafeHandle transformer, NativeDateTimeParameterX64 input, IntPtr output, out IntPtr errorHandle);
private static extern bool CompleteTrainingNative(TransformerEstimatorSafeHandle estimator, out IntPtr errorHandle);
private static extern bool FitNative(TransformerEstimatorSafeHandle estimator, in float input, out FitResult fitResult, out IntPtr errorHandle);
private protected abstract bool GetStateHelper(TransformerEstimatorSafeHandle estimator, out TrainingState trainingState, out IntPtr errorHandle);
