private protected unsafe TransformerEstimatorSafeHandle CreateTransformerFromEstimatorBase(DateTimeEstimator.HolidayList country) { bool success; IntPtr errorHandle; IntPtr estimator; if (country == DateTimeEstimator.HolidayList.None) { success = CreateEstimatorHelper(null, null, out estimator, out errorHandle); } else { fixed(byte *dataRootDir = Encoding.UTF8.GetBytes(AppDomain.CurrentDomain.BaseDirectory + char.MinValue)) fixed(byte *countryPointer = Encoding.UTF8.GetBytes(Enum.GetName(typeof(DateTimeEstimator.HolidayList), country) + char.MinValue)) { success = CreateEstimatorHelper(countryPointer, dataRootDir, out estimator, out errorHandle); } } if (!success) { throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle)); } using (var estimatorHandler = new TransformerEstimatorSafeHandle(estimator, DestroyEstimatorHelper)) { success = CreateTransformerFromEstimatorHelper(estimatorHandler, out IntPtr transformer, out errorHandle); if (!success) { throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle)); } return(new TransformerEstimatorSafeHandle(transformer, DestroyTransformerHelper)); } }
private protected override unsafe TransformerEstimatorSafeHandle CreateTransformerFromSavedDataHelper(byte *rawData, IntPtr dataSize) { var result = CreateTransformerFromSavedDataNative(rawData, dataSize, out IntPtr transformer, out IntPtr errorHandle); if (!result) { throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle)); } var handle = new TransformerEstimatorSafeHandle(transformer, DestroyTransformerNative); // Get the result of the transform and cache it. Pass in null so we get the Mode back. result = TransformDataNative(handle, null, out IntPtr output, out errorHandle); if (!result) { throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle)); } using (var handler = new TransformedDataSafeHandle(output, DestroyTransformedDataNative)) { Result = PointerToString(output).AsMemory(); } return(handle); }
// Normal constructor. internal TimeSeriesImputerTransformer(IHostEnvironment host, TimeSeriesImputerEstimator.Options options, IDataView input) { _host = host.Register(nameof(TimeSeriesImputerTransformer)); _timeSeriesColumn = options.TimeSeriesColumn; _grainColumns = options.GrainColumns; _imputeMode = options.ImputeMode; _suppressTypeErrors = options.SupressTypeErrors; IEnumerable <string> tempDataColumns; if (options.FilterMode == TimeSeriesImputerEstimator.FilterMode.Exclude) { tempDataColumns = input.Schema.Where(x => !options.FilterColumns.Contains(x.Name)).Select(x => x.Name); } else if (options.FilterMode == TimeSeriesImputerEstimator.FilterMode.Include) { tempDataColumns = input.Schema.Where(x => options.FilterColumns.Contains(x.Name)).Select(x => x.Name); } else { tempDataColumns = input.Schema.Select(x => x.Name); } // Time series and Grain columns should never be included in the data columns _dataColumns = tempDataColumns.Where(x => x != _timeSeriesColumn && !_grainColumns.Contains(x)).ToArray(); // 1 is for the time series column. Make one array in the correct order of all the columns. // Order is Timeseries column, All grain columns, All data columns. _allColumnNames = new string[1 + _grainColumns.Length + _dataColumns.Length]; _allColumnNames[0] = _timeSeriesColumn; Array.Copy(_grainColumns, 0, _allColumnNames, 1, _grainColumns.Length); Array.Copy(_dataColumns, 0, _allColumnNames, 1 + _grainColumns.Length, _dataColumns.Length); TransformerHandle = CreateTransformerFromEstimator(input); }
private protected unsafe override bool FitHelper(TransformerEstimatorSafeHandle estimator, ReadOnlyMemory <char> input, out FitResult fitResult, out IntPtr errorHandle) { var inputAsString = input.ToString(); fixed(byte *interopInput = string.IsNullOrEmpty(inputAsString)?null : Encoding.UTF8.GetBytes(inputAsString + char.MinValue)) { return(FitNative(estimator, interopInput, out fitResult, out errorHandle)); } }
internal unsafe void CreateTransformerFromSavedData(byte[] data) { fixed(byte *rawData = data) { IntPtr dataSize = new IntPtr(data.Count()); TransformerHandler = CreateTransformerFromSavedDataHelper(rawData, dataSize); } }
private protected override unsafe void CreateTransformerFromSavedDataHelper(byte *rawData, IntPtr dataSize) { fixed(byte *dataRootDir = Encoding.UTF8.GetBytes(AppDomain.CurrentDomain.BaseDirectory + char.MinValue)) { var result = CreateTransformerFromSavedDataNative(rawData, dataSize, dataRootDir, out IntPtr transformer, out IntPtr errorHandle); if (!result) { throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle)); } _transformerHandler = new TransformerEstimatorSafeHandle(transformer, DestroyTransformerNative); } }
// Factory method for SignatureLoadModel. internal TimeSeriesImputerTransformer(IHostEnvironment host, ModelLoadContext ctx) { _host = host.Register(nameof(TimeSeriesImputerTransformer)); _host.Check(!CommonExtensions.OsIsCentOS7(), "CentOS7 is not supported"); // *** Binary format *** // name of time series column // length of grain column array // all column names in grain column array // length of filter column array // all column names in filter column array // byte value of filter mode // byte value of impute mode // length of C++ state array // C++ byte state array _timeSeriesColumn = ctx.Reader.ReadString(); _grainColumns = new string[ctx.Reader.ReadInt32()]; for (int i = 0; i < _grainColumns.Length; i++) { _grainColumns[i] = ctx.Reader.ReadString(); } _dataColumns = new string[ctx.Reader.ReadInt32()]; for (int i = 0; i < _dataColumns.Length; i++) { _dataColumns[i] = ctx.Reader.ReadString(); } _imputeMode = (TimeSeriesImputerEstimator.ImputationStrategy)ctx.Reader.ReadByte(); _allColumnNames = new string[1 + _grainColumns.Length + _dataColumns.Length]; _allColumnNames[0] = _timeSeriesColumn; Array.Copy(_grainColumns, 0, _allColumnNames, 1, _grainColumns.Length); Array.Copy(_dataColumns, 0, _allColumnNames, 1 + _grainColumns.Length, _dataColumns.Length); var nativeState = ctx.Reader.ReadByteArray(); TransformerHandle = CreateTransformerFromSavedData(nativeState); }
private protected override unsafe TransformerEstimatorSafeHandle CreateTransformerFromSavedDataHelper(byte *rawData, IntPtr dataSize) { var success = CreateTransformerFromSavedDataNative(rawData, dataSize, out IntPtr transformer, out IntPtr errorHandle); if (!success) { throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle)); } var handle = new TransformerEstimatorSafeHandle(transformer, DestroyTransformerNative); // Get the result of the transform and cache it so its cached before transform is called. Pass in float.NaN so we get the Mode back. success = TransformDataNative(handle, float.NaN, out float output, out errorHandle); if (!success) { throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle)); } Result = output; return(handle); }
private static extern bool GetStateNative(TransformerEstimatorSafeHandle estimator, out TrainingState trainingState, out IntPtr errorHandle);
private unsafe TransformerEstimatorSafeHandle CreateTransformerFromEstimator(IDataView input) { IntPtr estimator; IntPtr errorHandle; bool success; var allColumns = input.Schema.Where(x => _allColumnNames.Contains(x.Name)).Select(x => TypedColumn.CreateTypedColumn(x, _dataColumns)).ToDictionary(x => x.Column.Name); // Create buffer to hold binary data var columnBuffer = new byte[4096]; // Create TypeId[] for types of grain and data columns; var dataColumnTypes = new TypeId[_dataColumns.Length]; var grainColumnTypes = new TypeId[_grainColumns.Length]; foreach (var column in _grainColumns.Select((value, index) => new { index, value })) { grainColumnTypes[column.index] = allColumns[column.value].GetTypeId(); } foreach (var column in _dataColumns.Select((value, index) => new { index, value })) { dataColumnTypes[column.index] = allColumns[column.value].GetTypeId(); fixed(bool *suppressErrors = &_suppressTypeErrors) fixed(TypeId * rawDataColumnTypes = dataColumnTypes) fixed(TypeId * rawGrainColumnTypes = grainColumnTypes) { success = CreateEstimatorNative(rawGrainColumnTypes, new IntPtr(grainColumnTypes.Length), rawDataColumnTypes, new IntPtr(dataColumnTypes.Length), _imputeMode, suppressErrors, out estimator, out errorHandle); } if (!success) { throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle)); } using (var estimatorHandler = new TransformerEstimatorSafeHandle(estimator, DestroyEstimatorNative)) { var fitResult = FitResult.Continue; while (fitResult != FitResult.Complete) { using (var cursor = input.GetRowCursorForAllColumns()) { // Initialize getters for start of loop foreach (var column in allColumns.Values) { column.InitializeGetter(cursor); } while ((fitResult == FitResult.Continue || fitResult == FitResult.ResetAndContinue) && cursor.MoveNext()) { BuildColumnByteArray(allColumns, ref columnBuffer, out int serializedDataLength); fixed(byte *bufferPointer = columnBuffer) { var binaryArchiveData = new NativeBinaryArchiveData() { Data = bufferPointer, DataSize = new IntPtr(serializedDataLength) }; success = FitNative(estimatorHandler, binaryArchiveData, out fitResult, out errorHandle); } if (!success) { throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle)); } } success = CompleteTrainingNative(estimatorHandler, out fitResult, out errorHandle); if (!success) { throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle)); } } } success = CreateTransformerFromEstimatorNative(estimatorHandler, out IntPtr transformer, out errorHandle); if (!success) { throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle)); } return(new TransformerEstimatorSafeHandle(transformer, DestroyTransformerNative)); } }
private protected override bool CompleteTrainingHelper(TransformerEstimatorSafeHandle estimator, out IntPtr errorHandle) => CompleteTrainingNative(estimator, out errorHandle);
private protected override bool FitHelper(TransformerEstimatorSafeHandle estimator, double input, out FitResult fitResult, out IntPtr errorHandle) => FitNative(estimator, input, out fitResult, out errorHandle);
private static unsafe extern bool TransformDataNative(TransformerEstimatorSafeHandle transformer, in float input, out float output, out IntPtr errorHandle);
private static unsafe extern bool FitNative(TransformerEstimatorSafeHandle estimator, byte *input, out FitResult fitResult, out IntPtr errorHandle);
private protected override bool OnDataCompletedHelper(TransformerEstimatorSafeHandle estimator, out IntPtr errorHandle) => OnDataCompletedNative(estimator, out errorHandle);
private static extern bool OnDataCompletedNative(TransformerEstimatorSafeHandle estimator, out IntPtr errorHandle);
private protected override bool GetStateHelper(TransformerEstimatorSafeHandle estimator, out TrainingState trainingState, out IntPtr errorHandle) => GetStateNative(estimator, out trainingState, out errorHandle);
private unsafe TransformerEstimatorSafeHandle CreateTransformerFromEstimator(IDataView input) { IntPtr estimator; IntPtr errorHandle; bool success; var allColumns = input.Schema.Where(x => _allColumnNames.Contains(x.Name)).Select(x => TypedColumn.CreateTypedColumn(x, _dataColumns)).ToDictionary(x => x.Column.Name); // Create TypeId[] for types of grain and data columns; var dataColumnTypes = new TypeId[_dataColumns.Length]; var grainColumnTypes = new TypeId[_grainColumns.Length]; foreach (var column in _grainColumns.Select((value, index) => new { index, value })) { grainColumnTypes[column.index] = allColumns[column.value].GetTypeId(); } foreach (var column in _dataColumns.Select((value, index) => new { index, value })) { dataColumnTypes[column.index] = allColumns[column.value].GetTypeId(); fixed(bool *suppressErrors = &_suppressTypeErrors) fixed(TypeId * rawDataColumnTypes = dataColumnTypes) fixed(TypeId * rawGrainColumnTypes = grainColumnTypes) { success = CreateEstimatorNative(rawGrainColumnTypes, new IntPtr(grainColumnTypes.Length), rawDataColumnTypes, new IntPtr(dataColumnTypes.Length), _imputeMode, suppressErrors, out estimator, out errorHandle); } if (!success) { throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle)); } using (var estimatorHandle = new TransformerEstimatorSafeHandle(estimator, DestroyEstimatorNative)) { TrainingState trainingState; FitResult fitResult; // Create buffer to hold binary data var memoryStream = new MemoryStream(4096); var binaryWriter = new BinaryWriter(memoryStream, Encoding.UTF8); // Can't use a using with this because it potentially needs to be reset. Manually disposing as needed. var cursor = input.GetRowCursorForAllColumns(); // Initialize getters foreach (var column in allColumns.Values) { column.InitializeGetter(cursor); } // Start the loop with the cursor in a valid state already. var valid = cursor.MoveNext(); // Make sure its not an empty data frame Debug.Assert(valid); while (true) { // Get the state of the native estimator. success = GetStateNative(estimatorHandle, out trainingState, out errorHandle); if (!success) { throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle)); } // If we are no longer training then exit loop. if (trainingState != TrainingState.Training) { break; } // Build byte array to send column data to native featurizer BuildColumnByteArray(allColumns, ref binaryWriter); // Fit the estimator fixed(byte *bufferPointer = memoryStream.GetBuffer()) { var binaryArchiveData = new NativeBinaryArchiveData() { Data = bufferPointer, DataSize = new IntPtr(memoryStream.Position) }; success = FitNative(estimatorHandle, binaryArchiveData, out fitResult, out errorHandle); } // Reset memory stream to 0 memoryStream.Position = 0; if (!success) { throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle)); } // If we need to reset the data to the beginning. if (fitResult == FitResult.ResetAndContinue) { ResetCursor(input, ref cursor, allColumns); } // If we are at the end of the data. if (!cursor.MoveNext()) { // If we get here fitResult should never be ResetAndContinue Debug.Assert(fitResult != FitResult.ResetAndContinue); OnDataCompletedNative(estimatorHandle, out errorHandle); if (!success) { throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle)); } ResetCursor(input, ref cursor, allColumns); } } // When done training complete the estimator. success = CompleteTrainingNative(estimatorHandle, out errorHandle); if (!success) { throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle)); } // Create the native transformer from the estimator; success = CreateTransformerFromEstimatorNative(estimatorHandle, out IntPtr transformer, out errorHandle); if (!success) { throw new Exception(GetErrorDetailsAndFreeNativeMemory(errorHandle)); } // Manually dispose of the IEnumerator since we don't have a using statement; cursor.Dispose(); return(new TransformerEstimatorSafeHandle(transformer, DestroyTransformerNative)); } }
private protected abstract bool CompleteTrainingHelper(TransformerEstimatorSafeHandle estimator, out IntPtr errorHandle);
private static extern bool TransformDataNativeX64(TransformerEstimatorSafeHandle transformer, NativeDateTimeParameterX64 input, IntPtr output, out IntPtr errorHandle);
private static extern bool CompleteTrainingNative(TransformerEstimatorSafeHandle estimator, out IntPtr errorHandle);
private static extern bool FitNative(TransformerEstimatorSafeHandle estimator, in float input, out FitResult fitResult, out IntPtr errorHandle);
private protected abstract bool GetStateHelper(TransformerEstimatorSafeHandle estimator, out TrainingState trainingState, out IntPtr errorHandle);