/// <summary>
///
/// </summary>
/// <param name="options"></param>
public DataImportTask(DataImportTaskOptions options)
{
_options = options;
string tmpGuid = Guid.NewGuid().ToString();
_harvester = new Harvester <T>(_options.ThreadCount);
if (options.Integration != null)
{
if (options.Integration.Fields.Count == 0)
{
throw new InvalidOperationException("Integration needs to have at least 1 field.");
}
_integration = options.Integration;
_integration.APIKey = _options.ApiKey;
if (string.IsNullOrEmpty(_integration.FeaturesCollection))
{
_integration.FeaturesCollection = $"{_integration.Collection}_features";
}
if (string.IsNullOrEmpty(_integration.Collection))
{
_integration.Collection = tmpGuid;
}
_harvester.AddIntegration(options.Integration, _options.Source);
}
else
{
_integration = _harvester.AddIntegrationSource(_options.Source, _options.ApiKey,
_options.IntegrationName, tmpGuid);
}
var outCollection = new DestinationCollection(_integration.Collection, _integration.GetReducedCollectionName());
OutputDestinationCollection = outCollection;
if (options.TotalEntryLimit > 0)
{
_harvester.LimitEntries(options.TotalEntryLimit);
}
if (options.ShardLimit > 0)
{
_harvester.LimitShards(options.ShardLimit);
}
this.EncodeOnImport = options.EncodeInput;
if (this.EncodeOnImport)
{
_encoder = FieldEncoder.Factory.Create(_integration);
}
// new OneHotEncoding(new FieldEncodingOptions { Integration = _integration });
}