public void PreExecute(ISyncContext context, RecordConfiguration configuration)
{
this.PreExecuteInternal(context, configuration);
}
protected override void PreExecuteInternal(ISyncContext context, RecordConfiguration configuration)
{
SchemaBuilder schemaBuilder;
ISchema schema;
IEnumerable <TTextualFieldSpec> headers;
TTextualSpec spec;
if ((object)context == null)
{
throw new ArgumentNullException(nameof(context));
}
if ((object)configuration == null)
{
throw new ArgumentNullException(nameof(configuration));
}
this.AssertValidConfiguration();
TTextualConnectorSpecificConfiguration fsConfig = this.Configuration.StageSpecificConfiguration;
spec = fsConfig.TextualConfiguration.MapToSpec();
if ((object)spec == null)
{
throw new SyncPremException(nameof(spec));
}
this.TextualReader = this.CreateTextualReader(new StreamReader(File.Open(fsConfig.TextualFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)), spec);
headers = this.TextualReader.ReadHeaderFields();
if ((object)headers == null)
{
throw new SyncPremException(nameof(headers));
}
if (!context.LocalState.TryGetValue(this, out IDictionary <string, object> localState))
{
localState = new Dictionary <string, object>(StringComparer.OrdinalIgnoreCase);
context.LocalState.Add(this, localState);
}
schemaBuilder = SchemaBuilder.Create();
headers.ForceEnumeration();
foreach (TTextualFieldSpec header in spec.TextualHeaderSpecs)
{
schemaBuilder.AddField(header.FieldTitle, header.FieldType.ToClrType(), header.IsFieldRequired, header.IsFieldIdentity);
}
schema = schemaBuilder.Build();
if ((object)schema == null)
{
throw new SyncPremException(nameof(schema));
}
localState.Add(Constants.ContextComponentScopedSchema, schema);
}
public IEnumerable <TargetRecord> Write(IEnumerable <TargetRecord> targetRecords, RecordConfiguration recordConfiguration)
{
TargetRecords.AddRange(targetRecords);
return(TargetRecords);
}
protected override async Task PreExecuteAsyncInternal(IAsyncContext asyncContext, RecordConfiguration configuration, CancellationToken cancellationToken)
{
SchemaBuilder schemaBuilder;
ISchema schema;
IList <ISchema> schemas;
IAsyncEnumerable <IAdoNetStreamingResult> results;
IAsyncEnumerator <IAdoNetStreamingResult> resultz;
IAsyncEnumerable <IPayload> records;
IAsyncEnumerator <IPayload> recordz;
IEnumerable <DbParameter> dbParameters;
if ((object)asyncContext == null)
{
throw new ArgumentNullException(nameof(asyncContext));
}
if ((object)configuration == null)
{
throw new ArgumentNullException(nameof(configuration));
}
this.AssertValidConfiguration();
AdoNetConnectorSpecificConfiguration fsConfig = this.Configuration.StageSpecificConfiguration;
this.SourceUnitOfWork = fsConfig.GetUnitOfWork();
if (fsConfig.PreExecuteCommand != null &&
!SolderFascadeAccessor.DataTypeFascade.IsNullOrWhiteSpace(fsConfig.PreExecuteCommand.CommandText))
{
dbParameters = fsConfig.PreExecuteCommand.GetDbDataParameters(this.SourceUnitOfWork);
results = this.SourceUnitOfWork.ExecuteResultsAsync(fsConfig.PreExecuteCommand.CommandType ?? CommandType.Text,
fsConfig.PreExecuteCommand.CommandText,
dbParameters, cancellationToken);
if ((object)results == null)
{
throw new SyncPremException(nameof(results));
}
await results.ForceAsyncEnumeration(cancellationToken); // force execution
}
// execute schema only
if (fsConfig.ExecuteCommand != null &&
!SolderFascadeAccessor.DataTypeFascade.IsNullOrWhiteSpace(fsConfig.ExecuteCommand.CommandText))
{
dbParameters = fsConfig.ExecuteCommand.GetDbDataParameters(this.SourceUnitOfWork);
results = this.SourceUnitOfWork.ExecuteSchemaResultsAsync(fsConfig.ExecuteCommand.CommandType ?? CommandType.Text,
fsConfig.ExecuteCommand.CommandText,
dbParameters, cancellationToken);
if ((object)results == null)
{
throw new SyncPremException(nameof(results));
}
resultz = results.GetEnumerator();
if ((object)resultz == null)
{
throw new SyncPremException(nameof(resultz));
}
schemas = new List <ISchema>();
while (await resultz.MoveNext(cancellationToken))
{
IAdoNetStreamingResult result = resultz.Current;
records = result.AsyncRecords;
if ((object)records == null)
{
throw new SyncPremException(nameof(results));
}
recordz = records.GetEnumerator();
if ((object)recordz == null)
{
throw new SyncPremException(nameof(recordz));
}
schemaBuilder = SchemaBuilder.Create();
while (await recordz.MoveNext(cancellationToken))
{
IPayload record = recordz.Current;
string fieldName;
Type fieldType;
bool isKey;
bool isNullable;
fieldName = (string)record[nameof(DbColumn.ColumnName)];
fieldType = (Type)record[nameof(DbColumn.DataType)];
isKey = (bool?)record[nameof(DbColumn.IsKey)] ?? false;
isNullable = (bool?)record[nameof(DbColumn.AllowDBNull)] ?? true;
// TODO ensure nullable type
schemaBuilder.AddField(fieldName, fieldType, isNullable, isKey);
}
schema = schemaBuilder.Build();
if ((object)schema == null)
{
throw new SyncPremException(nameof(schema));
}
schemas.Add(schema);
}
if (!asyncContext.LocalState.TryGetValue(this, out IDictionary <string, object> localState))
{
localState = new Dictionary <string, object>(StringComparer.OrdinalIgnoreCase);
asyncContext.LocalState.Add(this, localState);
}
localState.Add(Constants.ContextComponentScopedSchema, schemas);
}
}
protected abstract ISyncChannel ProduceInternal(ISyncContext context, RecordConfiguration configuration);
/// <summary>
/// Writes targetRecords to configured logger for this type
/// </summary>
/// <param name="targetRecords"></param>
/// <param name="recordConfiguration"></param>
/// <returns>Lazy enumerable of <see cref="TargetRecord" /></returns>
public IEnumerable <TargetRecord> Write(IEnumerable <TargetRecord> targetRecords, RecordConfiguration recordConfiguration)
{
var recordNumber = 1;
var logText = new StringBuilder();
var maxFieldLength = int.MinValue;
foreach (var targetRecord in targetRecords)
{
if (maxFieldLength <= 0)
{
maxFieldLength = targetRecord.Fields.Max(f => f.Name.Length) + 2;
}
logText.AppendLine($"TARGET RECORD #[{recordNumber.ToString(CultureInfo.InvariantCulture).PadLeft(4, '0')}]");
logText.AppendLine("--------------------------------------------------------------------");
foreach (var targetField in targetRecord.Fields)
{
logText.AppendLine($"{targetField.Name.PadLeft(maxFieldLength)}:\t{targetField.TypedValue} ({targetField.TypedValue.GetType().Name})");
}
_logger.LogInformation(logText.ToString());
logText.Clear();
yield return(targetRecord);
recordNumber++;
}
}
/// <summary>
/// Formats the given records for the given configuration.
/// </summary>
/// <param name="targetRecords">Lazily evaluated enumerable of populated <see cref="TargetRecord" /> records</param>
/// <param name="recordConfiguration">The <see cref="RecordConfiguration" /> being used to transform these source/target records</param>
/// <returns>Enumerable of <see cref="TargetRecord" /></returns>
public IEnumerable <TargetRecord> Format(IEnumerable <TargetRecord> targetRecords, RecordConfiguration recordConfiguration)
{
var recordNumber = 1;
foreach (var targetRecord in targetRecords)
{
TargetRecord yieldRecord = null;
try
{
yieldRecord = FormatRecord(targetRecord, recordConfiguration);
}
catch (Exception x) when(!TextWranglerConfig.OnException(x, $"Could not format target record in [{_formatterName}] from built record [{recordNumber}]"))
{
// OnException handler says not to rethrow, so keep on going, skipping this record
}
if (yieldRecord != null)
{
yield return(yieldRecord);
}
recordNumber++;
}
}
protected override async Task PreExecuteAsyncInternal(IAsyncContext asyncContext, RecordConfiguration configuration, CancellationToken cancellationToken)
{
ISchema schema;
if ((object)asyncContext == null)
{
throw new ArgumentNullException(nameof(asyncContext));
}
if ((object)configuration == null)
{
throw new ArgumentNullException(nameof(configuration));
}
this.AssertValidConfiguration();
if (!asyncContext.LocalState.TryGetValue(this, out IDictionary <string, object> localState))
{
localState = new Dictionary <string, object>(StringComparer.OrdinalIgnoreCase);
asyncContext.LocalState.Add(this, localState);
}
schema = await GetRandomSchemaAsync(cancellationToken);
if ((object)schema == null)
{
throw new SyncPremException(nameof(schema));
}
localState.Add(Constants.ContextComponentScopedSchema, schema);
}
protected override async Task ConsumeAsyncInternal(IAsyncContext asyncContext, RecordConfiguration configuration, IAsyncChannel asyncChannel, CancellationToken cancellationToken)
{
IAsyncEnumerable <IAsyncRecord> records;
if ((object)asyncContext == null)
{
throw new ArgumentNullException(nameof(asyncContext));
}
if ((object)configuration == null)
{
throw new ArgumentNullException(nameof(configuration));
}
if ((object)asyncChannel == null)
{
throw new ArgumentNullException(nameof(asyncChannel));
}
this.AssertValidConfiguration();
records = asyncChannel.Records;
if ((object)records == null)
{
throw new SyncPremException(nameof(records));
}
IAsyncEnumerator <IAsyncRecord> recordz;
recordz = records.GetEnumerator();
if ((object)recordz == null)
{
throw new InvalidOperationException(nameof(recordz));
}
while (await recordz.MoveNext(cancellationToken))
{
IAsyncRecord record = recordz.Current;
TextWriter.WriteLine(record);
}
}
public IEnumerable <TargetRecord> Write(IEnumerable <TargetRecord> targetRecords, RecordConfiguration recordConfiguration) => targetRecords;
protected override async Task ConsumeAsyncInternal(IAsyncContext asyncContext, RecordConfiguration configuration, IAsyncChannel asyncChannel, CancellationToken cancellationToken)
{
IAsyncEnumerable <IAsyncRecord> asyncRecords;
if ((object)asyncContext == null)
{
throw new ArgumentNullException(nameof(asyncContext));
}
if ((object)configuration == null)
{
throw new ArgumentNullException(nameof(configuration));
}
if ((object)asyncChannel == null)
{
throw new ArgumentNullException(nameof(asyncChannel));
}
this.AssertValidConfiguration();
asyncRecords = asyncChannel.Records;
if ((object)asyncRecords == null)
{
throw new SyncPremException(nameof(asyncRecords));
}
await asyncRecords.ForceAsyncEnumeration(cancellationToken); // force execution
}
protected abstract Task <IAsyncChannel> ProduceAsyncInternal(IAsyncContext asyncContext, RecordConfiguration configuration, CancellationToken cancellationToken);
public IEnumerable <TargetRecord> Filter(IEnumerable <TargetRecord> targetRecords, RecordConfiguration recordConfiguration)
{
var recordNumber = 1;
foreach (var targetRecord in _innerFieldFilterService.Filter(targetRecords, recordConfiguration))
{
var recordFiltered = false;
try
{
foreach (var targetField in targetRecord.Fields
.Where(f => !f.Type.IsNullOrEmpty()))
{
targetField.TypedValue = targetField.Value.ConvertToType(targetField.Type.GetSystemType());
}
recordFiltered = true;
}
catch (Exception x) when(!TextWranglerConfig.OnException(x, $"Could not convert target record [{recordNumber}] fields to types specified"))
{
// OnException handler says not to rethrow, so keep on going, skipping this record
}
if (recordFiltered)
{
yield return(targetRecord);
}
recordNumber++;
}
}
protected abstract void PreExecuteInternal(ISyncContext context, RecordConfiguration configuration);
protected abstract Task <IAsyncChannel> ProcessAsyncInternal(IAsyncContext asyncContext, RecordConfiguration configuration, IAsyncChannel asyncChannel, AsyncProcessDelegate asyncNext, CancellationToken cancellationToken);
protected override async Task ConsumeMessageReaderAsync(IAsyncContext asyncContext, RecordConfiguration configuration, DbDataReader sourceDataReader, CancellationToken cancellationToken)
{
long recordCount = 0;
//SqlRowsCopiedEventHandler callback;
if ((object)asyncContext == null)
{
throw new ArgumentNullException(nameof(asyncContext));
}
if ((object)configuration == null)
{
throw new ArgumentNullException(nameof(configuration));
}
if ((object)sourceDataReader == null)
{
throw new ArgumentNullException(nameof(sourceDataReader));
}
this.AssertValidConfiguration();
AdoNetConnectorSpecificConfiguration fsConfig = this.Configuration.StageSpecificConfiguration;
if ((object)fsConfig.ExecuteCommand == null)
{
throw new InvalidOperationException(string.Format("Configuration missing: '{0}'.", nameof(fsConfig.ExecuteCommand)));
}
if (SolderFascadeAccessor.DataTypeFascade.IsNullOrWhiteSpace(fsConfig.ExecuteCommand.CommandText))
{
throw new InvalidOperationException(string.Format("Configuration missing: '{0}.{1}'.", nameof(fsConfig.ExecuteCommand), nameof(fsConfig.ExecuteCommand.CommandText)));
}
using (SqlBulkCopy sqlBulkCopy = new SqlBulkCopy((SqlConnection)this.DestinationUnitOfWork.Connection, SqlBulkCopyOptions.Default, (SqlTransaction)this.DestinationUnitOfWork.Transaction))
{
//callback = (sender, e) => Console.WriteLine(_rowsCopied = e.RowsCopied);
foreach (FieldConfiguration columnConfiguration in configuration.ColumnConfigurations)
{
sqlBulkCopy.ColumnMappings.Add(columnConfiguration.FieldName, columnConfiguration.FieldName);
}
sqlBulkCopy.EnableStreaming = true;
sqlBulkCopy.BatchSize = 2500;
//sqlBulkCopy.NotifyAfter = 2500;
//sqlBulkCopy.SqlRowsCopied += callback;
sqlBulkCopy.DestinationTableName = fsConfig.ExecuteCommand.CommandText;
await sqlBulkCopy.WriteToServerAsync(sourceDataReader, cancellationToken);
//sqlBulkCopy.SqlRowsCopied -= callback;
}
}
protected override async Task ConsumeMessageReaderAsync(IAsyncContext asyncContext, RecordConfiguration configuration, DbDataReader sourceDataReader, CancellationToken cancellationToken)
{
IAsyncEnumerable <IAdoNetStreamingResult> results;
IEnumerable <DbParameter> dbParameters;
long recordCount = 0;
if ((object)asyncContext == null)
{
throw new ArgumentNullException(nameof(asyncContext));
}
if ((object)configuration == null)
{
throw new ArgumentNullException(nameof(configuration));
}
if ((object)sourceDataReader == null)
{
throw new ArgumentNullException(nameof(sourceDataReader));
}
this.AssertValidConfiguration();
AdoNetConnectorSpecificConfiguration fsConfig = this.Configuration.StageSpecificConfiguration;
if ((object)fsConfig.ExecuteCommand == null)
{
throw new InvalidOperationException(string.Format("Configuration missing: '{0}'.", nameof(fsConfig.ExecuteCommand)));
}
if (SolderFascadeAccessor.DataTypeFascade.IsNullOrWhiteSpace(fsConfig.ExecuteCommand.CommandText))
{
throw new InvalidOperationException(string.Format("Configuration missing: '{0}.{1}'.", nameof(fsConfig.ExecuteCommand), nameof(fsConfig.ExecuteCommand.CommandText)));
}
do
{
while (await sourceDataReader.ReadAsync(cancellationToken))
{
dbParameters = fsConfig.ExecuteCommand.GetDbDataParameters(this.DestinationUnitOfWork);
dbParameters = dbParameters.Select(p =>
{
// prevent modified closure bug
DbDataReader _sourceDataReader = sourceDataReader;
// lazy load
p.Value = _sourceDataReader[p.SourceColumn];
return(p);
});
results = this.DestinationUnitOfWork.ExecuteResultsAsync(fsConfig.ExecuteCommand.CommandType ?? CommandType.Text, fsConfig.ExecuteCommand.CommandText, dbParameters, cancellationToken);
await results.ForceAsyncEnumeration(cancellationToken); // force execution
recordCount++;
}
}while (await sourceDataReader.NextResultAsync(cancellationToken));
//System.Console.WriteLine("DESTINATION (update): recordCount={0}", recordCount);
}
protected override async Task PreExecuteAsyncInternal(IAsyncContext asyncContext, RecordConfiguration configuration, CancellationToken cancellationToken)
{
SchemaBuilder schemaBuilder;
ISchema schema;
string line;
string[] fieldNames;
if ((object)asyncContext == null)
{
throw new ArgumentNullException(nameof(asyncContext));
}
if ((object)configuration == null)
{
throw new ArgumentNullException(nameof(configuration));
}
this.AssertValidConfiguration();
schemaBuilder = SchemaBuilder.Create();
await TextWriter.WriteLineAsync("Enter list of schema field names separated by pipe character: ");
line = await TextReader.ReadLineAsync();
if (!SolderFascadeAccessor.DataTypeFascade.IsNullOrEmpty(line))
{
fieldNames = line.Split('|');
if ((object)fieldNames == null || fieldNames.Length <= 0)
{
await TextWriter.WriteLineAsync("List of schema field names was invalid; using default (blank).");
schemaBuilder.AddField(string.Empty, typeof(string), false, true);
}
else
{
for (long fieldIndex = 0; fieldIndex < fieldNames.Length; fieldIndex++)
{
string fieldName;
fieldName = fieldNames[fieldIndex];
if ((fieldName ?? string.Empty).Trim() == string.Empty)
{
continue;
}
schemaBuilder.AddField(fieldName, typeof(string), false, true);
}
await TextWriter.WriteLineAsync(string.Format("Building KEY schema: '{0}'", string.Join(" | ", fieldNames)));
}
}
if (!asyncContext.LocalState.TryGetValue(this, out IDictionary <string, object> localState))
{
localState = new Dictionary <string, object>(StringComparer.OrdinalIgnoreCase);
asyncContext.LocalState.Add(this, localState);
}
schema = schemaBuilder.Build();
if ((object)schema == null)
{
throw new SyncPremException(nameof(schema));
}
localState.Add(Constants.ContextComponentScopedSchema, schema);
}
protected abstract TargetRecord FormatRecord(TargetRecord record, RecordConfiguration recordConfiguration);
protected override Task <IAsyncChannel> ProduceAsyncInternal(IAsyncContext asyncContext, RecordConfiguration configuration, CancellationToken cancellationToken)
{
IAsyncChannel asyncChannel;
ISchema schema;
IAsyncEnumerable <IPayload> payloads;
if ((object)asyncContext == null)
{
throw new ArgumentNullException(nameof(asyncContext));
}
if ((object)configuration == null)
{
throw new ArgumentNullException(nameof(configuration));
}
this.AssertValidConfiguration();
if (!asyncContext.LocalState.TryGetValue(this, out IDictionary <string, object> localState))
{
localState = new Dictionary <string, object>(StringComparer.OrdinalIgnoreCase);
asyncContext.LocalState.Add(this, localState);
}
schema = localState[Constants.ContextComponentScopedSchema] as ISchema;
if ((object)schema == null)
{
throw new SyncPremException(nameof(schema));
}
payloads = this.GetYieldViaConsoleAsync(schema, cancellationToken);
if ((object)payloads == null)
{
throw new SyncPremException(nameof(payloads));
}
var records = payloads.Select(rec => new DefaultAsyncRecord(schema, rec, string.Empty, Partition.None, Offset.None));
asyncChannel = asyncContext.CreateChannelAsync(records);
return(Task.FromResult(asyncChannel));
}
protected override async Task PostExecuteAsyncInternal(IAsyncContext asyncContext, RecordConfiguration configuration, CancellationToken cancellationToken)
{
IAsyncEnumerable <IAdoNetStreamingResult> results;
IEnumerable <DbParameter> dbParameters;
if ((object)asyncContext == null)
{
throw new ArgumentNullException(nameof(asyncContext));
}
if ((object)configuration == null)
{
throw new ArgumentNullException(nameof(configuration));
}
this.AssertValidConfiguration();
AdoNetConnectorSpecificConfiguration fsConfig = this.Configuration.StageSpecificConfiguration;
if (fsConfig.PostExecuteCommand != null &&
!SolderFascadeAccessor.DataTypeFascade.IsNullOrWhiteSpace(fsConfig.PostExecuteCommand.CommandText))
{
dbParameters = fsConfig.PostExecuteCommand.GetDbDataParameters(this.SourceUnitOfWork);
results = this.SourceUnitOfWork.ExecuteSchemaResultsAsync(fsConfig.PostExecuteCommand.CommandType ?? CommandType.Text,
fsConfig.PostExecuteCommand.CommandText,
dbParameters, cancellationToken);
if ((object)results == null)
{
throw new SyncPremException(nameof(results));
}
await results.ForceAsyncEnumeration(cancellationToken); // force execution
}
if ((object)this.SourceUnitOfWork != null)
{
this.SourceUnitOfWork.Dispose();
}
this.SourceUnitOfWork = null;
}
/// <summary> /// Formats each record in the incoming enumerable through each of this composite's <see cref="IFieldFormatter" />s /// </summary> /// <param name="records"></param> /// <param name="recordConfiguration"></param> /// <returns>Lazy enumerable of <see cref="TargetRecord" /></returns> public IEnumerable <TargetRecord> Format(IEnumerable <TargetRecord> records, RecordConfiguration recordConfiguration) => _formatters.Aggregate(records, (prs, rf) => rf.Format(prs, recordConfiguration));
protected override Task <IAsyncChannel> ProduceAsyncInternal(IAsyncContext asyncContext, RecordConfiguration configuration, CancellationToken cancellationToken)
{
IAsyncChannel asyncChannel;
IAsyncEnumerable <IAdoNetStreamingResult> results;
IEnumerable <DbParameter> dbParameters;
if ((object)asyncContext == null)
{
throw new ArgumentNullException(nameof(asyncContext));
}
if ((object)configuration == null)
{
throw new ArgumentNullException(nameof(configuration));
}
this.AssertValidConfiguration();
AdoNetConnectorSpecificConfiguration fsConfig = this.Configuration.StageSpecificConfiguration;
if ((object)fsConfig.ExecuteCommand == null)
{
throw new InvalidOperationException(string.Format("Configuration missing: '{0}'.", nameof(fsConfig.ExecuteCommand)));
}
if (SolderFascadeAccessor.DataTypeFascade.IsNullOrWhiteSpace(fsConfig.ExecuteCommand.CommandText))
{
throw new InvalidOperationException(string.Format("Configuration missing: '{0}.{1}'.", nameof(fsConfig.ExecuteCommand), nameof(fsConfig.ExecuteCommand.CommandText)));
}
dbParameters = fsConfig.ExecuteCommand.GetDbDataParameters(this.SourceUnitOfWork);
results = this.SourceUnitOfWork.ExecuteResultsAsync(fsConfig.ExecuteCommand.CommandType ?? CommandType.Text, fsConfig.ExecuteCommand.CommandText, dbParameters, cancellationToken);
if ((object)results == null)
{
throw new SyncPremException(nameof(results));
}
var records = this.GetMultiplexedRecords(asyncContext, results, cancellationToken);
asyncChannel = asyncContext.CreateChannelAsync(records);
return(Task.FromResult(asyncChannel));
}
/// <summary>
/// Lazily writes the CSV from the enumerable provided.
/// If the configuration specifies inclusion of a header record, it will be written _before_ the records to be written begin enumeration, so if
/// there are zero records in the target provided, a file with just a header will be written.
/// </summary>
/// <param name="targetRecords"></param>
/// <param name="recordConfiguration"></param>
/// <returns>A lazily-produced enumerable of records sucessfully written</returns>
public IEnumerable <TargetRecord> Write(IEnumerable <TargetRecord> targetRecords, RecordConfiguration recordConfiguration)
{
// Write the header if appropriate
if (_csvWriter.Configuration.HasHeaderRecord)
{
try
{
foreach (var headerField in recordConfiguration.Fields
.Select(f => f.Name))
{
_csvWriter.WriteField(headerField);
}
_csvWriter.NextRecord();
}
catch (Exception x) when(!TextWranglerConfig.OnException(x, "Could not write CSV header record"))
{
// OnException handler says not to rethrow, so keep on going, skipping this record
}
}
// Now the records themselves
foreach (var targetRecord in targetRecords)
{
// CountProcessed is the index of the record currently being written...
CountProcessed++;
var recordWritten = false;
try
{
foreach (var targetFieldValue in targetRecord.Fields
.Select(f => f.TypedValue))
{
_csvWriter.WriteField(targetFieldValue);
}
_csvWriter.NextRecord();
recordWritten = true;
}
catch (Exception x) when(!TextWranglerConfig.OnException(x, "Could not write CSV header record"))
{
// OnException handler says not to rethrow, so keep on going, skipping this record
}
if (recordWritten)
{
yield return(targetRecord);
}
else
{
CountFail++;
}
}
}
protected abstract ISyncChannel ProcessInternal(ISyncContext context, RecordConfiguration configuration, ISyncChannel channel, SyncProcessDelegate next);
private Task <IAsyncChannel> TransformAsync(IAsyncContext asyncContext, RecordConfiguration configuration, IAsyncChannel asyncChannel, CancellationToken cancellationToken)
{
Console.WriteLine("voo doo!");
return(this.AsyncProcessToNext(asyncContext, configuration, asyncChannel, this.AsyncNext, cancellationToken));
}
protected abstract void ConsumeMessageReader(ISyncContext context, RecordConfiguration configuration, DbDataReader sourceDataReader);
/// <summary>
/// Formats the given records for the given configuration. Replaces delimited source-field names in the target fields with
/// either the source field value (for non-format delimited fields) or the ordinal position of the source field in the configuration
/// for use with string.format formatting
/// </summary>
/// <param name="targetRecords">Lazily evaluated enumerable of populated <see cref="TargetRecord" /> records</param>
/// <param name="recordConfiguration">The <see cref="RecordConfiguration" /> being used to transform these source/target records</param>
/// <returns>Enumerable of <see cref="TargetRecord" /></returns>
public IEnumerable <TargetRecord> Format(IEnumerable <TargetRecord> targetRecords, RecordConfiguration recordConfiguration)
{
var sourceFieldReplacementFormatMap = new Dictionary <string, (string Static, string Format)>(StringComparer.OrdinalIgnoreCase);
foreach (var targetRecord in targetRecords)
{
var recordNumber = 1;
var targetRecordFormatted = false;
try
{
foreach (var targetField in targetRecord.Fields
.Where(f => !f.Sources.IsNullOrEmpty()))
{
var sourceFieldIndex = 0;
foreach (var sourceValue in targetField.Sources)
{
// Build up a map of source field values to the delimited string we use to replace it with
// This is simply a bit of a memory-management/GC optimization so we don't have a ton of string objects getting created/destroyed on large files
if (!sourceFieldReplacementFormatMap.ContainsKey(sourceValue.Name))
{
sourceFieldReplacementFormatMap.Add(sourceValue.Name, (string.Concat("<", sourceValue.Name, ">"),
string.Concat("{<", sourceValue.Name, ">")));
}
// Another simple optimization of memory/gc at the expense of an extra lookup into the field value to avoid creating a new string unnecessarily
if (targetField.Value.IndexOf(sourceFieldReplacementFormatMap[sourceValue.Name].Static, StringComparison.OrdinalIgnoreCase) < 0)
{
continue;
}
// Replace all instances of the format-string delimited source field name with the index of the source field within the
// target field config list (for those that are being used in a format-string case)
// OR the raw value (for those that are not being used in a format string case)
targetField.Value = targetField.Value
.Replace(sourceFieldReplacementFormatMap[sourceValue.Name].Format,
string.Concat("{", sourceFieldIndex))
.Replace(sourceFieldReplacementFormatMap[sourceValue.Name].Static,
sourceValue.Value);
sourceFieldIndex++;
}
}
targetRecordFormatted = true;
}
catch (Exception x) when(!TextWranglerConfig.OnException(x, $"Could not format target record in [{GetType().Name}] from built record [{recordNumber}]"))
{
// OnException handler says not to rethrow, so keep on going, skipping this record
}
if (targetRecordFormatted)
{
yield return(targetRecord);
}
recordNumber++;
}
}
protected override async Task ConsumeAsyncInternal(IAsyncContext asyncContext, RecordConfiguration configuration, IAsyncChannel asyncChannel, CancellationToken cancellationToken)
{
IAsyncEnumerable <IRecord> records;
if ((object)asyncContext == null)
{
throw new ArgumentNullException(nameof(asyncContext));
}
if ((object)configuration == null)
{
throw new ArgumentNullException(nameof(configuration));
}
if ((object)asyncChannel == null)
{
throw new ArgumentNullException(nameof(asyncChannel));
}
this.AssertValidConfiguration();
RestfulWebApiConnectorSpecificConfiguration fsConfig = this.Configuration.StageSpecificConfiguration;
records = asyncChannel.Records;
if ((object)records == null)
{
throw new SyncPremException(nameof(records));
}
using (HttpClient httpClient = new HttpClient())
{
using (HttpContent httpContent = new PushStreamContent((s) => this.SerializeRecordsToStream(s, records)))
{
using (HttpResponseMessage result = await httpClient.PostAsync(fsConfig.WebEndpointUri, httpContent))
{
await result.Content.ReadAsStreamAsync();
result.EnsureSuccessStatusCode();
}
}
}
}
protected abstract Task PreExecuteAsyncInternal(IAsyncContext asyncContext, RecordConfiguration configuration, CancellationToken cancellationToken);