public void PreExecute(ISyncContext context, RecordConfiguration configuration)
 {
     this.PreExecuteInternal(context, configuration);
 }
        protected override void PreExecuteInternal(ISyncContext context, RecordConfiguration configuration)
        {
            SchemaBuilder schemaBuilder;
            ISchema       schema;
            IEnumerable <TTextualFieldSpec> headers;
            TTextualSpec spec;

            if ((object)context == null)
            {
                throw new ArgumentNullException(nameof(context));
            }

            if ((object)configuration == null)
            {
                throw new ArgumentNullException(nameof(configuration));
            }

            this.AssertValidConfiguration();

            TTextualConnectorSpecificConfiguration fsConfig = this.Configuration.StageSpecificConfiguration;

            spec = fsConfig.TextualConfiguration.MapToSpec();

            if ((object)spec == null)
            {
                throw new SyncPremException(nameof(spec));
            }

            this.TextualReader = this.CreateTextualReader(new StreamReader(File.Open(fsConfig.TextualFilePath, FileMode.Open, FileAccess.Read, FileShare.Read)), spec);

            headers = this.TextualReader.ReadHeaderFields();

            if ((object)headers == null)
            {
                throw new SyncPremException(nameof(headers));
            }

            if (!context.LocalState.TryGetValue(this, out IDictionary <string, object> localState))
            {
                localState = new Dictionary <string, object>(StringComparer.OrdinalIgnoreCase);
                context.LocalState.Add(this, localState);
            }

            schemaBuilder = SchemaBuilder.Create();

            headers.ForceEnumeration();

            foreach (TTextualFieldSpec header in spec.TextualHeaderSpecs)
            {
                schemaBuilder.AddField(header.FieldTitle, header.FieldType.ToClrType(), header.IsFieldRequired, header.IsFieldIdentity);
            }

            schema = schemaBuilder.Build();

            if ((object)schema == null)
            {
                throw new SyncPremException(nameof(schema));
            }

            localState.Add(Constants.ContextComponentScopedSchema, schema);
        }
Exemple #3
0
        public IEnumerable <TargetRecord> Write(IEnumerable <TargetRecord> targetRecords, RecordConfiguration recordConfiguration)
        {
            TargetRecords.AddRange(targetRecords);

            return(TargetRecords);
        }
Exemple #4
0
        protected override async Task PreExecuteAsyncInternal(IAsyncContext asyncContext, RecordConfiguration configuration, CancellationToken cancellationToken)
        {
            SchemaBuilder   schemaBuilder;
            ISchema         schema;
            IList <ISchema> schemas;

            IAsyncEnumerable <IAdoNetStreamingResult> results;
            IAsyncEnumerator <IAdoNetStreamingResult> resultz;

            IAsyncEnumerable <IPayload> records;
            IAsyncEnumerator <IPayload> recordz;

            IEnumerable <DbParameter> dbParameters;

            if ((object)asyncContext == null)
            {
                throw new ArgumentNullException(nameof(asyncContext));
            }

            if ((object)configuration == null)
            {
                throw new ArgumentNullException(nameof(configuration));
            }

            this.AssertValidConfiguration();

            AdoNetConnectorSpecificConfiguration fsConfig = this.Configuration.StageSpecificConfiguration;

            this.SourceUnitOfWork = fsConfig.GetUnitOfWork();

            if (fsConfig.PreExecuteCommand != null &&
                !SolderFascadeAccessor.DataTypeFascade.IsNullOrWhiteSpace(fsConfig.PreExecuteCommand.CommandText))
            {
                dbParameters = fsConfig.PreExecuteCommand.GetDbDataParameters(this.SourceUnitOfWork);

                results = this.SourceUnitOfWork.ExecuteResultsAsync(fsConfig.PreExecuteCommand.CommandType ?? CommandType.Text,
                                                                    fsConfig.PreExecuteCommand.CommandText,
                                                                    dbParameters, cancellationToken);

                if ((object)results == null)
                {
                    throw new SyncPremException(nameof(results));
                }

                await results.ForceAsyncEnumeration(cancellationToken);                 // force execution
            }

            // execute schema only
            if (fsConfig.ExecuteCommand != null &&
                !SolderFascadeAccessor.DataTypeFascade.IsNullOrWhiteSpace(fsConfig.ExecuteCommand.CommandText))
            {
                dbParameters = fsConfig.ExecuteCommand.GetDbDataParameters(this.SourceUnitOfWork);

                results = this.SourceUnitOfWork.ExecuteSchemaResultsAsync(fsConfig.ExecuteCommand.CommandType ?? CommandType.Text,
                                                                          fsConfig.ExecuteCommand.CommandText,
                                                                          dbParameters, cancellationToken);

                if ((object)results == null)
                {
                    throw new SyncPremException(nameof(results));
                }

                resultz = results.GetEnumerator();

                if ((object)resultz == null)
                {
                    throw new SyncPremException(nameof(resultz));
                }

                schemas = new List <ISchema>();

                while (await resultz.MoveNext(cancellationToken))
                {
                    IAdoNetStreamingResult result = resultz.Current;

                    records = result.AsyncRecords;

                    if ((object)records == null)
                    {
                        throw new SyncPremException(nameof(results));
                    }

                    recordz = records.GetEnumerator();

                    if ((object)recordz == null)
                    {
                        throw new SyncPremException(nameof(recordz));
                    }

                    schemaBuilder = SchemaBuilder.Create();

                    while (await recordz.MoveNext(cancellationToken))
                    {
                        IPayload record = recordz.Current;

                        string fieldName;
                        Type   fieldType;
                        bool   isKey;
                        bool   isNullable;

                        fieldName  = (string)record[nameof(DbColumn.ColumnName)];
                        fieldType  = (Type)record[nameof(DbColumn.DataType)];
                        isKey      = (bool?)record[nameof(DbColumn.IsKey)] ?? false;
                        isNullable = (bool?)record[nameof(DbColumn.AllowDBNull)] ?? true;

                        // TODO ensure nullable type
                        schemaBuilder.AddField(fieldName, fieldType, isNullable, isKey);
                    }

                    schema = schemaBuilder.Build();

                    if ((object)schema == null)
                    {
                        throw new SyncPremException(nameof(schema));
                    }

                    schemas.Add(schema);
                }

                if (!asyncContext.LocalState.TryGetValue(this, out IDictionary <string, object> localState))
                {
                    localState = new Dictionary <string, object>(StringComparer.OrdinalIgnoreCase);
                    asyncContext.LocalState.Add(this, localState);
                }

                localState.Add(Constants.ContextComponentScopedSchema, schemas);
            }
        }
 protected abstract ISyncChannel ProduceInternal(ISyncContext context, RecordConfiguration configuration);
Exemple #6
0
        /// <summary>
        /// Writes targetRecords to configured logger for this type
        /// </summary>
        /// <param name="targetRecords"></param>
        /// <param name="recordConfiguration"></param>
        /// <returns>Lazy enumerable of <see cref="TargetRecord" /></returns>
        public IEnumerable <TargetRecord> Write(IEnumerable <TargetRecord> targetRecords, RecordConfiguration recordConfiguration)
        {
            var recordNumber   = 1;
            var logText        = new StringBuilder();
            var maxFieldLength = int.MinValue;

            foreach (var targetRecord in targetRecords)
            {
                if (maxFieldLength <= 0)
                {
                    maxFieldLength = targetRecord.Fields.Max(f => f.Name.Length) + 2;
                }

                logText.AppendLine($"TARGET RECORD #[{recordNumber.ToString(CultureInfo.InvariantCulture).PadLeft(4, '0')}]");
                logText.AppendLine("--------------------------------------------------------------------");

                foreach (var targetField in targetRecord.Fields)
                {
                    logText.AppendLine($"{targetField.Name.PadLeft(maxFieldLength)}:\t{targetField.TypedValue} ({targetField.TypedValue.GetType().Name})");
                }

                _logger.LogInformation(logText.ToString());

                logText.Clear();

                yield return(targetRecord);

                recordNumber++;
            }
        }
        /// <summary>
        /// Formats the given records for the given configuration.
        /// </summary>
        /// <param name="targetRecords">Lazily evaluated enumerable of populated <see cref="TargetRecord" /> records</param>
        /// <param name="recordConfiguration">The <see cref="RecordConfiguration" /> being used to transform these source/target records</param>
        /// <returns>Enumerable of <see cref="TargetRecord" /></returns>
        public IEnumerable <TargetRecord> Format(IEnumerable <TargetRecord> targetRecords, RecordConfiguration recordConfiguration)
        {
            var recordNumber = 1;

            foreach (var targetRecord in targetRecords)
            {
                TargetRecord yieldRecord = null;

                try
                {
                    yieldRecord = FormatRecord(targetRecord, recordConfiguration);
                }
                catch (Exception x) when(!TextWranglerConfig.OnException(x, $"Could not format target record in [{_formatterName}] from built record [{recordNumber}]"))
                {
                    // OnException handler says not to rethrow, so keep on going, skipping this record
                }

                if (yieldRecord != null)
                {
                    yield return(yieldRecord);
                }

                recordNumber++;
            }
        }
        protected override async Task PreExecuteAsyncInternal(IAsyncContext asyncContext, RecordConfiguration configuration, CancellationToken cancellationToken)
        {
            ISchema schema;

            if ((object)asyncContext == null)
            {
                throw new ArgumentNullException(nameof(asyncContext));
            }

            if ((object)configuration == null)
            {
                throw new ArgumentNullException(nameof(configuration));
            }

            this.AssertValidConfiguration();

            if (!asyncContext.LocalState.TryGetValue(this, out IDictionary <string, object> localState))
            {
                localState = new Dictionary <string, object>(StringComparer.OrdinalIgnoreCase);
                asyncContext.LocalState.Add(this, localState);
            }

            schema = await GetRandomSchemaAsync(cancellationToken);

            if ((object)schema == null)
            {
                throw new SyncPremException(nameof(schema));
            }

            localState.Add(Constants.ContextComponentScopedSchema, schema);
        }
        protected override async Task ConsumeAsyncInternal(IAsyncContext asyncContext, RecordConfiguration configuration, IAsyncChannel asyncChannel, CancellationToken cancellationToken)
        {
            IAsyncEnumerable <IAsyncRecord> records;

            if ((object)asyncContext == null)
            {
                throw new ArgumentNullException(nameof(asyncContext));
            }

            if ((object)configuration == null)
            {
                throw new ArgumentNullException(nameof(configuration));
            }

            if ((object)asyncChannel == null)
            {
                throw new ArgumentNullException(nameof(asyncChannel));
            }

            this.AssertValidConfiguration();

            records = asyncChannel.Records;

            if ((object)records == null)
            {
                throw new SyncPremException(nameof(records));
            }

            IAsyncEnumerator <IAsyncRecord> recordz;

            recordz = records.GetEnumerator();

            if ((object)recordz == null)
            {
                throw new InvalidOperationException(nameof(recordz));
            }

            while (await recordz.MoveNext(cancellationToken))
            {
                IAsyncRecord record = recordz.Current;
                TextWriter.WriteLine(record);
            }
        }
Exemple #10
0
 public IEnumerable <TargetRecord> Write(IEnumerable <TargetRecord> targetRecords, RecordConfiguration recordConfiguration)
 => targetRecords;
        protected override async Task ConsumeAsyncInternal(IAsyncContext asyncContext, RecordConfiguration configuration, IAsyncChannel asyncChannel, CancellationToken cancellationToken)
        {
            IAsyncEnumerable <IAsyncRecord> asyncRecords;

            if ((object)asyncContext == null)
            {
                throw new ArgumentNullException(nameof(asyncContext));
            }

            if ((object)configuration == null)
            {
                throw new ArgumentNullException(nameof(configuration));
            }

            if ((object)asyncChannel == null)
            {
                throw new ArgumentNullException(nameof(asyncChannel));
            }

            this.AssertValidConfiguration();

            asyncRecords = asyncChannel.Records;

            if ((object)asyncRecords == null)
            {
                throw new SyncPremException(nameof(asyncRecords));
            }

            await asyncRecords.ForceAsyncEnumeration(cancellationToken);             // force execution
        }
 protected abstract Task <IAsyncChannel> ProduceAsyncInternal(IAsyncContext asyncContext, RecordConfiguration configuration, CancellationToken cancellationToken);
        public IEnumerable <TargetRecord> Filter(IEnumerable <TargetRecord> targetRecords, RecordConfiguration recordConfiguration)
        {
            var recordNumber = 1;

            foreach (var targetRecord in _innerFieldFilterService.Filter(targetRecords, recordConfiguration))
            {
                var recordFiltered = false;

                try
                {
                    foreach (var targetField in targetRecord.Fields
                             .Where(f => !f.Type.IsNullOrEmpty()))
                    {
                        targetField.TypedValue = targetField.Value.ConvertToType(targetField.Type.GetSystemType());
                    }

                    recordFiltered = true;
                }
                catch (Exception x) when(!TextWranglerConfig.OnException(x, $"Could not convert target record [{recordNumber}] fields to types specified"))
                {
                    // OnException handler says not to rethrow, so keep on going, skipping this record
                }

                if (recordFiltered)
                {
                    yield return(targetRecord);
                }

                recordNumber++;
            }
        }
 protected abstract void PreExecuteInternal(ISyncContext context, RecordConfiguration configuration);
 protected abstract Task <IAsyncChannel> ProcessAsyncInternal(IAsyncContext asyncContext, RecordConfiguration configuration, IAsyncChannel asyncChannel, AsyncProcessDelegate asyncNext, CancellationToken cancellationToken);
Exemple #16
0
        protected override async Task ConsumeMessageReaderAsync(IAsyncContext asyncContext, RecordConfiguration configuration, DbDataReader sourceDataReader, CancellationToken cancellationToken)
        {
            long recordCount = 0;

            //SqlRowsCopiedEventHandler callback;

            if ((object)asyncContext == null)
            {
                throw new ArgumentNullException(nameof(asyncContext));
            }

            if ((object)configuration == null)
            {
                throw new ArgumentNullException(nameof(configuration));
            }

            if ((object)sourceDataReader == null)
            {
                throw new ArgumentNullException(nameof(sourceDataReader));
            }

            this.AssertValidConfiguration();

            AdoNetConnectorSpecificConfiguration fsConfig = this.Configuration.StageSpecificConfiguration;

            if ((object)fsConfig.ExecuteCommand == null)
            {
                throw new InvalidOperationException(string.Format("Configuration missing: '{0}'.", nameof(fsConfig.ExecuteCommand)));
            }

            if (SolderFascadeAccessor.DataTypeFascade.IsNullOrWhiteSpace(fsConfig.ExecuteCommand.CommandText))
            {
                throw new InvalidOperationException(string.Format("Configuration missing: '{0}.{1}'.", nameof(fsConfig.ExecuteCommand), nameof(fsConfig.ExecuteCommand.CommandText)));
            }

            using (SqlBulkCopy sqlBulkCopy = new SqlBulkCopy((SqlConnection)this.DestinationUnitOfWork.Connection, SqlBulkCopyOptions.Default, (SqlTransaction)this.DestinationUnitOfWork.Transaction))
            {
                //callback = (sender, e) => Console.WriteLine(_rowsCopied = e.RowsCopied);

                foreach (FieldConfiguration columnConfiguration in configuration.ColumnConfigurations)
                {
                    sqlBulkCopy.ColumnMappings.Add(columnConfiguration.FieldName, columnConfiguration.FieldName);
                }

                sqlBulkCopy.EnableStreaming = true;
                sqlBulkCopy.BatchSize       = 2500;
                //sqlBulkCopy.NotifyAfter = 2500;
                //sqlBulkCopy.SqlRowsCopied += callback;
                sqlBulkCopy.DestinationTableName = fsConfig.ExecuteCommand.CommandText;

                await sqlBulkCopy.WriteToServerAsync(sourceDataReader, cancellationToken);

                //sqlBulkCopy.SqlRowsCopied -= callback;
            }
        }
        protected override async Task ConsumeMessageReaderAsync(IAsyncContext asyncContext, RecordConfiguration configuration, DbDataReader sourceDataReader, CancellationToken cancellationToken)
        {
            IAsyncEnumerable <IAdoNetStreamingResult> results;
            IEnumerable <DbParameter> dbParameters;
            long recordCount = 0;

            if ((object)asyncContext == null)
            {
                throw new ArgumentNullException(nameof(asyncContext));
            }

            if ((object)configuration == null)
            {
                throw new ArgumentNullException(nameof(configuration));
            }

            if ((object)sourceDataReader == null)
            {
                throw new ArgumentNullException(nameof(sourceDataReader));
            }

            this.AssertValidConfiguration();

            AdoNetConnectorSpecificConfiguration fsConfig = this.Configuration.StageSpecificConfiguration;

            if ((object)fsConfig.ExecuteCommand == null)
            {
                throw new InvalidOperationException(string.Format("Configuration missing: '{0}'.", nameof(fsConfig.ExecuteCommand)));
            }

            if (SolderFascadeAccessor.DataTypeFascade.IsNullOrWhiteSpace(fsConfig.ExecuteCommand.CommandText))
            {
                throw new InvalidOperationException(string.Format("Configuration missing: '{0}.{1}'.", nameof(fsConfig.ExecuteCommand), nameof(fsConfig.ExecuteCommand.CommandText)));
            }

            do
            {
                while (await sourceDataReader.ReadAsync(cancellationToken))
                {
                    dbParameters = fsConfig.ExecuteCommand.GetDbDataParameters(this.DestinationUnitOfWork);

                    dbParameters = dbParameters.Select(p =>
                    {
                        // prevent modified closure bug
                        DbDataReader _sourceDataReader = sourceDataReader;
                        // lazy load
                        p.Value = _sourceDataReader[p.SourceColumn];
                        return(p);
                    });

                    results = this.DestinationUnitOfWork.ExecuteResultsAsync(fsConfig.ExecuteCommand.CommandType ?? CommandType.Text, fsConfig.ExecuteCommand.CommandText, dbParameters, cancellationToken);

                    await results.ForceAsyncEnumeration(cancellationToken);                     // force execution

                    recordCount++;
                }
            }while (await sourceDataReader.NextResultAsync(cancellationToken));

            //System.Console.WriteLine("DESTINATION (update): recordCount={0}", recordCount);
        }
Exemple #18
0
        protected override async Task PreExecuteAsyncInternal(IAsyncContext asyncContext, RecordConfiguration configuration, CancellationToken cancellationToken)
        {
            SchemaBuilder schemaBuilder;
            ISchema       schema;

            string line;

            string[] fieldNames;

            if ((object)asyncContext == null)
            {
                throw new ArgumentNullException(nameof(asyncContext));
            }

            if ((object)configuration == null)
            {
                throw new ArgumentNullException(nameof(configuration));
            }

            this.AssertValidConfiguration();

            schemaBuilder = SchemaBuilder.Create();

            await TextWriter.WriteLineAsync("Enter list of schema field names separated by pipe character: ");

            line = await TextReader.ReadLineAsync();

            if (!SolderFascadeAccessor.DataTypeFascade.IsNullOrEmpty(line))
            {
                fieldNames = line.Split('|');

                if ((object)fieldNames == null || fieldNames.Length <= 0)
                {
                    await TextWriter.WriteLineAsync("List of schema field names was invalid; using default (blank).");

                    schemaBuilder.AddField(string.Empty, typeof(string), false, true);
                }
                else
                {
                    for (long fieldIndex = 0; fieldIndex < fieldNames.Length; fieldIndex++)
                    {
                        string fieldName;

                        fieldName = fieldNames[fieldIndex];

                        if ((fieldName ?? string.Empty).Trim() == string.Empty)
                        {
                            continue;
                        }

                        schemaBuilder.AddField(fieldName, typeof(string), false, true);
                    }

                    await TextWriter.WriteLineAsync(string.Format("Building KEY schema: '{0}'", string.Join(" | ", fieldNames)));
                }
            }

            if (!asyncContext.LocalState.TryGetValue(this, out IDictionary <string, object> localState))
            {
                localState = new Dictionary <string, object>(StringComparer.OrdinalIgnoreCase);
                asyncContext.LocalState.Add(this, localState);
            }

            schema = schemaBuilder.Build();

            if ((object)schema == null)
            {
                throw new SyncPremException(nameof(schema));
            }

            localState.Add(Constants.ContextComponentScopedSchema, schema);
        }
 protected abstract TargetRecord FormatRecord(TargetRecord record, RecordConfiguration recordConfiguration);
Exemple #20
0
        protected override Task <IAsyncChannel> ProduceAsyncInternal(IAsyncContext asyncContext, RecordConfiguration configuration, CancellationToken cancellationToken)
        {
            IAsyncChannel asyncChannel;
            ISchema       schema;
            IAsyncEnumerable <IPayload> payloads;

            if ((object)asyncContext == null)
            {
                throw new ArgumentNullException(nameof(asyncContext));
            }

            if ((object)configuration == null)
            {
                throw new ArgumentNullException(nameof(configuration));
            }

            this.AssertValidConfiguration();

            if (!asyncContext.LocalState.TryGetValue(this, out IDictionary <string, object> localState))
            {
                localState = new Dictionary <string, object>(StringComparer.OrdinalIgnoreCase);
                asyncContext.LocalState.Add(this, localState);
            }

            schema = localState[Constants.ContextComponentScopedSchema] as ISchema;

            if ((object)schema == null)
            {
                throw new SyncPremException(nameof(schema));
            }

            payloads = this.GetYieldViaConsoleAsync(schema, cancellationToken);

            if ((object)payloads == null)
            {
                throw new SyncPremException(nameof(payloads));
            }

            var records = payloads.Select(rec => new DefaultAsyncRecord(schema, rec, string.Empty, Partition.None, Offset.None));

            asyncChannel = asyncContext.CreateChannelAsync(records);

            return(Task.FromResult(asyncChannel));
        }
Exemple #21
0
        protected override async Task PostExecuteAsyncInternal(IAsyncContext asyncContext, RecordConfiguration configuration, CancellationToken cancellationToken)
        {
            IAsyncEnumerable <IAdoNetStreamingResult> results;
            IEnumerable <DbParameter> dbParameters;

            if ((object)asyncContext == null)
            {
                throw new ArgumentNullException(nameof(asyncContext));
            }

            if ((object)configuration == null)
            {
                throw new ArgumentNullException(nameof(configuration));
            }

            this.AssertValidConfiguration();

            AdoNetConnectorSpecificConfiguration fsConfig = this.Configuration.StageSpecificConfiguration;

            if (fsConfig.PostExecuteCommand != null &&
                !SolderFascadeAccessor.DataTypeFascade.IsNullOrWhiteSpace(fsConfig.PostExecuteCommand.CommandText))
            {
                dbParameters = fsConfig.PostExecuteCommand.GetDbDataParameters(this.SourceUnitOfWork);

                results = this.SourceUnitOfWork.ExecuteSchemaResultsAsync(fsConfig.PostExecuteCommand.CommandType ?? CommandType.Text,
                                                                          fsConfig.PostExecuteCommand.CommandText,
                                                                          dbParameters, cancellationToken);

                if ((object)results == null)
                {
                    throw new SyncPremException(nameof(results));
                }

                await results.ForceAsyncEnumeration(cancellationToken);                 // force execution
            }

            if ((object)this.SourceUnitOfWork != null)
            {
                this.SourceUnitOfWork.Dispose();
            }

            this.SourceUnitOfWork = null;
        }
 /// <summary>
 /// Formats each record in the incoming enumerable through each of this composite's <see cref="IFieldFormatter" />s
 /// </summary>
 /// <param name="records"></param>
 /// <param name="recordConfiguration"></param>
 /// <returns>Lazy enumerable of <see cref="TargetRecord" /></returns>
 public IEnumerable <TargetRecord> Format(IEnumerable <TargetRecord> records, RecordConfiguration recordConfiguration)
 => _formatters.Aggregate(records, (prs, rf) => rf.Format(prs, recordConfiguration));
Exemple #23
0
        protected override Task <IAsyncChannel> ProduceAsyncInternal(IAsyncContext asyncContext, RecordConfiguration configuration, CancellationToken cancellationToken)
        {
            IAsyncChannel asyncChannel;

            IAsyncEnumerable <IAdoNetStreamingResult> results;
            IEnumerable <DbParameter> dbParameters;

            if ((object)asyncContext == null)
            {
                throw new ArgumentNullException(nameof(asyncContext));
            }

            if ((object)configuration == null)
            {
                throw new ArgumentNullException(nameof(configuration));
            }

            this.AssertValidConfiguration();

            AdoNetConnectorSpecificConfiguration fsConfig = this.Configuration.StageSpecificConfiguration;

            if ((object)fsConfig.ExecuteCommand == null)
            {
                throw new InvalidOperationException(string.Format("Configuration missing: '{0}'.", nameof(fsConfig.ExecuteCommand)));
            }

            if (SolderFascadeAccessor.DataTypeFascade.IsNullOrWhiteSpace(fsConfig.ExecuteCommand.CommandText))
            {
                throw new InvalidOperationException(string.Format("Configuration missing: '{0}.{1}'.", nameof(fsConfig.ExecuteCommand), nameof(fsConfig.ExecuteCommand.CommandText)));
            }

            dbParameters = fsConfig.ExecuteCommand.GetDbDataParameters(this.SourceUnitOfWork);

            results = this.SourceUnitOfWork.ExecuteResultsAsync(fsConfig.ExecuteCommand.CommandType ?? CommandType.Text, fsConfig.ExecuteCommand.CommandText, dbParameters, cancellationToken);

            if ((object)results == null)
            {
                throw new SyncPremException(nameof(results));
            }

            var records = this.GetMultiplexedRecords(asyncContext, results, cancellationToken);

            asyncChannel = asyncContext.CreateChannelAsync(records);

            return(Task.FromResult(asyncChannel));
        }
Exemple #24
0
        /// <summary>
        /// Lazily writes the CSV from the enumerable provided.
        /// If the configuration specifies inclusion of a header record, it will be written _before_ the records to be written begin enumeration, so if
        /// there are zero records in the target provided, a file with just a header will be written.
        /// </summary>
        /// <param name="targetRecords"></param>
        /// <param name="recordConfiguration"></param>
        /// <returns>A lazily-produced enumerable of records sucessfully written</returns>
        public IEnumerable <TargetRecord> Write(IEnumerable <TargetRecord> targetRecords, RecordConfiguration recordConfiguration)
        {
            // Write the header if appropriate
            if (_csvWriter.Configuration.HasHeaderRecord)
            {
                try
                {
                    foreach (var headerField in recordConfiguration.Fields
                             .Select(f => f.Name))
                    {
                        _csvWriter.WriteField(headerField);
                    }

                    _csvWriter.NextRecord();
                }
                catch (Exception x) when(!TextWranglerConfig.OnException(x, "Could not write CSV header record"))
                {
                    // OnException handler says not to rethrow, so keep on going, skipping this record
                }
            }

            // Now the records themselves
            foreach (var targetRecord in targetRecords)
            {
                // CountProcessed is the index of the record currently being written...
                CountProcessed++;

                var recordWritten = false;

                try
                {
                    foreach (var targetFieldValue in targetRecord.Fields
                             .Select(f => f.TypedValue))
                    {
                        _csvWriter.WriteField(targetFieldValue);
                    }

                    _csvWriter.NextRecord();

                    recordWritten = true;
                }
                catch (Exception x) when(!TextWranglerConfig.OnException(x, "Could not write CSV header record"))
                {
                    // OnException handler says not to rethrow, so keep on going, skipping this record
                }

                if (recordWritten)
                {
                    yield return(targetRecord);
                }
                else
                {
                    CountFail++;
                }
            }
        }
Exemple #25
0
 protected abstract ISyncChannel ProcessInternal(ISyncContext context, RecordConfiguration configuration, ISyncChannel channel, SyncProcessDelegate next);
Exemple #26
0
 private Task <IAsyncChannel> TransformAsync(IAsyncContext asyncContext, RecordConfiguration configuration, IAsyncChannel asyncChannel, CancellationToken cancellationToken)
 {
     Console.WriteLine("voo doo!");
     return(this.AsyncProcessToNext(asyncContext, configuration, asyncChannel, this.AsyncNext, cancellationToken));
 }
Exemple #27
0
 protected abstract void ConsumeMessageReader(ISyncContext context, RecordConfiguration configuration, DbDataReader sourceDataReader);
        /// <summary>
        /// Formats the given records for the given configuration. Replaces delimited source-field names in the target fields with
        /// either the source field value (for non-format delimited fields) or the ordinal position of the source field in the configuration
        /// for use with string.format formatting
        /// </summary>
        /// <param name="targetRecords">Lazily evaluated enumerable of populated <see cref="TargetRecord" /> records</param>
        /// <param name="recordConfiguration">The <see cref="RecordConfiguration" /> being used to transform these source/target records</param>
        /// <returns>Enumerable of <see cref="TargetRecord" /></returns>
        public IEnumerable <TargetRecord> Format(IEnumerable <TargetRecord> targetRecords, RecordConfiguration recordConfiguration)
        {
            var sourceFieldReplacementFormatMap = new Dictionary <string, (string Static, string Format)>(StringComparer.OrdinalIgnoreCase);

            foreach (var targetRecord in targetRecords)
            {
                var recordNumber          = 1;
                var targetRecordFormatted = false;

                try
                {
                    foreach (var targetField in targetRecord.Fields
                             .Where(f => !f.Sources.IsNullOrEmpty()))
                    {
                        var sourceFieldIndex = 0;

                        foreach (var sourceValue in targetField.Sources)
                        {
                            // Build up a map of source field values to the delimited string we use to replace it with
                            // This is simply a bit of a memory-management/GC optimization so we don't have a ton of string objects getting created/destroyed on large files
                            if (!sourceFieldReplacementFormatMap.ContainsKey(sourceValue.Name))
                            {
                                sourceFieldReplacementFormatMap.Add(sourceValue.Name, (string.Concat("<", sourceValue.Name, ">"),
                                                                                       string.Concat("{<", sourceValue.Name, ">")));
                            }

                            // Another simple optimization of memory/gc at the expense of an extra lookup into the field value to avoid creating a new string unnecessarily
                            if (targetField.Value.IndexOf(sourceFieldReplacementFormatMap[sourceValue.Name].Static, StringComparison.OrdinalIgnoreCase) < 0)
                            {
                                continue;
                            }

                            // Replace all instances of the format-string delimited source field name with the index of the source field within the
                            // target field config list (for those that are being used in a format-string case)
                            // OR the raw value (for those that are not being used in a format string case)
                            targetField.Value = targetField.Value
                                                .Replace(sourceFieldReplacementFormatMap[sourceValue.Name].Format,
                                                         string.Concat("{", sourceFieldIndex))
                                                .Replace(sourceFieldReplacementFormatMap[sourceValue.Name].Static,
                                                         sourceValue.Value);

                            sourceFieldIndex++;
                        }
                    }

                    targetRecordFormatted = true;
                }
                catch (Exception x) when(!TextWranglerConfig.OnException(x, $"Could not format target record in [{GetType().Name}] from built record [{recordNumber}]"))
                {
                    // OnException handler says not to rethrow, so keep on going, skipping this record
                }

                if (targetRecordFormatted)
                {
                    yield return(targetRecord);
                }

                recordNumber++;
            }
        }
Exemple #29
0
        protected override async Task ConsumeAsyncInternal(IAsyncContext asyncContext, RecordConfiguration configuration, IAsyncChannel asyncChannel, CancellationToken cancellationToken)
        {
            IAsyncEnumerable <IRecord> records;

            if ((object)asyncContext == null)
            {
                throw new ArgumentNullException(nameof(asyncContext));
            }

            if ((object)configuration == null)
            {
                throw new ArgumentNullException(nameof(configuration));
            }

            if ((object)asyncChannel == null)
            {
                throw new ArgumentNullException(nameof(asyncChannel));
            }

            this.AssertValidConfiguration();

            RestfulWebApiConnectorSpecificConfiguration fsConfig = this.Configuration.StageSpecificConfiguration;

            records = asyncChannel.Records;

            if ((object)records == null)
            {
                throw new SyncPremException(nameof(records));
            }

            using (HttpClient httpClient = new HttpClient())
            {
                using (HttpContent httpContent = new PushStreamContent((s) => this.SerializeRecordsToStream(s, records)))
                {
                    using (HttpResponseMessage result = await httpClient.PostAsync(fsConfig.WebEndpointUri, httpContent))
                    {
                        await result.Content.ReadAsStreamAsync();

                        result.EnsureSuccessStatusCode();
                    }
                }
            }
        }
 protected abstract Task PreExecuteAsyncInternal(IAsyncContext asyncContext, RecordConfiguration configuration, CancellationToken cancellationToken);