public IEnumerable <T> WriteTo <T>(object writer, IEnumerable <object> records, Func <object, bool> predicate = null)
{
Configuration.Init();
if (records == null)
{
yield break;
}
var recEnum = records.GetEnumerator();
object notNullRecord = GetFirstNotNullRecord(recEnum);
if (notNullRecord == null)
{
yield break;
}
DiscoverKnownTypes(notNullRecord);
StreamWriter sw = null;
if (writer is Lazy <StreamWriter> )
{
var lsw = writer as Lazy <StreamWriter>;
ChoGuard.ArgumentNotNull(lsw, "StreamWriter");
_sw = sw = lsw.Value;
if (!Configuration.UseAvroSerializer)
{
if (_avroWriter == null)
{
_avroWriter = new SequentialWriter <T>(CreateAvroWriter <T>(sw), Configuration.SyncNumberOfObjects);
}
}
else
{
_avroSerializer = CreateAvroSerializer <T>();
}
}
else
{
_avroWriter = writer as IAvroWriter <T>;
if (_avroWriter == null)
{
throw new ChoParserException("Missing valid writer object passed.");
}
}
if (!BeginWrite.Value)
{
yield break;
}
CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture;
System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture;
object recOutput = null;
bool abortRequested = false;
try
{
foreach (object record in GetRecords(recEnum))
{
_index++;
if (TraceSwitch.TraceVerbose)
{
if (record is IChoETLNameableObject)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index));
}
}
recOutput = record;
if (record != null)
{
if (predicate == null || predicate(record))
{
if (!RaiseBeforeRecordWrite(record, _index, ref recOutput))
{
yield break;
}
if (recOutput == null)
{
continue;
}
else if (!(recOutput is T))
{
continue;
}
try
{
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
record.DoObjectLevelValidation(Configuration, Configuration.AvroRecordFieldConfigurations);
}
if (recOutput is T)
{
if (recOutput is ChoDynamicObject)
{
recOutput = new Dictionary <string, object>((ChoDynamicObject)recOutput);
}
if (_sw != null)
{
if (Configuration.UseAvroSerializer)
{
IAvroSerializer <T> avroSerializer = _avroSerializer as IAvroSerializer <T>;
avroSerializer.Serialize(sw.BaseStream, (T)(recOutput as object));
}
else
{
SequentialWriter <T> avroWriter = _avroWriter as SequentialWriter <T>;
avroWriter.Write((T)(recOutput as object));
}
}
else
{
SequentialWriter <T> avroWriter = _avroWriter as SequentialWriter <T>;
avroWriter.Write((T)(recOutput as object));
}
if (!RaiseAfterRecordWrite(record, _index, recOutput))
{
yield break;
}
}
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ref ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordWriteError(record, _index, recOutput, ex))
{
throw;
}
else
{
//ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
}
}
else
{
throw;
}
}
}
}
yield return((T)recOutput);
recOutput = null;
if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0)
{
if (RaisedRowsWritten(_index))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
abortRequested = true;
yield break;
}
}
}
if (!abortRequested && recOutput != null)
{
RaisedRowsWritten(_index, true);
}
}
finally
{
System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo;
}
}
private bool ToText(long index, object rec, out string recText)
{
if (typeof(IChoScalarObject).IsAssignableFrom(Configuration.RecordType))
{
rec = Activator.CreateInstance(Configuration.RecordType, rec);
}
recText = null;
StringBuilder msg = new StringBuilder();
object fieldValue = null;
string fieldText = null;
ChoFixedLengthRecordFieldConfiguration fieldConfig = null;
if (Configuration.ColumnCountStrict)
{
CheckColumnsStrict(rec);
}
//bool firstColumn = true;
PropertyInfo pi = null;
foreach (KeyValuePair <string, ChoFixedLengthRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict)
{
fieldConfig = kvp.Value;
fieldValue = null;
fieldText = String.Empty;
if (Configuration.PIDict != null)
{
Configuration.PIDict.TryGetValue(kvp.Key, out pi);
}
if (Configuration.ThrowAndStopOnMissingField)
{
if (Configuration.IsDynamicObject)
{
var dict = rec.ToDynamicObject() as IDictionary <string, Object>;
if (!dict.ContainsKey(kvp.Key))
{
throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' FixedLength column.".FormatString(fieldConfig.FieldName));
}
}
else
{
if (pi == null)
{
throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' FixedLength column.".FormatString(fieldConfig.FieldName));
}
}
}
try
{
if (Configuration.IsDynamicObject)
{
IDictionary <string, Object> dict = rec.ToDynamicObject() as IDictionary <string, Object>;
fieldValue = dict[kvp.Key]; // dict.GetValue(kvp.Key, Configuration.FileHeaderConfiguration.IgnoreCase, Configuration.Culture);
if (kvp.Value.FieldType == null)
{
if (fieldValue == null)
{
kvp.Value.FieldType = typeof(string);
}
else
{
kvp.Value.FieldType = fieldValue.GetType();
}
}
}
else
{
if (pi != null)
{
fieldValue = ChoType.GetPropertyValue(rec, pi);
if (kvp.Value.FieldType == null)
{
kvp.Value.FieldType = pi.PropertyType;
}
}
else
{
kvp.Value.FieldType = typeof(string);
}
}
//Discover default value, use it if null
if (fieldValue == null)
{
if (fieldConfig.IsDefaultValueSpecified)
{
fieldValue = fieldConfig.DefaultValue;
}
}
if (!RaiseBeforeRecordFieldWrite(rec, index, kvp.Key, ref fieldValue))
{
return(false);
}
if (fieldConfig.ValueConverter != null)
{
fieldValue = fieldConfig.ValueConverter(fieldValue);
}
else
{
rec.GetNConvertMemberValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue);
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.MemberLevel)
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
if (!RaiseAfterRecordFieldWrite(rec, index, kvp.Key, fieldValue))
{
return(false);
}
}
catch (ChoParserException)
{
throw;
}
catch (ChoMissingRecordFieldException)
{
if (Configuration.ThrowAndStopOnMissingField)
{
throw;
}
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ex);
if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
{
throw;
}
try
{
if (Configuration.IsDynamicObject)
{
var dict = rec.ToDynamicObject() as IDictionary <string, Object>;
if (dict.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else if (dict.GetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else
{
throw new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
}
}
else if (pi != null)
{
if (rec.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (rec.GetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else
{
throw new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
}
}
else
{
throw new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
}
}
catch (Exception innerEx)
{
if (ex == innerEx.InnerException)
{
if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
continue;
}
else
{
if (!RaiseRecordFieldWriteError(rec, index, kvp.Key, fieldText, ex))
{
throw new ChoWriterException($"Failed to write '{fieldValue}' value of '{kvp.Key}' member.", ex);
}
}
}
else
{
throw new ChoWriterException("Failed to use '{0}' fallback value for '{1}' member.".FormatString(fieldValue, kvp.Key), innerEx);
}
}
}
if (fieldValue == null)
{
fieldText = String.Empty;
}
else
{
fieldText = fieldValue.ToString();
}
msg.Append(NormalizeFieldValue(kvp.Key, fieldText, kvp.Value.Size, kvp.Value.Truncate, kvp.Value.QuoteField, GetFieldValueJustification(kvp.Value.FieldValueJustification, kvp.Value.FieldType), GetFillChar(kvp.Value.FillChar, kvp.Value.FieldType), false));
}
recText = msg.ToString();
return(true);
}
private bool LoadLine(Tuple <long, string> pair, ref object rec)
{
Type recType = rec.GetType();
try
{
if (!RaiseBeforeRecordLoad(rec, ref pair))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping...");
rec = null;
return(true);
}
if (pair.Item2 == null)
{
rec = null;
return(true);
}
else if (pair.Item2 == String.Empty)
{
return(true);
}
if (!pair.Item2.IsNullOrWhiteSpace())
{
var config = GetConfiguration(rec.GetType());
if (config == null)
{
throw new ChoParserException("No parser found to parse record line.");
}
if (config.GetType() == typeof(ChoCSVRecordConfiguration))
{
var r = ChoCSVReader.LoadText(rec.GetType(), pair.Item2, config as ChoCSVRecordConfiguration, Configuration.Encoding, Configuration.BufferSize, TraceSwitch, Reader);
rec = r.FirstOrDefault <object>();
}
else if (config.GetType() == typeof(ChoFixedLengthRecordConfiguration))
{
var r = ChoFixedLengthReader.LoadText(rec.GetType(), pair.Item2, config as ChoFixedLengthRecordConfiguration, Configuration.Encoding, Configuration.BufferSize, TraceSwitch, Reader);
rec = r.FirstOrDefault <object>();
}
else
{
throw new ChoParserException("Unsupported record line found to parse.");
}
}
bool skip = false;
if (!RaiseAfterRecordLoad(rec, pair, ref skip))
{
return(false);
}
else if (skip)
{
rec = null;
return(true);
}
}
catch (ChoParserException pEx)
{
throw new ChoParserException($"Failed to parse line to '{recType}' object.", pEx);
}
catch (ChoMissingRecordFieldException mEx)
{
throw new ChoParserException($"Failed to parse line to '{recType}' object.", mEx);
}
catch (Exception ex)
{
Reader.IsValid = false;
ChoETLFramework.HandleException(ref ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
rec = null;
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordLoadError(rec, pair, ex))
{
throw;
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
rec = null;
}
}
else
{
throw;
}
//if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
//{
// rec = null;
//}
//else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
//{
// if (!RaiseRecordLoadError(rec, pair, ex))
// throw new ChoReaderException($"Failed to parse line to '{recType}' object.", ex);
//}
//else
// throw new ChoReaderException($"Failed to parse line to '{recType}' object.", ex);
return(true);
}
return(true);
}
private bool FillRecord(ref object rec, Tuple <long, IDictionary <string, object> > pair)
{
long lineNo;
lineNo = pair.Item1;
var node = pair.Item2;
fieldValue = null;
fieldConfig = null;
pi = null;
object rootRec = rec;
foreach (KeyValuePair <string, ChoParquetRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict)
{
if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject)
{
if (Configuration.IgnoredFields.Contains(kvp.Key))
{
continue;
}
}
fieldValue = null;
fieldConfig = kvp.Value;
if (Configuration.PIDict != null)
{
// if FieldName is set
if (!string.IsNullOrEmpty(fieldConfig.FieldName))
{
// match using FieldName
Configuration.PIDict.TryGetValue(fieldConfig.FieldName, out pi);
}
else
{
// otherwise match usign the property name
Configuration.PIDict.TryGetValue(kvp.Key, out pi);
}
}
rec = GetDeclaringRecord(kvp.Value.DeclaringMember, rootRec);
if (!node.ContainsKey(kvp.Value.FieldName))
{
if (Configuration.ColumnCountStrict)
{
throw new ChoParserException("No matching '{0}' field found.".FormatString(fieldConfig.FieldName));
}
}
fieldValue = node[kvp.Value.FieldName];
if (!RaiseBeforeRecordFieldLoad(rec, pair.Item1, kvp.Key, ref fieldValue))
{
continue;
}
try
{
if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject)
{
}
else
{
if (pi != null)
{
kvp.Value.FieldType = pi.PropertyType;
}
else
{
kvp.Value.FieldType = typeof(string);
}
}
object v1 = node;
if (fieldConfig.CustomSerializer != null)
{
fieldValue = fieldConfig.CustomSerializer(v1);
}
else if (RaiseRecordFieldDeserialize(rec, pair.Item1, kvp.Key, ref v1))
{
fieldValue = v1;
}
else if (fieldConfig.PropCustomSerializer != null)
{
fieldValue = ChoCustomSerializer.Deserialize(v1, fieldConfig.FieldType, fieldConfig.PropCustomSerializer, fieldConfig.PropCustomSerializerParams, Configuration.Culture, fieldConfig.Name);
}
else
{
if (!node.ContainsKey(kvp.Value.FieldName))
{
if (Configuration.ColumnCountStrict)
{
throw new ChoParserException("No matching '{0}' field found.".FormatString(fieldConfig.FieldName));
}
}
else
{
fieldValue = node[kvp.Value.FieldName];
}
}
bool ignoreFieldValue = fieldValue.IgnoreFieldValue(fieldConfig.IgnoreFieldValueMode);
if (ignoreFieldValue)
{
fieldValue = fieldConfig.IsDefaultValueSpecified ? fieldConfig.DefaultValue : null;
}
if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject)
{
var dict = rec as IDictionary <string, Object>;
dict.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel)
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
}
else
{
if (Configuration.SupportsMultiRecordTypes)
{
ChoType.TryGetProperty(rec.GetType(), kvp.Key, out pi);
//*** TODO
//fieldConfig.PI = pi;
//fieldConfig.PropConverters = ChoTypeDescriptor.GetTypeConverters(fieldConfig.PI);
//fieldConfig.PropConverterParams = ChoTypeDescriptor.GetTypeConverterParams(fieldConfig.PI);
}
if (pi != null)
{
rec.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
}
else if (RecordType.IsSimple())
{
rec = ChoConvert.ConvertTo(fieldValue, RecordType, Configuration.Culture);
}
else
{
throw new ChoMissingRecordFieldException("Missing '{0}' property in {1} type.".FormatString(kvp.Key, ChoType.GetTypeName(rec)));
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel)
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
}
if (!RaiseAfterRecordFieldLoad(rec, pair.Item1, kvp.Key, fieldValue))
{
return(false);
}
}
catch (ChoParserException)
{
Reader.IsValid = false;
throw;
}
catch (ChoMissingRecordFieldException)
{
Reader.IsValid = false;
if (Configuration.ThrowAndStopOnMissingField)
{
throw;
}
}
catch (Exception ex)
{
Reader.IsValid = false;
ChoETLFramework.HandleException(ref ex);
if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
{
throw;
}
try
{
if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject)
{
var dict = rec as IDictionary <string, Object>;
if (dict.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (dict.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (ex is ValidationException)
{
throw;
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
else if (pi != null)
{
if (rec.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (rec.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (ex is ValidationException)
{
throw;
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
catch (Exception innerEx)
{
if (ex == innerEx.InnerException || ex is ValidationException)
{
if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
continue;
}
else
{
if (!RaiseRecordFieldLoadError(rec, pair.Item1, kvp.Key, ref fieldValue, ex))
{
if (ex is ValidationException)
{
throw;
}
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
else
{
try
{
if (Configuration.IsDynamicObject)
{
var dict = rec as IDictionary <string, Object>;
dict.ConvertNSetMemberValue(kvp.Key, fieldConfig, ref fieldValue, Configuration.Culture);
}
else
{
if (pi != null)
{
rec.ConvertNSetMemberValue(kvp.Key, fieldConfig, ref fieldValue, Configuration.Culture);
}
else
{
throw new ChoMissingRecordFieldException("Missing '{0}' property in {1} type.".FormatString(kvp.Key, ChoType.GetTypeName(rec)));
}
}
}
catch { }
}
}
}
else
{
throw new ChoReaderException("Failed to assign '{0}' fallback value to '{1}' field.".FormatString(fieldValue, fieldConfig.FieldName), innerEx);
}
}
}
}
rec = rootRec;
return(true);
}
private bool LoadLines(Tuple <long, List <Tuple <long, string> > > pairs, ref object rec)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading [{0}] record...".FormatString(pairs.Item1));
Tuple <long, string> pair = null;
long recNo = pairs.Item1;
if (!Configuration.AutoDiscoveredColumns)
{
if (Configuration.ColumnCountStrict)
{
if (pairs.Item2.Count != Configuration.KVPRecordFieldConfigurations.Count)
{
throw new ChoParserException("Incorrect number of field values found at record [{2}]. Expected [{0}] field values. Found [{1}] field values.".FormatString(Configuration.KVPRecordFieldConfigurations.Count, pairs.Item2.Count, recNo));
}
List <string> keys = (from p in pairs.Item2
select ToKVP(p.Item1, p.Item2).Key).ToList();
if (!Enumerable.SequenceEqual(keys.OrderBy(t => t), Configuration.FCArray.Select(a => a.Value.FieldName).OrderBy(t => t), Configuration.FileHeaderConfiguration.StringComparer))
{
throw new ChoParserException("Column count mismatch detected at [{0}] record.".FormatString(recNo));
}
}
if (Configuration.ColumnOrderStrict)
{
List <string> keys = (from p in pairs.Item2
select ToKVP(p.Item1, p.Item2).Key).ToList();
int runnngIndex = -1;
foreach (var k in keys)
{
runnngIndex++;
if (runnngIndex < Configuration.FCArray.Length)
{
if (Configuration.FileHeaderConfiguration.IsEqual(Configuration.FCArray[runnngIndex].Value.FieldName, k))
{
continue;
}
}
throw new ChoParserException("Found incorrect order on '{1}' column at [{0}] record.".FormatString(recNo, k));
}
}
}
object fieldValue = String.Empty;
PropertyInfo pi = null;
object rootRec = rec;
//Set default values
foreach (KeyValuePair <string, ChoKVPRecordFieldConfiguration> kvp in Configuration.FCArray)
{
rec = GetDeclaringRecord(kvp.Value.DeclaringMember, rootRec);
if (Configuration.PIDict != null)
{
Configuration.PIDict.TryGetValue(kvp.Key, out pi);
}
try
{
if (kvp.Value.IsDefaultValueSpecified)
{
fieldValue = kvp.Value.DefaultValue;
}
if (Configuration.IsDynamicObject)
{
var dict = rec as IDictionary <string, Object>;
dict.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
}
else
{
if (pi != null)
{
rec.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
}
}
}
catch
{
}
}
rec = rootRec;
_propInit.Clear();
foreach (var pair1 in pairs.Item2)
{
pair = pair1;
try
{
if (!RaiseBeforeRecordLoad(rec, ref pair))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping...");
rec = null;
return(true);
}
if (pair.Item2 == null)
{
rec = null;
return(true);
}
else if (pair.Item2 == String.Empty)
{
return(true);
}
if (!pair.Item2.IsNullOrWhiteSpace())
{
if (!FillRecord(rec, pair))
{
return(false);
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
rec.DoObjectLevelValidation(Configuration, Configuration.KVPRecordFieldConfigurations);
}
}
bool skip = false;
if (!RaiseAfterRecordLoad(rec, pair, ref skip))
{
return(false);
}
else if (skip)
{
rec = null;
return(true);
}
}
//catch (ChoParserException)
//{
// throw;
//}
//catch (ChoMissingRecordFieldException)
//{
// throw;
//}
catch (Exception ex)
{
Reader.IsValid = false;
if (ex is ChoMissingRecordFieldException && Configuration.ThrowAndStopOnMissingField)
{
throw;
}
ChoETLFramework.HandleException(ref ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
rec = null;
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordLoadError(rec, pair, ex))
{
throw;
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
rec = null;
}
}
else
{
throw;
}
return(true);
}
}
return(true);
}
private bool ToText(long index, object rec, out string recText)
{
if (typeof(IChoScalarObject).IsAssignableFrom(Configuration.RecordType))
{
rec = ChoActivator.CreateInstance(Configuration.RecordType, rec);
}
if (!Configuration.IsDynamicObject)
{
if (rec.ToTextIfCustomSerialization(out recText))
{
return(true);
}
//Check if KVP object
if (rec.GetType().IsKeyValueType())
{
recText = SerializeObject(rec);
return(true);
}
}
recText = null;
if (Configuration.UseYamlSerialization)
{
recText = new Serializer().Serialize(rec, rec.GetType());
return(true);
}
StringBuilder msg = new StringBuilder();
object fieldValue = null;
string fieldText = null;
ChoYamlRecordFieldConfiguration fieldConfig = null;
string fieldName = null;
if (Configuration.ColumnCountStrict)
{
CheckColumnsStrict(rec);
}
//bool firstColumn = true;
PropertyInfo pi = null;
bool isFirst = true;
object rootRec = rec;
Dictionary <string, object> output = new Dictionary <string, object>();
foreach (KeyValuePair <string, ChoYamlRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict.OrderBy(kvp => kvp.Value.Order))
{
//if (Configuration.IsDynamicObject)
//{
if (Configuration.IgnoredFields.Contains(kvp.Key))
{
continue;
}
//}
fieldConfig = kvp.Value;
fieldName = fieldConfig.FieldName;
fieldValue = null;
fieldText = String.Empty;
if (Configuration.PIDict != null)
{
// if FieldName is set
if (!string.IsNullOrEmpty(fieldConfig.FieldName))
{
// match using FieldName
Configuration.PIDict.TryGetValue(fieldConfig.FieldName, out pi);
}
else
{
// otherwise match usign the property name
Configuration.PIDict.TryGetValue(kvp.Key, out pi);
}
}
rec = GetDeclaringRecord(kvp.Value.DeclaringMember, rootRec);
if (Configuration.ThrowAndStopOnMissingField)
{
if (Configuration.IsDynamicObject)
{
var dict = rec.ToDynamicObject() as IDictionary <string, Object>;
if (!dict.ContainsKey(kvp.Key))
{
throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' Yaml node.".FormatString(fieldConfig.FieldName));
}
}
else
{
if (pi == null)
{
if (!RecordType.IsSimple())
{
throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' Yaml node.".FormatString(fieldConfig.FieldName));
}
}
}
}
try
{
if (Configuration.IsDynamicObject)
{
IDictionary <string, Object> dict = rec.ToDynamicObject() as IDictionary <string, Object>;
fieldValue = dict[kvp.Key]; // dict.GetValue(kvp.Key, Configuration.FileHeaderConfiguration.IgnoreCase, Configuration.Culture);
if (rec is ChoDynamicObject)
{
}
if (kvp.Value.FieldType == null)
{
if (rec is ChoDynamicObject)
{
var dobj = rec as ChoDynamicObject;
kvp.Value.FieldType = dobj.GetMemberType(kvp.Key);
}
if (kvp.Value.FieldType == null)
{
if (ElementType == null)
{
kvp.Value.FieldType = typeof(object);
}
else
{
kvp.Value.FieldType = ElementType;
}
}
}
}
else
{
if (pi != null)
{
fieldValue = ChoType.GetPropertyValue(rec, pi);
if (kvp.Value.FieldType == null)
{
kvp.Value.FieldType = pi.PropertyType;
}
}
else
{
kvp.Value.FieldType = typeof(string);
}
}
//Discover default value, use it if null
//if (fieldValue == null)
//{
// if (fieldConfig.IsDefaultValueSpecified)
// fieldValue = fieldConfig.DefaultValue;
//}
bool ignoreFieldValue = fieldValue.IgnoreFieldValue(fieldConfig.IgnoreFieldValueMode);
if (ignoreFieldValue)
{
fieldValue = fieldConfig.IsDefaultValueSpecified ? fieldConfig.DefaultValue : null;
}
if (!RaiseBeforeRecordFieldWrite(rec, index, kvp.Key, ref fieldValue))
{
return(false);
}
if (fieldConfig.ValueConverter != null)
{
fieldValue = fieldConfig.ValueConverter(fieldValue);
}
else if (RecordType.IsSimple())
{
fieldValue = new List <object> {
rec
}
}
;
else
{
rec.GetNConvertMemberValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue, true);
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.MemberLevel)
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
if (!RaiseAfterRecordFieldWrite(rec, index, kvp.Key, fieldValue))
{
return(false);
}
}
catch (ChoParserException)
{
throw;
}
catch (ChoMissingRecordFieldException)
{
if (Configuration.ThrowAndStopOnMissingField)
{
throw;
}
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ref ex);
if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
{
throw;
}
try
{
if (Configuration.IsDynamicObject)
{
var dict = rec.ToDynamicObject() as IDictionary <string, Object>;
if (dict.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else if (dict.GetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else
{
var ex1 = new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
fieldValue = null;
throw ex1;
}
}
else if (pi != null)
{
if (rec.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (rec.GetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else
{
var ex1 = new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
fieldValue = null;
throw ex1;
}
}
else
{
var ex1 = new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
fieldValue = null;
throw ex1;
}
}
catch (Exception innerEx)
{
if (ex == innerEx.InnerException)
{
if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
continue;
}
else
{
if (!RaiseRecordFieldWriteError(rec, index, kvp.Key, ref fieldValue, ex))
{
throw new ChoWriterException($"Failed to write '{fieldValue}' value of '{kvp.Key}' member.", ex);
}
}
}
else
{
throw new ChoWriterException("Failed to use '{0}' fallback value for '{1}' member.".FormatString(fieldValue, kvp.Key), innerEx);
}
}
}
bool isSimple = true;
Type ft = fieldValue == null ? typeof(object) : fieldValue.GetType();
if (fieldConfig.CustomSerializer != null)
{
fieldText = fieldConfig.CustomSerializer(fieldValue) as string;
}
else if (RaiseRecordFieldSerialize(rec, index, kvp.Key, ref fieldValue))
{
fieldText = fieldValue as string;
}
else if (fieldConfig.PropCustomSerializer != null)
{
fieldText = ChoCustomSerializer.Serialize(fieldValue, typeof(string), fieldConfig.PropCustomSerializer, fieldConfig.PropCustomSerializerParams, Configuration.Culture, fieldConfig.Name) as string;
}
else
{
//if (fieldConfig.IgnoreFieldValue(fieldValue))
// fieldText = null;
bool ignoreFieldValue = fieldValue.IgnoreFieldValue(fieldConfig.IgnoreFieldValueMode);
if (ignoreFieldValue)
{
fieldText = null;
}
else if (fieldValue == null)
{
//if (fieldConfig.FieldType == null || fieldConfig.FieldType == typeof(object))
//{
// if (fieldConfig.NullValue == null)
// fieldText = !fieldConfig.IsArray ? "null" : "[]";
// else
// fieldText = fieldConfig.NullValue;
//}
if (Configuration.NullValueHandling == ChoNullValueHandling.Ignore)
{
fieldText = null;
}
else if (Configuration.NullValueHandling == ChoNullValueHandling.Default)
{
fieldText = ChoActivator.CreateInstance(fieldConfig.FieldType).ToNString();
}
else if (Configuration.NullValueHandling == ChoNullValueHandling.Empty && fieldConfig.FieldType == typeof(string))
{
fieldText = String.Empty;
}
else
{
if (fieldConfig.NullValue == null)
{
}
else
{
fieldText = fieldConfig.NullValue;
}
}
}
else if (ft == typeof(string) || ft == typeof(char))
{
fieldText = NormalizeFieldValue(kvp.Key, fieldValue.ToString(), kvp.Value.Size, kvp.Value.Truncate, false, GetFieldValueJustification(kvp.Value.FieldValueJustification, kvp.Value.FieldType),
GetFillChar(kvp.Value.FillChar, kvp.Value.FieldType), false, kvp.Value.GetFieldValueTrimOption(kvp.Value.FieldType, Configuration.FieldValueTrimOption));
}
else if (ft == typeof(DateTime) || ft == typeof(TimeSpan))
{
fieldText = ChoConvert.ConvertTo(fieldValue, typeof(String), Configuration.Culture) as string;
}
else if (ft.IsEnum)
{
fieldText = ChoConvert.ConvertTo(fieldValue, typeof(String), Configuration.Culture) as string;
}
else if (ft == typeof(ChoCurrency))
{
fieldText = fieldValue.ToString();
}
else if (ft == typeof(bool))
{
fieldText = ChoConvert.ConvertTo(fieldValue, typeof(String), Configuration.Culture) as string;
}
else if (ft.IsNumeric())
{
fieldText = ChoConvert.ConvertTo(fieldValue, typeof(String), Configuration.Culture) as string;
}
else
{
isSimple = false;
}
}
object objValue = null;
if (isSimple)
{
objValue = fieldText;
}
else
{
Writer.ContractResolverState = new ChoContractResolverState
{
Name = kvp.Key,
Index = index,
Record = rec,
FieldConfig = kvp.Value
};
var json = JsonConvert.SerializeObject(fieldValue, Configuration.JsonSerializerSettings);
if (RecordType.IsSimple())
{
objValue = JsonConvert.DeserializeObject <IList <object> >(json);
}
else if (typeof(IList).IsAssignableFrom(ft))
{
objValue = JsonConvert.DeserializeObject <IList>(json);
}
else
{
objValue = JsonConvert.DeserializeObject <IDictionary <string, object> >(json);
}
}
if (!RecordType.IsSimple())
{
output.Add(fieldName, objValue);
}
else
{
fieldValue = objValue;
}
}
if (!RecordType.IsSimple())
{
recText = Configuration.YamlSerializer.Serialize(output, output.GetType());
}
else
{
recText = Configuration.YamlSerializer.Serialize(fieldValue, fieldValue.GetType());
}
return(true);
}
public override IEnumerable <object> WriteTo(object writer, IEnumerable <object> records, Func <object, bool> predicate = null)
{
TextWriter sw = writer as TextWriter;
ChoGuard.ArgumentNotNull(sw, "TextWriter");
if (records == null)
{
yield break;
}
if (!RaiseBeginWrite(sw))
{
yield break;
}
CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture;
System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture;
string recText = String.Empty;
long recCount = 0;
var recEnum = records.GetEnumerator();
try
{
object notNullRecord = GetFirstNotNullRecord(recEnum);
if (notNullRecord == null)
{
yield break;
}
if (Configuration.IsDynamicObject)
{
if (Configuration.MaxScanRows > 0)
{
List <string> fns = new List <string>();
foreach (object record1 in GetRecords(recEnum))
{
recCount++;
if (record1 != null)
{
if (recCount <= Configuration.MaxScanRows)
{
if (!record1.GetType().IsDynamicType())
{
throw new ChoParserException("Invalid record found.");
}
_recBuffer.Value.Add(record1);
fns = fns.Union(GetFields(record1)).ToList();
if (recCount == Configuration.MaxScanRows)
{
Configuration.Validate(fns.ToArray());
WriteHeaderLine(sw);
_configCheckDone = true;
break;
}
}
}
}
}
}
foreach (object record in GetRecords(recEnum))
{
_index++;
if (TraceSwitch.TraceVerbose)
{
if (record is IChoETLNameableObject)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index));
}
}
recText = String.Empty;
if (record != null)
{
if (predicate == null || predicate(record))
{
//Discover and load FixedLength columns from first record
if (!_configCheckDone)
{
if (notNullRecord != null)
{
string[] fieldNames = GetFields(notNullRecord);
Configuration.Validate(fieldNames);
WriteHeaderLine(sw);
_configCheckDone = true;
}
}
//Check record
if (record != null)
{
Type rt = record.GetType().ResolveType();
if (Configuration.IsDynamicObject)
{
if (ElementType != null)
{
}
else if (!rt.IsDynamicType())
{
throw new ChoWriterException("Invalid record found.");
}
}
else
{
if (rt != Configuration.RecordType)
{
throw new ChoWriterException("Invalid record found.");
}
}
}
if (!RaiseBeforeRecordWrite(record, _index, ref recText))
{
yield break;
}
if (recText == null)
{
continue;
}
else if (recText.Length > 0)
{
sw.Write("{1}{0}", recText, _hadHeaderWritten ? Configuration.EOLDelimiter : "");
continue;
}
try
{
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
record.DoObjectLevelValidation(Configuration, Configuration.FixedLengthRecordFieldConfigurations);
}
if (ToText(_index, record, out recText))
{
if (_index == 1)
{
sw.Write("{1}{0}", recText, _hadHeaderWritten ? Configuration.EOLDelimiter : "");
}
else
{
sw.Write("{1}{0}", recText, Configuration.EOLDelimiter);
}
if (!RaiseAfterRecordWrite(record, _index, recText))
{
yield break;
}
}
}
//catch (ChoParserException)
//{
// throw;
//}
catch (Exception ex)
{
ChoETLFramework.HandleException(ref ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordWriteError(record, _index, recText, ex))
{
throw;
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
}
}
else
{
throw;
}
}
}
}
yield return(record);
if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0)
{
if (RaisedRowsWritten(_index))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
yield break;
}
}
}
}
finally
{
System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo;
}
RaiseEndWrite(sw);
}
private bool FillRecord(object rec, Tuple <int, string> pair)
{
int lineNo;
string line;
lineNo = pair.Item1;
line = pair.Item2;
object fieldValue = null;
string[] fieldValues = (from x in line.Split(Configuration.Delimiter, Configuration.StringSplitOptions)
select x).ToArray();
if (Configuration.ColumnCountStrict)
{
if (fieldValues.Length != Configuration.RecordFieldConfigurations.Count)
{
throw new ChoParserException("Incorrect number of field values found at line [{2}]. Expected [{0}] field values. Found [{1}] field values.".FormatString(Configuration.RecordFieldConfigurations.Count, fieldValues.Length, pair.Item1));
}
}
Dictionary <string, string> _fieldNameValues = ToFieldNameValues(fieldValues);
ValidateLine(pair.Item1, fieldValues);
ChoCSVRecordFieldConfiguration fieldConfig = null;
foreach (KeyValuePair <string, ChoCSVRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict)
{
fieldConfig = kvp.Value;
if (Configuration.CSVFileHeaderConfiguration.HasHeaderRecord)
{
if (_fieldNameValues.ContainsKey(fieldConfig.FieldName))
{
fieldValue = _fieldNameValues[fieldConfig.FieldName];
}
else if (Configuration.ColumnCountStrict)
{
throw new ChoParserException("No matching '{0}' field header found.".FormatString(fieldConfig.FieldName));
}
}
else
{
if (fieldConfig.FieldPosition - 1 < fieldValues.Length)
{
fieldValue = fieldValues[fieldConfig.FieldPosition - 1];
}
else if (Configuration.ColumnCountStrict)
{
throw new ChoParserException("Missing field value for {0} [Position: {1}] field.".FormatString(fieldConfig.FieldName, fieldConfig.FieldPosition));
}
}
fieldValue = CleanFieldValue(fieldConfig, fieldValue as string);
if (!BeforeRecordFieldLoad(pair.Item1, kvp.Key, ref fieldValue))
{
continue;
}
try
{
bool ignoreFieldValue = fieldConfig.IgnoreFieldValue(fieldValue);
if (!ignoreFieldValue)
{
if (rec is ExpandoObject)
{
if (fieldConfig.FieldType != typeof(string))
{
fieldValue = ChoConvert.ConvertTo(fieldValue, fieldConfig.FieldType, Configuration.Culture);
}
var x = rec as IDictionary <string, Object>;
x.Add(kvp.Key, fieldValue);
}
else
{
ChoType.ConvertNSetMemberValue(rec, kvp.Key, fieldValue);
fieldValue = ChoType.GetMemberValue(rec, kvp.Key);
ChoValidator.ValididateFor(rec, kvp.Key);
}
}
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ex);
if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
{
throw;
}
try
{
ChoFallbackValueAttribute fbAttr = ChoTypeDescriptor.GetPropetyAttribute <ChoFallbackValueAttribute>(rec.GetType(), kvp.Key);
if (fbAttr != null)
{
if (!fbAttr.Value.IsNullOrDbNull())
{
ChoType.ConvertNSetMemberValue(rec, kvp.Key, fbAttr.Value);
}
}
else
{
throw;
}
}
catch
{
if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
continue;
}
else if (fieldConfig.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RecordFieldLoadError(pair.Item1, kvp.Key, fieldValue, ex))
{
throw;
}
}
else
{
throw;
}
}
}
if (!AfterRecordFieldLoad(pair.Item1, kvp.Key, fieldValue))
{
return(false);
}
}
return(true);
}
private bool LoadNode(Tuple <long, JObject> pair, ref object rec)
{
if (!Configuration.UseJSONSerialization)
{
rec = Configuration.IsDynamicObject ? new ChoDynamicObject()
{
ThrowExceptionIfPropNotExists = true
}
}
: Activator.CreateInstance(RecordType);
try
{
if (!RaiseBeforeRecordLoad(rec, ref pair))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping...");
rec = null;
return(true);
}
if (pair.Item2 == null)
{
rec = null;
return(true);
}
if (!Configuration.UseJSONSerialization)
{
if (!FillRecord(rec, pair))
{
return(false);
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
rec.DoObjectLevelValidation(Configuration, Configuration.JSONRecordFieldConfigurations);
}
}
else
{
//rec = _se.Value != null ? pair.Item2.ToObject(RecordType, _se.Value) : pair.Item2.ToObject(RecordType);
if (Configuration.IsDynamicObject)
{
rec = JsonConvert.DeserializeObject <ExpandoObject>(pair.Item2.ToString(), new ExpandoObjectConverter());
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.Off) != ChoObjectValidationMode.Off)
{
rec.DoObjectLevelValidation(Configuration, Configuration.JSONRecordFieldConfigurations);
}
}
else
{
rec = Configuration.JsonSerializerSettings == null?JsonConvert.DeserializeObject(pair.Item2.ToString(), RecordType) : JsonConvert.DeserializeObject(pair.Item2.ToString(), RecordType, Configuration.JsonSerializerSettings);
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.Off) != ChoObjectValidationMode.Off)
{
rec.DoObjectLevelValidation(Configuration, Configuration.JSONRecordFieldConfigurations);
}
}
}
bool skip = false;
if (!RaiseAfterRecordLoad(rec, pair, ref skip))
{
return(false);
}
else if (skip)
{
rec = null;
return(true);
}
}
//catch (ChoParserException)
//{
// throw;
//}
//catch (ChoMissingRecordFieldException)
//{
// throw;
//}
catch (Exception ex)
{
Reader.IsValid = false;
if (ex is ChoMissingRecordFieldException && Configuration.ThrowAndStopOnMissingField)
{
throw;
}
ChoETLFramework.HandleException(ref ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
rec = null;
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordLoadError(rec, pair, ex))
{
throw;
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
rec = null;
}
}
else
{
throw;
}
return(true);
}
return(true);
}
public override IEnumerable <object> WriteTo(object writer, IEnumerable <object> records, Func <object, bool> predicate = null)
{
TextWriter sw = writer as TextWriter;
ChoGuard.ArgumentNotNull(sw, "TextWriter");
if (records == null)
{
yield break;
}
CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture;
System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture;
_se = new Lazy <XmlSerializer>(() => Configuration.XmlSerializer == null ? null : Configuration.XmlSerializer);
string recText = String.Empty;
try
{
foreach (object record in records)
{
_index++;
if (TraceSwitch.TraceVerbose)
{
if (record is IChoETLNameableObject)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index));
}
}
recText = String.Empty;
if (record != null)
{
if (predicate == null || predicate(record))
{
//Discover and load Xml columns from first record
if (!_configCheckDone)
{
string[] fieldNames = null;
Configuration.IsDynamicObject = record.GetType().IsDynamicType();
if (!Configuration.IsDynamicObject)
{
Configuration.RecordType = record.GetType();
}
if (Configuration.IsDynamicObject)
{
var dict = record.ToDynamicObject() as IDictionary <string, Object>;
fieldNames = dict.Keys.ToArray();
}
else
{
fieldNames = ChoTypeDescriptor.GetProperties <ChoXmlNodeRecordFieldAttribute>(record.GetType()).Select(pd => pd.Name).ToArray();
if (fieldNames.Length == 0)
{
fieldNames = ChoType.GetProperties(record.GetType()).Select(p => p.Name).ToArray();
}
}
Configuration.Validate(fieldNames);
_configCheckDone = true;
if (!RaiseBeginWrite(sw))
{
yield break;
}
sw.Write("<{0}{1}>".FormatString(Configuration.RootName, GetNamespaceText()));
}
if (!RaiseBeforeRecordWrite(record, _index, ref recText))
{
yield break;
}
if (recText == null)
{
continue;
}
try
{
if (!Configuration.UseXmlSerialization)
{
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
record.DoObjectLevelValidation(Configuration, Configuration.XmlRecordFieldConfigurations);
}
if (ToText(_index, record, out recText))
{
sw.Write("{1}{0}", recText, Configuration.EOLDelimiter);
if (!RaiseAfterRecordWrite(record, _index, recText))
{
yield break;
}
}
}
else
{
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.Off) != ChoObjectValidationMode.Off)
{
record.DoObjectLevelValidation(Configuration, Configuration.XmlRecordFieldConfigurations);
}
if (_se.Value != null)
{
_se.Value.Serialize(sw, record);
}
else
{
sw.Write("{1}{0}", ChoUtility.XmlSerialize(record).Indent(2, Configuration.IndentChar.ToString()), Configuration.EOLDelimiter);
}
if (!RaiseAfterRecordWrite(record, _index, null))
{
yield break;
}
}
}
catch (ChoParserException)
{
throw;
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordWriteError(record, _index, recText, ex))
{
throw;
}
}
else
{
throw;
}
}
}
}
yield return(record);
if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0)
{
if (RaisedRowsWritten(_index))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
yield break;
}
}
}
}
finally
{
System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo;
}
}
private bool ToText(long index, object rec, out string recText)
{
if (typeof(IChoScalarObject).IsAssignableFrom(Configuration.RecordType))
{
rec = ChoActivator.CreateInstance(Configuration.RecordType, rec);
}
recText = null;
msg.Clear();
if (Configuration.ColumnCountStrict)
{
CheckColumnsStrict(rec);
}
bool firstColumn = true;
PropertyInfo pi = null;
object rootRec = rec;
foreach (KeyValuePair <string, ChoCSVRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict)
{
if (Configuration.IsDynamicObject)
{
if (Configuration.IgnoredFields.Contains(kvp.Key))
{
continue;
}
}
fieldConfig = kvp.Value;
fieldValue = null;
fieldText = String.Empty;
if (Configuration.PIDict != null)
{
Configuration.PIDict.TryGetValue(kvp.Key, out pi);
}
rec = GetDeclaringRecord(kvp.Value.DeclaringMember, rootRec);
dict = rec.ToDynamicObject() as IDictionary <string, Object>;
if (Configuration.IsDynamicObject)
{
dict = dict.Flatten().ToDictionary();
}
if (Configuration.ThrowAndStopOnMissingField)
{
if (Configuration.IsDynamicObject)
{
if (!dict.ContainsKey(kvp.Key))
{
if (fieldConfig.FieldPosition > dict.Count)
{
throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' CSV column.".FormatString(fieldConfig.FieldName));
}
}
}
else
{
if (pi == null)
{
pi = Configuration.PIDict.Where(kvp1 => kvp.Value.FieldPosition == kvp.Value.FieldPosition).FirstOrDefault().Value;
}
if (pi == null)
{
throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' CSV column.".FormatString(fieldConfig.FieldName));
}
}
}
try
{
if (Configuration.IsDynamicObject)
{
fieldValue = dict.ContainsKey(kvp.Key) ? dict[kvp.Key] : dict[dict.Keys.ElementAt(fieldConfig.FieldPosition - 1)]; // dict.GetValue(kvp.Key, Configuration.FileHeaderConfiguration.IgnoreCase, Configuration.Culture);
if (rec is ChoDynamicObject)
{
var dobj = rec as ChoDynamicObject;
kvp.Value.FieldType = dobj.GetMemberType(kvp.Key);
}
if (kvp.Value.FieldType == null)
{
if (fieldValue == null)
{
kvp.Value.FieldType = typeof(string);
}
else
{
kvp.Value.FieldType = fieldValue.GetType();
}
}
}
else
{
if (pi != null)
{
fieldValue = ChoType.GetPropertyValue(rec, pi);
if (kvp.Value.FieldType == null)
{
kvp.Value.FieldType = pi.PropertyType;
}
}
else
{
kvp.Value.FieldType = typeof(string);
}
}
//Discover default value, use it if null
if (fieldValue == null)
{
if (fieldConfig.IsDefaultValueSpecified)
{
fieldValue = fieldConfig.DefaultValue;
}
}
if (!RaiseBeforeRecordFieldWrite(rec, index, kvp.Key, ref fieldValue))
{
return(false);
}
if (fieldConfig.ValueSelector == null)
{
if (fieldConfig.ValueConverter != null)
{
fieldValue = fieldConfig.ValueConverter(fieldValue);
}
else
{
rec.GetNConvertMemberValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue);
}
}
else
{
fieldValue = fieldConfig.ValueSelector(fieldValue);
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.MemberLevel)
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
if (!RaiseAfterRecordFieldWrite(rec, index, kvp.Key, fieldValue))
{
return(false);
}
}
catch (ChoParserException)
{
throw;
}
catch (ChoMissingRecordFieldException)
{
if (Configuration.ThrowAndStopOnMissingField)
{
throw;
}
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ref ex);
if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
{
throw;
}
try
{
if (Configuration.IsDynamicObject)
{
if (dict.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else if (dict.GetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else
{
var ex1 = new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
fieldValue = null;
throw ex1;
}
}
else if (pi != null)
{
if (rec.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (rec.GetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else
{
var ex1 = new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
fieldValue = null;
throw ex1;
}
}
else
{
var ex1 = new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
fieldValue = null;
throw ex1;
}
}
catch (Exception innerEx)
{
if (ex == innerEx.InnerException)
{
if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
continue;
}
else
{
if (!RaiseRecordFieldWriteError(rec, index, kvp.Key, fieldText, ex))
{
throw new ChoWriterException($"Failed to write '{fieldValue}' value of '{kvp.Key}' member.", ex);
}
}
}
else
{
throw new ChoWriterException("Failed to use '{0}' fallback value for '{1}' member.".FormatString(fieldValue, kvp.Key), innerEx);
}
}
}
if (fieldValue == null)
{
fieldText = String.Empty;
}
else
{
if (fieldValue is IList)
{
StringBuilder sb = new StringBuilder();
bool first = true;
foreach (var item in (IList)fieldValue)
{
if (first)
{
sb.Append(NormalizeFieldValue(kvp.Key, item.ToNString(), null, false, null, ChoFieldValueJustification.None, GetFillChar(kvp.Value.FillChar)));
first = false;
}
else
{
sb.AppendFormat("{0}{1}", Configuration.Delimiter, NormalizeFieldValue(kvp.Key, item.ToNString(), null, false, null, ChoFieldValueJustification.None, GetFillChar(kvp.Value.FillChar)));
}
}
fieldText = sb.ToString();
}
else
{
fieldText = fieldValue.ToString();
}
}
if (firstColumn)
{
msg.Append(NormalizeFieldValue(kvp.Key, fieldText, kvp.Value.Size, kvp.Value.Truncate, kvp.Value.QuoteField,
GetFieldValueJustification(kvp.Value.FieldValueJustification), GetFillChar(kvp.Value.FillChar),
false, kvp.Value.NullValue, kvp.Value.GetFieldValueTrimOption(kvp.Value.FieldType), fieldConfig));
firstColumn = false;
}
else
{
msg.AppendFormat("{0}{1}", Configuration.Delimiter, NormalizeFieldValue(kvp.Key, fieldText, kvp.Value.Size, kvp.Value.Truncate, kvp.Value.QuoteField, GetFieldValueJustification(kvp.Value.FieldValueJustification),
GetFillChar(kvp.Value.FillChar), false, kvp.Value.NullValue, kvp.Value.GetFieldValueTrimOption(kvp.Value.FieldType), fieldConfig));
}
}
recText = msg.ToString();
return(true);
}
private bool ToText(long index, object rec, out string recText)
{
recText = null;
StringBuilder msg = new StringBuilder();
object fieldValue = null;
string fieldText = null;
ChoXmlRecordFieldConfiguration fieldConfig = null;
if (Configuration.ColumnCountStrict)
{
CheckColumnsStrict(rec);
}
//bool firstColumn = true;
PropertyInfo pi = null;
bool isElementClosed = false;
msg.Append("<{0}".FormatString(Configuration.NodeName).Indent(Configuration.Indent, Configuration.IndentChar.ToString()));
foreach (KeyValuePair <string, ChoXmlRecordFieldConfiguration> kvp in GetOrderedKVP())
{
fieldConfig = kvp.Value;
fieldValue = null;
fieldText = String.Empty;
if (Configuration.PIDict != null)
{
Configuration.PIDict.TryGetValue(kvp.Key, out pi);
}
if (Configuration.ThrowAndStopOnMissingField)
{
if (Configuration.IsDynamicObject)
{
var dict = rec.ToDynamicObject() as IDictionary <string, Object>;
if (!dict.ContainsKey(kvp.Key))
{
throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' Xml node.".FormatString(fieldConfig.FieldName));
}
}
else
{
if (pi == null)
{
throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' Xml node.".FormatString(fieldConfig.FieldName));
}
}
}
try
{
if (Configuration.IsDynamicObject)
{
IDictionary <string, Object> dict = rec.ToDynamicObject() as IDictionary <string, Object>;
fieldValue = dict[kvp.Key]; // dict.GetValue(kvp.Key, Configuration.FileHeaderConfiguration.IgnoreCase, Configuration.Culture);
if (kvp.Value.FieldType == null)
{
if (fieldValue == null)
{
kvp.Value.FieldType = typeof(string);
}
else
{
kvp.Value.FieldType = fieldValue.GetType();
}
}
}
else
{
if (pi != null)
{
fieldValue = ChoType.GetPropertyValue(rec, pi);
if (kvp.Value.FieldType == null)
{
kvp.Value.FieldType = pi.PropertyType;
}
}
else
{
kvp.Value.FieldType = typeof(string);
}
}
//Discover default value, use it if null
if (fieldValue == null)
{
if (fieldConfig.IsDefaultValueSpecified)
{
fieldValue = fieldConfig.DefaultValue;
}
}
if (!RaiseBeforeRecordFieldWrite(rec, index, kvp.Key, ref fieldValue))
{
return(false);
}
if (fieldConfig.ValueConverter != null)
{
fieldValue = fieldConfig.ValueConverter(fieldValue);
}
else
{
rec.GetNConvertMemberValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue, true);
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.MemberLevel)
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
if (!RaiseAfterRecordFieldWrite(rec, index, kvp.Key, fieldValue))
{
return(false);
}
}
catch (ChoParserException)
{
throw;
}
catch (ChoMissingRecordFieldException)
{
if (Configuration.ThrowAndStopOnMissingField)
{
throw;
}
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ex);
if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
{
throw;
}
try
{
if (Configuration.IsDynamicObject)
{
var dict = rec.ToDynamicObject() as IDictionary <string, Object>;
if (dict.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else if (dict.GetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else
{
throw new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
}
}
else if (pi != null)
{
if (rec.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (rec.GetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else
{
throw new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
}
}
else
{
throw new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
}
}
catch (Exception innerEx)
{
if (ex == innerEx.InnerException)
{
if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
continue;
}
else
{
if (!RaiseRecordFieldWriteError(rec, index, kvp.Key, fieldText, ex))
{
throw new ChoWriterException($"Failed to write '{fieldValue}' value of '{kvp.Key}' member.", ex);
}
}
}
else
{
throw new ChoWriterException("Failed to use '{0}' fallback value for '{1}' member.".FormatString(fieldValue, kvp.Key), innerEx);
}
}
}
if (fieldValue == null)
{
fieldText = String.Empty;
}
else
{
if (fieldValue.GetType().IsSimple())
{
fieldText = fieldValue.ToString();
if (fieldConfig.IsXmlAttribute)
{
msg.Append(@" {0}=""{1}""".FormatString(fieldConfig.FieldName, NormalizeFieldValue(kvp.Key, fieldText, kvp.Value.Size, kvp.Value.Truncate, kvp.Value.QuoteField, GetFieldValueJustification(kvp.Value.FieldValueJustification, kvp.Value.FieldType), GetFillChar(kvp.Value.FillChar, kvp.Value.FieldType), false)));
}
else
{
if (!isElementClosed)
{
msg.AppendFormat(">{0}", Configuration.EOLDelimiter);
isElementClosed = true;
}
msg.Append("<{0}>{1}</{0}>{2}".FormatString(fieldConfig.FieldName,
NormalizeFieldValue(kvp.Key, fieldText, kvp.Value.Size, kvp.Value.Truncate, kvp.Value.QuoteField, GetFieldValueJustification(kvp.Value.FieldValueJustification, kvp.Value.FieldType), GetFillChar(kvp.Value.FillChar, kvp.Value.FieldType), false),
Configuration.EOLDelimiter).Indent(Configuration.Indent * 2, Configuration.IndentChar.ToString()));
}
}
else
{
fieldText = ChoUtility.XmlSerialize(fieldValue);
fieldText = _beginTagRegex.Replace(fieldText, delegate(Match thisMatch)
{
return("<{0}>".FormatString(fieldConfig.FieldName));
});
fieldText = _endTagRegex.Replace(fieldText, delegate(Match thisMatch)
{
return("</{0}>".FormatString(fieldConfig.FieldName));
});
if (!isElementClosed)
{
msg.AppendFormat(">{0}", Configuration.EOLDelimiter);
isElementClosed = true;
}
msg.Append("{0}{1}".FormatString(fieldText,
Configuration.EOLDelimiter).Indent(Configuration.Indent * 2, Configuration.IndentChar.ToString()));
}
}
}
if (!isElementClosed)
{
msg.AppendFormat(">{0}", Configuration.EOLDelimiter);
isElementClosed = true;
}
msg.Append("</{0}>".FormatString(Configuration.NodeName).Indent(Configuration.Indent, Configuration.IndentChar.ToString()));
recText = msg.ToString();
return(true);
}
public override IEnumerable <object> WriteTo(object writer, IEnumerable <object> records, Func <object, bool> predicate = null)
{
StreamWriter sw = writer as StreamWriter;
ChoGuard.ArgumentNotNull(sw, "StreamWriter");
if (records == null)
{
yield break;
}
if (!RaiseBeginWrite(sw))
{
yield break;
}
CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture;
System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture;
string recText = String.Empty;
try
{
foreach (object record in records)
{
_index++;
if (record is IChoETLNameableObject)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index));
}
recText = String.Empty;
if (record != null)
{
if (predicate == null || predicate(record))
{
//Discover and load CSV columns from first record
if (!_configCheckDone)
{
string[] fieldNames = null;
if (record is ExpandoObject)
{
var dict = record as IDictionary <string, Object>;
fieldNames = dict.Keys.ToArray();
}
else
{
fieldNames = ChoTypeDescriptor.GetProperties <ChoCSVRecordFieldAttribute>(record.GetType()).Select(pd => pd.Name).ToArray();
if (fieldNames.Length == 0)
{
fieldNames = ChoType.GetProperties(record.GetType()).Select(p => p.Name).ToArray();
}
}
Configuration.Validate(fieldNames);
WriteHeaderLine(sw);
_configCheckDone = true;
}
if (!RaiseBeforeRecordWrite(record, _index, ref recText))
{
yield break;
}
if (recText == null)
{
continue;
}
else if (recText.Length > 0)
{
sw.Write("{1}{0}", recText, Configuration.FileHeaderConfiguration.HasHeaderRecord || HasExcelSeparator ? Configuration.EOLDelimiter : "");
continue;
}
try
{
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
record.DoObjectLevelValidatation(Configuration.CSVRecordFieldConfigurations.Cast <ChoRecordFieldConfiguration>().ToArray());
}
if (ToText(_index, record, out recText))
{
if (_index == 1)
{
sw.Write("{1}{0}", recText, Configuration.FileHeaderConfiguration.HasHeaderRecord || HasExcelSeparator ? Configuration.EOLDelimiter : "");
}
else
{
sw.Write("{1}{0}", recText, Configuration.EOLDelimiter);
}
if (!RaiseAfterRecordWrite(record, _index, recText))
{
yield break;
}
}
}
catch (ChoParserException)
{
throw;
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordWriteError(record, _index, recText, ex))
{
throw;
}
}
else
{
throw;
}
}
}
}
yield return(record);
}
}
finally
{
System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo;
}
RaiseEndWrite(sw);
}
private bool ToText(int index, object rec, out string recText)
{
recText = null;
StringBuilder msg = new StringBuilder();
object fieldValue = null;
string fieldText = null;
ChoCSVRecordFieldConfiguration fieldConfig = null;
if (Configuration.ColumnCountStrict)
{
CheckColumnsStrict(rec);
}
bool firstColumn = true;
foreach (KeyValuePair <string, ChoCSVRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict)
{
fieldConfig = kvp.Value;
fieldValue = null;
fieldText = String.Empty;
if (Configuration.ThrowAndStopOnMissingField)
{
if (rec is ExpandoObject)
{
var dict = rec as IDictionary <string, Object>;
if (!dict.Keys.Contains(kvp.Key, Configuration.FileHeaderConfiguration.StringComparer))
{
throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' CSV column.".FormatString(fieldConfig.FieldName));
}
}
else
{
if (!ChoType.HasProperty(rec.GetType(), kvp.Key))
{
throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' CSV column.".FormatString(fieldConfig.FieldName));
}
}
}
try
{
if (rec is ExpandoObject)
{
IDictionary <string, Object> dict = rec as IDictionary <string, Object>;
fieldValue = dict.GetValue(kvp.Key, Configuration.FileHeaderConfiguration.IgnoreCase, Configuration.Culture);
if (kvp.Value.FieldType == null)
{
if (fieldValue == null)
{
kvp.Value.FieldType = typeof(string);
}
else
{
kvp.Value.FieldType = fieldValue.GetType();
}
}
}
else
{
if (ChoType.HasProperty(rec.GetType(), kvp.Key))
{
fieldValue = ChoType.GetPropertyValue(rec, kvp.Key);
if (kvp.Value.FieldType == null)
{
kvp.Value.FieldType = ChoType.GetMemberType(rec.GetType(), kvp.Key);
}
}
else
{
kvp.Value.FieldType = typeof(string);
}
}
//Discover default value, use it if null
if (fieldValue == null)
{
fieldValue = rec.GetDefaultValue(kvp.Key, fieldConfig);
}
if (!RaiseBeforeRecordFieldWrite(rec, index, kvp.Key, ref fieldValue))
{
return(false);
}
fieldValue = rec.GetNConvertMemberValue(kvp.Key, kvp.Value, Configuration.Culture, fieldValue);
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
if (!RaiseAfterRecordFieldWrite(rec, index, kvp.Key, fieldValue))
{
return(false);
}
}
catch (ChoParserException)
{
throw;
}
catch (ChoMissingRecordFieldException)
{
if (Configuration.ThrowAndStopOnMissingField)
{
throw;
}
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ex);
if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
{
throw;
}
try
{
if (rec is ExpandoObject)
{
var dict = rec as IDictionary <string, Object>;
if (dict.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else
{
throw new ChoParserException($"Failed to write '{fieldValue}' value of '{kvp.Key}' member.", ex);
}
}
else if (ChoType.HasProperty(rec.GetType(), kvp.Key) && rec.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else
{
throw new ChoParserException($"Failed to write '{fieldValue}' value of '{kvp.Key}' member.", ex);
}
}
catch (Exception innerEx)
{
if (ex == innerEx.InnerException)
{
if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
continue;
}
else
{
if (!RaiseRecordFieldWriteError(rec, index, kvp.Key, fieldText, ex))
{
throw new ChoParserException($"Failed to write '{fieldValue}' value of '{kvp.Key}' member.", ex);
}
}
}
else
{
throw new ChoParserException("Failed to use '{0}' fallback value for '{1}' member.".FormatString(fieldValue, kvp.Key), innerEx);
}
}
}
if (fieldValue == null)
{
fieldText = String.Empty;
}
else
{
fieldText = fieldValue.ToString();
}
if (firstColumn)
{
msg.Append(NormalizeFieldValue(kvp.Key, fieldText, kvp.Value.Size, kvp.Value.Truncate, kvp.Value.QuoteField, GetFieldValueJustification(kvp.Value.FieldValueJustification), GetFillChar(kvp.Value.FillChar), false));
firstColumn = false;
}
else
{
msg.AppendFormat("{0}{1}", Configuration.Delimiter, NormalizeFieldValue(kvp.Key, fieldText, kvp.Value.Size, kvp.Value.Truncate, kvp.Value.QuoteField, GetFieldValueJustification(kvp.Value.FieldValueJustification),
GetFillChar(kvp.Value.FillChar), false));
}
}
recText = msg.ToString();
return(true);
}
public override IEnumerable <object> WriteTo(object writer, IEnumerable <object> records, Func <object, bool> predicate = null)
{
TextWriter sw = writer as TextWriter;
ChoGuard.ArgumentNotNull(sw, "TextWriter");
if (records == null)
{
yield break;
}
if (!RaiseBeginWrite(sw))
{
yield break;
}
CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture;
System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture;
string recText = String.Empty;
Type recType;
try
{
foreach (object record in records)
{
_index++;
recType = record.GetType();
if (record is IChoETLNameableObject)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index));
}
recText = String.Empty;
if (record != null)
{
if (predicate == null || predicate(record))
{
//Discover and load Manifold columns from first record
if (!_configCheckDone)
{
//string[] fieldNames = null;
//if (record is ExpandoObject)
//{
// var dict = record as IDictionary<string, Object>;
// fieldNames = dict.Keys.ToArray();
//}
//else
//{
// fieldNames = ChoTypeDescriptor.GetProperties<ChoManifoldRecordFieldAttribute>(record.GetType()).Select(pd => pd.Name).ToArray();
// if (fieldNames.Length == 0)
// {
// fieldNames = ChoType.GetProperties(record.GetType()).Select(p => p.Name).ToArray();
// }
//}
//Configuration.Validate(fieldNames);
//WriteHeaderLine(sw);
_configCheckDone = true;
}
if (!RaiseBeforeRecordWrite(record, _index, ref recText))
{
yield break;
}
if (recText == null)
{
continue;
}
else if (recText.Length > 0)
{
sw.Write("{1}{0}", recText, Configuration.FileHeaderConfiguration.HasHeaderRecord ? Configuration.EOLDelimiter : "");
continue;
}
try
{
var config = GetConfiguration(record.GetType());
if (config == null)
{
throw new ChoParserException("No writer found to write record.");
}
if (config.GetType() == typeof(ChoCSVRecordConfiguration))
{
recText = ChoCSVWriter.ToText(record, config as ChoCSVRecordConfiguration, Configuration.Encoding, Configuration.BufferSize, _offSwitch);
}
else if (config.GetType() == typeof(ChoFixedLengthRecordConfiguration))
{
recText = ChoFixedLengthWriter.ToText(record, config as ChoFixedLengthRecordConfiguration, Configuration.Encoding, Configuration.BufferSize, _offSwitch);
}
else
{
throw new ChoParserException("Unsupported record found to write.");
}
if (recText != null)
{
if (_index == 1)
{
sw.Write("{1}{0}", recText, Configuration.FileHeaderConfiguration.HasHeaderRecord ? Configuration.EOLDelimiter : "");
}
else
{
sw.Write("{1}{0}", recText, Configuration.EOLDelimiter);
}
if (!RaiseAfterRecordWrite(record, _index, recText))
{
yield break;
}
}
}
//catch (ChoParserException pEx)
//{
// throw new ChoParserException($"Failed to write line for '{recType}' object.", pEx);
//}
catch (Exception ex)
{
ChoETLFramework.HandleException(ref ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordWriteError(record, _index, recText, ex))
{
throw new ChoWriterException($"Failed to write line for '{recType}' object.", ex);
}
else
{
//ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
}
}
else
{
throw new ChoWriterException($"Failed to write line for '{recType}' object.", ex);
}
}
}
}
yield return(record);
if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0)
{
if (RaisedRowsWritten(_index))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
yield break;
}
}
}
}
finally
{
System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo;
}
RaiseEndWrite(sw);
}
private bool FillRecord(object rec, Tuple <long, JObject> pair)
{
long lineNo;
JObject node;
JToken jToken = null;
JToken[] jTokens = null;
lineNo = pair.Item1;
node = pair.Item2;
fieldValue = null;
fieldConfig = null;
pi = null;
//IDictionary<string, object> dictValues = ToDictionary(node);
if (!Configuration.IsDynamicObject)
{
if (rec.FillIfCustomSerialization(pair.Item2))
{
return(true);
}
if (FillIfKeyValueObject(rec, pair))
{
return(true);
}
}
object rootRec = rec;
foreach (KeyValuePair <string, ChoJSONRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict)
{
fieldValue = null;
fieldConfig = kvp.Value;
if (Configuration.PIDict != null)
{
Configuration.PIDict.TryGetValue(kvp.Key, out pi);
}
rec = GetDeclaringRecord(kvp.Value.DeclaringMember, rootRec);
//fieldValue = dictValues[kvp.Key];
if (!kvp.Value.JSONPath.IsNullOrWhiteSpace())
{
jTokens = node.SelectTokens(kvp.Value.JSONPath).ToArray();
jToken = jTokens.FirstOrDefault();
if (jToken == null)
{
if (Configuration.ColumnCountStrict)
{
throw new ChoParserException("No matching '{0}' field found.".FormatString(fieldConfig.FieldName));
}
}
}
else
{
if (!node.TryGetValue(kvp.Key, StringComparison.CurrentCultureIgnoreCase, out jToken))
{
if (Configuration.ColumnCountStrict)
{
throw new ChoParserException("No matching '{0}' field found.".FormatString(fieldConfig.FieldName));
}
}
}
fieldValue = !jTokens.IsNullOrEmpty() ? (object)jTokens : jToken;
if (!RaiseBeforeRecordFieldLoad(rec, pair.Item1, kvp.Key, ref fieldValue))
{
continue;
}
if (Configuration.IsDynamicObject) //rec is ExpandoObject)
{
}
else
{
if (pi != null)
{
kvp.Value.FieldType = pi.PropertyType;
}
else
{
kvp.Value.FieldType = typeof(string);
}
}
if (fieldConfig.ValueConverter != null)
{
fieldValue = fieldConfig.ValueConverter(fieldValue);
}
else
{
if (fieldConfig.FieldType == null)
{
if (!fieldConfig.IsArray && fieldValue is JToken[])
{
fieldValue = ((JToken[])fieldValue).FirstOrDefault();
if (fieldValue is JArray)
{
fieldValue = ((JArray)fieldValue).FirstOrDefault();
}
}
}
else
{
if (!fieldConfig.FieldType.IsCollection() && fieldValue is JToken[])
{
fieldValue = ((JToken[])fieldValue).FirstOrDefault();
//if (fieldValue is JArray)
//{
// fieldValue = ((JArray)fieldValue).FirstOrDefault();
//}
}
}
if (fieldConfig.FieldType == null ||
fieldConfig.FieldType == typeof(object) ||
fieldConfig.FieldType.GetItemType() == typeof(object))
{
if (fieldValue is JToken)
{
if (fieldConfig.ItemConverter != null)
{
fieldValue = fieldConfig.ItemConverter(fieldValue);
}
else
{
fieldValue = ToObject((JToken)fieldValue, null);
}
}
else if (fieldValue is JToken[])
{
List <object> arr = new List <object>();
foreach (var ele in (JToken[])fieldValue)
{
if (fieldConfig.ItemConverter != null)
{
arr.Add(fieldConfig.ItemConverter(ele));
}
else
{
arr.Add(ToObject(ele, null));
}
}
fieldValue = arr.ToArray();
}
}
else if (fieldConfig.FieldType == typeof(string) || fieldConfig.FieldType.IsSimple())
{
if (fieldValue is JToken[])
{
fieldValue = ((JToken[])fieldValue).FirstOrDefault();
}
if (fieldValue is JToken)
{
if (fieldConfig.ItemConverter != null)
{
fieldValue = fieldConfig.ItemConverter(fieldValue);
}
else
{
fieldValue = ToObject((JToken)fieldValue, fieldConfig.FieldType);
}
}
}
//else if (fieldConfig.FieldType.IsCollection())
//{
// List<object> list = new List<object>();
// Type itemType = fieldConfig.FieldType.GetItemType().GetUnderlyingType();
// if (fieldValue is JToken)
// {
// if (fieldConfig.ItemConverter != null)
// fieldValue = fieldConfig.ItemConverter(fieldValue);
// else
// fieldValue = ToObject((JToken)fieldValue, itemType);
// }
// else if (fieldValue is JToken[])
// {
// foreach (var ele in (JToken[])fieldValue)
// {
// if (fieldConfig.ItemConverter != null)
// list.Add(fieldConfig.ItemConverter(ele));
// else
// {
// fieldValue = ToObject(ele, itemType);
// }
// }
// fieldValue = list.ToArray();
// }
//}
else
{
List <object> list = new List <object>();
Type itemType = fieldConfig.FieldType.GetUnderlyingType().GetItemType().GetUnderlyingType();
if (fieldValue is JToken)
{
if (fieldConfig.ItemConverter != null)
{
fieldValue = fieldConfig.ItemConverter(fieldValue);
}
else
{
fieldValue = ToObject((JToken)fieldValue, itemType);
}
}
else if (fieldValue is JToken[])
{
var fi = ((JToken[])fieldValue).FirstOrDefault();
if (fi is JArray && !itemType.IsCollection())
{
if (fieldConfig.ItemConverter != null)
{
fieldValue = fieldConfig.ItemConverter(fi);
}
else
{
fieldValue = ToObject(fi, fieldConfig.FieldType);
}
}
else
{
foreach (var ele in (JToken[])fieldValue)
{
if (fieldConfig.ItemConverter != null)
{
list.Add(fieldConfig.ItemConverter(ele));
}
else
{
list.Add(ToObject(ele, itemType));
}
}
fieldValue = list.ToArray();
}
}
}
}
if (!(fieldValue is ICollection))
{
if (fieldValue is string)
{
fieldValue = CleanFieldValue(fieldConfig, kvp.Value.FieldType, fieldValue as string);
}
else if (fieldValue is JValue)
{
if (((JValue)fieldValue).Value is string)
{
fieldValue = CleanFieldValue(fieldConfig, kvp.Value.FieldType, fieldValue.ToString());
}
else
{
fieldValue = ((JValue)fieldValue).Value;
}
}
}
try
{
bool ignoreFieldValue = fieldConfig.IgnoreFieldValue(fieldValue);
if (ignoreFieldValue)
{
fieldValue = fieldConfig.IsDefaultValueSpecified ? fieldConfig.DefaultValue : null;
}
if (Configuration.IsDynamicObject)
{
var dict = rec as IDictionary <string, Object>;
dict.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel)
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
}
else
{
if (pi != null)
{
rec.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
}
else
{
throw new ChoMissingRecordFieldException("Missing '{0}' property in {1} type.".FormatString(kvp.Key, ChoType.GetTypeName(rec)));
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel)
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
}
if (!RaiseAfterRecordFieldLoad(rec, pair.Item1, kvp.Key, fieldValue))
{
return(false);
}
}
catch (ChoParserException)
{
Reader.IsValid = false;
throw;
}
catch (ChoMissingRecordFieldException)
{
Reader.IsValid = false;
if (Configuration.ThrowAndStopOnMissingField)
{
throw;
}
}
catch (Exception ex)
{
Reader.IsValid = false;
ChoETLFramework.HandleException(ref ex);
if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
{
throw;
}
try
{
if (Configuration.IsDynamicObject)
{
var dict = rec as IDictionary <string, Object>;
if (dict.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (dict.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (ex is ValidationException)
{
throw;
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
else if (pi != null)
{
if (rec.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (rec.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (ex is ValidationException)
{
throw;
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
catch (Exception innerEx)
{
if (ex == innerEx.InnerException || ex is ValidationException)
{
if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
continue;
}
else
{
if (!RaiseRecordFieldLoadError(rec, pair.Item1, kvp.Key, fieldValue, ex))
{
if (ex is ValidationException)
{
throw;
}
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
}
else
{
throw new ChoReaderException("Failed to assign '{0}' fallback value to '{1}' field.".FormatString(fieldValue, fieldConfig.FieldName), innerEx);
}
}
}
}
return(true);
}
private bool ToText(long index, object rec, out string recText)
{
if (typeof(IChoScalarObject).IsAssignableFrom(Configuration.RecordType))
{
rec = Activator.CreateInstance(Configuration.RecordType, rec);
}
if (!Configuration.IsDynamicObject)
{
if (rec.ToTextIfCustomSerialization(out recText))
{
return(true);
}
//Check if KVP object
if (rec.GetType().IsKeyValueType())
{
recText = SerializeObject(rec);
return(true);
}
}
recText = null;
if (rec == null)
{
if (Configuration.NullValueHandling == ChoNullValueHandling.Ignore)
{
return(false);
}
else if (Configuration.NullValueHandling == ChoNullValueHandling.Default)
{
rec = Activator.CreateInstance(Configuration.RecordType);
}
else
{
recText = "{{{0}}}".FormatString(Configuration.Formatting == Formatting.Indented ? Configuration.EOLDelimiter : String.Empty);
return(true);
}
}
StringBuilder msg = new StringBuilder();
object fieldValue = null;
string fieldText = null;
ChoJSONRecordFieldConfiguration fieldConfig = null;
if (Configuration.ColumnCountStrict)
{
CheckColumnsStrict(rec);
}
//bool firstColumn = true;
PropertyInfo pi = null;
bool isFirst = true;
object rootRec = rec;
msg.AppendFormat("{{{0}", Configuration.Formatting == Formatting.Indented ? Configuration.EOLDelimiter : String.Empty);
foreach (KeyValuePair <string, ChoJSONRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict)
{
fieldConfig = kvp.Value;
fieldValue = null;
fieldText = String.Empty;
if (Configuration.PIDict != null)
{
Configuration.PIDict.TryGetValue(kvp.Key, out pi);
}
rec = GetDeclaringRecord(kvp.Value.DeclaringMember, rootRec);
if (Configuration.ThrowAndStopOnMissingField)
{
if (Configuration.IsDynamicObject)
{
var dict = rec.ToDynamicObject() as IDictionary <string, Object>;
if (!dict.ContainsKey(kvp.Key))
{
throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' JSON node.".FormatString(fieldConfig.FieldName));
}
}
else
{
if (pi == null)
{
throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' JSON node.".FormatString(fieldConfig.FieldName));
}
}
}
try
{
if (Configuration.IsDynamicObject)
{
IDictionary <string, Object> dict = rec.ToDynamicObject() as IDictionary <string, Object>;
fieldValue = dict[kvp.Key]; // dict.GetValue(kvp.Key, Configuration.FileHeaderConfiguration.IgnoreCase, Configuration.Culture);
if (kvp.Value.FieldType == null)
{
if (fieldValue == null)
{
kvp.Value.FieldType = typeof(string);
}
else
{
kvp.Value.FieldType = fieldValue.GetType();
}
}
}
else
{
if (pi != null)
{
fieldValue = ChoType.GetPropertyValue(rec, pi);
if (kvp.Value.FieldType == null)
{
kvp.Value.FieldType = pi.PropertyType;
}
}
else
{
kvp.Value.FieldType = typeof(string);
}
}
//Discover default value, use it if null
if (fieldValue == null)
{
if (fieldConfig.IsDefaultValueSpecified)
{
fieldValue = fieldConfig.DefaultValue;
}
}
if (!RaiseBeforeRecordFieldWrite(rec, index, kvp.Key, ref fieldValue))
{
return(false);
}
if (fieldConfig.ValueConverter != null)
{
fieldValue = fieldConfig.ValueConverter(fieldValue);
}
else
{
rec.GetNConvertMemberValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue, true);
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.MemberLevel)
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
if (!RaiseAfterRecordFieldWrite(rec, index, kvp.Key, fieldValue))
{
return(false);
}
}
catch (ChoParserException)
{
throw;
}
catch (ChoMissingRecordFieldException)
{
if (Configuration.ThrowAndStopOnMissingField)
{
throw;
}
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ref ex);
if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
{
throw;
}
try
{
if (Configuration.IsDynamicObject)
{
var dict = rec.ToDynamicObject() as IDictionary <string, Object>;
if (dict.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else if (dict.GetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else
{
var ex1 = new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
fieldValue = null;
throw ex1;
}
}
else if (pi != null)
{
if (rec.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (rec.GetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else
{
var ex1 = new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
fieldValue = null;
throw ex1;
}
}
else
{
var ex1 = new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
fieldValue = null;
throw ex1;
}
}
catch (Exception innerEx)
{
if (ex == innerEx.InnerException)
{
if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
continue;
}
else
{
if (!RaiseRecordFieldWriteError(rec, index, kvp.Key, fieldText, ex))
{
throw new ChoWriterException($"Failed to write '{fieldValue}' value of '{kvp.Key}' member.", ex);
}
}
}
else
{
throw new ChoWriterException("Failed to use '{0}' fallback value for '{1}' member.".FormatString(fieldValue, kvp.Key), innerEx);
}
}
}
bool isSimple = true;
if (RaiseRecordFieldSerialize(rec, index, kvp.Key, ref fieldValue))
{
if (isFirst)
{
msg.AppendFormat("{1}{0}", fieldValue.ToNString(),
Configuration.Formatting == Formatting.Indented ? " " : String.Empty);
}
else
{
msg.AppendFormat(",{1}{2}{0}", fieldValue.ToNString(),
Configuration.Formatting == Formatting.Indented ? Configuration.EOLDelimiter : String.Empty,
Configuration.Formatting == Formatting.Indented ? " " : String.Empty);
}
}
else
{
Type ft = fieldValue == null ? typeof(object) : fieldValue.GetType();
if (fieldValue == null)
{
fieldText = "null";
}
else if (ft == typeof(string) || ft == typeof(char))
{
fieldText = JsonConvert.SerializeObject(NormalizeFieldValue(kvp.Key, fieldValue.ToString(), kvp.Value.Size, kvp.Value.Truncate, false, GetFieldValueJustification(kvp.Value.FieldValueJustification, kvp.Value.FieldType), GetFillChar(kvp.Value.FillChar, kvp.Value.FieldType), false));
}
else if (ft == typeof(DateTime))
{
fieldText = JsonConvert.SerializeObject(fieldValue);
}
else if (ft.IsEnum)
{
fieldText = JsonConvert.SerializeObject(fieldValue);
}
else if (ft == typeof(ChoCurrency))
{
fieldText = "\"{0}\"".FormatString(fieldValue.ToString());
}
else if (ft == typeof(bool))
{
fieldText = JsonConvert.SerializeObject(fieldValue);
}
else if (ft.IsNumeric())
{
fieldText = fieldValue.ToString();
}
else
{
isSimple = false;
}
if (isFirst)
{
msg.AppendFormat("{2}\"{0}\":{1}", fieldConfig.FieldName, isSimple ? " {0}".FormatString(fieldText) :
Configuration.Formatting == Formatting.Indented ? SerializeObject(fieldValue, fieldConfig.UseJSONSerialization).Indent(1, " ") : SerializeObject(fieldValue, fieldConfig.UseJSONSerialization),
Configuration.Formatting == Formatting.Indented ? " " : String.Empty);
}
else
{
msg.AppendFormat(",{2}{3}\"{0}\":{1}", fieldConfig.FieldName, isSimple ? " {0}".FormatString(fieldText) :
Configuration.Formatting == Formatting.Indented ? SerializeObject(fieldValue, fieldConfig.UseJSONSerialization).Indent(1, " ") : SerializeObject(fieldValue, fieldConfig.UseJSONSerialization),
Configuration.Formatting == Formatting.Indented ? Configuration.EOLDelimiter : String.Empty, Configuration.Formatting == Formatting.Indented ? " " : String.Empty);
}
}
isFirst = false;
}
msg.AppendFormat("{0}}}", Configuration.Formatting == Formatting.Indented ? Configuration.EOLDelimiter : String.Empty);
recText = msg.ToString();
return(true);
}
private bool LoadNode(Tuple <long, XElement> pair, ref object rec)
{
if (!Configuration.UseXmlSerialization || Configuration.IsDynamicObject)
{
rec = Configuration.IsDynamicObject ? new ChoDynamicObject()
{
ThrowExceptionIfPropNotExists = true
}
}
: Activator.CreateInstance(RecordType);
try
{
if (!RaiseBeforeRecordLoad(rec, ref pair))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping...");
rec = null;
return(true);
}
if (pair.Item2 == null)
{
rec = null;
return(true);
}
if (!Configuration.UseXmlSerialization)
{
if (!FillRecord(rec, pair))
{
return(false);
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
rec.DoObjectLevelValidation(Configuration, Configuration.XmlRecordFieldConfigurations);
}
}
else
{
if (Configuration.IsDynamicObject)
{
Parse(rec, pair.Item2);
}
else
{
rec = _se.Value.Deserialize(pair.Item2.CreateReader());
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.Off) != ChoObjectValidationMode.Off)
{
rec.DoObjectLevelValidation(Configuration, Configuration.XmlRecordFieldConfigurations);
}
}
bool skip = false;
if (!RaiseAfterRecordLoad(rec, pair, ref skip))
{
return(false);
}
else if (skip)
{
rec = null;
return(true);
}
}
//catch (ChoParserException)
//{
// throw;
//}
catch (ChoMissingRecordFieldException)
{
throw;
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
rec = null;
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordLoadError(rec, pair, ex))
{
throw;
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
rec = null;
}
}
else
{
throw;
}
return(true);
}
return(true);
}
public override IEnumerable <object> WriteTo(object writer, IEnumerable <object> records, Func <object, bool> predicate = null)
{
_sw = writer;
TextWriter sw = writer as TextWriter;
ChoGuard.ArgumentNotNull(sw, "TextWriter");
if (Configuration.JsonSerializerSettings.ContractResolver is ChoPropertyRenameAndIgnoreSerializerContractResolver)
{
ChoPropertyRenameAndIgnoreSerializerContractResolver cr = Configuration.JsonSerializerSettings.ContractResolver as ChoPropertyRenameAndIgnoreSerializerContractResolver;
cr.CallbackRecordFieldWrite = _callbackRecordFieldWrite;
cr.Writer = Writer;
}
if (records == null)
{
yield break;
}
if (Configuration.SingleDocument == null)
{
Configuration.SingleDocument = false;
}
CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture;
System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture;
string recText = String.Empty;
bool recordIgnored = false;
try
{
foreach (object record in records)
{
_index++;
//if (!isFirstRec)
//{
// if (!recordIgnored)
// sw.Write(",");
// else
// recordIgnored = false;
//}
if (TraceSwitch.TraceVerbose)
{
if (record is IChoETLNameableObject)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index));
}
}
recText = String.Empty;
if (predicate == null || predicate(record))
{
//Discover and load Xml columns from first record
if (!Configuration.IsInitialized)
{
if (record == null)
{
continue;
}
string[] fieldNames = null;
if (Configuration.RecordType == typeof(object))
{
Type recordType = ElementType == null?record.GetType() : ElementType;
RecordType = Configuration.RecordType = recordType.GetUnderlyingType(); //.ResolveType();
Configuration.IsDynamicObject = recordType.IsDynamicType();
}
if (typeof(IDictionary).IsAssignableFrom(Configuration.RecordType) ||
typeof(IList).IsAssignableFrom(Configuration.RecordType))
{
Configuration.UseYamlSerialization = true;
}
if (!Configuration.IsDynamicObject)
{
if (Configuration.YamlRecordFieldConfigurations.Count == 0)
{
Configuration.MapRecordFields(Configuration.RecordType);
}
}
if (Configuration.IsDynamicObject)
{
var dict = record.ToDynamicObject() as IDictionary <string, Object>;
fieldNames = dict.Keys.ToArray();
}
else
{
fieldNames = ChoTypeDescriptor.GetProperties <ChoYamlRecordFieldAttribute>(Configuration.RecordType).Select(pd => pd.Name).ToArray();
if (fieldNames.Length == 0)
{
fieldNames = ChoType.GetProperties(Configuration.RecordType).Select(p => p.Name).ToArray();
}
}
Configuration.Validate(fieldNames);
Configuration.IsInitialized = true;
if (!BeginWrite.Value)
{
yield break;
}
}
if (!RaiseBeforeRecordWrite(record, _index, ref recText))
{
yield break;
}
if (recText == null)
{
continue;
}
try
{
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
record.DoObjectLevelValidation(Configuration, Configuration.YamlRecordFieldConfigurations);
}
if (ToText(_index, record, out recText))
{
if (recText.EndsWith("..."))
{
recText = recText.Left(recText.Length - 3);
}
if (recText.EndsWith($"...{Environment.NewLine}"))
{
recText = recText.Left(recText.Length - $"...{Environment.NewLine}".Length);
}
if (!Configuration.SingleDocument.Value)
{
sw.Write($"---{EOLDelimiter}");
}
sw.Write("{0}", recText);
if (!Configuration.SingleDocument.Value)
{
sw.Write($"...{EOLDelimiter}");
}
if (!RaiseAfterRecordWrite(record, _index, recText))
{
yield break;
}
}
}
//catch (ChoParserException)
//{
// throw;
//}
catch (Exception ex)
{
ChoETLFramework.HandleException(ref ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
recordIgnored = true;
ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordWriteError(record, _index, recText, ex))
{
throw;
}
else
{
recordIgnored = true;
//ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
}
}
else
{
throw;
}
}
}
yield return(record);
if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0)
{
if (RaisedRowsWritten(_index))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
yield break;
}
}
}
}
finally
{
System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo;
}
}
private bool FillRecord(object rec, Tuple <long, XElement> pair)
{
long lineNo;
XElement node;
lineNo = pair.Item1;
node = pair.Item2;
fXElements = null;
fieldValue = null;
fieldConfig = null;
pi = null;
xpn = node.CreateNavigator(Configuration.NamespaceManager.NameTable);
ToDictionary(node);
foreach (KeyValuePair <string, ChoXmlRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict)
{
fieldValue = null;
fieldConfig = kvp.Value;
if (Configuration.PIDict != null)
{
Configuration.PIDict.TryGetValue(kvp.Key, out pi);
}
if (fieldConfig.XPath == "text()")
{
if (Configuration.GetNameWithNamespace(node.Name) == fieldConfig.FieldName)
{
object value = node;
if (!RaiseBeforeRecordFieldLoad(rec, pair.Item1, kvp.Key, ref value))
{
continue;
}
if (fieldConfig.ValueConverter != null)
{
value = fieldConfig.ValueConverter(value);
}
fieldValue = value is XElement ? node.Value : value;
}
else if (Configuration.ColumnCountStrict)
{
throw new ChoParserException("Missing '{0}' xml node.".FormatString(fieldConfig.FieldName));
}
}
else
{
if (/*!fieldConfig.UseCache && */ !xDict.ContainsKey(fieldConfig.FieldName))
{
xNodes.Clear();
foreach (XPathNavigator z in xpn.Select(fieldConfig.XPath, Configuration.NamespaceManager))
{
xNodes.Add(z.UnderlyingObject);
}
object value = xNodes;
if (!RaiseBeforeRecordFieldLoad(rec, pair.Item1, kvp.Key, ref value))
{
continue;
}
if (fieldConfig.ValueConverter != null)
{
fieldValue = fieldConfig.ValueConverter(value);
}
else
{
//object[] xNodes = ((IEnumerable)node.XPathEvaluate(fieldConfig.XPath, Configuration.NamespaceManager)).OfType<object>().ToArray();
//continue;
XAttribute fXAttribute = xNodes.OfType <XAttribute>().FirstOrDefault();
if (fXAttribute != null)
{
fieldValue = fXAttribute.Value;
}
else
{
fXElements = xNodes.OfType <XElement>().ToArray();
if (fXElements != null)
{
if (Configuration.IsDynamicObject)
{
if (fieldConfig.IsArray != null && fieldConfig.IsArray.Value)
{
List <object> list = new List <object>();
foreach (var ele in fXElements)
{
if (fieldConfig.FieldType.IsSimple())
{
list.Add(ChoConvert.ConvertTo(ele.Value, fieldConfig.FieldType.GetItemType()));
}
else
{
list.Add(ele.GetOuterXml().ToObjectFromXml(fieldConfig.FieldType.GetItemType()));
}
}
fieldValue = list.ToArray();
}
else
{
if (fieldConfig.FieldType == typeof(string) || fieldConfig.FieldType.IsSimple())
{
XElement fXElement = fXElements.FirstOrDefault();
if (fXElement != null)
{
fieldValue = fXElement.Value;
}
}
else
{
XmlAttributeOverrides overrides = null;
var xattribs = new XmlAttributes();
var xroot = new XmlRootAttribute(fieldConfig.FieldName);
xattribs.XmlRoot = xroot;
overrides = new XmlAttributeOverrides();
overrides.Add(fieldConfig.FieldType, xattribs);
XElement fXElement = fXElements.FirstOrDefault();
if (fXElement != null)
{
fieldValue = fXElement.GetOuterXml().ToObjectFromXml(fieldConfig.FieldType, overrides);
}
}
}
}
else
{
XmlAttributeOverrides overrides = null;
var xattribs = new XmlAttributes();
var xroot = new XmlRootAttribute(fieldConfig.FieldName);
xattribs.XmlRoot = xroot;
overrides = new XmlAttributeOverrides();
overrides.Add(fieldConfig.FieldType, xattribs);
XElement fXElement = fXElements.FirstOrDefault();
if (fXElement != null)
{
fieldValue = fXElement.GetOuterXml().ToObjectFromXml(fieldConfig.FieldType, overrides);
}
}
}
else if (Configuration.ColumnCountStrict)
{
throw new ChoParserException("Missing '{0}' xml node.".FormatString(fieldConfig.FieldName));
}
}
}
}
else
{
if (xDict[fieldConfig.FieldName].Count == 1)
{
fieldValue = xDict[fieldConfig.FieldName][0];
}
else
{
fieldValue = xDict[fieldConfig.FieldName];
}
if (!RaiseBeforeRecordFieldLoad(rec, pair.Item1, kvp.Key, ref fieldValue))
{
continue;
}
if (fieldConfig.ValueConverter != null)
{
fieldValue = fieldConfig.ValueConverter(fieldValue);
}
}
}
if (Configuration.IsDynamicObject)
{
if (kvp.Value.FieldType == null)
{
kvp.Value.FieldType = fieldValue is ICollection ? typeof(string[]) : DiscoverFieldType(fieldValue as string);
}
}
else
{
if (pi != null)
{
kvp.Value.FieldType = pi.PropertyType;
}
else
{
kvp.Value.FieldType = typeof(string);
}
}
if (fieldValue is string)
{
fieldValue = CleanFieldValue(fieldConfig, kvp.Value.FieldType, fieldValue as string);
}
try
{
bool ignoreFieldValue = fieldConfig.IgnoreFieldValue(fieldValue);
if (ignoreFieldValue)
{
fieldValue = fieldConfig.IsDefaultValueSpecified ? fieldConfig.DefaultValue : null;
}
if (Configuration.IsDynamicObject)
{
var dict = rec as IDictionary <string, Object>;
if (!fieldConfig.IsArray.CastTo <bool>())
{
dict.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
}
else
{
dict[kvp.Key] = fieldValue;
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel)
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
}
else
{
if (pi != null)
{
rec.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
}
else
{
throw new ChoMissingRecordFieldException("Missing '{0}' property in {1} type.".FormatString(kvp.Key, ChoType.GetTypeName(rec)));
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel)
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
}
if (!RaiseAfterRecordFieldLoad(rec, pair.Item1, kvp.Key, fieldValue))
{
return(false);
}
}
catch (ChoParserException)
{
throw;
}
catch (ChoMissingRecordFieldException)
{
if (Configuration.ThrowAndStopOnMissingField)
{
throw;
}
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ex);
if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
{
throw;
}
try
{
if (Configuration.IsDynamicObject)
{
var dict = rec as IDictionary <string, Object>;
if (dict.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (dict.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
else if (pi != null)
{
if (rec.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (rec.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
catch (Exception innerEx)
{
if (ex == innerEx.InnerException)
{
if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
continue;
}
else
{
if (!RaiseRecordFieldLoadError(rec, pair.Item1, kvp.Key, fieldValue, ex))
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
}
else
{
throw new ChoReaderException("Failed to assign '{0}' fallback value to '{1}' field.".FormatString(fieldValue, fieldConfig.FieldName), innerEx);
}
}
}
}
return(true);
}
private bool LoadNode(Tuple <long, IDictionary <string, object> > pair, ref object rec)
{
bool ignoreFieldValue = pair.Item2.IgnoreFieldValue(Configuration.IgnoreFieldValueMode);
if (ignoreFieldValue)
{
return(false);
}
else if (pair.Item2 == null && !Configuration.IsDynamicObject)
{
rec = RecordType.CreateInstanceAndDefaultToMembers(Configuration.RecordFieldConfigurationsDict.ToDictionary(kvp => kvp.Key, kvp => kvp.Value as ChoRecordFieldConfiguration));
return(true);
}
if (Configuration.SupportsMultiRecordTypes && Configuration.RecordSelector != null)
{
Type recType = Configuration.RecordSelector(pair);
if (recType == null)
{
if (Configuration.IgnoreIfNoRecordTypeFound)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, $"No record type found for [{pair.Item1}] line to parse.");
return(true);
}
else
{
throw new ChoParserException($"No record type found for [{pair.Item1}] line to parse.");
}
}
if (!Configuration.RecordTypeMapped)
{
Configuration.MapRecordFields(recType);
Configuration.Validate(null);
}
rec = recType.IsDynamicType() ? new ChoDynamicObject()
{
ThrowExceptionIfPropNotExists = true
} : ChoActivator.CreateInstance(recType);
RecordType = recType;
}
else if (Configuration.IsDynamicObject)
{
rec = Configuration.IsDynamicObject ? new ChoDynamicObject()
{
ThrowExceptionIfPropNotExists = true
}
}
: ChoActivator.CreateInstance(RecordType);
try
{
if (!RaiseBeforeRecordLoad(rec, ref pair))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping...");
rec = null;
return(true);
}
//if (Configuration.CustomNodeSelecter != null)
//{
// pair = new Tuple<long, IDictionary<string, object>>(pair.Item1, Configuration.CustomNodeSelecter(pair.Item2));
//}
if (pair.Item2 == null)
{
rec = null;
return(true);
}
if (!FillRecord(ref rec, pair))
{
return(false);
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
rec.DoObjectLevelValidation(Configuration, Configuration.ParquetRecordFieldConfigurations);
}
bool skip = false;
if (!RaiseAfterRecordLoad(rec, pair, ref skip))
{
return(false);
}
else if (skip)
{
rec = null;
return(true);
}
}
catch (Exception ex)
{
Reader.IsValid = false;
if (ex is ChoMissingRecordFieldException && Configuration.ThrowAndStopOnMissingField)
{
if (!RaiseRecordLoadError(rec, pair, ex))
{
throw;
}
else
{
//ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
//rec = null;
}
}
else
{
ChoETLFramework.HandleException(ref ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
rec = null;
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordLoadError(rec, pair, ex))
{
throw;
}
else
{
//ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
//rec = null;
}
}
else
{
throw;
}
}
return(true);
}
return(true);
}
object fieldValue = null;
ChoParquetRecordFieldConfiguration fieldConfig = null;
PropertyInfo pi = null;
private bool FillRecord(object rec, Tuple <long, string> pair)
{
long lineNo;
string line;
lineNo = pair.Item1;
line = pair.Item2;
if (line.Length != Configuration.RecordLength)
{
throw new ChoParserException("Incorrect record length [Length: {0}] found. Expected record length: {1}".FormatString(line.Length, Configuration.RecordLength));
}
object fieldValue = null;
ChoFixedLengthRecordFieldConfiguration fieldConfig = null;
PropertyInfo pi = null;
object rootRec = rec;
foreach (KeyValuePair <string, ChoFixedLengthRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict)
{
if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject)
{
if (Configuration.IgnoredFields.Contains(kvp.Key))
{
continue;
}
}
fieldValue = null;
fieldConfig = kvp.Value;
if (Configuration.PIDict != null)
{
Configuration.PIDict.TryGetValue(kvp.Key, out pi);
}
rec = GetDeclaringRecord(kvp.Value.DeclaringMember, rootRec);
try
{
if (fieldConfig.ValueSelector == null)
{
if (fieldConfig.StartIndex + fieldConfig.Size > line.Length)
{
if (Configuration.ColumnCountStrict)
{
throw new ChoParserException("Missing '{0}' field value.".FormatString(kvp.Key));
}
}
else
{
fieldValue = line.Substring(fieldConfig.StartIndex, fieldConfig.Size.Value);
}
}
else
{
fieldValue = fieldConfig.ValueSelector(new ChoDynamicObject(new Dictionary <string, object>()
{
{ "Line", line }
}));
}
if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject)
{
if (kvp.Value.FieldType == null)
{
kvp.Value.FieldType = Configuration.MaxScanRows == -1 ? DiscoverFieldType(fieldValue as string, Configuration) : typeof(string);
}
}
else
{
if (pi != null)
{
kvp.Value.FieldType = pi.PropertyType;
}
else
{
kvp.Value.FieldType = typeof(string);
}
}
fieldValue = fieldValue is string?CleanFieldValue(fieldConfig, kvp.Value.FieldType, fieldValue as string) : fieldValue;
if (!RaiseBeforeRecordFieldLoad(rec, pair.Item1, kvp.Key, ref fieldValue))
{
continue;
}
bool ignoreFieldValue = fieldConfig.IgnoreFieldValue(fieldValue);
if (ignoreFieldValue)
{
fieldValue = fieldConfig.IsDefaultValueSpecified ? fieldConfig.DefaultValue : null;
}
if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject)
{
var dict = rec as IDictionary <string, Object>;
dict.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel)
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
}
else
{
if (Configuration.SupportsMultiRecordTypes)
{
ChoType.TryGetProperty(rec.GetType(), kvp.Key, out pi);
fieldConfig.PI = pi;
fieldConfig.PropConverters = ChoTypeDescriptor.GetTypeConverters(fieldConfig.PI);
fieldConfig.PropConverterParams = ChoTypeDescriptor.GetTypeConverterParams(fieldConfig.PI);
}
if (pi != null)
{
rec.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
}
else if (!Configuration.SupportsMultiRecordTypes)
{
throw new ChoMissingRecordFieldException("Missing '{0}' property in {1} type.".FormatString(kvp.Key, ChoType.GetTypeName(rec)));
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel)
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
}
if (!RaiseAfterRecordFieldLoad(rec, pair.Item1, kvp.Key, fieldValue))
{
return(false);
}
}
catch (ChoParserException)
{
Reader.IsValid = false;
throw;
}
catch (ChoMissingRecordFieldException)
{
Reader.IsValid = false;
if (Configuration.ThrowAndStopOnMissingField)
{
throw;
}
}
catch (Exception ex)
{
Reader.IsValid = false;
ChoETLFramework.HandleException(ref ex);
if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
{
throw;
}
try
{
if (Configuration.IsDynamicObject)
{
var dict = rec as IDictionary <string, Object>;
if (dict.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (dict.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (ex is ValidationException)
{
throw;
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
else if (pi != null)
{
if (rec.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (rec.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (ex is ValidationException)
{
throw;
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
catch (Exception innerEx)
{
if (ex == innerEx.InnerException || ex is ValidationException)
{
if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
continue;
}
else
{
if (!RaiseRecordFieldLoadError(rec, pair.Item1, kvp.Key, fieldValue, ex))
{
if (ex is ValidationException)
{
throw;
}
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
}
else
{
throw new ChoReaderException("Failed to assign '{0}' fallback value to '{1}' field.".FormatString(fieldValue, fieldConfig.FieldName), innerEx);
}
}
}
}
return(true);
}
private bool FillRecord(object rec, Tuple <long, string> pair)
{
long lineNo;
string line;
lineNo = pair.Item1;
line = pair.Item2;
string[] fieldValues = line.Split(Configuration.Delimiter, Configuration.StringSplitOptions, Configuration.QuoteChar);
if (Configuration.ColumnCountStrict)
{
if (fieldValues.Length != Configuration.CSVRecordFieldConfigurations.Count)
{
throw new ChoParserException("Incorrect number of field values found at line [{2}]. Expected [{0}] field values. Found [{1}] field values.".FormatString(Configuration.CSVRecordFieldConfigurations.Count, fieldValues.Length, pair.Item1));
}
}
if (_fieldNames != null) //Configuration.FileHeaderConfiguration.HasHeaderRecord && Configuration.ColumnOrderStrict)
{
if (this.fieldNameValues == null)
{
this.fieldNameValues = InitFieldNameValuesDict();
}
ToFieldNameValues(fieldNameValues, fieldValues);
}
ValidateLine(pair.Item1, fieldValues);
object fieldValue = null;
ChoCSVRecordFieldConfiguration fieldConfig = null;
PropertyInfo pi = null;
foreach (KeyValuePair <string, ChoCSVRecordFieldConfiguration> kvp in Configuration.FCArray)
{
fieldValue = null;
fieldConfig = kvp.Value;
if (Configuration.PIDict != null)
{
Configuration.PIDict.TryGetValue(kvp.Key, out pi);
}
try
{
if (fieldNameValues != null)
{
if (fieldNameValues.ContainsKey(fieldConfig.FieldName))
{
fieldValue = fieldNameValues[fieldConfig.FieldName];
}
else if (Configuration.ThrowAndStopOnMissingField)
{
throw new ChoMissingRecordFieldException("Missing '{0}' field in CSV file.".FormatString(fieldConfig.FieldName));
//if (Configuration.ColumnOrderStrict)
// throw new ChoParserException("No matching '{0}' field header found.".FormatString(fieldConfig.FieldName));
}
}
else
{
if (fieldConfig.FieldPosition - 1 < fieldValues.Length)
{
fieldValue = fieldValues[fieldConfig.FieldPosition - 1];
}
else if (Configuration.ThrowAndStopOnMissingField)
{
throw new ChoMissingRecordFieldException("Missing field value at [Position: {1}] in CSV file.".FormatString(fieldConfig.FieldName, fieldConfig.FieldPosition));
}
}
//if (Configuration.FileHeaderConfiguration.HasHeaderRecord && Configuration.ColumnOrderStrict)
//{
// if (fieldNameValues.ContainsKey(fieldConfig.FieldName))
// fieldValue = fieldNameValues[fieldConfig.FieldName];
// else if (Configuration.ColumnCountStrict)
// throw new ChoParserException("No matching '{0}' field header found.".FormatString(fieldConfig.FieldName));
//}
//else
//{
// if (fieldConfig.FieldPosition - 1 < fieldValues.Length)
// fieldValue = fieldValues[fieldConfig.FieldPosition - 1];
// else if (Configuration.ColumnCountStrict)
// throw new ChoParserException("Missing field value for '{0}' [Position: {1}] field.".FormatString(fieldConfig.FieldName, fieldConfig.FieldPosition));
//}
if (Configuration.IsDynamicObject)
{
if (kvp.Value.FieldType == null)
{
kvp.Value.FieldType = DiscoverFieldType(fieldValue as string);
}
}
else
{
if (pi != null)
{
kvp.Value.FieldType = pi.PropertyType;
}
else
{
kvp.Value.FieldType = typeof(string);
}
}
fieldValue = CleanFieldValue(fieldConfig, kvp.Value.FieldType, fieldValue as string);
if (!RaiseBeforeRecordFieldLoad(rec, pair.Item1, kvp.Key, ref fieldValue))
{
continue;
}
bool ignoreFieldValue = fieldConfig.IgnoreFieldValue(fieldValue);
if (ignoreFieldValue)
{
fieldValue = fieldConfig.IsDefaultValueSpecified ? fieldConfig.DefaultValue : null;
}
if (Configuration.IsDynamicObject)
{
var dict = rec as IDictionary <string, Object>;
dict.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel)
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
}
else
{
if (pi != null)
{
rec.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
}
else
{
throw new ChoMissingRecordFieldException("Missing '{0}' property in {1} type.".FormatString(kvp.Key, ChoType.GetTypeName(rec)));
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel)
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
}
if (!RaiseAfterRecordFieldLoad(rec, pair.Item1, kvp.Key, fieldValue))
{
return(false);
}
}
catch (ChoParserException)
{
throw;
}
catch (ChoMissingRecordFieldException)
{
if (Configuration.ThrowAndStopOnMissingField)
{
throw;
}
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ex);
if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
{
throw;
}
try
{
if (Configuration.IsDynamicObject)
{
var dict = rec as IDictionary <string, Object>;
if (dict.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (dict.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
else if (pi != null)
{
if (rec.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (rec.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
catch (Exception innerEx)
{
if (ex == innerEx.InnerException)
{
if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
continue;
}
else
{
if (!RaiseRecordFieldLoadError(rec, pair.Item1, kvp.Key, fieldValue, ex))
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
}
else
{
throw new ChoReaderException("Failed to assign '{0}' fallback value to '{1}' field.".FormatString(fieldValue, fieldConfig.FieldName), innerEx);
}
}
}
}
return(true);
}
private bool FillRecord(object rec, Tuple <int, string> pair)
{
int lineNo;
string line;
lineNo = pair.Item1;
line = pair.Item2;
object fieldValue = null;
string[] fieldValues = (from x in line.Split(Configuration.Delimiter, Configuration.StringSplitOptions, Configuration.QuoteChar)
select x).ToArray();
if (Configuration.ColumnCountStrict)
{
if (fieldValues.Length != Configuration.CSVRecordFieldConfigurations.Count)
{
throw new ChoParserException("Incorrect number of field values found at line [{2}]. Expected [{0}] field values. Found [{1}] field values.".FormatString(Configuration.CSVRecordFieldConfigurations.Count, fieldValues.Length, pair.Item1));
}
}
Dictionary <string, string> fieldNameValues = ToFieldNameValues(fieldValues);
ValidateLine(pair.Item1, fieldValues);
ChoCSVRecordFieldConfiguration fieldConfig = null;
foreach (KeyValuePair <string, ChoCSVRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict)
{
fieldValue = null;
fieldConfig = kvp.Value;
if (Configuration.FileHeaderConfiguration.HasHeaderRecord)
{
if (fieldNameValues.ContainsKey(fieldConfig.FieldName))
{
fieldValue = fieldNameValues[fieldConfig.FieldName];
}
else if (Configuration.ColumnCountStrict)
{
throw new ChoParserException("No matching '{0}' field header found.".FormatString(fieldConfig.FieldName));
}
}
else
{
if (fieldConfig.FieldPosition - 1 < fieldValues.Length)
{
fieldValue = fieldValues[fieldConfig.FieldPosition - 1];
}
else if (Configuration.ColumnCountStrict)
{
throw new ChoParserException("Missing field value for {0} [Position: {1}] field.".FormatString(fieldConfig.FieldName, fieldConfig.FieldPosition));
}
}
fieldValue = CleanFieldValue(fieldConfig, fieldValue as string);
if (!RaiseBeforeRecordFieldLoad(rec, pair.Item1, kvp.Key, ref fieldValue))
{
return(false);
}
try
{
bool ignoreFieldValue = fieldConfig.IgnoreFieldValue(fieldValue);
if (ignoreFieldValue)
{
fieldValue = null;
}
if (rec is ExpandoObject)
{
var dict = rec as IDictionary <string, Object>;
dict.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture);
if (ignoreFieldValue)
{
dict.AddOrUpdate(kvp.Key, fieldValue);
}
else
{
dict.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
}
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else
{
if (ChoType.HasProperty(rec.GetType(), kvp.Key))
{
rec.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture);
if (!ignoreFieldValue)
{
rec.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
}
}
else
{
throw new ChoMissingRecordFieldException("Missing '{0}' property in {1} type.".FormatString(kvp.Key, ChoType.GetTypeName(rec)));
}
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
if (!RaiseAfterRecordFieldLoad(rec, pair.Item1, kvp.Key, fieldValue))
{
return(false);
}
}
catch (ChoParserException)
{
throw;
}
catch (ChoMissingRecordFieldException)
{
if (Configuration.ThrowAndStopOnMissingField)
{
throw;
}
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ex);
if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
{
throw;
}
try
{
if (rec is ExpandoObject)
{
var dict = rec as IDictionary <string, Object>;
if (dict.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else
{
throw new ChoParserException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
else if (ChoType.HasProperty(rec.GetType(), kvp.Key) && rec.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else
{
throw new ChoParserException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
catch (Exception innerEx)
{
if (ex == innerEx.InnerException)
{
if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
continue;
}
else
{
if (!RaiseRecordFieldLoadError(rec, pair.Item1, kvp.Key, fieldValue, ex))
{
throw new ChoParserException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
}
else
{
throw new ChoParserException("Failed to assign '{0}' fallback value to '{1}' field.".FormatString(fieldValue, fieldConfig.FieldName), innerEx);
}
}
}
}
return(true);
}
private bool FillRecord(object rec, Tuple <long, string> pair)
{
long lineNo;
string line;
lineNo = pair.Item1;
line = pair.Item2;
var tokens = ToKVP(pair.Item1, pair.Item2);
//ValidateLine(pair.Item1, fieldValues);
object fieldValue = tokens.Value;
string key = tokens.Key;
if (!Configuration.RecordFieldConfigurationsDict2.ContainsKey(key))
{
return(true);
}
key = Configuration.RecordFieldConfigurationsDict2[key].Name;
ChoKVPRecordFieldConfiguration fieldConfig = Configuration.RecordFieldConfigurationsDict[key];
PropertyInfo pi = null;
try
{
if (_propInit.ContainsKey(key))
{
return(true);
}
_propInit.Add(key, true);
fieldValue = CleanFieldValue(fieldConfig, fieldConfig.FieldType, fieldValue as string);
if (Configuration.IsDynamicObject)
{
if (fieldConfig.FieldType == null)
{
fieldConfig.FieldType = typeof(string);
}
}
else
{
if (Configuration.PIDict != null)
{
Configuration.PIDict.TryGetValue(key, out pi);
}
if (pi != null)
{
fieldConfig.FieldType = pi.PropertyType;
}
else
{
fieldConfig.FieldType = typeof(string);
}
}
fieldValue = CleanFieldValue(fieldConfig, fieldConfig.FieldType, fieldValue as string);
if (!RaiseBeforeRecordFieldLoad(rec, pair.Item1, key, ref fieldValue))
{
return(true);
}
bool ignoreFieldValue = fieldConfig.IgnoreFieldValue(fieldValue);
if (ignoreFieldValue)
{
fieldValue = fieldConfig.IsDefaultValueSpecified ? fieldConfig.DefaultValue : null;
}
if (Configuration.IsDynamicObject)
{
var dict = rec as IDictionary <string, Object>;
dict.ConvertNSetMemberValue(key, fieldConfig, ref fieldValue, Configuration.Culture);
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel)
{
dict.DoMemberLevelValidation(key, fieldConfig, Configuration.ObjectValidationMode);
}
}
else
{
if (pi != null)
{
rec.ConvertNSetMemberValue(key, fieldConfig, ref fieldValue, Configuration.Culture);
}
else
{
throw new ChoMissingRecordFieldException("Missing '{0}' property in {1} type.".FormatString(key, ChoType.GetTypeName(rec)));
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel)
{
rec.DoMemberLevelValidation(key, fieldConfig, Configuration.ObjectValidationMode);
}
}
if (!RaiseAfterRecordFieldLoad(rec, pair.Item1, key, fieldValue))
{
return(false);
}
}
catch (ChoParserException)
{
Reader.IsValid = false;
throw;
}
catch (ChoMissingRecordFieldException)
{
Reader.IsValid = false;
if (Configuration.ThrowAndStopOnMissingField)
{
throw;
}
}
catch (Exception ex)
{
Reader.IsValid = false;
ChoETLFramework.HandleException(ref ex);
if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
{
throw;
}
try
{
if (Configuration.IsDynamicObject)
{
var dict = rec as IDictionary <string, Object>;
if (dict.SetFallbackValue(key, fieldConfig, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(key, fieldConfig, Configuration.ObjectValidationMode);
}
else if (dict.SetDefaultValue(key, fieldConfig, Configuration.Culture))
{
dict.DoMemberLevelValidation(key, fieldConfig, Configuration.ObjectValidationMode);
}
else if (ex is ValidationException)
{
throw;
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
else if (pi != null)
{
if (rec.SetFallbackValue(key, fieldConfig, Configuration.Culture))
{
rec.DoMemberLevelValidation(key, fieldConfig, Configuration.ObjectValidationMode);
}
else if (rec.SetDefaultValue(key, fieldConfig, Configuration.Culture))
{
rec.DoMemberLevelValidation(key, fieldConfig, Configuration.ObjectValidationMode);
}
else if (ex is ValidationException)
{
throw;
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
catch (Exception innerEx)
{
if (ex == innerEx.InnerException || ex is ValidationException)
{
if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
}
else
{
if (!RaiseRecordFieldLoadError(rec, pair.Item1, key, fieldValue, ex))
{
if (ex is ValidationException)
{
throw;
}
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
}
else
{
throw new ChoReaderException("Failed to assign '{0}' fallback value to '{1}' field.".FormatString(fieldValue, fieldConfig.FieldName), innerEx);
}
}
}
return(true);
}
private bool LoadLine(Tuple <int, string> pair, ref object rec)
{
try
{
if (!RaiseBeforeRecordLoad(rec, ref pair))
{
return(false);
}
if (pair.Item2 == null)
{
rec = null;
return(true);
}
else if (pair.Item2 == String.Empty)
{
return(true);
}
if (!pair.Item2.IsNullOrWhiteSpace())
{
if (!FillRecord(rec, pair))
{
return(false);
}
rec.DoObjectLevelValidation(Configuration, Configuration.FixedLengthRecordFieldConfigurations.ToArray());
}
if (!RaiseAfterRecordLoad(rec, pair))
{
return(false);
}
}
catch (ChoParserException)
{
throw;
}
catch (ChoMissingRecordFieldException)
{
throw;
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
rec = null;
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordLoadError(rec, pair, ex))
{
throw;
}
}
else
{
throw;
}
return(true);
}
return(true);
}
public override IEnumerable <object> WriteTo(object writer, IEnumerable <object> records, Func <object, bool> predicate = null)
{
TextWriter sw = writer as TextWriter;
ChoGuard.ArgumentNotNull(sw, "TextWriter");
if (records == null)
{
yield break;
}
if (!RaiseBeginWrite(sw))
{
yield break;
}
CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture;
System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture;
string recText = String.Empty;
try
{
foreach (object record in records)
{
_index++;
if (TraceSwitch.TraceVerbose)
{
if (record is IChoETLNameableObject)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index));
}
}
recText = String.Empty;
if (record != null)
{
if (predicate == null || predicate(record))
{
//Discover and load FixedLength columns from first record
if (!_configCheckDone)
{
string[] fieldNames = null;
Type recordType = ElementType == null?record.GetType() : ElementType;
if (typeof(ICollection).IsAssignableFrom(recordType))
{
recordType = recordType.GetEnumerableItemType().GetUnderlyingType();
}
else
{
recordType = recordType.GetUnderlyingType();
}
Configuration.IsDynamicObject = recordType.IsDynamicType();
if (!Configuration.IsDynamicObject)
{
if (recordType.IsSimple())
{
Configuration.RecordType = typeof(ChoScalarObject);
}
else
{
Configuration.RecordType = recordType;
}
}
if (Configuration.IsDynamicObject)
{
var dict = record.ToDynamicObject() as IDictionary <string, Object>;
fieldNames = dict.Keys.ToArray();
}
else
{
fieldNames = ChoTypeDescriptor.GetProperties <ChoFixedLengthRecordFieldAttribute>(Configuration.RecordType).Select(pd => pd.Name).ToArray();
if (fieldNames.Length == 0)
{
fieldNames = ChoType.GetProperties(Configuration.RecordType).Select(p => p.Name).ToArray();
}
}
Configuration.Validate(fieldNames);
WriteHeaderLine(sw);
_configCheckDone = true;
}
if (!RaiseBeforeRecordWrite(record, _index, ref recText))
{
yield break;
}
if (recText == null)
{
continue;
}
else if (recText.Length > 0)
{
sw.Write("{1}{0}", recText, _hadHeaderWritten ? Configuration.EOLDelimiter : "");
continue;
}
try
{
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
record.DoObjectLevelValidation(Configuration, Configuration.FixedLengthRecordFieldConfigurations);
}
if (ToText(_index, record, out recText))
{
if (_index == 1)
{
sw.Write("{1}{0}", recText, _hadHeaderWritten ? Configuration.EOLDelimiter : "");
}
else
{
sw.Write("{1}{0}", recText, Configuration.EOLDelimiter);
}
if (!RaiseAfterRecordWrite(record, _index, recText))
{
yield break;
}
}
}
//catch (ChoParserException)
//{
// throw;
//}
catch (Exception ex)
{
ChoETLFramework.HandleException(ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordWriteError(record, _index, recText, ex))
{
throw;
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
}
}
else
{
throw;
}
}
}
}
yield return(record);
if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0)
{
if (RaisedRowsWritten(_index))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
yield break;
}
}
}
}
finally
{
System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo;
}
RaiseEndWrite(sw);
}
private bool FillRecord(object rec, Tuple <int, string> pair)
{
int lineNo;
string line;
lineNo = pair.Item1;
line = pair.Item2;
if (line.Length != Configuration.RecordLength)
{
throw new ChoParserException("Incorrect record length [Length: {0}] found. Expected record length: {1}".FormatString(line.Length, Configuration.RecordLength));
}
object fieldValue = null;
ChoFixedLengthRecordFieldConfiguration fieldConfig = null;
foreach (KeyValuePair <string, ChoFixedLengthRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict)
{
fieldValue = null;
fieldConfig = kvp.Value;
if (fieldConfig.StartIndex + fieldConfig.Size > line.Length)
{
if (Configuration.ColumnCountStrict)
{
throw new ChoParserException("Missing '{0}' field value.".FormatString(kvp.Key));
}
}
else
{
fieldValue = line.Substring(fieldConfig.StartIndex, fieldConfig.Size.Value);
}
fieldValue = CleanFieldValue(fieldConfig, fieldValue as string);
if (!RaiseBeforeRecordFieldLoad(rec, pair.Item1, kvp.Key, ref fieldValue))
{
return(false);
}
try
{
bool ignoreFieldValue = fieldConfig.IgnoreFieldValue(fieldValue);
if (ignoreFieldValue)
{
fieldValue = null;
}
if (rec is ExpandoObject)
{
var dict = rec as IDictionary <string, Object>;
dict.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture);
if (ignoreFieldValue)
{
dict.AddOrUpdate(kvp.Key, fieldValue);
}
else
{
dict.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
}
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else
{
if (ChoType.HasProperty(rec.GetType(), kvp.Key))
{
rec.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture);
if (!ignoreFieldValue)
{
rec.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
}
}
else
{
throw new ChoMissingRecordFieldException("Missing '{0}' property in {1} type.".FormatString(kvp.Key, ChoType.GetTypeName(rec)));
}
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
if (!RaiseAfterRecordFieldLoad(rec, pair.Item1, kvp.Key, fieldValue))
{
return(false);
}
}
catch (ChoParserException)
{
throw;
}
catch (ChoMissingRecordFieldException)
{
if (Configuration.ThrowAndStopOnMissingField)
{
throw;
}
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ex);
if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
{
throw;
}
try
{
if (rec is ExpandoObject)
{
var dict = rec as IDictionary <string, Object>;
if (dict.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else
{
throw new ChoParserException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
else if (ChoType.HasProperty(rec.GetType(), kvp.Key) && rec.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else
{
throw new ChoParserException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
catch (Exception innerEx)
{
if (ex == innerEx.InnerException)
{
if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
continue;
}
else
{
if (!RaiseRecordFieldLoadError(rec, pair.Item1, kvp.Key, fieldValue, ex))
{
throw new ChoParserException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
}
else
{
throw new ChoParserException("Failed to assign '{0}' fallback value to '{1}' field.".FormatString(fieldValue, fieldConfig.FieldName), innerEx);
}
}
}
}
return(true);
}
private bool LoadLine(Tuple <long, string> pair, ref object rec)
{
try
{
if (!RaiseBeforeRecordLoad(rec, ref pair))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping...");
rec = null;
return(true);
}
if (pair.Item2 == null)
{
rec = null;
return(true);
}
else if (pair.Item2 == String.Empty)
{
return(true);
}
if (!pair.Item2.IsNullOrWhiteSpace())
{
if (!FillRecord(rec, pair))
{
return(false);
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
rec.DoObjectLevelValidation(Configuration, Configuration.FixedLengthRecordFieldConfigurations);
}
}
bool skip = false;
if (!RaiseAfterRecordLoad(rec, pair, ref skip))
{
return(false);
}
else if (skip)
{
rec = null;
return(true);
}
}
//catch (ChoParserException)
//{
// throw;
//}
//catch (ChoMissingRecordFieldException)
//{
// throw;
//}
catch (Exception ex)
{
Reader.IsValid = false;
if (ex is ChoMissingRecordFieldException && Configuration.ThrowAndStopOnMissingField)
{
if (!RaiseRecordLoadError(rec, pair, ex))
{
throw;
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
rec = null;
}
}
else
{
ChoETLFramework.HandleException(ref ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
rec = null;
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordLoadError(rec, pair, ex))
{
throw;
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
rec = null;
}
}
else
{
throw;
}
}
return(true);
}
return(true);
}
public override IEnumerable <object> WriteTo(object writer, IEnumerable <object> records, Func <object, bool> predicate = null)
{
StreamWriter sw = writer as StreamWriter;
ChoGuard.ArgumentNotNull(sw, "StreamWriter");
if (records == null)
{
yield break;
}
if (!RaiseBeginWrite(sw))
{
yield break;
}
WriteHeaderLine(sw);
foreach (object record in records)
{
if (record != null)
{
if (predicate == null || predicate(record))
{
//Discover and load CSV columns from first record
if (!_configCheckDone)
{
string[] fieldNames = null;
if (record is ExpandoObject)
{
var x = record as IDictionary <string, Object>;
fieldNames = x.Keys.ToArray();
}
else
{
fieldNames = ChoTypeDescriptor.GetProperties <ChoCSVRecordFieldAttribute>(record.GetType()).Select(pd => pd.Name).ToArray();
}
Configuration.Validate(fieldNames);
_configCheckDone = true;
}
if (!RaiseBeforeRecordWrite(record))
{
yield break;
}
try
{
sw.Write("{1}{0}", ToText(record), Configuration.EOLDelimiter);
if (!RaiseAfterRecordWrite(record))
{
yield break;
}
}
catch (ChoParserException)
{
throw;
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordWriteError(record, ex))
{
throw;
}
}
else
{
throw;
}
}
}
}
yield return(record);
}
RaiseEndWrite(sw);
}
