private IEnumerable <object> AsEnumerable(IEnumerable <JObject> jObjects, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null)
{
TraceSwitch = traceSwitch;
long counter = 0;
Tuple <long, JObject> pair = null;
bool?skip = false;
bool?skipUntil = true;
bool?doWhile = true;
bool abortRequested = false;
_se = new Lazy <JsonSerializer>(() => Configuration.JsonSerializerSettings != null ? JsonSerializer.Create(Configuration.JsonSerializerSettings) : null);
foreach (var obj in jObjects)
{
pair = new Tuple <long, JObject>(++counter, obj);
skip = false;
if (skipUntil != null)
{
if (skipUntil.Value)
{
skipUntil = RaiseSkipUntil(pair);
if (skipUntil == null)
{
}
else
{
if (skipUntil.Value)
{
skip = skipUntil;
}
else
{
skip = true;
}
}
}
}
if (skip == null)
{
break;
}
if (skip.Value)
{
continue;
}
if (!_configCheckDone)
{
Configuration.Validate(pair);
var dict = Configuration.JSONRecordFieldConfigurations.ToDictionary(i => i.Name, i => i.FieldType == null ? null : i.FieldType);
RaiseMembersDiscovered(dict);
_configCheckDone = true;
}
object rec = null;
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading node [{0}]...".FormatString(pair.Item1));
}
if (!LoadNode(pair, ref rec))
{
yield break;
}
if (rec == null)
{
continue;
}
yield return(rec);
if (Configuration.NotifyAfter > 0 && pair.Item1 % Configuration.NotifyAfter == 0)
{
if (RaisedRowsLoaded(pair.Item1))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
abortRequested = true;
yield break;
}
}
if (doWhile != null)
{
doWhile = RaiseDoWhile(pair);
if (doWhile != null && doWhile.Value)
{
break;
}
}
}
if (!abortRequested && pair != null)
{
RaisedRowsLoaded(pair.Item1);
}
}
private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null)
{
TraceSwitch = traceSwitch;
TextReader sr = source as TextReader;
ChoGuard.ArgumentNotNull(sr, "TextReader");
if (sr is StreamReader)
{
((StreamReader)sr).Seek(0, SeekOrigin.Begin);
}
if (!RaiseBeginLoad(sr))
{
yield break;
}
string[] commentTokens = Configuration.Comments;
bool? skip = false;
bool isRecordStartFound = false;
bool isRecordEndFound = false;
long seekOriginPos = sr is StreamReader ? ((StreamReader)sr).BaseStream.Position : 0;
List <string> headers = new List <string>();
Tuple <long, string> lastLine = null;
List <Tuple <long, string> > recLines = new List <Tuple <long, string> >();
long recNo = 0;
int loopCount = Configuration.AutoDiscoverColumns && Configuration.KVPRecordFieldConfigurations.Count == 0 ? 2 : 1;
bool isHeaderFound = loopCount == 1;
bool IsHeaderLoaded = false;
Tuple <long, string> pairIn;
bool abortRequested = false;
bool?skipUntil = true;
bool?doWhile = true;
for (int i = 0; i < loopCount; i++)
{
if (i == 1)
{
if (sr is StreamReader)
{
((StreamReader)sr).Seek(seekOriginPos, SeekOrigin.Begin);
}
TraceSwitch = traceSwitch;
}
else
{
TraceSwitch = ChoETLFramework.TraceSwitchOff;
}
lastLine = null;
recLines.Clear();
isRecordEndFound = false;
isRecordStartFound = false;
using (ChoPeekEnumerator <Tuple <long, string> > e = new ChoPeekEnumerator <Tuple <long, string> >(
new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar, Configuration.MayContainEOLInData)).ToEnumerable(),
(pair) =>
{
//bool isStateAvail = IsStateAvail();
skip = false;
if (skipUntil != null)
{
if (skipUntil.Value)
{
skipUntil = RaiseSkipUntil(pair);
if (skipUntil == null)
{
}
else
{
skip = skipUntil.Value;
}
}
}
//if (isStateAvail)
//{
// if (!IsStateMatches(item))
// {
// skip = filterFunc != null ? filterFunc(item) : false;
// }
// else
// skip = true;
//}
//else
// skip = filterFunc != null ? filterFunc(item) : false;
if (skip == null)
{
return(null);
}
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine);
if (!skip.Value)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1));
}
}
if (skip.Value)
{
return(skip);
}
//if (!(sr.BaseStream is MemoryStream))
// ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1));
//if (Task != null)
// return !IsStateNOTExistsOrNOTMatch(item);
if (pair.Item2.IsNullOrWhiteSpace())
{
if (!Configuration.IgnoreEmptyLine)
{
throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pair.Item1));
}
else
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1));
}
return(true);
}
}
if (commentTokens != null && commentTokens.Length > 0)
{
foreach (string comment in commentTokens)
{
if (!pair.Item2.IsNull() && pair.Item2.StartsWith(comment, StringComparison.Ordinal)) //, true, Configuration.Culture))
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1));
}
return(true);
}
}
}
if (!_configCheckDone)
{
Configuration.Validate(null);
var dict = Configuration.KVPRecordFieldConfigurations.ToDictionary(i1 => i1.Name, i1 => i1.FieldType == null ? null : i1.FieldType);
RaiseMembersDiscovered(dict);
Configuration.UpdateFieldTypesIfAny(dict);
_configCheckDone = true;
}
return(false);
}))
{
while (true)
{
pairIn = e.Peek;
if (!isRecordStartFound)
{
if (pairIn == null)
{
break;
}
lastLine = null;
recLines.Clear();
isRecordEndFound = false;
isRecordStartFound = true;
if (!Configuration.RecordStart.IsNullOrWhiteSpace())
{
//Move to record start
while (!(Configuration.IsRecordStartMatch(e.Peek.Item2)))
{
e.MoveNext();
if (e.Peek == null)
{
break;
}
}
}
if (e.Peek != null)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Record start found at [{0}] line...".FormatString(e.Peek.Item1));
}
e.MoveNext();
continue;
}
else
{
string recordEnd = !Configuration.RecordEnd.IsNullOrWhiteSpace() ? Configuration.RecordEnd : Configuration.RecordStart;
if (!recordEnd.IsNullOrWhiteSpace())
{
if (e.Peek == null)
{
if (Configuration.RecordEnd.IsNullOrWhiteSpace())
{
isRecordEndFound = true;
}
else
{
break;
}
}
else
{
//Move to record start
if (Configuration.IsRecordEndMatch(e.Peek.Item2))
{
isRecordEndFound = true;
isRecordStartFound = false;
if (e.Peek != null)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Record end found at [{0}] line...".FormatString(e.Peek.Item1));
}
}
}
}
else if (e.Peek == null)
{
isRecordEndFound = true;
}
if (!isHeaderFound)
{
//if (isRecordEndFound && headers.Count == 0)
//{
// //throw new ChoParserException("Unexpected EOF found.");
//}
if (!isRecordEndFound)
{
e.MoveNext();
if (e.Peek != null)
{
//If line empty or line continuation, skip
if (pairIn.Item2.IsNullOrWhiteSpace() || IsLineContinuationCharFound(pairIn.Item2)) //.Item2[0] == ' ' || pairIn.Item2[0] == '\t')
{
}
else
{
string header = ToKVP(pairIn.Item1, pairIn.Item2).Key;
if (!header.IsNullOrWhiteSpace())
{
headers.Add(header);
}
}
}
}
else
{
Configuration.Validate(headers.ToArray());
isHeaderFound = true;
isRecordEndFound = false;
IsHeaderLoaded = true;
break;
}
}
else
{
if (!IsHeaderLoaded)
{
Configuration.Validate(new string[] { });
IsHeaderLoaded = true;
}
if (isRecordEndFound && recLines.Count == 0)
{
//throw new ChoParserException("Unexpected EOF found.");
}
else if (!isRecordEndFound)
{
e.MoveNext();
if (e.Peek != null)
{
//If line empty or line continuation, skip
if (pairIn.Item2.IsNullOrWhiteSpace())
{
if (!Configuration.IgnoreEmptyLine)
{
throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pairIn.Item1));
}
else
{
Tuple <long, string> t = new Tuple <long, string>(lastLine.Item1, lastLine.Item2 + Configuration.EOLDelimiter);
recLines.RemoveAt(recLines.Count - 1);
recLines.Add(t);
}
}
else if (IsLineContinuationCharFound(pairIn.Item2)) //pairIn.Item2[0] == ' ' || pairIn.Item2[0] == '\t')
{
if (lastLine == null)
{
throw new ChoParserException("Unexpected line continuation found at {0} location.".FormatString(pairIn.Item1));
}
else
{
Tuple <long, string> t = new Tuple <long, string>(lastLine.Item1, lastLine.Item2 + Configuration.EOLDelimiter + pairIn.Item2);
recLines.RemoveAt(recLines.Count - 1);
recLines.Add(t);
}
}
else
{
lastLine = pairIn;
recLines.Add(pairIn);
}
}
}
else
{
object rec = Configuration.IsDynamicObject ? new ChoDynamicObject(new Dictionary <string, object>(Configuration.FileHeaderConfiguration.StringComparer))
{
ThrowExceptionIfPropNotExists = true,
AlternativeKeys = Configuration.AlternativeKeys
} : ChoActivator.CreateInstance(RecordType);
if (!LoadLines(new Tuple <long, List <Tuple <long, string> > >(++recNo, recLines), ref rec))
{
yield break;
}
isRecordStartFound = false;
//StoreState(e.Current, rec != null);
if (!Configuration.RecordEnd.IsNullOrWhiteSpace())
{
e.MoveNext();
}
if (rec == null)
{
continue;
}
yield return(rec);
if (Configuration.NotifyAfter > 0 && recNo % Configuration.NotifyAfter == 0)
{
if (RaisedRowsLoaded(recNo))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
abortRequested = true;
yield break;
}
}
if (e.Peek == null)
{
break;
}
}
}
}
if (doWhile != null)
{
doWhile = RaiseDoWhile(pairIn);
if (doWhile != null && doWhile.Value)
{
break;
}
}
}
}
}
if (!abortRequested)
{
RaisedRowsLoaded(recNo, true);
}
RaiseEndLoad(sr);
}
private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null)
{
TraceSwitch = traceSwitch;
TextReader sr = source as TextReader;
ChoGuard.ArgumentNotNull(sr, "TextReader");
if (sr is StreamReader)
{
((StreamReader)sr).Seek(0, SeekOrigin.Begin);
}
if (!RaiseBeginLoad(sr))
{
yield break;
}
string[] commentTokens = Configuration.Comments;
bool? skip = false;
bool abortRequested = false;
long runningCount = 0;
using (ChoPeekEnumerator <Tuple <long, string> > e = new ChoPeekEnumerator <Tuple <long, string> >(
new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar, Configuration.MayContainEOLInData)).ToEnumerable(),
(pair) =>
{
//bool isStateAvail = IsStateAvail();
skip = false;
//if (isStateAvail)
//{
// if (!IsStateMatches(item))
// {
// skip = filterFunc != null ? filterFunc(item) : false;
// }
// else
// skip = true;
//}
//else
// skip = filterFunc != null ? filterFunc(item) : false;
if (skip == null)
{
return(null);
}
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine);
if (!skip.Value)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1));
}
}
if (skip.Value)
{
return(skip);
}
//if (!(sr.BaseStream is MemoryStream))
// ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1));
//if (Task != null)
// return !IsStateNOTExistsOrNOTMatch(item);
if (pair.Item2.IsNullOrWhiteSpace())
{
if (!Configuration.IgnoreEmptyLine)
{
throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pair.Item1));
}
else
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1));
}
return(true);
}
}
if (commentTokens != null && commentTokens.Length > 0)
{
foreach (string comment in commentTokens)
{
if (!pair.Item2.IsNull() && pair.Item2.StartsWith(comment, StringComparison.Ordinal)) //, true, Configuration.Culture))
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1));
}
return(true);
}
}
}
if (!_configCheckDone)
{
Configuration.Validate(pair); // GetHeaders(pair.Item2));
RaiseMembersDiscovered(Configuration.FixedLengthRecordFieldConfigurations.Select(i => new KeyValuePair <string, Type>(i.Name, i.FieldType == null ? typeof(string) : i.FieldType)).ToArray());
_configCheckDone = true;
}
//LoadHeader if any
if (Configuration.FileHeaderConfiguration.HasHeaderRecord &&
!_headerFound)
{
if (Configuration.FileHeaderConfiguration.IgnoreHeader)
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring header line at [{0}]...".FormatString(pair.Item1));
}
}
else
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading header line at [{0}]...".FormatString(pair.Item1));
}
LoadHeaderLine(pair);
}
_headerFound = true;
return(true);
}
return(false);
}))
{
while (true)
{
Tuple <long, string> pair = e.Peek;
if (pair == null)
{
if (!abortRequested)
{
RaisedRowsLoaded(runningCount);
}
RaiseEndLoad(sr);
yield break;
}
runningCount = pair.Item1;
object rec = Activator.CreateInstance(RecordType);
if (!LoadLine(pair, ref rec))
{
yield break;
}
//StoreState(e.Current, rec != null);
e.MoveNext();
if (rec == null)
{
continue;
}
yield return(rec);
if (Configuration.NotifyAfter > 0 && pair.Item1 % Configuration.NotifyAfter == 0)
{
if (RaisedRowsLoaded(pair.Item1))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
yield break;
}
}
}
}
}
private void LoadValueConverters(ChoIniFile iniFile1)
{
ChoIniFile iniFile = iniFile1.GetSection("CONVERTER");
var dict = new Dictionary <string, List <Tuple <IValueConverter, string> > >();
IValueConverter value = null;
Dictionary <string, string> iniKeyValuesDict = iniFile.KeyValuesDict;
foreach (string fieldName in iniKeyValuesDict.Keys) //GetFieldNames(iniFile1))
{
if (fieldName.IsNullOrWhiteSpace())
{
continue;
}
if (!iniKeyValuesDict.ContainsKey(fieldName))
{
continue;
}
if (dict.ContainsKey(fieldName))
{
continue;
}
dict.Add(fieldName, new List <Tuple <IValueConverter, string> >());
if (!iniKeyValuesDict[fieldName].IsNullOrWhiteSpace())
{
foreach (string val in iniKeyValuesDict[fieldName].SplitNTrim(";"))
{
if (val.IsNullOrWhiteSpace())
{
continue;
}
string vcType = null;
string vcParam = null;
foreach (KeyValuePair <string, string> convParams in val.ToKeyValuePairs('&'))
{
if (convParams.Key == "Type")
{
vcType = convParams.Value;
}
else if (convParams.Key == "Parameter")
{
vcParam = convParams.Value;
}
}
if (!vcType.IsNullOrWhiteSpace())
{
try
{
value = Activator.CreateInstance(ChoType.GetType(vcType)) as IValueConverter;
if (value != null)
{
dict[fieldName].Add(new Tuple <IValueConverter, string>(value, vcParam));
}
}
catch (Exception ex)
{
ChoETLFramework.WriteLog(ChoETLFramework.Switch.TraceError, "Error while loading format value '{0}' for '{1}' member. {2}".FormatString(value, fieldName, ex.Message));
}
}
}
}
}
_valueConverterCache.AddOrUpdate(iniFile1.Key, dict);
}
public override IEnumerable <object> WriteTo(object writer, IEnumerable <object> records, Func <object, bool> predicate = null)
{
TextWriter sw = writer as TextWriter;
ChoGuard.ArgumentNotNull(sw, "TextWriter");
if (records == null)
{
yield break;
}
if (!RaiseBeginWrite(sw))
{
yield break;
}
CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture;
System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture;
string recText = String.Empty;
long recCount = 0;
var recEnum = records.GetEnumerator();
try
{
object notNullRecord = GetFirstNotNullRecord(recEnum);
if (notNullRecord == null)
{
yield break;
}
if (Configuration.IsDynamicObject)
{
if (Configuration.MaxScanRows > 0)
{
List <string> fns = new List <string>();
foreach (object record1 in GetRecords(recEnum))
{
recCount++;
if (record1 != null)
{
if (recCount <= Configuration.MaxScanRows)
{
if (!record1.GetType().IsDynamicType())
{
throw new ChoParserException("Invalid record found.");
}
_recBuffer.Value.Add(record1);
fns = fns.Union(GetFields(record1)).ToList();
if (recCount == Configuration.MaxScanRows)
{
Configuration.Validate(fns.ToArray());
WriteHeaderLine(sw);
_configCheckDone = true;
break;
}
}
}
}
}
}
foreach (object record in GetRecords(recEnum))
{
_index++;
if (TraceSwitch.TraceVerbose)
{
if (record is IChoETLNameableObject)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index));
}
}
recText = String.Empty;
if (record != null)
{
if (predicate == null || predicate(record))
{
//Discover and load CSV columns from first record
if (!_configCheckDone)
{
if (notNullRecord != null)
{
string[] fieldNames = GetFields(notNullRecord);
Configuration.Validate(fieldNames);
WriteHeaderLine(sw);
_configCheckDone = true;
}
}
//Check record
if (record != null)
{
Type rt = record.GetType().ResolveType();
if (Configuration.IsDynamicObject)
{
if (ElementType != null)
{
}
else if (!rt.IsDynamicType())
{
throw new ChoWriterException("Invalid record found.");
}
}
else
{
if (rt != Configuration.RecordType)
{
throw new ChoWriterException("Invalid record found.");
}
}
}
if (!RaiseBeforeRecordWrite(record, _index, ref recText))
{
yield break;
}
if (recText == null)
{
continue;
}
else if (recText.Length > 0)
{
sw.Write("{1}{0}", recText, _hadHeaderWritten || HasExcelSeparator ? Configuration.EOLDelimiter : "");
continue;
}
try
{
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
record.DoObjectLevelValidation(Configuration, Configuration.CSVRecordFieldConfigurations);
}
if (ToText(_index, record, out recText))
{
if (_index == 1)
{
sw.Write("{1}{0}", recText, _hadHeaderWritten || HasExcelSeparator ? Configuration.EOLDelimiter : "");
}
else
{
sw.Write("{1}{0}", recText, Configuration.EOLDelimiter);
}
if (!RaiseAfterRecordWrite(record, _index, recText))
{
yield break;
}
}
}
//catch (ChoParserException)
//{
// throw;
//}
catch (Exception ex)
{
ChoETLFramework.HandleException(ref ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordWriteError(record, _index, recText, ex))
{
throw;
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
}
}
else
{
throw;
}
}
}
}
yield return(record);
if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0)
{
if (RaisedRowsWritten(_index))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
yield break;
}
}
}
}
finally
{
System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo;
}
RaiseEndWrite(sw);
}
public static void Warning(string msg)
{
ChoETLFramework.WriteLog(ChoETLFramework.Switch.TraceWarning, msg);
}
public static void Write(string msg)
{
ChoETLFramework.WriteLog(msg);
}
private bool LoadNode(Tuple <long, JObject> pair, ref object rec)
{
if (!Configuration.UseJSONSerialization)
{
rec = Configuration.IsDynamicObject ? new ChoDynamicObject()
{
ThrowExceptionIfPropNotExists = true
}
}
: Activator.CreateInstance(RecordType);
try
{
if (!RaiseBeforeRecordLoad(rec, ref pair))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping...");
rec = null;
return(true);
}
if (pair.Item2 == null)
{
rec = null;
return(true);
}
if (!Configuration.UseJSONSerialization)
{
if (!FillRecord(rec, pair))
{
return(false);
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
rec.DoObjectLevelValidation(Configuration, Configuration.JSONRecordFieldConfigurations);
}
}
else
{
//rec = _se.Value != null ? pair.Item2.ToObject(RecordType, _se.Value) : pair.Item2.ToObject(RecordType);
if (Configuration.IsDynamicObject)
{
rec = JsonConvert.DeserializeObject <ExpandoObject>(pair.Item2.ToString(), new ExpandoObjectConverter());
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.Off) != ChoObjectValidationMode.Off)
{
rec.DoObjectLevelValidation(Configuration, Configuration.JSONRecordFieldConfigurations);
}
}
else
{
rec = JsonConvert.DeserializeObject(pair.Item2.ToString(), RecordType);
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.Off) != ChoObjectValidationMode.Off)
{
rec.DoObjectLevelValidation(Configuration, Configuration.JSONRecordFieldConfigurations);
}
}
}
bool skip = false;
if (!RaiseAfterRecordLoad(rec, pair, ref skip))
{
return(false);
}
else if (skip)
{
rec = null;
return(true);
}
}
//catch (ChoParserException)
//{
// throw;
//}
catch (ChoMissingRecordFieldException)
{
throw;
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
rec = null;
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordLoadError(rec, pair, ex))
{
throw;
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
rec = null;
}
}
else
{
throw;
}
return(true);
}
return(true);
}
private IEnumerable <object> AsEnumerable(IEnumerable <object> objs, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null)
{
TraceSwitch = traceSwitch;
long counter = 0;
Tuple <long, object> pair = null;
bool? skip = false;
bool? skipUntil = true;
bool? doWhile = true;
bool abortRequested = false;
List <object> buffer = new List <object>();
IDictionary <string, Type> recFieldTypes = null;
foreach (var obj in objs)
{
pair = new Tuple <long, object>(++counter, obj);
skip = false;
if (skipUntil != null)
{
if (skipUntil.Value)
{
skipUntil = RaiseSkipUntil(pair);
if (skipUntil == null)
{
}
else
{
skip = skipUntil.Value;
}
}
}
if (skip == null)
{
break;
}
if (skip.Value)
{
continue;
}
if (!_configCheckDone)
{
Configuration.Validate(pair);
_configCheckDone = true;
}
object rec = null;
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading node [{0}]...".FormatString(pair.Item1));
}
if (!LoadNode(pair, ref rec))
{
yield break;
}
if (rec == null)
{
continue;
}
yield return(rec);
if (Configuration.NotifyAfter > 0 && pair.Item1 % Configuration.NotifyAfter == 0)
{
if (RaisedRowsLoaded(pair.Item1))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
abortRequested = true;
yield break;
}
}
if (doWhile != null)
{
doWhile = RaiseDoWhile(pair);
if (doWhile != null && doWhile.Value)
{
break;
}
}
pair = null;
}
if (!abortRequested && pair != null)
{
RaisedRowsLoaded(pair.Item1, true);
}
}
public override IEnumerable <object> WriteTo(object writer, IEnumerable <object> records, Func <object, bool> predicate = null)
{
_sw = writer;
TextWriter sw = writer as TextWriter;
ChoGuard.ArgumentNotNull(sw, "TextWriter");
if (Configuration.JsonSerializerSettings.ContractResolver is ChoPropertyRenameAndIgnoreSerializerContractResolver)
{
ChoPropertyRenameAndIgnoreSerializerContractResolver cr = Configuration.JsonSerializerSettings.ContractResolver as ChoPropertyRenameAndIgnoreSerializerContractResolver;
cr.CallbackRecordFieldWrite = _callbackRecordFieldWrite;
cr.Writer = Writer;
}
if (records == null)
{
yield break;
}
if (Configuration.SingleDocument == null)
{
Configuration.SingleDocument = false;
}
CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture;
System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture;
string recText = String.Empty;
bool recordIgnored = false;
try
{
foreach (object record in records)
{
_index++;
//if (!isFirstRec)
//{
// if (!recordIgnored)
// sw.Write(",");
// else
// recordIgnored = false;
//}
if (TraceSwitch.TraceVerbose)
{
if (record is IChoETLNameableObject)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index));
}
}
recText = String.Empty;
if (predicate == null || predicate(record))
{
//Discover and load Xml columns from first record
if (!Configuration.IsInitialized)
{
if (record == null)
{
continue;
}
string[] fieldNames = null;
if (Configuration.RecordType == typeof(object))
{
Type recordType = ElementType == null?record.GetType() : ElementType;
RecordType = Configuration.RecordType = recordType.GetUnderlyingType(); //.ResolveType();
Configuration.IsDynamicObject = recordType.IsDynamicType();
}
if (typeof(IDictionary).IsAssignableFrom(Configuration.RecordType) ||
typeof(IList).IsAssignableFrom(Configuration.RecordType))
{
Configuration.UseYamlSerialization = true;
}
if (!Configuration.IsDynamicObject)
{
if (Configuration.YamlRecordFieldConfigurations.Count == 0)
{
Configuration.MapRecordFields(Configuration.RecordType);
}
}
if (Configuration.IsDynamicObject)
{
var dict = record.ToDynamicObject() as IDictionary <string, Object>;
fieldNames = dict.Keys.ToArray();
}
else
{
fieldNames = ChoTypeDescriptor.GetProperties <ChoYamlRecordFieldAttribute>(Configuration.RecordType).Select(pd => pd.Name).ToArray();
if (fieldNames.Length == 0)
{
fieldNames = ChoType.GetProperties(Configuration.RecordType).Select(p => p.Name).ToArray();
}
}
Configuration.Validate(fieldNames);
Configuration.IsInitialized = true;
if (!BeginWrite.Value)
{
yield break;
}
}
if (!RaiseBeforeRecordWrite(record, _index, ref recText))
{
yield break;
}
if (recText == null)
{
continue;
}
try
{
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
record.DoObjectLevelValidation(Configuration, Configuration.YamlRecordFieldConfigurations);
}
if (ToText(_index, record, out recText))
{
if (recText.EndsWith("..."))
{
recText = recText.Left(recText.Length - 3);
}
if (recText.EndsWith($"...{Environment.NewLine}"))
{
recText = recText.Left(recText.Length - $"...{Environment.NewLine}".Length);
}
if (!Configuration.SingleDocument.Value)
{
sw.Write($"---{EOLDelimiter}");
}
sw.Write("{0}", recText);
if (!Configuration.SingleDocument.Value)
{
sw.Write($"...{EOLDelimiter}");
}
if (!RaiseAfterRecordWrite(record, _index, recText))
{
yield break;
}
}
}
//catch (ChoParserException)
//{
// throw;
//}
catch (Exception ex)
{
ChoETLFramework.HandleException(ref ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
recordIgnored = true;
ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordWriteError(record, _index, recText, ex))
{
throw;
}
else
{
recordIgnored = true;
//ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
}
}
else
{
throw;
}
}
}
yield return(record);
if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0)
{
if (RaisedRowsWritten(_index))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
yield break;
}
}
}
}
finally
{
System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo;
}
}
private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null)
{
TraceSwitch = traceSwitch;
TextReader sr = source as TextReader;
ChoGuard.ArgumentNotNull(sr, "TextReader");
if (sr is StreamReader)
{
((StreamReader)sr).Seek(0, SeekOrigin.Begin);
}
if (!RaiseBeginLoad(sr))
{
yield break;
}
string[] commentTokens = Configuration.Comments;
bool? skip = false;
bool abortRequested = false;
long runningCount = 0;
using (ChoPeekEnumerator <Tuple <long, string> > e = new ChoPeekEnumerator <Tuple <long, string> >(
new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar, Configuration.MayContainEOLInData)).ToEnumerable(),
(pair) =>
{
//bool isStateAvail = IsStateAvail();
skip = false;
//if (isStateAvail)
//{
// if (!IsStateMatches(item))
// {
// skip = filterFunc != null ? filterFunc(item) : false;
// }
// else
// skip = true;
//}
//else
// skip = filterFunc != null ? filterFunc(item) : false;
if (skip == null)
{
return(null);
}
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine);
if (!skip.Value)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1));
}
}
if (skip.Value)
{
return(skip);
}
//if (!(sr.BaseStream is MemoryStream))
// ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1));
//if (Task != null)
// return !IsStateNOTExistsOrNOTMatch(item);
if (pair.Item2.IsNullOrWhiteSpace())
{
if (!Configuration.IgnoreEmptyLine)
{
throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pair.Item1));
}
else
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1));
}
return(true);
}
}
//LoadExcelSeparator if any
if (pair.Item1 == 1 &&
!_excelSeparatorFound)
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Inspecting for excel separator at [{0}]...".FormatString(pair.Item1));
}
bool retVal = LoadExcelSeperatorIfAny(pair);
_excelSeparatorFound = true;
if (Configuration.HasExcelSeparator != null &&
Configuration.HasExcelSeparator.Value &&
!retVal)
{
throw new ChoParserException("Missing excel separator header line in the file.");
}
if (retVal)
{
return(true);
}
}
if (commentTokens != null && commentTokens.Length > 0)
{
foreach (string comment in commentTokens)
{
if (!pair.Item2.IsNull() && pair.Item2.StartsWith(comment, StringComparison.Ordinal)) //, true, Configuration.Culture))
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1));
}
return(true);
}
}
}
if (Configuration.FileHeaderConfiguration.HeaderLineAt > 0)
{
if (pair.Item1 < Configuration.FileHeaderConfiguration.HeaderLineAt)
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Header line at {1}. Skipping [{0}] line...".FormatString(pair.Item1, Configuration.FileHeaderConfiguration.HeaderLineAt));
}
return(true);
}
}
if (!_configCheckDone)
{
Configuration.Validate(GetHeaders(pair.Item2));
var dict = Configuration.CSVRecordFieldConfigurations.ToDictionary(i => i.Name, i => i.FieldType == null ? null : i.FieldType);
RaiseMembersDiscovered(ref dict);
Configuration.UpdateFieldTypesIfAny(dict);
_configCheckDone = true;
}
//LoadHeader if any
if ((Configuration.FileHeaderConfiguration.HasHeaderRecord ||
Configuration.FileHeaderConfiguration.HeaderLineAt > 0) &&
!_headerFound)
{
if (Configuration.FileHeaderConfiguration.IgnoreHeader)
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring header line at [{0}]...".FormatString(pair.Item1));
}
}
else
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading header line at [{0}]...".FormatString(pair.Item1));
}
LoadHeaderLine(pair);
}
_headerFound = true;
return(true);
}
return(false);
}))
{
while (true)
{
Tuple <long, string> pair = e.Peek;
if (pair == null)
{
if (!abortRequested)
{
RaisedRowsLoaded(runningCount);
}
RaiseEndLoad(sr);
yield break;
}
runningCount = pair.Item1;
object rec = Configuration.IsDynamicObject ? new ChoDynamicObject(new Dictionary <string, object>(Configuration.FileHeaderConfiguration.StringComparer))
{
ThrowExceptionIfPropNotExists = true,
KeyResolver = (name) =>
{
if (Configuration.RecordFieldConfigurationsDict2.ContainsKey(name))
{
return(Configuration.RecordFieldConfigurationsDict2[name].Name);
}
else
{
return(name);
}
}
} : Activator.CreateInstance(RecordType);
if (!LoadLine(pair, ref rec))
{
yield break;
}
//StoreState(e.Current, rec != null);
e.MoveNext();
if (rec == null)
{
continue;
}
yield return(rec);
if (Configuration.NotifyAfter > 0 && pair.Item1 % Configuration.NotifyAfter == 0)
{
if (RaisedRowsLoaded(pair.Item1))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
abortRequested = true;
yield break;
}
}
}
}
}
public override IEnumerable <object> WriteTo(object writer, IEnumerable <object> records, Func <object, bool> predicate = null)
{
TextWriter sw = writer as TextWriter;
ChoGuard.ArgumentNotNull(sw, "TextWriter");
if (records == null)
{
yield break;
}
CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture;
System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture;
string recText = String.Empty;
bool recordIgnored = false;
try
{
foreach (object record in records)
{
_index++;
if (!isFirstRec)
{
if (!recordIgnored)
{
sw.Write(",");
}
else
{
recordIgnored = false;
}
}
if (TraceSwitch.TraceVerbose)
{
if (record is IChoETLNameableObject)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index));
}
}
recText = String.Empty;
if (predicate == null || predicate(record))
{
//Discover and load Xml columns from first record
if (!_configCheckDone)
{
if (record == null)
{
continue;
}
string[] fieldNames = null;
Type recordType = ElementType == null?record.GetType() : ElementType;
if (typeof(ICollection).IsAssignableFrom(recordType))
{
recordType = recordType.GetEnumerableItemType().GetUnderlyingType();
}
else
{
recordType = recordType.GetUnderlyingType();
}
Configuration.IsDynamicObject = recordType.IsDynamicType();
if (!Configuration.IsDynamicObject)
{
if (recordType.IsSimple())
{
Configuration.RecordType = typeof(ChoScalarObject <>).MakeGenericType(recordType);
}
else
{
Configuration.RecordType = recordType;
}
}
if (Configuration.IsDynamicObject)
{
var dict = record.ToDynamicObject() as IDictionary <string, Object>;
fieldNames = dict.Keys.ToArray();
}
else
{
fieldNames = ChoTypeDescriptor.GetProperties <ChoJSONRecordFieldAttribute>(Configuration.RecordType).Select(pd => pd.Name).ToArray();
if (fieldNames.Length == 0)
{
fieldNames = ChoType.GetProperties(Configuration.RecordType).Select(p => p.Name).ToArray();
}
}
Configuration.Validate(fieldNames);
_configCheckDone = true;
if (!RaiseBeginWrite(sw))
{
yield break;
}
if (!SupportMultipleContent)
{
sw.Write("[");
}
}
if (!RaiseBeforeRecordWrite(record, _index, ref recText))
{
yield break;
}
if (recText == null)
{
continue;
}
try
{
if (!Configuration.UseJSONSerialization)
{
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
record.DoObjectLevelValidation(Configuration, Configuration.JSONRecordFieldConfigurations);
}
if (ToText(_index, record, out recText))
{
if (!recText.IsNullOrEmpty())
{
if (!SupportMultipleContent)
{
sw.Write("{1}{0}", Configuration.Formatting == Formatting.Indented ? recText.Indent(1, " ") : recText, Configuration.EOLDelimiter);
}
else
{
if (_index == 1)
{
sw.Write("{0}", recText);
}
else
{
sw.Write("{1}{0}", recText, Configuration.EOLDelimiter);
}
}
if (!RaiseAfterRecordWrite(record, _index, recText))
{
yield break;
}
}
}
}
else
{
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.Off) != ChoObjectValidationMode.Off)
{
record.DoObjectLevelValidation(Configuration, Configuration.JSONRecordFieldConfigurations);
}
recText = JsonConvert.SerializeObject(record, Configuration.Formatting);
if (!SupportMultipleContent)
{
sw.Write("{1}{0}", Configuration.Formatting == Formatting.Indented ? recText.Indent(1, " ") : recText, Configuration.EOLDelimiter);
}
else
{
if (_index == 1)
{
sw.Write("{0}", recText);
}
else
{
sw.Write("{1}{0}", recText, Configuration.EOLDelimiter);
}
}
if (!RaiseAfterRecordWrite(record, _index, recText))
{
yield break;
}
}
}
//catch (ChoParserException)
//{
// throw;
//}
catch (Exception ex)
{
ChoETLFramework.HandleException(ref ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
recordIgnored = true;
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordWriteError(record, _index, recText, ex))
{
throw;
}
else
{
recordIgnored = true;
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
}
}
else
{
throw;
}
}
}
yield return(record);
if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0)
{
if (RaisedRowsWritten(_index))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
yield break;
}
}
isFirstRec = false;
}
}
finally
{
System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo;
}
}
private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null)
{
TraceSwitch = traceSwitch;
StreamReader sr = source as StreamReader;
ChoGuard.ArgumentNotNull(sr, "StreamReader");
sr.Seek(0, SeekOrigin.Begin);
if (!RaiseBeginLoad(sr))
{
yield break;
}
string[] commentTokens = Configuration.Comments;
using (ChoPeekEnumerator <Tuple <int, string> > e = new ChoPeekEnumerator <Tuple <int, string> >(
new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar)).ToEnumerable(),
(pair) =>
{
//bool isStateAvail = IsStateAvail();
bool?skip = false;
//if (isStateAvail)
//{
// if (!IsStateMatches(item))
// {
// skip = filterFunc != null ? filterFunc(item) : false;
// }
// else
// skip = true;
//}
//else
// skip = filterFunc != null ? filterFunc(item) : false;
if (skip == null)
{
return(null);
}
//if (!(sr.BaseStream is MemoryStream))
//{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine);
if (!skip.Value)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1));
}
//}
if (skip.Value)
{
return(skip);
}
//if (!(sr.BaseStream is MemoryStream))
// ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1));
//if (Task != null)
// return !IsStateNOTExistsOrNOTMatch(item);
if (pair.Item2.IsNullOrWhiteSpace())
{
if (!Configuration.IgnoreEmptyLine)
{
throw new ChoParserException("Empty line found at {0} location.".FormatString(pair.Item1));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1));
return(true);
}
}
if (commentTokens == null)
{
return(false);
}
else
{
var x = (from comment in commentTokens
where !pair.Item2.IsNull() && pair.Item2.StartsWith(comment, true, Configuration.Culture)
select comment).FirstOrDefault();
if (x != null)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1));
return(true);
}
}
if (!_configCheckDone)
{
Configuration.Validate(pair); // GetHeaders(pair.Item2));
_configCheckDone = true;
}
//LoadHeader if any
if (Configuration.FileHeaderConfiguration.HasHeaderRecord &&
!_headerFound)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading header line at [{0}]...".FormatString(pair.Item1));
LoadHeaderLine(pair);
_headerFound = true;
return(true);
}
return(false);
}))
{
while (true)
{
Tuple <int, string> pair = e.Peek;
if (pair == null)
{
RaiseEndLoad(sr);
yield break;
}
object rec = ChoActivator.CreateInstance(RecordType);
if (!LoadLine(pair, ref rec))
{
yield break;
}
//StoreState(e.Current, rec != null);
e.MoveNext();
if (rec == null)
{
continue;
}
yield return(rec);
}
}
}
private bool FillRecord(ref object rec, Tuple <long, IDictionary <string, object> > pair)
{
long lineNo;
lineNo = pair.Item1;
var node = pair.Item2;
fieldValue = null;
fieldConfig = null;
pi = null;
object rootRec = rec;
foreach (KeyValuePair <string, ChoParquetRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict)
{
if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject)
{
if (Configuration.IgnoredFields.Contains(kvp.Key))
{
continue;
}
}
fieldValue = null;
fieldConfig = kvp.Value;
if (Configuration.PIDict != null)
{
// if FieldName is set
if (!string.IsNullOrEmpty(fieldConfig.FieldName))
{
// match using FieldName
Configuration.PIDict.TryGetValue(fieldConfig.FieldName, out pi);
}
if (pi == null)
{
// otherwise match usign the property name
Configuration.PIDict.TryGetValue(kvp.Key, out pi);
}
}
rec = GetDeclaringRecord(kvp.Value.DeclaringMember, rootRec);
if (!node.ContainsKey(kvp.Value.FieldName))
{
if (Configuration.ColumnCountStrict)
{
throw new ChoParserException("No matching '{0}' field found.".FormatString(fieldConfig.FieldName));
}
}
fieldValue = node[kvp.Value.FieldName];
if (!RaiseBeforeRecordFieldLoad(rec, pair.Item1, kvp.Key, ref fieldValue))
{
continue;
}
try
{
if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject)
{
}
else
{
if (pi != null)
{
kvp.Value.FieldType = pi.PropertyType;
}
else
{
kvp.Value.FieldType = typeof(string);
}
}
object v1 = node;
if (fieldConfig.CustomSerializer != null)
{
fieldValue = fieldConfig.CustomSerializer(v1);
}
else if (RaiseRecordFieldDeserialize(rec, pair.Item1, kvp.Key, ref v1))
{
fieldValue = v1;
}
else if (fieldConfig.PropCustomSerializer != null)
{
fieldValue = ChoCustomSerializer.Deserialize(v1, fieldConfig.FieldType, fieldConfig.PropCustomSerializer, fieldConfig.PropCustomSerializerParams, Configuration.Culture, fieldConfig.Name);
}
else
{
if (!node.ContainsKey(kvp.Value.FieldName))
{
if (Configuration.ColumnCountStrict)
{
throw new ChoParserException("No matching '{0}' field found.".FormatString(fieldConfig.FieldName));
}
}
else
{
fieldValue = node[kvp.Value.FieldName];
}
}
bool ignoreFieldValue = fieldValue.IgnoreFieldValue(fieldConfig.IgnoreFieldValueMode);
if (ignoreFieldValue && fieldConfig.IsDefaultValueSpecified)
{
fieldValue = fieldConfig.DefaultValue;
}
ignoreFieldValue = fieldValue.IgnoreFieldValue(fieldConfig.IgnoreFieldValueMode);
if (ignoreFieldValue)
{
continue;
}
if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject)
{
var dict = rec as IDictionary <string, Object>;
dict.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel)
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
}
else
{
if (Configuration.SupportsMultiRecordTypes)
{
ChoType.TryGetProperty(rec.GetType(), kvp.Key, out pi);
//*** TODO
//fieldConfig.PI = pi;
//fieldConfig.PropConverters = ChoTypeDescriptor.GetTypeConverters(fieldConfig.PI);
//fieldConfig.PropConverterParams = ChoTypeDescriptor.GetTypeConverterParams(fieldConfig.PI);
}
if (pi != null)
{
rec.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
}
else if (RecordType.IsSimple())
{
rec = ChoConvert.ConvertTo(fieldValue, RecordType, Configuration.Culture);
}
else
{
throw new ChoMissingRecordFieldException("Missing '{0}' property in {1} type.".FormatString(kvp.Key, ChoType.GetTypeName(rec)));
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.MemberLevel) == ChoObjectValidationMode.MemberLevel)
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
}
if (!RaiseAfterRecordFieldLoad(rec, pair.Item1, kvp.Key, fieldValue))
{
return(false);
}
}
catch (ChoParserException)
{
Reader.IsValid = false;
throw;
}
catch (ChoMissingRecordFieldException)
{
Reader.IsValid = false;
if (Configuration.ThrowAndStopOnMissingField)
{
throw;
}
}
catch (Exception ex)
{
Reader.IsValid = false;
ChoETLFramework.HandleException(ref ex);
if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
{
throw;
}
try
{
if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject)
{
var dict = rec as IDictionary <string, Object>;
if (dict.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (dict.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (ex is ValidationException)
{
throw;
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
else if (pi != null)
{
if (rec.SetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (rec.SetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (ex is ValidationException)
{
throw;
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
else
{
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
}
catch (Exception innerEx)
{
if (ex == innerEx.InnerException || ex is ValidationException)
{
if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring field...".FormatString(ex.Message));
continue;
}
else
{
if (!RaiseRecordFieldLoadError(rec, pair.Item1, kvp.Key, ref fieldValue, ex))
{
if (ex is ValidationException)
{
throw;
}
throw new ChoReaderException($"Failed to parse '{fieldValue}' value for '{fieldConfig.FieldName}' field.", ex);
}
else
{
try
{
if (Configuration.IsDynamicObject)
{
var dict = rec as IDictionary <string, Object>;
dict.ConvertNSetMemberValue(kvp.Key, fieldConfig, ref fieldValue, Configuration.Culture);
}
else
{
if (pi != null)
{
rec.ConvertNSetMemberValue(kvp.Key, fieldConfig, ref fieldValue, Configuration.Culture);
}
else
{
throw new ChoMissingRecordFieldException("Missing '{0}' property in {1} type.".FormatString(kvp.Key, ChoType.GetTypeName(rec)));
}
}
}
catch { }
}
}
}
else
{
throw new ChoReaderException("Failed to assign '{0}' fallback value to '{1}' field.".FormatString(fieldValue, fieldConfig.FieldName), innerEx);
}
}
}
}
rec = rootRec;
return(true);
}
public static void Verbose(string msg)
{
ChoETLFramework.WriteLog(ChoETLFramework.Switch.TraceVerbose, msg);
}
private bool LoadNode(Tuple <long, object> pair, ref object rec)
{
bool ignoreFieldValue = pair.Item2.IgnoreFieldValue(Configuration.IgnoreFieldValueMode);
if (ignoreFieldValue)
{
return(false);
}
else if (pair.Item2 == null && !Configuration.IsDynamicObject)
{
rec = RecordType.CreateInstanceAndDefaultToMembers(Configuration.RecordFieldConfigurationsDict.ToDictionary(kvp => kvp.Key, kvp => kvp.Value as ChoRecordFieldConfiguration));
return(true);
}
try
{
if (!RaiseBeforeRecordLoad(rec, ref pair))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping...");
rec = null;
return(true);
}
//if (Configuration.CustomNodeSelecter != null)
//{
// pair = new Tuple<long, object>(pair.Item1, Configuration.CustomNodeSelecter(pair.Item2));
//}
if (pair.Item2 == null)
{
rec = null;
return(true);
}
rec = pair.Item2;
if (rec is AvroRecord)
{
rec = ToDynamicObject(rec as AvroRecord);
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
rec.DoObjectLevelValidation(Configuration, Configuration.AvroRecordFieldConfigurations);
}
bool skip = false;
if (!RaiseAfterRecordLoad(rec, pair, ref skip))
{
return(false);
}
else if (skip)
{
rec = null;
return(true);
}
}
catch (Exception ex)
{
Reader.IsValid = false;
if (ex is ChoMissingRecordFieldException && Configuration.ThrowAndStopOnMissingField)
{
if (!RaiseRecordLoadError(rec, pair, ex))
{
throw;
}
else
{
//ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
//rec = null;
}
}
else
{
ChoETLFramework.HandleException(ref ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
rec = null;
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordLoadError(rec, pair, ex))
{
throw;
}
else
{
//ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
//rec = null;
}
}
else
{
throw;
}
}
return(true);
}
return(true);
}
public static void Info(string msg)
{
ChoETLFramework.WriteLog(ChoETLFramework.Switch.TraceInfo, msg);
}
private IEnumerable <object> AsEnumerable(IEnumerable <DataColumn[]> dataColumns, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null)
{
TraceSwitch = traceSwitch;
long counter = 0;
Tuple <long, IDictionary <string, object> > pair = null;
bool? skip = false;
bool? skipUntil = true;
bool? doWhile = true;
bool abortRequested = false;
List <object> buffer = new List <object>();
IDictionary <string, Type> recFieldTypes = null;
foreach (var dc in dataColumns)
{
foreach (var obj in Unpack(dc))
{
pair = new Tuple <long, IDictionary <string, object> >(++counter, obj);
skip = false;
if (skipUntil != null)
{
if (skipUntil.Value)
{
skipUntil = RaiseSkipUntil(pair);
if (skipUntil == null)
{
}
else
{
skip = skipUntil.Value;
}
}
}
if (skip == null)
{
break;
}
if (skip.Value)
{
continue;
}
if (!_configCheckDone)
{
if (Configuration.SupportsMultiRecordTypes && Configuration.RecordSelector != null && !Configuration.RecordTypeMapped)
{
}
else
{
Configuration.Validate(pair);
}
var dict = Configuration.ParquetRecordFieldConfigurations.ToDictionary(i => i.Name, i => i.FieldType == null ? null : i.FieldType);
if (Configuration.MaxScanRows == 0)
{
RaiseMembersDiscovered(dict);
}
Configuration.UpdateFieldTypesIfAny(dict);
_configCheckDone = true;
}
object rec = null;
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading node [{0}]...".FormatString(pair.Item1));
}
if (!LoadNode(pair, ref rec))
{
yield break;
}
if (rec == null)
{
continue;
}
if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject)
{
if (Configuration.AreAllFieldTypesNull && Configuration.AutoDiscoverFieldTypes && Configuration.MaxScanRows > 0 && counter <= Configuration.MaxScanRows)
{
buffer.Add(rec);
if (recFieldTypes == null)
{
recFieldTypes = Configuration.ParquetRecordFieldConfigurations.ToDictionary(i => i.FieldName, i => i.FieldType == null ? null : i.FieldType);
}
RaiseRecordFieldTypeAssessment(recFieldTypes, (IDictionary <string, object>)rec, counter == Configuration.MaxScanRows);
if (counter == Configuration.MaxScanRows)
{
Configuration.UpdateFieldTypesIfAny(recFieldTypes);
var dict = recFieldTypes = Configuration.ParquetRecordFieldConfigurations.ToDictionary(i => i.FieldName, i => i.FieldType == null ? null : i.FieldType);
RaiseMembersDiscovered(dict);
foreach (object rec1 in buffer)
{
yield return(new ChoDynamicObject(MigrateToNewSchema(rec1 as IDictionary <string, object>, recFieldTypes)));
}
buffer.Clear();
}
}
else
{
yield return(rec);
}
}
else
{
yield return(rec);
}
if (Configuration.NotifyAfter > 0 && pair.Item1 % Configuration.NotifyAfter == 0)
{
if (RaisedRowsLoaded(pair.Item1))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
abortRequested = true;
yield break;
}
}
if (doWhile != null)
{
doWhile = RaiseDoWhile(pair);
if (doWhile != null && doWhile.Value)
{
break;
}
}
}
}
if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject)
{
if (buffer.Count > 0)
{
Configuration.UpdateFieldTypesIfAny(recFieldTypes);
var dict = recFieldTypes = Configuration.ParquetRecordFieldConfigurations.ToDictionary(i => i.FieldName, i => i.FieldType == null ? null : i.FieldType);
RaiseMembersDiscovered(dict);
foreach (object rec1 in buffer)
{
yield return(new ChoDynamicObject(MigrateToNewSchema(rec1 as IDictionary <string, object>, recFieldTypes)));
}
}
}
if (!abortRequested && pair != null)
{
RaisedRowsLoaded(pair.Item1);
}
}
public static void Error(string msg)
{
ChoETLFramework.WriteLog(ChoETLFramework.Switch.TraceError, msg);
}
private bool LoadNode(Tuple <long, IDictionary <string, object> > pair, ref object rec)
{
bool ignoreFieldValue = pair.Item2.IgnoreFieldValue(Configuration.IgnoreFieldValueMode);
if (ignoreFieldValue)
{
return(false);
}
else if (pair.Item2 == null && !Configuration.IsDynamicObject)
{
rec = RecordType.CreateInstanceAndDefaultToMembers(Configuration.RecordFieldConfigurationsDict.ToDictionary(kvp => kvp.Key, kvp => kvp.Value as ChoRecordFieldConfiguration));
return(true);
}
if (Configuration.SupportsMultiRecordTypes && Configuration.RecordSelector != null)
{
Type recType = Configuration.RecordSelector(pair);
if (recType == null)
{
if (Configuration.IgnoreIfNoRecordTypeFound)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, $"No record type found for [{pair.Item1}] line to parse.");
return(true);
}
else
{
throw new ChoParserException($"No record type found for [{pair.Item1}] line to parse.");
}
}
if (!Configuration.RecordTypeMapped)
{
Configuration.MapRecordFields(recType);
Configuration.Validate(null);
}
rec = recType.IsDynamicType() ? new ChoDynamicObject()
{
ThrowExceptionIfPropNotExists = true
} : ChoActivator.CreateInstance(recType);
}
else if (Configuration.IsDynamicObject)
{
rec = Configuration.IsDynamicObject ? new ChoDynamicObject()
{
ThrowExceptionIfPropNotExists = true
}
}
: ChoActivator.CreateInstance(RecordType);
try
{
if (!RaiseBeforeRecordLoad(rec, ref pair))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping...");
rec = null;
return(true);
}
//if (Configuration.CustomNodeSelecter != null)
//{
// pair = new Tuple<long, IDictionary<string, object>>(pair.Item1, Configuration.CustomNodeSelecter(pair.Item2));
//}
if (pair.Item2 == null)
{
rec = null;
return(true);
}
if (!FillRecord(ref rec, pair))
{
return(false);
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
rec.DoObjectLevelValidation(Configuration, Configuration.ParquetRecordFieldConfigurations);
}
bool skip = false;
if (!RaiseAfterRecordLoad(rec, pair, ref skip))
{
return(false);
}
else if (skip)
{
rec = null;
return(true);
}
}
catch (Exception ex)
{
Reader.IsValid = false;
if (ex is ChoMissingRecordFieldException && Configuration.ThrowAndStopOnMissingField)
{
if (!RaiseRecordLoadError(rec, pair, ex))
{
throw;
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
rec = null;
}
}
else
{
ChoETLFramework.HandleException(ref ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
rec = null;
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordLoadError(rec, pair, ex))
{
throw;
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
rec = null;
}
}
else
{
throw;
}
}
return(true);
}
return(true);
}
object fieldValue = null;
ChoParquetRecordFieldConfiguration fieldConfig = null;
PropertyInfo pi = null;
public static void Write(bool condition, string msg)
{
ChoETLFramework.WriteLog(condition, msg);
}
private bool LoadLine(Tuple <long, string> pair, ref object rec)
{
Type recType = rec.GetType();
try
{
if (!RaiseBeforeRecordLoad(rec, ref pair))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping...");
rec = null;
return(true);
}
if (pair.Item2 == null)
{
rec = null;
return(true);
}
else if (pair.Item2 == String.Empty)
{
return(true);
}
if (!pair.Item2.IsNullOrWhiteSpace())
{
var config = GetConfiguration(rec.GetType());
if (config == null)
{
throw new ChoParserException("No parser found to parse record line.");
}
if (config.GetType() == typeof(ChoCSVRecordConfiguration))
{
var r = ChoCSVReader.LoadText(rec.GetType(), pair.Item2, config as ChoCSVRecordConfiguration, Configuration.Encoding, Configuration.BufferSize, TraceSwitch, Reader);
rec = r.FirstOrDefault <object>();
}
else if (config.GetType() == typeof(ChoFixedLengthRecordConfiguration))
{
var r = ChoFixedLengthReader.LoadText(rec.GetType(), pair.Item2, config as ChoFixedLengthRecordConfiguration, Configuration.Encoding, Configuration.BufferSize, TraceSwitch, Reader);
rec = r.FirstOrDefault <object>();
}
else
{
throw new ChoParserException("Unsupported record line found to parse.");
}
}
bool skip = false;
if (!RaiseAfterRecordLoad(rec, pair, ref skip))
{
return(false);
}
else if (skip)
{
rec = null;
return(true);
}
}
catch (ChoParserException pEx)
{
throw new ChoParserException($"Failed to parse line to '{recType}' object.", pEx);
}
catch (ChoMissingRecordFieldException mEx)
{
throw new ChoParserException($"Failed to parse line to '{recType}' object.", mEx);
}
catch (Exception ex)
{
Reader.IsValid = false;
ChoETLFramework.HandleException(ref ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
rec = null;
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordLoadError(rec, pair, ex))
{
throw;
}
else
{
//ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
//rec = null;
}
}
else
{
throw;
}
//if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
//{
// rec = null;
//}
//else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
//{
// if (!RaiseRecordLoadError(rec, pair, ex))
// throw new ChoReaderException($"Failed to parse line to '{recType}' object.", ex);
//}
//else
// throw new ChoReaderException($"Failed to parse line to '{recType}' object.", ex);
return(true);
}
return(true);
}
public override IEnumerable <object> WriteTo(object writer, IEnumerable <object> records, Func <object, bool> predicate = null)
{
TextWriter sw = writer as TextWriter;
ChoGuard.ArgumentNotNull(sw, "TextWriter");
if (records == null)
{
yield break;
}
if (!RaiseBeginWrite(sw))
{
yield break;
}
CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture;
System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture;
string recText = String.Empty;
try
{
foreach (object record in records)
{
_index++;
if (TraceSwitch.TraceVerbose)
{
if (record is IChoETLNameableObject)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index));
}
}
recText = String.Empty;
if (record != null)
{
if (predicate == null || predicate(record))
{
//Discover and load FixedLength columns from first record
if (!_configCheckDone)
{
string[] fieldNames = null;
if (Configuration.IsDynamicObject)
{
var dict = record.ToDynamicObject() as IDictionary <string, Object>;
fieldNames = dict.Keys.ToArray();
}
else
{
fieldNames = ChoTypeDescriptor.GetProperties <ChoFixedLengthRecordFieldAttribute>(record.GetType()).Select(pd => pd.Name).ToArray();
if (fieldNames.Length == 0)
{
fieldNames = ChoType.GetProperties(record.GetType()).Select(p => p.Name).ToArray();
}
}
Configuration.Validate(fieldNames);
WriteHeaderLine(sw);
_configCheckDone = true;
}
if (!RaiseBeforeRecordWrite(record, _index, ref recText))
{
yield break;
}
if (recText == null)
{
continue;
}
else if (recText.Length > 0)
{
sw.Write("{1}{0}", recText, Configuration.FileHeaderConfiguration.HasHeaderRecord ? Configuration.EOLDelimiter : "");
continue;
}
try
{
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
record.DoObjectLevelValidation(Configuration, Configuration.FixedLengthRecordFieldConfigurations);
}
if (ToText(_index, record, out recText))
{
if (_index == 1)
{
sw.Write("{1}{0}", recText, Configuration.FileHeaderConfiguration.HasHeaderRecord ? Configuration.EOLDelimiter : "");
}
else
{
sw.Write("{1}{0}", recText, Configuration.EOLDelimiter);
}
if (!RaiseAfterRecordWrite(record, _index, recText))
{
yield break;
}
}
}
catch (ChoParserException)
{
throw;
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordWriteError(record, _index, recText, ex))
{
throw;
}
}
else
{
throw;
}
}
}
}
yield return(record);
if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0)
{
if (RaisedRowsWritten(_index))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
yield break;
}
}
}
}
finally
{
System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo;
}
RaiseEndWrite(sw);
}
private bool LoadNode(Tuple <long, XElement> pair, ref object rec)
{
if (!Configuration.UseXmlSerialization || Configuration.IsDynamicObject)
{
rec = Configuration.IsDynamicObject ? new ExpandoObject() : Activator.CreateInstance(RecordType);
}
try
{
if (!RaiseBeforeRecordLoad(rec, ref pair))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping...");
rec = null;
return(true);
}
if (pair.Item2 == null)
{
rec = null;
return(true);
}
if (!Configuration.UseXmlSerialization)
{
if (!FillRecord(rec, pair))
{
return(false);
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
rec.DoObjectLevelValidation(Configuration, Configuration.XmlRecordFieldConfigurations);
}
}
else
{
if (Configuration.IsDynamicObject)
{
Parse(rec, pair.Item2);
}
else
{
rec = _se.Value.Deserialize(pair.Item2.CreateReader());
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.Off) != ChoObjectValidationMode.Off)
{
rec.DoObjectLevelValidation(Configuration, Configuration.XmlRecordFieldConfigurations);
}
}
if (!RaiseAfterRecordLoad(rec, pair))
{
return(false);
}
}
catch (ChoParserException)
{
throw;
}
catch (ChoMissingRecordFieldException)
{
throw;
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
rec = null;
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordLoadError(rec, pair, ex))
{
throw;
}
}
else
{
throw;
}
return(true);
}
return(true);
}
private bool LoadLines(Tuple <long, List <Tuple <long, string> > > pairs, ref object rec)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading [{0}] record...".FormatString(pairs.Item1));
Tuple <long, string> pair = null;
long recNo = pairs.Item1;
if (!Configuration.AutoDiscoveredColumns)
{
if (Configuration.ColumnCountStrict)
{
if (pairs.Item2.Count != Configuration.KVPRecordFieldConfigurations.Count)
{
throw new ChoParserException("Incorrect number of field values found at record [{2}]. Expected [{0}] field values. Found [{1}] field values.".FormatString(Configuration.KVPRecordFieldConfigurations.Count, pairs.Item2.Count, recNo));
}
List <string> keys = (from p in pairs.Item2
select ToKVP(p.Item1, p.Item2).Key).ToList();
if (!Enumerable.SequenceEqual(keys.OrderBy(t => t), Configuration.FCArray.Select(a => a.Value.FieldName).OrderBy(t => t), Configuration.FileHeaderConfiguration.StringComparer))
{
throw new ChoParserException("Column count mismatch detected at [{0}] record.".FormatString(recNo));
}
}
if (Configuration.ColumnOrderStrict)
{
List <string> keys = (from p in pairs.Item2
select ToKVP(p.Item1, p.Item2).Key).ToList();
int runnngIndex = -1;
foreach (var k in keys)
{
runnngIndex++;
if (runnngIndex < Configuration.FCArray.Length)
{
if (Configuration.FileHeaderConfiguration.IsEqual(Configuration.FCArray[runnngIndex].Value.FieldName, k))
{
continue;
}
}
throw new ChoParserException("Found incorrect order on '{1}' column at [{0}] record.".FormatString(recNo, k));
}
}
}
object fieldValue = String.Empty;
PropertyInfo pi = null;
object rootRec = rec;
//Set default values
foreach (KeyValuePair <string, ChoKVPRecordFieldConfiguration> kvp in Configuration.FCArray)
{
rec = GetDeclaringRecord(kvp.Value.DeclaringMember, rootRec);
if (Configuration.PIDict != null)
{
Configuration.PIDict.TryGetValue(kvp.Key, out pi);
}
try
{
if (kvp.Value.IsDefaultValueSpecified)
{
fieldValue = kvp.Value.DefaultValue;
}
if (Configuration.IsDynamicObject)
{
var dict = rec as IDictionary <string, Object>;
dict.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
}
else
{
if (pi != null)
{
rec.ConvertNSetMemberValue(kvp.Key, kvp.Value, ref fieldValue, Configuration.Culture);
}
}
}
catch
{
}
}
rec = rootRec;
_propInit.Clear();
foreach (var pair1 in pairs.Item2)
{
pair = pair1;
try
{
if (!RaiseBeforeRecordLoad(rec, ref pair))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping...");
rec = null;
return(true);
}
if (pair.Item2 == null)
{
rec = null;
return(true);
}
else if (pair.Item2 == String.Empty)
{
return(true);
}
if (!pair.Item2.IsNullOrWhiteSpace())
{
if (!FillRecord(rec, pair))
{
return(false);
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
rec.DoObjectLevelValidation(Configuration, Configuration.KVPRecordFieldConfigurations);
}
}
bool skip = false;
if (!RaiseAfterRecordLoad(rec, pair, ref skip))
{
return(false);
}
else if (skip)
{
rec = null;
return(true);
}
}
//catch (ChoParserException)
//{
// throw;
//}
//catch (ChoMissingRecordFieldException)
//{
// throw;
//}
catch (Exception ex)
{
Reader.IsValid = false;
if (ex is ChoMissingRecordFieldException && Configuration.ThrowAndStopOnMissingField)
{
if (!RaiseRecordLoadError(rec, pair, ex))
{
throw;
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
rec = null;
}
}
else
{
ChoETLFramework.HandleException(ref ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
rec = null;
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordLoadError(rec, pair, ex))
{
throw;
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
rec = null;
}
}
else
{
throw;
}
}
return(true);
}
}
return(true);
}
private bool LoadLine(Tuple <long, string> pair, ref object rec)
{
try
{
if (!RaiseBeforeRecordLoad(rec, ref pair))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping...");
rec = null;
return(true);
}
if (pair.Item2 == null)
{
rec = null;
return(true);
}
else if (pair.Item2 == String.Empty)
{
return(true);
}
if (!pair.Item2.IsNullOrWhiteSpace())
{
if (!FillRecord(rec, pair))
{
return(false);
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
rec.DoObjectLevelValidation(Configuration, Configuration.FixedLengthRecordFieldConfigurations);
}
}
bool skip = false;
if (!RaiseAfterRecordLoad(rec, pair, ref skip))
{
return(false);
}
else if (skip)
{
rec = null;
return(true);
}
}
//catch (ChoParserException)
//{
// throw;
//}
//catch (ChoMissingRecordFieldException)
//{
// throw;
//}
catch (Exception ex)
{
Reader.IsValid = false;
if (ex is ChoMissingRecordFieldException && Configuration.ThrowAndStopOnMissingField)
{
throw;
}
ChoETLFramework.HandleException(ref ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
rec = null;
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordLoadError(rec, pair, ex))
{
throw;
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
rec = null;
}
}
else
{
throw;
}
return(true);
}
return(true);
}
private bool LoadLine(Tuple <long, string> pair, ref object rec)
{
try
{
if (!RaiseBeforeRecordLoad(rec, ref pair))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping...");
rec = null;
return(true);
}
if (pair.Item2 == null)
{
rec = null;
return(true);
}
else if (pair.Item2 == String.Empty)
{
return(true);
}
if (!pair.Item2.IsNullOrWhiteSpace())
{
if (!FillRecord(rec, pair))
{
return(false);
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
rec.DoObjectLevelValidation(Configuration, Configuration.FixedLengthRecordFieldConfigurations);
}
}
if (!RaiseAfterRecordLoad(rec, pair))
{
return(false);
}
}
catch (ChoParserException)
{
throw;
}
catch (ChoMissingRecordFieldException)
{
throw;
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
rec = null;
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordLoadError(rec, pair, ex))
{
throw;
}
}
else
{
throw;
}
return(true);
}
return(true);
}
private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null)
{
TraceSwitch = traceSwitch;
TextReader sr = source as TextReader;
ChoGuard.ArgumentNotNull(sr, "TextReader");
if (sr is StreamReader)
{
((StreamReader)sr).Seek(0, SeekOrigin.Begin);
}
if (!RaiseBeginLoad(sr))
{
yield break;
}
string[] commentTokens = Configuration.Comments;
bool? skip = false;
bool abortRequested = false;
long runningCount = 0;
long recCount = 0;
bool headerLineLoaded = false;
List <object> buffer = new List <object>();
IDictionary <string, Type> recFieldTypes = null;
bool?skipUntil = true;
bool?doWhile = true;
using (ChoPeekEnumerator <Tuple <long, string> > e = new ChoPeekEnumerator <Tuple <long, string> >(
new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar, false /*Configuration.MayContainEOLInData*/)).ToEnumerable(),
(pair) =>
{
//bool isStateAvail = IsStateAvail();
skip = false;
if (skipUntil != null)
{
if (skipUntil.Value)
{
skipUntil = RaiseSkipUntil(pair);
if (skipUntil == null)
{
}
else
{
if (skipUntil.Value)
{
skip = skipUntil;
}
else
{
skip = true;
}
}
}
}
//if (isStateAvail)
//{
// if (!IsStateMatches(item))
// {
// skip = filterFunc != null ? filterFunc(item) : false;
// }
// else
// skip = true;
//}
//else
// skip = filterFunc != null ? filterFunc(item) : false;
if (skip == null)
{
return(null);
}
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine);
if (!skip.Value)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1));
}
}
if (skip.Value)
{
return(skip);
}
//if (!(sr.BaseStream is MemoryStream))
// ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1));
//if (Task != null)
// return !IsStateNOTExistsOrNOTMatch(item);
if (pair.Item2.IsNullOrWhiteSpace())
{
if (!Configuration.IgnoreEmptyLine)
{
throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pair.Item1));
}
else
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1));
}
return(true);
}
}
if (commentTokens != null && commentTokens.Length > 0)
{
foreach (string comment in commentTokens)
{
if (!pair.Item2.IsNull() && pair.Item2.StartsWith(comment, StringComparison.Ordinal)) //, true, Configuration.Culture))
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1));
}
return(true);
}
}
}
if (Configuration.FileHeaderConfiguration.HeaderLineAt > 0)
{
if (pair.Item1 < Configuration.FileHeaderConfiguration.HeaderLineAt)
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Header line at {1}. Skipping [{0}] line...".FormatString(pair.Item1, Configuration.FileHeaderConfiguration.HeaderLineAt));
}
return(true);
}
}
if (!_configCheckDone)
{
if (Configuration.SupportsMultiRecordTypes && Configuration.RecordSelector != null && !Configuration.RecordTypeMapped)
{
}
else
{
Configuration.Validate(pair); // GetHeaders(pair.Item2));
}
var dict = recFieldTypes = Configuration.FixedLengthRecordFieldConfigurations.ToDictionary(i => i.Name, i => i.FieldType == null ? null : i.FieldType);
RaiseMembersDiscovered(dict);
Configuration.UpdateFieldTypesIfAny(dict);
_configCheckDone = true;
}
//LoadHeader if any
if ((Configuration.FileHeaderConfiguration.HasHeaderRecord ||
Configuration.FileHeaderConfiguration.HeaderLineAt > 0) &&
!_headerFound)
{
if (Configuration.FileHeaderConfiguration.IgnoreHeader)
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring header line at [{0}]...".FormatString(pair.Item1));
}
}
else
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading header line at [{0}]...".FormatString(pair.Item1));
}
headerLineLoaded = true;
LoadHeaderLine(pair);
}
_headerFound = true;
return(true);
}
return(false);
}))
{
while (true)
{
recCount++;
Tuple <long, string> pair = e.Peek;
if (pair == null)
{
if (!abortRequested)
{
RaisedRowsLoaded(runningCount);
}
RaiseEndLoad(sr);
yield break;
}
runningCount = pair.Item1;
object rec = null;
if (Configuration.SupportsMultiRecordTypes && Configuration.RecordSelector != null)
{
Type recType = Configuration.RecordSelector(pair);
if (recType == null)
{
if (Configuration.IgnoreIfNoRecordTypeFound)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, $"No record type found for [{pair.Item1}] line to parse.");
e.MoveNext();
continue;
}
else
{
throw new ChoParserException($"No record type found for [{pair.Item1}] line to parse.");
}
}
if (!Configuration.RecordTypeMapped)
{
Configuration.MapRecordFields(recType);
Configuration.Validate(null);
}
rec = recType.IsDynamicType() ? new ChoDynamicObject(new Dictionary <string, object>(Configuration.FileHeaderConfiguration.StringComparer))
{
ThrowExceptionIfPropNotExists = true,
AlternativeKeys = Configuration.AlternativeKeys
} : ChoActivator.CreateInstance(recType);
}
else
{
rec = Configuration.IsDynamicObject ? new ChoDynamicObject(new Dictionary <string, object>(Configuration.FileHeaderConfiguration.StringComparer))
{
ThrowExceptionIfPropNotExists = true,
AlternativeKeys = Configuration.AlternativeKeys
} : ChoActivator.CreateInstance(RecordType);
}
if (!LoadLine(pair, ref rec))
{
yield break;
}
//StoreState(e.Current, rec != null);
e.MoveNext();
if (rec == null)
{
continue;
}
if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject)
{
if (Configuration.AreAllFieldTypesNull && Configuration.AutoDiscoverFieldTypes && Configuration.MaxScanRows > 0 && recCount <= Configuration.MaxScanRows)
{
buffer.Add(rec);
RaiseRecordFieldTypeAssessment(recFieldTypes, (IDictionary <string, object>)rec, recCount == Configuration.MaxScanRows);
if (recCount == Configuration.MaxScanRows || e.Peek == null)
{
Configuration.UpdateFieldTypesIfAny(recFieldTypes);
var dict = recFieldTypes = Configuration.FixedLengthRecordFieldConfigurations.ToDictionary(i => i.Name, i => i.FieldType == null ? null : i.FieldType);
RaiseMembersDiscovered(dict);
foreach (object rec1 in buffer)
{
yield return(ConvertToNestedObjectIfApplicable(new ChoDynamicObject(MigrateToNewSchema(rec1 as IDictionary <string, object>, recFieldTypes)) as object, headerLineLoaded));
}
}
}
else
{
yield return(ConvertToNestedObjectIfApplicable(rec, headerLineLoaded));
}
}
else
{
yield return(rec);
}
if (Configuration.NotifyAfter > 0 && pair.Item1 % Configuration.NotifyAfter == 0)
{
if (RaisedRowsLoaded(pair.Item1))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
yield break;
}
}
if (doWhile != null)
{
doWhile = RaiseDoWhile(pair);
if (doWhile != null && doWhile.Value)
{
break;
}
}
}
}
}
public override IEnumerable <object> WriteTo(object writer, IEnumerable <object> records, Func <object, bool> predicate = null)
{
TextWriter sw = writer as TextWriter;
ChoGuard.ArgumentNotNull(sw, "TextWriter");
if (records == null)
{
yield break;
}
if (!RaiseBeginWrite(sw))
{
yield break;
}
CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture;
System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture;
string recText = String.Empty;
Type recType;
try
{
foreach (object record in records)
{
_index++;
recType = record.GetType();
if (record is IChoETLNameableObject)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index));
}
recText = String.Empty;
if (record != null)
{
if (predicate == null || predicate(record))
{
//Discover and load Manifold columns from first record
if (!_configCheckDone)
{
//string[] fieldNames = null;
//if (record is ExpandoObject)
//{
// var dict = record as IDictionary<string, Object>;
// fieldNames = dict.Keys.ToArray();
//}
//else
//{
// fieldNames = ChoTypeDescriptor.GetProperties<ChoManifoldRecordFieldAttribute>(record.GetType()).Select(pd => pd.Name).ToArray();
// if (fieldNames.Length == 0)
// {
// fieldNames = ChoType.GetProperties(record.GetType()).Select(p => p.Name).ToArray();
// }
//}
//Configuration.Validate(fieldNames);
//WriteHeaderLine(sw);
_configCheckDone = true;
}
if (!RaiseBeforeRecordWrite(record, _index, ref recText))
{
yield break;
}
if (recText == null)
{
continue;
}
else if (recText.Length > 0)
{
sw.Write("{1}{0}", recText, Configuration.FileHeaderConfiguration.HasHeaderRecord ? Configuration.EOLDelimiter : "");
continue;
}
try
{
var config = GetConfiguration(record.GetType());
if (config == null)
{
throw new ChoParserException("No writer found to write record.");
}
if (config.GetType() == typeof(ChoCSVRecordConfiguration))
{
recText = ChoCSVWriter.ToText(record, config as ChoCSVRecordConfiguration, Configuration.Encoding, Configuration.BufferSize);
}
else if (config.GetType() == typeof(ChoFixedLengthRecordConfiguration))
{
recText = ChoFixedLengthWriter.ToText(record, config as ChoFixedLengthRecordConfiguration, Configuration.Encoding, Configuration.BufferSize);
}
else
{
throw new ChoParserException("Unsupported record found to write.");
}
if (recText != null)
{
if (_index == 1)
{
sw.Write("{1}{0}", recText, Configuration.FileHeaderConfiguration.HasHeaderRecord ? Configuration.EOLDelimiter : "");
}
else
{
sw.Write("{1}{0}", recText, Configuration.EOLDelimiter);
}
if (!RaiseAfterRecordWrite(record, _index, recText))
{
yield break;
}
}
}
catch (ChoParserException pEx)
{
throw new ChoParserException($"Failed to write line for '{recType}' object.", pEx);
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordWriteError(record, _index, recText, ex))
{
throw new ChoWriterException($"Failed to write line for '{recType}' object.", ex);
}
}
else
{
throw new ChoWriterException($"Failed to write line for '{recType}' object.", ex);
}
}
}
}
yield return(record);
if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0)
{
if (RaisedRowsWritten(_index))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
yield break;
}
}
}
}
finally
{
System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo;
}
RaiseEndWrite(sw);
}
public IEnumerable <T> WriteTo <T>(object writer, IEnumerable <object> records, Func <object, bool> predicate = null)
{
Configuration.Init();
if (records == null)
{
yield break;
}
var recEnum = records.GetEnumerator();
object notNullRecord = GetFirstNotNullRecord(recEnum);
if (notNullRecord == null)
{
yield break;
}
DiscoverKnownTypes(notNullRecord);
StreamWriter sw = null;
if (writer is Lazy <StreamWriter> )
{
var lsw = writer as Lazy <StreamWriter>;
ChoGuard.ArgumentNotNull(lsw, "StreamWriter");
_sw = sw = lsw.Value;
if (!Configuration.UseAvroSerializer)
{
if (_avroWriter == null)
{
_avroWriter = new SequentialWriter <T>(CreateAvroWriter <T>(sw), Configuration.SyncNumberOfObjects);
}
}
else
{
_avroSerializer = CreateAvroSerializer <T>();
}
}
else
{
_avroWriter = writer as IAvroWriter <T>;
if (_avroWriter == null)
{
throw new ChoParserException("Missing valid writer object passed.");
}
}
if (!BeginWrite.Value)
{
yield break;
}
CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture;
System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture;
object recOutput = null;
bool abortRequested = false;
try
{
foreach (object record in GetRecords(recEnum))
{
_index++;
if (TraceSwitch.TraceVerbose)
{
if (record is IChoETLNameableObject)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index));
}
}
recOutput = record;
if (record != null)
{
if (predicate == null || predicate(record))
{
if (!RaiseBeforeRecordWrite(record, _index, ref recOutput))
{
yield break;
}
if (recOutput == null)
{
continue;
}
else if (!(recOutput is T))
{
continue;
}
try
{
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
record.DoObjectLevelValidation(Configuration, Configuration.AvroRecordFieldConfigurations);
}
if (recOutput is T)
{
if (recOutput is ChoDynamicObject)
{
recOutput = new Dictionary <string, object>((ChoDynamicObject)recOutput);
}
if (_sw != null)
{
if (Configuration.UseAvroSerializer)
{
IAvroSerializer <T> avroSerializer = _avroSerializer as IAvroSerializer <T>;
avroSerializer.Serialize(sw.BaseStream, (T)(recOutput as object));
}
else
{
SequentialWriter <T> avroWriter = _avroWriter as SequentialWriter <T>;
avroWriter.Write((T)(recOutput as object));
}
}
else
{
SequentialWriter <T> avroWriter = _avroWriter as SequentialWriter <T>;
avroWriter.Write((T)(recOutput as object));
}
if (!RaiseAfterRecordWrite(record, _index, recOutput))
{
yield break;
}
}
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ref ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordWriteError(record, _index, recOutput, ex))
{
throw;
}
else
{
//ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
}
}
else
{
throw;
}
}
}
}
yield return((T)recOutput);
recOutput = null;
if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0)
{
if (RaisedRowsWritten(_index))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
abortRequested = true;
yield break;
}
}
}
if (!abortRequested && recOutput != null)
{
RaisedRowsWritten(_index, true);
}
}
finally
{
System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo;
}
}