public static TValue GetValue <TValue>(this IDictionary <string, TValue> dict, string key, bool ignoreCase = false, CultureInfo culture = null, TValue defaultValue = default(TValue))
{
ChoGuard.ArgumentNotNull(dict, "Dictionary");
if (culture == null)
{
culture = System.Threading.Thread.CurrentThread.CurrentCulture;
}
string cultureSpecificKeyName = dict.Keys.Where(i => String.Compare(i, key, ignoreCase, culture) == 0).FirstOrDefault();
if (!cultureSpecificKeyName.IsNullOrWhiteSpace())
{
return(dict[cultureSpecificKeyName]);
}
else
{
return(defaultValue);
}
}
public static IEnumerable <T> GetPropetyAttributes <T>(Type type, string propName)
where T : Attribute
{
ChoGuard.ArgumentNotNull(type, "Type");
ChoGuard.ArgumentNotNullOrEmpty(propName, "PropName");
Init(type);
PropertyDescriptor pd = _pdDict[type].Where(pd1 =>
pd1.Name == propName && pd1.Attributes.OfType <T>().Any()).FirstOrDefault();
if (pd == null)
{
return(Enumerable.Empty <T>());
}
else
{
return(GetPropetyAttributes <T>(pd));
}
}
public void Write(DataTable dt)
{
ChoGuard.ArgumentNotNull(dt, "DataTable");
if (Configuration.UseJSONSerialization)
{
Write(dt);
return;
}
DataTable schemaTable = dt;
if (Configuration.JSONRecordFieldConfigurations.IsNullOrEmpty())
{
string colName = null;
Type colType = null;
int startIndex = 0;
int fieldLength = 0;
foreach (DataColumn col in schemaTable.Columns)
{
colName = col.ColumnName;
colType = col.DataType;
//if (!colType.IsSimple()) continue;
var obj = new ChoJSONRecordFieldConfiguration(colName, jsonPath: null);
Configuration.JSONRecordFieldConfigurations.Add(obj);
startIndex += fieldLength;
}
}
Configuration.RootName = dt.TableName.IsNullOrWhiteSpace() ? null : dt.TableName;
foreach (DataRow row in dt.Rows)
{
dynamic expando = new ExpandoObject();
var expandoDic = (IDictionary <string, object>)expando;
foreach (var fc in Configuration.JSONRecordFieldConfigurations)
{
expandoDic.Add(fc.Name, row[fc.Name] == DBNull.Value ? null : row[fc.Name]);
}
Write(expando);
}
}
public static void AddOrUpdateValue <TValue>(this IDictionary <string, TValue> dict, string key, TValue value, bool ignoreCase = false, CultureInfo culture = null)
{
ChoGuard.ArgumentNotNull(dict, "Dictionary");
if (culture == null)
{
culture = System.Threading.Thread.CurrentThread.CurrentCulture;
}
string cultureSpecificKeyName = dict.Keys.Where(i => String.Compare(i, key, ignoreCase, culture) == 0).FirstOrDefault();
if (cultureSpecificKeyName.IsNullOrWhiteSpace())
{
dict.Add(cultureSpecificKeyName, value);
}
else
{
dict[cultureSpecificKeyName] = value;
}
}
public void Write(IDataReader dr)
{
ChoGuard.ArgumentNotNull(dr, "DataReader");
DataTable schemaTable = dr.GetSchemaTable();
dynamic expando = new ExpandoObject();
var expandoDic = (IDictionary <string, object>)expando;
Configuration.UseNestedKeyFormat = false;
int ordinal = 0;
if (Configuration.CSVRecordFieldConfigurations.IsNullOrEmpty())
{
string colName = null;
Type colType = null;
foreach (DataRow row in schemaTable.Rows)
{
colName = row["ColumnName"].CastTo <string>();
colType = row["DataType"] as Type;
//if (!colType.IsSimple()) continue;
Configuration.CSVRecordFieldConfigurations.Add(new ChoCSVRecordFieldConfiguration(colName, ++ordinal)
{
FieldType = colType
});
}
}
var ordinals = Configuration.CSVRecordFieldConfigurations.ToDictionary(c => c.Name, c => dr.HasColumn(c.Name) ? dr.GetOrdinal(c.Name) : -1);
while (dr.Read())
{
expandoDic.Clear();
foreach (var fc in ordinals)
{
expandoDic.Add(fc.Key, fc.Value == -1 ? null : dr[fc.Value]);
}
Write(expando);
}
}
public void Write(IDataReader dr)
{
ChoGuard.ArgumentNotNull(dr, "DataReader");
DataTable schemaTable = dr.GetSchemaTable();
dynamic expando = new ExpandoObject();
var expandoDic = (IDictionary <string, object>)expando;
Configuration.UseNestedKeyFormat = false;
//int ordinal = 0;
if (Configuration.FixedLengthRecordFieldConfigurations.IsNullOrEmpty())
{
string colName = null;
Type colType = null;
int startIndex = 0;
int fieldLength = 0;
foreach (DataRow row in schemaTable.Rows)
{
colName = row["ColumnName"].CastTo <string>();
colType = row["DataType"] as Type;
//if (!colType.IsSimple()) continue;
fieldLength = ChoFixedLengthFieldDefaultSizeConfiguation.Instance.GetSize(colType);
var obj = new ChoFixedLengthRecordFieldConfiguration(colName, startIndex, fieldLength);
Configuration.FixedLengthRecordFieldConfigurations.Add(obj);
startIndex += fieldLength;
}
}
while (dr.Read())
{
expandoDic.Clear();
foreach (var fc in Configuration.FixedLengthRecordFieldConfigurations)
{
expandoDic.Add(fc.Name, dr[fc.Name]);
}
Write(expando);
}
}
public static string JsonSerialize(object target, Encoding encoding = null)
{
ChoGuard.ArgumentNotNull(target, "Target");
encoding = encoding == null ? Encoding.UTF8 : encoding;
StringBuilder JsonString = new StringBuilder();
using (MemoryStream ms = new MemoryStream())
{
using (var writer = JsonReaderWriterFactory.CreateJsonWriter(
ms, encoding, true, true, " "))
{
DataContractJsonSerializer serializer = new DataContractJsonSerializer(target.GetType(), jsonSettings);
serializer.WriteObject(writer, target);
writer.Flush();
byte[] json = ms.ToArray();
return(encoding.GetString(json, 0, json.Length));
}
}
}
private static Assembly GetWebApplicationAssembly(HttpContext context)
{
ChoGuard.ArgumentNotNull(context, "context");
IHttpHandler handler = context.CurrentHandler;
if (handler == null)
{
return(null);
}
Type type = handler.GetType();
while (type != null && type != typeof(object) && type.Namespace == AspNetNamespace)
{
type = type.BaseType;
}
return(type.Assembly);
}
public override IEnumerable <object> AsEnumerable(object source, Func <object, bool?> filterFunc = null)
{
ParquetReader sr = source as ParquetReader;
ChoGuard.ArgumentNotNull(sr, "ParquetReader");
InitializeRecordConfiguration(Configuration);
if (!RaiseBeginLoad(sr))
{
yield break;
}
foreach (var item in AsEnumerable(ReadObjects(sr), TraceSwitch, filterFunc))
{
yield return(item);
}
RaiseEndLoad(sr);
}
public ChoJObjectWriter(string propName, ChoJObjectWriter writer)
{
ChoGuard.ArgumentNotNull(writer, nameof(writer));
ChoGuard.ArgumentNotNull(propName, nameof(propName));
_writer = writer._writer;
_options = writer._options;
_formatting = writer.Formatting;
_writeStartObject = new Lazy <bool>(() =>
{
_writer.WriteStartObject();
return(true);
});
var _ = writer._writeStartObject.Value;
_writer.WritePropertyName(propName);
var x = _writeStartObject.Value;
}
public static void Write(ChoFixedLengthWriter <dynamic> w, DataTable dt)
{
ChoGuard.ArgumentNotNull(w, "Writer");
ChoGuard.ArgumentNotNull(dt, "DataTable");
DataTable schemaTable = dt;
w.Configuration.UseNestedKeyFormat = false;
if (w.Configuration.FixedLengthRecordFieldConfigurations.IsNullOrEmpty())
{
string colName = null;
Type colType = null;
int startIndex = 0;
int fieldLength = 0;
foreach (DataColumn col in schemaTable.Columns)
{
colName = col.ColumnName;
colType = col.DataType;
//if (!colType.IsSimple()) continue;
fieldLength = ChoFixedLengthFieldDefaultSizeConfiguation.Instance.GetSize(colType);
var obj = new ChoFixedLengthRecordFieldConfiguration(colName, startIndex, fieldLength);
w.Configuration.FixedLengthRecordFieldConfigurations.Add(obj);
startIndex += fieldLength;
}
}
foreach (DataRow row in dt.Rows)
{
dynamic expando = new ExpandoObject();
var expandoDic = (IDictionary <string, object>)expando;
foreach (var fc in w.Configuration.FixedLengthRecordFieldConfigurations)
{
expandoDic.Add(fc.Name, row[fc.Name]);
}
w.Write(expando);
}
}
public void Write(DataTable dt)
{
ChoGuard.ArgumentNotNull(dt, "DataTable");
DataTable schemaTable = dt;
dynamic expando = new ExpandoObject();
var expandoDic = (IDictionary <string, object>)expando;
Configuration.UseNestedKeyFormat = false;
int ordinal = 0;
if (Configuration.CSVRecordFieldConfigurations.IsNullOrEmpty())
{
string colName = null;
Type colType = null;
foreach (DataColumn col in schemaTable.Columns)
{
colName = col.ColumnName;
colType = col.DataType;
//if (!colType.IsSimple()) continue;
Configuration.CSVRecordFieldConfigurations.Add(new ChoCSVRecordFieldConfiguration(colName, ++ordinal)
{
FieldType = colType
});
}
}
foreach (DataRow row in dt.Rows)
{
expandoDic.Clear();
foreach (var fc in Configuration.CSVRecordFieldConfigurations)
{
expandoDic.Add(fc.Name, row[fc.Name]);
}
Write(expando);
}
}
public ChoFixedLengthRecordWriter(Type recordType, ChoFixedLengthRecordConfiguration configuration) : base(recordType)
{
ChoGuard.ArgumentNotNull(configuration, "Configuration");
Configuration = configuration;
_callbackFileHeaderArrange = ChoMetadataObjectCache.CreateMetadataObject <IChoNotifyFileHeaderArrange>(recordType);
_callbackFileHeaderWrite = ChoMetadataObjectCache.CreateMetadataObject <IChoNotifyFileHeaderWrite>(recordType);
_callbackRecordWrite = ChoMetadataObjectCache.CreateMetadataObject <IChoNotifyRecordWrite>(recordType);
_callbackFileWrite = ChoMetadataObjectCache.CreateMetadataObject <IChoNotifyFileWrite>(recordType);
_callbackRecordFieldWrite = ChoMetadataObjectCache.CreateMetadataObject <IChoNotifyRecordFieldWrite>(recordType);
_recBuffer = new Lazy <List <object> >(() =>
{
if (Writer != null)
{
var b = Writer.Context.ContainsKey("RecBuffer") ? Writer.Context.RecBuffer : null;
if (b == null)
{
Writer.Context.RecBuffer = new List <object>();
}
return(Writer.Context.RecBuffer);
}
else
{
return(new List <object>());
}
});
BeginWrite = new Lazy <bool>(() =>
{
TextWriter sw = _sw as TextWriter;
if (sw != null)
{
return(RaiseBeginWrite(sw));
}
return(false);
});
//Configuration.Validate();
}
public void Write(IDataReader dr)
{
throw new NotSupportedException();
ChoGuard.ArgumentNotNull(dr, "DataReader");
DataTable schemaTable = dr.GetSchemaTable();
dynamic expando = new ExpandoObject();
var expandoDic = (IDictionary <string, object>)expando;
Configuration.UseNestedKeyFormat = false;
if (Configuration.AvroRecordFieldConfigurations.IsNullOrEmpty())
{
string colName = null;
Type colType = null;
foreach (DataRow row in schemaTable.Rows)
{
colName = row["ColumnName"].CastTo <string>();
colType = row["DataType"] as Type;
//if (!colType.IsSimple()) continue;
Configuration.AvroRecordFieldConfigurations.Add(new ChoBSONRecordFieldConfiguration(colName)
{
FieldType = colType
});
}
}
while (dr.Read())
{
expandoDic.Clear();
foreach (var fc in Configuration.AvroRecordFieldConfigurations)
{
expandoDic.Add(fc.Name, dr[fc.Name]);
}
Write(expando);
}
}
public object GetDefaultValue(MemberInfo mi)
{
ChoGuard.ArgumentNotNull(mi, "MemberInfo");
Type declaringType = mi.ReflectedType;
if (!typeof(ChoRecord).IsAssignableFrom(declaringType) ||
declaringType.IsDynamicRecord())
{
return(null);
}
InitDefaults(declaringType);
if (!_defaultsCache.ContainsKey(declaringType) ||
!_defaultsCache[declaringType].ContainsKey(mi.Name))
{
return(null);
}
return(_defaultsCache[declaringType][mi.Name]);
}
public object[] GetConverterParams(MemberInfo mi)
{
ChoGuard.ArgumentNotNull(mi, "MemberInfo");
Type declaringType = mi.ReflectedType;
if (!typeof(ChoRecord).IsAssignableFrom(declaringType) ||
declaringType.IsDynamicRecord())
{
return(null);
}
InitConverterParams(declaringType);
if (!_converterParams.ContainsKey(declaringType) ||
!_converterParams[declaringType].ContainsKey(mi.Name))
{
return(null);
}
return(_converterParams[declaringType][mi.Name]);
}
public bool IsRequired(MemberInfo mi)
{
ChoGuard.ArgumentNotNull(mi, "MemberInfo");
Type declaringType = mi.ReflectedType;
if (!typeof(ChoRecord).IsAssignableFrom(declaringType) ||
declaringType.IsDynamicRecord())
{
return(false);
}
InitIsRequireds(declaringType);
if (!_isReqCache.ContainsKey(declaringType) ||
!_isReqCache[declaringType].ContainsKey(mi.Name))
{
return(false);
}
return(_isReqCache[declaringType][mi.Name]);
}
public ChoEnumerableDataReader(IEnumerable collection, IChoDeferedObjectMemberDiscoverer dom)
{
ChoGuard.ArgumentNotNull(collection, "Collection");
ChoGuard.ArgumentNotNull(dom, "DeferedObjectMemberDiscoverer");
_isDeferred = true;
_type = GetElementType(collection);
dom.MembersDiscovered += (o, e) =>
{
_dynamicMembersInfo = e.Value.ToArray();
SetFields(_type, e.Value.ToArray());
};
_enumerator = collection.GetEnumerator();
_firstElementExists = _enumerator.MoveNext();
if (_enumerator.Current is ChoDynamicObject && ((ChoDynamicObject)_enumerator.Current).IsHeaderOnlyObject)
{
_firstElementExists = false;
}
}
public static IEnumerable <string> ReadLines(this StreamReader reader, char delimeter)
{
ChoGuard.ArgumentNotNull(reader, "TextReader");
List <char> chars = new List <char>();
while (reader.Peek() >= 0)
{
char c = (char)reader.Read();
if (c == delimeter)
{
yield return(new String(chars.ToArray()));
chars.Clear();
continue;
}
chars.Add(c);
}
}
public static void RegisterConverters(Type type, TypeConverter[] typeConverters)
{
ChoGuard.ArgumentNotNull(type, "Type");
lock (_typeMemberTypeConverterCacheLockObject)
{
if (typeConverters == null)
{
typeConverters = EmptyTypeConverters;
}
if (_typeTypeConverterCache.ContainsKey(type))
{
_typeTypeConverterCache[type] = typeConverters;
}
else
{
_typeTypeConverterCache.Add(type, typeConverters);
}
}
}
public static void RegisterConverters(MemberInfo memberInfo, TypeConverter[] typeConverters)
{
ChoGuard.ArgumentNotNull(memberInfo, "MemberInfo");
lock (_typeMemberTypeConverterCacheLockObject)
{
if (typeConverters == null)
{
typeConverters = EmptyTypeConverters;
}
if (_typeMemberTypeConverterCache.ContainsKey(memberInfo))
{
_typeMemberTypeConverterCache[memberInfo] = typeConverters;
}
else
{
_typeMemberTypeConverterCache.Add(memberInfo, typeConverters);
}
}
}
public ChoAvroRecordWriter(Type recordType, ChoBSONRecordConfiguration configuration) : base(recordType)
{
ChoGuard.ArgumentNotNull(configuration, "Configuration");
Configuration = configuration;
_callbackFileHeaderArrange = ChoMetadataObjectCache.CreateMetadataObject <IChoNotifyFileHeaderArrange>(recordType);
_callbackRecordWrite = ChoMetadataObjectCache.CreateMetadataObject <IChoNotifyRecordWrite>(recordType);
_callbackFileWrite = ChoMetadataObjectCache.CreateMetadataObject <IChoNotifyFileWrite>(recordType);
_callbackRecordFieldWrite = ChoMetadataObjectCache.CreateMetadataObject <IChoNotifyRecordFieldWrite>(recordType);
System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture;
_recBuffer = new Lazy <List <object> >(() =>
{
if (Writer != null)
{
var b = Writer.Context.ContainsKey("RecBuffer") ? Writer.Context.RecBuffer : null;
if (b == null)
{
Writer.Context.RecBuffer = new List <object>();
}
return(Writer.Context.RecBuffer);
}
else
{
return(new List <object>());
}
}, true);
BeginWrite = new Lazy <bool>(() =>
{
if (_sw != null)
{
return(RaiseBeginWrite(_sw));
}
return(false);
});
//Configuration.Validate();
}
public static Dictionary <string, object> ToDictionary(this object target)
{
ChoGuard.ArgumentNotNull(target, "Target");
if (target is IDictionary <string, object> )
{
return((Dictionary <string, object>)target);
}
if (target is IDictionary)
{
Dictionary <string, object> dict1 = new Dictionary <string, object>();
foreach (var kvp in ((IDictionary)target).Keys)
{
dict1.Add(kvp.ToNString(), ((IDictionary)target)[kvp]);
}
return(dict1);
}
if (target is IEnumerable <KeyValuePair <string, object> > )
{
return(new List <KeyValuePair <string, object> >(target as IEnumerable <KeyValuePair <string, object> >).ToDictionary(x => x.Key, x => x.Value));
}
if (target is IEnumerable <Tuple <string, object> > )
{
return(new List <Tuple <string, object> >(target as IEnumerable <Tuple <string, object> >).ToDictionary(x => x.Item1, x => x.Item2));
}
if (target is IList)
{
return(((IList)(target)).OfType <object>().Select((item, index) => new KeyValuePair <string, object>("Column_{0}".FormatString(index), item)).ToDictionary(kvp => kvp.Key, kvp => kvp.Value));
}
Dictionary <string, object> dict = new Dictionary <string, object>();
foreach (PropertyDescriptor pd in ChoTypeDescriptor.GetProperties(target.GetType()))
{
dict.Add(pd.Name, ChoType.GetPropertyValue(target, pd.Name));
}
return(dict);
}
public void Write(IDataReader dr)
{
ChoGuard.ArgumentNotNull(dr, "DataReader");
DataTable schemaTable = dr.GetSchemaTable();
dynamic expando = new ExpandoObject();
var expandoDic = (IDictionary <string, object>)expando;
int ordinal = 0;
if (Configuration.CSVRecordFieldConfigurations.IsNullOrEmpty())
{
string colName = null;
Type colType = null;
foreach (DataRow row in schemaTable.Rows)
{
colName = row["ColumnName"].CastTo <string>();
colType = row["DataType"] as Type;
//if (!colType.IsSimple()) continue;
Configuration.CSVRecordFieldConfigurations.Add(new ChoCSVRecordFieldConfiguration(colName, ++ordinal)
{
FieldType = colType
});
}
}
while (dr.Read())
{
expandoDic.Clear();
foreach (var fc in Configuration.CSVRecordFieldConfigurations)
{
expandoDic.Add(fc.Name, dr[fc.Name]);
}
Write(expando);
}
}
public ChoYamlRecordWriter(Type recordType, ChoYamlRecordConfiguration configuration) : base(recordType, true)
{
ChoGuard.ArgumentNotNull(configuration, "Configuration");
Configuration = configuration;
_callbackRecordWrite = ChoMetadataObjectCache.CreateMetadataObject <IChoNotifyRecordWrite>(recordType);
_callbackFileWrite = ChoMetadataObjectCache.CreateMetadataObject <IChoNotifyFileWrite>(recordType);
_callbackRecordFieldWrite = ChoMetadataObjectCache.CreateMetadataObject <IChoNotifyRecordFieldWrite>(recordType);
_callbackRecordSeriablizable = ChoMetadataObjectCache.CreateMetadataObject <IChoRecordFieldSerializable>(recordType);
System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture;
//_recBuffer = new Lazy<List<object>>(() =>
//{
// if (Writer != null)
// {
// var b = Writer.Context.ContainsKey("RecBuffer") ? Writer.Context.RecBuffer : null;
// if (b == null)
// Writer.Context.RecBuffer = new List<object>();
// return Writer.Context.RecBuffer;
// }
// else
// return new List<object>();
//}, true);
//Configuration.Validate();
BeginWrite = new Lazy <bool>(() =>
{
TextWriter sw = _sw as TextWriter;
if (sw != null)
{
return(RaiseBeginWrite(sw));
}
return(false);
});
}
private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null)
{
TraceSwitch = traceSwitch;
TextReader sr = source as TextReader;
ChoGuard.ArgumentNotNull(sr, "TextReader");
if (sr is StreamReader)
{
((StreamReader)sr).Seek(0, SeekOrigin.Begin);
}
if (!RaiseBeginLoad(sr))
{
yield break;
}
string[] commentTokens = Configuration.Comments;
bool? skip = false;
bool isRecordStartFound = false;
bool isRecordEndFound = false;
long seekOriginPos = sr is StreamReader ? ((StreamReader)sr).BaseStream.Position : 0;
List <string> headers = new List <string>();
Tuple <long, string> lastLine = null;
List <Tuple <long, string> > recLines = new List <Tuple <long, string> >();
long recNo = 0;
int loopCount = Configuration.AutoDiscoverColumns && Configuration.KVPRecordFieldConfigurations.Count == 0 ? 2 : 1;
bool isHeaderFound = loopCount == 1;
bool IsHeaderLoaded = false;
Tuple <long, string> pairIn;
bool abortRequested = false;
for (int i = 0; i < loopCount; i++)
{
if (i == 1)
{
if (sr is StreamReader)
{
((StreamReader)sr).Seek(seekOriginPos, SeekOrigin.Begin);
}
TraceSwitch = traceSwitch;
}
else
{
TraceSwitch = ChoETLFramework.TraceSwitchOff;
}
lastLine = null;
recLines.Clear();
isRecordEndFound = false;
isRecordStartFound = false;
using (ChoPeekEnumerator <Tuple <long, string> > e = new ChoPeekEnumerator <Tuple <long, string> >(
new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar, Configuration.MayContainEOLInData)).ToEnumerable(),
(pair) =>
{
//bool isStateAvail = IsStateAvail();
skip = false;
//if (isStateAvail)
//{
// if (!IsStateMatches(item))
// {
// skip = filterFunc != null ? filterFunc(item) : false;
// }
// else
// skip = true;
//}
//else
// skip = filterFunc != null ? filterFunc(item) : false;
if (skip == null)
{
return(null);
}
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine);
if (!skip.Value)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1));
}
}
if (skip.Value)
{
return(skip);
}
//if (!(sr.BaseStream is MemoryStream))
// ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1));
//if (Task != null)
// return !IsStateNOTExistsOrNOTMatch(item);
if (pair.Item2.IsNullOrWhiteSpace())
{
if (!Configuration.IgnoreEmptyLine)
{
throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pair.Item1));
}
else
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1));
}
return(true);
}
}
if (commentTokens != null && commentTokens.Length > 0)
{
foreach (string comment in commentTokens)
{
if (!pair.Item2.IsNull() && pair.Item2.StartsWith(comment, StringComparison.Ordinal)) //, true, Configuration.Culture))
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1));
}
return(true);
}
}
}
if (!_configCheckDone)
{
Configuration.Validate(null);
var dict = Configuration.KVPRecordFieldConfigurations.ToDictionary(i1 => i1.Name, i1 => i1.FieldType == null ? null : i1.FieldType);
RaiseMembersDiscovered(dict);
Configuration.UpdateFieldTypesIfAny(dict);
_configCheckDone = true;
}
return(false);
}))
{
while (true)
{
pairIn = e.Peek;
if (!isRecordStartFound)
{
if (pairIn == null)
{
break;
}
lastLine = null;
recLines.Clear();
isRecordEndFound = false;
isRecordStartFound = true;
if (!Configuration.RecordStart.IsNullOrWhiteSpace())
{
//Move to record start
while (!(Configuration.IsRecordStartMatch(e.Peek.Item2)))
{
e.MoveNext();
if (e.Peek == null)
{
break;
}
}
}
if (e.Peek != null)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Record start found at [{0}] line...".FormatString(e.Peek.Item1));
}
e.MoveNext();
continue;
}
else
{
string recordEnd = !Configuration.RecordEnd.IsNullOrWhiteSpace() ? Configuration.RecordEnd : Configuration.RecordStart;
if (!recordEnd.IsNullOrWhiteSpace())
{
if (e.Peek == null)
{
if (Configuration.RecordEnd.IsNullOrWhiteSpace())
{
isRecordEndFound = true;
}
else
{
break;
}
}
else
{
//Move to record start
if (Configuration.IsRecordEndMatch(e.Peek.Item2))
{
isRecordEndFound = true;
isRecordStartFound = false;
if (e.Peek != null)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Record end found at [{0}] line...".FormatString(e.Peek.Item1));
}
}
}
}
else if (e.Peek == null)
{
isRecordEndFound = true;
}
if (!isHeaderFound)
{
//if (isRecordEndFound && headers.Count == 0)
//{
// //throw new ChoParserException("Unexpected EOF found.");
//}
if (!isRecordEndFound)
{
e.MoveNext();
if (e.Peek != null)
{
//If line empty or line continuation, skip
if (pairIn.Item2.IsNullOrWhiteSpace() || IsLineContinuationCharFound(pairIn.Item2)) //.Item2[0] == ' ' || pairIn.Item2[0] == '\t')
{
}
else
{
string header = ToKVP(pairIn.Item1, pairIn.Item2).Key;
if (!header.IsNullOrWhiteSpace())
{
headers.Add(header);
}
}
}
}
else
{
Configuration.Validate(headers.ToArray());
isHeaderFound = true;
isRecordEndFound = false;
IsHeaderLoaded = true;
break;
}
}
else
{
if (!IsHeaderLoaded)
{
Configuration.Validate(new string[] { });
IsHeaderLoaded = true;
}
if (isRecordEndFound && recLines.Count == 0)
{
//throw new ChoParserException("Unexpected EOF found.");
}
else if (!isRecordEndFound)
{
e.MoveNext();
if (e.Peek != null)
{
//If line empty or line continuation, skip
if (pairIn.Item2.IsNullOrWhiteSpace())
{
if (!Configuration.IgnoreEmptyLine)
{
throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pairIn.Item1));
}
else
{
Tuple <long, string> t = new Tuple <long, string>(lastLine.Item1, lastLine.Item2 + Configuration.EOLDelimiter);
recLines.RemoveAt(recLines.Count - 1);
recLines.Add(t);
}
}
else if (IsLineContinuationCharFound(pairIn.Item2)) //pairIn.Item2[0] == ' ' || pairIn.Item2[0] == '\t')
{
if (lastLine == null)
{
throw new ChoParserException("Unexpected line continuation found at {0} location.".FormatString(pairIn.Item1));
}
else
{
Tuple <long, string> t = new Tuple <long, string>(lastLine.Item1, lastLine.Item2 + Configuration.EOLDelimiter + pairIn.Item2);
recLines.RemoveAt(recLines.Count - 1);
recLines.Add(t);
}
}
else
{
lastLine = pairIn;
recLines.Add(pairIn);
}
}
}
else
{
object rec = Configuration.IsDynamicObject ? new ChoDynamicObject(new Dictionary <string, object>(Configuration.FileHeaderConfiguration.StringComparer))
{
ThrowExceptionIfPropNotExists = true,
AlternativeKeys = Configuration.AlternativeKeys
} : Activator.CreateInstance(RecordType);
if (!LoadLines(new Tuple <long, List <Tuple <long, string> > >(++recNo, recLines), ref rec))
{
yield break;
}
isRecordStartFound = false;
//StoreState(e.Current, rec != null);
if (!Configuration.RecordEnd.IsNullOrWhiteSpace())
{
e.MoveNext();
}
if (rec == null)
{
continue;
}
yield return(rec);
if (Configuration.NotifyAfter > 0 && recNo % Configuration.NotifyAfter == 0)
{
if (RaisedRowsLoaded(recNo))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
abortRequested = true;
yield break;
}
}
if (e.Peek == null)
{
break;
}
}
}
}
}
}
}
if (!abortRequested)
{
RaisedRowsLoaded(recNo, true);
}
RaiseEndLoad(sr);
}
public override IEnumerable <object> WriteTo(object writer, IEnumerable <object> records, Func <object, bool> predicate = null)
{
TextWriter sw = writer as TextWriter;
ChoGuard.ArgumentNotNull(sw, "TextWriter");
if (records == null)
{
yield break;
}
if (!RaiseBeginWrite(sw))
{
yield break;
}
CultureInfo prevCultureInfo = System.Threading.Thread.CurrentThread.CurrentCulture;
System.Threading.Thread.CurrentThread.CurrentCulture = Configuration.Culture;
string recText = String.Empty;
long recCount = 0;
var recEnum = records.GetEnumerator();
try
{
object notNullRecord = GetFirstNotNullRecord(recEnum);
if (notNullRecord == null)
{
yield break;
}
if (Configuration.IsDynamicObject)
{
if (Configuration.MaxScanRows > 0)
{
List <string> fns = new List <string>();
foreach (object record1 in GetRecords(recEnum))
{
recCount++;
if (record1 != null)
{
if (recCount <= Configuration.MaxScanRows)
{
if (!record1.GetType().IsDynamicType())
{
throw new ChoParserException("Invalid record found.");
}
_recBuffer.Value.Add(record1);
fns = fns.Union(GetFields(record1)).ToList();
if (recCount == Configuration.MaxScanRows)
{
Configuration.Validate(fns.ToArray());
WriteHeaderLine(sw);
_configCheckDone = true;
break;
}
}
}
}
}
}
foreach (object record in GetRecords(recEnum))
{
_index++;
if (TraceSwitch.TraceVerbose)
{
if (record is IChoETLNameableObject)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(((IChoETLNameableObject)record).Name));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Writing [{0}] object...".FormatString(_index));
}
}
recText = String.Empty;
if (record != null)
{
if (predicate == null || predicate(record))
{
//Discover and load FixedLength columns from first record
if (!_configCheckDone)
{
if (notNullRecord != null)
{
string[] fieldNames = GetFields(notNullRecord);
Configuration.Validate(fieldNames);
WriteHeaderLine(sw);
_configCheckDone = true;
}
}
//Check record
if (record != null)
{
Type rt = record.GetType().ResolveType();
if (Configuration.IsDynamicObject)
{
if (ElementType != null)
{
}
else if (!rt.IsDynamicType())
{
throw new ChoWriterException("Invalid record found.");
}
}
else
{
if (rt != Configuration.RecordType)
{
throw new ChoWriterException("Invalid record found.");
}
}
}
if (!RaiseBeforeRecordWrite(record, _index, ref recText))
{
yield break;
}
if (recText == null)
{
continue;
}
else if (recText.Length > 0)
{
sw.Write("{1}{0}", recText, _hadHeaderWritten ? Configuration.EOLDelimiter : "");
continue;
}
try
{
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
record.DoObjectLevelValidation(Configuration, Configuration.FixedLengthRecordFieldConfigurations);
}
if (ToText(_index, record, out recText))
{
if (_index == 1)
{
sw.Write("{1}{0}", recText, _hadHeaderWritten ? Configuration.EOLDelimiter : "");
}
else
{
sw.Write("{1}{0}", recText, Configuration.EOLDelimiter);
}
if (!RaiseAfterRecordWrite(record, _index, recText))
{
yield break;
}
}
}
//catch (ChoParserException)
//{
// throw;
//}
catch (Exception ex)
{
ChoETLFramework.HandleException(ref ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordWriteError(record, _index, recText, ex))
{
throw;
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
}
}
else
{
throw;
}
}
}
}
yield return(record);
if (Configuration.NotifyAfter > 0 && _index % Configuration.NotifyAfter == 0)
{
if (RaisedRowsWritten(_index))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
yield break;
}
}
}
}
finally
{
System.Threading.Thread.CurrentThread.CurrentCulture = prevCultureInfo;
}
RaiseEndWrite(sw);
}
internal ChoBSONRecordFieldConfigurationMap(ChoBSONRecordFieldConfiguration config)
{
ChoGuard.ArgumentNotNull(config, nameof(config));
_config = config;
}
public static IEnumerable <string> ReadLines(this TextReader reader, string EOLDelimiter = null, char quoteChar = ChoCharEx.NUL, bool mayContainEOLInData = false, int maxLineSize = 32768)
{
ChoGuard.ArgumentNotNull(reader, "TextReader");
EOLDelimiter = EOLDelimiter ?? Environment.NewLine;
if (!mayContainEOLInData)
{
if (EOLDelimiter == Environment.NewLine ||
(EOLDelimiter.Length == 1 && EOLDelimiter[0] == '\r') ||
(EOLDelimiter.Length == 1 && EOLDelimiter[0] == '\n')
)
{
string line;
while ((line = reader.ReadLine()) != null)
{
yield return(line);
}
}
else
{
string line;
while ((line = ReadLineWithFixedNewlineDelimeter(reader as StreamReader, EOLDelimiter, maxLineSize)) != null)
{
yield return(line);
}
}
}
bool isPrevEscape = false;
bool inQuote = false;
List <char> buffer = new List <char>();
char c = ChoCharEx.NUL;
CircularBuffer <char> delim_buffer = new CircularBuffer <char>(EOLDelimiter.Length);
while (reader.Peek() >= 0)
{
isPrevEscape = c == ChoCharEx.Backslash;
c = (char)reader.Read();
delim_buffer.Enqueue(c);
if (quoteChar != ChoCharEx.NUL && quoteChar == c)
{
if (!isPrevEscape)
{
inQuote = !inQuote;
}
else if (reader.Peek() == quoteChar)
{
inQuote = false;
}
}
if (!inQuote)
{
if (delim_buffer.ToString() == EOLDelimiter)
{
if (buffer.Count > 0)
{
string x = new String(buffer.ToArray());
yield return(x.Substring(0, x.Length - (EOLDelimiter.Length - 1)));
buffer.Clear();
}
continue;
}
}
buffer.Add(c);
if (buffer.Count > maxLineSize)
{
throw new ApplicationException("Large line found. Check and correct the end of line delimiter.");
}
}
if (buffer.Count > 0)
{
yield return(new String(buffer.ToArray()));
}
else
{
yield break;
}
}
private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null)
{
TraceSwitch = traceSwitch;
TextReader sr = source as TextReader;
ChoGuard.ArgumentNotNull(sr, "TextReader");
if (sr is StreamReader)
{
((StreamReader)sr).Seek(0, SeekOrigin.Begin);
}
if (Configuration.RecordSelector == null)
{
throw new ChoRecordConfigurationException("Missing record selector.");
}
if (!RaiseBeginLoad(sr))
{
yield break;
}
string[] commentTokens = Configuration.Comments;
bool? skip = false;
bool _headerFound = false;
bool? skipUntil = true;
bool? doWhile = true;
using (ChoPeekEnumerator <Tuple <long, string> > e = new ChoPeekEnumerator <Tuple <long, string> >(
new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar, Configuration.MayContainEOLInData, Configuration.MaxLineSize)).ToEnumerable(),
(pair) =>
{
//bool isStateAvail = IsStateAvail();
skip = false;
if (skipUntil != null)
{
if (skipUntil.Value)
{
skipUntil = RaiseSkipUntil(pair);
if (skipUntil == null)
{
}
else
{
skip = skipUntil.Value;
}
}
}
//if (isStateAvail)
//{
// if (!IsStateMatches(item))
// {
// skip = filterFunc != null ? filterFunc(item) : false;
// }
// else
// skip = true;
//}
//else
// skip = filterFunc != null ? filterFunc(item) : false;
if (skip == null)
{
return(null);
}
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine);
if (!skip.Value)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1));
}
}
if (skip.Value)
{
return(skip);
}
//if (!(sr.BaseStream is MemoryStream))
// ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1));
//if (Task != null)
// return !IsStateNOTExistsOrNOTMatch(item);
if (pair.Item2.IsNullOrWhiteSpace())
{
if (!Configuration.IgnoreEmptyLine)
{
throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pair.Item1));
}
else
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1));
}
return(true);
}
}
if (commentTokens != null && commentTokens.Length > 0)
{
foreach (string comment in commentTokens)
{
if (!pair.Item2.IsNull() && pair.Item2.StartsWith(comment, StringComparison.Ordinal)) //, true, Configuration.Culture))
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1));
}
return(true);
}
}
}
if (!_configCheckDone)
{
Configuration.Validate(pair); // GetHeaders(pair.Item2));
_configCheckDone = true;
}
//Ignore Header if any
if (Configuration.FileHeaderConfiguration.HasHeaderRecord &&
!_headerFound)
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring header line at [{0}]...".FormatString(pair.Item1));
}
_headerFound = true;
return(true);
}
return(false);
}))
{
while (true)
{
Tuple <long, string> pair = e.Peek;
if (pair == null)
{
RaiseEndLoad(sr);
yield break;
}
Type recType = Configuration.RecordSelector(pair.Item2);
if (recType == null)
{
if (Configuration.IgnoreIfNoRecordParserExists)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, $"No record type found for [{pair.Item1}] line to parse.");
}
else
{
throw new ChoParserException($"No record type found for [{pair.Item1}] line to parse.");
}
}
object rec = ChoActivator.CreateInstance(recType);
if (!LoadLine(pair, ref rec))
{
yield break;
}
//StoreState(e.Current, rec != null);
e.MoveNext();
if (rec == null)
{
continue;
}
yield return(rec);
if (Configuration.NotifyAfter > 0 && pair.Item1 % Configuration.NotifyAfter == 0)
{
if (RaisedRowsLoaded(pair.Item1))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
yield break;
}
}
if (doWhile != null)
{
doWhile = RaiseDoWhile(pair);
if (doWhile != null && doWhile.Value)
{
break;
}
}
}
}
}
