public static object CreateInstanceAndDefaultToMembers(this Type type, IDictionary <string, ChoRecordFieldConfiguration> fcs)
{
var obj = ChoActivator.CreateInstance(type);
object defaultValue = null;
foreach (PropertyDescriptor pd in ChoTypeDescriptor.GetProperties(type))
{
try
{
if (!fcs.ContainsKey(pd.Name) || !fcs[pd.Name].IsDefaultValueSpecified)
{
continue;
}
defaultValue = fcs[pd.Name].DefaultValue;
if (defaultValue != null)
{
ChoType.ConvertNSetPropertyValue(obj, pd.Name, defaultValue);
}
}
catch (Exception ex)
{
ChoETLFramework.WriteLog(ChoETLFramework.TraceSwitch.TraceError, "Error while assigning default value '{0}' to '{1}' member. {2}".FormatString(defaultValue, ChoType.GetMemberName(pd), ex.Message));
}
}
return(obj);
}
public static T CreateMetadataObject <T>(Type recordType)
where T : class
{
T callbackRecord = default(T);
try
{
MetadataTypeAttribute attr = recordType.GetCustomAttribute <MetadataTypeAttribute>();
if (attr == null)
{
if (typeof(T).IsAssignableFrom(recordType))
{
callbackRecord = ChoActivator.CreateInstance(recordType) as T;
}
}
else
{
if (attr.MetadataClassType != null && typeof(T).IsAssignableFrom(attr.MetadataClassType))
{
callbackRecord = ChoActivator.CreateInstance(attr.MetadataClassType) as T;
}
}
}
catch
{
}
return(callbackRecord);
}
public static object[] GetTypeConvertersForType(Type objType)
{
if (objType == null)
{
return(null);
}
lock (_typeMemberTypeConverterCacheLockObject)
{
if (_typeTypeConverterCache.ContainsKey(objType))
{
return(_typeTypeConverterCache[objType]);
}
else
{
if (!_typeTypeConverterCache.ContainsKey(objType))
{
Type type = objType;
if (ChoTypeConverter.Global.Contains(type))
{
_typeTypeConverterCache[type] = (from a1 in ChoTypeConverter.Global.GetAll()
where a1.Key == type
select a1.Value).ToArray();
return(_typeTypeConverterCache[type]);
}
Type[] types = ChoType.GetTypes(typeof(ChoTypeConverterAttribute)).Where(t => t.GetCustomAttribute <ChoTypeConverterAttribute>().ConverterType.IsAssignableFrom(objType)).ToArray();
if (types != null)
{
int index1 = 0;
SortedList <int, object> queue1 = new SortedList <int, object>();
SortedList <int, object> paramsQueue1 = new SortedList <int, object>();
foreach (Type t in types)
{
queue1.Add(index1, ChoActivator.CreateInstance(t));
index1++;
}
_typeTypeConverterCache.Add(objType, queue1.Values.ToArray());
return(_typeTypeConverterCache[objType]);
}
TypeConverter converter = TypeDescriptor.GetConverter(objType);
if (converter != null)
{
_typeTypeConverterCache.Add(objType, new object[] { converter });
}
else
{
_typeTypeConverterCache.Add(objType, EmptyTypeConverters);
}
_typeTypeConverterParamsCache.Add(objType, EmptyParams);
}
return(_typeTypeConverterCache[objType]);
}
}
}
public static object ToJSONObject(this IDictionary <string, object> dict, Type type)
{
object target = ChoActivator.CreateInstance(type);
string key = null;
foreach (var p in ChoType.GetProperties(type))
{
if (p.GetCustomAttribute <JsonIgnoreAttribute>() != null)
{
continue;
}
key = p.Name;
var attr = p.GetCustomAttribute <JsonPropertyAttribute>();
if (attr != null && !attr.PropertyName.IsNullOrWhiteSpace())
{
key = attr.PropertyName.NTrim();
}
if (!dict.ContainsKey(key))
{
continue;
}
p.SetValue(target, dict[key].CastObjectTo(p.PropertyType));
}
return(target);
}
protected virtual object Deserialize(object value, Type targetType, object parameter, System.Globalization.CultureInfo culture)
{
if (value != null && value.GetType().IsCollectionType())
{
if (targetType != typeof(object) && !targetType.IsSimple() && !typeof(ICollection).IsAssignableFrom(targetType))
{
IList coll = value as IList;
var itemType = value.GetType().GetItemType();
if (itemType == typeof(object) || itemType.IsSimple())
{
value = ChoActivator.CreateInstance(targetType);
foreach (var p in ChoTypeDescriptor.GetProperties <ChoArrayIndexAttribute>(targetType).Select(pd => new { pd, a = ChoTypeDescriptor.GetPropetyAttribute <ChoArrayIndexAttribute>(pd) })
.GroupBy(g => g.a.Position).Select(g => g.First()).Where(g => g.a.Position >= 0).OrderBy(g => g.a.Position))
{
if (p.a.Position < coll.Count)
{
ChoType.ConvertNSetPropertyValue(value, p.pd.Name, coll[p.a.Position], culture);
}
}
}
}
}
return(value);
}
public static object Default(this Type type)
{
if (type != null && type.IsValueType)
{
return(ChoActivator.CreateInstance(type));
}
return((object)null);
}
public ChoObjectPool(int size = 1024)
{
if (size > 1024)
{
_size = size;
}
_cache = new T[_size];
_backupCache = new T[_size];
Parallel.For(0, _size, i => _backupCache[i] = ChoActivator.CreateInstance <T>());
Initialize();
}
public static object ConvertToObject(this object source, Type type)
{
if (source == null)
{
return(source);
}
if (source is IDictionary <string, object> )
{
return(((IDictionary <string, object>)source).ToObject(type));
}
else
{
if (source is IDictionary)
{
var dict = ((IDictionary)source).ToDictionary();
return(dict.ToObject(type));
}
else
{
Type sourceType = source.GetType();
object target = ChoActivator.CreateInstance(type);
string key = null;
object value = null;
foreach (var p in ChoType.GetProperties(type))
{
if (p.GetCustomAttribute <ChoIgnoreMemberAttribute>() != null)
{
continue;
}
key = p.Name;
var attr = p.GetCustomAttribute <ChoPropertyAttribute>();
if (attr != null && !attr.Name.IsNullOrWhiteSpace())
{
key = attr.Name.NTrim();
}
if (!ChoType.HasProperty(sourceType, key))
{
continue;
}
value = ChoType.GetPropertyValue(source, key);
p.SetValue(target, value.CastObjectTo(p.PropertyType));
}
return(target);
}
}
}
public virtual object CreateInstance()
{
if (ConverterType == null)
{
return(null);
}
//if (ChoGuard.IsArgumentNotNullOrEmpty(Parameters) && ChoType.HasConstructor(ConverterType, ParametersArray))
// return ChoType.CreateInstance(ConverterType, ParametersArray);
//else if (ChoType.HasConstructor(ConverterType, new object[] { String.Empty }))
// return ChoType.CreateInstance(ConverterType, new object[] { ParametersArray != null && ParametersArray.Length > 0 ? ParametersArray[0] : String.Empty });
//else
return(ChoActivator.CreateInstance(ConverterType));
}
//protected Type DiscoverFieldType(string value, ChoFileRecordConfiguration config)
//{
// bool treatCurrencyAsDecimal = config.TreatCurrencyAsDecimal;
// long lresult = 0;
// double dresult = 0;
// DateTime dtresult;
// Decimal decResult = 0;
// ChoCurrency currResult = 0;
// Guid guidResult;
// if (value == null)
// return typeof(string);
// else if (long.TryParse(value, out lresult))
// return typeof(long);
// else if (double.TryParse(value, out dresult))
// return typeof(double);
// else if (RecordConfiguration.TypeConverterFormatSpec.DateTimeFormat.IsNullOrWhiteSpace()
// && ChoDateTime.TryParseExact(value, RecordConfiguration.TypeConverterFormatSpec.DateTimeFormat, CultureInfo.CurrentCulture, out dtResult))
// return typeof(DateTime);
// else if (DateTime.TryParse(value, out dtresult))
// return typeof(DateTime);
// else if (Guid.TryParse(value, out guidResult))
// return typeof(Guid);
// else if (!treatCurrencyAsDecimal && ChoCurrency.TryParse(value, out currResult))
// return typeof(ChoCurrency);
// else if (treatCurrencyAsDecimal && Decimal.TryParse(value, NumberStyles.Currency, CultureInfo.CurrentCulture, out decResult))
// return typeof(Decimal);
// else
// return typeof(string);
//}
protected object GetDeclaringRecord(string declaringMember, object rec, ChoFileRecordFieldConfiguration config = null)
{
if (rec == null)
{
return(null);
}
var obj = ChoType.GetDeclaringRecord(declaringMember, rec);
if (obj == null)
{
return(null);
}
Type recordType = obj.GetType();
if (config != null)
{
if (config.ArrayIndex != null &&
config.ArrayIndex.Value >= 0 &&
obj is IEnumerable &&
!(obj is ArrayList))
{
var item = Enumerable.Skip(((IEnumerable)obj).Cast <object>(), config.ArrayIndex.Value).FirstOrDefault();
if (item == null)
{
Type itemType = obj.GetType().GetItemType();
item = ChoActivator.CreateInstance(itemType);
}
if (obj is Array)
{
if (config.ArrayIndex.Value < ((Array)obj).Length)
{
((Array)obj).SetValue(item, config.ArrayIndex.Value);
}
}
else if (obj is IList)
{
((IList)obj).Add(item);
}
return(item);
}
}
return(obj);
}
/// <summary>
/// Advances the <see cref="T:System.Data.IDataReader"/> to the next record.
/// </summary>
/// <returns>
/// true if there are more rows; otherwise, false.
/// </returns>
/// <filterpriority>2</filterpriority>
public override bool Read()
{
bool returnValue = false;
if (_isDeferred)
{
returnValue = _firstElementExists;
_isDeferred = false;
}
else
{
returnValue = _enumerator.MoveNext();
}
_current = returnValue ? _enumerator.Current : _type.IsValueType ? ChoActivator.CreateInstance(_type) : null;
return(returnValue);
}
protected virtual object Serialize(object value, Type targetType, object parameter, System.Globalization.CultureInfo culture)
{
IList result = ChoActivator.CreateInstance(typeof(IList <>).MakeGenericType(targetType)) as IList;
if (value != null && !value.GetType().IsCollectionType())
{
if (targetType == typeof(object) || targetType.IsSimple())
{
foreach (var p in ChoTypeDescriptor.GetProperties(value.GetType()).Where(pd => ChoTypeDescriptor.GetPropetyAttribute <ChoIgnoreMemberAttribute>(pd) == null))
{
result.Add(ChoConvert.ConvertTo(ChoType.GetPropertyValue(value, p.Name), targetType, culture));
}
}
}
return(result.OfType <object>().ToArray());
}
protected virtual void Init(Type recordType)
{
if (recordType == null)
{
return;
}
_recObject = new Lazy <object>(() => ChoActivator.CreateInstance(RecordType));
ChoRecordObjectAttribute recObjAttr = ChoType.GetAttribute <ChoRecordObjectAttribute>(recordType);
if (recObjAttr != null)
{
ErrorMode = recObjAttr.ErrorMode;
IgnoreFieldValueMode = recObjAttr.IgnoreFieldValueMode;
ThrowAndStopOnMissingField = recObjAttr.ThrowAndStopOnMissingField;
ObjectValidationMode = recObjAttr.ObjectValidationMode;
}
}
protected virtual void Init(Type recordType)
{
if (recordType == null)
{
return;
}
var tc = recordType.GetCustomAttribute(typeof(ChoTypeConverterAttribute)) as ChoTypeConverterAttribute;
if (tc != null)
{
var c = tc.CreateInstance();
if (c is IChoValueConverter)
{
ChoTypeConverter.Global.Add(recordType, c as IChoValueConverter);
}
else if (c is TypeConverter)
{
ChoTypeConverter.Global.Add(recordType, c as TypeConverter);
}
#if !NETSTANDARD2_0
else if (c is IValueConverter)
{
ChoTypeConverter.Global.Add(recordType, c as IValueConverter);
}
#endif
}
var st = recordType.GetCustomAttribute(typeof(ChoSourceTypeAttribute)) as ChoSourceTypeAttribute;
if (st != null)
{
SourceType = st.Type;
}
_recObject = new Lazy <object>(() => ChoActivator.CreateInstance(RecordType));
ChoRecordObjectAttribute recObjAttr = ChoType.GetAttribute <ChoRecordObjectAttribute>(recordType);
if (recObjAttr != null)
{
ErrorMode = recObjAttr.ErrorMode;
IgnoreFieldValueMode = recObjAttr.IgnoreFieldValueMode;
ThrowAndStopOnMissingField = recObjAttr.ThrowAndStopOnMissingField;
ObjectValidationMode = recObjAttr.ObjectValidationMode;
}
}
private List <JsonConverter> GetJSONConverters()
{
List <JsonConverter> converters = new List <JsonConverter>();
converters.Add(new ExpandoObjectConverter());
foreach (var kvp in NodeConvertersForType)
{
if (kvp.Value == null)
{
continue;
}
converters.Add(ChoActivator.CreateInstance(typeof(ChoJSONNodeConverter <>).MakeGenericType(kvp.Key), kvp.Value) as JsonConverter);
ChoTypeConverter.Global.Add(kvp.Key, ChoActivator.CreateInstance(typeof(ChoJSONTypeConverter <>).MakeGenericType(kvp.Key), kvp.Value) as IChoValueConverter);
}
return(converters);
}
protected void InitializeRecordConfiguration(ChoRecordConfiguration configuration)
{
if (configuration == null || configuration.IsDynamicObject || configuration.RecordType == null)
{
return;
}
if (!typeof(IChoNotifyRecordConfigurable).IsAssignableFrom(configuration.RecordMapType))
{
return;
}
var obj = ChoActivator.CreateInstance(configuration.RecordMapType) as IChoNotifyRecordConfigurable;
if (obj != null)
{
obj.RecondConfigure(configuration);
}
}
public static object CreateMoqInstance(Type objType, ChoIniFile iniFile = null)
{
//if (typeof(ChoRecord).IsAssignableFrom(objType))
// return CreateDynamicMoqInstance(objType, iniFile);
object obj = ChoActivator.CreateInstance(objType);
foreach (KeyValuePair <MemberInfo, Attribute> kv in ChoUtility.DiscoverMembers(objType, typeof(ChoRandomAttribute)))
{
if (!(kv.Value is ChoRandomAttribute))
{
continue;
}
ChoType.SetMemberValue(obj, kv.Key, ((ChoRandomAttribute)kv.Value).NextValue());
}
return(obj);
}
public virtual object CreateInstance()
{
if (ConverterType == null)
{
return(null);
}
//if (ChoGuard.IsArgumentNotNullOrEmpty(Parameters) && ChoType.HasConstructor(ConverterType, ParametersArray1))
// return ChoType.CreateInstance(ConverterType, ParametersArray1);
//else if (ChoType.HasConstructor(ConverterType, new object[] { String.Empty }))
// return ChoType.CreateInstance(ConverterType, new object[] { ParametersArray1 != null && ParametersArray1.Length > 0 ? ParametersArray1[0] : String.Empty });
//else
if ((ParametersArray == null || ParametersArray.Length == 0) && (ParametersDict == null || ParametersDict.Count == 0))
{
return(ChoActivator.CreateInstance(ConverterType));
}
else
{
if (ParametersDict != null || ParametersDict.Count > 0)
{
try
{
return(ChoActivator.CreateInstance(ConverterType, ParametersDict));
}
catch
{
return(ChoActivator.CreateInstance(ConverterType));
}
}
else
{
try
{
return(ChoActivator.CreateInstance(ConverterType, ParametersArray));
}
catch
{
return(ChoActivator.CreateInstance(ConverterType));
}
}
}
}
public void ScanAndLoad()
{
var types = ChoType.GetAllTypes().Where(t => typeof(IComparer).IsAssignableFrom(t)).ToArray();
foreach (Type compType in types)
{
try
{
var comp = ChoActivator.CreateInstance(compType) as IComparer;
if (comp != null)
{
if (compType.IsGenericType)
{
Type type = comp.GetType().GetGenericArguments()[0];
Add(type, comp);
}
else
{
Type objType = compType.GetAllInterfaces().Where(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(IComparer <>))
.Select(i => i.GetGenericArguments()[0]).FirstOrDefault();
if (objType != null)
{
Add(objType, comp);
}
else
{
var attr = compType.GetCustomAttribute <ChoComparerObjectTypeAttribute>();
if (attr != null && attr.Type != null)
{
Add(attr.Type, comp);
}
}
}
}
}
catch
{
}
}
}
public object GetMetadataObject(object @this)
{
if (@this == null)
{
return(@this);
}
Type type = @this.GetType();
if (_objectCache.ContainsKey(type))
{
return(_objectCache[type] != null ? _objectCache[type] : @this);
}
MetadataTypeAttribute attr = type.GetCustomAttribute <MetadataTypeAttribute>();
if (attr == null || attr.MetadataClassType == null)
{
return(@this);
}
else
{
lock (_padLock)
{
if (!_objectCache.ContainsKey(type))
{
object obj = null;
try
{
obj = ChoActivator.CreateInstance(attr.MetadataClassType);
}
catch { }
_objectCache.Add(type, obj);
}
return(_objectCache[type] != null ? _objectCache[type] : @this);
}
}
}
public static object ChangeType <T>(object value, Type conversionType)
where T : Attribute
{
if (value == null)
{
return(ChoActivator.CreateInstance(conversionType));
}
if (conversionType.IsAssignableFrom(value.GetType()))
{
return(value);
}
object dest = null;
if (value is IConvertible)
{
dest = Convert.ChangeType(value, conversionType);
}
else
{
dest = ChoActivator.CreateInstance(conversionType);
value.CloneTo <T>(dest);
}
if (dest != null && !dest.GetType().IsSimple())
{
ChoObjectValidationMode m = GetValidationMode(value);
if (m == ChoObjectValidationMode.MemberLevel)
{
ChoValidator.Validate(dest);
}
else if (m == ChoObjectValidationMode.ObjectLevel)
{
ChoValidator.Validate(dest);
}
}
return(dest);
}
public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
{
if (typeof(IList).IsAssignableFrom(objectType))
{
Type itemType = objectType.GetItemType();
var arr = JArray.Load(reader);
IList result = ChoActivator.CreateInstance(typeof(IList <>).MakeGenericType(itemType)) as IList;
foreach (var jo in arr.OfType <JObject>())
{
result.Add(jo.ToObjectEx(ResolveType(jo, itemType), serializer));
}
return(typeof(Array).IsAssignableFrom(objectType) ? result.ConvertToArray() : result);
}
else
{
var jo = JObject.Load(reader);
var newType = ResolveType(jo, objectType);
return(jo.ToObjectEx(newType, serializer));
}
}
internal ChoJSONRecordConfiguration(Type recordType) : base(recordType)
{
_JsonSerializer = new Lazy <JsonSerializer>(() =>
{
return(JsonSerializerSettings == null ? null : JsonSerializer.Create(JsonSerializerSettings));
});
_JSONConverters = new Lazy <List <JsonConverter> >(() =>
{
List <JsonConverter> converters = new List <JsonConverter>();
converters.Add(new ExpandoObjectConverter());
converters.Add(ChoDynamicObjectConverter.Instance);
foreach (var kvp in NodeConvertersForType)
{
if (kvp.Value == null)
{
continue;
}
converters.Add(ChoActivator.CreateInstance(typeof(ChoJSONNodeConverter <>).MakeGenericType(kvp.Key), kvp.Value) as JsonConverter);
ChoTypeConverter.Global.Add(kvp.Key, ChoActivator.CreateInstance(typeof(ChoJSONTypeConverter <>).MakeGenericType(kvp.Key), kvp.Value) as IChoValueConverter);
}
return(converters);
});
DefaultArrayHandling = recordType == null || recordType.IsDynamicType() ? true : false;
JSONRecordFieldConfigurations = new List <ChoJSONRecordFieldConfiguration>();
WithJSONConverter(ChoDynamicObjectConverter.Instance);
LineBreakChars = Environment.NewLine;
Formatting = Newtonsoft.Json.Formatting.Indented;
if (recordType != null)
{
Init(recordType);
}
}
public override object ReadJson(JsonReader reader, Type objectType,
object existingValue, JsonSerializer serializer)
{
var dict = serializer.Deserialize <Dictionary <string, string> >(reader);
var item = dict.First();
var rec = ChoActivator.CreateInstance(objectType);
if (typeof(IChoKeyValueType).IsAssignableFrom(objectType))
{
IChoKeyValueType kvp = rec as IChoKeyValueType;
kvp.Key = item.Key;
kvp.Value = item.Value;
}
else
{
var kP = ChoTypeDescriptor.GetProperties <ChoKeyAttribute>(objectType).FirstOrDefault();
var vP = ChoTypeDescriptor.GetProperties <ChoValueAttribute>(objectType).FirstOrDefault();
ChoType.SetPropertyValue(rec, kP.Name, item.Key);
ChoType.SetPropertyValue(rec, vP.Name, item.Value);
}
return(rec);
}
private bool ToText(long index, object rec, out string recText)
{
if (typeof(IChoScalarObject).IsAssignableFrom(Configuration.RecordType))
{
rec = ChoActivator.CreateInstance(Configuration.RecordType, rec);
}
recText = null;
StringBuilder msg = new StringBuilder();
object fieldValue = null;
string fieldText = null;
ChoFixedLengthRecordFieldConfiguration fieldConfig = null;
if (Configuration.ColumnCountStrict)
{
CheckColumnsStrict(rec);
}
//bool firstColumn = true;
PropertyInfo pi = null;
object rootRec = rec;
IDictionary <string, Object> dict = null;
foreach (KeyValuePair <string, ChoFixedLengthRecordFieldConfiguration> kvp in Configuration.RecordFieldConfigurationsDict)
{
fieldConfig = kvp.Value;
fieldValue = null;
fieldText = String.Empty;
if (Configuration.PIDict != null)
{
Configuration.PIDict.TryGetValue(kvp.Key, out pi);
}
rec = GetDeclaringRecord(kvp.Value.DeclaringMember, rootRec);
dict = rec.ToDynamicObject() as IDictionary <string, Object>;
if (Configuration.IsDynamicObject)
{
dict = dict.Flatten().ToDictionary();
}
if (Configuration.ThrowAndStopOnMissingField)
{
if (Configuration.IsDynamicObject)
{
if (!dict.ContainsKey(kvp.Key))
{
throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' FixedLength column.".FormatString(fieldConfig.FieldName));
}
}
else
{
if (pi == null)
{
throw new ChoMissingRecordFieldException("No matching property found in the object for '{0}' FixedLength column.".FormatString(fieldConfig.FieldName));
}
}
}
try
{
if (Configuration.IsDynamicObject)
{
fieldValue = dict[kvp.Key]; // dict.GetValue(kvp.Key, Configuration.FileHeaderConfiguration.IgnoreCase, Configuration.Culture);
if (kvp.Value.FieldType == null)
{
if (fieldValue == null)
{
kvp.Value.FieldType = typeof(string);
}
else
{
kvp.Value.FieldType = fieldValue.GetType();
}
}
}
else
{
if (pi != null)
{
fieldValue = ChoType.GetPropertyValue(rec, pi);
if (kvp.Value.FieldType == null)
{
kvp.Value.FieldType = pi.PropertyType;
}
}
else
{
kvp.Value.FieldType = typeof(string);
}
}
//Discover default value, use it if null
if (fieldValue == null)
{
if (fieldConfig.IsDefaultValueSpecified)
{
fieldValue = fieldConfig.DefaultValue;
}
}
if (!RaiseBeforeRecordFieldWrite(rec, index, kvp.Key, ref fieldValue))
{
return(false);
}
if (fieldConfig.ValueConverter != null)
{
fieldValue = fieldConfig.ValueConverter(fieldValue);
}
else
{
rec.GetNConvertMemberValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue);
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.MemberLevel)
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
if (!RaiseAfterRecordFieldWrite(rec, index, kvp.Key, fieldValue))
{
return(false);
}
}
catch (ChoParserException)
{
throw;
}
catch (ChoMissingRecordFieldException)
{
if (Configuration.ThrowAndStopOnMissingField)
{
throw;
}
}
catch (Exception ex)
{
ChoETLFramework.HandleException(ref ex);
if (fieldConfig.ErrorMode == ChoErrorMode.ThrowAndStop)
{
throw;
}
try
{
if (Configuration.IsDynamicObject)
{
if (dict.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else if (dict.GetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
dict.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else
{
var ex1 = new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
fieldValue = null;
throw ex1;
}
}
else if (pi != null)
{
if (rec.GetFallbackValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode);
}
else if (rec.GetDefaultValue(kvp.Key, kvp.Value, Configuration.Culture, ref fieldValue))
{
rec.DoMemberLevelValidation(kvp.Key, kvp.Value, Configuration.ObjectValidationMode, fieldValue);
}
else
{
var ex1 = new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
fieldValue = null;
throw ex1;
}
}
else
{
var ex1 = new ChoWriterException($"Failed to write '{fieldValue}' value for '{fieldConfig.FieldName}' member.", ex);
fieldValue = null;
throw ex1;
}
}
catch (Exception innerEx)
{
if (ex == innerEx.InnerException)
{
if (fieldConfig.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
continue;
}
else
{
if (!RaiseRecordFieldWriteError(rec, index, kvp.Key, fieldText, ex))
{
throw new ChoWriterException($"Failed to write '{fieldValue}' value of '{kvp.Key}' member.", ex);
}
}
}
else
{
throw new ChoWriterException("Failed to use '{0}' fallback value for '{1}' member.".FormatString(fieldValue, kvp.Key), innerEx);
}
}
}
if (fieldValue == null)
{
fieldText = String.Empty;
}
else
{
fieldText = fieldValue.ToString();
}
msg.Append(NormalizeFieldValue(kvp.Key, fieldText, kvp.Value.Size, kvp.Value.Truncate, kvp.Value.QuoteField,
GetFieldValueJustification(kvp.Value.FieldValueJustification, kvp.Value.FieldType),
GetFillChar(kvp.Value.FillChar, kvp.Value.FieldType), false, kvp.Value.NullValue,
kvp.Value.GetFieldValueTrimOption(kvp.Value.FieldType)));
}
recText = msg.ToString();
return(true);
}
private bool LoadNode(Tuple <long, IDictionary <string, object> > pair, ref object rec)
{
bool ignoreFieldValue = pair.Item2.IgnoreFieldValue(Configuration.IgnoreFieldValueMode);
if (ignoreFieldValue)
{
return(false);
}
else if (pair.Item2 == null && !Configuration.IsDynamicObject)
{
rec = RecordType.CreateInstanceAndDefaultToMembers(Configuration.RecordFieldConfigurationsDict.ToDictionary(kvp => kvp.Key, kvp => kvp.Value as ChoRecordFieldConfiguration));
return(true);
}
if (Configuration.SupportsMultiRecordTypes && Configuration.RecordTypeSelector != null)
{
Type recType = Configuration.RecordTypeSelector(pair);
if (recType == null)
{
if (Configuration.IgnoreIfNoRecordTypeFound)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, $"No record type found for [{pair.Item1}] line to parse.");
return(true);
}
else
{
throw new ChoParserException($"No record type found for [{pair.Item1}] line to parse.");
}
}
if (!Configuration.RecordTypeMapped)
{
Configuration.MapRecordFields(recType);
Configuration.Validate(null);
}
rec = recType.IsDynamicType() ? new ChoDynamicObject()
{
ThrowExceptionIfPropNotExists = Configuration.ThrowExceptionIfDynamicPropNotExists == null ? ChoDynamicObjectSettings.ThrowExceptionIfPropNotExists : Configuration.ThrowExceptionIfDynamicPropNotExists.Value,
} : ChoActivator.CreateInstance(recType);
RecordType = recType;
}
else if (Configuration.IsDynamicObject)
{
rec = Configuration.IsDynamicObject ? new ChoDynamicObject()
{
ThrowExceptionIfPropNotExists = Configuration.ThrowExceptionIfDynamicPropNotExists == null ? ChoDynamicObjectSettings.ThrowExceptionIfPropNotExists : Configuration.ThrowExceptionIfDynamicPropNotExists.Value,
}
}
: ChoActivator.CreateInstance(RecordType);
try
{
if (!RaiseBeforeRecordLoad(rec, ref pair))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping...");
rec = null;
return(true);
}
//if (Configuration.CustomNodeSelecter != null)
//{
// pair = new Tuple<long, IDictionary<string, object>>(pair.Item1, Configuration.CustomNodeSelecter(pair.Item2));
//}
if (pair.Item2 == null)
{
rec = null;
return(true);
}
if (!FillRecord(ref rec, pair))
{
return(false);
}
if ((Configuration.ObjectValidationMode & ChoObjectValidationMode.ObjectLevel) == ChoObjectValidationMode.ObjectLevel)
{
rec.DoObjectLevelValidation(Configuration, Configuration.ParquetRecordFieldConfigurations);
}
bool skip = false;
if (!RaiseAfterRecordLoad(rec, pair, ref skip))
{
return(false);
}
else if (skip)
{
rec = null;
return(true);
}
}
catch (Exception ex)
{
Reader.IsValid = false;
if (ex is ChoMissingRecordFieldException && Configuration.ThrowAndStopOnMissingField)
{
if (!RaiseRecordLoadError(rec, pair, ex))
{
throw;
}
else
{
//ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
//rec = null;
}
}
else
{
ChoETLFramework.HandleException(ref ex);
if (Configuration.ErrorMode == ChoErrorMode.IgnoreAndContinue)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
rec = null;
}
else if (Configuration.ErrorMode == ChoErrorMode.ReportAndContinue)
{
if (!RaiseRecordLoadError(rec, pair, ex))
{
throw;
}
else
{
//ChoETLFramework.WriteLog(TraceSwitch.TraceError, "Error [{0}] found. Ignoring record...".FormatString(ex.Message));
//rec = null;
}
}
else
{
throw;
}
}
return(true);
}
return(true);
}
object fieldValue = null;
ChoParquetRecordFieldConfiguration fieldConfig = null;
PropertyInfo pi = null;
private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null)
{
TraceSwitch = traceSwitch;
TextReader sr = source as TextReader;
ChoGuard.ArgumentNotNull(sr, "TextReader");
if (sr is StreamReader)
{
((StreamReader)sr).Seek(0, SeekOrigin.Begin);
}
if (Configuration.RecordSelector == null)
{
throw new ChoRecordConfigurationException("Missing record selector.");
}
if (!RaiseBeginLoad(sr))
{
yield break;
}
string[] commentTokens = Configuration.Comments;
bool? skip = false;
bool _headerFound = false;
bool? skipUntil = true;
bool? doWhile = true;
using (ChoPeekEnumerator <Tuple <long, string> > e = new ChoPeekEnumerator <Tuple <long, string> >(
new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar, Configuration.MayContainEOLInData, Configuration.MaxLineSize)).ToEnumerable(),
(pair) =>
{
//bool isStateAvail = IsStateAvail();
skip = false;
if (skipUntil != null)
{
if (skipUntil.Value)
{
skipUntil = RaiseSkipUntil(pair);
if (skipUntil == null)
{
}
else
{
skip = skipUntil.Value;
}
}
}
//if (isStateAvail)
//{
// if (!IsStateMatches(item))
// {
// skip = filterFunc != null ? filterFunc(item) : false;
// }
// else
// skip = true;
//}
//else
// skip = filterFunc != null ? filterFunc(item) : false;
if (skip == null)
{
return(null);
}
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine);
if (!skip.Value)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1));
}
}
if (skip.Value)
{
return(skip);
}
//if (!(sr.BaseStream is MemoryStream))
// ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1));
//if (Task != null)
// return !IsStateNOTExistsOrNOTMatch(item);
if (pair.Item2.IsNullOrWhiteSpace())
{
if (!Configuration.IgnoreEmptyLine)
{
throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pair.Item1));
}
else
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1));
}
return(true);
}
}
if (commentTokens != null && commentTokens.Length > 0)
{
foreach (string comment in commentTokens)
{
if (!pair.Item2.IsNull() && pair.Item2.StartsWith(comment, StringComparison.Ordinal)) //, true, Configuration.Culture))
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1));
}
return(true);
}
}
}
if (!_configCheckDone)
{
Configuration.Validate(pair); // GetHeaders(pair.Item2));
_configCheckDone = true;
}
//Ignore Header if any
if (Configuration.FileHeaderConfiguration.HasHeaderRecord &&
!_headerFound)
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring header line at [{0}]...".FormatString(pair.Item1));
}
_headerFound = true;
return(true);
}
return(false);
}))
{
while (true)
{
Tuple <long, string> pair = e.Peek;
if (pair == null)
{
RaiseEndLoad(sr);
yield break;
}
Type recType = Configuration.RecordSelector(pair.Item2);
if (recType == null)
{
if (Configuration.IgnoreIfNoRecordParserExists)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, $"No record type found for [{pair.Item1}] line to parse.");
}
else
{
throw new ChoParserException($"No record type found for [{pair.Item1}] line to parse.");
}
}
object rec = ChoActivator.CreateInstance(recType);
if (!LoadLine(pair, ref rec))
{
yield break;
}
//StoreState(e.Current, rec != null);
e.MoveNext();
if (rec == null)
{
continue;
}
yield return(rec);
if (Configuration.NotifyAfter > 0 && pair.Item1 % Configuration.NotifyAfter == 0)
{
if (RaisedRowsLoaded(pair.Item1))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
yield break;
}
}
if (doWhile != null)
{
doWhile = RaiseDoWhile(pair);
if (doWhile != null && doWhile.Value)
{
break;
}
}
}
}
}
private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null)
{
TraceSwitch = traceSwitch;
StreamReader sr = source as StreamReader;
ChoGuard.ArgumentNotNull(sr, "StreamReader");
if (Configuration.RecordSelector == null)
{
throw new ChoRecordConfigurationException("Missing record selector.");
}
sr.Seek(0, SeekOrigin.Begin);
if (!RaiseBeginLoad(sr))
{
yield break;
}
string[] commentTokens = Configuration.Comments;
bool _headerFound = false;
using (ChoPeekEnumerator <Tuple <int, string> > e = new ChoPeekEnumerator <Tuple <int, string> >(
new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar)).ToEnumerable(),
(pair) =>
{
//bool isStateAvail = IsStateAvail();
bool?skip = false;
//if (isStateAvail)
//{
// if (!IsStateMatches(item))
// {
// skip = filterFunc != null ? filterFunc(item) : false;
// }
// else
// skip = true;
//}
//else
// skip = filterFunc != null ? filterFunc(item) : false;
if (skip == null)
{
return(null);
}
//if (!(sr.BaseStream is MemoryStream))
//{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine);
if (!skip.Value)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1));
}
//}
if (skip.Value)
{
return(skip);
}
//if (!(sr.BaseStream is MemoryStream))
// ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1));
//if (Task != null)
// return !IsStateNOTExistsOrNOTMatch(item);
if (pair.Item2.IsNullOrWhiteSpace())
{
if (!Configuration.IgnoreEmptyLine)
{
throw new ChoParserException("Empty line found at {0} location.".FormatString(e.Peek.Item1));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Empty line found at [{0}]...".FormatString(pair.Item1));
return(true);
}
}
if (commentTokens == null)
{
return(false);
}
else
{
var x = (from comment in commentTokens
where !pair.Item2.IsNull() && pair.Item2.StartsWith(comment, true, Configuration.Culture)
select comment).FirstOrDefault();
if (x != null)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1));
return(true);
}
}
if (!_configCheckDone)
{
Configuration.Validate(pair); // GetHeaders(pair.Item2));
_configCheckDone = true;
}
//Ignore Header if any
if (Configuration.FileHeaderConfiguration.HasHeaderRecord &&
!_headerFound)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring header line at [{0}]...".FormatString(pair.Item1));
_headerFound = true;
return(true);
}
return(false);
}))
{
while (true)
{
Tuple <int, string> pair = e.Peek;
if (pair == null)
{
RaiseEndLoad(sr);
yield break;
}
Type recType = Configuration.RecordSelector(pair.Item2);
if (recType == null)
{
if (Configuration.IgnoreIfNoRecordParserExists)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, $"No record type found for [{pair.Item1}] line to parse.");
}
else
{
throw new ChoParserException($"No record type found for [{pair.Item1}] line to parse.");
}
}
object rec = ChoActivator.CreateInstance(recType);
if (!LoadLine(pair, ref rec))
{
yield break;
}
//StoreState(e.Current, rec != null);
e.MoveNext();
if (rec == null)
{
continue;
}
yield return(rec);
}
}
}
private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null)
{
TraceSwitch = traceSwitch;
TextReader sr = source as TextReader;
ChoGuard.ArgumentNotNull(sr, "TextReader");
if (sr is StreamReader)
{
((StreamReader)sr).Seek(0, SeekOrigin.Begin);
}
if (!RaiseBeginLoad(sr))
{
yield break;
}
string[] commentTokens = Configuration.Comments;
bool? skip = false;
bool abortRequested = false;
long runningCount = 0;
long recCount = 0;
bool headerLineLoaded = false;
List <object> buffer = new List <object>();
IDictionary <string, Type> recFieldTypes = null;
bool?skipUntil = true;
bool?doWhile = true;
using (ChoPeekEnumerator <Tuple <long, string> > e = new ChoPeekEnumerator <Tuple <long, string> >(
new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar, false /*Configuration.MayContainEOLInData*/)).ToEnumerable(),
(pair) =>
{
//bool isStateAvail = IsStateAvail();
skip = false;
if (skipUntil != null)
{
if (skipUntil.Value)
{
skipUntil = RaiseSkipUntil(pair);
if (skipUntil == null)
{
}
else
{
if (skipUntil.Value)
{
skip = skipUntil;
}
else
{
skip = true;
}
}
}
}
//if (isStateAvail)
//{
// if (!IsStateMatches(item))
// {
// skip = filterFunc != null ? filterFunc(item) : false;
// }
// else
// skip = true;
//}
//else
// skip = filterFunc != null ? filterFunc(item) : false;
if (skip == null)
{
return(null);
}
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine);
if (!skip.Value)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1));
}
}
if (skip.Value)
{
return(skip);
}
//if (!(sr.BaseStream is MemoryStream))
// ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1));
//if (Task != null)
// return !IsStateNOTExistsOrNOTMatch(item);
if (pair.Item2.IsNullOrWhiteSpace())
{
if (!Configuration.IgnoreEmptyLine)
{
throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pair.Item1));
}
else
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1));
}
return(true);
}
}
if (commentTokens != null && commentTokens.Length > 0)
{
foreach (string comment in commentTokens)
{
if (!pair.Item2.IsNull() && pair.Item2.StartsWith(comment, StringComparison.Ordinal)) //, true, Configuration.Culture))
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1));
}
return(true);
}
}
}
if (Configuration.FileHeaderConfiguration.HeaderLineAt > 0)
{
if (pair.Item1 < Configuration.FileHeaderConfiguration.HeaderLineAt)
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Header line at {1}. Skipping [{0}] line...".FormatString(pair.Item1, Configuration.FileHeaderConfiguration.HeaderLineAt));
}
return(true);
}
}
if (!_configCheckDone)
{
if (Configuration.SupportsMultiRecordTypes && Configuration.RecordSelector != null && !Configuration.RecordTypeMapped)
{
}
else
{
Configuration.Validate(pair); // GetHeaders(pair.Item2));
}
var dict = recFieldTypes = Configuration.FixedLengthRecordFieldConfigurations.ToDictionary(i => i.Name, i => i.FieldType == null ? null : i.FieldType);
RaiseMembersDiscovered(dict);
Configuration.UpdateFieldTypesIfAny(dict);
_configCheckDone = true;
}
//LoadHeader if any
if ((Configuration.FileHeaderConfiguration.HasHeaderRecord ||
Configuration.FileHeaderConfiguration.HeaderLineAt > 0) &&
!_headerFound)
{
if (Configuration.FileHeaderConfiguration.IgnoreHeader)
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring header line at [{0}]...".FormatString(pair.Item1));
}
}
else
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading header line at [{0}]...".FormatString(pair.Item1));
}
headerLineLoaded = true;
LoadHeaderLine(pair);
}
_headerFound = true;
return(true);
}
return(false);
}))
{
while (true)
{
recCount++;
Tuple <long, string> pair = e.Peek;
if (pair == null)
{
if (!abortRequested)
{
RaisedRowsLoaded(runningCount);
}
RaiseEndLoad(sr);
yield break;
}
runningCount = pair.Item1;
object rec = null;
if (Configuration.SupportsMultiRecordTypes && Configuration.RecordSelector != null)
{
Type recType = Configuration.RecordSelector(pair);
if (recType == null)
{
if (Configuration.IgnoreIfNoRecordTypeFound)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, $"No record type found for [{pair.Item1}] line to parse.");
e.MoveNext();
continue;
}
else
{
throw new ChoParserException($"No record type found for [{pair.Item1}] line to parse.");
}
}
if (!Configuration.RecordTypeMapped)
{
Configuration.MapRecordFields(recType);
Configuration.Validate(null);
}
rec = recType.IsDynamicType() ? new ChoDynamicObject(new Dictionary <string, object>(Configuration.FileHeaderConfiguration.StringComparer))
{
ThrowExceptionIfPropNotExists = true,
AlternativeKeys = Configuration.AlternativeKeys
} : ChoActivator.CreateInstance(recType);
}
else
{
rec = Configuration.IsDynamicObject ? new ChoDynamicObject(new Dictionary <string, object>(Configuration.FileHeaderConfiguration.StringComparer))
{
ThrowExceptionIfPropNotExists = true,
AlternativeKeys = Configuration.AlternativeKeys
} : ChoActivator.CreateInstance(RecordType);
}
if (!LoadLine(pair, ref rec))
{
yield break;
}
//StoreState(e.Current, rec != null);
e.MoveNext();
if (rec == null)
{
continue;
}
if (!Configuration.SupportsMultiRecordTypes && Configuration.IsDynamicObject)
{
if (Configuration.AreAllFieldTypesNull && Configuration.AutoDiscoverFieldTypes && Configuration.MaxScanRows > 0 && recCount <= Configuration.MaxScanRows)
{
buffer.Add(rec);
RaiseRecordFieldTypeAssessment(recFieldTypes, (IDictionary <string, object>)rec, recCount == Configuration.MaxScanRows);
if (recCount == Configuration.MaxScanRows || e.Peek == null)
{
Configuration.UpdateFieldTypesIfAny(recFieldTypes);
var dict = recFieldTypes = Configuration.FixedLengthRecordFieldConfigurations.ToDictionary(i => i.Name, i => i.FieldType == null ? null : i.FieldType);
RaiseMembersDiscovered(dict);
foreach (object rec1 in buffer)
{
yield return(ConvertToNestedObjectIfApplicable(new ChoDynamicObject(MigrateToNewSchema(rec1 as IDictionary <string, object>, recFieldTypes)) as object, headerLineLoaded));
}
}
}
else
{
yield return(ConvertToNestedObjectIfApplicable(rec, headerLineLoaded));
}
}
else
{
yield return(rec);
}
if (Configuration.NotifyAfter > 0 && pair.Item1 % Configuration.NotifyAfter == 0)
{
if (RaisedRowsLoaded(pair.Item1))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
yield break;
}
}
if (doWhile != null)
{
doWhile = RaiseDoWhile(pair);
if (doWhile != null && doWhile.Value)
{
break;
}
}
}
}
}
private IEnumerable <object> AsEnumerable(object source, TraceSwitch traceSwitch, Func <object, bool?> filterFunc = null)
{
TraceSwitch = traceSwitch;
TextReader sr = source as TextReader;
ChoGuard.ArgumentNotNull(sr, "TextReader");
if (sr is StreamReader)
{
((StreamReader)sr).Seek(0, SeekOrigin.Begin);
}
if (!RaiseBeginLoad(sr))
{
yield break;
}
string[] commentTokens = Configuration.Comments;
bool? skip = false;
bool isRecordStartFound = false;
bool isRecordEndFound = false;
long seekOriginPos = sr is StreamReader ? ((StreamReader)sr).BaseStream.Position : 0;
List <string> headers = new List <string>();
Tuple <long, string> lastLine = null;
List <Tuple <long, string> > recLines = new List <Tuple <long, string> >();
long recNo = 0;
int loopCount = Configuration.AutoDiscoverColumns && Configuration.KVPRecordFieldConfigurations.Count == 0 ? 2 : 1;
bool isHeaderFound = loopCount == 1;
bool IsHeaderLoaded = false;
Tuple <long, string> pairIn;
bool abortRequested = false;
bool?skipUntil = true;
bool?doWhile = true;
for (int i = 0; i < loopCount; i++)
{
if (i == 1)
{
if (sr is StreamReader)
{
((StreamReader)sr).Seek(seekOriginPos, SeekOrigin.Begin);
}
TraceSwitch = traceSwitch;
}
else
{
TraceSwitch = ChoETLFramework.TraceSwitchOff;
}
lastLine = null;
recLines.Clear();
isRecordEndFound = false;
isRecordStartFound = false;
using (ChoPeekEnumerator <Tuple <long, string> > e = new ChoPeekEnumerator <Tuple <long, string> >(
new ChoIndexedEnumerator <string>(sr.ReadLines(Configuration.EOLDelimiter, Configuration.QuoteChar, Configuration.MayContainEOLInData)).ToEnumerable(),
(pair) =>
{
//bool isStateAvail = IsStateAvail();
skip = false;
if (skipUntil != null)
{
if (skipUntil.Value)
{
skipUntil = RaiseSkipUntil(pair);
if (skipUntil == null)
{
}
else
{
if (skipUntil.Value)
{
skip = skipUntil;
}
else
{
skip = true;
}
}
}
}
//if (isStateAvail)
//{
// if (!IsStateMatches(item))
// {
// skip = filterFunc != null ? filterFunc(item) : false;
// }
// else
// skip = true;
//}
//else
// skip = filterFunc != null ? filterFunc(item) : false;
if (skip == null)
{
return(null);
}
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, Environment.NewLine);
if (!skip.Value)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Loading line [{0}]...".FormatString(pair.Item1));
}
else
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Skipping line [{0}]...".FormatString(pair.Item1));
}
}
if (skip.Value)
{
return(skip);
}
//if (!(sr.BaseStream is MemoryStream))
// ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, ChoETLFramework.Switch.TraceVerbose, "Loading line [{0}]...".FormatString(item.Item1));
//if (Task != null)
// return !IsStateNOTExistsOrNOTMatch(item);
if (pair.Item2.IsNullOrWhiteSpace())
{
if (!Configuration.IgnoreEmptyLine)
{
throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pair.Item1));
}
else
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Ignoring empty line found at [{0}].".FormatString(pair.Item1));
}
return(true);
}
}
if (commentTokens != null && commentTokens.Length > 0)
{
foreach (string comment in commentTokens)
{
if (!pair.Item2.IsNull() && pair.Item2.StartsWith(comment, StringComparison.Ordinal)) //, true, Configuration.Culture))
{
if (TraceSwitch.TraceVerbose)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Comment line found at [{0}]...".FormatString(pair.Item1));
}
return(true);
}
}
}
if (!_configCheckDone)
{
Configuration.Validate(null);
var dict = Configuration.KVPRecordFieldConfigurations.ToDictionary(i1 => i1.Name, i1 => i1.FieldType == null ? null : i1.FieldType);
RaiseMembersDiscovered(dict);
Configuration.UpdateFieldTypesIfAny(dict);
_configCheckDone = true;
}
return(false);
}))
{
while (true)
{
pairIn = e.Peek;
if (!isRecordStartFound)
{
if (pairIn == null)
{
break;
}
lastLine = null;
recLines.Clear();
isRecordEndFound = false;
isRecordStartFound = true;
if (!Configuration.RecordStart.IsNullOrWhiteSpace())
{
//Move to record start
while (!(Configuration.IsRecordStartMatch(e.Peek.Item2)))
{
e.MoveNext();
if (e.Peek == null)
{
break;
}
}
}
if (e.Peek != null)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Record start found at [{0}] line...".FormatString(e.Peek.Item1));
}
e.MoveNext();
continue;
}
else
{
string recordEnd = !Configuration.RecordEnd.IsNullOrWhiteSpace() ? Configuration.RecordEnd : Configuration.RecordStart;
if (!recordEnd.IsNullOrWhiteSpace())
{
if (e.Peek == null)
{
if (Configuration.RecordEnd.IsNullOrWhiteSpace())
{
isRecordEndFound = true;
}
else
{
break;
}
}
else
{
//Move to record start
if (Configuration.IsRecordEndMatch(e.Peek.Item2))
{
isRecordEndFound = true;
isRecordStartFound = false;
if (e.Peek != null)
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Record end found at [{0}] line...".FormatString(e.Peek.Item1));
}
}
}
}
else if (e.Peek == null)
{
isRecordEndFound = true;
}
if (!isHeaderFound)
{
//if (isRecordEndFound && headers.Count == 0)
//{
// //throw new ChoParserException("Unexpected EOF found.");
//}
if (!isRecordEndFound)
{
e.MoveNext();
if (e.Peek != null)
{
//If line empty or line continuation, skip
if (pairIn.Item2.IsNullOrWhiteSpace() || IsLineContinuationCharFound(pairIn.Item2)) //.Item2[0] == ' ' || pairIn.Item2[0] == '\t')
{
}
else
{
string header = ToKVP(pairIn.Item1, pairIn.Item2).Key;
if (!header.IsNullOrWhiteSpace())
{
headers.Add(header);
}
}
}
}
else
{
Configuration.Validate(headers.ToArray());
isHeaderFound = true;
isRecordEndFound = false;
IsHeaderLoaded = true;
break;
}
}
else
{
if (!IsHeaderLoaded)
{
Configuration.Validate(new string[] { });
IsHeaderLoaded = true;
}
if (isRecordEndFound && recLines.Count == 0)
{
//throw new ChoParserException("Unexpected EOF found.");
}
else if (!isRecordEndFound)
{
e.MoveNext();
if (e.Peek != null)
{
//If line empty or line continuation, skip
if (pairIn.Item2.IsNullOrWhiteSpace())
{
if (!Configuration.IgnoreEmptyLine)
{
throw new ChoParserException("Empty line found at [{0}] location.".FormatString(pairIn.Item1));
}
else
{
Tuple <long, string> t = new Tuple <long, string>(lastLine.Item1, lastLine.Item2 + Configuration.EOLDelimiter);
recLines.RemoveAt(recLines.Count - 1);
recLines.Add(t);
}
}
else if (IsLineContinuationCharFound(pairIn.Item2)) //pairIn.Item2[0] == ' ' || pairIn.Item2[0] == '\t')
{
if (lastLine == null)
{
throw new ChoParserException("Unexpected line continuation found at {0} location.".FormatString(pairIn.Item1));
}
else
{
Tuple <long, string> t = new Tuple <long, string>(lastLine.Item1, lastLine.Item2 + Configuration.EOLDelimiter + pairIn.Item2);
recLines.RemoveAt(recLines.Count - 1);
recLines.Add(t);
}
}
else
{
lastLine = pairIn;
recLines.Add(pairIn);
}
}
}
else
{
object rec = Configuration.IsDynamicObject ? new ChoDynamicObject(new Dictionary <string, object>(Configuration.FileHeaderConfiguration.StringComparer))
{
ThrowExceptionIfPropNotExists = true,
AlternativeKeys = Configuration.AlternativeKeys
} : ChoActivator.CreateInstance(RecordType);
if (!LoadLines(new Tuple <long, List <Tuple <long, string> > >(++recNo, recLines), ref rec))
{
yield break;
}
isRecordStartFound = false;
//StoreState(e.Current, rec != null);
if (!Configuration.RecordEnd.IsNullOrWhiteSpace())
{
e.MoveNext();
}
if (rec == null)
{
continue;
}
yield return(rec);
if (Configuration.NotifyAfter > 0 && recNo % Configuration.NotifyAfter == 0)
{
if (RaisedRowsLoaded(recNo))
{
ChoETLFramework.WriteLog(TraceSwitch.TraceVerbose, "Abort requested.");
abortRequested = true;
yield break;
}
}
if (e.Peek == null)
{
break;
}
}
}
}
if (doWhile != null)
{
doWhile = RaiseDoWhile(pairIn);
if (doWhile != null && doWhile.Value)
{
break;
}
}
}
}
}
if (!abortRequested)
{
RaisedRowsLoaded(recNo, true);
}
RaiseEndLoad(sr);
}