public static CsvReader CreateReader(string csvFilePath, CsvClassMap classMap, CsvConfiguration csvConfiguration = null, CultureInfo culture = null) { var serviceLocator = ServiceLocator.Default; var csvReaderService = serviceLocator.ResolveType <ICsvReaderService>(); return(csvReaderService.CreateReader(csvFilePath, classMap, csvConfiguration, culture)); }
private static IEnumerable <T> ParseNutsFromData <T>(string filePath, CsvClassMap classMap) where T : class, INutEntry { using (var file = File.OpenText(filePath)) { // skip header file.ReadLine(); using (var csv = new CsvReader(file)) { csv.Configuration.HasHeaderRecord = true; csv.Configuration.CultureInfo = CultureInfo.InvariantCulture; csv.Configuration.RegisterClassMap(classMap); while (csv.Read()) { T entry = null; try { entry = csv.GetRecord <T>(); } catch (Exception e) { Console.WriteLine(e); } if (entry != null) { yield return(entry); } } } } }
private static void CsvParser <T, TMap>( string file, List <T> dataList, // ReSharper disable once SuggestBaseTypeForParameter CsvClassMap <TMap> typeMap) { try { using (var fs = new FileStream(Directory.GetCurrentDirectory() + file, FileMode.Open)) { using (var fileStreamReader = new StreamReader(fs)) { using (var fileReader = new CsvReader(fileStreamReader)) { fileReader.Configuration.AllowComments = true; fileReader.Configuration.Comment = '#'; fileReader.Configuration.RegisterClassMap(typeMap); dataList.AddRange(fileReader.GetRecords <T>()); } } } } catch (Exception e) { Console.WriteLine(@"Some exception loading file...."); Console.WriteLine(e.StackTrace + "\n\n"); Console.WriteLine(e.Message + "\n\n"); } }
public static IEnumerable <T> ReadCsv <T>(string csvFilePath, CsvClassMap map, Action <T> initializer = null, CsvConfiguration csvConfiguration = null, bool throwOnError = false, CultureInfo culture = null) { var serviceLocator = ServiceLocator.Default; var csvReaderService = serviceLocator.ResolveType <ICsvReaderService>(); return(csvReaderService.ReadCsv <T>(csvFilePath, map, initializer, csvConfiguration, throwOnError, culture)); }
public virtual IEnumerable <T> ReadCsv <T>(string csvFilePath, CsvClassMap map, Action <T> initializer = null, CsvConfiguration csvConfiguration = null, bool throwOnError = false, CultureInfo culture = null) { using (var csvReader = CreateReader(csvFilePath, map, csvConfiguration, culture)) { return(ReadData(csvFilePath, initializer, throwOnError, csvReader)); } }
public CsvReader CreateReader(string csvFilePath, CsvClassMap classMap, CsvConfiguration csvConfiguration = null, CultureInfo culture = null) { var csvReader = CreateCsvReader(csvFilePath, csvConfiguration ?? CreateDefaultCsvConfiguration(culture)); csvReader.Configuration.RegisterClassMap(classMap); return(csvReader); }
private ITextFileResultModel GetCsvTextFileResult <T>(IEnumerable <T> data, CsvClassMap <T> csvProfile, string fileName) { return(new TextFileResultModel { FileName = fileName, Content = csvHelper.ToCsv(data, csvProfile) }); }
/// <summary> /// Adds the properties from the mapping. This will recursively /// traverse the mapping tree and add all properties for /// reference maps. /// </summary> /// <param name="properties">The properties to be added to.</param> /// <param name="mapping">The mapping where the properties are added from.</param> protected virtual void AddProperties(CsvPropertyMapCollection properties, CsvClassMap mapping) { properties.AddRange(mapping.PropertyMaps); foreach (var refMap in mapping.ReferenceMaps) { AddProperties(properties, refMap.Mapping); } }
public ITextFileResultModel GetGroupedTasksCsv <T>( IGroupedTasksCsvRequest <T> csvChartRequest, string chartName, CsvClassMap <TaskInfo> csvProfile, bool showLastCompletedOrCanceledStatus, bool onlyWithTime = false) { IEnumerable <IDayAssign> dayAssigns = dayAssignService.GetByIds(csvChartRequest.GroupedTasksIds.SelectMany(g => g.Value)); IEnumerable <TaskInfo> taskInfoList = tasksInfoBuilder.GetTaskInfoList(dayAssigns, showLastCompletedOrCanceledStatus, onlyWithTime); string fileName = GetChartCsvFileName(chartName, csvChartRequest.RangeDateString); var result = GetCsvTextFileResult(taskInfoList, csvProfile, fileName); return(result); }
private static CsvReader GetCsvReader(StreamReader sr, CsvClassMap mapper) { var csv = new CsvReader(sr); csv.Configuration.IsHeaderCaseSensitive = false; csv.Configuration.WillThrowOnMissingField = false; // set true to enforce required columns csv.Configuration.RegisterClassMap(mapper); return(csv); }
private void RemoveReadOnlyProps(CsvClassMap map) { var parametersProperties = map.PropertyMaps; var readonlyProps = parametersProperties .Where(pm => !pm.Data.Property.CanWrite) .ToArray(); foreach (var prop in readonlyProps) { parametersProperties.Remove(prop); } }
/// <summary> /// Adds the specified map for it's record type. If a map /// already exists for the record type, the specified /// map will replace it. /// </summary> /// <param name="map">The map.</param> public void Add(CsvClassMap map) { var type = GetGenericCsvClassMapType(map.GetType()).GetGenericArguments().First(); if (data.ContainsKey(type)) { data[type] = map; } else { data.Add(type, map); } }
public string ToCsv <T>(IEnumerable <T> records, CsvClassMap <T> csvClassMap) { using (var writer = new StringWriter()) using (var csv = new CsvWriter(writer)) { csv.Configuration.Delimiter = appSettingHelper.GetAppSetting <string>(Constants.Constants.AppSetting.CsvSeparator); csv.Configuration.Encoding = Encoding.Unicode; csv.Configuration.RegisterClassMap(csvClassMap); csv.Configuration.Quote = '"'; csv.WriteRecords(records); var result = writer.ToString(); return(result); } }
private Dataset <T> GetDataset <T>(string path1, string path2, CsvClassMap <T> map) { IDatasetReader <T> reader1 = null, reader2 = null; Dataset <T> dataset; switch (Parameters.GetValueOrDefault("data-format", "")) { case ("csv"): string sep = ","; if (Parameters.GetValueOrDefault("csv-sep", "") != "") { sep = Parameters["csv-sep"]; } var csvConfing = new CsvConfiguration() { Delimiter = sep }; reader1 = new CsvReader <T>(path1, csvConfing, map); if (path2 != "") { reader2 = new CsvReader <T>(path2, csvConfing, map); } break; case (""): throw new WrapRecException("The parameter 'data-format' should be specified."); break; } if (path2 == "") { if (Parameters.GetValueOrDefault("testportion", "") == "") { throw new WrapRecException("The parameter 'testportion' should be specified."); } dataset = new Dataset <T>(reader1, float.Parse(Parameters["testportion"])); } else { dataset = new Dataset <T>(reader1, reader2); } return(dataset); }
private static void Configure <T>(CsvClassMap map) { map.ReferenceMaps.Clear(); //http://stackoverflow.com/questions/1571022/how-would-i-know-if-a-property-is-a-generic-collection Type tColl = typeof(ICollection <>); foreach (var pm in map.PropertyMaps.Where(p => { Type t = p.Data.Member.MemberType(); return(t.IsGenericType && tColl.IsAssignableFrom(t.GetGenericTypeDefinition()) || t.GetInterfaces().Any(x => x.IsGenericType && x.GetGenericTypeDefinition() == tColl)); }).ToArray()) { map.PropertyMaps.Remove(pm); } }
/// <summary> /// Adds the specified map for it's record type. If a map /// already exists for the record type, the specified /// map will replace it. /// </summary> /// <param name="map">The map.</param> public void Add(CsvClassMap map) { #if WINRT_4_5 var type = map.GetType().GetTypeInfo().BaseType.GenericTypeArguments.First(); #else var type = map.GetType().BaseType.GetGenericArguments().First(); #endif if (data.ContainsKey(type)) { data[type] = map; } else { data.Add(type, map); } }
/// <summary> /// Creates the property bindings for the given <see cref="CsvClassMap"/>. /// </summary> /// <param name="mapping">The mapping to create the bindings for.</param> /// <param name="recordType">The type of record.</param> /// <param name="bindings">The bindings that will be added to from the mapping.</param> protected virtual void CreatePropertyBindingsForMapping(CsvClassMap mapping, Type recordType, List <MemberBinding> bindings) { AddPropertyBindings(mapping.PropertyMaps, bindings); foreach (var referenceMap in mapping.ReferenceMaps) { if (!CanRead(referenceMap)) { continue; } var referenceBindings = new List <MemberBinding>(); CreatePropertyBindingsForMapping(referenceMap.Mapping, referenceMap.Property.PropertyType, referenceBindings); var referenceBody = Expression.MemberInit(Expression.New(referenceMap.Property.PropertyType), referenceBindings); bindings.Add(Expression.Bind(referenceMap.Property, referenceBody)); } }
public static void SeedFromStream <T>(this IDbSet <T> dbSet, Stream stream, Expression <Func <T, object> > identifierExpression, params CsvColumnMapping <T>[] additionalMappings) where T : class { try { using (var reader = new StreamReader(stream)) { var csvReader = new CsvReader(reader); CsvClassMap map = csvReader.Configuration.AutoMap <T>(); map.ReferenceMaps.Clear(); csvReader.Configuration.RegisterClassMap(map); csvReader.Configuration.WillThrowOnMissingField = false; while (csvReader.Read()) { var entity = csvReader.GetRecord <T>(); foreach (var additionalMapping in additionalMappings) { additionalMapping.Execute(entity, csvReader.GetField(additionalMapping.CsvColumnName)); } try { dbSet.AddOrUpdate(identifierExpression, entity); } catch (Exception exception) { string message = string.Format("Error adding {0} to column {1}", entity, identifierExpression); throw new Exception(message, exception); } } } } catch (Exception exception) { string message = string.Format("Error seeding dbSet<{0}>: {1}", dbSet.GetType().GenericTypeArguments[0].FullName, exception); Exception innerException = exception.GetType().IsSerializable ? exception : null; throw new Exception(message, innerException); } }
/// <summary> /// Creates the property bindings for the given <see cref="CsvClassMap"/>. /// </summary> /// <param name="mapping">The mapping to create the bindings for.</param> /// <param name="recordType">The type of record.</param> /// <param name="bindings">The bindings that will be added to from the mapping.</param> protected virtual void CreatePropertyBindingsForMapping(CsvClassMap mapping, Type recordType, List <MemberBinding> bindings) { // If there is no property mappings yet, use attribute mappings. if (mapping == null) { // TODO: auto class mapping throw new CsvConfigurationException("No mapping has been created. Use Configuration.ClassMapping to create a map."); } AddPropertyBindings(mapping.PropertyMaps, bindings); foreach (var referenceMap in mapping.ReferenceMaps) { var referenceBindings = new List <MemberBinding>(); CreatePropertyBindingsForMapping(referenceMap.Mapping, referenceMap.Property.PropertyType, referenceBindings); var referenceBody = Expression.MemberInit(Expression.New(referenceMap.Property.PropertyType), referenceBindings); bindings.Add(Expression.Bind(referenceMap.Property, referenceBody)); } }
/// <summary> /// /// </summary> /// <param name="filePath">csv path</param> /// <param name="map">e.g. new ClassLib.ConfirmationCSVMap()</param> /// <param name="type">e.g. typeof(ClassLib.ConfirmationCSV)</param> /// <returns></returns> public List <object> ReadCSVToList(string filePath, CsvClassMap map, Type type, CsvConfiguration csvConfig) { try { StreamReader file = new StreamReader(filePath); using (CsvReader objCsvReader = new CsvReader(file)) { objCsvReader.Configuration.RegisterClassMap(map); objCsvReader.Configuration.Delimiter = csvConfig.Delimiter; objCsvReader.Configuration.TrimFields = true; objCsvReader.Configuration.WillThrowOnMissingField = true; objCsvReader.Configuration.HasHeaderRecord = csvConfig.HasHeaderRecord; //default is true return(objCsvReader.GetRecords(type).ToList()); } } catch (Exception ex) { throw new Exception("Fail to Build CSV List from CSV File" + ex.Message); } }
/// <summary> /// Creates a property expression for the given property on the record. /// This will recursively traverse the mapping to find the property /// and create a safe property accessor for each level as it goes. /// </summary> /// <param name="recordExpression">The current property expression.</param> /// <param name="mapping">The mapping to look for the property to map on.</param> /// <param name="propertyMap">The property map to look for on the mapping.</param> /// <returns>An Expression to access the given property.</returns> protected virtual Expression CreatePropertyExpression(Expression recordExpression, CsvClassMap mapping, CsvPropertyMap propertyMap) { if (mapping.PropertyMaps.Any(pm => pm == propertyMap)) { // The property is on this level. return(Expression.Property(recordExpression, propertyMap.Data.Property)); } // The property isn't on this level of the mapping. // We need to search down through the reference maps. foreach (var refMap in mapping.ReferenceMaps) { var wrapped = Expression.Property(recordExpression, refMap.Data.Property); var propertyExpression = CreatePropertyExpression(wrapped, refMap.Data.Mapping, propertyMap); if (propertyExpression == null) { continue; } if (refMap.Data.Property.PropertyType.GetTypeInfo().IsValueType) { return(propertyExpression); } var nullCheckExpression = Expression.Equal(wrapped, Expression.Constant(null)); var isValueType = propertyMap.Data.Property.PropertyType.GetTypeInfo().IsValueType; var isGenericType = isValueType && propertyMap.Data.Property.PropertyType.GetTypeInfo().IsGenericType; Type propertyType; if (isValueType && !isGenericType && !configuration.UseNewObjectForNullReferenceProperties) { propertyType = typeof(Nullable <>).MakeGenericType(propertyMap.Data.Property.PropertyType); propertyExpression = Expression.Convert(propertyExpression, propertyType); } else { propertyType = propertyMap.Data.Property.PropertyType; } var defaultValueExpression = isValueType && !isGenericType ? (Expression)Expression.New(propertyType) : Expression.Constant(null, propertyType); var conditionExpression = Expression.Condition(nullCheckExpression, defaultValueExpression, propertyExpression); return(conditionExpression); } return(null); }
/// <summary> /// Creates a property expression for the given property on the record. /// This will recursively traverse the mapping to find the property /// and create a safe property accessor for each level as it goes. /// </summary> /// <param name="recordExpression">The current property expression.</param> /// <param name="mapping">The mapping to look for the property to map on.</param> /// <param name="propertyMap">The property map to look for on the mapping.</param> /// <returns>An Expression to access the given property.</returns> protected virtual Expression CreatePropertyExpression(Expression recordExpression, CsvClassMap mapping, CsvPropertyMap propertyMap) { if (mapping.PropertyMaps.Any(pm => pm == propertyMap)) { // The property is on this level. return(Expression.Property(recordExpression, propertyMap.Data.Property)); } // The property isn't on this level of the mapping. // We need to search down through the reference maps. foreach (var refMap in mapping.ReferenceMaps) { var wrapped = Expression.Property(recordExpression, refMap.Property); var propertyExpression = CreatePropertyExpression(wrapped, refMap.Mapping, propertyMap); if (propertyExpression == null) { continue; } var isReferenceValueType = refMap.Property.PropertyType.IsValueType; if (isReferenceValueType) { return(propertyExpression); } var nullCheckExpression = Expression.Equal(wrapped, Expression.Constant(null)); var isValueType = propertyMap.Data.Property.PropertyType.IsValueType; var defaultValueExpression = isValueType ? (Expression)Expression.New(propertyMap.Data.Property.PropertyType) : Expression.Constant(null, propertyMap.Data.Property.PropertyType); var conditionExpression = Expression.Condition(nullCheckExpression, defaultValueExpression, propertyExpression); return(conditionExpression); } return(null); }
protected void DoExport <TCsvClass, TClass>(string fileName, int chunkSize, string entitiesType, Func <ICollection <TCsvClass> > entityFactory, CsvClassMap <TClass> entityClassMap, ExportPushNotification notification) { Action <ExportImportProgressInfo> progressCallback = x => { notification.Description = x.Description; notification.TotalCount = x.TotalCount; notification.ProcessedCount = x.ProcessedCount; notification.Errors = x.Errors; _pushNotifier.Upsert(notification); }; var progressInfo = new ExportImportProgressInfo { Description = string.Format("Loading {0}...", entitiesType) }; progressCallback(progressInfo); var updateProgress = new Action(() => { progressInfo.Description = string.Format("{0} of {1} {2} processed", progressInfo.ProcessedCount, progressInfo.TotalCount, entitiesType); progressCallback(progressInfo); }); var updateProgressWithThrottling = updateProgress.Throttle(TimeSpan.FromSeconds(1)); var relativeUrl = "temp/" + fileName + ".zip"; using (var blobStream = _blobStorageProvider.OpenWrite(relativeUrl)) { try { // Because of Recommendation API file uploading limit, we need to split our csv file to parts with size no more than this limit using (var archive = new ZipArchive(blobStream, ZipArchiveMode.Create, true, new UTF8Encoding(false))) { var partIndex = 1; using (var stream = new MemoryStream()) { using (var streamWriter = new StreamWriter(stream, new UTF8Encoding(false), 1024, true) { AutoFlush = true }) { using (var csvWriter = new CsvWriter(streamWriter)) { progressCallback(progressInfo); var entities = entityFactory().ToArray(); csvWriter.Configuration.Delimiter = ","; csvWriter.Configuration.RegisterClassMap(entityClassMap); progressInfo.TotalCount = entities.Length; for (var index = 0; index < entities.Length; index++) { try { var previousSize = (int)stream.Length; csvWriter.WriteRecord(entities[index]); if (stream.Length > chunkSize) { WriteEntry(archive, fileName, ref partIndex, x => x.Write(stream.GetBuffer(), 0, previousSize - 2)); stream.SetLength(0); --index; } } catch (Exception ex) { progressInfo.Errors.Add(ex.ToString()); progressCallback(progressInfo); } progressInfo.ProcessedCount = index + 1; updateProgressWithThrottling(); } } } WriteEntry(archive, fileName, ref partIndex, stream.WriteTo, true); } } updateProgress(); notification.DownloadUrl = _blobUrlResolver.GetAbsoluteUrl(relativeUrl); } catch (Exception ex) { notification.Description = "Export failed"; notification.Errors.Add(ex.ExpandExceptionMessage()); } finally { notification.Description = "Export finished"; notification.Finished = DateTime.UtcNow; _pushNotifier.Upsert(notification); } } }
public virtual void WriteCsv(IEnumerable records, StreamWriter streamWriter, Type recordType, CsvClassMap csvMap, CsvConfiguration csvConfiguration = null, bool throwOnError = false, CultureInfo cultureInfo = null) { using (var csvWriter = CreateWriter(streamWriter, csvConfiguration ?? CreateDefaultCsvConfiguration(cultureInfo))) { if (csvMap != null) { csvWriter.Configuration.RegisterClassMap(csvMap); } WriteRecords(records, recordType, throwOnError, csvWriter); } }
/// <summary> /// Creates a property expression for the given property on the record. /// This will recursively traverse the mapping to find the property /// and create a safe property accessor for each level as it goes. /// </summary> /// <param name="recordExpression">The current property expression.</param> /// <param name="mapping">The mapping to look for the property to map on.</param> /// <param name="propertyMap">The property map to look for on the mapping.</param> /// <returns>An Expression to access the given property.</returns> protected virtual Expression CreatePropertyExpression(Expression recordExpression, CsvClassMap mapping, CsvPropertyMap propertyMap) { Type propertyType; Expression expression; Expression expression1; if (mapping.PropertyMaps.Any(pm => pm == propertyMap)) { return(Expression.Property(recordExpression, propertyMap.Data.Property)); } var enumerator = mapping.ReferenceMaps.GetEnumerator(); try { while (enumerator.MoveNext()) { var current = enumerator.Current; var memberExpression = Expression.Property(recordExpression, current.Data.Property); var expression2 = CreatePropertyExpression(memberExpression, current.Data.Mapping, propertyMap); if (expression2 == null) { continue; } if (!current.Data.Property.PropertyType.GetTypeInfo().IsValueType) { var binaryExpression = Expression.Equal(memberExpression, Expression.Constant(null)); bool isValueType = propertyMap.Data.Property.PropertyType.GetTypeInfo().IsValueType; bool flag = !isValueType ? false : propertyMap.Data.Property.PropertyType.GetTypeInfo().IsGenericType; if (!isValueType || flag || configuration.UseNewObjectForNullReferenceProperties) { propertyType = propertyMap.Data.Property.PropertyType; } else { propertyType = typeof(Nullable <>).MakeGenericType(propertyMap.Data.Property.PropertyType); expression2 = Expression.Convert(expression2, propertyType); } if (!isValueType || flag) { expression1 = Expression.Constant(null, propertyType); } else { expression1 = Expression.New(propertyType); } expression = Expression.Condition(binaryExpression, expression1, expression2); return(expression); } else { expression = expression2; return(expression); } } return(null); } finally { ((IDisposable)enumerator).Dispose(); } return(expression); }
public static CsvReader GetCsvReader(string filePath, bool skipFirstLine = true, string delimiter = ",", Func <string[], bool> shouldSkipRecord = null, CsvClassMap csvClassMap = null) { CsvReader csvReader = new CsvReader( File.OpenText(filePath), new CsvConfiguration { Delimiter = delimiter, HasHeaderRecord = skipFirstLine, SkipEmptyRecords = true, ShouldSkipRecord = shouldSkipRecord, IgnoreBlankLines = true, DetectColumnCountChanges = true, WillThrowOnMissingField = true }); if (csvClassMap != null) { csvReader.Configuration.RegisterClassMap(csvClassMap); } return(csvReader); }
public static CsvPropertyMap GetPropertyMap <T>(this CsvClassMap <T> classMap, Expression <Func <T, object> > propertyExpression) { var property = propertyExpression.GetProperty(); return(classMap.PropertyMaps.Single(cpm => cpm.Data.Property.PropertyInfoMetaDataEquality(property))); }
/// <summary> /// Creates a parameter for the given property. This will /// recursively traverse the mapping to to find property /// mapping and create a new property access for each /// reference map it goes through. /// </summary> /// <param name="parameter">The current parameter.</param> /// <param name="mapping">The mapping to look for the property map on.</param> /// <param name="propertyMap">The property map to look for on the mapping.</param> /// <returns>A <see cref="ParameterExpression"/> to access the given property map.</returns> protected virtual Expression CreateParameterForProperty(Expression parameter, CsvClassMap mapping, CsvPropertyMap propertyMap) { var propertyMapping = mapping.PropertyMaps.SingleOrDefault(pm => pm == propertyMap); if (propertyMapping != null) { // If the property map exists on this level of the class // mapping, we can return the parameter. return(parameter); } // The property isn't on this level of the mapping. // We need to search down through the reference maps. foreach (var refMap in mapping.ReferenceMaps) { var wrappedParameter = Expression.Property(parameter, refMap.Property); var param = CreateParameterForProperty(wrappedParameter, refMap.Mapping, propertyMap); if (param != null) { return(param); } } return(null); }
private static IEnumerable <CsvType> CsvParseFile <CsvType>(string ctaDataFile, CsvClassMap classMap) { using (TextReader textReader = File.OpenText(ctaDataFile)) { var csv = new CsvReader(textReader); csv.Configuration.RegisterClassMap(classMap); csv.Configuration.WillThrowOnMissingField = false; csv.Configuration.TrimFields = true; var records = csv.GetRecords <CsvType>(); return(records.ToList()); } }
public virtual void WriteCsv <TRecord>(IEnumerable <TRecord> records, string csvFilePath, CsvClassMap csvMap, CsvConfiguration csvConfiguration = null, bool throwOnError = false, CultureInfo cultureInfo = null) { WriteCsv(records, csvFilePath, typeof(TRecord), csvMap, csvConfiguration, throwOnError, cultureInfo); }