public string SerializeList <T>(List <T> list)
{
Type t = typeof(T);
PropertyInfo[] properties = t.GetRuntimeProperties().Where(x => !x.SetMethod.IsVirtual).ToArray();
CsvClassMap map = new DefaultCsvClassMap <T>();
foreach (PropertyInfo pi in properties)
{
CsvPropertyMap propMap = new CsvPropertyMap(pi);
propMap.Name(pi.Name);
propMap.Index(0);
map.PropertyMaps.Add(propMap);
}
StringWriter sw = new StringWriter();
CsvWriter writer = new CsvWriter(sw);
writer.Configuration.Delimiter = delimiter;
writer.Configuration.RegisterClassMap(map);
writer.WriteRecords(list);
return(sw.ToString());
}
public List <T> DeserializeList <T>(string list)
{
if (list != "")
{
PropertyInfo[] properties = typeof(T).GetRuntimeProperties().Where(x => !x.SetMethod.IsVirtual).ToArray();
CsvClassMap map = new DefaultCsvClassMap <T>();
foreach (PropertyInfo pi in properties)
{
CsvPropertyMap propMap = new CsvPropertyMap(pi);
propMap.Name(pi.Name);
propMap.Index(0);
map.PropertyMaps.Add(propMap);
}
TextReader tr = new StringReader(list);
CsvReader reader = new CsvReader(tr);
reader.Configuration.Delimiter = delimiter;
reader.Configuration.RegisterClassMap(map);
List <T> result = new List <T>();
foreach (T record in reader.GetRecords <T>())
{
result.Add(record);
}
return(result);
}
return(new List <T>());
}
public async Task <List <T> > ReadRecords <T>(IS3Path path, Func <T, bool> predicate, bool isOptional = false)
{
_logger.Information("Reading {Type} records from {@S3Path}", typeof(T).Name, path);
var configuration = new CsvConfiguration
{
Delimiter = "|",
QuoteNoFields = true,
HasHeaderRecord = true
};
var customerMap = new DefaultCsvClassMap <T>();
foreach (var prop in typeof(T).GetProperties())
{
var name = prop.Name.ToSnakeCase();
var map = new CsvPropertyMap(prop).Name(name);
customerMap.PropertyMaps.Add(map);
}
configuration.RegisterClassMap(customerMap);
try
{
using (var response = await _s3Client.GetObjectAsync(new GetObjectRequest {
BucketName = path.BucketName, Key = path.Key
}))
{
using (var responseStream = response.ResponseStream)
{
using (var streamReader = new StreamReader(responseStream))
{
using (var reader = new CsvReader(streamReader, configuration))
{
var results = reader.GetRecords <T>();
if (predicate != null)
{
results = results.Where(predicate);
}
return(results.ToList());
}
}
}
}
}
catch (AmazonS3Exception ex)
{
if (ex.ErrorCode == "NoSuchKey" && isOptional)
{
return(new List <T>());
}
throw;
}
catch (Exception e)
{
_logger.Fatal("Exception when trying to read csv file: {@Message}", e.Message);
throw;
}
}
public DataImportClient()
{
/*
* groups = fields.GroupBy(
* f => f.Substring(
* 0, f.IndexOf('.')
* ), f => f.Remove(
* 0, f.IndexOf('.') + 1
* )
* ).Select(g => new Group(
* g.Key, g.ToArray()
* )).OrderBy(g => g.prefix).ToArray();*/
cfg = new CsvConfiguration();
cfg.Delimiter = "\t";
cfg.DetectColumnCountChanges = true;
cfg.TrimFields = true;
//cfg.RegisterClassMap(typeof(Group));
csvMapping = (Lookup <string, Lookup <string, string> >)fields.ToLookup(
f => f.Substring(0, f.IndexOf('.')),
f => (Lookup <string, string>)Enumerable
.Empty <string>().ToLookup(
p => f.Remove(
0, f.IndexOf('.'))));
/* var csvMap = new DefaultCsvClassMap<Lookup<string, string>>();
* foreach(var group in csvMapping)
* {
* foreach(var subgroup in group)
* {
* foreach(var item in subgroup)
* {
* var newMap = new CsvPropertyMap(
* typeof(IGrouping<string, string>).GetType().GetProperty(item.Key)
* );
* }
* }
* }*/
fileFormatMap = new DefaultCsvClassMap <FileFormatSkeleton>();
Type ffs = typeof(FileFormatSkeleton);
ffsFields = ffs.GetFields(BindingFlags.SetProperty).ToDictionary(f => f.Name, f => f);
var fieldNames = fields.Select(f => new KeyValuePair <string, string>(f, f.Replace('.', '_')));
foreach (var field in fieldNames)
{
var newMap = new CsvPropertyMap(
ffs.GetProperty(field.Value));
newMap.Name(field.Key);
fileFormatMap.PropertyMaps.Add(newMap);
}
cfg.RegisterClassMap(fileFormatMap);
}
static void Main()
{
using (TextReader tr = File.OpenText("exp.csv"))
{
using (CsvHelper.CsvReader reader = new CsvHelper.CsvReader(tr))
{
var tmp2 = new DefaultCsvClassMap <Record>();
tmp2.Map(c => c.Date);
var tmp = reader.GetRecords <Record>().ToList();
}
}
}
public DataImportClient()
{
fields.InsertRange(
fields.IndexOf(
fields.Last(
f => f.StartsWith("L.")
)
) + 1, Enumerable.Range(1, 31).Select(
n => "L.UserDef" + n.ToString()
)
);
fields.RemoveRange(fields.IndexOf("L.UserDef14"), 10);
fields.Remove("L.UserDef4");
fields.Remove("L.UserDef6");
fields.Remove("L.UserDef7");
fields.Remove("L.UserDef9");
recordDict = new List <Dictionary <string, string> >();
/*
* groups = fields.GroupBy(
* f => f.Substring(
* 0, f.IndexOf('.')
* ), f => f.Remove(
* 0, f.IndexOf('.') + 1
* )
* ).Select(g => new Group(
* g.Key, g.ToArray()
* )).OrderBy(g => g.prefix).ToArray();*/
cfg = new CsvConfiguration();
cfg.Delimiter = "\t";
cfg.DetectColumnCountChanges = true;
cfg.TrimFields = true;
//cfg.RegisterClassMap(typeof(Group));
var grps = fields.GroupBy(f => f.Substring(0, f.IndexOf('.')));
csvMapping = (
Lookup <string, Lookup <string, string> >
)grps.ToLookup(
f => f.Key, f => (
Lookup <string, string>
)f.ToLookup(
g => g.Remove(
0, g.IndexOf('.') + 1)));
/* var csvMap = new DefaultCsvClassMap<Lookup<string, string>>();
* foreach(var group in csvMapping)
* {
* foreach(var subgroup in group)
* {
* foreach(var item in subgroup)
* {
* var newMap = new CsvPropertyMap(
* typeof(IGrouping<string, string>).GetType().GetProperty(item.Key)
* );
* }
* }
* }*/
fileFormatMap = new DefaultCsvClassMap <FileFormatSkeleton>();
Type ffs = typeof(FileFormatSkeleton);
ffsFields = ffs.GetRuntimeFields() /*ffs.GetFields(BindingFlags.SetProperty)*/.ToDictionary(f => f.Name, f => f);
var fieldNames = fields.Select(f => new KeyValuePair <string, string>(f, f.Replace('.', '_')));
foreach (var field in fieldNames)
{
try
{
var propInfo = ffs.GetProperty(field.Value);
var newMap = new CsvPropertyMap(propInfo);
newMap.Name(field.Key);
fileFormatMap.PropertyMaps.Add(newMap);
}
catch
{
break;
}
}
//cfg.RegisterClassMap(fileFormatMap);
/*FileStream fs = new FileStream(
* @"C:\DocUploads\logs\Export.txt",
* FileMode.Create,
* FileAccess.ReadWrite,
* FileShare.Read
* );
* StreamWriter sw = new StreamWriter(fs);
* CsvWriter writer = new CsvWriter(sw, cfg);*/
}
public async Task WriteRecords <T>(IS3Path path, IEnumerable <T> records)
{
_logger.Information("Writing {Type} records to {@S3Path}", typeof(T).Name, path);
var configuration = new CsvConfiguration
{
Delimiter = "|",
QuoteNoFields = true
};
var customerMap = new DefaultCsvClassMap <T>();
foreach (var prop in typeof(T).GetProperties())
{
var name = prop.Name.ToSnakeCase();
var map = new CsvPropertyMap(prop).Name(name);
customerMap.PropertyMaps.Add(map);
}
configuration.RegisterClassMap(customerMap);
var recordCount = 0;
using (var temp = new TempFile(".md-ra-poc.csv"))
{
_logger.Debug("Writing records to temp path {TempPath}", temp.FullPath);
using (var textWriter = File.CreateText(temp.FullPath))
{
using (var csvWriter = new CsvWriter(textWriter, configuration))
{
// Write header manually because WriteHeader append an index
foreach (var property in csvWriter.Configuration.Maps[typeof(T)].PropertyMaps)
{
csvWriter.WriteField(property.Data.Names.FirstOrDefault());
}
// Advance past header
csvWriter.NextRecord();
foreach (var record in records)
{
csvWriter.WriteRecord(record);
csvWriter.NextRecord();
recordCount++;
}
}
}
_logger.Debug("Wrote CSV to {HierarchyPath}. Records: {Records}. Size: {Bytes} bytes", temp.FullPath, recordCount, new FileInfo(temp.FullPath).Length);
_logger.Debug("Uploading {TempPath} to S3 HierarchyPath: {@S3Path}", temp.FullPath, path);
var transferUtility = new TransferUtility(_s3Client);
await transferUtility.UploadAsync(new TransferUtilityUploadRequest
{
BucketName = path.BucketName,
Key = path.Key,
FilePath = temp.FullPath
}, CancellationToken.None);
_logger.Debug("Uploaded {TempPath} to S3 HierarchyPath: {@S3Path}", temp.FullPath, path);
}
_logger.Information("Uploaded {Records} {Type} records to {@S3Path}", recordCount, typeof(T).Name, path);
}
