/// <summary>
/// Sets the parameterss of the courses and adds them to the courses list.
/// </summary>
/// <param name="reader">The CsvReader.</param>
/// <returns>courseList</returns>
private static List <Course> parseRecordsForCourse(CsvHelper.CsvReader reader)
{
// make sure not at header or end of file
List <Course> courseList = new List <Course>();
Regex rgx = new Regex(@"(\S*,.*?)\s*\(\d*\)");
Match match;
string NewInstructor;
while ((fileHasMoreRecords = reader.Read()) && courseHasMoreRecords(reader))
{
Course course = reader.GetRecord <Course>();
course.SetAllDerivedProperties();
NewInstructor = "";
match = rgx.Match(course.Instructor);
while (match.Success)
{
NewInstructor += match.Groups[1].Value + "; ";
match = match.NextMatch();
}
if (NewInstructor != "")
{
course.Instructor = NewInstructor.Substring(0, NewInstructor.Length - 2);
}
courseList.Add(course); //add course to course list.
}
return(courseList);
}
public static void ReadCsv <TRow>(
string currentFolder,
string filename,
Action <TRow> rowReadCallback,
Action <IReaderConfiguration> changeConfiguration = null)
{
using (var csvStream = File.OpenText(Path.Combine(currentFolder, "DataFiles", filename)))
using (var csvReader = new CsvReader(csvStream, true))
{
changeConfiguration?.Invoke(csvReader.Configuration);
if (csvReader.Read() && csvReader.ReadHeader()) //precaution
{
csvReader.ValidateHeader <TRow>();
csvReader.Configuration.BadDataFound = null;
while (csvReader.Read())
{
try
{
rowReadCallback(csvReader.GetRecord <TRow>());
}
catch (Exception)
{
}
}
}
}
}
/// <summary>
/// Read the contents of a CSV file following the CSVRecord format and place its strings into the provided string collection
/// </summary>
/// <param name="fileName"></param>
/// <param name="collection"></param>
public void ReadCSVFile(string fileName, StringCollection collection)
{
FileStream fs = new FileStream(fileName, FileMode.Open);
StreamReader sr = new StreamReader(fs);
var csvReader = new CsvHelper.CsvReader(sr);
csvReader.Configuration.MissingFieldFound = null;
csvReader.Configuration.RegisterClassMap <CSVRecordMap>();
csvReader.Read();
csvReader.ReadHeader(); // not checking correctness of header atm
while (csvReader.Read())
{
var record = csvReader.GetRecord <CSVRecord>();
var id = record.stringID;
var translatedText = record.translatedText;
if (translatedText == "")
{
translatedText = record.originalText; // replace translated text with original text if it's blank
}
collection.AddString(id, translatedText);
}
sr.Close();
fs.Close();
}
private void ReadSymbolFile()
{
string folder = GFile.ExePath;
string filename = "system.symbol_ids.DF.csv";
StreamReader reader = null;
try
{
reader = new StreamReader(Path.Combine(folder, filename));
var csv = new CsvHelper.CsvReader(reader);
csv.Read();
csv.ReadHeader();
//csv.ValidateHeader<XSymbol>();
while (csv.Read())
{
var entry = csv.GetRecord <XSymbol>();
if (!m_symbols.ContainsKey(entry.Exchange))
{
m_symbols[entry.Exchange] = new SortedDictionary <string, XSymbol>();
}
m_symbols[entry.Exchange][entry.SymbolId] = entry;
}
}
catch (Exception ex)
{
Console.WriteLine("SymbolManager ERROR: {0}", ex.Message);
}
finally
{
if (reader != null)
{
reader.Close();
}
}
}
public async Task Load()
{
HttpWebRequest req = WebRequest.CreateHttp(Url + Build);
using (HttpWebResponse resp = (HttpWebResponse)await req.GetResponseAsync())
using (var respStream = resp.GetResponseStream())
using (var sr = new StreamReader(respStream))
using (var csv = new CsvHelper.CsvReader(sr))
{
while (await csv.ReadAsync())
{
var record = csv.GetRecord <dynamic>();
uint id = uint.Parse(record.ID);
ComponentGender gender = (ComponentGender)byte.Parse(record.GenderIndex);
ComponentRace race = (ComponentRace)byte.Parse(record.RaceID);
if (race == ComponentRace.None)
{
Add(id, "");
}
else
{
Add(id, $"_{race}_{gender}");
}
}
}
}
private void OpenFileBtn_Click(object sender, EventArgs e)
{
try
{
if (openFileDialog1.ShowDialog() == DialogResult.OK)
{
fileReader = new StreamReader(openFileDialog1.FileName);
fileDirectoryTxtB.Text = openFileDialog1.FileName;
CsvHelper.CsvReader reader = new CsvHelper.CsvReader(fileReader);
int count = 0;
while (reader.Read() && count <= 500000)
{
count++;
retailDatas.Add(reader.GetRecord <RetailData>());
}
}
PopuateData(retailDatas);
tabControl1.Visible = true;
tabControl1.Enabled = true;
}
catch (Exception ex)
{
MessageBox.Show(ex.Message);
}
}
public async Task Load()
{
HttpWebRequest req = WebRequest.CreateHttp(Url + Build);
using (HttpWebResponse resp = (HttpWebResponse)await req.GetResponseAsync())
using (var respStream = resp.GetResponseStream())
using (var sr = new StreamReader(respStream))
using (var csv = new CsvHelper.CsvReader(sr))
{
while (await csv.ReadAsync())
{
var record = csv.GetRecord <dynamic>();
uint fileDataId = uint.Parse(record.FileDataID);
uint materialResourcesId = uint.Parse(record.MaterialResourcesID);
Add(fileDataId, materialResourcesId);
if (!ReverseMap.ContainsKey(materialResourcesId))
{
ReverseMap[materialResourcesId] = new HashSet <uint>();
}
ReverseMap[materialResourcesId].Add(fileDataId);
}
}
}
public async Task Load()
{
HttpWebRequest req = WebRequest.CreateHttp(Url + Build);
using (HttpWebResponse resp = (HttpWebResponse)await req.GetResponseAsync())
using (var respStream = resp.GetResponseStream())
using (var sr = new StreamReader(respStream))
using (var csv = new CsvHelper.CsvReader(sr))
{
while (await csv.ReadAsync())
{
var record = csv.GetRecord <dynamic>() as IDictionary <string, object>;
Add(new Item()
{
ModelResourcesID = new HashSet <uint> {
uint.Parse(record["ModelResourcesID[0]"].ToString()),
uint.Parse(record["ModelResourcesID[1]"].ToString())
},
ModelMaterialResourcesID = new HashSet <uint> {
uint.Parse(record["ModelMaterialResourcesID[0]"].ToString()),
uint.Parse(record["ModelMaterialResourcesID[1]"].ToString())
}
});
}
foreach (var record in this)
{
record.ModelMaterialResourcesID.Remove(0);
record.ModelResourcesID.Remove(0);
}
}
}
/*
* function will init Scans data from the CSV filepath saved in the variable path
*/
public static void InitData(dynamic vaProxy)
{
try
{
var path = vaProxy.GetText("path");
vaProxy.WriteToLog(path, "green");
var tempList = new List <ScanEntry>();
using (var reader = new StreamReader(path))
using (var csv = new CsvHelper.CsvReader(reader))
{
while (csv.Read())
{
var body = csv.GetRecord <ScanEntry>();
body.Body = NATO.Aggregate(body.Body.Replace(body.System, ""), (current, value) => current.Replace(value.Key, value.Value));
vaProxy.WriteToLog(body.ToString());
//first system?
if (CurrSystem == "")
{
CurrSystem = body.System;
tempList.Add(body);
}
//multiple planets
else if (CurrSystem == body.System)
{
tempList.Add(body);
}
//next system
else
{
//update system
CurrSystem = body.System;
Scans.Add(tempList);
tempList = new List <ScanEntry>();
tempList.Add(body);
}
}
//add last record
if (tempList[0].System == Scans[Scans.Count - 1][0].System)
{
Scans[Scans.Count - 1].Add(tempList[0]);
}
else
{
Scans.Add(tempList);
}
CurrIndex = 0;
}
vaProxy.SetText("r2r_sayData", "Initialization complete");
vaProxy.WriteToLog(String.Format("{0} systems to scan", Scans.Count), "blue");
}
catch (Exception e)
{
vaProxy.WriteToLog("Error - init" + e.ToString(), "red");
return;
}
}
public tEntity Deserialize <tEntity>(string value) where tEntity : BaseEntity
{
using (TextReader reader = new StringReader(value))
{
using (CsvHelper.CsvReader readerCsv = new CsvHelper.CsvReader(reader))
{
return(readerCsv.GetRecord <tEntity>());
}
}
}
public async Task RunCsvHelper_Async()
{
StringReader reader = new StringReader(data);
var csvReader = new CsvHelper.CsvReader(reader, new Configuration());
List <Person> people = new List <Person>();
await csvReader.ReadAsync().ConfigureAwait(false);
csvReader.ReadHeader();
while (await csvReader.ReadAsync().ConfigureAwait(false))
{
people.Add(csvReader.GetRecord <Person>());
}
}
private static List <Course> parseRecordsForCourse(CsvHelper.CsvReader reader)
{
// make sure not at header or end of file
List <Course> courseList = new List <Course>();
while ((fileHasMoreRecords = reader.Read()) && courseHasMoreRecords(reader))
{
Course course = reader.GetRecord <Course>();
course.SetDerivedProperties();
courseList.Add(course);
}
return(courseList);
}
public static List <T> Read <T>(CsvConfiguration configuration, string filePath)
{
using (var reader = new StreamReader(filePath))
using (var csv = new CsvHelper.CsvReader(reader, configuration))
{
csv.Context.RegisterClassMap <LeaseMap>();
List <T> list = new List <T>();
while (csv.Read())
{
list.Add(csv.GetRecord <T>());
}
return(list);
}
}
public Country[] ReadFirstNCountries(int nCountries)
{
Country[] countries = new Country[nCountries];
using (StreamReader sr = new StreamReader(this._csvFilePath))
using (var csv = new CsvHelper.CsvReader(sr))
{
for (int i = 0; i < nCountries; i++)
{
var country = csv.GetRecord <Country>();
countries[i] = country;
}
}
return(countries);
}
public async Task ImportPostcodes(Stream csvStream)
{
using (var streamReader = new StreamReader(csvStream))
{
using (var csvReader = new CsvHelper.CsvReader(streamReader))
{
var client = new MongoClient("mongodb://localhost:32773");
var database = client.GetDatabase("HousePrice");
database.DropCollection("Postcodes");
var collection = database.GetCollection <PostcodeData>("Postcodes");
var batch = new List <PostcodeData>();
long lastId = 0;
while (csvReader.Read())
{
try
{
var record = csvReader.GetRecord <PostcodeData>();
record.Postcode = record.Postcode.Replace(" ", String.Empty);
lastId = record.Id;
batch.Add(record);
}
catch (TypeConverterException)
{
// invalid lat/long so ignore
}
if (batch.Count == 1000)
{
await collection.InsertManyAsync(batch);
batch.Clear();
}
}
if (batch.Any())
{
await collection.InsertManyAsync(batch);
}
Console.WriteLine("Generating index...");
var notificationLogBuilder = Builders <PostcodeData> .IndexKeys;
var indexModel = new CreateIndexModel <PostcodeData>(notificationLogBuilder.Ascending(x => x.Postcode));
await collection.Indexes.CreateOneAsync(indexModel).ConfigureAwait(false);
}
}
}
public IEnumerable <T> Load()
{
var data = new List <T>();
using (var reader = new CsvHelper.CsvReader(_reader, new CsvHelper.Configuration.CsvConfiguration {
HasHeaderRecord = _hasHeader
}))
{
while (reader.Read())
{
data.Add(reader.GetRecord <T>());
}
}
return(data);
}
public ActionResult Upload(HttpPostedFileBase file)
{
//http://joshclose.github.io/CsvHelper/
Stream stream = file.InputStream;
StreamReader streamReader = new StreamReader(stream);
TextReader textReader = (TextReader)streamReader;
var csv = new CsvHelper.CsvReader(textReader);
List<ImportDTO> listOfDTOs = new List<ImportDTO>();
while (csv.Read())
{
ImportDTO record = csv.GetRecord<ImportDTO>();
listOfDTOs.Add(record);
}
//Map DTO to models
foreach (ImportDTO item in listOfDTOs)
{
//Get countries
List<string> countries = item.Countries.Split(new char[] { '|' }).ToList();
Collection<Country> countryCollection = new Collection<Country>();
foreach (string countryName in countries)
{
countryCollection.Add(new Country { Name = countryName });
}
//Get start and end times
DateTime start = DateTime.ParseExact(item.StartDate + " " + item.StartTime, "dd/MM/yyyy HH:mm",
System.Globalization.CultureInfo.InvariantCulture);
DateTime end = DateTime.ParseExact(item.EndDate + " " + item.EndTime, "dd/MM/yyyy HH:mm",
System.Globalization.CultureInfo.InvariantCulture);
Film newFilm = new Film();
newFilm.Country = countryCollection;
TimeSlot slot = new TimeSlot();
slot.StartTime = start;
slot.EndTime = end;
}
return View("Index");
}
public Dictionary <string, Country> ReadAllCountriesDict()
{
var countries = new Dictionary <string, Country>();
using (var sr = new StreamReader(this._csvFilePath))
using (var csv = new CsvHelper.CsvReader(sr))
{
csv.Configuration.TypeConverterCache.AddConverter <int>(new CustomIntConverter());
while (csv.Read())
{
var country = csv.GetRecord <Country>();
countries.Add(country.Code, country);
}
}
return(countries);
}
public async Task Load()
{
HttpWebRequest req = WebRequest.CreateHttp(Url + Build);
using (HttpWebResponse resp = (HttpWebResponse)await req.GetResponseAsync())
using (var respStream = resp.GetResponseStream())
using (var sr = new StreamReader(respStream))
using (var csv = new CsvHelper.CsvReader(sr))
{
while (await csv.ReadAsync())
{
var record = csv.GetRecord <dynamic>();
Add(uint.Parse(record.ID), uint.Parse(record.FileDataID));
}
}
}
public static IEnumerable <CSVExerciseVM> ReadCSV(string fileName)
{
using (var fileReader = System.IO.File.OpenText(fileName))
using (var csvReader = new CsvHelper.CsvReader(fileReader))
{
csvReader.Configuration.HasHeaderRecord = true;
csvReader.Configuration.IgnoreHeaderWhiteSpace = false;
csvReader.Configuration.IsHeaderCaseSensitive = true;
csvReader.Configuration.RegisterClassMap <CSVExerciseMap>();
while (csvReader.Read())
{
var record = csvReader.GetRecord <CSVExerciseVM>();
yield return(record);
}
}
}
public IList <CityEvent> ReadAll(StreamReader stream)
{
var list = new List <CityEvent>();
var rr = new CsvHelper.CsvReader(stream, _conf);
if (!rr.ReadHeader())
{
throw new InvalidOperationException();
}
while (rr.Read())
{
list.Add(rr.GetRecord <CityEvent>());
}
return(list);
}
public ExtractPaymentCsv(HttpPostedFile file, int accountId)
{
using (StreamReader reader = new StreamReader(file.InputStream))
{
string content = reader.ReadToEnd();
using (TextReader textReader = reader)
{
CsvHelper.CsvReader csvReader = new CsvHelper.CsvReader(textReader);
while (csvReader.Read())
{
PaymentDto payment = csvReader.GetRecord <PaymentDto>();
payment.AccountId = accountId;
Payments.Add(payment);
}
}
};
}
public async Task LoadFile <T>(string filename) where T : CsvBaseObject
{
LastEntity = typeof(T).Name; // kludge
DateTime now = DateTime.UtcNow;
string filePath = Path.Combine(BasePath, filename);
string table = typeof(T).Name;
using (var file = File.OpenText(filePath))
{
using (var csv = new CsvHelper.CsvReader(file))
{
csv.Configuration.MissingFieldFound = null;
csv.Configuration.HasHeaderRecord = true;
csv.Read();
csv.ReadHeader();
for (int i = 0; await csv.ReadAsync(); i++)
{
T record = null;
try
{
record = csv.GetRecord <T>();
await ProcessRecord(record, table, now);
}
catch (Exception ex)
{
if (ex is ProcessingException)
{
throw;
}
string o = record == null ? "(null)" : JsonConvert.SerializeObject(record);
throw new ProcessingException(Logger.Here(), $"Unhandled error processing {typeof(T).Name}: {o}", ex);
}
await Repo.Committer.InvokeIfChunk();
}
// commit any last changes
await Repo.Committer.InvokeIfAny();
}
Logger.Here().LogInformation($"Processed Csv file {filePath}");
}
}
/// <summary>
/// Used to populate test data: https://www.mockaroo.com/
/// </summary>
/// <param name="csvFile"></param>
/// <param name="tableName"></param>
protected void ImportTableFromCSV(string csvFile, string tableName, int maximum = 1000)
{
csvFile = ResolvePath(csvFile);
if (!File.Exists(csvFile))
{
logger.Warn("CSV file '{0}' does not exist", csvFile);
return;
}
logger.Info("Importing data from {0} into database table {1}", csvFile, tableName);
using (var reader = new StreamReader(csvFile))
using (var csv = new CsvHelper.CsvReader(reader))
using (var conn = SimpleDbConnection())
{
var records = new List <dynamic>();
// Assume the CSV header matches the database fields
csv.Read();
csv.ReadHeader();
for (int i = 0; i < maximum && csv.Read(); i++)
{
records.Add(csv.GetRecord <dynamic>());
}
// Extract the header names
var tableFields = new List <string>();
var objectFields = new List <string>();
foreach (var header in csv.Context.HeaderRecord)
{
tableFields.Add(header);
objectFields.Add("@" + header);
}
// Insert the records
var sql = String.Format(@"
INSERT INTO {0}({1})
VALUES ({2})
", tableName, String.Join(", ", tableFields), String.Join(", ", objectFields));
conn.Execute(sql, records);
}
}
public async Task Load()
{
HttpWebRequest req = WebRequest.CreateHttp(Url + Build);
using (HttpWebResponse resp = (HttpWebResponse)await req.GetResponseAsync())
using (var respStream = resp.GetResponseStream())
using (var sr = new StreamReader(respStream))
using (var csv = new CsvHelper.CsvReader(sr))
{
while (await csv.ReadAsync())
{
var record = csv.GetRecord <dynamic>();
uint id = uint.Parse(record.ItemDisplayInfoID);
uint icon = uint.Parse(record.DefaultIconFileDataID);
this[id] = icon;
}
}
}
public IList <Country> ReadAllCountries()
{
IList <Country> countries = new List <Country>();
using (StreamReader sr = new StreamReader(this._csvFilePath))
using (var csv = new CsvHelper.CsvReader(sr))
{
csv.Configuration.IgnoreBlankLines = false;
csv.Configuration.MissingFieldFound = null;
csv.Configuration.TypeConverterCache.AddConverter <int>(new CustomIntConverter());
// var records = csv.GetRecords<Country>();
while (csv.Read())
{
Country country = csv.GetRecord <Country>();
countries.Add(country);
}
}
return(countries);
}
public void TestMethod1()
{
var pathToMyDocs = Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments);
var pathToDataFile = Path.Combine(pathToMyDocs, "data1.csv");
using (var textReader = File.OpenText(pathToDataFile))
{
var csv = new CsvHelper.CsvReader(textReader);
csv.Read(); // first row
csv.Read(); // second row
csv.Read(); // third rows
csv.Read(); // fouth row: course title
csv.Read(); // fifth row: actual record
csv.Configuration.HasHeaderRecord = false;
csv.Configuration.RegisterClassMap(new CourseClassMap());
Course c = csv.GetRecord <Course>();
}
}
public async Task ConvertCsv()
{
var csvPath = @"C:\Users\sweet\Downloads\Beat Buddy Midi Drum Map - Map (1).csv";
using (var reader = new StreamReader(csvPath))
using (var writer = new StreamWriter("map.csv"))
using (var csvReader = new CsvHelper.CsvReader(reader, CultureInfo.InvariantCulture))
using (var csvWriter = new CsvHelper.CsvWriter(writer, new CsvConfiguration(CultureInfo.InvariantCulture)))
{
await csvReader.ReadAsync();
csvReader.ReadHeader();
csvWriter.WriteHeader <Record>();
await csvWriter.NextRecordAsync();
while (await csvReader.ReadAsync())
{
var inRecord = csvReader.GetRecord <dynamic>() as IDictionary <string, object>;
var keys = inRecord.Keys;
var outRecord = new Record()
{
Number = Convert.ToInt32(inRecord["Number"]),
MidiNote = Convert.ToString(inRecord["Note"]),
OctaveMiddleC3 = Convert.ToInt32(inRecord["Octave (Middle C3)"]),
OctaveMiddleC4 = Convert.ToInt32(inRecord["Octave (Middle C4)"])
};
for (var k = 4; k < keys.Count; k++)
{
var key = keys.ElementAt(k);
var value = Convert.ToString(inRecord[key]);
// if (string.IsNullOrWhiteSpace(value))
// continue;
var instrimentNote = GetInstrumentNote(value);
outRecord.Kit = key;
outRecord.Instrument = instrimentNote.Instrument;
outRecord.KitNote = instrimentNote.KitNote;
csvWriter.WriteRecord(outRecord);
await csvWriter.NextRecordAsync();
}
}
}
}
public async Task Load()
{
HttpWebRequest req = WebRequest.CreateHttp(Url + Build);
using (HttpWebResponse resp = (HttpWebResponse)await req.GetResponseAsync())
using (var respStream = resp.GetResponseStream())
using (var sr = new StreamReader(respStream))
using (var csv = new CsvHelper.CsvReader(sr))
{
while (await csv.ReadAsync())
{
var record = csv.GetRecord <dynamic>();
this[uint.Parse(record.MaterialResourcesID)] = new MaterialRes()
{
ItemDisplayInfoID = uint.Parse(record.ItemDisplayInfoID),
Section = (ComponentSection)byte.Parse(record.ComponentSection)
};
}
}
}
public async Task <List <PersonImport> > ReadAllAsync()
{
var people = new List <PersonImport>();
if (string.IsNullOrWhiteSpace(pathToCsvFile) || !File.Exists(pathToCsvFile))
{
return(people);
}
using (var sr = new StreamReader(pathToCsvFile))
{
using (var csv = new CsvHelper.CsvReader(sr, System.Globalization.CultureInfo.InvariantCulture))
{
csv.Configuration.RegisterClassMap <CsvTreeReaderMap>();
csv.Configuration.BadDataFound = context =>
{
logger?.LogError((int)LogEventIds.BadDataInReaderImport, $"Bad data found on row '{context.RawRow}'.");
};
while (await csv.ReadAsync())
{
try
{
var person = csv.GetRecord <PersonImport>();
people.Add(person);
}
catch (Exception ex)
{
logger?.LogError((int)LogEventIds.ErrorReadingCsvInReader, ex, $"Error occurred reading CSV.");
}
}
}
}
return(people);
}
public async Task Import(string name, Stream csvStream)
{
await _mongoContext.ExecuteActionAsync <HousePrice>("Transactions", async (collection) =>
{
using (var streamReader = new StreamReader(csvStream))
{
using (var csvReader = new CsvHelper.CsvReader(streamReader))
{
csvReader.Configuration.HasHeaderRecord = false;
csvReader.Configuration.RegisterClassMap <HousePriceMap>();
var batch = new List <HousePrice>();
while (csvReader.Read())
{
try
{
var record = csvReader.GetRecord <HousePrice>();
record.Postcode = record.Postcode.Replace(" ", String.Empty);
}
catch (Exception e)
{
Log.Error(e.Message);
throw;
}
}
if (batch.Count % 1000 == 0)
{
Console.WriteLine($"{name}: {batch.Count}");
}
await collection.InsertManyAsync(batch);
}
}
});
}