public async Task<IEnumerable<ChimetDataRecord>> DownloadData(DateTime day) { WebRequest dateRequest = HttpWebRequest.Create(string.Format("http://{1}/archive/{0:yyyy}/{0:MMMM}/CSV/{2}{0:ddMMMyyyy}.csv", day, this.Station.Address, this.Station.ShortName)); WebResponse data = await dateRequest.GetResponseAsync().ConfigureAwait(false); CsvHelper.Configuration.CsvConfiguration config = new CsvHelper.Configuration.CsvConfiguration(); config.RegisterClassMap<ChimetDataRecordMap>(); using (var reader = new CsvHelper.CsvReader(new System.IO.StreamReader(data.GetResponseStream()), config)) { return reader.GetRecords<ChimetDataRecord>().ToList().AsReadOnly(); } }
public CsvReader(string path, CsvConfiguration config, Domain domain, bool isTestReader) { Path = path; _reader = new CsvHelper.CsvReader(File.OpenText(this.Path), config); Domain = domain; IsTestReader = isTestReader; }
/// <summary> /// CSVを読み込み、薬品情報のリストを返す /// </summary> /// <param name="path">CSVファイルの絶対パス</param> /// <returns>薬品情報のリスト</returns> static List<Yakuhin> ReadCsv(string path) { List<Yakuhin> list = new List<Yakuhin>(); var enc = new System.Text.UTF8Encoding(false); using (var reader = new System.IO.StreamReader(path, enc)) { var csv = new CsvHelper.CsvReader(reader); while (csv.Read()) { string drugCode = csv.GetField<string>(0); string clsCode = csv.GetField<string>(1); string clsName = csv.GetField<string>(2); string drugName = csv.GetField<string>(3); string company = csv.GetField<string>(4); var yakuhin = new Yakuhin { DrugCode = drugCode, ClassificationCode = clsCode, ClassificationName = clsName, DrugName = drugName, Company = company }; list.Add(yakuhin); } } return list; }
public IEnumerable<UnsettledBetItem> GetUnsettledBets() { using (var fileReader = File.OpenText(GetFilePath(UnsettledBetsFileName))) using (var csvReader = new CsvHelper.CsvReader(fileReader)) { return csvReader.GetRecords<UnsettledBetItem>().ToList(); } }
public void ReadGraph(TextReader reader) { var g = new BidirectionalGraph<MyType, IEdge<MyType>>(); try { var csv = new CsvHelper.CsvReader(reader); while (csv.Read()) { if (!csv.CurrentRecord.Any()) { continue; } var node = csv.CurrentRecord.First(); var edges = csv.CurrentRecord.Skip(1) .Where(x => !string.IsNullOrEmpty(x)) .Select(x => x.Trim()) .Where(x => !string.IsNullOrEmpty(x)); foreach (var edge in edges) { g.AddVerticesAndEdge(new Edge<MyType>(node, edge)); } } var dict = new Dictionary<int, HashSet<MyType>>(); HashSet<MyType> set; foreach (var v in g.Vertices) { var edgeCount = g.InEdges(v).Count(); if (!dict.TryGetValue(edgeCount, out set)) { set = new HashSet<MyType>(); dict.Add(edgeCount, set); } set.Add(v); } Graph = g; Summary = string.Join(Environment.NewLine, dict .OrderBy(kvp => kvp.Key) .Select(kvp => kvp.Key + ": " + string.Join(", ", kvp.Value))) + Environment.NewLine + Environment.NewLine + string.Join(Environment.NewLine, g.Vertices .Where(v => g.OutEdges(v).Count() != 3) .Select(v => v + ": " + g.OutEdges(v).Count())); //Summary = string.Join( // Environment.NewLine, // graph.Vertices // .OrderBy(x => graph.InEdges(x).Count()) // .Select(v => v + ": " + graph.InEdges(v).Count())); } catch (Exception e) { Summary = "Failed to read:" + Environment.NewLine + e; } }
public override dynamic Load(Slice slice) { var ret = new List<dynamic>(); if (!Blob.Exists()) return ret; using (var reader = new CsvHelper.CsvReader(new StreamReader(Blob.OpenRead(), Configuration.Encoding))) { var records = reader.GetRecords<dynamic>(); foreach (IDictionary<string, object> record in records) ret.Add(Helpers.DictionaryToObject(record, Structure)); } return ret; }
private IEnumerable<Employee> GetEmployeeAdjustments(Stream content) { using (var reader = new CsvHelper.CsvReader(new StreamReader(content))) { return reader.GetRecords<CsvAdjustment>().Where(x => x.Year > 0) .Select(x => new Employee { Id = 0, Ssn = x.Ssn, DeductionSchema = new DeductionSchema { Adjustment = x.Amount, Year = x.Year } }) .ToList(); } }
public ActionResult Upload(HttpPostedFileBase file) { //http://joshclose.github.io/CsvHelper/ Stream stream = file.InputStream; StreamReader streamReader = new StreamReader(stream); TextReader textReader = (TextReader)streamReader; var csv = new CsvHelper.CsvReader(textReader); List<ImportDTO> listOfDTOs = new List<ImportDTO>(); while (csv.Read()) { ImportDTO record = csv.GetRecord<ImportDTO>(); listOfDTOs.Add(record); } //Map DTO to models foreach (ImportDTO item in listOfDTOs) { //Get countries List<string> countries = item.Countries.Split(new char[] { '|' }).ToList(); Collection<Country> countryCollection = new Collection<Country>(); foreach (string countryName in countries) { countryCollection.Add(new Country { Name = countryName }); } //Get start and end times DateTime start = DateTime.ParseExact(item.StartDate + " " + item.StartTime, "dd/MM/yyyy HH:mm", System.Globalization.CultureInfo.InvariantCulture); DateTime end = DateTime.ParseExact(item.EndDate + " " + item.EndTime, "dd/MM/yyyy HH:mm", System.Globalization.CultureInfo.InvariantCulture); Film newFilm = new Film(); newFilm.Country = countryCollection; TimeSlot slot = new TimeSlot(); slot.StartTime = start; slot.EndTime = end; } return View("Index"); }
public List<PublicationTarget> Read() { using (var tr = new StreamReader(_filename)) { var csv = new CsvHelper.CsvReader(tr); csv.Configuration.Delimiter = ","; csv.Configuration.IsHeaderCaseSensitive = false; csv.Configuration.SkipEmptyRecords = true; csv.Configuration.IgnoreBlankLines = true; csv.Configuration.RegisterClassMap<PublicationTargetCsvMapping>(); //csv.Configuration.TrimHeaders = true; var records = csv.GetRecords<PublicationTarget>().ToList(); return records; } }
public Fact[] Read(string message) { var csvConfig = new CsvConfiguration { Delimiter = "|", Encoding = Encoding.UTF8, HasHeaderRecord = true, }; csvConfig.RegisterClassMap<FactMap>(); csvConfig.CultureInfo = CultureInfo.InvariantCulture; using (var reader = new StringReader(message)) using (var parser = new CsvHelper.CsvReader(reader, csvConfig)) { return parser.GetRecords<Fact>().ToArray(); } }
public static void Train() { using (System.IO.TextReader fs = System.IO.File.OpenText("dfsafias")) { using (CsvHelper.CsvReader reader = new CsvHelper.CsvReader(fs)) { //reader.GetField } } OpenCvSharp.ML.RTrees forest = OpenCvSharp.ML.RTrees.Create(); // forest.Predict(); // forest. }
private bool LoadData() { var listLog = new List<LogEntry>(); try { if (File.Exists(Debug.ErrFile[CurrentView])) { using (var csv = new CsvHelper.CsvReader(new StreamReader(Debug.ErrFile[CurrentView], Encoding.UTF8, true))) { csv.Configuration.Delimiter = Debug.CharSeparate; csv.Configuration.HasHeaderRecord = false; while (csv.Read()) { string[] r = csv.CurrentRecord; if (r.Length >= 4) { var item = new LogEntry { TimeStamp = r[0], Level = r[1], Message = r[2], ErrorMessage = r[3] }; listLog.Add(item); } else { lblErr.Text = @"Log file wrong format"; return false; } } } listLog.Reverse(); grid.DataSource = listLog; grid.DataBind(); return true; } return false; } catch (Exception ex) { lblErr.Text = ex.Message; return false; } }
public static List<Users.User> getStudents() { CsvHelper.CsvReader reader; List<Users.User> userList = new List<Users.User>(); reader = new CsvHelper.CsvReader(File.OpenText("student_accounts.csv")); while (reader.Read()) { string TNumber = reader.GetField<string>("T_NUMBER"); string firstName = reader.GetField<string>("FIRSTNAME"); string lastName = reader.GetField<string>("LAST_NAME"); string userName = reader.GetField<string>("USERNAME"); string Email = reader.GetField<string>("EMAILADDRESS"); long secretKey = reader.GetField<long>("CHECK-N ID"); userList.Add(new Users.StudentWorker(firstName, lastName, TNumber, secretKey, userName, Email, 110)); } return userList; }
public static IFileReader CreateFromFileName(FileInfoBase fileInfoBase) { if (IsCsvFile(fileInfoBase.FullName)) { var reader = new CsvHelper.CsvReader(new StreamReader(fileInfoBase.OpenRead()), new CsvConfiguration { HasHeaderRecord = true, Delimiter = ";" }); return new CsvReader(reader); } if (IsExcelFile(fileInfoBase.FullName)) { var excelReader = ExcelReaderFactory.CreateOpenXmlReader(fileInfoBase.Open(FileMode.Open, FileAccess.Read)); excelReader.IsFirstRowAsColumnNames = true; return new ExcelReader(excelReader); } throw new InvalidOperationException(); }
private void Load <T>(IDbConnection conn, IDbTransaction trans) { var dataFileName = Path.ChangeExtension(typeof(T).Name, ".tsv"); var tr = File.OpenText(Path.Combine(_dataDirectory, dataFileName)); var config = new Configuration { HasHeaderRecord = true, Delimiter = "\t" }; foreach (var classMap in GetClassMapTypes()) { config.RegisterClassMap(classMap); } var csv = new CsvHelper.CsvReader(tr, config); var records = csv.GetRecords <T>().ToList(); Console.WriteLine($" {dataFileName} ..."); conn.Insert(records, trans); }
public async Task <List <VaccineHospital> > GetVaccineHospitalsAsync() { var url = @"http://www.cdc.gov.tw/downloadfile.aspx?fid=2AE494CD4C1EAFBD"; var str = await _netService.GetStringAsync(new Uri(url)); if (string.IsNullOrWhiteSpace(str)) { return(new List <VaccineHospital>()); } List <VaccineHospital> items; using (var reader = new StringReader(str)) using (var csv = new CsvHelper.CsvReader(reader)) { csv.Configuration.HasHeaderRecord = true; csv.Configuration.RegisterClassMap <VaccineHospitalMap>(); items = csv.GetRecords <VaccineHospital>() .OrderBy(i => i.縣市) .ThenBy(i => i.鎮市區) .ThenBy(i => i.機構類別) .ThenBy(i => i.合約醫療院所名稱) .ToList(); } foreach (var item in items) { item.Id = item.GetHashCode().ToString(); item.縣市 = item.縣市.Replace(" ", ""); item.鎮市區 = item.鎮市區.Replace(" ", ""); item.合約醫療院所名稱 = item.合約醫療院所名稱.Replace("\n", ""); item.公費疫苗 = item.公費疫苗.Replace("\n", "").Replace("、", Environment.NewLine); item.自費疫苗 = item.自費疫苗.Replace("\n", "").Replace("、", Environment.NewLine); } return(items); }
/// <summary> /// Delete customer from file. File must contain the CustomerCid from CSP /// </summary> /// <param name="filename">Path of csv file contain customers to delete</param> /// <param name="defaultDomain">default domain of the reseller</param> /// <param name="appId">appid that is registered for this application in Azure Active Directory (AAD)</param> /// <param name="key">Key for this application in Azure Active Directory</param> /// <param name="resellerMicrosoftId">Microsoft Id of the reseller</param> internal static void DeleteCustomers(string filename, string defaultDomain, string appId, string key, string resellerMicrosoftId) { GetTokens(defaultDomain, appId, key, resellerMicrosoftId); using (var reader = new CsvHelper.CsvReader(new StreamReader(filename))) { while (reader.Read()) { string displayName = reader.GetField <string>("displayName"); string cspCustomerCid = reader.GetField <string>("cspCustomerCid"); if (cspCustomerCid.Trim().Length == 0) { Console.WriteLine("Unable to delete {0} because CustomerCid is missing.", displayName); } else { Console.WriteLine("Deleting {0}", displayName); Customer.DeleteCustomer(resellerCid, cspCustomerCid, saAuthorizationToken.AccessToken); } } } }
void Form1_DragDrop(object sender, DragEventArgs e) { listBox1.ValueMember = ""; string[] files = (string[])e.Data.GetData(DataFormats.FileDrop); if (files.Length > 1 || Path.GetExtension(files[0]).ToLower() != ".csv") { MessageBox.Show("Drag 1 file, needs to be csv"); } else { StreamReader sr = new StreamReader(files[0], Encoding.Default); CsvHelper.CsvReader rdr = new CsvHelper.CsvReader(sr); rdr.Configuration.RegisterClassMap <MyClassMap>(); rdr.Configuration.Delimiter = ";"; var test = rdr.GetRecords <Student>().ToList(); foreach (var item in test) { listBox1.Items.Add(item); } } }
public void ImportCharityData() { var dbConnectionFactory = appHost.Container.Resolve<IDbConnectionFactory>(); using (var db = dbConnectionFactory.OpenDbConnection()) { db.DropAndCreateTable<Charity>(); } using (var db = dbConnectionFactory.OpenDbConnection()) { string currentPath = AppDomain.CurrentDomain.BaseDirectory; var filestream = File.OpenRead(Path.Combine(currentPath, ConfigurationManager.AppSettings["CharityPath"])); TextReader textreader = new StreamReader(filestream); var reader = new CsvHelper.CsvReader(textreader); var lol = reader.GetRecords<dynamic>(); var charityJson = lol.ToJson(); var data = JsonSerializer.DeserializeFromString<List<Charity>>(charityJson); db.InsertAll(data); } }
public override void ImportDate() { var csv = new CsvHelper.CsvReader(System.IO.File.OpenText(DownloadFullPath)); while (csv.Read()) { string CsvStockNum = ""; try { CsvStockNum = csv.GetField <string>(0); CsvStockNum = CsvStockNum.Replace(" ", ""); if (CsvStockNum.Length != 4) { continue; } Int32.Parse(CsvStockNum); } catch { continue; } double CsvPrice = 0; try { CsvPrice = csv.GetField <double>(10); } catch { } StockPricesList.Add(new StockPriceDataModel { StockNum = CsvStockNum, StockPrice = Math.Round(CsvPrice, 2) }); } UploadData(); UpdateLocalDB(); }
public void HandleBackedOutWork(Backout backout) { var file = Path.Combine(CSVBasePath, "bookings.csv"); File.WriteAllText(Path.Combine(CSVBasePath, "latest-backout-id"), backout.BackoutId.ToString()); var orders = new List <UnscheduledCsvRow>(); if (File.Exists(file)) { using (var f = File.OpenRead(file)) using (var csv = new CsvHelper.CsvReader(new StreamReader(f), CultureInfo.InvariantCulture)) { orders.AddRange(csv.GetRecords <UnscheduledCsvRow>()); } } foreach (var p in backout.Parts) { orders.Add(new UnscheduledCsvRow() { Id = "Reschedule-" + p.Part + "-" + DateTime.UtcNow.ToString("yyy-MM-ddTHH-mm-ssZ"), DueDate = DateTime.Today, Priority = 100, Part = p.Part, Quantity = p.Quantity }); } using (var s = new StreamWriter(file)) using (var csv = new CsvHelper.CsvWriter(s, CultureInfo.InvariantCulture)) { csv.Configuration.TypeConverterOptionsCache.GetOptions <DateTime>().Formats = new string[] { "yyyy-MM-dd" }; csv.WriteRecords(orders); } }
static void Main(string[] args) { string dir = Path.Combine(Directory.GetParent(typeof(Program).Assembly.Location).ToString(), "ProductUploadTemplate.csv"); FileStream fileStream = new FileStream(dir, FileMode.Open); MemoryStream ms = new MemoryStream(); Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); Encoding encoding = CodePagesEncodingProvider.Instance.GetEncoding("big5"); fileStream.CopyTo(ms); string content = encoding.GetString(ms.ToArray()); StringReader reader = new StringReader(content); CsvHelper.CsvReader csv = new CsvHelper.CsvReader(reader, CultureInfo.InvariantCulture); csv.Read(); csv.ReadHeader(); while (csv.Read()) { string value = csv.GetField <string>("DESCRIPTION"); Console.WriteLine(value); } Console.ReadKey(); }
private void AssertCsvSerialization <T>(T value, string expected, ITypeConverter converter) { string actual; using (var writer = new StringWriter()) { var serializer = new CsvHelper.CsvWriter(writer, CultureInfo.InvariantCulture); serializer.Context.TypeConverterCache.AddConverter <T>(converter); serializer.WriteField(value); serializer.Flush(); actual = writer.ToString(); Assert.Equal(expected, actual); } var reader = new CsvHelper.CsvReader(new StringReader(actual), CultureInfo.InvariantCulture); reader.Context.TypeConverterCache.AddConverter <T>(converter); reader.Read(); var deserializedValue = reader.GetField <T>(0); Assert.Equal(value, deserializedValue); }
public static Dictionary <Guid, User> GetUsers(string path) { Dictionary <Guid, User> users = new Dictionary <Guid, User>(); using (var reader = File.OpenText(path + "users.csv")) { using (var csv = new CsvHelper.CsvReader(reader)) { while (csv.Read()) { User usr = new User() { Id = csv.GetField <Guid>(0), Name = csv.GetField(1), Email = csv.GetField(2) }; users.Add(usr.Id, usr); } } } return(users); }
/// <summary> /// parse the file, and get information /// <remark>skip the headers</remark> /// </summary> /// <param name="file"></param> /// <returns>courseForFile</returns> static List <Course> parseFile(string file) { var coursesForFile = new List <Course>(); using (StreamReader fileStream = File.OpenText(file)) { // Use CsvHelper library to read the file. var csvReader = new CsvHelper.CsvReader(fileStream); // configure csv reader csvReader.Configuration.HasHeaderRecord = false; csvReader.Configuration.RegisterClassMap <CourseClassMap>(); skipHeaders(csvReader); csvReader.Read(); // read first header and skip it. while (fileHasMoreRecords) { coursesForFile.AddRange(parseRecordsForCourse(csvReader)); } } return(coursesForFile); }
public void ReadAllHistoricDates() { try { using (var reader = new StreamReader(HistoricalFileName)) using (var csv = new CsvHelper.CsvReader(reader, CultureInfo.InvariantCulture)) { var records = new List <CurencyDataViewModel>(); csv.Read(); csv.ReadHeader(); List <string> allDates = new List <string>(); while (csv.Read()) { var dateAsString = csv.GetField(0); allDates.Add(dateAsString); } AllHistoricDates = allDates; } } catch (Exception ex) { AllHistoricDates = null; } }
private void btnAddPath_Click(object sender, EventArgs e) { StreamReader sr; CsvHelper.CsvReader csvread; sr = new StreamReader(txtFilePath.Text); csvread = new CsvHelper.CsvReader(sr); KlijentIzCsv kicsv = new KlijentIzCsv(); csvread.Configuration.HasHeaderRecord = false; int i = 1; while (csvread.Read()) { if (i == 1) { kicsv.klijent = csvread[4].Trim(); kicsv.nazivMonitoringa = csvread[5].Trim(); } Console.WriteLine(csvread[0]); if (csvread[0].Trim() != "") { kicsv.nicsvList.Add(new NadgledanaIzCsv(csvread[0].Trim(), csvread[1].Trim(), csvread[2].Trim(), csvread[3].Trim())); } i++; } DBCommOsnovnePostavke.connection = new SqlConnection(DBBlokadeData.connectionString); DBCommOsnovnePostavke.connection.Open(); DBCommOsnovnePostavke.addKlijentIzCsv(kicsv); DBCommOsnovnePostavke.connection.Close(); txtMaticniBroj.Clear(); txtNaziv.Clear(); txtFilePath.Clear(); ucitajKlijente(1); }
public static Dictionary <Guid, Product> GetProducts(string path) { Dictionary <Guid, Product> products = new Dictionary <Guid, Product>(); using (var reader = File.OpenText(path + "products.csv")) { using (var csv = new CsvHelper.CsvReader(reader)) { while (csv.Read()) { Product p = new Product() { Id = csv.GetField <Guid>(0), Name = csv.GetField(1), Description = csv.GetField(2), ImageUrl = csv.GetField(3), Brand = csv.GetField <Guid>(4), Category = csv.GetField(5), Created = DateTime.Now }; Stock s = new Stock() { Quantity = csv.GetField <int>(6), Cost = csv.GetField <double>(7), Price = csv.GetField <double>(8) }; p.Stock = s; products.Add(p.Id, p); } } } return(products); }
static List <Address> ReadCSV(string path) { List <Address> list = new List <Address>(); var encode = new System.Text.UTF8Encoding(false); using (var reader = new System.IO.StreamReader(path, encode)) { var csv = new CsvHelper.CsvReader(reader); bool isSkip = true; while (csv.Read()) { if (isSkip) { isSkip = false; continue; } //氏名,氏名(カタカナ),電話番号,メールアドレス,郵便番号,住所,,,, Address address = new Address() { Name = csv.GetField <string>(0), Kana = csv.GetField <string>(1), Telephone = csv.GetField <string>(2), Mail = csv.GetField <string>(3), ZipCode = csv.GetField <string>(4).Replace("-", ""), Prefecture = csv.GetField <string>(5), StreetAddress = $"{csv.GetField<string>(6)}{csv.GetField<string>(7)}{csv.GetField<string>(8)}{csv.GetField<string>(9)}" }; list.Add(address); } } return(list); }
static void Main(string[] args) { var dir = new DirectoryInfo(Directory.GetCurrentDirectory()); var file = new FileInfo($"{dir.FullName}\\color-keywords.csv"); using var fs = new StreamReader(file.FullName); var colors = new CsvHelper .CsvReader(fs, CultureInfo.InvariantCulture) .GetRecords <CsvColor>() .Select(c => new ColorData(c)) .OrderBy(c => c.Color.GetHue()) .ThenBy(c => c.Color.GetSaturation()) .ThenBy(c => c.Color.GetBrightness()) .ToList(); var doc = new StringBuilder(); doc.AppendLine("<style>"); doc.AppendLine("table img {"); doc.AppendLine("\tbox-shadow: 1px 1px 8px rgba(0,0,0,.67);"); doc.AppendLine("\tborder: 1px solid rgba(0,0,0,.67);"); doc.AppendLine("}"); doc.AppendLine("</style>"); doc.AppendLine(); doc.AppendLine("Keyword | RGB Hex Value | Color"); doc.AppendLine("--------|---------------|------"); foreach (var color in colors) { doc.AppendLine($"{color.Name} | {color.Hex} | {color.Preview}"); } using var sw = new StreamWriter($"{dir.FullName}\\color-keywords.md"); sw.Write(doc.ToString()); }
public static Dictionary <Guid, Brand> GetBrands(string path) { Dictionary <Guid, Brand> brands = new Dictionary <Guid, Brand>(); using (var reader = File.OpenText(path + "brands.csv")) { using (var csv = new CsvHelper.CsvReader(reader)) { while (csv.Read()) { Brand b = new Brand() { Id = csv.GetField <Guid>(0), Name = csv.GetField(1), Description = csv.GetField(2) }; brands.Add(b.Id, b); } } } return(brands); }
static void Main(string[] args) { CultureInfo.DefaultThreadCurrentCulture = new CultureInfo("en-US"); CultureInfo.DefaultThreadCurrentUICulture = new CultureInfo("en-US"); List <Wine> wines; using (var stream = new StreamReader(".\\winequality-red.csv")) using (var helper = new CsvHelper.CsvReader(stream)) { helper.Configuration.RegisterClassMap <WineMap>(); wines = helper.GetRecords <Wine>().ToList(); } var testingSet = wines.Take(400).ToList(); var trainingSet = wines.Skip(400).ToList(); double[] result = LinearRegression(testingSet, trainingSet); decimal good = result.Zip(testingSet, (x, y) => Math.Abs(x - y.Quality) <= 0.2).Where(x => x).Count(); decimal all = 400; Debug.WriteLine($"Correctness: {good / all}"); ScatterplotBox.Show("Expected results", result.Select(x => (double)x).ToArray(), testingSet.Select(x => (double)x.Quality).ToArray()).Hold(); }
public async Task <List <PersonImport> > ReadAllAsync() { var people = new List <PersonImport>(); if (string.IsNullOrWhiteSpace(pathToCsvFile) || !File.Exists(pathToCsvFile)) { return(people); } using (var sr = new StreamReader(pathToCsvFile)) { using (var csv = new CsvHelper.CsvReader(sr, System.Globalization.CultureInfo.InvariantCulture)) { csv.Configuration.RegisterClassMap <CsvTreeReaderMap>(); csv.Configuration.BadDataFound = context => { logger?.LogError((int)LogEventIds.BadDataInReaderImport, $"Bad data found on row '{context.RawRow}'."); }; while (await csv.ReadAsync()) { try { var person = csv.GetRecord <PersonImport>(); people.Add(person); } catch (Exception ex) { logger?.LogError((int)LogEventIds.ErrorReadingCsvInReader, ex, $"Error occurred reading CSV."); } } } } return(people); }
static ParseRAKBankStatementResult ParseRAKBankStatement(string strFilePath) { ParseRAKBankStatementResult result = new ParseRAKBankStatementResult(); var strFileContent = System.IO.File.ReadAllText(strFilePath); var strFileContentLines = strFileContent.Split(new char[] { '\n' }); result.strAccountName = strFileContentLines[3].Split(new char[] { ',' })[2]; result.strAccountNumber = strFileContentLines[4].Split(new char[] { '.' })[1].Split(new char[] { ']' })[0]; result.strBranchName = strFileContentLines[6].Split(new char[] { '[' })[1].Split(new char[] { ']' })[0]; result.d8From = Convert.ToDateTime(DateTime.ParseExact(strFileContentLines[7].Split(new char[] { ',' })[2], "dd/MM/yyyy", System.Globalization.CultureInfo.InvariantCulture)); result.d8To = Convert.ToDateTime(DateTime.ParseExact(strFileContentLines[8].Split(new char[] { ',' })[2], "dd/MM/yyyy", System.Globalization.CultureInfo.InvariantCulture)); string strCSVContent = string.Empty; for (int i = 10; i < strFileContentLines.Length - 2; i++) { if (i == 10) { var Headers = strFileContentLines[i].Split(new char[] { ',' }); strCSVContent += $"EmptyHeader1,{Headers[1]},EmptyHeader2,{Headers[3]},{Headers[4]},{Headers[5]},{Headers[6]},{Headers[7]}{Environment.NewLine}"; } else { strCSVContent += strFileContentLines[i] + Environment.NewLine; } } var textReader = new System.IO.StringReader(strCSVContent); var csvr = new CsvHelper.CsvReader(textReader); csvr.Configuration.PrepareHeaderForMatch = (string header, int index) => header?.Trim(); csvr.Configuration.PrepareHeaderForMatch = (string header, int index) => header.Replace(" ", string.Empty); result.CSVRecords = csvr.GetRecords <RAKBankCSVRecord>(); return(result); }
//[Benchmark] public List <HalfNumbers> CsvHelperReadAll() { var reader = new CsvHelper.CsvReader(new StringReader(Data), new Configuration() { HeaderValidated = null }); while (reader.Read()) { var record = new HalfNumbers(); record.A = reader.GetField <string>(0); record.C = reader.GetField <string>(2); record.E = reader.GetField <string>(4); record.G = reader.GetField <string>(6); record.I = reader.GetField <string>(8); record.B = reader.GetField <int>(1); record.D = reader.GetField <int>(3); record.F = reader.GetField <int>(5); record.H = reader.GetField <int>(7); record.J = reader.GetField <int>(9); } return(reader.GetRecords <HalfNumbers>().ToList()); }
public async Task Import(string name, Stream csvStream) { await _mongoContext.ExecuteActionAsync <HousePrice>("Transactions", async (collection) => { using (var streamReader = new StreamReader(csvStream)) { using (var csvReader = new CsvHelper.CsvReader(streamReader)) { csvReader.Configuration.HasHeaderRecord = false; csvReader.Configuration.RegisterClassMap <HousePriceMap>(); var batch = new List <HousePrice>(); while (csvReader.Read()) { try { var record = csvReader.GetRecord <HousePrice>(); record.Postcode = record.Postcode.Replace(" ", String.Empty); } catch (Exception e) { Log.Error(e.Message); throw; } } if (batch.Count % 1000 == 0) { Console.WriteLine($"{name}: {batch.Count}"); } await collection.InsertManyAsync(batch); } } }); }
private Task <IEnumerable <CaseStatus> > ReadInternal(string filePath) { var result = new List <CaseStatus>(); CultureInfo provider = CultureInfo.InvariantCulture; using (var reader = new StreamReader(filePath)) using (var csv = new CsvHelper.CsvReader(reader, CultureInfo.InvariantCulture)) { var dataRead = csv.GetRecords <dynamic>().ToList(); foreach (var item in dataRead) { if (string.IsNullOrWhiteSpace(item.Date)) { continue; } var valueDictionary = new RouteValueDictionary(item); result.AddRange(Enum.GetNames(typeof(District)).Select(x => new CaseStatus { District = (District)Enum.Parse(typeof(District), x), Date = System.DateTime.ParseExact(item.Date, "dd-MM-yyyy", provider), Count = int.TryParse((string)valueDictionary["TotalSamplesTested"], out var value) ? value : 0 }));
private async void ImportPathfinderSpellsCSV(IStorageFile file) { using (var reader = new StreamReader(await file.OpenStreamForReadAsync())) { var csvHelper = new CsvHelper.CsvReader(reader); var spells = csvHelper.GetRecords<PathfinderSpellsCSV>(); foreach(var spell in spells) { //spell. } } }
public Dictionary <string, Booking> LoadUnscheduledBookings(int?lookaheadDays = null) { var bookingMap = new Dictionary <string, Booking>(); var pth = Path.Combine(CSVBasePath, "bookings.csv"); if (!File.Exists(pth)) { CreateEmptyBookingFile(pth); return(bookingMap); } using (var f = File.OpenRead(pth)) using (var sreader = new StreamReader(f)) using (var csv = new CsvHelper.CsvReader(sreader, CultureInfo.InvariantCulture)) { var orderCntr = 0; foreach (var row in csv.GetRecords <UnscheduledCsvRow>()) { var bookingId = row.Id; if (string.IsNullOrEmpty(bookingId)) { bookingId = "B" + GetUtcNow().ToString("yyyy-MM-dd-HH-mm-ss") + "-" + orderCntr.ToString(); orderCntr += 1; } Booking book; if (bookingMap.ContainsKey(bookingId)) { book = bookingMap[bookingId]; } else { book = new Booking { BookingId = bookingId, Priority = row.Priority, DueDate = row.DueDate, Parts = new List <BookingDemand>(), ScheduleId = null }; bookingMap.Add(bookingId, book); } book.Parts.Add(new BookingDemand { BookingId = bookingId, Part = row.Part, Quantity = row.Quantity, }); } } DateTime?endDate = null; if (lookaheadDays.HasValue && lookaheadDays.Value > 0) { endDate = DateTime.Today.AddDays(lookaheadDays.Value); } foreach (var id in bookingMap.Keys.ToList()) { var f = Path.Combine(CSVBasePath, Path.Combine(ScheduledBookingsPath, id + ".csv")); if (File.Exists(f) || endDate.HasValue && bookingMap[id].DueDate > endDate) { bookingMap.Remove(id); } } return(bookingMap); }
public ActionResult UploadExerciseVideosAndExcel() { StringBuilder traceLog = null; if (Login.IsSessionExpire(Convert.ToString(HttpContext.Session[ConstantHelper.constLoginUserName])) || Convert.ToString(HttpContext.Session[ConstantHelper.constUserType]) != ConstantHelper.constLoginadmin) { return(RedirectToAction(ConstantHelper.constLogin, ConstantHelper.constLogin)); } DataTable dt = null; try { traceLog = new StringBuilder(); traceLog.AppendLine("Start: UploadExerciseVideos() in Execise Controller"); var httpRequest = System.Web.HttpContext.Current.Request; if (httpRequest.Files.Count > 0) { Dictionary <string, string> uploadedExeciseVideos = new Dictionary <string, string>(); List <UpdateExerciseSproutLink> FailToUploadedExeciseVideos = new List <UpdateExerciseSproutLink>(); for (int i = 0; i < httpRequest.Files.Count; i++) { var fileDoc = httpRequest.Files[i]; IExcelDataReader reader = null; string sPath = ""; sPath = Server.MapPath("~/Temp/"); string newVideoId = Path.GetFileName(fileDoc.FileName); string execiseName = Path.GetFileNameWithoutExtension(fileDoc.FileName); if (fileDoc.FileName.EndsWith(".xls", StringComparison.OrdinalIgnoreCase)) { using (Stream stream = fileDoc.InputStream) { reader = ExcelReaderFactory.CreateBinaryReader(stream); reader.IsFirstRowAsColumnNames = true; DataSet result = reader.AsDataSet(); reader.Close(); dt.Clear(); dt = result.Tables[0]; } } else if (fileDoc.FileName.EndsWith(".csv", StringComparison.OrdinalIgnoreCase)) { dt = new DataTable(); dt.Locale = CultureInfo.InvariantCulture; dt.Clear(); dt.Columns.Add("Name"); dt.Columns.Add("Index"); dt.Columns.Add("Team ID"); dt.Columns.Add("Trainer ID"); if (!System.IO.File.Exists(sPath + newVideoId)) { fileDoc.SaveAs(sPath + newVideoId); } using (var fileReader = System.IO.File.OpenText(sPath + newVideoId)) using (var csvReader = new CsvHelper.CsvReader(fileReader)) { csvReader.Configuration.HasHeaderRecord = true; csvReader.Configuration.IgnoreHeaderWhiteSpace = false; csvReader.Configuration.IsHeaderCaseSensitive = true; csvReader.Configuration.RegisterClassMap <CSVExerciseMap>(); while (csvReader.Read()) { var record = csvReader.GetRecord <CSVExerciseVM>(); if (record != null) { DataRow rowdata = dt.NewRow(); rowdata["Name"] = record.Name; rowdata["Index"] = record.Index; rowdata["Team ID"] = record.TeamID; rowdata["Trainer ID"] = record.TrainerID; dt.Rows.Add(rowdata); } } } } else if (fileDoc.FileName.EndsWith(".xlsx", StringComparison.OrdinalIgnoreCase)) { using (Stream stream = fileDoc.InputStream) { reader = ExcelReaderFactory.CreateOpenXmlReader(stream); reader.IsFirstRowAsColumnNames = true; DataSet result = reader.AsDataSet(); reader.Close(); dt = result.Tables[0]; } } else if (fileDoc.FileName.EndsWith(".mp4", StringComparison.OrdinalIgnoreCase)) { uploadedExeciseVideos.Add(execiseName, sPath + newVideoId); } else { var FailToUpload = new UpdateExerciseSproutLink { ExerciseName = execiseName, Index = string.Empty, TeamID = string.Empty, TrainerID = string.Empty, FailedVideoName = fileDoc.FileName, }; FailToUploadedExeciseVideos.Add(FailToUpload); } // Save the files in local directory if (!System.IO.File.Exists(sPath + newVideoId)) { fileDoc.SaveAs(sPath + newVideoId); } } // varify execel execisename and upload on sprout server foreach (DataRow row in dt.Rows) { string execiseName = string.Empty; string index = string.Empty; string teamID = string.Empty; string trainerID = string.Empty; foreach (DataColumn col in dt.Columns) { switch (col.ColumnName) { case "Name": execiseName = Convert.ToString(row[col.ColumnName]); break; case "Index": index = Convert.ToString(row[col.ColumnName]); break; case "Team ID": teamID = Convert.ToString(row[col.ColumnName]); break; case "Trainer ID": trainerID = Convert.ToString(row[col.ColumnName]); break; } } // uploaded Execise Videos on sprout server if (uploadedExeciseVideos.ContainsKey(execiseName)) { // find out local stored file path from dictionary string localUploadPath = string.Empty; if (uploadedExeciseVideos.TryGetValue(execiseName, out localUploadPath)) { if (!string.IsNullOrEmpty(localUploadPath)) { // Upload on Execise Video on sprout server, get response and add to Fitcom database; string uploadesfile = UploadExeciseVideoToFitcom(localUploadPath); if (!string.IsNullOrEmpty(uploadesfile)) { ExerciseVideoResponse response = JsonConvert.DeserializeObject <ExerciseVideoResponse>(uploadesfile); if (response != null) { UpdateExerciseSproutLink objUpdateExerciseSproutLink = new UpdateExerciseSproutLink { SecuryId = response.security_token, VideoId = response.id, V1080pUrl = response.assets.videos._1080p, V240pUrl = response.assets.videos._240p, V360pUrl = response.assets.videos._360p, V480pUrl = response.assets.videos._480p, V720pUrl = response.assets.videos._720p, ThumnailUrl = response.assets.thumbnails[0], ExerciseName = execiseName, IsActive = true, Index = index, Description = string.Empty, IsUpdated = true, TeamID = teamID, TrainerID = trainerID, VideoLink = execiseName + ConstantHelper.constMP4ExesionWithdot, SourceUrl = string.Empty }; // Add Execise Details ExerciseBL.AddExerciseSproutData(objUpdateExerciseSproutLink); } } else { var FailToUpload = new UpdateExerciseSproutLink { ExerciseName = execiseName, Index = index, TeamID = teamID, TrainerID = trainerID, FailedVideoName = execiseName + ConstantHelper.constMP4ExesionWithdot, }; FailToUploadedExeciseVideos.Add(FailToUpload); } } // Delete Uploaded Execise if (System.IO.File.Exists(localUploadPath)) { System.IO.File.Delete(localUploadPath); } localUploadPath = string.Empty; } } else if (!string.IsNullOrEmpty(execiseName)) { var FailToUpload = new UpdateExerciseSproutLink { ExerciseName = execiseName, Index = index, TeamID = teamID, TrainerID = trainerID, FailedVideoName = string.Empty, }; FailToUploadedExeciseVideos.Add(FailToUpload); } } // Save To Failed Upload data in database ExerciseBL.SaveFailedSproutServer(FailToUploadedExeciseVideos); } else { return(Json(new { Message = "No files selected.", StatusCode = 100 }, JsonRequestBehavior.AllowGet)); } return(Json(new { Message = "File Uploaded Successfully!", StatusCode = 101 }, JsonRequestBehavior.AllowGet)); } catch (Exception ex) { LogManager.LogManagerInstance.WriteErrorLog(ex); return(Json(new { Message = "Error occurred. Error details: " + ex.Message, StatusCode = 100 }, JsonRequestBehavior.AllowGet)); } finally { traceLog.AppendLine("UploadExerciseVideos end() : --- " + DateTime.Now.ToLongDateString()); LogManager.LogManagerInstance.WriteTraceLog(traceLog); if (dt != null) { dt.Dispose(); } } }
private static void ValidateAcademicSessionsCSVFile(List <string> lstFileMissingHeaders, List <string> lstFileInvalidInfo, CsvHelper.CsvReader csvreader) { if (csvreader.ReadHeader()) { if (!csvreader.FieldHeaders.Contains("sourcedId")) { lstFileMissingHeaders.Add("sourcedId"); } if (!csvreader.FieldHeaders.Contains("title")) { lstFileMissingHeaders.Add("title"); } if (!csvreader.FieldHeaders.Contains("type")) { lstFileMissingHeaders.Add("type"); } if (!csvreader.FieldHeaders.Contains("startDate")) { lstFileMissingHeaders.Add("startDate"); } if (!csvreader.FieldHeaders.Contains("endDate")) { lstFileMissingHeaders.Add("endDate"); } } int iRow = 1; while (csvreader.Read()) { ValidateField("sourcedId", null, lstFileInvalidInfo, iRow, csvreader); ValidateField("title", null, lstFileInvalidInfo, iRow, csvreader); //ValidateField("type", new string[] {"term", "gradingPeriod", "schoolYear", "semester" }, lstFileInvalidInfo, iRow, csvreader); ValidateField("startDate", null, lstFileInvalidInfo, iRow, csvreader); ValidateField("endDate", null, lstFileInvalidInfo, iRow, csvreader); iRow++; } }
private AdvException GetError(Debug.ErrType errType, string atTime) { try { if (File.Exists(Debug.ErrFile[errType])) { using (var csv = new CsvHelper.CsvReader(new StreamReader(Debug.ErrFile[errType], Encoding.UTF8, true))) { csv.Configuration.Delimiter = Debug.CharSeparate; csv.Configuration.HasHeaderRecord = false; while (csv.Read()) { string[] r = csv.CurrentRecord; if (atTime == r[0] && r.Length >= 5) { var ex1 = AdvException.GetFromJsonString(r[4]); ex1.ExceptionData.ManualMessage = r[2]; return ex1; } } } } } catch (Exception ex) { lblErr.Text = ex.Message; } return null; }
public ActionResult ImportDataProcess(string id, CreateSimpleModel model, FormCollection form) { var email = Request?.RequestContext?.HttpContext?.User?.Identity?.Name; if (Request.IsAuthenticated == false) { //https://www.hlidacstatu.cz/account/Login?returnUrl=%2F%3Frnd%3D0036bd9be9bc42d4bdf449492968846e return(RedirectToAction("Login", "Account", new { returnUrl = this.Request.Url.PathAndQuery })); } ViewBag.NumOfRows = 0; model.DatasetId = id; string[] csvHeaders = null; if (string.IsNullOrEmpty(id)) { return(Redirect("/data")); } var ds = DataSet.CachedDatasets.Get(id); if (ds == null) { return(Redirect("/data")); } if (ds.HasAdminAccess(email) == false) { return(View("NoAccess")); } datasetIndexStatCache.Invalidate(); if (ds.IsFlatStructure() == false) { return(RedirectToAction("ImportData", new { id = ds.DatasetId, fileId = model.FileId, delimiter = model.GetValidDelimiter() })); } var uTmp = new Lib.IO.UploadedTmpFile(); var path = uTmp.GetFullPath(model.FileId.ToString(), model.FileId.ToString() + ".csv"); if (!System.IO.File.Exists(path)) { return(RedirectToAction("ImportData", new { id = ds.DatasetId })); } RuntimeClassBuilder rcb = new RuntimeClassBuilder(ds.GetPropertyNamesTypesFromSchema().ToDictionary(m => m.Key, v => v.Value.Type)); string[] formsHeaders = form["sheaders"].Split('|'); List <MappingCSV> mappingProps = new List <MappingCSV>(); for (int i = 0; i < formsHeaders.Length + 3; i++) //+3 a little bit more, at least +1 for id column { if (!string.IsNullOrEmpty(form["source_" + i]) && !string.IsNullOrEmpty(form["target_" + i]) && !string.IsNullOrEmpty(form["transform_" + i]) ) { mappingProps.Add(new MappingCSV() { sourceCSV = form["source_" + i], TargetJSON = form["target_" + i], Transform = form["transform_" + i] } ); } } System.Collections.Concurrent.ConcurrentBag <Exception> errors = new System.Collections.Concurrent.ConcurrentBag <Exception>(); List <Tuple <object, string> > items = new List <Tuple <object, string> >(); try { using (System.IO.StreamReader r = new System.IO.StreamReader(path)) { if (model.Delimiter == "auto") { model.Delimiter = Devmasters.IO.IOTools.DetectCSVDelimiter(r); } var csv = new CsvHelper.CsvReader(r, new CsvHelper.Configuration.CsvConfiguration(Util.Consts.csCulture) { HasHeaderRecord = true , Delimiter = model.GetValidDelimiter() //,MissingFieldFound = null } ); csv.Read(); csv.ReadHeader(); csvHeaders = csv.HeaderRecord.Where(m => !string.IsNullOrEmpty(m?.Trim())).ToArray(); //for future control while (csv.Read()) { var newObj = rcb.CreateObject(); for (int m = 0; m < mappingProps.Count; m++) { Type destType = ds.GetPropertyNameTypeFromSchema(mappingProps[m].TargetJSON).FirstOrDefault().Value.Type; object value = null; string[] specialValues = new string[] { "-skip-", "-gen-", "--" }; if (specialValues.Contains(mappingProps[m].sourceCSV)) { if (mappingProps[m].sourceCSV == "-gen-") { value = Guid.NewGuid().ToString("N"); rcb.SetPropertyValue(newObj, mappingProps[m].TargetJSON, value); } else { continue; // -skip- skip } } else { string svalue = null; try { svalue = csv.GetField(mappingProps[m].sourceCSV); if (destType == typeof(string)) { value = svalue; } else if (destType == typeof(DateTime) || destType == typeof(DateTime?)) { value = Devmasters.DT.Util.ToDateTime(svalue); } else if (destType == typeof(decimal) || destType == typeof(decimal?)) { value = Util.ParseTools.ToDecimal(svalue); if (value == null) { value = Util.ParseTools.FromTextToDecimal(svalue); } } else if (destType == typeof(long) || destType == typeof(long?) || destType == typeof(int) || destType == typeof(int?)) { value = Devmasters.DT.Util.ToDate(svalue); } else if (destType == typeof(bool) || destType == typeof(bool?)) { if (bool.TryParse(svalue, out bool tryp)) { value = tryp; } } else { value = svalue; } if (mappingProps[m].Transform == "normalize" && destType == typeof(string) ) { value = DataSet.NormalizeValueForId((string)value); } else if (mappingProps[m].Transform == "findico" && destType == typeof(string) ) { value = Lib.Validators.IcosInText((string)value).FirstOrDefault(); } else //copy { } rcb.SetPropertyValue(newObj, mappingProps[m].TargetJSON, value); } catch (Exception mex) { errors.Add(mex); } } } //for string idPropName = "id"; string idVal = rcb.GetPropertyValue(newObj, "id")?.ToString(); if (string.IsNullOrEmpty(idVal)) { idVal = rcb.GetPropertyValue(newObj, "Id")?.ToString(); idPropName = "Id"; } if (string.IsNullOrEmpty(idVal)) { idVal = rcb.GetPropertyValue(newObj, "iD")?.ToString(); idPropName = "iD"; } if (string.IsNullOrEmpty(idVal)) { idVal = rcb.GetPropertyValue(newObj, "ID")?.ToString(); idPropName = "ID"; } try { //var debugJson = Newtonsoft.Json.JsonConvert.SerializeObject(newObj); //normalize ID idVal = DataSet.NormalizeValueForId(idVal); rcb.SetPropertyValue(newObj, idPropName, idVal); items.Add(new Tuple <object, string>(newObj, idVal)); model.NumOfRows++; } catch (DataSetException dex) { errors.Add(dex); } catch (Exception ex) { errors.Add(ex); } } } } catch (Exception ex) { errors.Add(ex); } try { Devmasters.Batch.Manager.DoActionForAll <Tuple <object, string> >(items, (item) => { try { ds.AddData(item.Item1, item.Item2, email, true); } catch (Exception ex) { errors.Add(ex); } return(new Devmasters.Batch.ActionOutputData()); }, true ); } catch (Exception ex) { errors.Add(ex); } if (errors?.Count > 0) { HlidacStatu.Util.Consts.Logger.Error("ImportDataProcess exceptions \n" + errors.Select(m => m.Message).Aggregate((f, s) => f + "\n" + s)); } ViewBag.ApiResponseError = ApiResponseStatus.Error(-99, "Chyba při importu dat"); ViewBag.Errors = errors.ToList(); return(View(model)); }
public ActionResult CreateSimple2(CreateSimpleModel model) { var email = Request?.RequestContext?.HttpContext?.User?.Identity?.Name; if (Request.IsAuthenticated == false) { return(RedirectToAction("Login", "Account", new { returnUrl = this.Request.Url.PathAndQuery })); } var uTmp = new Lib.IO.UploadedTmpFile(); var path = uTmp.GetFullPath(model.FileId.ToString(), model.FileId.ToString() + ".csv"); if (!System.IO.File.Exists(path)) { return(RedirectToAction("CreateSimple")); } try { using (System.IO.StreamReader r = new System.IO.StreamReader(path)) { var config = new CsvHelper.Configuration.CsvConfiguration(Util.Consts.csCulture); config.HasHeaderRecord = true; config.Delimiter = model.GetValidDelimiter(); var csv = new CsvHelper.CsvReader(r, config); csv.Read(); csv.ReadHeader(); model.Headers = csv.HeaderRecord.Where(m => !string.IsNullOrEmpty(m?.Trim())).ToArray(); //read first lines with data and guest type List <string[]> lines = new List <string[]>(); for (int row = 0; row < 10; row++) { if (csv.Read()) { lines.Add(csv.GetRecords <string>().ToArray()); } } List <string> colTypes = null; if (lines.Count > 0) { colTypes = new List <string>(); for (int cx = 0; cx < lines[0].Length; cx++) { string t = GuestBestCSVValueType(lines[0][cx]); for (int line = 1; line < lines.Count; line++) { var nextT = GuestBestCSVValueType(lines[line][cx]); if (nextT != t) { t = "string"; //kdyz jsou ruzne typy ve stejnem sloupci v ruznych radcich, } //fallback na string } colTypes.Add(t); } } ViewBag.ColTypes = colTypes.ToArray(); return(View(model)); } } catch (Exception e) { ViewBag.ApiResponseError = ApiResponseStatus.Error(-99, "Soubor není ve formátu CSV.", e.ToString()); return(View(model)); } }
public IEnumerable<PlateletActive.Core.Models.HplcData> GetLogFileData(string path) { importing = true; var filePaths = System.IO.Directory.GetFiles(path, "*.csv"); var hplcDatas = new List<PlateletActive.Core.Models.HplcData>(); filesImported = new List<string>(); int concentrationColumn = -1; foreach (var filePath in filePaths) { var valid = true; using (System.IO.StreamReader reader = System.IO.File.OpenText(filePath)) { using (var csv = new CsvHelper.CsvReader(reader)) { int row = 2; var hplcData = new Core.Models.HplcData(); while (csv.Read()) { if (!valid) { continue; } string fieldATemp = null; var fieldA = csv.TryGetField<string>(0, out fieldATemp) ? fieldATemp : null; string fieldBTemp = null; var fieldB = csv.TryGetField<string>(1, out fieldBTemp) ? fieldBTemp : null; var fieldBDateTime = new DateTime(); if (fieldA == "Generated" && DateTime.TryParse(fieldB, out fieldBDateTime)) { hplcData.Timestamp = fieldBDateTime; } if (fieldA == "Sample Name") { hplcData.SampleName = fieldB; var sampleNameParts = fieldB.Split('-'); int batchId = 0; if (sampleNameParts.Count() == 4 && Int32.TryParse(sampleNameParts.ElementAt(1), out batchId)) { hplcData.BatchID = batchId; if (hplcData.BatchID == null) { valid = false; continue; } hplcData.SampleLocation = sampleNameParts.First(); hplcData.SampleAge = sampleNameParts.ElementAt(2); hplcData.User = sampleNameParts.Last(); } else if (sampleNameParts.Count() == 4) { hplcData.SampleLocation = sampleNameParts.First(); hplcData.User = sampleNameParts.Last(); } else if (sampleNameParts.Count() == 3 && Int32.TryParse(sampleNameParts.ElementAt(0), out batchId)) { hplcData.BatchID = batchId; if (hplcData.BatchID == null) { valid = false; continue; } hplcData.SampleLocation = sampleNameParts.ElementAt(1); hplcData.User = sampleNameParts.Last(); } else if (sampleNameParts.Count() == 3 && Int32.TryParse(sampleNameParts.ElementAt(1), out batchId)) { hplcData.BatchID = batchId; if (hplcData.BatchID == null) { valid = false; continue; } hplcData.SampleLocation = sampleNameParts.First(); hplcData.User = sampleNameParts.Last(); } else if (sampleNameParts.Count() == 3) { hplcData.SampleLocation = sampleNameParts.ElementAt(1); hplcData.User = sampleNameParts.Last(); } else if (sampleNameParts.Count() == 2) { hplcData.SampleLocation = sampleNameParts.First(); hplcData.User = sampleNameParts.Last(); } else if (sampleNameParts.Count() == 1) { hplcData.User = sampleNameParts.First(); } } // Find concentration column. if (fieldA.Trim() == "ID#") { string concentrationColumnCheck = string.Empty; for (int i = 0; i < 10; i++) { // If this is the concenctration column then break out of the loop and use this column to get // the concentrations. if (csv.TryGetField<string>(i, out concentrationColumnCheck) && concentrationColumnCheck.Trim() == "Conc.") { concentrationColumn = i; break; } } // If the concentration column was not found then mark this file as invalid. if (concentrationColumn == -1) { valid = false; continue; } } // Get concentrations. if (fieldB == "DP4") { double hplcDataDp4Temp = -1; if (!csv.TryGetField<double>(concentrationColumn, out hplcDataDp4Temp)) { valid = false; continue; } hplcData.DP4 = hplcDataDp4Temp; } if (fieldB == "DP3") { double hplcDataDp3Temp = -1; if (!csv.TryGetField<double>(concentrationColumn, out hplcDataDp3Temp)) { valid = false; continue; } hplcData.DP3 = hplcDataDp3Temp; } if (fieldB == "DP2 MALTOSE") { double hplcDataDp2MaltoseTemp = -1; if (!csv.TryGetField<double>(concentrationColumn, out hplcDataDp2MaltoseTemp)) { valid = false; continue; } hplcData.DP2Maltose = hplcDataDp2MaltoseTemp; } if (fieldB == "DP1 GLUCOSE") { double hplcDataDp1GlucoseTemp = -1; if (!csv.TryGetField<double>(concentrationColumn, out hplcDataDp1GlucoseTemp)) { valid = false; continue; } hplcData.DP1Glucose = hplcDataDp1GlucoseTemp; } if (fieldB == "LACTIC ACID") { double hplcDataLacticAcidTemp = -1; if (!csv.TryGetField<double>(concentrationColumn, out hplcDataLacticAcidTemp)) { valid = false; continue; } hplcData.LacticAcid = hplcDataLacticAcidTemp; } if (fieldB == "GLYCEROL") { double hplcDataGlycerolTemp = -1; if (!csv.TryGetField<double>(concentrationColumn, out hplcDataGlycerolTemp)) { valid = false; continue; } hplcData.Glycerol = hplcDataGlycerolTemp; } if (fieldB == "ACETIC ACID") { double hplcDataAceticAcidTemp = -1; if (!csv.TryGetField<double>(concentrationColumn, out hplcDataAceticAcidTemp)) { valid = false; continue; } hplcData.AceticAcid = hplcDataAceticAcidTemp; } if (fieldB == "ETHANOL") { double hplcDataEthanolTemp = -1; if (!csv.TryGetField<double>(concentrationColumn, out hplcDataEthanolTemp)) { valid = false; continue; } hplcData.Ethanol = hplcDataEthanolTemp; } row++; } if (valid) { hplcDatas.Add(hplcData); ((List<string>)filesImported).Add(filePath); } } } } importing = false; return hplcDatas; }
/// <summary> /// /// </summary> /// <param name="outputDirectory"></param> private void ProcessAttachmentHashes(string outputDirectory) { if (File.Exists(System.IO.Path.Combine(outputDirectory, "Attachment.Hashes.csv")) == false) { OnError("Cannot locate the \"Attachment.Hashes.csv\" files"); return; } CsvConfiguration csvConfig = new CsvConfiguration(); Regex regex = new Regex("<(.*?)>", RegexOptions.IgnoreCase); using (StreamReader sr = new StreamReader(System.IO.Path.Combine(outputDirectory, "Attachment.Hashes.csv"))) using (CsvHelper.CsvReader csvReader = new CsvHelper.CsvReader(sr, csvConfig)) { List<Attachment> attachments = new List<Attachment>(); while (csvReader.Read()) { var md5 = csvReader.GetField(0); var file = csvReader.GetField(1); var fileName = System.IO.Path.GetFileName(file); var srcIp = csvReader.GetField(2); var srcPort = csvReader.GetField(3); var dstIp = csvReader.GetField(4); var dstPort = csvReader.GetField(5); var to = csvReader.GetField(6); var dateSent = csvReader.GetField(11); List<string> tempTo = new List<string>(to.Split(',')); for (int index = tempTo.Count() - 1; index > -1; index--) { string person = tempTo[index].Trim().ToLower(); if (person.IndexOf("@") == -1) { tempTo.RemoveAt(index); continue; } Match match = regex.Match(person); if (match.Success == true) { person = match.Groups[1].Value; } person = person.Replace(@"""", string.Empty); tempTo[index] = person; } var from = csvReader.GetField(6); var sender = csvReader.GetField(7); var subject = csvReader.GetField(10); var attachment = (from a in attachments where a.Md5 == md5.ToLower() select a).SingleOrDefault(); if (attachment == null) { attachment = new Attachment(); attachment.Md5 = md5.ToLower(); attachment.Recipients.AddRange(tempTo); attachment.Subjects.Add(subject); attachment.Senders.Add(sender); if (fileName.Length > 0) { attachment.FileNames.Add(fileName); } attachment.DateSent = dateSent; attachments.Add(attachment); SubjectRecipents subjectRecipient = new SubjectRecipents(); subjectRecipient.Subject = subject; subjectRecipient.File = file; subjectRecipient.Sender = sender; foreach (string person in tempTo) { subjectRecipient.Recipients.Add(person); } attachment.SubjectRecipents.Add(subjectRecipient); } else { foreach (string person in tempTo) { var tempPerson = from r in attachment.Recipients where r == person select r; if (tempPerson.Any() == false) { attachment.Recipients.Add(person); } } var tempFileName = from s in attachment.FileNames where s == fileName select s; if (tempFileName.Any() == false) { attachment.FileNames.Add(fileName); } var tempSubject = from s in attachment.Subjects where s.ToLower() == subject select s; if (tempSubject.Any() == false) { attachment.Subjects.Add(subject); } var tempSender = from s in attachment.Senders where s.ToLower() == sender select s; if (tempSender.Any() == false) { attachment.Senders.Add(sender); } var subjectRecipient = (from s in attachment.SubjectRecipents where s.Subject.ToLower() == subject.ToLower() select s).SingleOrDefault(); if (subjectRecipient == null) { subjectRecipient = new SubjectRecipents(); subjectRecipient.Subject = subject; subjectRecipient.File = file; subjectRecipient.Sender = sender; foreach (string person in tempTo) { subjectRecipient.Recipients.Add(person); } attachment.SubjectRecipents.Add(subjectRecipient); } else { foreach (string person in tempTo) { var tempPerson = from r in subjectRecipient.Recipients where r == person select r; if (tempPerson.Any() == false) { subjectRecipient.Recipients.Add(person); } } } } } OutputSummary(outputDirectory, attachments); } }
public static SqlTable CsvToSqlTable(TextReader textReader, ISqlWriter sqlWriter, bool hasHeader = true, string delimiter = ",", int count = -1) { var config = new CsvHelper.Configuration.CsvConfiguration(); config.Delimiter = delimiter; config.HasHeaderRecord = hasHeader; var reader = new CsvHelper.CsvReader(textReader, config); var rows = reader.GetRecords<object>().ToList(); textReader.Close(); SqlTable sqlTable = null; if (rows.Count > 0) { sqlTable = new SqlTable(); sqlTable.Headers = new List<SqlColumnTypeDefinition>(); sqlTable.Rows = new List<SqlRow>(); var i = 0; foreach (var row in rows) { if (i == count) break; i++; var columns = (System.Dynamic.ExpandoObject)row; var sqlRow = new SqlRow(); sqlTable.Rows.Add(sqlRow); var countUnkown = 1; foreach (var col in columns) { // set rows var sqlColumn = new SqlColumn(); sqlColumn.Type = sqlWriter.SqlTypeToCSharpType(col.Value); sqlColumn.Value = col.Value; sqlRow.Columns.Add(sqlColumn); // set headers var key = col.Key; if (string.IsNullOrWhiteSpace(key)) { key = "Unkown" + countUnkown; countUnkown++; } SqlColumnTypeDefinition sqlColumnTypeDefinition = sqlTable.Headers.FirstOrDefault(f => f.Name == key); if (sqlColumnTypeDefinition == null) { sqlColumnTypeDefinition = new SqlColumnTypeDefinition(); sqlColumnTypeDefinition.Name = col.Key; sqlColumnTypeDefinition.Type = sqlColumn.Type; sqlTable.Headers.Add(sqlColumnTypeDefinition); } sqlColumn.ColumnTypeDefinition = sqlColumnTypeDefinition; // override type of header to STRING because exists a string in the column if (sqlColumnTypeDefinition.Type != typeof(string) && sqlColumn.Type == typeof(string)) sqlColumnTypeDefinition.Type = typeof(string); } } // Fix type to STRING if all values is 'NULL' foreach (var sqlHeader in sqlTable.Headers) { if (sqlHeader.Type == null) sqlHeader.Type = typeof(string); sqlHeader.TypeFormatted = sqlWriter.CSharpTypeToSqlType(sqlHeader.Type); } } return sqlTable; }
public ActionResult ImportData(string id, CreateSimpleModel model) { var email = Request?.RequestContext?.HttpContext?.User?.Identity?.Name; if (Request.IsAuthenticated == false) { //https://www.hlidacstatu.cz/account/Login?returnUrl=%2F%3Frnd%3D0036bd9be9bc42d4bdf449492968846e return(RedirectToAction("Login", "Account", new { returnUrl = this.Request.Url.PathAndQuery })); } model = model ?? new CreateSimpleModel(); model.DatasetId = id; if (string.IsNullOrEmpty(id)) { return(Redirect("/data")); } var ds = DataSet.CachedDatasets.Get(id); if (ds == null) { return(Redirect("/data")); } if (ds.HasAdminAccess(email) == false) { return(View("NoAccess")); } model.DatasetId = ds.DatasetId; if (ds.IsFlatStructure() == false) { ViewBag.Mode = "notflat"; return(View("ImportData.noflat", model)); } if (model.FileId.HasValue) { ViewBag.Mode = "mapping"; Guid fileId = model.FileId.Value; var uTmp = new Lib.IO.UploadedTmpFile(); var path = uTmp.GetFullPath(fileId.ToString(), fileId.ToString() + ".csv"); if (!System.IO.File.Exists(path)) { return(RedirectToAction("ImportData", new { id = model.DatasetId })); } using (System.IO.StreamReader r = new System.IO.StreamReader(path)) { if (model.Delimiter == "auto") { model.Delimiter = Devmasters.IO.IOTools.DetectCSVDelimiter(r); } var csv = new CsvHelper.CsvReader(r, new CsvHelper.Configuration.CsvConfiguration(Util.Consts.csCulture) { HasHeaderRecord = true, Delimiter = model.GetValidDelimiter() }); csv.Read(); csv.ReadHeader(); model.Headers = csv.HeaderRecord.Where(m => !string.IsNullOrEmpty(m?.Trim())).ToArray(); } return(View("ImportData.mapping", model)); } else { ViewBag.Mode = "upload"; } return(View(model)); }
public static void LoadStellarClasses() { using(var stream = new System.IO.StreamReader(StellarClassDefinitionFile)) { var csvReader = new CsvHelper.CsvReader(stream); // skip 2 header rows csvReader.ReadHeader(); csvReader.Read(); while (csvReader.Read()) { var item = new StarClass { Name = csvReader.GetField(0), Mass = csvReader.GetField<double>(1), Luminosity = csvReader.GetField<double>(2), Radius = csvReader.GetField<double>(3), Temperature = csvReader.GetField<double>(4), ColorIndex = csvReader.GetField<float>(5), AbsoluteMagnitude = csvReader.GetField<float>(6), BolometricCorrection = csvReader.GetField<float>(7), BolometricMagnitude = csvReader.GetField<float>(8), }; var rgb = csvReader.GetField(9).Split(' ').Select(s => byte.Parse(s)).ToArray(); item.Color = new Color(rgb[0], rgb[1], rgb[2]); StellarDefinitions[item.Name] = item; } } }
//******************** //* * //* UploadEntities * //* * //******************** // Upload a collection of entities from a local file in a selected download format. private void UploadEntities(String tableName, String format, String inputFile, String outerElementName, String partitionKeyColumnName, String rowKeyColumnName, bool stopOnError) { int recordNumber = 1; int recordsAdded = 0; int recordErrors = 0; String serializationError = null; String xpath = outerElementName; // Create task action. String message = null; message = "Uploading entities to table " + tableName + " from file " + inputFile; Action action = new Action() { Id = NextAction++, ActionType = Action.ACTION_DOWNLOAD_ENTITIES, IsCompleted = false, Message = message }; Actions.Add(action.Id, action); UpdateStatus(); // Execute background task to perform the downloading. Task task = Task.Factory.StartNew(() => { CloudTable table = tableClient.GetTableReference(tableName); //table.CreateIfNotExists(); // Upload data using the specified format (csv, json, or xml). switch (format) { case "json": { // JSON format upload // { // "Entities": [ // { // "RowKey": "Batman", // "PartitionKey": "DC Comics", // "Timestamp": "8/6/2014 4:07:06 AM +00:00", // "Debut": "5/1/1939 12:00:00 AM", // "SecretIdentity": "Bruce Wayne" // }, // { // "RowKey": "Green Lantern", // "PartitionKey": "DC Comics", // "Timestamp": "8/6/2014 4:10:52 AM +00:00", // "Debut": "7/1/1940 12:00:00 AM", // "SecretIdentity": "Hal Jordan" // }, // ... // ] //} Dictionary<String, Object> entries = null; // Parse the data in the JavaScriptSerializer. try { JavaScriptSerializer ser = new JavaScriptSerializer(); entries = ser.DeserializeObject(File.ReadAllText(inputFile)) as Dictionary<String, Object>; } catch (Exception ex) { serializationError = "An error occurred deserializing the JSON file: " + ex.Message; } // Walk the result object graph and extract entities. if (entries != null) { foreach (KeyValuePair<String, Object> entry in entries) { if (entry.Key == outerElementName) { Object[] entities = entry.Value as Object[]; foreach (object ent in entities) { if (ent is Dictionary<String, Object>) { if (WriteEntity(tableName, ent as Dictionary<String, Object>, partitionKeyColumnName, rowKeyColumnName)) { recordsAdded++; recordNumber++; } else { recordErrors++; if (stopOnError) { recordNumber++; break; } } } } } } } } break; case "xml": { // XML format upload XmlDocument doc = null; try { doc = new XmlDocument(); doc.LoadXml(File.ReadAllText(inputFile)); //foreach (XmlElement entities in doc.DocumentElement.GetElementsByTagName(outerElementName)) XmlNodeList nodes = doc.DocumentElement.SelectNodes(xpath); //foreach (XmlElement entities in doc.DocumentElement) foreach (XmlElement entities in nodes) { //<Entities xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> // <Entity> // <RowKey>Batman</RowKey> // <PartitionKey>DC Comics</PartitionKey> // <Timestamp>8/6/2014 4:07:06 AM +00:00</Timestamp> // <Debut>5/1/1939 12:00:00 AM</Debut> // <SecretIdentity>Bruce Wayne</SecretIdentity> // </Entity> List<String> columns = new List<string>(); List<String> values = new List<string>(); foreach (XmlElement entity in entities.ChildNodes) // .GetElementsByTagName("Entity")) //foreach (XmlElement entity in entities.GetElementsByTagName(outerElementName)) { foreach (XmlNode field in entity.ChildNodes) { if (field is XmlText) { XmlText node = field as XmlText; columns.Add(node.ParentNode.Name); values.Add(node.Value); } } } if (WriteEntity(tableName, columns.ToArray(), values.ToArray(), partitionKeyColumnName, rowKeyColumnName)) { recordsAdded++; recordNumber++; } else { recordErrors++; if (stopOnError) { recordNumber++; break; } } } } catch (Exception ex) { serializationError = "An error occurred parsing the XML file: " + ex.Message; } } break; case "csv": { // CSV format upload. Uses CsvHelper from http://joshclose.github.io/CsvHelper/. try { using (TextReader reader = File.OpenText(inputFile)) { var csv = new CsvHelper.CsvReader(reader); String[] columns = null; String[] values = null; while(csv.Read()) { // If first pass, retrieve column names. if (recordNumber == 1) { columns = csv.FieldHeaders; values = new String[columns.Length]; } recordNumber++; // Retrieve record values. int col = 0; foreach (String column in columns) { values[col] = csv.GetField(column); col++; } // Write entity. if (WriteEntity(tableName, columns, values, partitionKeyColumnName, rowKeyColumnName)) { recordsAdded++; recordNumber++; } else { recordErrors++; if (stopOnError) { recordNumber++; break; } } } } } catch (Exception ex) { serializationError = "An error occurred parsing the CSV file: " + ex.Message; } } break; default: ShowError("Cannot upload - unknown format '" + format + "'"); break; } Actions[action.Id].IsCompleted = true; }); // Task complete - update UI. task.ContinueWith((t) => { if (serializationError != null) { ShowError(serializationError); } switch (recordErrors) { case 0: break; case 1: ShowError("An error occurred inserting entity nunber " + (recordNumber-1).ToString() + "."); break; default: ShowError(recordErrors.ToString() + " errors occurred inserting entities."); break; } UpdateStatus(); ShowTableContainer(SelectedTableContainer); }, TaskScheduler.FromCurrentSynchronizationContext()); }
private void ProcessCsv(string fullPath) { CommonStatistic.Init(); CommonStatistic.IsRun = true; var fieldMapping = new Dictionary<string, int>(); using (var csv = new CsvHelper.CsvReader(new StreamReader(fullPath, Encoding.UTF8))) { csv.Configuration.Delimiter = ";"; csv.Configuration.HasHeaderRecord = false; csv.Read(); for (int i = 0; i < csv.CurrentRecord.Length; i++) { if (csv.CurrentRecord[i] == ProductFields.GetStringNameByEnum(ProductFields.Fields.None)) continue; if (!fieldMapping.ContainsKey(csv.CurrentRecord[i])) fieldMapping.Add(csv.CurrentRecord[i], i); } } using (var csv = new CsvHelper.CsvReader(new StreamReader(fullPath, Encoding.UTF8))) { csv.Configuration.Delimiter = ";"; csv.Configuration.HasHeaderRecord = true; while (csv.Read()) { if (!CommonStatistic.IsRun) { csv.Dispose(); FileHelpers.DeleteFile(fullPath); return; } CommonStatistic.RowPosition++; try { // Step by rows var productInStrings = new Dictionary<ProductFields.Fields, string>(); string nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.Sku); if (fieldMapping.ContainsKey(nameField)) { productInStrings.Add(ProductFields.Fields.Sku, SQLDataHelper.GetString(csv[fieldMapping[nameField]])); } nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.Name).Trim('*'); if (fieldMapping.ContainsKey(nameField)) { var name = SQLDataHelper.GetString(csv[fieldMapping[nameField]]); if (!string.IsNullOrEmpty(name)) { productInStrings.Add(ProductFields.Fields.Name, name); } else { Log(string.Format(Resource.Admin_ImportCsv_CanNotEmpty, ProductFields.GetDisplayNameByEnum(ProductFields.Fields.Name), CommonStatistic.RowPosition + 2), "InvalidData"); continue; } } nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.Enabled).Trim('*'); if (fieldMapping.ContainsKey(nameField)) { string enabled = SQLDataHelper.GetString(csv[fieldMapping[nameField]]); productInStrings.Add(ProductFields.Fields.Enabled, enabled); } nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.Discount); if (fieldMapping.ContainsKey(nameField)) { string discount = SQLDataHelper.GetString(csv[fieldMapping[nameField]]); if (string.IsNullOrEmpty(discount)) discount = "0"; float tmp; if (float.TryParse(discount, out tmp)) { productInStrings.Add(ProductFields.Fields.Discount, tmp.ToString()); } else if (float.TryParse(discount, NumberStyles.Any, CultureInfo.InvariantCulture, out tmp)) { productInStrings.Add(ProductFields.Fields.Discount, tmp.ToString()); } else { Log(string.Format(Resource.Admin_ImportCsv_MustBeNumber, ProductFields.GetDisplayNameByEnum(ProductFields.Fields.Discount), CommonStatistic.RowPosition + 2), "InvalidData"); continue; } } nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.Weight); if (fieldMapping.ContainsKey(nameField)) { string weight = SQLDataHelper.GetString(csv[fieldMapping[nameField]]); if (string.IsNullOrEmpty(weight)) weight = "0"; float tmp; if (float.TryParse(weight, out tmp)) { productInStrings.Add(ProductFields.Fields.Weight, tmp.ToString()); } else if (float.TryParse(weight, NumberStyles.Any, CultureInfo.InvariantCulture, out tmp)) { productInStrings.Add(ProductFields.Fields.Weight, tmp.ToString()); } else { Log(string.Format(Resource.Admin_ImportCsv_MustBeNumber, ProductFields.GetDisplayNameByEnum(ProductFields.Fields.Weight), CommonStatistic.RowPosition + 2), "InvalidData"); continue; } } nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.Size); if (fieldMapping.ContainsKey(nameField)) { productInStrings.Add(ProductFields.Fields.Size, SQLDataHelper.GetString(csv[fieldMapping[nameField]])); } nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.BriefDescription); if (fieldMapping.ContainsKey(nameField)) { productInStrings.Add(ProductFields.Fields.BriefDescription, SQLDataHelper.GetString(csv[fieldMapping[nameField]])); } nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.Description); if (fieldMapping.ContainsKey(nameField)) { productInStrings.Add(ProductFields.Fields.Description, SQLDataHelper.GetString(csv[fieldMapping[nameField]])); } nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.MultiOffer); if (fieldMapping.ContainsKey(nameField)) { var multiOffer = SQLDataHelper.GetString(csv[fieldMapping[nameField]]); if (!string.IsNullOrEmpty(multiOffer)) { productInStrings.Add(ProductFields.Fields.MultiOffer, multiOffer); } } nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.Price).Trim('*'); if (fieldMapping.ContainsKey(nameField)) { string price = SQLDataHelper.GetString(csv[fieldMapping[nameField]]); if (string.IsNullOrEmpty(price)) price = "0"; float tmp; if (float.TryParse(price, out tmp)) { productInStrings.Add(ProductFields.Fields.Price, tmp.ToString()); } else if (float.TryParse(price, NumberStyles.Any, CultureInfo.InvariantCulture, out tmp)) { productInStrings.Add(ProductFields.Fields.Price, tmp.ToString()); } else { Log(string.Format(Resource.Admin_ImportCsv_MustBeNumber, ProductFields.GetDisplayNameByEnum(ProductFields.Fields.Price), CommonStatistic.RowPosition + 2), "InvalidData"); continue; } } nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.PurchasePrice); if (fieldMapping.ContainsKey(nameField)) { string sypplyprice = SQLDataHelper.GetString(csv[fieldMapping[nameField]]); if (string.IsNullOrEmpty(sypplyprice)) sypplyprice = "0"; float tmp; if (float.TryParse(sypplyprice, out tmp)) { productInStrings.Add(ProductFields.Fields.PurchasePrice, tmp.ToString()); } else if (float.TryParse(sypplyprice, NumberStyles.Any, CultureInfo.InvariantCulture, out tmp)) { productInStrings.Add(ProductFields.Fields.PurchasePrice, tmp.ToString()); } else { Log(string.Format(Resource.Admin_ImportCsv_MustBeNumber, ProductFields.GetDisplayNameByEnum(ProductFields.Fields.PurchasePrice), CommonStatistic.RowPosition + 2), "InvalidData"); continue; } } nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.ShippingPrice); if (fieldMapping.ContainsKey(nameField)) { string shippingPrice = SQLDataHelper.GetString(csv[fieldMapping[nameField]]); if (string.IsNullOrEmpty(shippingPrice)) shippingPrice = "0"; float tmp; if (float.TryParse(shippingPrice, out tmp)) { productInStrings.Add(ProductFields.Fields.ShippingPrice, tmp.ToString()); } else if (float.TryParse(shippingPrice, NumberStyles.Any, CultureInfo.InvariantCulture, out tmp)) { productInStrings.Add(ProductFields.Fields.ShippingPrice, tmp.ToString()); } else { Log(string.Format(Resource.Admin_ImportCsv_MustBeNumber, ProductFields.GetDisplayNameByEnum(ProductFields.Fields.ShippingPrice), CommonStatistic.RowPosition + 2), "InvalidData"); continue; } } nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.Amount); if (fieldMapping.ContainsKey(nameField)) { string amount = SQLDataHelper.GetString(csv[fieldMapping[nameField]]); if (string.IsNullOrEmpty(amount)) amount = "0"; int tmp; if (int.TryParse(amount, out tmp)) { productInStrings.Add(ProductFields.Fields.Amount, amount); } else { Log(string.Format(Resource.Admin_ImportCsv_MustBeNumber, ProductFields.GetDisplayNameByEnum(ProductFields.Fields.Amount), CommonStatistic.RowPosition + 2), "InvalidData"); continue; } } nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.Unit); if (fieldMapping.ContainsKey(nameField)) { productInStrings.Add(ProductFields.Fields.Unit, SQLDataHelper.GetString(csv[fieldMapping[nameField]])); } nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.ParamSynonym); if (fieldMapping.ContainsKey(nameField)) { string rewurl = SQLDataHelper.GetString(csv[fieldMapping[nameField]]); productInStrings.Add(ProductFields.Fields.ParamSynonym, rewurl); } nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.Title); if (fieldMapping.ContainsKey(nameField)) { productInStrings.Add(ProductFields.Fields.Title, SQLDataHelper.GetString(csv[fieldMapping[nameField]])); } nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.MetaKeywords); if (fieldMapping.ContainsKey(nameField)) { productInStrings.Add(ProductFields.Fields.MetaKeywords, SQLDataHelper.GetString(csv[fieldMapping[nameField]])); } nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.MetaDescription); if (fieldMapping.ContainsKey(nameField)) { productInStrings.Add(ProductFields.Fields.MetaDescription, SQLDataHelper.GetString(csv[fieldMapping[nameField]])); } nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.Photos); if (fieldMapping.ContainsKey(nameField)) { productInStrings.Add(ProductFields.Fields.Photos, SQLDataHelper.GetString(csv[fieldMapping[nameField]])); } nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.Markers); if (fieldMapping.ContainsKey(nameField)) { productInStrings.Add(ProductFields.Fields.Markers, SQLDataHelper.GetString(csv[fieldMapping[nameField]])); } nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.Properties); if (fieldMapping.ContainsKey(nameField)) { productInStrings.Add(ProductFields.Fields.Properties, SQLDataHelper.GetString(csv[fieldMapping[nameField]])); } nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.Producer); if (fieldMapping.ContainsKey(nameField)) { productInStrings.Add(ProductFields.Fields.Producer, SQLDataHelper.GetString(csv[fieldMapping[nameField]])); } nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.Category).Trim('*'); if (fieldMapping.ContainsKey(nameField)) { var parentCategory = SQLDataHelper.GetString(csv[fieldMapping[nameField]]); if (!string.IsNullOrEmpty(parentCategory)) { productInStrings.Add(ProductFields.Fields.Category, parentCategory); } } ImportProduct.UpdateInsertProduct(productInStrings); } catch (Exception ex) { Log(ex.Message, "InvalidData"); } } CategoryService.RecalculateProductsCountManual(); } CommonStatistic.IsRun = false; FileHelpers.DeleteFile(fullPath); }
static async Task <int> Main(string[] args) { try { // // Parse options // Options = new AppOptions(); Options.Parse(args); if (Options.ShowList) { } if (Options.Exit) { return(-1); } if (string.IsNullOrEmpty(Options.FileName)) { throw new ApplicationException("Please use --file to specify which file to use"); } // // Init module client // if (Options.UseEdge) { Log.WriteLine($"{AppOptions.AppName} module starting."); await BlockTimer("Initializing Azure IoT Edge", async() => await InitEdge()); } cts = new CancellationTokenSource(); AssemblyLoadContext.Default.Unloading += (ctx) => cts.Cancel(); Console.CancelKeyPress += (sender, cpe) => cts.Cancel(); // // Load model // MLModel model = null; Console.WriteLine($"Loading model from: '{Options.ModelPath}', Exists: '{File.Exists(Options.ModelPath)}'"); await BlockTimer($"Loading modelfile '{Options.ModelPath}' on the {(Options.UseGpu ? "GPU" : "CPU")}", async() => { var d = Directory.GetCurrentDirectory(); var path = d + "\\" + Options.ModelPath; StorageFile modelFile = await AsAsync(StorageFile.GetFileFromPathAsync(path)); model = await MLModel.CreateFromStreamAsync(modelFile); }); do { // // Open file // var rows = new List <DataRow>(); try { using (var fs = new StreamReader(Options.FileName)) { // I just need this one line to load the records from the file in my List<CsvLine> rows = new CsvHelper.CsvReader(fs).GetRecords <DataRow>().ToList(); Console.WriteLine($"Loaded {rows.Count} row(s)"); } } catch (Exception e) { Console.WriteLine(e); } Console.WriteLine(rows); // // Main loop // foreach (var row in rows) { // // Evaluate model // var inputShape = new long[2] { 1, 4 }; var inputFeatures = new float[4] { row.Temperature, row.Pressure, row.Humidity, row.ExternalTemperature }; MLModelVariable result = null; var evalticks = await BlockTimer("Running the model", async() => { result = await model.EvaluateAsync(new MLModelVariable() { Variable = TensorFloat.CreateFromArray(inputShape, inputFeatures) }); }); // // Print results // var message = new MessageBody { result = result.Variable.GetAsVectorView().First() }; message.metrics.evaltimeinms = evalticks; var json = JsonConvert.SerializeObject(message); Log.WriteLineRaw($"Recognized {json}"); // // Send results to Edge // if (Options.UseEdge) { var eventMessage = new Message(Encoding.UTF8.GetBytes(json)); await ioTHubModuleClient.SendEventAsync("resultsOutput", eventMessage); // Let's not totally spam Edge :) await Task.Delay(500); } Console.WriteLine("Waiting 1 second..."); Thread.Sleep(1000); } }while (Options.RunForever && !cts.Token.IsCancellationRequested); } catch (Exception ex) { Console.WriteLine(ex); return(-1); } return(0); }
public ActionResult Create(CoordinatorSmsAndEmailModel model) { var trickleId = Guid.NewGuid(); Session.Add("CoordinatorSmsAndEmailModel", model); Session.Add("trickleId", trickleId.ToString()); var hpf = Request.Files[0]; if (hpf.ContentLength == 0) throw new ArgumentException("no content"); var csvFileContents = new CsvFileContents(); using (var csvReader = new CsvHelper.CsvReader(new StreamReader(hpf.InputStream), new CsvConfiguration { HasHeaderRecord = false })) { // TODO : Not reading first row properly while (csvReader.Read()) { csvFileContents.Rows.Add(csvReader.CurrentRecord); } } var fileContentsId = trickleId.ToString() + "_fileContents"; using (var session = Raven.GetStore().OpenSession()) { session.Store(csvFileContents, fileContentsId); session.SaveChanges(); } var dataColumnPicker = new DataColumnPicker {TrickleId = trickleId, FirstRowIsHeader = true}; if (!string.IsNullOrWhiteSpace(model.TemplateName)) { using (var session = Raven.GetStore().OpenSession("Configuration")) { var communicationTemplate = session.Load<CommunicationTemplate>(model.TemplateName); if (communicationTemplate.TemplateVariables != null) communicationTemplate.TemplateVariables.ForEach(t => dataColumnPicker.TemplateVariableColumns.Add(t.VariableName, null)); } } var dropDownList = csvFileContents.CreateSelectList(); ViewData.Add("selectListData", dropDownList); return View("CreateSmsAndEmailPickRows", dataColumnPicker); }
public async Task <(ImportStatus, string)> FromCsvAsync(StreamReader csvStream) { var notes = new List <string>(); using var csv = new CsvHelper.CsvReader(csvStream, CultureInfo.InvariantCulture); try { int recordCount = 0; int issues = 0; int districtsAdded = 0; int schoolsAdded = 0; var districts = await _schoolService.GetDistrictsAsync(); var schools = await _schoolService.GetForExportAsync(); var districtIndex = districts.ToDictionary(_ => _.Name, _ => _.Id); foreach (var record in csv.GetRecords <SchoolImportExport>()) { try { if (string.IsNullOrEmpty(record.District)) { throw new GraException($"School district is blank on record {recordCount + 2}"); } if (string.IsNullOrEmpty(record.Name)) { throw new GraException($"School name is blank on record {recordCount + 2}"); } if (record.District.Length > 255) { record.District = record.District.Substring(0, 255); string warning = $"<li>District too long, truncated: <strong>{record.District}</strong>.</li>"; if (!notes.Contains(warning)) { notes.Add(warning); } } if (record.Name.Length > 255) { record.Name = record.Name.Substring(0, 255); string warning = $"<li>Type too long, truncated: <strong>{record.Name}</strong>.</li>"; if (!notes.Contains(warning)) { notes.Add(warning); } } int districtId; if (districtIndex.Keys.Contains(record.District.Trim())) { districtId = districtIndex[record.District.Trim()]; } else { var district = await _schoolService.AddDistrict(new SchoolDistrict { Name = record.District.Trim() }); districtIndex.Add(record.District.Trim(), district.Id); districtId = district.Id; districtsAdded++; } var schoolExists = schools.Any(_ => _.District == record.District.Trim() && _.Name == record.Name.Trim()); if (!schoolExists) { await _schoolService .AddSchool(record.Name.Trim(), districtId); schoolsAdded++; } recordCount++; } catch (Exception rex) { issues++; if (rex.InnerException != null) { _logger.LogError($"School import error: {rex.InnerException.Message}"); notes.Add($"<li>Problem inserting record {recordCount + 2}: {rex.InnerException.Message}</li>"); } else { _logger.LogError($"School import error: {rex.Message}"); notes.Add($"<li>Problem inserting record {recordCount + 2}: {rex.Message}</li>"); } } } var returnMessage = new StringBuilder($"<p><strong>Read {recordCount} records (added {districtsAdded} districts, {schoolsAdded} schools) and skipped {issues} rows due to issues.</strong></p>"); foreach (var note in notes) { returnMessage.AppendLine(note); } if (issues > 0) { return(ImportStatus.Warning, returnMessage.ToString()); } if (notes.Count > 0) { return(ImportStatus.Info, returnMessage.ToString()); } return(ImportStatus.Success, returnMessage.ToString()); } catch (Exception ex) { string error = $"CSV parsing error: {ex.Message}"; _logger.LogError(error); return(ImportStatus.Danger, error); } }
/// <summary> /// Delete customer from file. File must contain the CustomerCid from CSP /// </summary> /// <param name="filename">Path of csv file contain customers to delete</param> /// <param name="defaultDomain">default domain of the reseller</param> /// <param name="appId">appid that is registered for this application in Azure Active Directory (AAD)</param> /// <param name="key">Key for this application in Azure Active Directory</param> /// <param name="resellerMicrosoftId">Microsoft Id of the reseller</param> internal static void DeleteCustomers(string filename, string defaultDomain, string appId, string key, string resellerMicrosoftId) { GetTokens(defaultDomain, appId, key, resellerMicrosoftId); using (var reader = new CsvHelper.CsvReader(new StreamReader(filename))) { while (reader.Read()) { string displayName = reader.GetField<string>("displayName"); string cspCustomerCid = reader.GetField<string>("cspCustomerCid"); if (cspCustomerCid.Trim().Length == 0) Console.WriteLine("Unable to delete {0} because CustomerCid is missing.", displayName); else { Console.WriteLine("Deleting {0}", displayName); Customer.DeleteCustomer(resellerCid, cspCustomerCid, saAuthorizationToken.AccessToken); } } } }