public async Task<IEnumerable<ChimetDataRecord>> DownloadData(DateTime day)
{
WebRequest dateRequest = HttpWebRequest.Create(string.Format("http://{1}/archive/{0:yyyy}/{0:MMMM}/CSV/{2}{0:ddMMMyyyy}.csv", day, this.Station.Address, this.Station.ShortName));
WebResponse data = await dateRequest.GetResponseAsync().ConfigureAwait(false);
CsvHelper.Configuration.CsvConfiguration config = new CsvHelper.Configuration.CsvConfiguration();
config.RegisterClassMap<ChimetDataRecordMap>();
using (var reader = new CsvHelper.CsvReader(new System.IO.StreamReader(data.GetResponseStream()), config))
{
return reader.GetRecords<ChimetDataRecord>().ToList().AsReadOnly();
}
}
public CsvReader(string path, CsvConfiguration config, Domain domain, bool isTestReader)
{
Path = path;
_reader = new CsvHelper.CsvReader(File.OpenText(this.Path), config);
Domain = domain;
IsTestReader = isTestReader;
}
/// <summary>
/// CSVを読み込み、薬品情報のリストを返す
/// </summary>
/// <param name="path">CSVファイルの絶対パス</param>
/// <returns>薬品情報のリスト</returns>
static List<Yakuhin> ReadCsv(string path)
{
List<Yakuhin> list = new List<Yakuhin>();
var enc = new System.Text.UTF8Encoding(false);
using (var reader = new System.IO.StreamReader(path, enc))
{
var csv = new CsvHelper.CsvReader(reader);
while (csv.Read())
{
string drugCode = csv.GetField<string>(0);
string clsCode = csv.GetField<string>(1);
string clsName = csv.GetField<string>(2);
string drugName = csv.GetField<string>(3);
string company = csv.GetField<string>(4);
var yakuhin = new Yakuhin
{
DrugCode = drugCode,
ClassificationCode = clsCode,
ClassificationName = clsName,
DrugName = drugName,
Company = company
};
list.Add(yakuhin);
}
}
return list;
}
public IEnumerable<UnsettledBetItem> GetUnsettledBets()
{
using (var fileReader = File.OpenText(GetFilePath(UnsettledBetsFileName)))
using (var csvReader = new CsvHelper.CsvReader(fileReader))
{
return csvReader.GetRecords<UnsettledBetItem>().ToList();
}
}
public void ReadGraph(TextReader reader)
{
var g = new BidirectionalGraph<MyType, IEdge<MyType>>();
try {
var csv = new CsvHelper.CsvReader(reader);
while (csv.Read()) {
if (!csv.CurrentRecord.Any()) {
continue;
}
var node = csv.CurrentRecord.First();
var edges = csv.CurrentRecord.Skip(1)
.Where(x => !string.IsNullOrEmpty(x))
.Select(x => x.Trim())
.Where(x => !string.IsNullOrEmpty(x));
foreach (var edge in edges) {
g.AddVerticesAndEdge(new Edge<MyType>(node, edge));
}
}
var dict = new Dictionary<int, HashSet<MyType>>();
HashSet<MyType> set;
foreach (var v in g.Vertices) {
var edgeCount = g.InEdges(v).Count();
if (!dict.TryGetValue(edgeCount, out set)) {
set = new HashSet<MyType>();
dict.Add(edgeCount, set);
}
set.Add(v);
}
Graph = g;
Summary = string.Join(Environment.NewLine,
dict
.OrderBy(kvp => kvp.Key)
.Select(kvp => kvp.Key + ": " + string.Join(", ", kvp.Value)))
+ Environment.NewLine
+ Environment.NewLine
+ string.Join(Environment.NewLine,
g.Vertices
.Where(v => g.OutEdges(v).Count() != 3)
.Select(v => v + ": " + g.OutEdges(v).Count()));
//Summary = string.Join(
// Environment.NewLine,
// graph.Vertices
// .OrderBy(x => graph.InEdges(x).Count())
// .Select(v => v + ": " + graph.InEdges(v).Count()));
} catch (Exception e) {
Summary = "Failed to read:" + Environment.NewLine + e;
}
}
public override dynamic Load(Slice slice)
{
var ret = new List<dynamic>();
if (!Blob.Exists())
return ret;
using (var reader = new CsvHelper.CsvReader(new StreamReader(Blob.OpenRead(), Configuration.Encoding)))
{
var records = reader.GetRecords<dynamic>();
foreach (IDictionary<string, object> record in records)
ret.Add(Helpers.DictionaryToObject(record, Structure));
}
return ret;
}
private IEnumerable<Employee> GetEmployeeAdjustments(Stream content)
{
using (var reader = new CsvHelper.CsvReader(new StreamReader(content))) {
return reader.GetRecords<CsvAdjustment>().Where(x => x.Year > 0)
.Select(x => new Employee {
Id = 0,
Ssn = x.Ssn,
DeductionSchema = new DeductionSchema {
Adjustment = x.Amount,
Year = x.Year
}
})
.ToList();
}
}
public ActionResult Upload(HttpPostedFileBase file)
{
//http://joshclose.github.io/CsvHelper/
Stream stream = file.InputStream;
StreamReader streamReader = new StreamReader(stream);
TextReader textReader = (TextReader)streamReader;
var csv = new CsvHelper.CsvReader(textReader);
List<ImportDTO> listOfDTOs = new List<ImportDTO>();
while (csv.Read())
{
ImportDTO record = csv.GetRecord<ImportDTO>();
listOfDTOs.Add(record);
}
//Map DTO to models
foreach (ImportDTO item in listOfDTOs)
{
//Get countries
List<string> countries = item.Countries.Split(new char[] { '|' }).ToList();
Collection<Country> countryCollection = new Collection<Country>();
foreach (string countryName in countries)
{
countryCollection.Add(new Country { Name = countryName });
}
//Get start and end times
DateTime start = DateTime.ParseExact(item.StartDate + " " + item.StartTime, "dd/MM/yyyy HH:mm",
System.Globalization.CultureInfo.InvariantCulture);
DateTime end = DateTime.ParseExact(item.EndDate + " " + item.EndTime, "dd/MM/yyyy HH:mm",
System.Globalization.CultureInfo.InvariantCulture);
Film newFilm = new Film();
newFilm.Country = countryCollection;
TimeSlot slot = new TimeSlot();
slot.StartTime = start;
slot.EndTime = end;
}
return View("Index");
}
public List<PublicationTarget> Read()
{
using (var tr = new StreamReader(_filename))
{
var csv = new CsvHelper.CsvReader(tr);
csv.Configuration.Delimiter = ",";
csv.Configuration.IsHeaderCaseSensitive = false;
csv.Configuration.SkipEmptyRecords = true;
csv.Configuration.IgnoreBlankLines = true;
csv.Configuration.RegisterClassMap<PublicationTargetCsvMapping>();
//csv.Configuration.TrimHeaders = true;
var records = csv.GetRecords<PublicationTarget>().ToList();
return records;
}
}
public Fact[] Read(string message)
{
var csvConfig = new CsvConfiguration
{
Delimiter = "|",
Encoding = Encoding.UTF8,
HasHeaderRecord = true,
};
csvConfig.RegisterClassMap<FactMap>();
csvConfig.CultureInfo = CultureInfo.InvariantCulture;
using (var reader = new StringReader(message))
using (var parser = new CsvHelper.CsvReader(reader, csvConfig))
{
return parser.GetRecords<Fact>().ToArray();
}
}
public static void Train()
{
using (System.IO.TextReader fs = System.IO.File.OpenText("dfsafias"))
{
using (CsvHelper.CsvReader reader = new CsvHelper.CsvReader(fs))
{
//reader.GetField
}
}
OpenCvSharp.ML.RTrees forest = OpenCvSharp.ML.RTrees.Create();
// forest.Predict();
// forest.
}
private bool LoadData()
{
var listLog = new List<LogEntry>();
try
{
if (File.Exists(Debug.ErrFile[CurrentView]))
{
using (var csv = new CsvHelper.CsvReader(new StreamReader(Debug.ErrFile[CurrentView], Encoding.UTF8, true)))
{
csv.Configuration.Delimiter = Debug.CharSeparate;
csv.Configuration.HasHeaderRecord = false;
while (csv.Read())
{
string[] r = csv.CurrentRecord;
if (r.Length >= 4)
{
var item = new LogEntry
{
TimeStamp = r[0],
Level = r[1],
Message = r[2],
ErrorMessage = r[3]
};
listLog.Add(item);
}
else
{
lblErr.Text = @"Log file wrong format";
return false;
}
}
}
listLog.Reverse();
grid.DataSource = listLog;
grid.DataBind();
return true;
}
return false;
}
catch (Exception ex)
{
lblErr.Text = ex.Message;
return false;
}
}
public static List<Users.User> getStudents()
{
CsvHelper.CsvReader reader;
List<Users.User> userList = new List<Users.User>();
reader = new CsvHelper.CsvReader(File.OpenText("student_accounts.csv"));
while (reader.Read())
{
string TNumber = reader.GetField<string>("T_NUMBER");
string firstName = reader.GetField<string>("FIRSTNAME");
string lastName = reader.GetField<string>("LAST_NAME");
string userName = reader.GetField<string>("USERNAME");
string Email = reader.GetField<string>("EMAILADDRESS");
long secretKey = reader.GetField<long>("CHECK-N ID");
userList.Add(new Users.StudentWorker(firstName, lastName, TNumber, secretKey, userName, Email, 110));
}
return userList;
}
public static IFileReader CreateFromFileName(FileInfoBase fileInfoBase)
{
if (IsCsvFile(fileInfoBase.FullName))
{
var reader = new CsvHelper.CsvReader(new StreamReader(fileInfoBase.OpenRead()), new CsvConfiguration
{
HasHeaderRecord = true,
Delimiter = ";"
});
return new CsvReader(reader);
}
if (IsExcelFile(fileInfoBase.FullName))
{
var excelReader = ExcelReaderFactory.CreateOpenXmlReader(fileInfoBase.Open(FileMode.Open, FileAccess.Read));
excelReader.IsFirstRowAsColumnNames = true;
return new ExcelReader(excelReader);
}
throw new InvalidOperationException();
}
private void Load <T>(IDbConnection conn, IDbTransaction trans)
{
var dataFileName = Path.ChangeExtension(typeof(T).Name, ".tsv");
var tr = File.OpenText(Path.Combine(_dataDirectory, dataFileName));
var config = new Configuration
{
HasHeaderRecord = true,
Delimiter = "\t"
};
foreach (var classMap in GetClassMapTypes())
{
config.RegisterClassMap(classMap);
}
var csv = new CsvHelper.CsvReader(tr, config);
var records = csv.GetRecords <T>().ToList();
Console.WriteLine($" {dataFileName} ...");
conn.Insert(records, trans);
}
public async Task <List <VaccineHospital> > GetVaccineHospitalsAsync()
{
var url = @"http://www.cdc.gov.tw/downloadfile.aspx?fid=2AE494CD4C1EAFBD";
var str = await _netService.GetStringAsync(new Uri(url));
if (string.IsNullOrWhiteSpace(str))
{
return(new List <VaccineHospital>());
}
List <VaccineHospital> items;
using (var reader = new StringReader(str))
using (var csv = new CsvHelper.CsvReader(reader))
{
csv.Configuration.HasHeaderRecord = true;
csv.Configuration.RegisterClassMap <VaccineHospitalMap>();
items = csv.GetRecords <VaccineHospital>()
.OrderBy(i => i.縣市)
.ThenBy(i => i.鎮市區)
.ThenBy(i => i.機構類別)
.ThenBy(i => i.合約醫療院所名稱)
.ToList();
}
foreach (var item in items)
{
item.Id = item.GetHashCode().ToString();
item.縣市 = item.縣市.Replace(" ", "");
item.鎮市區 = item.鎮市區.Replace(" ", "");
item.合約醫療院所名稱 = item.合約醫療院所名稱.Replace("\n", "");
item.公費疫苗 = item.公費疫苗.Replace("\n", "").Replace("、", Environment.NewLine);
item.自費疫苗 = item.自費疫苗.Replace("\n", "").Replace("、", Environment.NewLine);
}
return(items);
}
/// <summary>
/// Delete customer from file. File must contain the CustomerCid from CSP
/// </summary>
/// <param name="filename">Path of csv file contain customers to delete</param>
/// <param name="defaultDomain">default domain of the reseller</param>
/// <param name="appId">appid that is registered for this application in Azure Active Directory (AAD)</param>
/// <param name="key">Key for this application in Azure Active Directory</param>
/// <param name="resellerMicrosoftId">Microsoft Id of the reseller</param>
internal static void DeleteCustomers(string filename, string defaultDomain, string appId, string key, string resellerMicrosoftId)
{
GetTokens(defaultDomain, appId, key, resellerMicrosoftId);
using (var reader = new CsvHelper.CsvReader(new StreamReader(filename)))
{
while (reader.Read())
{
string displayName = reader.GetField <string>("displayName");
string cspCustomerCid = reader.GetField <string>("cspCustomerCid");
if (cspCustomerCid.Trim().Length == 0)
{
Console.WriteLine("Unable to delete {0} because CustomerCid is missing.", displayName);
}
else
{
Console.WriteLine("Deleting {0}", displayName);
Customer.DeleteCustomer(resellerCid, cspCustomerCid, saAuthorizationToken.AccessToken);
}
}
}
}
void Form1_DragDrop(object sender, DragEventArgs e)
{
listBox1.ValueMember = "";
string[] files = (string[])e.Data.GetData(DataFormats.FileDrop);
if (files.Length > 1 || Path.GetExtension(files[0]).ToLower() != ".csv")
{
MessageBox.Show("Drag 1 file, needs to be csv");
}
else
{
StreamReader sr = new StreamReader(files[0], Encoding.Default);
CsvHelper.CsvReader rdr = new CsvHelper.CsvReader(sr);
rdr.Configuration.RegisterClassMap <MyClassMap>();
rdr.Configuration.Delimiter = ";";
var test = rdr.GetRecords <Student>().ToList();
foreach (var item in test)
{
listBox1.Items.Add(item);
}
}
}
public void ImportCharityData()
{
var dbConnectionFactory = appHost.Container.Resolve<IDbConnectionFactory>();
using (var db = dbConnectionFactory.OpenDbConnection())
{
db.DropAndCreateTable<Charity>();
}
using (var db = dbConnectionFactory.OpenDbConnection())
{
string currentPath = AppDomain.CurrentDomain.BaseDirectory;
var filestream = File.OpenRead(Path.Combine(currentPath, ConfigurationManager.AppSettings["CharityPath"]));
TextReader textreader = new StreamReader(filestream);
var reader = new CsvHelper.CsvReader(textreader);
var lol = reader.GetRecords<dynamic>();
var charityJson = lol.ToJson();
var data = JsonSerializer.DeserializeFromString<List<Charity>>(charityJson);
db.InsertAll(data);
}
}
public override void ImportDate()
{
var csv = new CsvHelper.CsvReader(System.IO.File.OpenText(DownloadFullPath));
while (csv.Read())
{
string CsvStockNum = "";
try
{
CsvStockNum = csv.GetField <string>(0);
CsvStockNum = CsvStockNum.Replace(" ", "");
if (CsvStockNum.Length != 4)
{
continue;
}
Int32.Parse(CsvStockNum);
}
catch
{
continue;
}
double CsvPrice = 0;
try
{
CsvPrice = csv.GetField <double>(10);
}
catch { }
StockPricesList.Add(new StockPriceDataModel
{
StockNum = CsvStockNum,
StockPrice = Math.Round(CsvPrice, 2)
});
}
UploadData();
UpdateLocalDB();
}
public void HandleBackedOutWork(Backout backout)
{
var file = Path.Combine(CSVBasePath, "bookings.csv");
File.WriteAllText(Path.Combine(CSVBasePath, "latest-backout-id"), backout.BackoutId.ToString());
var orders = new List <UnscheduledCsvRow>();
if (File.Exists(file))
{
using (var f = File.OpenRead(file))
using (var csv = new CsvHelper.CsvReader(new StreamReader(f), CultureInfo.InvariantCulture))
{
orders.AddRange(csv.GetRecords <UnscheduledCsvRow>());
}
}
foreach (var p in backout.Parts)
{
orders.Add(new UnscheduledCsvRow()
{
Id = "Reschedule-" + p.Part + "-" + DateTime.UtcNow.ToString("yyy-MM-ddTHH-mm-ssZ"),
DueDate = DateTime.Today,
Priority = 100,
Part = p.Part,
Quantity = p.Quantity
});
}
using (var s = new StreamWriter(file))
using (var csv = new CsvHelper.CsvWriter(s, CultureInfo.InvariantCulture))
{
csv.Configuration.TypeConverterOptionsCache.GetOptions <DateTime>().Formats
= new string[] { "yyyy-MM-dd" };
csv.WriteRecords(orders);
}
}
static void Main(string[] args)
{
string dir = Path.Combine(Directory.GetParent(typeof(Program).Assembly.Location).ToString(), "ProductUploadTemplate.csv");
FileStream fileStream = new FileStream(dir, FileMode.Open);
MemoryStream ms = new MemoryStream();
Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
Encoding encoding = CodePagesEncodingProvider.Instance.GetEncoding("big5");
fileStream.CopyTo(ms);
string content = encoding.GetString(ms.ToArray());
StringReader reader = new StringReader(content);
CsvHelper.CsvReader csv = new CsvHelper.CsvReader(reader, CultureInfo.InvariantCulture);
csv.Read();
csv.ReadHeader();
while (csv.Read())
{
string value = csv.GetField <string>("DESCRIPTION");
Console.WriteLine(value);
}
Console.ReadKey();
}
private void AssertCsvSerialization <T>(T value, string expected, ITypeConverter converter)
{
string actual;
using (var writer = new StringWriter())
{
var serializer = new CsvHelper.CsvWriter(writer, CultureInfo.InvariantCulture);
serializer.Context.TypeConverterCache.AddConverter <T>(converter);
serializer.WriteField(value);
serializer.Flush();
actual = writer.ToString();
Assert.Equal(expected, actual);
}
var reader = new CsvHelper.CsvReader(new StringReader(actual), CultureInfo.InvariantCulture);
reader.Context.TypeConverterCache.AddConverter <T>(converter);
reader.Read();
var deserializedValue = reader.GetField <T>(0);
Assert.Equal(value, deserializedValue);
}
public static Dictionary <Guid, User> GetUsers(string path)
{
Dictionary <Guid, User> users = new Dictionary <Guid, User>();
using (var reader = File.OpenText(path + "users.csv"))
{
using (var csv = new CsvHelper.CsvReader(reader))
{
while (csv.Read())
{
User usr = new User()
{
Id = csv.GetField <Guid>(0),
Name = csv.GetField(1),
Email = csv.GetField(2)
};
users.Add(usr.Id, usr);
}
}
}
return(users);
}
/// <summary>
/// parse the file, and get information
/// <remark>skip the headers</remark>
/// </summary>
/// <param name="file"></param>
/// <returns>courseForFile</returns>
static List <Course> parseFile(string file)
{
var coursesForFile = new List <Course>();
using (StreamReader fileStream = File.OpenText(file))
{
// Use CsvHelper library to read the file.
var csvReader = new CsvHelper.CsvReader(fileStream);
// configure csv reader
csvReader.Configuration.HasHeaderRecord = false;
csvReader.Configuration.RegisterClassMap <CourseClassMap>();
skipHeaders(csvReader);
csvReader.Read(); // read first header and skip it.
while (fileHasMoreRecords)
{
coursesForFile.AddRange(parseRecordsForCourse(csvReader));
}
}
return(coursesForFile);
}
public void ReadAllHistoricDates()
{
try
{
using (var reader = new StreamReader(HistoricalFileName))
using (var csv = new CsvHelper.CsvReader(reader, CultureInfo.InvariantCulture))
{
var records = new List <CurencyDataViewModel>();
csv.Read();
csv.ReadHeader();
List <string> allDates = new List <string>();
while (csv.Read())
{
var dateAsString = csv.GetField(0);
allDates.Add(dateAsString);
}
AllHistoricDates = allDates;
}
}
catch (Exception ex) {
AllHistoricDates = null;
}
}
private void btnAddPath_Click(object sender, EventArgs e)
{
StreamReader sr;
CsvHelper.CsvReader csvread;
sr = new StreamReader(txtFilePath.Text);
csvread = new CsvHelper.CsvReader(sr);
KlijentIzCsv kicsv = new KlijentIzCsv();
csvread.Configuration.HasHeaderRecord = false;
int i = 1;
while (csvread.Read())
{
if (i == 1)
{
kicsv.klijent = csvread[4].Trim();
kicsv.nazivMonitoringa = csvread[5].Trim();
}
Console.WriteLine(csvread[0]);
if (csvread[0].Trim() != "")
{
kicsv.nicsvList.Add(new NadgledanaIzCsv(csvread[0].Trim(), csvread[1].Trim(), csvread[2].Trim(), csvread[3].Trim()));
}
i++;
}
DBCommOsnovnePostavke.connection = new SqlConnection(DBBlokadeData.connectionString);
DBCommOsnovnePostavke.connection.Open();
DBCommOsnovnePostavke.addKlijentIzCsv(kicsv);
DBCommOsnovnePostavke.connection.Close();
txtMaticniBroj.Clear();
txtNaziv.Clear();
txtFilePath.Clear();
ucitajKlijente(1);
}
public static Dictionary <Guid, Product> GetProducts(string path)
{
Dictionary <Guid, Product> products = new Dictionary <Guid, Product>();
using (var reader = File.OpenText(path + "products.csv"))
{
using (var csv = new CsvHelper.CsvReader(reader))
{
while (csv.Read())
{
Product p = new Product()
{
Id = csv.GetField <Guid>(0),
Name = csv.GetField(1),
Description = csv.GetField(2),
ImageUrl = csv.GetField(3),
Brand = csv.GetField <Guid>(4),
Category = csv.GetField(5),
Created = DateTime.Now
};
Stock s = new Stock()
{
Quantity = csv.GetField <int>(6),
Cost = csv.GetField <double>(7),
Price = csv.GetField <double>(8)
};
p.Stock = s;
products.Add(p.Id, p);
}
}
}
return(products);
}
static List <Address> ReadCSV(string path)
{
List <Address> list = new List <Address>();
var encode = new System.Text.UTF8Encoding(false);
using (var reader = new System.IO.StreamReader(path, encode))
{
var csv = new CsvHelper.CsvReader(reader);
bool isSkip = true;
while (csv.Read())
{
if (isSkip)
{
isSkip = false;
continue;
}
//氏名,氏名（カタカナ）,電話番号,メールアドレス,郵便番号,住所,,,,
Address address = new Address()
{
Name = csv.GetField <string>(0),
Kana = csv.GetField <string>(1),
Telephone = csv.GetField <string>(2),
Mail = csv.GetField <string>(3),
ZipCode = csv.GetField <string>(4).Replace("-", ""),
Prefecture = csv.GetField <string>(5),
StreetAddress = $"{csv.GetField<string>(6)}{csv.GetField<string>(7)}{csv.GetField<string>(8)}{csv.GetField<string>(9)}"
};
list.Add(address);
}
}
return(list);
}
static void Main(string[] args)
{
var dir = new DirectoryInfo(Directory.GetCurrentDirectory());
var file = new FileInfo($"{dir.FullName}\\color-keywords.csv");
using var fs = new StreamReader(file.FullName);
var colors = new CsvHelper
.CsvReader(fs, CultureInfo.InvariantCulture)
.GetRecords <CsvColor>()
.Select(c => new ColorData(c))
.OrderBy(c => c.Color.GetHue())
.ThenBy(c => c.Color.GetSaturation())
.ThenBy(c => c.Color.GetBrightness())
.ToList();
var doc = new StringBuilder();
doc.AppendLine("<style>");
doc.AppendLine("table img {");
doc.AppendLine("\tbox-shadow: 1px 1px 8px rgba(0,0,0,.67);");
doc.AppendLine("\tborder: 1px solid rgba(0,0,0,.67);");
doc.AppendLine("}");
doc.AppendLine("</style>");
doc.AppendLine();
doc.AppendLine("Keyword | RGB Hex Value | Color");
doc.AppendLine("--------|---------------|------");
foreach (var color in colors)
{
doc.AppendLine($"{color.Name} | {color.Hex} | {color.Preview}");
}
using var sw = new StreamWriter($"{dir.FullName}\\color-keywords.md");
sw.Write(doc.ToString());
}
public static Dictionary <Guid, Brand> GetBrands(string path)
{
Dictionary <Guid, Brand> brands = new Dictionary <Guid, Brand>();
using (var reader = File.OpenText(path + "brands.csv"))
{
using (var csv = new CsvHelper.CsvReader(reader))
{
while (csv.Read())
{
Brand b = new Brand()
{
Id = csv.GetField <Guid>(0),
Name = csv.GetField(1),
Description = csv.GetField(2)
};
brands.Add(b.Id, b);
}
}
}
return(brands);
}
static void Main(string[] args)
{
CultureInfo.DefaultThreadCurrentCulture = new CultureInfo("en-US");
CultureInfo.DefaultThreadCurrentUICulture = new CultureInfo("en-US");
List <Wine> wines;
using (var stream = new StreamReader(".\\winequality-red.csv"))
using (var helper = new CsvHelper.CsvReader(stream))
{
helper.Configuration.RegisterClassMap <WineMap>();
wines = helper.GetRecords <Wine>().ToList();
}
var testingSet = wines.Take(400).ToList();
var trainingSet = wines.Skip(400).ToList();
double[] result = LinearRegression(testingSet, trainingSet);
decimal good = result.Zip(testingSet, (x, y) => Math.Abs(x - y.Quality) <= 0.2).Where(x => x).Count();
decimal all = 400;
Debug.WriteLine($"Correctness: {good / all}");
ScatterplotBox.Show("Expected results", result.Select(x => (double)x).ToArray(), testingSet.Select(x => (double)x.Quality).ToArray()).Hold();
}
public async Task <List <PersonImport> > ReadAllAsync()
{
var people = new List <PersonImport>();
if (string.IsNullOrWhiteSpace(pathToCsvFile) || !File.Exists(pathToCsvFile))
{
return(people);
}
using (var sr = new StreamReader(pathToCsvFile))
{
using (var csv = new CsvHelper.CsvReader(sr, System.Globalization.CultureInfo.InvariantCulture))
{
csv.Configuration.RegisterClassMap <CsvTreeReaderMap>();
csv.Configuration.BadDataFound = context =>
{
logger?.LogError((int)LogEventIds.BadDataInReaderImport, $"Bad data found on row '{context.RawRow}'.");
};
while (await csv.ReadAsync())
{
try
{
var person = csv.GetRecord <PersonImport>();
people.Add(person);
}
catch (Exception ex)
{
logger?.LogError((int)LogEventIds.ErrorReadingCsvInReader, ex, $"Error occurred reading CSV.");
}
}
}
}
return(people);
}
static ParseRAKBankStatementResult ParseRAKBankStatement(string strFilePath)
{
ParseRAKBankStatementResult result = new ParseRAKBankStatementResult();
var strFileContent = System.IO.File.ReadAllText(strFilePath);
var strFileContentLines = strFileContent.Split(new char[] { '\n' });
result.strAccountName = strFileContentLines[3].Split(new char[] { ',' })[2];
result.strAccountNumber = strFileContentLines[4].Split(new char[] { '.' })[1].Split(new char[] { ']' })[0];
result.strBranchName = strFileContentLines[6].Split(new char[] { '[' })[1].Split(new char[] { ']' })[0];
result.d8From = Convert.ToDateTime(DateTime.ParseExact(strFileContentLines[7].Split(new char[] { ',' })[2], "dd/MM/yyyy", System.Globalization.CultureInfo.InvariantCulture));
result.d8To = Convert.ToDateTime(DateTime.ParseExact(strFileContentLines[8].Split(new char[] { ',' })[2], "dd/MM/yyyy", System.Globalization.CultureInfo.InvariantCulture));
string strCSVContent = string.Empty;
for (int i = 10; i < strFileContentLines.Length - 2; i++)
{
if (i == 10)
{
var Headers = strFileContentLines[i].Split(new char[] { ',' });
strCSVContent += $"EmptyHeader1,{Headers[1]},EmptyHeader2,{Headers[3]},{Headers[4]},{Headers[5]},{Headers[6]},{Headers[7]}{Environment.NewLine}";
}
else
{
strCSVContent += strFileContentLines[i] + Environment.NewLine;
}
}
var textReader = new System.IO.StringReader(strCSVContent);
var csvr = new CsvHelper.CsvReader(textReader);
csvr.Configuration.PrepareHeaderForMatch = (string header, int index) => header?.Trim();
csvr.Configuration.PrepareHeaderForMatch = (string header, int index) => header.Replace(" ", string.Empty);
result.CSVRecords = csvr.GetRecords <RAKBankCSVRecord>();
return(result);
}
//[Benchmark]
public List <HalfNumbers> CsvHelperReadAll()
{
var reader = new CsvHelper.CsvReader(new StringReader(Data), new Configuration()
{
HeaderValidated = null
});
while (reader.Read())
{
var record = new HalfNumbers();
record.A = reader.GetField <string>(0);
record.C = reader.GetField <string>(2);
record.E = reader.GetField <string>(4);
record.G = reader.GetField <string>(6);
record.I = reader.GetField <string>(8);
record.B = reader.GetField <int>(1);
record.D = reader.GetField <int>(3);
record.F = reader.GetField <int>(5);
record.H = reader.GetField <int>(7);
record.J = reader.GetField <int>(9);
}
return(reader.GetRecords <HalfNumbers>().ToList());
}
public async Task Import(string name, Stream csvStream)
{
await _mongoContext.ExecuteActionAsync <HousePrice>("Transactions", async (collection) =>
{
using (var streamReader = new StreamReader(csvStream))
{
using (var csvReader = new CsvHelper.CsvReader(streamReader))
{
csvReader.Configuration.HasHeaderRecord = false;
csvReader.Configuration.RegisterClassMap <HousePriceMap>();
var batch = new List <HousePrice>();
while (csvReader.Read())
{
try
{
var record = csvReader.GetRecord <HousePrice>();
record.Postcode = record.Postcode.Replace(" ", String.Empty);
}
catch (Exception e)
{
Log.Error(e.Message);
throw;
}
}
if (batch.Count % 1000 == 0)
{
Console.WriteLine($"{name}: {batch.Count}");
}
await collection.InsertManyAsync(batch);
}
}
});
}
private Task <IEnumerable <CaseStatus> > ReadInternal(string filePath)
{
var result = new List <CaseStatus>();
CultureInfo provider = CultureInfo.InvariantCulture;
using (var reader = new StreamReader(filePath))
using (var csv = new CsvHelper.CsvReader(reader, CultureInfo.InvariantCulture))
{
var dataRead = csv.GetRecords <dynamic>().ToList();
foreach (var item in dataRead)
{
if (string.IsNullOrWhiteSpace(item.Date))
{
continue;
}
var valueDictionary = new RouteValueDictionary(item);
result.AddRange(Enum.GetNames(typeof(District)).Select(x => new CaseStatus
{
District = (District)Enum.Parse(typeof(District), x),
Date = System.DateTime.ParseExact(item.Date, "dd-MM-yyyy", provider),
Count = int.TryParse((string)valueDictionary["TotalSamplesTested"], out var value) ? value : 0
}));
private async void ImportPathfinderSpellsCSV(IStorageFile file)
{
using (var reader = new StreamReader(await file.OpenStreamForReadAsync()))
{
var csvHelper = new CsvHelper.CsvReader(reader);
var spells = csvHelper.GetRecords<PathfinderSpellsCSV>();
foreach(var spell in spells)
{
//spell.
}
}
}
public Dictionary <string, Booking> LoadUnscheduledBookings(int?lookaheadDays = null)
{
var bookingMap = new Dictionary <string, Booking>();
var pth = Path.Combine(CSVBasePath, "bookings.csv");
if (!File.Exists(pth))
{
CreateEmptyBookingFile(pth);
return(bookingMap);
}
using (var f = File.OpenRead(pth))
using (var sreader = new StreamReader(f))
using (var csv = new CsvHelper.CsvReader(sreader, CultureInfo.InvariantCulture))
{
var orderCntr = 0;
foreach (var row in csv.GetRecords <UnscheduledCsvRow>())
{
var bookingId = row.Id;
if (string.IsNullOrEmpty(bookingId))
{
bookingId = "B" + GetUtcNow().ToString("yyyy-MM-dd-HH-mm-ss") + "-" + orderCntr.ToString();
orderCntr += 1;
}
Booking book;
if (bookingMap.ContainsKey(bookingId))
{
book = bookingMap[bookingId];
}
else
{
book = new Booking
{
BookingId = bookingId,
Priority = row.Priority,
DueDate = row.DueDate,
Parts = new List <BookingDemand>(),
ScheduleId = null
};
bookingMap.Add(bookingId, book);
}
book.Parts.Add(new BookingDemand
{
BookingId = bookingId,
Part = row.Part,
Quantity = row.Quantity,
});
}
}
DateTime?endDate = null;
if (lookaheadDays.HasValue && lookaheadDays.Value > 0)
{
endDate = DateTime.Today.AddDays(lookaheadDays.Value);
}
foreach (var id in bookingMap.Keys.ToList())
{
var f = Path.Combine(CSVBasePath, Path.Combine(ScheduledBookingsPath, id + ".csv"));
if (File.Exists(f) || endDate.HasValue && bookingMap[id].DueDate > endDate)
{
bookingMap.Remove(id);
}
}
return(bookingMap);
}
public ActionResult UploadExerciseVideosAndExcel()
{
StringBuilder traceLog = null;
if (Login.IsSessionExpire(Convert.ToString(HttpContext.Session[ConstantHelper.constLoginUserName])) ||
Convert.ToString(HttpContext.Session[ConstantHelper.constUserType]) != ConstantHelper.constLoginadmin)
{
return(RedirectToAction(ConstantHelper.constLogin, ConstantHelper.constLogin));
}
DataTable dt = null;
try
{
traceLog = new StringBuilder();
traceLog.AppendLine("Start: UploadExerciseVideos() in Execise Controller");
var httpRequest = System.Web.HttpContext.Current.Request;
if (httpRequest.Files.Count > 0)
{
Dictionary <string, string> uploadedExeciseVideos = new Dictionary <string, string>();
List <UpdateExerciseSproutLink> FailToUploadedExeciseVideos = new List <UpdateExerciseSproutLink>();
for (int i = 0; i < httpRequest.Files.Count; i++)
{
var fileDoc = httpRequest.Files[i];
IExcelDataReader reader = null;
string sPath = "";
sPath = Server.MapPath("~/Temp/");
string newVideoId = Path.GetFileName(fileDoc.FileName);
string execiseName = Path.GetFileNameWithoutExtension(fileDoc.FileName);
if (fileDoc.FileName.EndsWith(".xls", StringComparison.OrdinalIgnoreCase))
{
using (Stream stream = fileDoc.InputStream)
{
reader = ExcelReaderFactory.CreateBinaryReader(stream);
reader.IsFirstRowAsColumnNames = true;
DataSet result = reader.AsDataSet();
reader.Close();
dt.Clear();
dt = result.Tables[0];
}
}
else if (fileDoc.FileName.EndsWith(".csv", StringComparison.OrdinalIgnoreCase))
{
dt = new DataTable();
dt.Locale = CultureInfo.InvariantCulture;
dt.Clear();
dt.Columns.Add("Name");
dt.Columns.Add("Index");
dt.Columns.Add("Team ID");
dt.Columns.Add("Trainer ID");
if (!System.IO.File.Exists(sPath + newVideoId))
{
fileDoc.SaveAs(sPath + newVideoId);
}
using (var fileReader = System.IO.File.OpenText(sPath + newVideoId))
using (var csvReader = new CsvHelper.CsvReader(fileReader))
{
csvReader.Configuration.HasHeaderRecord = true;
csvReader.Configuration.IgnoreHeaderWhiteSpace = false;
csvReader.Configuration.IsHeaderCaseSensitive = true;
csvReader.Configuration.RegisterClassMap <CSVExerciseMap>();
while (csvReader.Read())
{
var record = csvReader.GetRecord <CSVExerciseVM>();
if (record != null)
{
DataRow rowdata = dt.NewRow();
rowdata["Name"] = record.Name;
rowdata["Index"] = record.Index;
rowdata["Team ID"] = record.TeamID;
rowdata["Trainer ID"] = record.TrainerID;
dt.Rows.Add(rowdata);
}
}
}
}
else if (fileDoc.FileName.EndsWith(".xlsx", StringComparison.OrdinalIgnoreCase))
{
using (Stream stream = fileDoc.InputStream)
{
reader = ExcelReaderFactory.CreateOpenXmlReader(stream);
reader.IsFirstRowAsColumnNames = true;
DataSet result = reader.AsDataSet();
reader.Close();
dt = result.Tables[0];
}
}
else if (fileDoc.FileName.EndsWith(".mp4", StringComparison.OrdinalIgnoreCase))
{
uploadedExeciseVideos.Add(execiseName, sPath + newVideoId);
}
else
{
var FailToUpload = new UpdateExerciseSproutLink
{
ExerciseName = execiseName,
Index = string.Empty,
TeamID = string.Empty,
TrainerID = string.Empty,
FailedVideoName = fileDoc.FileName,
};
FailToUploadedExeciseVideos.Add(FailToUpload);
}
// Save the files in local directory
if (!System.IO.File.Exists(sPath + newVideoId))
{
fileDoc.SaveAs(sPath + newVideoId);
}
}
// varify execel execisename and upload on sprout server
foreach (DataRow row in dt.Rows)
{
string execiseName = string.Empty;
string index = string.Empty;
string teamID = string.Empty;
string trainerID = string.Empty;
foreach (DataColumn col in dt.Columns)
{
switch (col.ColumnName)
{
case "Name":
execiseName = Convert.ToString(row[col.ColumnName]);
break;
case "Index":
index = Convert.ToString(row[col.ColumnName]);
break;
case "Team ID":
teamID = Convert.ToString(row[col.ColumnName]);
break;
case "Trainer ID":
trainerID = Convert.ToString(row[col.ColumnName]);
break;
}
}
// uploaded Execise Videos on sprout server
if (uploadedExeciseVideos.ContainsKey(execiseName))
{
// find out local stored file path from dictionary
string localUploadPath = string.Empty;
if (uploadedExeciseVideos.TryGetValue(execiseName, out localUploadPath))
{
if (!string.IsNullOrEmpty(localUploadPath))
{
// Upload on Execise Video on sprout server, get response and add to Fitcom database;
string uploadesfile = UploadExeciseVideoToFitcom(localUploadPath);
if (!string.IsNullOrEmpty(uploadesfile))
{
ExerciseVideoResponse response = JsonConvert.DeserializeObject <ExerciseVideoResponse>(uploadesfile);
if (response != null)
{
UpdateExerciseSproutLink objUpdateExerciseSproutLink = new UpdateExerciseSproutLink
{
SecuryId = response.security_token,
VideoId = response.id,
V1080pUrl = response.assets.videos._1080p,
V240pUrl = response.assets.videos._240p,
V360pUrl = response.assets.videos._360p,
V480pUrl = response.assets.videos._480p,
V720pUrl = response.assets.videos._720p,
ThumnailUrl = response.assets.thumbnails[0],
ExerciseName = execiseName,
IsActive = true,
Index = index,
Description = string.Empty,
IsUpdated = true,
TeamID = teamID,
TrainerID = trainerID,
VideoLink = execiseName + ConstantHelper.constMP4ExesionWithdot,
SourceUrl = string.Empty
};
// Add Execise Details
ExerciseBL.AddExerciseSproutData(objUpdateExerciseSproutLink);
}
}
else
{
var FailToUpload = new UpdateExerciseSproutLink
{
ExerciseName = execiseName,
Index = index,
TeamID = teamID,
TrainerID = trainerID,
FailedVideoName = execiseName + ConstantHelper.constMP4ExesionWithdot,
};
FailToUploadedExeciseVideos.Add(FailToUpload);
}
}
// Delete Uploaded Execise
if (System.IO.File.Exists(localUploadPath))
{
System.IO.File.Delete(localUploadPath);
}
localUploadPath = string.Empty;
}
}
else if (!string.IsNullOrEmpty(execiseName))
{
var FailToUpload = new UpdateExerciseSproutLink
{
ExerciseName = execiseName,
Index = index,
TeamID = teamID,
TrainerID = trainerID,
FailedVideoName = string.Empty,
};
FailToUploadedExeciseVideos.Add(FailToUpload);
}
}
// Save To Failed Upload data in database
ExerciseBL.SaveFailedSproutServer(FailToUploadedExeciseVideos);
}
else
{
return(Json(new { Message = "No files selected.", StatusCode = 100 }, JsonRequestBehavior.AllowGet));
}
return(Json(new { Message = "File Uploaded Successfully!", StatusCode = 101 }, JsonRequestBehavior.AllowGet));
}
catch (Exception ex)
{
LogManager.LogManagerInstance.WriteErrorLog(ex);
return(Json(new { Message = "Error occurred. Error details: " + ex.Message, StatusCode = 100 }, JsonRequestBehavior.AllowGet));
}
finally
{
traceLog.AppendLine("UploadExerciseVideos end() : --- " + DateTime.Now.ToLongDateString());
LogManager.LogManagerInstance.WriteTraceLog(traceLog);
if (dt != null)
{
dt.Dispose();
}
}
}
private static void ValidateAcademicSessionsCSVFile(List <string> lstFileMissingHeaders, List <string> lstFileInvalidInfo, CsvHelper.CsvReader csvreader)
{
if (csvreader.ReadHeader())
{
if (!csvreader.FieldHeaders.Contains("sourcedId"))
{
lstFileMissingHeaders.Add("sourcedId");
}
if (!csvreader.FieldHeaders.Contains("title"))
{
lstFileMissingHeaders.Add("title");
}
if (!csvreader.FieldHeaders.Contains("type"))
{
lstFileMissingHeaders.Add("type");
}
if (!csvreader.FieldHeaders.Contains("startDate"))
{
lstFileMissingHeaders.Add("startDate");
}
if (!csvreader.FieldHeaders.Contains("endDate"))
{
lstFileMissingHeaders.Add("endDate");
}
}
int iRow = 1;
while (csvreader.Read())
{
ValidateField("sourcedId", null, lstFileInvalidInfo, iRow, csvreader);
ValidateField("title", null, lstFileInvalidInfo, iRow, csvreader);
//ValidateField("type", new string[] {"term", "gradingPeriod", "schoolYear", "semester" }, lstFileInvalidInfo, iRow, csvreader);
ValidateField("startDate", null, lstFileInvalidInfo, iRow, csvreader);
ValidateField("endDate", null, lstFileInvalidInfo, iRow, csvreader);
iRow++;
}
}
private AdvException GetError(Debug.ErrType errType, string atTime)
{
try
{
if (File.Exists(Debug.ErrFile[errType]))
{
using (var csv = new CsvHelper.CsvReader(new StreamReader(Debug.ErrFile[errType], Encoding.UTF8, true)))
{
csv.Configuration.Delimiter = Debug.CharSeparate;
csv.Configuration.HasHeaderRecord = false;
while (csv.Read())
{
string[] r = csv.CurrentRecord;
if (atTime == r[0] && r.Length >= 5)
{
var ex1 = AdvException.GetFromJsonString(r[4]);
ex1.ExceptionData.ManualMessage = r[2];
return ex1;
}
}
}
}
}
catch (Exception ex)
{
lblErr.Text = ex.Message;
}
return null;
}
public ActionResult ImportDataProcess(string id, CreateSimpleModel model, FormCollection form)
{
var email = Request?.RequestContext?.HttpContext?.User?.Identity?.Name;
if (Request.IsAuthenticated == false)
{
//https://www.hlidacstatu.cz/account/Login?returnUrl=%2F%3Frnd%3D0036bd9be9bc42d4bdf449492968846e
return(RedirectToAction("Login", "Account", new { returnUrl = this.Request.Url.PathAndQuery }));
}
ViewBag.NumOfRows = 0;
model.DatasetId = id;
string[] csvHeaders = null;
if (string.IsNullOrEmpty(id))
{
return(Redirect("/data"));
}
var ds = DataSet.CachedDatasets.Get(id);
if (ds == null)
{
return(Redirect("/data"));
}
if (ds.HasAdminAccess(email) == false)
{
return(View("NoAccess"));
}
datasetIndexStatCache.Invalidate();
if (ds.IsFlatStructure() == false)
{
return(RedirectToAction("ImportData", new { id = ds.DatasetId, fileId = model.FileId, delimiter = model.GetValidDelimiter() }));
}
var uTmp = new Lib.IO.UploadedTmpFile();
var path = uTmp.GetFullPath(model.FileId.ToString(), model.FileId.ToString() + ".csv");
if (!System.IO.File.Exists(path))
{
return(RedirectToAction("ImportData", new { id = ds.DatasetId }));
}
RuntimeClassBuilder rcb = new RuntimeClassBuilder(ds.GetPropertyNamesTypesFromSchema().ToDictionary(m => m.Key, v => v.Value.Type));
string[] formsHeaders = form["sheaders"].Split('|');
List <MappingCSV> mappingProps = new List <MappingCSV>();
for (int i = 0; i < formsHeaders.Length + 3; i++) //+3 a little bit more, at least +1 for id column
{
if (!string.IsNullOrEmpty(form["source_" + i]) &&
!string.IsNullOrEmpty(form["target_" + i]) &&
!string.IsNullOrEmpty(form["transform_" + i])
)
{
mappingProps.Add(new MappingCSV()
{
sourceCSV = form["source_" + i],
TargetJSON = form["target_" + i],
Transform = form["transform_" + i]
}
);
}
}
System.Collections.Concurrent.ConcurrentBag <Exception> errors = new System.Collections.Concurrent.ConcurrentBag <Exception>();
List <Tuple <object, string> > items = new List <Tuple <object, string> >();
try
{
using (System.IO.StreamReader r = new System.IO.StreamReader(path))
{
if (model.Delimiter == "auto")
{
model.Delimiter = Devmasters.IO.IOTools.DetectCSVDelimiter(r);
}
var csv = new CsvHelper.CsvReader(r, new CsvHelper.Configuration.CsvConfiguration(Util.Consts.csCulture)
{
HasHeaderRecord = true
,
Delimiter = model.GetValidDelimiter()
//,MissingFieldFound = null
}
);
csv.Read(); csv.ReadHeader();
csvHeaders = csv.HeaderRecord.Where(m => !string.IsNullOrEmpty(m?.Trim())).ToArray(); //for future control
while (csv.Read())
{
var newObj = rcb.CreateObject();
for (int m = 0; m < mappingProps.Count; m++)
{
Type destType = ds.GetPropertyNameTypeFromSchema(mappingProps[m].TargetJSON).FirstOrDefault().Value.Type;
object value = null;
string[] specialValues = new string[] { "-skip-", "-gen-", "--" };
if (specialValues.Contains(mappingProps[m].sourceCSV))
{
if (mappingProps[m].sourceCSV == "-gen-")
{
value = Guid.NewGuid().ToString("N");
rcb.SetPropertyValue(newObj, mappingProps[m].TargetJSON, value);
}
else
{
continue; // -skip- skip
}
}
else
{
string svalue = null;
try
{
svalue = csv.GetField(mappingProps[m].sourceCSV);
if (destType == typeof(string))
{
value = svalue;
}
else if (destType == typeof(DateTime) || destType == typeof(DateTime?))
{
value = Devmasters.DT.Util.ToDateTime(svalue);
}
else if (destType == typeof(decimal) || destType == typeof(decimal?))
{
value = Util.ParseTools.ToDecimal(svalue);
if (value == null)
{
value = Util.ParseTools.FromTextToDecimal(svalue);
}
}
else if (destType == typeof(long) || destType == typeof(long?) ||
destType == typeof(int) || destType == typeof(int?))
{
value = Devmasters.DT.Util.ToDate(svalue);
}
else if (destType == typeof(bool) || destType == typeof(bool?))
{
if (bool.TryParse(svalue, out bool tryp))
{
value = tryp;
}
}
else
{
value = svalue;
}
if (mappingProps[m].Transform == "normalize" &&
destType == typeof(string)
)
{
value = DataSet.NormalizeValueForId((string)value);
}
else if (mappingProps[m].Transform == "findico" &&
destType == typeof(string)
)
{
value = Lib.Validators.IcosInText((string)value).FirstOrDefault();
}
else //copy
{
}
rcb.SetPropertyValue(newObj, mappingProps[m].TargetJSON, value);
}
catch (Exception mex)
{
errors.Add(mex);
}
}
} //for
string idPropName = "id";
string idVal = rcb.GetPropertyValue(newObj, "id")?.ToString();
if (string.IsNullOrEmpty(idVal))
{
idVal = rcb.GetPropertyValue(newObj, "Id")?.ToString();
idPropName = "Id";
}
if (string.IsNullOrEmpty(idVal))
{
idVal = rcb.GetPropertyValue(newObj, "iD")?.ToString();
idPropName = "iD";
}
if (string.IsNullOrEmpty(idVal))
{
idVal = rcb.GetPropertyValue(newObj, "ID")?.ToString();
idPropName = "ID";
}
try
{
//var debugJson = Newtonsoft.Json.JsonConvert.SerializeObject(newObj);
//normalize ID
idVal = DataSet.NormalizeValueForId(idVal);
rcb.SetPropertyValue(newObj, idPropName, idVal);
items.Add(new Tuple <object, string>(newObj, idVal));
model.NumOfRows++;
}
catch (DataSetException dex)
{
errors.Add(dex);
}
catch (Exception ex)
{
errors.Add(ex);
}
}
}
}
catch (Exception ex)
{
errors.Add(ex);
}
try
{
Devmasters.Batch.Manager.DoActionForAll <Tuple <object, string> >(items,
(item) =>
{
try
{
ds.AddData(item.Item1, item.Item2, email, true);
}
catch (Exception ex)
{
errors.Add(ex);
}
return(new Devmasters.Batch.ActionOutputData());
}, true
);
}
catch (Exception ex)
{
errors.Add(ex);
}
if (errors?.Count > 0)
{
HlidacStatu.Util.Consts.Logger.Error("ImportDataProcess exceptions \n"
+ errors.Select(m => m.Message).Aggregate((f, s) => f + "\n" + s));
}
ViewBag.ApiResponseError = ApiResponseStatus.Error(-99, "Chyba při importu dat");
ViewBag.Errors = errors.ToList();
return(View(model));
}
public ActionResult CreateSimple2(CreateSimpleModel model)
{
var email = Request?.RequestContext?.HttpContext?.User?.Identity?.Name;
if (Request.IsAuthenticated == false)
{
return(RedirectToAction("Login", "Account", new { returnUrl = this.Request.Url.PathAndQuery }));
}
var uTmp = new Lib.IO.UploadedTmpFile();
var path = uTmp.GetFullPath(model.FileId.ToString(), model.FileId.ToString() + ".csv");
if (!System.IO.File.Exists(path))
{
return(RedirectToAction("CreateSimple"));
}
try
{
using (System.IO.StreamReader r = new System.IO.StreamReader(path))
{
var config = new CsvHelper.Configuration.CsvConfiguration(Util.Consts.csCulture);
config.HasHeaderRecord = true;
config.Delimiter = model.GetValidDelimiter();
var csv = new CsvHelper.CsvReader(r, config);
csv.Read(); csv.ReadHeader();
model.Headers = csv.HeaderRecord.Where(m => !string.IsNullOrEmpty(m?.Trim())).ToArray();
//read first lines with data and guest type
List <string[]> lines = new List <string[]>();
for (int row = 0; row < 10; row++)
{
if (csv.Read())
{
lines.Add(csv.GetRecords <string>().ToArray());
}
}
List <string> colTypes = null;
if (lines.Count > 0)
{
colTypes = new List <string>();
for (int cx = 0; cx < lines[0].Length; cx++)
{
string t = GuestBestCSVValueType(lines[0][cx]);
for (int line = 1; line < lines.Count; line++)
{
var nextT = GuestBestCSVValueType(lines[line][cx]);
if (nextT != t)
{
t = "string"; //kdyz jsou ruzne typy ve stejnem sloupci v ruznych radcich,
}
//fallback na string
}
colTypes.Add(t);
}
}
ViewBag.ColTypes = colTypes.ToArray();
return(View(model));
}
}
catch (Exception e)
{
ViewBag.ApiResponseError = ApiResponseStatus.Error(-99, "Soubor není ve formátu CSV.", e.ToString());
return(View(model));
}
}
public IEnumerable<PlateletActive.Core.Models.HplcData> GetLogFileData(string path)
{
importing = true;
var filePaths = System.IO.Directory.GetFiles(path, "*.csv");
var hplcDatas = new List<PlateletActive.Core.Models.HplcData>();
filesImported = new List<string>();
int concentrationColumn = -1;
foreach (var filePath in filePaths)
{
var valid = true;
using (System.IO.StreamReader reader = System.IO.File.OpenText(filePath))
{
using (var csv = new CsvHelper.CsvReader(reader))
{
int row = 2;
var hplcData = new Core.Models.HplcData();
while (csv.Read())
{
if (!valid)
{
continue;
}
string fieldATemp = null;
var fieldA = csv.TryGetField<string>(0, out fieldATemp) ? fieldATemp : null;
string fieldBTemp = null;
var fieldB = csv.TryGetField<string>(1, out fieldBTemp) ? fieldBTemp : null;
var fieldBDateTime = new DateTime();
if (fieldA == "Generated" && DateTime.TryParse(fieldB, out fieldBDateTime))
{
hplcData.Timestamp = fieldBDateTime;
}
if (fieldA == "Sample Name")
{
hplcData.SampleName = fieldB;
var sampleNameParts = fieldB.Split('-');
int batchId = 0;
if (sampleNameParts.Count() == 4 && Int32.TryParse(sampleNameParts.ElementAt(1), out batchId))
{
hplcData.BatchID = batchId;
if (hplcData.BatchID == null)
{
valid = false;
continue;
}
hplcData.SampleLocation = sampleNameParts.First();
hplcData.SampleAge = sampleNameParts.ElementAt(2);
hplcData.User = sampleNameParts.Last();
}
else if (sampleNameParts.Count() == 4)
{
hplcData.SampleLocation = sampleNameParts.First();
hplcData.User = sampleNameParts.Last();
}
else if (sampleNameParts.Count() == 3 && Int32.TryParse(sampleNameParts.ElementAt(0), out batchId))
{
hplcData.BatchID = batchId;
if (hplcData.BatchID == null)
{
valid = false;
continue;
}
hplcData.SampleLocation = sampleNameParts.ElementAt(1);
hplcData.User = sampleNameParts.Last();
}
else if (sampleNameParts.Count() == 3 && Int32.TryParse(sampleNameParts.ElementAt(1), out batchId))
{
hplcData.BatchID = batchId;
if (hplcData.BatchID == null)
{
valid = false;
continue;
}
hplcData.SampleLocation = sampleNameParts.First();
hplcData.User = sampleNameParts.Last();
}
else if (sampleNameParts.Count() == 3)
{
hplcData.SampleLocation = sampleNameParts.ElementAt(1);
hplcData.User = sampleNameParts.Last();
}
else if (sampleNameParts.Count() == 2)
{
hplcData.SampleLocation = sampleNameParts.First();
hplcData.User = sampleNameParts.Last();
}
else if (sampleNameParts.Count() == 1)
{
hplcData.User = sampleNameParts.First();
}
}
// Find concentration column.
if (fieldA.Trim() == "ID#")
{
string concentrationColumnCheck = string.Empty;
for (int i = 0; i < 10; i++)
{
// If this is the concenctration column then break out of the loop and use this column to get
// the concentrations.
if (csv.TryGetField<string>(i, out concentrationColumnCheck) && concentrationColumnCheck.Trim() == "Conc.")
{
concentrationColumn = i;
break;
}
}
// If the concentration column was not found then mark this file as invalid.
if (concentrationColumn == -1)
{
valid = false;
continue;
}
}
// Get concentrations.
if (fieldB == "DP4")
{
double hplcDataDp4Temp = -1;
if (!csv.TryGetField<double>(concentrationColumn, out hplcDataDp4Temp))
{
valid = false;
continue;
}
hplcData.DP4 = hplcDataDp4Temp;
}
if (fieldB == "DP3")
{
double hplcDataDp3Temp = -1;
if (!csv.TryGetField<double>(concentrationColumn, out hplcDataDp3Temp))
{
valid = false;
continue;
}
hplcData.DP3 = hplcDataDp3Temp;
}
if (fieldB == "DP2 MALTOSE")
{
double hplcDataDp2MaltoseTemp = -1;
if (!csv.TryGetField<double>(concentrationColumn, out hplcDataDp2MaltoseTemp))
{
valid = false;
continue;
}
hplcData.DP2Maltose = hplcDataDp2MaltoseTemp;
}
if (fieldB == "DP1 GLUCOSE")
{
double hplcDataDp1GlucoseTemp = -1;
if (!csv.TryGetField<double>(concentrationColumn, out hplcDataDp1GlucoseTemp))
{
valid = false;
continue;
}
hplcData.DP1Glucose = hplcDataDp1GlucoseTemp;
}
if (fieldB == "LACTIC ACID")
{
double hplcDataLacticAcidTemp = -1;
if (!csv.TryGetField<double>(concentrationColumn, out hplcDataLacticAcidTemp))
{
valid = false;
continue;
}
hplcData.LacticAcid = hplcDataLacticAcidTemp;
}
if (fieldB == "GLYCEROL")
{
double hplcDataGlycerolTemp = -1;
if (!csv.TryGetField<double>(concentrationColumn, out hplcDataGlycerolTemp))
{
valid = false;
continue;
}
hplcData.Glycerol = hplcDataGlycerolTemp;
}
if (fieldB == "ACETIC ACID")
{
double hplcDataAceticAcidTemp = -1;
if (!csv.TryGetField<double>(concentrationColumn, out hplcDataAceticAcidTemp))
{
valid = false;
continue;
}
hplcData.AceticAcid = hplcDataAceticAcidTemp;
}
if (fieldB == "ETHANOL")
{
double hplcDataEthanolTemp = -1;
if (!csv.TryGetField<double>(concentrationColumn, out hplcDataEthanolTemp))
{
valid = false;
continue;
}
hplcData.Ethanol = hplcDataEthanolTemp;
}
row++;
}
if (valid)
{
hplcDatas.Add(hplcData);
((List<string>)filesImported).Add(filePath);
}
}
}
}
importing = false;
return hplcDatas;
}
/// <summary>
///
/// </summary>
/// <param name="outputDirectory"></param>
private void ProcessAttachmentHashes(string outputDirectory)
{
if (File.Exists(System.IO.Path.Combine(outputDirectory, "Attachment.Hashes.csv")) == false)
{
OnError("Cannot locate the \"Attachment.Hashes.csv\" files");
return;
}
CsvConfiguration csvConfig = new CsvConfiguration();
Regex regex = new Regex("<(.*?)>", RegexOptions.IgnoreCase);
using (StreamReader sr = new StreamReader(System.IO.Path.Combine(outputDirectory, "Attachment.Hashes.csv")))
using (CsvHelper.CsvReader csvReader = new CsvHelper.CsvReader(sr, csvConfig))
{
List<Attachment> attachments = new List<Attachment>();
while (csvReader.Read())
{
var md5 = csvReader.GetField(0);
var file = csvReader.GetField(1);
var fileName = System.IO.Path.GetFileName(file);
var srcIp = csvReader.GetField(2);
var srcPort = csvReader.GetField(3);
var dstIp = csvReader.GetField(4);
var dstPort = csvReader.GetField(5);
var to = csvReader.GetField(6);
var dateSent = csvReader.GetField(11);
List<string> tempTo = new List<string>(to.Split(','));
for (int index = tempTo.Count() - 1; index > -1; index--)
{
string person = tempTo[index].Trim().ToLower();
if (person.IndexOf("@") == -1)
{
tempTo.RemoveAt(index);
continue;
}
Match match = regex.Match(person);
if (match.Success == true)
{
person = match.Groups[1].Value;
}
person = person.Replace(@"""", string.Empty);
tempTo[index] = person;
}
var from = csvReader.GetField(6);
var sender = csvReader.GetField(7);
var subject = csvReader.GetField(10);
var attachment = (from a in attachments where a.Md5 == md5.ToLower() select a).SingleOrDefault();
if (attachment == null)
{
attachment = new Attachment();
attachment.Md5 = md5.ToLower();
attachment.Recipients.AddRange(tempTo);
attachment.Subjects.Add(subject);
attachment.Senders.Add(sender);
if (fileName.Length > 0)
{
attachment.FileNames.Add(fileName);
}
attachment.DateSent = dateSent;
attachments.Add(attachment);
SubjectRecipents subjectRecipient = new SubjectRecipents();
subjectRecipient.Subject = subject;
subjectRecipient.File = file;
subjectRecipient.Sender = sender;
foreach (string person in tempTo)
{
subjectRecipient.Recipients.Add(person);
}
attachment.SubjectRecipents.Add(subjectRecipient);
}
else
{
foreach (string person in tempTo)
{
var tempPerson = from r in attachment.Recipients where r == person select r;
if (tempPerson.Any() == false)
{
attachment.Recipients.Add(person);
}
}
var tempFileName = from s in attachment.FileNames where s == fileName select s;
if (tempFileName.Any() == false)
{
attachment.FileNames.Add(fileName);
}
var tempSubject = from s in attachment.Subjects where s.ToLower() == subject select s;
if (tempSubject.Any() == false)
{
attachment.Subjects.Add(subject);
}
var tempSender = from s in attachment.Senders where s.ToLower() == sender select s;
if (tempSender.Any() == false)
{
attachment.Senders.Add(sender);
}
var subjectRecipient = (from s in attachment.SubjectRecipents where s.Subject.ToLower() == subject.ToLower() select s).SingleOrDefault();
if (subjectRecipient == null)
{
subjectRecipient = new SubjectRecipents();
subjectRecipient.Subject = subject;
subjectRecipient.File = file;
subjectRecipient.Sender = sender;
foreach (string person in tempTo)
{
subjectRecipient.Recipients.Add(person);
}
attachment.SubjectRecipents.Add(subjectRecipient);
}
else
{
foreach (string person in tempTo)
{
var tempPerson = from r in subjectRecipient.Recipients where r == person select r;
if (tempPerson.Any() == false)
{
subjectRecipient.Recipients.Add(person);
}
}
}
}
}
OutputSummary(outputDirectory, attachments);
}
}
public static SqlTable CsvToSqlTable(TextReader textReader, ISqlWriter sqlWriter, bool hasHeader = true, string delimiter = ",", int count = -1)
{
var config = new CsvHelper.Configuration.CsvConfiguration();
config.Delimiter = delimiter;
config.HasHeaderRecord = hasHeader;
var reader = new CsvHelper.CsvReader(textReader, config);
var rows = reader.GetRecords<object>().ToList();
textReader.Close();
SqlTable sqlTable = null;
if (rows.Count > 0)
{
sqlTable = new SqlTable();
sqlTable.Headers = new List<SqlColumnTypeDefinition>();
sqlTable.Rows = new List<SqlRow>();
var i = 0;
foreach (var row in rows)
{
if (i == count)
break;
i++;
var columns = (System.Dynamic.ExpandoObject)row;
var sqlRow = new SqlRow();
sqlTable.Rows.Add(sqlRow);
var countUnkown = 1;
foreach (var col in columns)
{
// set rows
var sqlColumn = new SqlColumn();
sqlColumn.Type = sqlWriter.SqlTypeToCSharpType(col.Value);
sqlColumn.Value = col.Value;
sqlRow.Columns.Add(sqlColumn);
// set headers
var key = col.Key;
if (string.IsNullOrWhiteSpace(key))
{
key = "Unkown" + countUnkown;
countUnkown++;
}
SqlColumnTypeDefinition sqlColumnTypeDefinition = sqlTable.Headers.FirstOrDefault(f => f.Name == key);
if (sqlColumnTypeDefinition == null)
{
sqlColumnTypeDefinition = new SqlColumnTypeDefinition();
sqlColumnTypeDefinition.Name = col.Key;
sqlColumnTypeDefinition.Type = sqlColumn.Type;
sqlTable.Headers.Add(sqlColumnTypeDefinition);
}
sqlColumn.ColumnTypeDefinition = sqlColumnTypeDefinition;
// override type of header to STRING because exists a string in the column
if (sqlColumnTypeDefinition.Type != typeof(string) && sqlColumn.Type == typeof(string))
sqlColumnTypeDefinition.Type = typeof(string);
}
}
// Fix type to STRING if all values is 'NULL'
foreach (var sqlHeader in sqlTable.Headers)
{
if (sqlHeader.Type == null)
sqlHeader.Type = typeof(string);
sqlHeader.TypeFormatted = sqlWriter.CSharpTypeToSqlType(sqlHeader.Type);
}
}
return sqlTable;
}
public ActionResult ImportData(string id, CreateSimpleModel model)
{
var email = Request?.RequestContext?.HttpContext?.User?.Identity?.Name;
if (Request.IsAuthenticated == false)
{
//https://www.hlidacstatu.cz/account/Login?returnUrl=%2F%3Frnd%3D0036bd9be9bc42d4bdf449492968846e
return(RedirectToAction("Login", "Account", new { returnUrl = this.Request.Url.PathAndQuery }));
}
model = model ?? new CreateSimpleModel();
model.DatasetId = id;
if (string.IsNullOrEmpty(id))
{
return(Redirect("/data"));
}
var ds = DataSet.CachedDatasets.Get(id);
if (ds == null)
{
return(Redirect("/data"));
}
if (ds.HasAdminAccess(email) == false)
{
return(View("NoAccess"));
}
model.DatasetId = ds.DatasetId;
if (ds.IsFlatStructure() == false)
{
ViewBag.Mode = "notflat";
return(View("ImportData.noflat", model));
}
if (model.FileId.HasValue)
{
ViewBag.Mode = "mapping";
Guid fileId = model.FileId.Value;
var uTmp = new Lib.IO.UploadedTmpFile();
var path = uTmp.GetFullPath(fileId.ToString(), fileId.ToString() + ".csv");
if (!System.IO.File.Exists(path))
{
return(RedirectToAction("ImportData", new { id = model.DatasetId }));
}
using (System.IO.StreamReader r = new System.IO.StreamReader(path))
{
if (model.Delimiter == "auto")
{
model.Delimiter = Devmasters.IO.IOTools.DetectCSVDelimiter(r);
}
var csv = new CsvHelper.CsvReader(r, new CsvHelper.Configuration.CsvConfiguration(Util.Consts.csCulture)
{
HasHeaderRecord = true, Delimiter = model.GetValidDelimiter()
});
csv.Read(); csv.ReadHeader();
model.Headers = csv.HeaderRecord.Where(m => !string.IsNullOrEmpty(m?.Trim())).ToArray();
}
return(View("ImportData.mapping", model));
}
else
{
ViewBag.Mode = "upload";
}
return(View(model));
}
public static void LoadStellarClasses()
{
using(var stream = new System.IO.StreamReader(StellarClassDefinitionFile))
{
var csvReader = new CsvHelper.CsvReader(stream);
// skip 2 header rows
csvReader.ReadHeader();
csvReader.Read();
while (csvReader.Read())
{
var item = new StarClass
{
Name = csvReader.GetField(0),
Mass = csvReader.GetField<double>(1),
Luminosity = csvReader.GetField<double>(2),
Radius = csvReader.GetField<double>(3),
Temperature = csvReader.GetField<double>(4),
ColorIndex = csvReader.GetField<float>(5),
AbsoluteMagnitude = csvReader.GetField<float>(6),
BolometricCorrection = csvReader.GetField<float>(7),
BolometricMagnitude = csvReader.GetField<float>(8),
};
var rgb = csvReader.GetField(9).Split(' ').Select(s => byte.Parse(s)).ToArray();
item.Color = new Color(rgb[0], rgb[1], rgb[2]);
StellarDefinitions[item.Name] = item;
}
}
}
//********************
//* *
//* UploadEntities *
//* *
//********************
// Upload a collection of entities from a local file in a selected download format.
private void UploadEntities(String tableName, String format, String inputFile, String outerElementName, String partitionKeyColumnName, String rowKeyColumnName, bool stopOnError)
{
int recordNumber = 1;
int recordsAdded = 0;
int recordErrors = 0;
String serializationError = null;
String xpath = outerElementName;
// Create task action.
String message = null;
message = "Uploading entities to table " + tableName + " from file " + inputFile;
Action action = new Action()
{
Id = NextAction++,
ActionType = Action.ACTION_DOWNLOAD_ENTITIES,
IsCompleted = false,
Message = message
};
Actions.Add(action.Id, action);
UpdateStatus();
// Execute background task to perform the downloading.
Task task = Task.Factory.StartNew(() =>
{
CloudTable table = tableClient.GetTableReference(tableName);
//table.CreateIfNotExists();
// Upload data using the specified format (csv, json, or xml).
switch (format)
{
case "json":
{
// JSON format upload
// {
// "Entities": [
// {
// "RowKey": "Batman",
// "PartitionKey": "DC Comics",
// "Timestamp": "8/6/2014 4:07:06 AM +00:00",
// "Debut": "5/1/1939 12:00:00 AM",
// "SecretIdentity": "Bruce Wayne"
// },
// {
// "RowKey": "Green Lantern",
// "PartitionKey": "DC Comics",
// "Timestamp": "8/6/2014 4:10:52 AM +00:00",
// "Debut": "7/1/1940 12:00:00 AM",
// "SecretIdentity": "Hal Jordan"
// },
// ...
// ]
//}
Dictionary<String, Object> entries = null;
// Parse the data in the JavaScriptSerializer.
try
{
JavaScriptSerializer ser = new JavaScriptSerializer();
entries = ser.DeserializeObject(File.ReadAllText(inputFile)) as Dictionary<String, Object>;
}
catch (Exception ex)
{
serializationError = "An error occurred deserializing the JSON file: " + ex.Message;
}
// Walk the result object graph and extract entities.
if (entries != null)
{
foreach (KeyValuePair<String, Object> entry in entries)
{
if (entry.Key == outerElementName)
{
Object[] entities = entry.Value as Object[];
foreach (object ent in entities)
{
if (ent is Dictionary<String, Object>)
{
if (WriteEntity(tableName, ent as Dictionary<String, Object>, partitionKeyColumnName, rowKeyColumnName))
{
recordsAdded++;
recordNumber++;
}
else
{
recordErrors++;
if (stopOnError)
{
recordNumber++;
break;
}
}
}
}
}
}
}
}
break;
case "xml":
{
// XML format upload
XmlDocument doc = null;
try
{
doc = new XmlDocument();
doc.LoadXml(File.ReadAllText(inputFile));
//foreach (XmlElement entities in doc.DocumentElement.GetElementsByTagName(outerElementName))
XmlNodeList nodes = doc.DocumentElement.SelectNodes(xpath);
//foreach (XmlElement entities in doc.DocumentElement)
foreach (XmlElement entities in nodes)
{
//<Entities xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
// <Entity>
// <RowKey>Batman</RowKey>
// <PartitionKey>DC Comics</PartitionKey>
// <Timestamp>8/6/2014 4:07:06 AM +00:00</Timestamp>
// <Debut>5/1/1939 12:00:00 AM</Debut>
// <SecretIdentity>Bruce Wayne</SecretIdentity>
// </Entity>
List<String> columns = new List<string>();
List<String> values = new List<string>();
foreach (XmlElement entity in entities.ChildNodes) // .GetElementsByTagName("Entity"))
//foreach (XmlElement entity in entities.GetElementsByTagName(outerElementName))
{
foreach (XmlNode field in entity.ChildNodes)
{
if (field is XmlText)
{
XmlText node = field as XmlText;
columns.Add(node.ParentNode.Name);
values.Add(node.Value);
}
}
}
if (WriteEntity(tableName, columns.ToArray(), values.ToArray(), partitionKeyColumnName, rowKeyColumnName))
{
recordsAdded++;
recordNumber++;
}
else
{
recordErrors++;
if (stopOnError)
{
recordNumber++;
break;
}
}
}
}
catch (Exception ex)
{
serializationError = "An error occurred parsing the XML file: " + ex.Message;
}
}
break;
case "csv":
{
// CSV format upload. Uses CsvHelper from http://joshclose.github.io/CsvHelper/.
try
{
using (TextReader reader = File.OpenText(inputFile))
{
var csv = new CsvHelper.CsvReader(reader);
String[] columns = null;
String[] values = null;
while(csv.Read())
{
// If first pass, retrieve column names.
if (recordNumber == 1)
{
columns = csv.FieldHeaders;
values = new String[columns.Length];
}
recordNumber++;
// Retrieve record values.
int col = 0;
foreach (String column in columns)
{
values[col] = csv.GetField(column);
col++;
}
// Write entity.
if (WriteEntity(tableName, columns, values, partitionKeyColumnName, rowKeyColumnName))
{
recordsAdded++;
recordNumber++;
}
else
{
recordErrors++;
if (stopOnError)
{
recordNumber++;
break;
}
}
}
}
}
catch (Exception ex)
{
serializationError = "An error occurred parsing the CSV file: " + ex.Message;
}
}
break;
default:
ShowError("Cannot upload - unknown format '" + format + "'");
break;
}
Actions[action.Id].IsCompleted = true;
});
// Task complete - update UI.
task.ContinueWith((t) =>
{
if (serializationError != null)
{
ShowError(serializationError);
}
switch (recordErrors)
{
case 0:
break;
case 1:
ShowError("An error occurred inserting entity nunber " + (recordNumber-1).ToString() + ".");
break;
default:
ShowError(recordErrors.ToString() + " errors occurred inserting entities.");
break;
}
UpdateStatus();
ShowTableContainer(SelectedTableContainer);
}, TaskScheduler.FromCurrentSynchronizationContext());
}
private void ProcessCsv(string fullPath)
{
CommonStatistic.Init();
CommonStatistic.IsRun = true;
var fieldMapping = new Dictionary<string, int>();
using (var csv = new CsvHelper.CsvReader(new StreamReader(fullPath, Encoding.UTF8)))
{
csv.Configuration.Delimiter = ";";
csv.Configuration.HasHeaderRecord = false;
csv.Read();
for (int i = 0; i < csv.CurrentRecord.Length; i++)
{
if (csv.CurrentRecord[i] == ProductFields.GetStringNameByEnum(ProductFields.Fields.None)) continue;
if (!fieldMapping.ContainsKey(csv.CurrentRecord[i]))
fieldMapping.Add(csv.CurrentRecord[i], i);
}
}
using (var csv = new CsvHelper.CsvReader(new StreamReader(fullPath, Encoding.UTF8)))
{
csv.Configuration.Delimiter = ";";
csv.Configuration.HasHeaderRecord = true;
while (csv.Read())
{
if (!CommonStatistic.IsRun)
{
csv.Dispose();
FileHelpers.DeleteFile(fullPath);
return;
}
CommonStatistic.RowPosition++;
try
{
// Step by rows
var productInStrings = new Dictionary<ProductFields.Fields, string>();
string nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.Sku);
if (fieldMapping.ContainsKey(nameField))
{
productInStrings.Add(ProductFields.Fields.Sku, SQLDataHelper.GetString(csv[fieldMapping[nameField]]));
}
nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.Name).Trim('*');
if (fieldMapping.ContainsKey(nameField))
{
var name = SQLDataHelper.GetString(csv[fieldMapping[nameField]]);
if (!string.IsNullOrEmpty(name))
{
productInStrings.Add(ProductFields.Fields.Name, name);
}
else
{
Log(string.Format(Resource.Admin_ImportCsv_CanNotEmpty, ProductFields.GetDisplayNameByEnum(ProductFields.Fields.Name), CommonStatistic.RowPosition + 2), "InvalidData");
continue;
}
}
nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.Enabled).Trim('*');
if (fieldMapping.ContainsKey(nameField))
{
string enabled = SQLDataHelper.GetString(csv[fieldMapping[nameField]]);
productInStrings.Add(ProductFields.Fields.Enabled, enabled);
}
nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.Discount);
if (fieldMapping.ContainsKey(nameField))
{
string discount = SQLDataHelper.GetString(csv[fieldMapping[nameField]]);
if (string.IsNullOrEmpty(discount))
discount = "0";
float tmp;
if (float.TryParse(discount, out tmp))
{
productInStrings.Add(ProductFields.Fields.Discount, tmp.ToString());
}
else if (float.TryParse(discount, NumberStyles.Any, CultureInfo.InvariantCulture, out tmp))
{
productInStrings.Add(ProductFields.Fields.Discount, tmp.ToString());
}
else
{
Log(string.Format(Resource.Admin_ImportCsv_MustBeNumber, ProductFields.GetDisplayNameByEnum(ProductFields.Fields.Discount), CommonStatistic.RowPosition + 2), "InvalidData");
continue;
}
}
nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.Weight);
if (fieldMapping.ContainsKey(nameField))
{
string weight = SQLDataHelper.GetString(csv[fieldMapping[nameField]]);
if (string.IsNullOrEmpty(weight))
weight = "0";
float tmp;
if (float.TryParse(weight, out tmp))
{
productInStrings.Add(ProductFields.Fields.Weight, tmp.ToString());
}
else if (float.TryParse(weight, NumberStyles.Any, CultureInfo.InvariantCulture, out tmp))
{
productInStrings.Add(ProductFields.Fields.Weight, tmp.ToString());
}
else
{
Log(string.Format(Resource.Admin_ImportCsv_MustBeNumber, ProductFields.GetDisplayNameByEnum(ProductFields.Fields.Weight), CommonStatistic.RowPosition + 2), "InvalidData");
continue;
}
}
nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.Size);
if (fieldMapping.ContainsKey(nameField))
{
productInStrings.Add(ProductFields.Fields.Size, SQLDataHelper.GetString(csv[fieldMapping[nameField]]));
}
nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.BriefDescription);
if (fieldMapping.ContainsKey(nameField))
{
productInStrings.Add(ProductFields.Fields.BriefDescription, SQLDataHelper.GetString(csv[fieldMapping[nameField]]));
}
nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.Description);
if (fieldMapping.ContainsKey(nameField))
{
productInStrings.Add(ProductFields.Fields.Description, SQLDataHelper.GetString(csv[fieldMapping[nameField]]));
}
nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.MultiOffer);
if (fieldMapping.ContainsKey(nameField))
{
var multiOffer = SQLDataHelper.GetString(csv[fieldMapping[nameField]]);
if (!string.IsNullOrEmpty(multiOffer))
{
productInStrings.Add(ProductFields.Fields.MultiOffer, multiOffer);
}
}
nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.Price).Trim('*');
if (fieldMapping.ContainsKey(nameField))
{
string price = SQLDataHelper.GetString(csv[fieldMapping[nameField]]);
if (string.IsNullOrEmpty(price))
price = "0";
float tmp;
if (float.TryParse(price, out tmp))
{
productInStrings.Add(ProductFields.Fields.Price, tmp.ToString());
}
else if (float.TryParse(price, NumberStyles.Any, CultureInfo.InvariantCulture, out tmp))
{
productInStrings.Add(ProductFields.Fields.Price, tmp.ToString());
}
else
{
Log(string.Format(Resource.Admin_ImportCsv_MustBeNumber, ProductFields.GetDisplayNameByEnum(ProductFields.Fields.Price), CommonStatistic.RowPosition + 2), "InvalidData");
continue;
}
}
nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.PurchasePrice);
if (fieldMapping.ContainsKey(nameField))
{
string sypplyprice = SQLDataHelper.GetString(csv[fieldMapping[nameField]]);
if (string.IsNullOrEmpty(sypplyprice))
sypplyprice = "0";
float tmp;
if (float.TryParse(sypplyprice, out tmp))
{
productInStrings.Add(ProductFields.Fields.PurchasePrice, tmp.ToString());
}
else if (float.TryParse(sypplyprice, NumberStyles.Any, CultureInfo.InvariantCulture, out tmp))
{
productInStrings.Add(ProductFields.Fields.PurchasePrice, tmp.ToString());
}
else
{
Log(string.Format(Resource.Admin_ImportCsv_MustBeNumber, ProductFields.GetDisplayNameByEnum(ProductFields.Fields.PurchasePrice), CommonStatistic.RowPosition + 2), "InvalidData");
continue;
}
}
nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.ShippingPrice);
if (fieldMapping.ContainsKey(nameField))
{
string shippingPrice = SQLDataHelper.GetString(csv[fieldMapping[nameField]]);
if (string.IsNullOrEmpty(shippingPrice))
shippingPrice = "0";
float tmp;
if (float.TryParse(shippingPrice, out tmp))
{
productInStrings.Add(ProductFields.Fields.ShippingPrice, tmp.ToString());
}
else if (float.TryParse(shippingPrice, NumberStyles.Any, CultureInfo.InvariantCulture, out tmp))
{
productInStrings.Add(ProductFields.Fields.ShippingPrice, tmp.ToString());
}
else
{
Log(string.Format(Resource.Admin_ImportCsv_MustBeNumber, ProductFields.GetDisplayNameByEnum(ProductFields.Fields.ShippingPrice), CommonStatistic.RowPosition + 2), "InvalidData");
continue;
}
}
nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.Amount);
if (fieldMapping.ContainsKey(nameField))
{
string amount = SQLDataHelper.GetString(csv[fieldMapping[nameField]]);
if (string.IsNullOrEmpty(amount))
amount = "0";
int tmp;
if (int.TryParse(amount, out tmp))
{
productInStrings.Add(ProductFields.Fields.Amount, amount);
}
else
{
Log(string.Format(Resource.Admin_ImportCsv_MustBeNumber, ProductFields.GetDisplayNameByEnum(ProductFields.Fields.Amount), CommonStatistic.RowPosition + 2), "InvalidData");
continue;
}
}
nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.Unit);
if (fieldMapping.ContainsKey(nameField))
{
productInStrings.Add(ProductFields.Fields.Unit, SQLDataHelper.GetString(csv[fieldMapping[nameField]]));
}
nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.ParamSynonym);
if (fieldMapping.ContainsKey(nameField))
{
string rewurl = SQLDataHelper.GetString(csv[fieldMapping[nameField]]);
productInStrings.Add(ProductFields.Fields.ParamSynonym, rewurl);
}
nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.Title);
if (fieldMapping.ContainsKey(nameField))
{
productInStrings.Add(ProductFields.Fields.Title, SQLDataHelper.GetString(csv[fieldMapping[nameField]]));
}
nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.MetaKeywords);
if (fieldMapping.ContainsKey(nameField))
{
productInStrings.Add(ProductFields.Fields.MetaKeywords, SQLDataHelper.GetString(csv[fieldMapping[nameField]]));
}
nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.MetaDescription);
if (fieldMapping.ContainsKey(nameField))
{
productInStrings.Add(ProductFields.Fields.MetaDescription, SQLDataHelper.GetString(csv[fieldMapping[nameField]]));
}
nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.Photos);
if (fieldMapping.ContainsKey(nameField))
{
productInStrings.Add(ProductFields.Fields.Photos, SQLDataHelper.GetString(csv[fieldMapping[nameField]]));
}
nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.Markers);
if (fieldMapping.ContainsKey(nameField))
{
productInStrings.Add(ProductFields.Fields.Markers, SQLDataHelper.GetString(csv[fieldMapping[nameField]]));
}
nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.Properties);
if (fieldMapping.ContainsKey(nameField))
{
productInStrings.Add(ProductFields.Fields.Properties, SQLDataHelper.GetString(csv[fieldMapping[nameField]]));
}
nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.Producer);
if (fieldMapping.ContainsKey(nameField))
{
productInStrings.Add(ProductFields.Fields.Producer, SQLDataHelper.GetString(csv[fieldMapping[nameField]]));
}
nameField = ProductFields.GetStringNameByEnum(ProductFields.Fields.Category).Trim('*');
if (fieldMapping.ContainsKey(nameField))
{
var parentCategory = SQLDataHelper.GetString(csv[fieldMapping[nameField]]);
if (!string.IsNullOrEmpty(parentCategory))
{
productInStrings.Add(ProductFields.Fields.Category, parentCategory);
}
}
ImportProduct.UpdateInsertProduct(productInStrings);
}
catch (Exception ex)
{
Log(ex.Message, "InvalidData");
}
}
CategoryService.RecalculateProductsCountManual();
}
CommonStatistic.IsRun = false;
FileHelpers.DeleteFile(fullPath);
}
static async Task <int> Main(string[] args)
{
try
{
//
// Parse options
//
Options = new AppOptions();
Options.Parse(args);
if (Options.ShowList)
{
}
if (Options.Exit)
{
return(-1);
}
if (string.IsNullOrEmpty(Options.FileName))
{
throw new ApplicationException("Please use --file to specify which file to use");
}
//
// Init module client
//
if (Options.UseEdge)
{
Log.WriteLine($"{AppOptions.AppName} module starting.");
await BlockTimer("Initializing Azure IoT Edge", async() => await InitEdge());
}
cts = new CancellationTokenSource();
AssemblyLoadContext.Default.Unloading += (ctx) => cts.Cancel();
Console.CancelKeyPress += (sender, cpe) => cts.Cancel();
//
// Load model
//
MLModel model = null;
Console.WriteLine($"Loading model from: '{Options.ModelPath}', Exists: '{File.Exists(Options.ModelPath)}'");
await BlockTimer($"Loading modelfile '{Options.ModelPath}' on the {(Options.UseGpu ? "GPU" : "CPU")}",
async() =>
{
var d = Directory.GetCurrentDirectory();
var path = d + "\\" + Options.ModelPath;
StorageFile modelFile = await AsAsync(StorageFile.GetFileFromPathAsync(path));
model = await MLModel.CreateFromStreamAsync(modelFile);
});
do
{
//
// Open file
//
var rows = new List <DataRow>();
try
{
using (var fs = new StreamReader(Options.FileName))
{
// I just need this one line to load the records from the file in my List<CsvLine>
rows = new CsvHelper.CsvReader(fs).GetRecords <DataRow>().ToList();
Console.WriteLine($"Loaded {rows.Count} row(s)");
}
}
catch (Exception e)
{
Console.WriteLine(e);
}
Console.WriteLine(rows);
//
// Main loop
//
foreach (var row in rows)
{
//
// Evaluate model
//
var inputShape = new long[2] {
1, 4
};
var inputFeatures = new float[4] {
row.Temperature, row.Pressure, row.Humidity, row.ExternalTemperature
};
MLModelVariable result = null;
var evalticks = await BlockTimer("Running the model",
async() =>
{
result = await model.EvaluateAsync(new MLModelVariable()
{
Variable = TensorFloat.CreateFromArray(inputShape, inputFeatures)
});
});
//
// Print results
//
var message = new MessageBody
{
result = result.Variable.GetAsVectorView().First()
};
message.metrics.evaltimeinms = evalticks;
var json = JsonConvert.SerializeObject(message);
Log.WriteLineRaw($"Recognized {json}");
//
// Send results to Edge
//
if (Options.UseEdge)
{
var eventMessage = new Message(Encoding.UTF8.GetBytes(json));
await ioTHubModuleClient.SendEventAsync("resultsOutput", eventMessage);
// Let's not totally spam Edge :)
await Task.Delay(500);
}
Console.WriteLine("Waiting 1 second...");
Thread.Sleep(1000);
}
}while (Options.RunForever && !cts.Token.IsCancellationRequested);
}
catch (Exception ex)
{
Console.WriteLine(ex);
return(-1);
}
return(0);
}
public ActionResult Create(CoordinatorSmsAndEmailModel model)
{
var trickleId = Guid.NewGuid();
Session.Add("CoordinatorSmsAndEmailModel", model);
Session.Add("trickleId", trickleId.ToString());
var hpf = Request.Files[0];
if (hpf.ContentLength == 0)
throw new ArgumentException("no content");
var csvFileContents = new CsvFileContents();
using (var csvReader = new CsvHelper.CsvReader(new StreamReader(hpf.InputStream), new CsvConfiguration { HasHeaderRecord = false }))
{
// TODO : Not reading first row properly
while (csvReader.Read())
{
csvFileContents.Rows.Add(csvReader.CurrentRecord);
}
}
var fileContentsId = trickleId.ToString() + "_fileContents";
using (var session = Raven.GetStore().OpenSession())
{
session.Store(csvFileContents, fileContentsId);
session.SaveChanges();
}
var dataColumnPicker = new DataColumnPicker {TrickleId = trickleId, FirstRowIsHeader = true};
if (!string.IsNullOrWhiteSpace(model.TemplateName))
{
using (var session = Raven.GetStore().OpenSession("Configuration"))
{
var communicationTemplate = session.Load<CommunicationTemplate>(model.TemplateName);
if (communicationTemplate.TemplateVariables != null)
communicationTemplate.TemplateVariables.ForEach(t => dataColumnPicker.TemplateVariableColumns.Add(t.VariableName, null));
}
}
var dropDownList = csvFileContents.CreateSelectList();
ViewData.Add("selectListData", dropDownList);
return View("CreateSmsAndEmailPickRows", dataColumnPicker);
}
public async Task <(ImportStatus, string)> FromCsvAsync(StreamReader csvStream)
{
var notes = new List <string>();
using var csv = new CsvHelper.CsvReader(csvStream, CultureInfo.InvariantCulture);
try
{
int recordCount = 0;
int issues = 0;
int districtsAdded = 0;
int schoolsAdded = 0;
var districts = await _schoolService.GetDistrictsAsync();
var schools = await _schoolService.GetForExportAsync();
var districtIndex = districts.ToDictionary(_ => _.Name, _ => _.Id);
foreach (var record in csv.GetRecords <SchoolImportExport>())
{
try
{
if (string.IsNullOrEmpty(record.District))
{
throw new GraException($"School district is blank on record {recordCount + 2}");
}
if (string.IsNullOrEmpty(record.Name))
{
throw new GraException($"School name is blank on record {recordCount + 2}");
}
if (record.District.Length > 255)
{
record.District = record.District.Substring(0, 255);
string warning = $"<li>District too long, truncated: <strong>{record.District}</strong>.</li>";
if (!notes.Contains(warning))
{
notes.Add(warning);
}
}
if (record.Name.Length > 255)
{
record.Name = record.Name.Substring(0, 255);
string warning = $"<li>Type too long, truncated: <strong>{record.Name}</strong>.</li>";
if (!notes.Contains(warning))
{
notes.Add(warning);
}
}
int districtId;
if (districtIndex.Keys.Contains(record.District.Trim()))
{
districtId = districtIndex[record.District.Trim()];
}
else
{
var district = await _schoolService.AddDistrict(new SchoolDistrict
{
Name = record.District.Trim()
});
districtIndex.Add(record.District.Trim(), district.Id);
districtId = district.Id;
districtsAdded++;
}
var schoolExists = schools.Any(_ => _.District == record.District.Trim() &&
_.Name == record.Name.Trim());
if (!schoolExists)
{
await _schoolService
.AddSchool(record.Name.Trim(), districtId);
schoolsAdded++;
}
recordCount++;
}
catch (Exception rex)
{
issues++;
if (rex.InnerException != null)
{
_logger.LogError($"School import error: {rex.InnerException.Message}");
notes.Add($"<li>Problem inserting record {recordCount + 2}: {rex.InnerException.Message}</li>");
}
else
{
_logger.LogError($"School import error: {rex.Message}");
notes.Add($"<li>Problem inserting record {recordCount + 2}: {rex.Message}</li>");
}
}
}
var returnMessage = new StringBuilder($"<p><strong>Read {recordCount} records (added {districtsAdded} districts, {schoolsAdded} schools) and skipped {issues} rows due to issues.</strong></p>");
foreach (var note in notes)
{
returnMessage.AppendLine(note);
}
if (issues > 0)
{
return(ImportStatus.Warning, returnMessage.ToString());
}
if (notes.Count > 0)
{
return(ImportStatus.Info, returnMessage.ToString());
}
return(ImportStatus.Success, returnMessage.ToString());
}
catch (Exception ex)
{
string error = $"CSV parsing error: {ex.Message}";
_logger.LogError(error);
return(ImportStatus.Danger, error);
}
}
/// <summary>
/// Delete customer from file. File must contain the CustomerCid from CSP
/// </summary>
/// <param name="filename">Path of csv file contain customers to delete</param>
/// <param name="defaultDomain">default domain of the reseller</param>
/// <param name="appId">appid that is registered for this application in Azure Active Directory (AAD)</param>
/// <param name="key">Key for this application in Azure Active Directory</param>
/// <param name="resellerMicrosoftId">Microsoft Id of the reseller</param>
internal static void DeleteCustomers(string filename, string defaultDomain, string appId, string key, string resellerMicrosoftId)
{
GetTokens(defaultDomain, appId, key, resellerMicrosoftId);
using (var reader = new CsvHelper.CsvReader(new StreamReader(filename)))
{
while (reader.Read())
{
string displayName = reader.GetField<string>("displayName");
string cspCustomerCid = reader.GetField<string>("cspCustomerCid");
if (cspCustomerCid.Trim().Length == 0)
Console.WriteLine("Unable to delete {0} because CustomerCid is missing.", displayName);
else
{
Console.WriteLine("Deleting {0}", displayName);
Customer.DeleteCustomer(resellerCid, cspCustomerCid, saAuthorizationToken.AccessToken);
}
}
}
}
