public ActionResult CreateSimple3(Guid?fileId) { var email = Request?.RequestContext?.HttpContext?.User?.Identity?.Name; if (Request.IsAuthenticated == false) { //https://www.hlidacstatu.cz/account/Login?returnUrl=%2F%3Frnd%3D0036bd9be9bc42d4bdf449492968846e return(RedirectToAction("Login", "Account", new { returnUrl = this.Request.Url.PathAndQuery })); } if (!fileId.HasValue) { return(RedirectToAction("CreateSimple")); } var uTmp = new Lib.IO.UploadedTmpFile(); var path = uTmp.GetFullPath(fileId.ToString(), fileId.ToString() + ".csv"); var pathJson = uTmp.GetFullPath(fileId.ToString(), fileId.ToString() + ".json"); if (!System.IO.File.Exists(path)) { return(RedirectToAction("CreateSimple")); } if (!System.IO.File.Exists(pathJson)) { return(RedirectToAction("CreateSimple")); } CreateSimpleModel model = CreateSimpleModel.Load(pathJson); if (model.NumOfRows == 0) { using (System.IO.StreamReader r = new System.IO.StreamReader(path)) { var csv = new CsvHelper.CsvReader(r, new CsvHelper.Configuration.CsvConfiguration(Util.Consts.csCulture) { HasHeaderRecord = true, Delimiter = model.GetValidDelimiter() }); csv.Read(); csv.ReadHeader(); while (csv.Read()) { model.NumOfRows++; } } } model.Save(pathJson); return(View(model)); }
public ActionResult ImportDataProcess(string id, CreateSimpleModel model, FormCollection form) { var email = Request?.RequestContext?.HttpContext?.User?.Identity?.Name; if (Request.IsAuthenticated == false) { //https://www.hlidacstatu.cz/account/Login?returnUrl=%2F%3Frnd%3D0036bd9be9bc42d4bdf449492968846e return(RedirectToAction("Login", "Account", new { returnUrl = this.Request.Url.PathAndQuery })); } ViewBag.NumOfRows = 0; model.DatasetId = id; string[] csvHeaders = null; if (string.IsNullOrEmpty(id)) { return(Redirect("/data")); } var ds = DataSet.CachedDatasets.Get(id); if (ds == null) { return(Redirect("/data")); } if (ds.HasAdminAccess(email) == false) { return(View("NoAccess")); } datasetIndexStatCache.Invalidate(); if (ds.IsFlatStructure() == false) { return(RedirectToAction("ImportData", new { id = ds.DatasetId, fileId = model.FileId, delimiter = model.GetValidDelimiter() })); } var uTmp = new Lib.IO.UploadedTmpFile(); var path = uTmp.GetFullPath(model.FileId.ToString(), model.FileId.ToString() + ".csv"); if (!System.IO.File.Exists(path)) { return(RedirectToAction("ImportData", new { id = ds.DatasetId })); } RuntimeClassBuilder rcb = new RuntimeClassBuilder(ds.GetPropertyNamesTypesFromSchema().ToDictionary(m => m.Key, v => v.Value.Type)); string[] formsHeaders = form["sheaders"].Split('|'); List <MappingCSV> mappingProps = new List <MappingCSV>(); for (int i = 0; i < formsHeaders.Length + 3; i++) //+3 a little bit more, at least +1 for id column { if (!string.IsNullOrEmpty(form["source_" + i]) && !string.IsNullOrEmpty(form["target_" + i]) && !string.IsNullOrEmpty(form["transform_" + i]) ) { mappingProps.Add(new MappingCSV() { sourceCSV = form["source_" + i], TargetJSON = form["target_" + i], Transform = form["transform_" + i] } ); } } System.Collections.Concurrent.ConcurrentBag <Exception> errors = new System.Collections.Concurrent.ConcurrentBag <Exception>(); List <Tuple <object, string> > items = new List <Tuple <object, string> >(); try { using (System.IO.StreamReader r = new System.IO.StreamReader(path)) { if (model.Delimiter == "auto") { model.Delimiter = Devmasters.IO.IOTools.DetectCSVDelimiter(r); } var csv = new CsvHelper.CsvReader(r, new CsvHelper.Configuration.CsvConfiguration(Util.Consts.csCulture) { HasHeaderRecord = true , Delimiter = model.GetValidDelimiter() //,MissingFieldFound = null } ); csv.Read(); csv.ReadHeader(); csvHeaders = csv.HeaderRecord.Where(m => !string.IsNullOrEmpty(m?.Trim())).ToArray(); //for future control while (csv.Read()) { var newObj = rcb.CreateObject(); for (int m = 0; m < mappingProps.Count; m++) { Type destType = ds.GetPropertyNameTypeFromSchema(mappingProps[m].TargetJSON).FirstOrDefault().Value.Type; object value = null; string[] specialValues = new string[] { "-skip-", "-gen-", "--" }; if (specialValues.Contains(mappingProps[m].sourceCSV)) { if (mappingProps[m].sourceCSV == "-gen-") { value = Guid.NewGuid().ToString("N"); rcb.SetPropertyValue(newObj, mappingProps[m].TargetJSON, value); } else { continue; // -skip- skip } } else { string svalue = null; try { svalue = csv.GetField(mappingProps[m].sourceCSV); if (destType == typeof(string)) { value = svalue; } else if (destType == typeof(DateTime) || destType == typeof(DateTime?)) { value = Devmasters.DT.Util.ToDateTime(svalue); } else if (destType == typeof(decimal) || destType == typeof(decimal?)) { value = Util.ParseTools.ToDecimal(svalue); if (value == null) { value = Util.ParseTools.FromTextToDecimal(svalue); } } else if (destType == typeof(long) || destType == typeof(long?) || destType == typeof(int) || destType == typeof(int?)) { value = Devmasters.DT.Util.ToDate(svalue); } else if (destType == typeof(bool) || destType == typeof(bool?)) { if (bool.TryParse(svalue, out bool tryp)) { value = tryp; } } else { value = svalue; } if (mappingProps[m].Transform == "normalize" && destType == typeof(string) ) { value = DataSet.NormalizeValueForId((string)value); } else if (mappingProps[m].Transform == "findico" && destType == typeof(string) ) { value = Lib.Validators.IcosInText((string)value).FirstOrDefault(); } else //copy { } rcb.SetPropertyValue(newObj, mappingProps[m].TargetJSON, value); } catch (Exception mex) { errors.Add(mex); } } } //for string idPropName = "id"; string idVal = rcb.GetPropertyValue(newObj, "id")?.ToString(); if (string.IsNullOrEmpty(idVal)) { idVal = rcb.GetPropertyValue(newObj, "Id")?.ToString(); idPropName = "Id"; } if (string.IsNullOrEmpty(idVal)) { idVal = rcb.GetPropertyValue(newObj, "iD")?.ToString(); idPropName = "iD"; } if (string.IsNullOrEmpty(idVal)) { idVal = rcb.GetPropertyValue(newObj, "ID")?.ToString(); idPropName = "ID"; } try { //var debugJson = Newtonsoft.Json.JsonConvert.SerializeObject(newObj); //normalize ID idVal = DataSet.NormalizeValueForId(idVal); rcb.SetPropertyValue(newObj, idPropName, idVal); items.Add(new Tuple <object, string>(newObj, idVal)); model.NumOfRows++; } catch (DataSetException dex) { errors.Add(dex); } catch (Exception ex) { errors.Add(ex); } } } } catch (Exception ex) { errors.Add(ex); } try { Devmasters.Batch.Manager.DoActionForAll <Tuple <object, string> >(items, (item) => { try { ds.AddData(item.Item1, item.Item2, email, true); } catch (Exception ex) { errors.Add(ex); } return(new Devmasters.Batch.ActionOutputData()); }, true ); } catch (Exception ex) { errors.Add(ex); } if (errors?.Count > 0) { HlidacStatu.Util.Consts.Logger.Error("ImportDataProcess exceptions \n" + errors.Select(m => m.Message).Aggregate((f, s) => f + "\n" + s)); } ViewBag.ApiResponseError = ApiResponseStatus.Error(-99, "Chyba při importu dat"); ViewBag.Errors = errors.ToList(); return(View(model)); }
public ActionResult ImportData(string id, string delimiter, string data, FormCollection form, HttpPostedFileBase file) { var email = Request?.RequestContext?.HttpContext?.User?.Identity?.Name; if (Request.IsAuthenticated == false) { //https://www.hlidacstatu.cz/account/Login?returnUrl=%2F%3Frnd%3D0036bd9be9bc42d4bdf449492968846e return(RedirectToAction("Login", "Account", new { returnUrl = this.Request.Url.PathAndQuery })); } if (string.IsNullOrEmpty(id)) { return(Redirect("/data")); } var ds = DataSet.CachedDatasets.Get(id); if (ds == null) { return(Redirect("/data")); } if (ds.HasAdminAccess(email) == false) { return(View("NoAccess")); } datasetIndexStatCache.Invalidate(); Guid fileId = Guid.NewGuid(); var uTmp = new Lib.IO.UploadedTmpFile(); if (string.IsNullOrEmpty(data)) { if (file == null) { ViewBag.ApiResponseError = ApiResponseStatus.Error(-99, "Źádné CSV jste nenahráli"); return(View()); } else { var path = uTmp.GetFullPath(fileId.ToString(), fileId.ToString() + ".csv"); file.SaveAs(path); return(RedirectToAction("ImportData", new { id = ds.DatasetId, fileId = fileId, delimiter = CreateSimpleModel.GetValidDelimiter(delimiter) })); } } else { var path = uTmp.GetFullPath(fileId.ToString(), fileId.ToString() + ".csv"); System.IO.File.WriteAllText(path, data); return(RedirectToAction("ImportData", new { id = ds.DatasetId, fileId = fileId, delimiter = "auto" })); } }
public ActionResult ImportData(string id, CreateSimpleModel model) { var email = Request?.RequestContext?.HttpContext?.User?.Identity?.Name; if (Request.IsAuthenticated == false) { //https://www.hlidacstatu.cz/account/Login?returnUrl=%2F%3Frnd%3D0036bd9be9bc42d4bdf449492968846e return(RedirectToAction("Login", "Account", new { returnUrl = this.Request.Url.PathAndQuery })); } model = model ?? new CreateSimpleModel(); model.DatasetId = id; if (string.IsNullOrEmpty(id)) { return(Redirect("/data")); } var ds = DataSet.CachedDatasets.Get(id); if (ds == null) { return(Redirect("/data")); } if (ds.HasAdminAccess(email) == false) { return(View("NoAccess")); } model.DatasetId = ds.DatasetId; if (ds.IsFlatStructure() == false) { ViewBag.Mode = "notflat"; return(View("ImportData.noflat", model)); } if (model.FileId.HasValue) { ViewBag.Mode = "mapping"; Guid fileId = model.FileId.Value; var uTmp = new Lib.IO.UploadedTmpFile(); var path = uTmp.GetFullPath(fileId.ToString(), fileId.ToString() + ".csv"); if (!System.IO.File.Exists(path)) { return(RedirectToAction("ImportData", new { id = model.DatasetId })); } using (System.IO.StreamReader r = new System.IO.StreamReader(path)) { if (model.Delimiter == "auto") { model.Delimiter = Devmasters.IO.IOTools.DetectCSVDelimiter(r); } var csv = new CsvHelper.CsvReader(r, new CsvHelper.Configuration.CsvConfiguration(Util.Consts.csCulture) { HasHeaderRecord = true, Delimiter = model.GetValidDelimiter() }); csv.Read(); csv.ReadHeader(); model.Headers = csv.HeaderRecord.Where(m => !string.IsNullOrEmpty(m?.Trim())).ToArray(); } return(View("ImportData.mapping", model)); } else { ViewBag.Mode = "upload"; } return(View(model)); }
public ActionResult CreateSimple2(CreateSimpleModel model) { var email = Request?.RequestContext?.HttpContext?.User?.Identity?.Name; if (Request.IsAuthenticated == false) { return(RedirectToAction("Login", "Account", new { returnUrl = this.Request.Url.PathAndQuery })); } var uTmp = new Lib.IO.UploadedTmpFile(); var path = uTmp.GetFullPath(model.FileId.ToString(), model.FileId.ToString() + ".csv"); if (!System.IO.File.Exists(path)) { return(RedirectToAction("CreateSimple")); } try { using (System.IO.StreamReader r = new System.IO.StreamReader(path)) { var config = new CsvHelper.Configuration.CsvConfiguration(Util.Consts.csCulture); config.HasHeaderRecord = true; config.Delimiter = model.GetValidDelimiter(); var csv = new CsvHelper.CsvReader(r, config); csv.Read(); csv.ReadHeader(); model.Headers = csv.HeaderRecord.Where(m => !string.IsNullOrEmpty(m?.Trim())).ToArray(); //read first lines with data and guest type List <string[]> lines = new List <string[]>(); for (int row = 0; row < 10; row++) { if (csv.Read()) { lines.Add(csv.GetRecords <string>().ToArray()); } } List <string> colTypes = null; if (lines.Count > 0) { colTypes = new List <string>(); for (int cx = 0; cx < lines[0].Length; cx++) { string t = GuestBestCSVValueType(lines[0][cx]); for (int line = 1; line < lines.Count; line++) { var nextT = GuestBestCSVValueType(lines[line][cx]); if (nextT != t) { t = "string"; //kdyz jsou ruzne typy ve stejnem sloupci v ruznych radcich, } //fallback na string } colTypes.Add(t); } } ViewBag.ColTypes = colTypes.ToArray(); return(View(model)); } } catch (Exception e) { ViewBag.ApiResponseError = ApiResponseStatus.Error(-99, "Soubor není ve formátu CSV.", e.ToString()); return(View(model)); } }
public ActionResult CreateSimple(string name, string delimiter, FormCollection form, HttpPostedFileBase file) { var email = Request?.RequestContext?.HttpContext?.User?.Identity?.Name; if (Request.IsAuthenticated == false) { return(RedirectToAction("Login", "Account", new { returnUrl = this.Request.Url.PathAndQuery })); } Guid fileId = Guid.NewGuid(); var uTmp = new Lib.IO.UploadedTmpFile(); if (string.IsNullOrEmpty(name)) { ViewBag.ApiResponseError = ApiResponseStatus.Error(-99, "Bez jména datasetu nemůžeme pokračovat. Prozraďte nám ho, prosím."); return(View()); } if (file == null) { ViewBag.ApiResponseError = ApiResponseStatus.Error(-99, "Źádné CSV jste nenahráli"); return(View()); } else { var path = uTmp.GetFullPath(fileId.ToString(), fileId.ToString() + ".csv"); file.SaveAs(path); return(RedirectToAction("CreateSimple2", new { fileId = fileId, delimiter = CreateSimpleModel.GetValidDelimiter(delimiter), name = name })); } }