public ActionResult ImportDataProcess(string id, CreateSimpleModel model, FormCollection form) { var email = Request?.RequestContext?.HttpContext?.User?.Identity?.Name; if (Request.IsAuthenticated == false) { //https://www.hlidacstatu.cz/account/Login?returnUrl=%2F%3Frnd%3D0036bd9be9bc42d4bdf449492968846e return(RedirectToAction("Login", "Account", new { returnUrl = this.Request.Url.PathAndQuery })); } ViewBag.NumOfRows = 0; model.DatasetId = id; string[] csvHeaders = null; if (string.IsNullOrEmpty(id)) { return(Redirect("/data")); } var ds = DataSet.CachedDatasets.Get(id); if (ds == null) { return(Redirect("/data")); } if (ds.HasAdminAccess(email) == false) { return(View("NoAccess")); } datasetIndexStatCache.Invalidate(); if (ds.IsFlatStructure() == false) { return(RedirectToAction("ImportData", new { id = ds.DatasetId, fileId = model.FileId, delimiter = model.GetValidDelimiter() })); } var uTmp = new Lib.IO.UploadedTmpFile(); var path = uTmp.GetFullPath(model.FileId.ToString(), model.FileId.ToString() + ".csv"); if (!System.IO.File.Exists(path)) { return(RedirectToAction("ImportData", new { id = ds.DatasetId })); } RuntimeClassBuilder rcb = new RuntimeClassBuilder(ds.GetPropertyNamesTypesFromSchema().ToDictionary(m => m.Key, v => v.Value.Type)); string[] formsHeaders = form["sheaders"].Split('|'); List <MappingCSV> mappingProps = new List <MappingCSV>(); for (int i = 0; i < formsHeaders.Length + 3; i++) //+3 a little bit more, at least +1 for id column { if (!string.IsNullOrEmpty(form["source_" + i]) && !string.IsNullOrEmpty(form["target_" + i]) && !string.IsNullOrEmpty(form["transform_" + i]) ) { mappingProps.Add(new MappingCSV() { sourceCSV = form["source_" + i], TargetJSON = form["target_" + i], Transform = form["transform_" + i] } ); } } System.Collections.Concurrent.ConcurrentBag <Exception> errors = new System.Collections.Concurrent.ConcurrentBag <Exception>(); List <Tuple <object, string> > items = new List <Tuple <object, string> >(); try { using (System.IO.StreamReader r = new System.IO.StreamReader(path)) { if (model.Delimiter == "auto") { model.Delimiter = Devmasters.IO.IOTools.DetectCSVDelimiter(r); } var csv = new CsvHelper.CsvReader(r, new CsvHelper.Configuration.CsvConfiguration(Util.Consts.csCulture) { HasHeaderRecord = true , Delimiter = model.GetValidDelimiter() //,MissingFieldFound = null } ); csv.Read(); csv.ReadHeader(); csvHeaders = csv.HeaderRecord.Where(m => !string.IsNullOrEmpty(m?.Trim())).ToArray(); //for future control while (csv.Read()) { var newObj = rcb.CreateObject(); for (int m = 0; m < mappingProps.Count; m++) { Type destType = ds.GetPropertyNameTypeFromSchema(mappingProps[m].TargetJSON).FirstOrDefault().Value.Type; object value = null; string[] specialValues = new string[] { "-skip-", "-gen-", "--" }; if (specialValues.Contains(mappingProps[m].sourceCSV)) { if (mappingProps[m].sourceCSV == "-gen-") { value = Guid.NewGuid().ToString("N"); rcb.SetPropertyValue(newObj, mappingProps[m].TargetJSON, value); } else { continue; // -skip- skip } } else { string svalue = null; try { svalue = csv.GetField(mappingProps[m].sourceCSV); if (destType == typeof(string)) { value = svalue; } else if (destType == typeof(DateTime) || destType == typeof(DateTime?)) { value = Devmasters.DT.Util.ToDateTime(svalue); } else if (destType == typeof(decimal) || destType == typeof(decimal?)) { value = Util.ParseTools.ToDecimal(svalue); if (value == null) { value = Util.ParseTools.FromTextToDecimal(svalue); } } else if (destType == typeof(long) || destType == typeof(long?) || destType == typeof(int) || destType == typeof(int?)) { value = Devmasters.DT.Util.ToDate(svalue); } else if (destType == typeof(bool) || destType == typeof(bool?)) { if (bool.TryParse(svalue, out bool tryp)) { value = tryp; } } else { value = svalue; } if (mappingProps[m].Transform == "normalize" && destType == typeof(string) ) { value = DataSet.NormalizeValueForId((string)value); } else if (mappingProps[m].Transform == "findico" && destType == typeof(string) ) { value = Lib.Validators.IcosInText((string)value).FirstOrDefault(); } else //copy { } rcb.SetPropertyValue(newObj, mappingProps[m].TargetJSON, value); } catch (Exception mex) { errors.Add(mex); } } } //for string idPropName = "id"; string idVal = rcb.GetPropertyValue(newObj, "id")?.ToString(); if (string.IsNullOrEmpty(idVal)) { idVal = rcb.GetPropertyValue(newObj, "Id")?.ToString(); idPropName = "Id"; } if (string.IsNullOrEmpty(idVal)) { idVal = rcb.GetPropertyValue(newObj, "iD")?.ToString(); idPropName = "iD"; } if (string.IsNullOrEmpty(idVal)) { idVal = rcb.GetPropertyValue(newObj, "ID")?.ToString(); idPropName = "ID"; } try { //var debugJson = Newtonsoft.Json.JsonConvert.SerializeObject(newObj); //normalize ID idVal = DataSet.NormalizeValueForId(idVal); rcb.SetPropertyValue(newObj, idPropName, idVal); items.Add(new Tuple <object, string>(newObj, idVal)); model.NumOfRows++; } catch (DataSetException dex) { errors.Add(dex); } catch (Exception ex) { errors.Add(ex); } } } } catch (Exception ex) { errors.Add(ex); } try { Devmasters.Batch.Manager.DoActionForAll <Tuple <object, string> >(items, (item) => { try { ds.AddData(item.Item1, item.Item2, email, true); } catch (Exception ex) { errors.Add(ex); } return(new Devmasters.Batch.ActionOutputData()); }, true ); } catch (Exception ex) { errors.Add(ex); } if (errors?.Count > 0) { HlidacStatu.Util.Consts.Logger.Error("ImportDataProcess exceptions \n" + errors.Select(m => m.Message).Aggregate((f, s) => f + "\n" + s)); } ViewBag.ApiResponseError = ApiResponseStatus.Error(-99, "Chyba při importu dat"); ViewBag.Errors = errors.ToList(); return(View(model)); }
public ActionResult CreateSimple2(CreateSimpleModel model, FormCollection form) { var email = Request?.RequestContext?.HttpContext?.User?.Identity?.Name; if (Request.IsAuthenticated == false) { return(RedirectToAction("Login", "Account", new { returnUrl = this.Request.Url.PathAndQuery })); } var uTmp = new Lib.IO.UploadedTmpFile(); var path = uTmp.GetFullPath(model.FileId.ToString(), model.FileId.ToString() + ".csv"); var pathModels = uTmp.GetFullPath(model.FileId.ToString(), model.FileId.ToString() + ".json"); model.Headers = (form["sheaders"] ?? "").Split('|'); //check Keycolumn List <CreateSimpleModel.Column> cols = new List <CreateSimpleModel.Column>(); int columns = model.Headers.Length; for (int i = 0; i < columns; i++) { string name = model.Headers[i]; if (form[$"include_{i}"] == "1") { cols.Add( new CreateSimpleModel.Column() { Name = name, NiceName = form[$"nicename_{i}"], ValType = form[$"typ_{i}"], ShowSearchFormat = form[$"show_search_{i}"] == "--" ? "string" : form[$"show_search_{i}"], ShowDetailFormat = form[$"show_detail_{i}"] == "--" ? "string" : form[$"show_detail_{i}"], } ); } } if (string.IsNullOrEmpty(model.KeyColumn) && !cols.Any(m => m.Name.ToLower() == "id")) { cols.Add(new CreateSimpleModel.Column() { Name = "id", NiceName = "Id", ValType = "string", ShowSearchFormat = "show", ShowDetailFormat = "hide", }); } model.Columns = cols.ToArray(); model.Save(pathModels); bool addIcoCol = false; Dictionary <string, Type> properties = new Dictionary <string, Type>(); //properties.Add("id", typeof(string)); foreach (var c in model.Columns) { switch (c.ValType) { case "number": properties.Add(c.NormalizedName(), typeof(Nullable <decimal>)); break; case "datetime": properties.Add(c.NormalizedName(), typeof(Nullable <DateTime>)); break; case "url": case "ico": default: properties.Add(c.NormalizedName(), typeof(string)); break; } } if (addIcoCol && !model.Columns.Any(m => m.NormalizedName() == "ico")) { properties.Add("ICO", typeof(string)); } if (!properties.Any(m => m.Key.ToLower() == "id")) { properties.Add("id", typeof(string)); } RuntimeClassBuilder rcb = new RuntimeClassBuilder(properties); var rcbObj = rcb.CreateObject(); Newtonsoft.Json.Schema.Generation.JSchemaGenerator jsonGen = new Newtonsoft.Json.Schema.Generation.JSchemaGenerator(); jsonGen.DefaultRequired = Newtonsoft.Json.Required.Default; var schema = jsonGen.Generate(rcbObj.GetType()); //JSON schema //create registration Registration reg = new Registration(); reg.allowWriteAccess = false; reg.betaversion = true; reg.jsonSchema = schema.ToString(); reg.name = model.Name; reg.NormalizeShortName(); reg.createdBy = email; HlidacStatu.Api.Dataset.Connector.ClassicTemplate.ClassicSearchResultTemplate search = new Api.Dataset.Connector.ClassicTemplate.ClassicSearchResultTemplate(); HlidacStatu.Api.Dataset.Connector.ClassicTemplate.ClassicDetailTemplate detail = new Api.Dataset.Connector.ClassicTemplate.ClassicDetailTemplate(); search.AddColumn("Detail", "<a href=\"{{ fn_DatasetItemUrl item.id }}\">Detail</a>"); foreach (var col in model.Columns) { if (col.NormalizedName().ToLower() != "id") { if (col.ShowSearchFormat == "price") { search.AddColumn(col.NiceName, "{{ fn_FormatPrice item." + col.NormalizedName() + " }}"); } else if (col.ShowSearchFormat == "show") { if (col.ValType == "number") { search.AddColumn(col.NiceName, "{{ fn_FormatNumber item." + col.NormalizedName() + " }}"); } else if (col.ValType == "datetime") { search.AddColumn(col.NiceName, "{{ fn_FormatDate item." + col.NormalizedName() + " }}"); } else if (col.ValType == "ico") { search.AddColumn(col.NiceName, "{{ fn_RenderCompanyWithLink item." + col.NormalizedName() + " }}"); } else if (col.ValType == "url") { search.AddColumn(col.NiceName, "<a href='{{ item." + col.NormalizedName() + " }}' target='_blank'>Odkaz</a>"); } else { search.AddColumn(col.NiceName, "{{ item." + col.NormalizedName() + " }}"); } } } if (col.ShowDetailFormat == "price") { detail.AddColumn(col.NiceName, "{{ fn_FormatPrice item." + col.NormalizedName() + " }}"); } else if (col.ShowDetailFormat == "show") { if (col.ValType == "number") { detail.AddColumn(col.NiceName, "{{ fn_FormatNumber item." + col.NormalizedName() + " }}"); } else if (col.ValType == "datetime") { detail.AddColumn(col.NiceName, "{{ fn_FormatDate item." + col.NormalizedName() + " }}"); } else if (col.ValType == "ico") { detail.AddColumn(col.NiceName, "{{ fn_RenderCompanyWithLink item." + col.NormalizedName() + " }}"); } else if (col.ValType == "url") { detail.AddColumn(col.NiceName, "<a href='{{ item." + col.NormalizedName() + " }}' target='_blank'>Odkaz</a>"); } else { detail.AddColumn(col.NiceName, "{{ item." + col.NormalizedName() + " }}"); } } } reg.detailTemplate = new Registration.Template() { body = detail.Body }; reg.searchResultTemplate = new Registration.Template() { body = search.Body }; if (DataSet.ExistsDataset(reg.datasetId)) { reg.datasetId = reg.datasetId + "-" + Devmasters.TextUtil.GenRandomString(5); } datasetIndexStatCache.Invalidate(); var status = DataSet.Api.Create(reg, email); if (status.valid == false) { if (DataSet.ExistsDataset((status.value?.ToString() ?? ""))) { DataSetDB.Instance.DeleteRegistration(status.value?.ToString(), email); } ViewBag.ApiResponseError = status; return(View(model)); } model.DatasetId = ((DataSet)status.value).DatasetId; model.Save(pathModels); return(RedirectToAction("createSimple3", model)); }