public ActionResult CreateSimple3(Guid?fileId)
{
var email = Request?.RequestContext?.HttpContext?.User?.Identity?.Name;
if (Request.IsAuthenticated == false)
{
//https://www.hlidacstatu.cz/account/Login?returnUrl=%2F%3Frnd%3D0036bd9be9bc42d4bdf449492968846e
return(RedirectToAction("Login", "Account", new { returnUrl = this.Request.Url.PathAndQuery }));
}
if (!fileId.HasValue)
{
return(RedirectToAction("CreateSimple"));
}
var uTmp = new Lib.IO.UploadedTmpFile();
var path = uTmp.GetFullPath(fileId.ToString(), fileId.ToString() + ".csv");
var pathJson = uTmp.GetFullPath(fileId.ToString(), fileId.ToString() + ".json");
if (!System.IO.File.Exists(path))
{
return(RedirectToAction("CreateSimple"));
}
if (!System.IO.File.Exists(pathJson))
{
return(RedirectToAction("CreateSimple"));
}
CreateSimpleModel model = CreateSimpleModel.Load(pathJson);
if (model.NumOfRows == 0)
{
using (System.IO.StreamReader r = new System.IO.StreamReader(path))
{
var csv = new CsvHelper.CsvReader(r, new CsvHelper.Configuration.CsvConfiguration(Util.Consts.csCulture)
{
HasHeaderRecord = true, Delimiter = model.GetValidDelimiter()
});
csv.Read(); csv.ReadHeader();
while (csv.Read())
{
model.NumOfRows++;
}
}
}
model.Save(pathJson);
return(View(model));
}
public ActionResult ImportDataProcess(string id, CreateSimpleModel model, FormCollection form)
{
var email = Request?.RequestContext?.HttpContext?.User?.Identity?.Name;
if (Request.IsAuthenticated == false)
{
//https://www.hlidacstatu.cz/account/Login?returnUrl=%2F%3Frnd%3D0036bd9be9bc42d4bdf449492968846e
return(RedirectToAction("Login", "Account", new { returnUrl = this.Request.Url.PathAndQuery }));
}
ViewBag.NumOfRows = 0;
model.DatasetId = id;
string[] csvHeaders = null;
if (string.IsNullOrEmpty(id))
{
return(Redirect("/data"));
}
var ds = DataSet.CachedDatasets.Get(id);
if (ds == null)
{
return(Redirect("/data"));
}
if (ds.HasAdminAccess(email) == false)
{
return(View("NoAccess"));
}
datasetIndexStatCache.Invalidate();
if (ds.IsFlatStructure() == false)
{
return(RedirectToAction("ImportData", new { id = ds.DatasetId, fileId = model.FileId, delimiter = model.GetValidDelimiter() }));
}
var uTmp = new Lib.IO.UploadedTmpFile();
var path = uTmp.GetFullPath(model.FileId.ToString(), model.FileId.ToString() + ".csv");
if (!System.IO.File.Exists(path))
{
return(RedirectToAction("ImportData", new { id = ds.DatasetId }));
}
RuntimeClassBuilder rcb = new RuntimeClassBuilder(ds.GetPropertyNamesTypesFromSchema().ToDictionary(m => m.Key, v => v.Value.Type));
string[] formsHeaders = form["sheaders"].Split('|');
List <MappingCSV> mappingProps = new List <MappingCSV>();
for (int i = 0; i < formsHeaders.Length + 3; i++) //+3 a little bit more, at least +1 for id column
{
if (!string.IsNullOrEmpty(form["source_" + i]) &&
!string.IsNullOrEmpty(form["target_" + i]) &&
!string.IsNullOrEmpty(form["transform_" + i])
)
{
mappingProps.Add(new MappingCSV()
{
sourceCSV = form["source_" + i],
TargetJSON = form["target_" + i],
Transform = form["transform_" + i]
}
);
}
}
System.Collections.Concurrent.ConcurrentBag <Exception> errors = new System.Collections.Concurrent.ConcurrentBag <Exception>();
List <Tuple <object, string> > items = new List <Tuple <object, string> >();
try
{
using (System.IO.StreamReader r = new System.IO.StreamReader(path))
{
if (model.Delimiter == "auto")
{
model.Delimiter = Devmasters.IO.IOTools.DetectCSVDelimiter(r);
}
var csv = new CsvHelper.CsvReader(r, new CsvHelper.Configuration.CsvConfiguration(Util.Consts.csCulture)
{
HasHeaderRecord = true
,
Delimiter = model.GetValidDelimiter()
//,MissingFieldFound = null
}
);
csv.Read(); csv.ReadHeader();
csvHeaders = csv.HeaderRecord.Where(m => !string.IsNullOrEmpty(m?.Trim())).ToArray(); //for future control
while (csv.Read())
{
var newObj = rcb.CreateObject();
for (int m = 0; m < mappingProps.Count; m++)
{
Type destType = ds.GetPropertyNameTypeFromSchema(mappingProps[m].TargetJSON).FirstOrDefault().Value.Type;
object value = null;
string[] specialValues = new string[] { "-skip-", "-gen-", "--" };
if (specialValues.Contains(mappingProps[m].sourceCSV))
{
if (mappingProps[m].sourceCSV == "-gen-")
{
value = Guid.NewGuid().ToString("N");
rcb.SetPropertyValue(newObj, mappingProps[m].TargetJSON, value);
}
else
{
continue; // -skip- skip
}
}
else
{
string svalue = null;
try
{
svalue = csv.GetField(mappingProps[m].sourceCSV);
if (destType == typeof(string))
{
value = svalue;
}
else if (destType == typeof(DateTime) || destType == typeof(DateTime?))
{
value = Devmasters.DT.Util.ToDateTime(svalue);
}
else if (destType == typeof(decimal) || destType == typeof(decimal?))
{
value = Util.ParseTools.ToDecimal(svalue);
if (value == null)
{
value = Util.ParseTools.FromTextToDecimal(svalue);
}
}
else if (destType == typeof(long) || destType == typeof(long?) ||
destType == typeof(int) || destType == typeof(int?))
{
value = Devmasters.DT.Util.ToDate(svalue);
}
else if (destType == typeof(bool) || destType == typeof(bool?))
{
if (bool.TryParse(svalue, out bool tryp))
{
value = tryp;
}
}
else
{
value = svalue;
}
if (mappingProps[m].Transform == "normalize" &&
destType == typeof(string)
)
{
value = DataSet.NormalizeValueForId((string)value);
}
else if (mappingProps[m].Transform == "findico" &&
destType == typeof(string)
)
{
value = Lib.Validators.IcosInText((string)value).FirstOrDefault();
}
else //copy
{
}
rcb.SetPropertyValue(newObj, mappingProps[m].TargetJSON, value);
}
catch (Exception mex)
{
errors.Add(mex);
}
}
} //for
string idPropName = "id";
string idVal = rcb.GetPropertyValue(newObj, "id")?.ToString();
if (string.IsNullOrEmpty(idVal))
{
idVal = rcb.GetPropertyValue(newObj, "Id")?.ToString();
idPropName = "Id";
}
if (string.IsNullOrEmpty(idVal))
{
idVal = rcb.GetPropertyValue(newObj, "iD")?.ToString();
idPropName = "iD";
}
if (string.IsNullOrEmpty(idVal))
{
idVal = rcb.GetPropertyValue(newObj, "ID")?.ToString();
idPropName = "ID";
}
try
{
//var debugJson = Newtonsoft.Json.JsonConvert.SerializeObject(newObj);
//normalize ID
idVal = DataSet.NormalizeValueForId(idVal);
rcb.SetPropertyValue(newObj, idPropName, idVal);
items.Add(new Tuple <object, string>(newObj, idVal));
model.NumOfRows++;
}
catch (DataSetException dex)
{
errors.Add(dex);
}
catch (Exception ex)
{
errors.Add(ex);
}
}
}
}
catch (Exception ex)
{
errors.Add(ex);
}
try
{
Devmasters.Batch.Manager.DoActionForAll <Tuple <object, string> >(items,
(item) =>
{
try
{
ds.AddData(item.Item1, item.Item2, email, true);
}
catch (Exception ex)
{
errors.Add(ex);
}
return(new Devmasters.Batch.ActionOutputData());
}, true
);
}
catch (Exception ex)
{
errors.Add(ex);
}
if (errors?.Count > 0)
{
HlidacStatu.Util.Consts.Logger.Error("ImportDataProcess exceptions \n"
+ errors.Select(m => m.Message).Aggregate((f, s) => f + "\n" + s));
}
ViewBag.ApiResponseError = ApiResponseStatus.Error(-99, "Chyba při importu dat");
ViewBag.Errors = errors.ToList();
return(View(model));
}
public ActionResult ImportData(string id, string delimiter, string data, FormCollection form, HttpPostedFileBase file)
{
var email = Request?.RequestContext?.HttpContext?.User?.Identity?.Name;
if (Request.IsAuthenticated == false)
{
//https://www.hlidacstatu.cz/account/Login?returnUrl=%2F%3Frnd%3D0036bd9be9bc42d4bdf449492968846e
return(RedirectToAction("Login", "Account", new { returnUrl = this.Request.Url.PathAndQuery }));
}
if (string.IsNullOrEmpty(id))
{
return(Redirect("/data"));
}
var ds = DataSet.CachedDatasets.Get(id);
if (ds == null)
{
return(Redirect("/data"));
}
if (ds.HasAdminAccess(email) == false)
{
return(View("NoAccess"));
}
datasetIndexStatCache.Invalidate();
Guid fileId = Guid.NewGuid();
var uTmp = new Lib.IO.UploadedTmpFile();
if (string.IsNullOrEmpty(data))
{
if (file == null)
{
ViewBag.ApiResponseError = ApiResponseStatus.Error(-99, "Źádné CSV jste nenahráli");
return(View());
}
else
{
var path = uTmp.GetFullPath(fileId.ToString(), fileId.ToString() + ".csv");
file.SaveAs(path);
return(RedirectToAction("ImportData", new { id = ds.DatasetId, fileId = fileId, delimiter = CreateSimpleModel.GetValidDelimiter(delimiter) }));
}
}
else
{
var path = uTmp.GetFullPath(fileId.ToString(), fileId.ToString() + ".csv");
System.IO.File.WriteAllText(path, data);
return(RedirectToAction("ImportData", new { id = ds.DatasetId, fileId = fileId, delimiter = "auto" }));
}
}
public ActionResult ImportData(string id, CreateSimpleModel model)
{
var email = Request?.RequestContext?.HttpContext?.User?.Identity?.Name;
if (Request.IsAuthenticated == false)
{
//https://www.hlidacstatu.cz/account/Login?returnUrl=%2F%3Frnd%3D0036bd9be9bc42d4bdf449492968846e
return(RedirectToAction("Login", "Account", new { returnUrl = this.Request.Url.PathAndQuery }));
}
model = model ?? new CreateSimpleModel();
model.DatasetId = id;
if (string.IsNullOrEmpty(id))
{
return(Redirect("/data"));
}
var ds = DataSet.CachedDatasets.Get(id);
if (ds == null)
{
return(Redirect("/data"));
}
if (ds.HasAdminAccess(email) == false)
{
return(View("NoAccess"));
}
model.DatasetId = ds.DatasetId;
if (ds.IsFlatStructure() == false)
{
ViewBag.Mode = "notflat";
return(View("ImportData.noflat", model));
}
if (model.FileId.HasValue)
{
ViewBag.Mode = "mapping";
Guid fileId = model.FileId.Value;
var uTmp = new Lib.IO.UploadedTmpFile();
var path = uTmp.GetFullPath(fileId.ToString(), fileId.ToString() + ".csv");
if (!System.IO.File.Exists(path))
{
return(RedirectToAction("ImportData", new { id = model.DatasetId }));
}
using (System.IO.StreamReader r = new System.IO.StreamReader(path))
{
if (model.Delimiter == "auto")
{
model.Delimiter = Devmasters.IO.IOTools.DetectCSVDelimiter(r);
}
var csv = new CsvHelper.CsvReader(r, new CsvHelper.Configuration.CsvConfiguration(Util.Consts.csCulture)
{
HasHeaderRecord = true, Delimiter = model.GetValidDelimiter()
});
csv.Read(); csv.ReadHeader();
model.Headers = csv.HeaderRecord.Where(m => !string.IsNullOrEmpty(m?.Trim())).ToArray();
}
return(View("ImportData.mapping", model));
}
else
{
ViewBag.Mode = "upload";
}
return(View(model));
}
public ActionResult CreateSimple2(CreateSimpleModel model)
{
var email = Request?.RequestContext?.HttpContext?.User?.Identity?.Name;
if (Request.IsAuthenticated == false)
{
return(RedirectToAction("Login", "Account", new { returnUrl = this.Request.Url.PathAndQuery }));
}
var uTmp = new Lib.IO.UploadedTmpFile();
var path = uTmp.GetFullPath(model.FileId.ToString(), model.FileId.ToString() + ".csv");
if (!System.IO.File.Exists(path))
{
return(RedirectToAction("CreateSimple"));
}
try
{
using (System.IO.StreamReader r = new System.IO.StreamReader(path))
{
var config = new CsvHelper.Configuration.CsvConfiguration(Util.Consts.csCulture);
config.HasHeaderRecord = true;
config.Delimiter = model.GetValidDelimiter();
var csv = new CsvHelper.CsvReader(r, config);
csv.Read(); csv.ReadHeader();
model.Headers = csv.HeaderRecord.Where(m => !string.IsNullOrEmpty(m?.Trim())).ToArray();
//read first lines with data and guest type
List <string[]> lines = new List <string[]>();
for (int row = 0; row < 10; row++)
{
if (csv.Read())
{
lines.Add(csv.GetRecords <string>().ToArray());
}
}
List <string> colTypes = null;
if (lines.Count > 0)
{
colTypes = new List <string>();
for (int cx = 0; cx < lines[0].Length; cx++)
{
string t = GuestBestCSVValueType(lines[0][cx]);
for (int line = 1; line < lines.Count; line++)
{
var nextT = GuestBestCSVValueType(lines[line][cx]);
if (nextT != t)
{
t = "string"; //kdyz jsou ruzne typy ve stejnem sloupci v ruznych radcich,
}
//fallback na string
}
colTypes.Add(t);
}
}
ViewBag.ColTypes = colTypes.ToArray();
return(View(model));
}
}
catch (Exception e)
{
ViewBag.ApiResponseError = ApiResponseStatus.Error(-99, "Soubor není ve formátu CSV.", e.ToString());
return(View(model));
}
}
public ActionResult CreateSimple(string name, string delimiter, FormCollection form, HttpPostedFileBase file)
{
var email = Request?.RequestContext?.HttpContext?.User?.Identity?.Name;
if (Request.IsAuthenticated == false)
{
return(RedirectToAction("Login", "Account", new { returnUrl = this.Request.Url.PathAndQuery }));
}
Guid fileId = Guid.NewGuid();
var uTmp = new Lib.IO.UploadedTmpFile();
if (string.IsNullOrEmpty(name))
{
ViewBag.ApiResponseError = ApiResponseStatus.Error(-99, "Bez jména datasetu nemůžeme pokračovat. Prozraďte nám ho, prosím.");
return(View());
}
if (file == null)
{
ViewBag.ApiResponseError = ApiResponseStatus.Error(-99, "Źádné CSV jste nenahráli");
return(View());
}
else
{
var path = uTmp.GetFullPath(fileId.ToString(), fileId.ToString() + ".csv");
file.SaveAs(path);
return(RedirectToAction("CreateSimple2", new { fileId = fileId, delimiter = CreateSimpleModel.GetValidDelimiter(delimiter), name = name }));
}
}