public FileImportException(string message, ImportError error, ImportFileType fileType, string fileName)
{
Message = message;
Error = error;
FileType = fileType;
FileName = fileName;
}
public DialogResult ShowDialog(string filePath, string searchDir, ImportError error, ImportFileType fileType, bool criticalError)
{
bool isUrl = filePath.ToLower().StartsWith("http");
findInFolderBtn.Visible = !isUrl;
folderDialog.InitialDirectory = searchDir;
string fileTypeString = fileType == ImportFileType.Note ? "Note" : "Audio";
if (error == ImportError.Missing)
{
errorLabel.Text = $"{fileTypeString} file missing: ";
}
else
{
errorLabel.Text = $"Invalid {fileTypeString.ToLower()} file format: ";
}
cancelBtn.Text = criticalError ? "Cancel" : "Ignore";
filePathTb.Text = filePath;
FilePath = filePath;
if (isUrl)
{
fileDialog.InitialDirectory = searchDir;
fileDialog.FileName = "";
}
else
{
fileDialog.InitialDirectory = Path.GetDirectoryName(FilePath);
fileDialog.FileName = FilePath;
}
return(base.ShowDialog());
}
public (Photo, ImportError) ReadPhoto(string filePath, bool readMetaData, bool readDateTimeFromFile)
{
ImportError readError = null;
Photo photo = null;
try
{
photo = Photo.Create(filePath);
if (readMetaData)
{
photo.ImageMetaData = ReadPhotoMetaData(filePath);
}
if (readDateTimeFromFile)
{
photo.ReadDateTimeFromFile();
}
}
catch (Exception exception)
{
readError = new ImportError(filePath, ImportErrorType.Read, exception);
}
return(photo, readError);
}
private static void HandleRequiredValueError(Action <ImportProgressInfo> progressCallback, ImportProgressInfo importProgress, ICsvCustomerImportReporter reporter, ReadingContext context, ImportErrorsContext errorsContext)
{
var fieldName = context.HeaderRecord[context.CurrentIndex];
var requiredFields = CsvCustomerImportHelper.GetImportCustomerRequiredColumns <ImportableContact>();
var missedValueColumns = new List <string>();
for (var i = 0; i < context.HeaderRecord.Length; i++)
{
if (requiredFields.Contains(context.HeaderRecord[i], StringComparer.InvariantCultureIgnoreCase) && context.Record[i].IsNullOrEmpty())
{
missedValueColumns.Add(context.HeaderRecord[i]);
}
}
var importError = new ImportError {
Error = $"The required value in column {fieldName} is missing.", RawRow = context.RawRecord
};
if (missedValueColumns.Count > 1)
{
importError.Error = $"The required values in columns: {string.Join(", ", missedValueColumns)} - are missing.";
}
reporter.WriteAsync(importError).GetAwaiter().GetResult();
errorsContext.ErrorsRows.Add(context.Row);
HandleError(progressCallback, importProgress);
}
public ImportError ImportErrorSave(ImportError importError)
{
ImportError retVal = null;
string procName = "dbo.ImportQueue_Edit";
using (IDbConnection conn = DbHelper.GetDBConnection())
{
try
{
var para = new DynamicParameters();
para.Add("@ImportQueueId", importError.ImportQueueId, DbType.Int32);
para.Add("@Error", dbType: DbType.String, size: -1);
conn.Execute(procName, para, commandType: CommandType.StoredProcedure);
retVal = importError;
}
catch (Exception ex)
{
Log.Error(ex);
throw;
}
}
return(retVal);
}
private async Task <ImportError> DeactivateException(ImportError exception)
{
var filter = ImportQueueFilter.FromExceptionId(exception.FdpImportErrorId);
filter.IsActive = false;
return(await DataContext.Import.SaveException(filter));
}
public void Write(ImportError error)
{
lock (_lock)
{
ReportIsNotEmpty = true;
_streamWriter.WriteLine(GetLine(error));
}
}
public async Task WriteAsync(ImportError error)
{
using (await AsyncLock.GetLockByKey(_filePath).LockAsync())
{
ReportIsNotEmpty = true;
await _streamWriter.WriteLineAsync(GetLine(error));
}
}
private static async Task HandleBadDataErrorAsync(Action <ImportProgressInfo> progressCallback, ImportProgressInfo importProgress, ICsvCustomerImportReporter reporter, ReadingContext context, ImportErrorsContext errorsContext)
{
var importError = new ImportError {
Error = "This row has invalid data. The data after field with not escaped quote was lost.", RawRow = context.RawRecord
};
await reporter.WriteAsync(importError);
errorsContext.ErrorsRows.Add(context.Row);
HandleError(progressCallback, importProgress);
}
private static void HandleNotClosedQuoteError(Action <ImportProgressInfo> progressCallback, ImportProgressInfo importProgress, ICsvCustomerImportReporter reporter, ReadingContext context, ImportErrorsContext errorsContext)
{
var importError = new ImportError {
Error = "This row has invalid data. Quotes should be closed.", RawRow = context.RawRecord
};
reporter.WriteAsync(importError).GetAwaiter().GetResult();
errorsContext.ErrorsRows.Add(context.Row);
HandleError(progressCallback, importProgress);
}
private static void HandleNotClosedQuoteError(Action <ImportProgressInfo> progressCallback, ImportProgressInfo importProgress, ICsvPriceImportReporter reporter, CsvContext context, ImportErrorsContext errorsContext)
{
var importError = new ImportError {
Error = "This row has invalid data. Quotes should be closed.", RawRow = context.Parser.RawRecord
};
reporter.Write(importError);
errorsContext.ErrorsRows.Add(context.Parser.Row);
HandleError(progressCallback, importProgress);
}
private static void HandleWrongValueError(Action <ImportProgressInfo> progressCallback, ImportProgressInfo importProgress, ICsvPriceImportReporter reporter, CsvContext context, ImportErrorsContext errorsContext)
{
var invalidFieldName = context.Reader.HeaderRecord[context.Reader.CurrentIndex];
var importError = new ImportError {
Error = $"This row has invalid value in the column {invalidFieldName}.", RawRow = context.Parser.RawRecord
};
reporter.Write(importError);
errorsContext.ErrorsRows.Add(context.Parser.Row);
HandleError(progressCallback, importProgress);
}
private static void HandleWrongValueError(Action <ImportProgressInfo> progressCallback, ImportProgressInfo importProgress, ICsvCustomerImportReporter reporter, ReadingContext context, ImportErrorsContext errorsContext)
{
var invalidFieldName = context.HeaderRecord[context.CurrentIndex];
var importError = new ImportError {
Error = string.Format(ModuleConstants.ValidationMessages[ModuleConstants.ValidationErrors.InvalidValue], invalidFieldName), RawRow = context.RawRecord
};
reporter.WriteAsync(importError).GetAwaiter().GetResult();
errorsContext.ErrorsRows.Add(context.Row);
HandleError(progressCallback, importProgress);
}
public async void ImportFromWebService()
{
var apiurl = Properties.Settings.Default.pmis_api_url;
var project = Properties.Settings.Default.pmis_project_code;
var authkey = Properties.Settings.Default.pmis_auth_key;
string url = String.Format("{0}/api/register/docs.action", apiurl);
try
{
using (var client = new HttpClient())
{
var values = new Dictionary <string, string> {
{ "forward", "json" },
{ "srch_show_hist", "1" },
{ "pjt_cd", project },
{ "access_token", authkey },
{ "pageScale", "200" },
{ "pageNo", "1" },
{ "login_locale", "ko_KR" }
};
var page = 1;
var total = 999;
while (page <= total)
{
values["pageNo"] = "" + page;
var content = new FormUrlEncodedContent(values);
var response = await client.PostAsync(url, content);
response.EnsureSuccessStatusCode();
var responseString = await response.Content.ReadAsStringAsync();
PmisJsonResponse <RegisterDocument> dt = JsonConvert.DeserializeObject <PmisJsonResponse <RegisterDocument> >(responseString);
ImportData(dt.List);
page = dt.PageInfo.CurrentPage + 1;
total = dt.PageInfo.TotalPages;
LogUtil.Log(dt.ToString());
}
}
}
catch (Exception ex)
{
ex.Log().Display();
ImportError?.Invoke(this, new ErrorEventArgs(ex));
}
finally
{
ImportComplete?.Invoke(this, EventArgs.Empty);
}
}
public static DataSet ExcelImport(string strFileName, Tk5ListMetaData metaInfos, ImportError importError)
{
DataSet dataSet = new DataSet();
DataTable dataTable = DataSetUtil.CreateDataTable(metaInfos.Table.TableName, metaInfos.Table.TableList);
dataTable.Columns.Add("OriginalRowNum", typeof(int));
IWorkbook workbook ;
ISheet sheet;
ExcelUtil.ReadExcle(strFileName, metaInfos.Table.TableDesc, out workbook, out sheet);
SheetImport(metaInfos, dataTable, sheet, importError);
dataSet.Tables.Add(dataTable);
return dataSet;
}
private static void HandleMissedColumnError(Action <ImportProgressInfo> progressCallback, ImportProgressInfo importProgress, ICsvPriceImportReporter reporter, CsvContext context, ImportErrorsContext errorsContext)
{
var headerColumns = context.Reader.HeaderRecord;
var recordFields = context.Parser.Record;
var missedColumns = headerColumns.Skip(recordFields.Length).ToArray();
var error = $"This row has next missing columns: {string.Join(", ", missedColumns)}.";
var importError = new ImportError {
Error = error, RawRow = context.Parser.RawRecord
};
reporter.Write(importError);
errorsContext.ErrorsRows.Add(context.Parser.Row);
HandleError(progressCallback, importProgress);
}
public override OutputData DoAction(IInputData input)
{
string source = input.QueryString["Source"];
string path = Path.Combine(BaseAppSetting.Current.XmlPath, @"Import", source + ".xml");
ImportConfigXml config = new ImportConfigXml();
config.ReadXmlFromFile(path);
string filePath = @"C:\Users\zll\Downloads\角色.xls";
var meta = config.Import.MetaData.CreateObject(input);
Tk5ListMetaData data = meta as Tk5ListMetaData;
var resolver = config.Import.Resolver.CreateObject(this);
MetaDataTableResolver metaResolver = resolver as MetaDataTableResolver;
TkDebug.AssertNotNull(metaResolver, "metaResolver", this);
ImportError errResult = new ImportError();
var dataSet = ExcelImporter.ExcelImport(filePath, data, errResult);
FieldErrorInfoCollection importResult = metaResolver.Import(dataSet, input);
if (importResult.Count > 0)
{
var positions = (from item in importResult
orderby item.Position descending
select item.Position).Distinct();
foreach (var errorInfo in importResult)
{
DataRow errorRow = dataSet.Tables[errorInfo.TableName].Rows[errorInfo.Position];
ImportWarningItem errorItem = new ImportWarningItem(errorRow["OriginalRowNum"].Value<int>(),
resolver.GetFieldInfo(errorInfo.NickName).DisplayName, (string)errorRow[errorInfo.NickName], errorInfo.Message);
errResult.Add(errorItem);
}
foreach (var index in positions)
{
resolver.HostTable.Rows.RemoveAt(index);
}
}
DataSet webReport = null;
if (errResult.Count > 0)
{
byte[] dataFile = ExcelUtil.ExcelReport(filePath, data, errResult);
webReport = ExcelUtil.DataSetReport(filePath, data, errResult);
}
return OutputData.CreateObject(new ImportResultData(DataSet, webReport, errResult));
}
public async Task ImportAsync_BadDataFoundDuringImport_WillReportError(string row)
{
// Arrange
var request = TestHelper.CreateImportDataRequest(ImportMode.CreateAndUpdate);
var cancellationTokenWrapper = GetCancellationTokenWrapper();
var progressInfos = new List <ImportProgressInfo>();
void ProgressCallback(ImportProgressInfo progressInfo)
{
progressInfos.Add((ImportProgressInfo)progressInfo.Clone());
}
var invalidRows = new[] { row };
var errorReporterStream = new MemoryStream();
var importReporterFactoryMock = new Mock <ICsvPriceImportReporterFactory>();
var importReporterMock = new Mock <ICsvPriceImportReporter>();
ImportError errorForAssertion = null;
importReporterMock.Setup(x => x.Write(It.IsAny <ImportError>()))
.Callback <ImportError>(error => errorForAssertion = error);
importReporterFactoryMock.Setup(x => x.CreateAsync(It.IsAny <string>(), It.IsAny <string>()))
.ReturnsAsync(importReporterMock.Object);
var allRows = validRows.Union(invalidRows).ToArray();
var importer = GetCsvPagedPriceDataImporter(GetBlobStorageProvider(CsvHeader, allRows, errorReporterStream), importReporterFactoryMock.Object);
// Act
await importer.ImportAsync(request, ProgressCallback, cancellationTokenWrapper);
// Assert
var errorProgressInfo = progressInfos.LastOrDefault();
Assert.Equal(validRows.Length + invalidRows.Length, errorProgressInfo?.ProcessedCount);
Assert.Equal(validRows.Length, errorProgressInfo?.CreatedCount);
Assert.Equal(0, errorProgressInfo?.UpdatedCount);
Assert.Equal(invalidRows.Length, errorProgressInfo?.ErrorCount);
Assert.NotNull(errorProgressInfo?.Description);
Assert.StartsWith("Import completed with errors", errorProgressInfo?.Description);
importReporterMock.Verify(x => x.Write(It.IsAny <ImportError>()), Times.Once);
Assert.Equal("This row has invalid data. The data after field with not escaped quote was lost.", errorForAssertion.Error);
Assert.Equal($"{invalidRows.First()}", errorForAssertion.RawRow.TrimEnd());
}
private async Task <ImportResult> ReProcessException(ImportError exception)
{
if (!ConfigurationSettings.GetBoolean("ReprocessImportAfterHandleError"))
{
return(null);
}
var queuedItem = new ImportQueue()
{
ImportId = exception.FdpImportId,
ImportQueueId = exception.ImportQueueId,
LineNumber = int.Parse(exception.LineNumber)
};
return(await Task.FromResult(DataContext.Import.ReprocessImportQueue(queuedItem)));
}
private static void HandleBadDataError(Action <ImportProgressInfo> progressCallback, ImportProgressInfo importProgress, ICsvPriceImportReporter reporter, CsvContext context, ImportErrorsContext errorsContext)
{
var importError = new ImportError {
Error = "This row has invalid data. The data after field with not escaped quote was lost.", RawRow = context.Parser.RawRecord
};
reporter.Write(importError);
if (context.Reader is VcCsvReader vcCsvReader)
{
vcCsvReader.IsFieldBadData = true;
}
errorsContext.ErrorsRows.Add(context.Parser.Row);
HandleError(progressCallback, importProgress);
}
public override ImportError HandleRequest(string logPath)
{
ImportError ie = Validate();
if (ie.Type == ErrorType.NOERROR)
{
try
{
DataTable dt = ReadData();
string jsonline = toJSONLine(dt);
System.IO.File.WriteAllText(logPath + "/" + this.Context.ID + ".json", jsonline + Environment.NewLine);
this.Context.Status = ImportStatus.pending;
var con = this.Context.Clone <ImportContext>();
con.View = null;
con.ComponentData = new List <ComponentData>();
con.Template.TemplateFields = new List <CompExtention.ImportTemplate.TemplateField>();
con.Template.View = null;
con.DataLocation = "";
Global.SaveImportContext(this.Context.ID, Newtonsoft.Json.JsonConvert.SerializeObject(con), 2,
this.Context.ActionBy,
this.Context.Connection, this.Context.ClientID
);
//Global.SaveImportContext(Newtonsoft.Json.JsonConvert.SerializeObject(this.Context), 2,
// this.Context.ID,
// this.Context.ActionBy,
// logPath
// );
return(new ImportError()
{
Message = "No Error", Type = ErrorType.NOERROR
});
}
catch (Exception Ex) {
throw Ex;
}
}
else
{
return(ie);
}
}
private static void SheetImport(Tk5ListMetaData metaInfos, DataTable dataTable, ISheet sheet,
ImportError importError)
{
if (sheet != null)
{
Dictionary<string, Tk5FieldInfoEx> dicOfInfo = new Dictionary<string, Tk5FieldInfoEx>();
foreach (Tk5FieldInfoEx info in metaInfos.Table.TableList)
{
dicOfInfo.Add(info.DisplayName, info);
}
IRow headerRow = sheet.GetRow(0);
IRow row = null;
string columnName = string.Empty;
string strValue = string.Empty;
ICell cell = null;
for (int i = (sheet.FirstRowNum + 1); i <= sheet.LastRowNum; i++)
{
row = sheet.GetRow(i);
DataRow dataRow = dataTable.NewRow();
bool rowError = false;
for (int j = headerRow.FirstCellNum; j < headerRow.LastCellNum; j++)
{
columnName = headerRow.GetCell(j).ToString();
cell = row.GetCell(j);
strValue = ((cell == null) ? null : cell.ToString());
var imResult = TablePadding(dataRow, columnName, dicOfInfo, strValue, i);
if (imResult != null)
{
importError.Add(imResult);
rowError = true;
}
}
if (!rowError)
{
dataRow["OriginalRowNum"] = i;
dataTable.Rows.Add(dataRow);
}
}
}
}
public bool AddDriverToDriverPack(Driver driver)
{
log.Debug("AddDriverToDriverPack: " + driver.Model);
List <int> contentIDs = new List <int>();
string query = string.Format("SELECT * FROM SMS_CIToContent WHERE CI_ID='{0}'", driver.Object["CI_ID"].IntegerValue);
foreach (IResultObject resultObject in connectionManager.QueryProcessor.ExecuteQuery(query))
{
contentIDs.Add(resultObject["ContentID"].IntegerValue);
}
List <string> packageSources = new List <string>
{
driver.Object["ContentSourcePath"].StringValue
};
Dictionary <string, object> methodParameters = new Dictionary <string, object>
{
{ "bRefreshDPs", false },
{ "ContentIDs", contentIDs.ToArray() },
{ "ContentSourcePath", packageSources.ToArray() }
};
try
{
Package.ExecuteMethod("AddDriverContent", methodParameters);
}
catch (SmsQueryException ex)
{
string str = string.Format("{0} ({1})", driver.Object["LocalizedDisplayName"].StringValue, driver.Object["DriverINFFile"].StringValue);
ManagementException managementException = ex.InnerException as ManagementException;
ImportError.Add(str, "Could not be added to package: " + managementException.ErrorInformation["Description"].ToString());
log.Error("AddDriverToDriverPack: " + managementException.ErrorInformation["Description"].ToString());
return(false);
}
return(true);
}
public bool RemoveDriverFromCategory(IResultObject driverObject)
{
log.Debug("RemoveDriverFromCategory: " + driverObject["LocalizedDisplayName"].StringValue);
// get category unique id
string categoryUniqueID = Category["CategoryInstance_UniqueID"].StringValue;
ArrayList categories = new ArrayList();
ArrayList removeCategory = new ArrayList();
// check if driver has any category
if (driverObject["CategoryInstance_UniqueIDs"].StringArrayValue != null)
{
// check if category is in driver categories list
categories.AddRange(driverObject["CategoryInstance_UniqueIDs"].StringArrayValue);
foreach (string str in categories)
{
// if category exist add it to list for removal
if (str != categoryUniqueID)
{
removeCategory.Add(str);
}
}
}
// remove category from driver
driverObject["CategoryInstance_UniqueIDs"].StringArrayValue = (string[])removeCategory.ToArray(typeof(string));
try
{
driverObject.Put();
driverObject.Get();
}
catch (SmsQueryException ex)
{
string str = string.Format("{0} ({1})", driverObject["LocalizedDisplayName"].StringValue, driverObject["DriverINFFile"].StringValue);
ManagementException managementException = ex.InnerException as ManagementException;
ImportError.Add(str, "Cannot be removed from category: " + managementException.ErrorInformation["Description"].ToString());
log.Error("RemoveDriverFromCategory: " + managementException.ErrorInformation["Description"].ToString());
return(false);
}
return(true);
}
public bool AddDriverToCategory(Driver driver)
{
log.Debug("AddDriverToCategory: " + driver.Model);
// get category unique id
string categoryUniqueID = Category["CategoryInstance_UniqueID"].StringValue;
ArrayList categories = new ArrayList();
// check if driver has any category
if (driver.Object["CategoryInstance_UniqueIDs"].StringArrayValue != null)
{
// check if category is in driver categories list
categories.AddRange(driver.Object["CategoryInstance_UniqueIDs"].StringArrayValue);
foreach (string str in categories)
{
// if category exist return
if (str == categoryUniqueID)
{
return(true);
}
}
}
// add category to driver
categories.Add(categoryUniqueID);
driver.Object["CategoryInstance_UniqueIDs"].StringArrayValue = (string[])categories.ToArray(typeof(string));
try
{
driver.Object.Put();
driver.Object.Get();
}
catch (SmsQueryException ex)
{
string str = string.Format("{0} ({1})", driver.Object["LocalizedDisplayName"].StringValue, driver.Object["DriverINFFile"].StringValue);
ManagementException managementException = ex.InnerException as ManagementException;
ImportError.Add(str, "Cannot be added to category: " + managementException.ErrorInformation["Description"].ToString());
log.Error("AddDriverToCategory: " + managementException.ErrorInformation["Description"].ToString());
return(false);
}
return(true);
}
public override ImportError HandleRequest(string logPath)
{
ImportError ie = Validate();
if (ie.Type == ErrorType.NOERROR)
{
this.Context.LoadComponentView();
if (this.Context.ID == null)
{
this.Context.ID = Shared.generateID();
}
this.Context.Status = ImportStatus.pending;
var con = this.Context.Clone <ImportContext>();
con.View = null;
con.ComponentData = new List <ComponentData>();
con.Template.TemplateFields = new List <CompExtention.ImportTemplate.TemplateField>();
con.Template.View = null;
con.DataLocation = "";
Global.SaveImportContext(this.Context.ID, Newtonsoft.Json.JsonConvert.SerializeObject(con), 1,
this.Context.ActionBy,
this.Context.Connection, this.Context.ClientID
);
return(new ImportError()
{
Message = "No Error", Type = ErrorType.NOERROR
});
}
else
{
return(ie);
}
}
protected async Task <ValidationResult> ValidateAsync(ImportRecord <TCsvMember>[] importRecords, ICsvCustomerImportReporter importReporter)
{
var validationResult = await _importRecordsValidator.ValidateAsync(importRecords);
var errorsInfos = validationResult.Errors.Select(x => new { Message = x.ErrorMessage, (x.CustomState as ImportValidationState <TCsvMember>)?.InvalidRecord }).ToArray();
// We need to order by row number because otherwise records will be written to report in random order
var errorsGroups = errorsInfos.OrderBy(x => x.InvalidRecord.Row).GroupBy(x => x.InvalidRecord);
foreach (var group in errorsGroups)
{
var importPrice = group.Key;
var errorMessages = string.Join(" ", group.Select(x => x.Message).ToArray());
var importError = new ImportError {
Error = errorMessages, RawRow = importPrice.RawRecord
};
await importReporter.WriteAsync(importError);
}
return(validationResult);
}
public static DataSet DataSetReport(string strFileName, Tk5ListMetaData metaInfos, ImportError importError)
{
DataSet dataSet = new DataSet();
DataTable dataTable = CreateDataTable(metaInfos.Table.TableName, metaInfos.Table.TableList);
dataTable.Columns.Add(ORIGINAL_ROWNUM, typeof(int));
IWorkbook workbook;
ISheet sheet;
string columnName = string.Empty;
ReadExcle(strFileName, metaInfos.Table.TableDesc, out workbook, out sheet);
var positions = (from index in importError
orderby index.IndexOfRow ascending
select index.IndexOfRow).Distinct();
IRow headerRow = sheet.GetRow(0);
foreach (var index in positions)
{
IRow row = sheet.GetRow(index);
DataRow dataRow = dataTable.NewRow();
for (int i = headerRow.FirstCellNum; i < headerRow.LastCellNum; i++)
{
dataRow[headerRow.GetCell(i).ToString()] = row.GetCell(i);
dataRow[ORIGINAL_ROWNUM] = index;
}
dataTable.Rows.Add(dataRow);
}
dataSet.Tables.Add(dataTable);
return dataSet;
}
public static byte[] ExcelReport(string strFileName, Tk5ListMetaData metaInfos, ImportError importError)
{
MemoryStream ms = new MemoryStream();
ExcelExporter exporter =
new ExcelExporter(false, ExcelContentFormat.DefaultHeader, ExcelContentFormat.DefaultContent, metaInfos);
using (ms)
{
IWorkbook workBook;
ISheet sheet;
ReadExcle(strFileName, metaInfos.Table.TableDesc, out workBook, out sheet);
PostilAdd(metaInfos, importError, workBook, sheet);
RowFilter(importError, sheet);
workBook.Write(ms);
byte[] filedata = ms.ToArray();
ms.Flush();
using (FileStream fs = new FileStream(@"C:\Users\zll\Downloads\ImportReport.xls", FileMode.Create))
{
BinaryWriter bw = new BinaryWriter(fs);
bw.Write(filedata);
bw.Close();
fs.Close();
}
return filedata;
}
}
private void DoSales
(
ExcelWorksheet worksheet,
Dictionary <string, string> colDic,
MasterMap mMap,
DTO.ImportPost ip
)
{
int iStart = worksheet.Dimension.Start.Row;
iStart = 2;
int iEnd = GetLastRow(worksheet);
if (iEnd > 0)
{
for (int i = iStart; i <= iEnd; i++)
{
var row = worksheet.Row(i);
string warnings = "";
string requires = "";
var rec = new Sale();
foreach (string key in colDic.Keys)
{
try
{
var els = colDic[key].Split('|');
int ix = int.Parse(els[0]);
string val = worksheet.GetValue(i, ix).ToString().Trim();
string pattern = els[1];
if (!String.IsNullOrEmpty(pattern))
{
Regex regex = new Regex(pattern);
Match match = regex.Match(val);
if (match.Success)
{
val = match.Groups[1].Value;
}
}
var colSpecs = mMap.Columns.First(o => o.ColumnName == key);
if (!String.IsNullOrEmpty(val))
{
dynamic realVal = null;
switch (colSpecs.ColumnDataType)
{
case 1:
realVal = val;
break;
case 2:
val = Regex.Replace(val, "[^0-9.]", "");
var ppos = val.IndexOf('.');
if (ppos != -1)
{
val = val.Substring(0, ppos - 1);
}
realVal = Convert.ToInt64(val);
break;
case 3:
val = Regex.Replace(val, "[^0-9.]", "");
realVal = Convert.ToDouble(val);
break;
case 4:
DateTime realDT;
if (DateTime.TryParse(val, out realDT))
{
realVal = realDT;
}
if (realVal == null)
{
long ldate = long.Parse(val);
realVal = DateTime.FromOADate(ldate);
}
break;
case 5:
realVal = OrmHelper.IsTrue(val);
break;
case 6:
val = Regex.Replace(val, "[^0-9.]", "");
realVal = Decimal.Parse(val);
break;
}
if (realVal != null)
{
OrmHelper.SetPropertyValue <Sale>(ref rec, key, realVal);
}
}
}
catch
{
warnings += $"'{key}',";
}
}
bool isFubar = false;
foreach (var col in mMap.Columns)
{
if (col.Required)
{
dynamic realVal = OrmHelper.GetPropertyValue <Sale>(rec, col.ColumnName);
if (realVal == null)
{
requires += $"'{col.ColumnName}',";
isFubar = true;
}
}
}
if (!isFubar)
{
rec.ImportId = ip.ImportId;
rec.Row = i;
rec.Sheet = ip.SheetNo;
OrmHelper.SetAuditColumns <Sale>(ref rec, true, ip.PersonId);
_db.Sale.Add(rec);
}
if (isFubar || warnings.Length > 0)
{
if (isFubar)
{
ip.SkipRows++;
}
if (warnings.Length > 0)
{
ip.WarnRows++;
}
string errInfo = "{";
errInfo += "warns:[" + warnings.TrimEnd(',') + "],";
errInfo += "skips:[" + requires.TrimEnd(',') + "]";
errInfo += "}";
var ie = new ImportError()
{
ImportId = ip.ImportId,
Status = isFubar ? 1 : 0,
Row = i,
Sheet = ip.SheetNo,
Errors = errInfo
};
OrmHelper.SetAuditColumns <ImportError>(ref ie, true, ip.PersonId);
_db.ImportError.Add(ie);
}
}
}
}
ImportInternal(string iniFile, bool returnUnmergedFMsList = false, FieldsToImport?fields = null)
{
string[] lines = await Task.Run(() => File.ReadAllLines(iniFile));
var fms = new List <FanMission>();
ImportError error = await Task.Run(() =>
{
bool archiveDirRead = false;
string archiveDir = "";
for (int i = 0; i < lines.Length; i++)
{
string line = lines[i];
#region Read archive directory
if (!archiveDirRead && line == "[Config]")
{
while (i < lines.Length - 1)
{
string lc = lines[i + 1];
if (lc.StartsWithFast_NoNullChecks("ArchiveRoot="))
{
archiveDir = lc.Substring(12).Trim();
break;
}
else if (!lc.IsEmpty() && lc[0] == '[' && lc[lc.Length - 1] == ']')
{
break;
}
i++;
}
if (archiveDir.IsEmpty())
{
return(ImportError.NoArchiveDirsFound);
}
i = -1;
archiveDirRead = true;
continue;
}
#endregion
#region Read FM entries
// MUST CHECK archiveDirRead OR IT ADDS EVERY FM TWICE!
if (archiveDirRead &&
line.Length >= 5 && line[0] == '[' && line[1] == 'F' && line[2] == 'M' && line[3] == '=')
{
// NOTE: There can be a problem like:
// installed name is CoolMission[1]
// it gets written like [FM=CoolMission[1]]
// it gets read and all [ and ] chars are removed
// it gets written back out like [FM=CoolMission1]
// Rare I guess, so just ignore?
string instName = line.Substring(4, line.Length - 5);
var fm = new FanMission {
InstalledDir = instName
};
// Unfortunately NDL doesn't store its archive names, so we have to do a file search
// similar to DarkLoader
try
{
// NDL always searches subdirectories as well
foreach (string f in Directory.EnumerateFiles(archiveDir, "*", SearchOption.AllDirectories))
{
// @DIRSEP: '/' conversion due to string.ContainsI()
if (!f.ToForwardSlashes().ContainsI("/.fix/"))
{
string fn = Path.GetFileNameWithoutExtension(f);
if (fn.ToInstDirNameNDL().EqualsI(instName) || fn.EqualsI(instName))
{
fm.Archive = Path.GetFileName(f);
break;
}
}
}
}
catch (Exception ex)
{
Log("Exception in NewDarkLoader archive dir file enumeration", ex);
}
while (i < lines.Length - 1)
{
string lineFM = lines[i + 1];
if (lineFM.StartsWithFast_NoNullChecks("NiceName="))
{
fm.Title = lineFM.Substring(9);
}
else if (lineFM.StartsWithFast_NoNullChecks("ReleaseDate="))
{
fm.ReleaseDate.UnixDateString = lineFM.Substring(12);
}
else if (lineFM.StartsWithFast_NoNullChecks("LastCompleted="))
{
fm.LastPlayed.UnixDateString = lineFM.Substring(14);
}
else if (lineFM.StartsWithFast_NoNullChecks("Finished="))
{
uint.TryParse(lineFM.Substring(9), out uint result);
// result will be 0 on fail, which is the empty value so it's fine
fm.FinishedOn = result;
}
else if (lineFM.StartsWithFast_NoNullChecks("Rating="))
{
fm.Rating = int.TryParse(lineFM.Substring(7), out int result) ? result : -1;
}
else if (lineFM.StartsWithFast_NoNullChecks("Comment="))
{
fm.Comment = lineFM.Substring(8);
}
else if (lineFM.StartsWithFast_NoNullChecks("ModExclude="))
{
string val = lineFM.Substring(11);
if (val == "*")
{
fm.DisableAllMods = true;
}
else
{
fm.DisabledMods = val;
}
}
else if (lineFM.StartsWithFast_NoNullChecks("Tags="))
{
string val = lineFM.Substring(5);
if (!val.IsEmpty() && val != "[none]")
{
fm.TagsString = val;
}
}
else if (lineFM.StartsWithFast_NoNullChecks("InfoFile="))
{
fm.SelectedReadme = lineFM.Substring(9);
}
else if (lineFM.StartsWithFast_NoNullChecks("FMSize="))
{
ulong.TryParse(lineFM.Substring(7), out ulong result);
fm.SizeBytes = result;
}
else if (!lineFM.IsEmpty() && lineFM[0] == '[' && lineFM[lineFM.Length - 1] == ']')
{
break;
}
i++;
}
fms.Add(fm);
}
#endregion
}
return(ImportError.None);
});
if (error != ImportError.None)
{
return(error, fms);
}
var importedFMs = returnUnmergedFMsList
? fms
: ImportCommon.MergeImportedFMData(ImportType.NewDarkLoader, fms, fields);
return(ImportError.None, importedFMs);
}
private static void RowFilter(ImportError importError, ISheet sheet)
{
int counter = 0;
var res = (from err in importError
select err.IndexOfRow).Distinct();
foreach (var rowNum in res)
{
sheet.ShiftRows(rowNum, rowNum, 1 - rowNum + counter);
counter++;
}
for (int rowNum = counter + 1; rowNum <= sheet.LastRowNum; rowNum++)
{
IRow row = sheet.GetRow(rowNum);
sheet.RemoveRow(row);
}
}
public async Task ImportAsync(ImportDataRequest request, Action <ImportProgressInfo> progressCallback, ICancellationToken cancellationToken)
{
ValidateParameters(request, progressCallback, cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
var errorsContext = new ImportErrorsContext();
var csvPriceDataValidationResult = await _csvPriceDataValidator.ValidateAsync(request.FilePath);
if (csvPriceDataValidationResult.Errors.Any())
{
throw new InvalidDataException();
}
var reportFilePath = GetReportFilePath(request.FilePath);
var configuration = _importConfigurationFactory.Create();
await using var importReporter = await _importReporterFactory.CreateAsync(reportFilePath, configuration.Delimiter);
cancellationToken.ThrowIfCancellationRequested();
var importProgress = new ImportProgressInfo {
ProcessedCount = 0, CreatedCount = 0, UpdatedCount = 0, Description = "Import has started"
};
SetupErrorHandlers(progressCallback, configuration, errorsContext, importProgress, importReporter);
using var dataSource = _dataSourceFactory.Create(request.FilePath, ModuleConstants.Settings.PageSize, configuration);
var headerRaw = dataSource.GetHeaderRaw();
if (!headerRaw.IsNullOrEmpty())
{
importReporter.WriteHeader(headerRaw);
}
importProgress.TotalCount = dataSource.GetTotalCount();
progressCallback(importProgress);
const string importDescription = "{0} out of {1} have been imported.";
try
{
importProgress.Description = "Fetching...";
progressCallback(importProgress);
var importProductPricesExistValidator = new ImportProductPricesExistenceValidator(_pricingSearchService, ImportProductPricesExistenceValidationMode.Exists);
while (await dataSource.FetchAsync())
{
cancellationToken.ThrowIfCancellationRequested();
var importProductPrices = dataSource.Items
// expect records that was parsed with errors
.Where(importProductPrice => !errorsContext.ErrorsRows.Contains(importProductPrice.Row))
.Select(importProductPrice =>
{
importProductPrice.Price.PricelistId = request.PricelistId;
return(importProductPrice);
}).ToArray();
try
{
var createdPrices = new List <Price>();
var updatedPrices = new List <Price>();
var validationResult = await _importProductPricesValidator.ValidateAsync(importProductPrices, options => options.IncludeRuleSets(request.ImportMode.ToString()));
var invalidImportProductPrices = validationResult.Errors.Select(x => (x.CustomState as ImportValidationState)?.InvalidImportProductPrice).Distinct().ToArray();
var errorsInfos = validationResult.Errors.Select(x => new { Message = x.ErrorMessage, ImportProductPrice = (x.CustomState as ImportValidationState)?.InvalidImportProductPrice }).ToArray();
var errorsGroups = errorsInfos.GroupBy(x => x.ImportProductPrice);
foreach (var group in errorsGroups)
{
var importPrice = group.Key;
var errorMessages = string.Join(" ", group.Select(x => x.Message).ToArray());
var importError = new ImportError {
Error = errorMessages, RawRow = importPrice.RawRecord
};
await importReporter.WriteAsync(importError);
}
importProgress.ErrorCount += invalidImportProductPrices.Length;
importProductPrices = importProductPrices.Except(invalidImportProductPrices).ToArray();
switch (request.ImportMode)
{
case ImportMode.CreateOnly:
createdPrices.AddRange(importProductPrices.Select(importProductPrice => importProductPrice.Price));
break;
case ImportMode.UpdateOnly:
var existingPrices = await GetAndPatchExistingPrices(request, importProductPrices);
updatedPrices.AddRange(existingPrices);
break;
case ImportMode.CreateAndUpdate:
var importProductPriceNotExistValidationResult = await importProductPricesExistValidator.ValidateAsync(importProductPrices);
var importProductPricesToCreate = importProductPriceNotExistValidationResult.Errors
.Select(x => (x.CustomState as ImportValidationState)?.InvalidImportProductPrice).Distinct().ToArray();
var importProductPricesToUpdate = importProductPrices.Except(importProductPricesToCreate).ToArray();
createdPrices.AddRange(importProductPricesToCreate.Select(importProductPrice => importProductPrice.Price));
var existingPricesToUpdate = await GetAndPatchExistingPrices(request, importProductPricesToUpdate);
updatedPrices.AddRange(existingPricesToUpdate);
break;
default:
throw new ArgumentException("Import mode has invalid value", nameof(request));
}
var allChangingPrices = createdPrices.Concat(updatedPrices).ToArray();
await _pricingService.SavePricesAsync(allChangingPrices);
importProgress.CreatedCount += createdPrices.Count;
importProgress.UpdatedCount += updatedPrices.Count;
}
catch (Exception e)
{
HandleError(progressCallback, importProgress, e.Message);
}
finally
{
importProgress.ProcessedCount = Math.Min(dataSource.CurrentPageNumber * dataSource.PageSize, importProgress.TotalCount);
}
if (importProgress.ProcessedCount != importProgress.TotalCount)
{
importProgress.Description = string.Format(importDescription, importProgress.ProcessedCount, importProgress.TotalCount);
progressCallback(importProgress);
}
}
}
catch (Exception e)
{
HandleError(progressCallback, importProgress, e.Message);
}
finally
{
var completedMessage = importProgress.ErrorCount > 0 ? "Import completed with errors" : "Import completed";
importProgress.Description = $"{completedMessage}: {string.Format(importDescription, importProgress.ProcessedCount, importProgress.TotalCount)}";
if (importReporter.ReportIsNotEmpty)
{
importProgress.ReportUrl = _blobUrlResolver.GetAbsoluteUrl(reportFilePath);
}
progressCallback(importProgress);
}
}
private string GetLine(ImportError importError)
{
var result = $"{importError.Error}{_delimiter}{importError.RawRow.TrimEnd()}";
return(result);
}
private void OnImportError(string msg)
{
errorOccurred = true;
ImportError?.Invoke(msg);
}
private void onImportError(string message)
{
ImportError?.Invoke(message);
}
private static void PostilAdd(Tk5ListMetaData metaInfos, ImportError importError, IWorkbook workBook, ISheet sheet)
{
IDrawing part = sheet.CreateDrawingPatriarch();
Dictionary<string, int> indexOfName = new Dictionary<string, int>();
int i = 0;
foreach (var info in metaInfos.Table.TableList)
{
indexOfName.Add(info.DisplayName, i);
i++;
}
foreach (var err in importError)
{
IRow row = sheet.GetRow(err.IndexOfRow);
IComment comment = null;
ICell cell = row.GetCell(indexOfName[err.ColumnName]);
ICreationHelper factory = workBook.GetCreationHelper();
IClientAnchor anchor = null;
anchor = factory.CreateClientAnchor();
anchor.Col1 = cell.ColumnIndex + 2;
anchor.Col2 = cell.ColumnIndex + 4;
anchor.Row1 = row.RowNum;
anchor.Row2 = row.RowNum + 3;
comment = part.CreateCellComment(anchor);
comment.Author = "mitu";
comment.String = new HSSFRichTextString(err.ErrorMsg);
cell.CellComment = comment;
}
}
