public async Task <IHttpActionResult> ProcessFiles()
{
logger.InfoFormat("Process File start at {0}", DateTime.Now);
// Moveing incoming files to processing folder
string[] fileEntries = Directory.GetFiles(this.IncomingFilePath, "*.csv", SearchOption.TopDirectoryOnly);
foreach (var fileName in fileEntries)
{
string fileNameWithoutPath = Path.GetFileName(fileName);
string processingFile = string.Format(@"{0}\{1}", this.ProcessingFilePath, fileNameWithoutPath);
try
{
// Move file to processing folder
if (File.Exists(processingFile))
{
File.Delete(processingFile);
}
File.Move(fileName, processingFile);
}
catch (Exception ex)
{
logger.Error(ex.Message, ex);
}
}
// Process the list of files found in the directory.
string[] processFileEntries = Directory.GetFiles(this.ProcessingFilePath, "*.csv", SearchOption.TopDirectoryOnly);
foreach (string fileName in processFileEntries)
{
string fileNameWithoutPath = Path.GetFileName(fileName);
string processingFile = string.Format(@"{0}\{1}", this.ProcessingFilePath, fileNameWithoutPath);
string archiveFile = string.Format(@"{0}\{1}", this.ArchiveFilePath, fileNameWithoutPath);
string errorFile = string.Format(@"{0}\{1}", this.ErrorFilePath, fileNameWithoutPath);
try
{
// Process file
FileProcessSummary summary = await ProcessSingleFile(processingFile);
logger.DebugFormat("File Name : {0}", fileNameWithoutPath);
logger.DebugFormat("Summary - Start Time: {0}", summary.StartTime);
logger.DebugFormat("Summary - End Time: {0}", summary.EndTime);
logger.DebugFormat("Summary - Total Line: {0}", summary.Total);
logger.DebugFormat("Summary - Success: {0}", summary.Success);
logger.DebugFormat("Summary - Failure: {0}", summary.Failure);
// Archive file
if (File.Exists(archiveFile))
{
archiveFile = GetFileName(archiveFile);
}
File.Move(processingFile, archiveFile);
// Error file
if (!summary.SuccessfullyProcessed)
{
errorFile = GetFileName(errorFile);
StreamWriter sw = File.CreateText(errorFile);
foreach (var errLine in summary.ErrorList)
{
await sw.WriteLineAsync(errLine);
}
sw.Close();
}
}
catch (Exception ex)
{
logger.ErrorFormat("Error File Name : {0}{1}{2}", fileNameWithoutPath, Environment.NewLine, ex.StackTrace);
try
{
if (File.Exists(errorFile))
{
File.Delete(errorFile);
}
File.Move(processingFile, errorFile);
}
catch (Exception moveEx)
{
logger.Error(moveEx.Message, moveEx);
}
}
}
logger.InfoFormat("Process Files end at {0}", DateTime.Now);
logger.InfoFormat(Environment.NewLine);
await ProcessStagingData();
return(StatusCode(HttpStatusCode.OK));
}
protected async Task <FileProcessSummary> ProcessSingleFile(string fileNamePath)
{
FileProcessSummary summary = new FileProcessSummary();
string[] lines = File.ReadAllLines(fileNamePath);
bool hasError = false;
long totalLine = lines.Length;
int recordAffected = 0;
if (totalLine > 0)
{
string headerLine = lines[0];
string[] header = headerLine.Split(new char[] { ',' });
if (header.Length != 7)
{
logger.Error(Consts.INTEGRATION_FILE_HEADER_LENGTH);
throw new FileProcessingException(Consts.INTEGRATION_FILE_HEADER_LENGTH, FileProcessingException.ErrorType.File);
}
summary.ErrorList = new List <string>();
summary.ErrorList.Add(headerLine);
summary.Total = lines.Length - 1;
summary.StartTime = DateTime.Now;
List <Monitoring> monitoringList = new List <Monitoring>();
for (long i = 0; i < lines.Length; i++)
{
long currentLine = i;
string line = lines[i];
Regex CSVParser = new Regex(",(?=(?:[^\"]*\"[^\"]*\")*(?![^\"]*\"))");
string[] raw = CSVParser.Split(line);
//string[] raw = line.Split(new char[] { ',' });
try
{
Monitoring monitor = new Monitoring();
// IMO Number
if (!string.IsNullOrWhiteSpace(raw[0]))
{
monitor.IMO_No = RemoveDoubleQuotes(raw[0]);
}
// Engine Serial No.
if (!string.IsNullOrWhiteSpace(raw[1]))
{
monitor.SerialNo = RemoveDoubleQuotes(raw[1]);
}
// Channel No
if (!string.IsNullOrWhiteSpace(raw[2]))
{
monitor.ChannelNo = RemoveDoubleQuotes(raw[2]);
}
// Channel Description
if (!string.IsNullOrWhiteSpace(raw[3]))
{
monitor.ChannelDescription = RemoveDoubleQuotes(raw[3]);
}
// TimeStamp
if (!string.IsNullOrWhiteSpace(raw[4]))
{
monitor.TimeStamp = DateTime.Parse(raw[4]).ToUniversalTime();
}
// Data Values
if (!string.IsNullOrWhiteSpace(raw[5]))
{
monitor.Value = RemoveDoubleQuotes(raw[5]);
}
// Unit of Measurement
if (!string.IsNullOrWhiteSpace(raw[6]))
{
monitor.Unit = RemoveDoubleQuotes(raw[6]);
}
// System Info
//monitor.Id = Guid.NewGuid();
//monitor.CreatedBy = Guid.Empty;
//monitor.CreatedOn = DateTime.Now;
//monitor.ModifiedBy = Guid.Empty;
//monitor.ModifiedOn = DateTime.Now;
monitor.DataRecord = line;
monitor.FileName = Path.GetFileName(fileNamePath);
monitor.TimeStampOriginal = raw[4];
monitoringList.Add(monitor);
summary.Success = summary.Success++;
}
catch (Exception ex)
{
summary.ErrorList.Add(string.Format("{0},{1},{2}", line, i.ToString(), ex.Message));
hasError = true;
summary.Failure++;
continue;
}
}
summary.EndTime = DateTime.Now;
Remax_Entities remax_Entities = new Remax_Entities();
remax_Entities.Monitorings.AddRange(monitoringList);
recordAffected = 0; // Do not remove this
User serviceUser = (from u in remax_Entities.Users
where (string.IsNullOrEmpty(u.FullName) ? "" : u.FullName).ToLower() == "service"
select u).FirstOrDefault();
remax_Entities.ServiceUser = serviceUser;
try
{
recordAffected = await remax_Entities.SaveChangesAsync();
}
catch (Exception ex)
{
hasError = true;
logger.Error(ex.Message, ex);
summary.Failure += (summary.Total - summary.Failure) - recordAffected;
}
}
summary.DatabaseInsert = recordAffected;
summary.Success = recordAffected;
summary.SuccessfullyProcessed = !hasError;
return(summary);
}