public FileProcessStatus ProcessFileArchival(FileProcessStatus fileProcessStatus, string fileName, string folderPath, IArchivalHandler archivalHandler)
{
try
{
log.Debug("Start ProcessFileArchival()");
FileProcessStatus archiveProcessStatus = FileProcessStatus.Undetermined;
if (fileProcessStatus == FileProcessStatus.FileSuccessfullyProccessed)
{
archiveProcessStatus = ArchiveFile(fileName, folderPath, FileArchivalType.Archive, archivalHandler);
archiveProcessStatus = FileProcessStatus.FileSuccessfullyArchived;
}
else if (fileProcessStatus == FileProcessStatus.FileRowsSkipped)
{
archiveProcessStatus = ArchiveFile(fileName, folderPath, FileArchivalType.PartiallyProccessed, archivalHandler);
archiveProcessStatus = FileProcessStatus.FileSuccessfullyArchivedToPartial;
}
else
{
archiveProcessStatus = ArchiveFile(fileName, folderPath, FileArchivalType.Error, archivalHandler);
archiveProcessStatus = FileProcessStatus.FileSuccessfullyArchivedToError;
}
return(archiveProcessStatus);
}
catch (Exception)
{
throw;
}
}
public void TestFileArchivalForPartialFolder()
{
Mock <IArchivalHandler> mockArchivalHandler = new Mock <IArchivalHandler>();
mockArchivalHandler.Setup(x => x.CreateDirectory(It.IsAny <string>())).Returns(true);
mockArchivalHandler.Setup(x => x.MoveFile(It.IsAny <string>(), It.IsAny <string>())).Returns(FileProcessStatus.FileSuccessfullyMoved);
FileProcessor fileProcessor = new FileProcessor();
FileProcessStatus fileProcessStatus = fileProcessor.ArchiveFile(It.IsAny <string>(), It.IsAny <string>(), FileArchivalType.PartiallyProccessed, mockArchivalHandler.Object);
Assert.AreEqual(FileProcessStatus.FileSuccessfullyMoved, fileProcessStatus);
}
public void TestOnlyHeaderFilesProcessing(string fileName)
{
string testFolder = GetTestDataFolder();
IFileProcessor fileProcessor = new FileProcessor();
FileTypes fileTypes = FileTypes.GetFileTypes();
DataFile testFile = fileProcessor.FetchAllDataFiles(fileTypes, testFolder)
.Find(x => x.FileName.Contains(fileName));
double divergencePercentage = 20;
FileProcessStatus fileProcessStatus = fileProcessor.ProcessInputFile(testFile, fileTypes, testFolder, divergencePercentage);
Assert.AreEqual(FileProcessStatus.FileOnlyHeaderFound, fileProcessStatus);
}
public FileProcessStatus ArchiveFile(string fileName, string folderPath, FileArchivalType fileArchivalType, IArchivalHandler archivalHandler)
{
try
{
log.Debug("Start ArchiveFile()");
string destinationFolderType;
FileProcessStatus fileProcessStatus = FileProcessStatus.Undetermined;
if (fileArchivalType == FileArchivalType.Archive)
{
destinationFolderType = "Archive";
}
else if (fileArchivalType == FileArchivalType.Error)
{
destinationFolderType = "Error";
}
else if (fileArchivalType == FileArchivalType.PartiallyProccessed)
{
destinationFolderType = "PartiallyProcessed";
}
else
{
destinationFolderType = "Archive";
}
string destinationFullPath = folderPath + "\\" + destinationFolderType + "\\" + fileName;
string sourceFullPath = folderPath + fileName;
if (archivalHandler.CreateDirectory(folderPath + "\\" + destinationFolderType))
{
if (!archivalHandler.FileExists(destinationFullPath))
{
fileProcessStatus = archivalHandler.MoveFile(sourceFullPath, destinationFullPath);
}
else
{
if (archivalHandler.DeleteFile(destinationFullPath))
{
fileProcessStatus = archivalHandler.MoveFile(sourceFullPath, destinationFullPath);
}
}
}
return(fileProcessStatus);
}
catch (Exception)
{
throw;
}
}
public void TestMissingColumnFile(string fileName)
{
string testFolder = GetTestDataFolder();
IFileProcessor fileProcessor = new FileProcessor();
FileTypes fileTypes = FileTypes.GetFileTypes();
DataFile testFile = fileProcessor.FetchAllDataFiles(fileTypes, testFolder)
.Find(x => x.FileName.Contains(fileName));
FileType fileType = new FileType();
FileProcessStatus validationSuccess = FileProcessStatus.Undetermined;
string[] file = fileProcessor.ValidateInputFile(testFile, fileTypes, ref fileType, testFolder, ref validationSuccess);
Assert.AreEqual(validationSuccess, FileProcessStatus.FileHeadersNotFound);
}
public void TestFileArchivalHeaderNotFound()
{
Mock <IArchivalHandler> mockArchivalHandler = new Mock <IArchivalHandler>();
mockArchivalHandler.Setup(x => x.CreateDirectory(It.IsAny <string>())).Returns(true);
mockArchivalHandler.Setup(x => x.FileExists(It.IsAny <string>())).Returns(true);
mockArchivalHandler.Setup(x => x.DeleteFile(It.IsAny <string>())).Returns(true);
mockArchivalHandler.Setup(x => x.MoveFile(It.IsAny <string>(), It.IsAny <string>())).Returns(FileProcessStatus.FileSuccessfullyArchived);
FileProcessor fileProcessor = new FileProcessor();
FileProcessStatus fileProcessStatus = FileProcessStatus.FileHeadersNotFound;
fileProcessStatus = fileProcessor.ProcessFileArchival(fileProcessStatus, "Foobar.csv", "Foobar/Foobar", mockArchivalHandler.Object);
Assert.AreEqual(FileProcessStatus.FileSuccessfullyArchivedToError, fileProcessStatus);
}
public bool ProcessAllFiles(string folderPath, IArchivalHandler archivalHandler, ref List <DataFile> dataFiles)
{
try
{
log.Debug("Start ProcessAllFiles()");
var fileTypes = FileTypes.GetFileTypes();
dataFiles = FetchAllDataFiles(fileTypes, folderPath);
bool atleastOneFileFound = true;
if (dataFiles.Count() == 0)
{
atleastOneFileFound = false;
log.Warn("No Files Found in Input Directory, Exiting Processing!");
}
else
{
double divergencePercentage = Convert.ToDouble(ConfigurationManager.AppSettings["DivergencePercentage"]);
foreach (DataFile dataFile in dataFiles)
{
try
{
FileProcessStatus fileProcessStatus = FileProcessStatus.Undetermined;
fileProcessStatus = ProcessInputFile(dataFile, fileTypes, folderPath, divergencePercentage);
fileProcessStatus = ProcessFileArchival(fileProcessStatus, dataFile.FileName, folderPath, archivalHandler);
dataFile.ProcessingStatus = fileProcessStatus;
}
catch (Exception ex)
{
log.Error("File processing has unexpectedly failed for " + dataFile.FileName + " Continuing processing for other files, if any!");
log.Error(ex);
}
}
OutputStatistics(dataFiles, divergencePercentage);
}
log.Info("Read Completed");
return(atleastOneFileFound);
}
catch (Exception)
{
throw;
}
}
public void TestValidateBlankFile(string fileName)
{
string testFolder = GetTestDataFolder();
IFileProcessor fileProcessor = new FileProcessor();
FileTypes fileTypes = FileTypes.GetFileTypes();
DataFile testFile = fileProcessor.FetchAllDataFiles(fileTypes, testFolder)
.Find(x => x.FileName.Contains(fileName));
FileType fileType = new FileType();
FileProcessStatus validationSuccess = FileProcessStatus.Undetermined;
string[] file = fileProcessor.ValidateInputFile(testFile, fileTypes, ref fileType, testFolder, ref validationSuccess);
//Test if the array returned by the validate function has nothing in it
Assert.AreEqual(0, file.Count());
//Test if correct fileprocessing status is returned
Assert.AreEqual(FileProcessStatus.BlankFileDetected, validationSuccess);
}
public void TestPositiveFilesProcessing(string fileName, double divergencePercentage, double medianResult, int noOfMoreDivergentResult, int noOfLessDivergentResult, int noOfLinesProcessedResult)
{
string testFolder = GetTestDataFolder();
IFileProcessor fileProcessor = new FileProcessor();
FileTypes fileTypes = FileTypes.GetFileTypes();
DataFile testFile = fileProcessor.FetchAllDataFiles(fileTypes, testFolder)
.Find(x => x.FileName.Contains(fileName));
FileProcessStatus fileProcessStatus = fileProcessor.ProcessInputFile(testFile, fileTypes, testFolder, divergencePercentage);
//Tests Median Value
Assert.AreEqual(medianResult, testFile.MedianValue);
//Tests The Number of values more than divergence percentage
Assert.AreEqual(noOfMoreDivergentResult, testFile.MeterReads.FindAll(x => x.Divergence == Divergence.MoreThanAcceptableDivergence).Count());
//Tests the Number of values less than divergence percentage
Assert.AreEqual(noOfLessDivergentResult, testFile.MeterReads.FindAll(x => x.Divergence == Divergence.LessThanAcceptableDivergence).Count());
//Test the number of files processed
Assert.AreEqual(noOfLinesProcessedResult, testFile.MeterReads.Count());
//Tests if file is sucessfully processed
Assert.AreEqual(FileProcessStatus.FileSuccessfullyProccessed, fileProcessStatus);
}
public FileProcessStatus ProcessInputFile(DataFile dataFile, FileTypes fileTypes, string folderPath, double divergencePercentage)
{
try
{
log.Debug("Start ProcessInputFile()");
log.Info("Processing " + dataFile.FileName);
#region Read data from each file and calculate median values
FileProcessStatus fileProcessStatus = FileProcessStatus.Undetermined;
FileType fileType = new FileType();
string[] file = ValidateInputFile(dataFile, fileTypes, ref fileType, folderPath, ref fileProcessStatus);
if (fileProcessStatus == FileProcessStatus.FileValidationSucceeded)
{
dataFile.MeterReads = ReadFile(file, fileType, ref fileProcessStatus);
if (fileProcessStatus == FileProcessStatus.FileReadSuccessfully)
{
double median = DataFile.CalculateMedian(dataFile.MeterReads);
foreach (FileRow meterRead in dataFile.MeterReads)
{
if (meterRead.DataValue > ((100 + divergencePercentage) * median / 100))
{
meterRead.Divergence = Divergence.MoreThanAcceptableDivergence;
}
else if (meterRead.DataValue < ((100 - divergencePercentage) * median / 100))
{
meterRead.Divergence = Divergence.LessThanAcceptableDivergence;
}
if ((meterRead.Divergence == Divergence.LessThanAcceptableDivergence) || (meterRead.Divergence == Divergence.MoreThanAcceptableDivergence))
{
Console.WriteLine(dataFile.FileName + " " + meterRead.OccurrenceDateTime.ToString() + " "
+ meterRead.DataValue.ToString()
+ " " + median.ToString());
}
}
dataFile.MedianValue = median;
}
}
else
{
return(fileProcessStatus);
}
#endregion
if (fileProcessStatus != FileProcessStatus.FileRowsSkipped)
{
if (fileProcessStatus != FileProcessStatus.FileOnlyHeaderFound)
{
fileProcessStatus = FileProcessStatus.FileSuccessfullyProccessed;
}
}
return(fileProcessStatus);
}
catch (Exception)
{
throw;
}
}
public List <FileRow> ReadFile(string[] csvLines, FileType fileType, ref FileProcessStatus readFileStatus)
{
try
{
log.Debug("Start ReadFile()");
List <FileRow> meterReads = new List <FileRow>();
int dateTimeColumnIndex = Convert.ToInt32(fileType.Items.FirstOrDefault(x => x.ValueType == "Occurence DateTime").Index);
int valueColumnIndex = Convert.ToInt32(fileType.Items.FirstOrDefault(x => x.ValueType == "Data Value").Index);
//Check if data exists other than the header
if (csvLines.Skip(1).Count() == 0)
{
readFileStatus = FileProcessStatus.FileOnlyHeaderFound;
return(null);
}
foreach (string csvLine in csvLines.Skip(1))
{
string[] values = csvLine.Split(',');
// Validation to check if the data is in the appropriate format
if (DateTime.TryParse(values[dateTimeColumnIndex], out DateTime tempDateTime))
{
tempDateTime = Convert.ToDateTime(values[dateTimeColumnIndex]);
}
else
{
readFileStatus = FileProcessStatus.FileRowsSkipped;
continue;
}
if (Double.TryParse(values[valueColumnIndex], out Double tempDouble))
{
tempDouble = Convert.ToDouble(values[valueColumnIndex]);
}
else
{
readFileStatus = FileProcessStatus.FileRowsSkipped;
continue;
}
FileRow meterRead = new FileRow(tempDateTime, tempDouble);
meterReads.Add(meterRead);
}
if (readFileStatus != FileProcessStatus.FileRowsSkipped)
{
if (csvLines.Count() - 1 == meterReads.Count())
{
readFileStatus = FileProcessStatus.FileReadSuccessfully;
}
}
return(meterReads);
}
catch (Exception)
{
throw;
}
}
public string[] ValidateInputFile(DataFile dataFile, FileTypes fileTypes, ref FileType fileType, string folderPath, ref FileProcessStatus validationStatus)
{
try
{
log.Debug("Start ValidateInputFile()");
//Check Blank files
if (new FileInfo(folderPath + dataFile.FileName).Length == 0)
{
log.Warn(dataFile.FileName + " is blank, nothing to read! ");
validationStatus = FileProcessStatus.BlankFileDetected;
return(Array.Empty <string>());
}
else
{
string[] csvLines = File.ReadAllLines(folderPath + dataFile.FileName);
string[] headers = csvLines[0].Split(',');
//Get the Column Names for the files and check if they contain the required column names, and check the indexes
fileType = fileTypes.FileType.FirstOrDefault(x => x.Identifier == dataFile.FileType);
foreach (Items item in fileType.Items)
{
item.Index = null;
for (int i = 0; i < headers.Count(); i++)
{
if (headers[i] == item.FileHeaderName)
{
item.Index = i;
break;
}
}
if (item.Index == null)
{
log.Error("Column " + item.FileHeaderName + " is missing in file: " + dataFile.FileName);
validationStatus = FileProcessStatus.FileHeadersNotFound;
return(Array.Empty <string>());
}
}
validationStatus = FileProcessStatus.FileValidationSucceeded;
return(csvLines);
}
}
catch (Exception)
{
throw;
}
}
