public static void UpdateNumberOfRowsInFile(UploadJob job, DataTable dataTable, UploadJobRepository repository,
            bool isSimpleUpload)
        {
            var rowCount = 0;

            if (isSimpleUpload)
            {
                for (var i = 0; i < dataTable.Rows.Count; i++)
                {
                    var row = dataTable.Rows[i];
                    var rowParser = new UploadSimpleRowParser(row);

                    if (rowParser.DoesRowContainData == false)
                    {
                        break;
                    }

                    rowCount++;
                }
            }
            else
            {
                rowCount = dataTable.Rows.Count;
            }

            job.TotalRows = rowCount;
            repository.UpdateJob(job);


        }
        public bool Check(List<int> indicators, UploadJob job, UploadJobErrorRepository errorRepository)
        {

            // Check permission
            CheckIndicatorPermissionForCurrentUser(indicators, job.UserId);

            // Check if all indicator exist
            if (noneExistingIndicators.Count > 0)
            {
                var error = job.JobType == UploadJobType.Simple
                    ? ErrorBuilder.GetSimplePermissionError(job.Guid, noneExistingIndicators, false)
                    : ErrorBuilder.GetBatchPermissionError(job.Guid, noneExistingIndicators, false);
                errorRepository.Log(error);
                return false;
            }

            // Check permissions
            if (indicatorsWithoutPermission.Count > 0)
            {
                var error = job.JobType == UploadJobType.Simple
                    ? ErrorBuilder.GetSimplePermissionError(job.Guid, indicatorsWithoutPermission, true)
                    : ErrorBuilder.GetBatchPermissionError(job.Guid, indicatorsWithoutPermission, true);
                errorRepository.Log(error);
                return false;
            }

            return true;
        }
 public void InProgress(UploadJob job)
 {
     job.Status = UploadJobStatus.InProgress;
     job.TotalRowsCommitted = 0;
     _jobRepository.UpdateJob(job);
     _logger.Info("Job# {0} status changed to {1} ", job.Guid, job.Status);
 }
        public void SuccessfulUpload(UploadJob job, int rowsUploaded)
        {
            job.Status = UploadJobStatus.SuccessfulUpload;
            job.TotalRowsCommitted = rowsUploaded;

            _jobRepository.UpdateJob(job);
            _logger.Info("Job# {0} successfully completed", job.Guid);
        }
 protected UploadJob GetJob(UploadJobType jobType, Guid jobGuid)
 {
     var job = new UploadJob
     {
         Guid = jobGuid,
         Status = UploadJobStatus.NotStart,
         DateCreated = DateTime.Now,
         JobType = jobType,
         Filename = @"fake.xls",
         UserId = 11,
         Username = @"phe\doris.hain"
     };
     return job;
 }
        public void UpdateJob(UploadJob uploadJob)
        {
            try
            {
                transaction = CurrentSession.BeginTransaction();

                CurrentSession.Update(uploadJob);
                CurrentSession.Flush();
                CurrentSession.Refresh(uploadJob);
                transaction.Commit();

            }
            catch (Exception exception)
            {
                HandleException(exception);
            }
        }
        public SimpleUpload UploadData(DataTable indicatorDetails, DataTable pholioData, UploadJob job)
        {
            var simpleUpload = new SimpleUpload();
            var rowCount = 0;

            var allowedData = new AllowedData(_profilesReader);

            // TODO: find out why were are calling it again here as its caled in Validate()
            ValidateIndicatorDetails(indicatorDetails, simpleUpload, allowedData);

            var dataToUpload = new List<UploadDataModel>();

            for (int i = 0; i < pholioData.Rows.Count; i++)
            {
                DataRow row = pholioData.Rows[i];
                var rowParser = new UploadSimpleRowParser(row);

                if (rowParser.DoesRowContainData == false)
                {
                    //There isn't an area code or value so assume the end of the data  
                    break;
                }

                rowCount++;

                var upload = rowParser.GetUploadDataModelWithUnparsedValuesSetToDefaults(simpleUpload);
                _coreDataRepository.InsertCoreData(upload.ToCoreDataSet(), job.Guid);
                dataToUpload.Add(upload);
            }

            simpleUpload.DataToUpload = dataToUpload;

            int uploadId = _loggingRepository.InsertUploadAudit(job.Guid, job.Username,
                rowCount, job.Filename, WorksheetNames.SimplePholio);

            simpleUpload.ShortFileName = Path.GetFileName(job.Filename);
            simpleUpload.TotalDataRows = rowCount;
            simpleUpload.UploadBatchId = job.Guid;
            simpleUpload.Id = uploadId;

            return simpleUpload;
        }
        public UploadJob SaveJob(UploadJob job)
        {
            try
            {
                transaction = CurrentSession.BeginTransaction();

                CurrentSession.Save(job);

                CurrentSession.Flush();
                CurrentSession.Refresh(job);
                transaction.Commit();

            }
            catch (Exception exception)
            {
                HandleException(exception);
            }

            return job;
        }
        public void StartJob(UploadJob job)
        {
            var validator = new WorksheetNameValidator();
            var actualFilePath = FilePathHelper.GetActualFilePath(job);
            var excelFileReader = new ExcelFileReader(actualFilePath);

            if (job.JobType == UploadJobType.Simple)
            {
                SetUsername(job);
                _logger.Info("Processing at Simple upload for {0} and jobid# is {1}", job.Username, job.Guid);
                var worker = new SimpleJobWorker();
                var processor = new SimpleWorksheetDataProcessor(_coreDataRepository, _loggingRepository);
                worker.ProcessJob(job, validator, processor, excelFileReader);
            }
            else
            {
                SetUsername(job);
                _logger.Info("Processing at Batch upload for {0} and jobid# is {1}", job.Username, job.Guid);
                var worker = new BatchJobWorker();
                var processor = new BatchWorksheetDataProcessor(_coreDataRepository, _loggingRepository);
                worker.ProcessJob(job, validator, processor, excelFileReader);
            }
        }
 private void UpdateJobProgress(UploadJob job, ProgressStage stage)
 {
     job.ProgressStage = stage;
     _jobRepository.UpdateJob(job);
 }
        private bool CheckWorksheets(UploadJob job, IWorksheetNameValidator nameValidator, List<string> worksheets)
        {
            if (nameValidator.ValidateBatch(worksheets)) return true;

            jobStatus.FailedValidation(job);

            var error = ErrorBuilder.GetWorkSheetNameValidationError(job);
            _jobErrorRepository.Log(error);
            _logger.Info("Job# {0} doesn't have required worksheets", job.Guid);
            _logger.Info("Job# {0} status changed to {1} ", job.Guid, job.Status);
            return false;
        }
        private bool CheckPermission(UploadJob job, List<int> indicatorIdsInBatch)
        {
            var hasPermission = new IndicatorPermission().Check(indicatorIdsInBatch, job, _jobErrorRepository);
            if (hasPermission) return true;
            jobStatus.FailedValidation(job);

            _logger.Info("Job# {0}, User doesn't have permission for indicator", job.Guid);
            return false;
        }
 private void UploadDataToCoreDataSet(UploadJob job, ISimpleWorksheetDataProcessor processor,
     DataTable indicatorDetails, DataTable pholioData)
 {
     // Upload to DB
     var rowsUploaded = processor.UploadData(indicatorDetails, pholioData, job).DataToUpload.Count;
     // All good job done
     jobStatus.SuccessfulUpload(job, rowsUploaded);
 }
 private UploadJob CreateJob(Guid guid)
 {
     var newJob = new UploadJob
     {
         Guid = guid,
         Username = @"phe\farrukh",
         Filename = "Fake.xls",
         DateCreated = DateTime.Now,
         JobType = UploadJobType.Simple,
         TotalRows = 0,
         Status = UploadJobStatus.NotStart
     };
     return newJob;
 }
        public void ProcessJob(UploadJob job, IWorksheetNameValidator nameValidator,
            IBatchWorksheetDataProcessor processor, IExcelFileReader excelFileReader)
        {
            try
            {
                _jobRepository = new UploadJobRepository();
                _jobErrorRepository = new UploadJobErrorRepository();
                var batchUpload = ToBatchUpload(job);
                jobStatus = new StatusHelper(_jobRepository, _logger);

                _logger.Info("Job# {0} current status is {1} ", job.Guid, job.Status);

                // If user wants to override duplications 
                if (job.Status == UploadJobStatus.ConfirmationGiven)
                {
                    jobStatus.InProgress(job);
                    // Read indicators in datatable
                    var batchDataTable = GetBatchData(excelFileReader);
                    // Save the total number of rows in file
                    WorkerHelper.UpdateNumberOfRowsInFile(job, batchDataTable, _jobRepository, true);
                    // 
                    //Perform validation once again to get the list 
                    // of duplicate rows in database
                    //
                    processor.Validate(batchDataTable, batchUpload);
                    // Remove duplications in file
                    CheckDuplicateRowsInWorksheet(job, batchUpload, ref batchDataTable);
                    // Archive rows
                    processor.ArchiveDuplicates(batchUpload.DuplicateRowInDatabaseErrors, job);
                    // Upload data to core data set
                    UploadDataToCoreDataSet(job, processor, batchDataTable);
                }
                else // If we have a new job
                {
                    jobStatus.InProgress(job);

                    // Get worksheets from file
                    var worksheets = excelFileReader.GetWorksheets();

                    UpdateJobProgress(job, ProgressStage.ValidatingWorksheets);

                    // Check worksheet names are correct
                    var worksheetsOk = CheckWorksheets(job, nameValidator, worksheets);
                    if (!worksheetsOk) return;


                    var batchDataTable = GetBatchData(excelFileReader);

                    // Save the total number of rows in file
                    WorkerHelper.UpdateNumberOfRowsInFile(job, batchDataTable, _jobRepository, true);

                    UpdateJobProgress(job, ProgressStage.ValidatingData);

                    processor.Validate(batchDataTable, batchUpload);

                    var indicatorIdsInBatch = processor.GetIndicatorIdsInBatch();

                    UpdateJobProgress(job, ProgressStage.CheckingPermission);

                    // Check user permission for indicators
                    var permissionsOk = CheckPermission(job, indicatorIdsInBatch);
                    if (!permissionsOk) return;

                    UpdateJobProgress(job, ProgressStage.CheckingDuplicationInFile);

                    // Check for duplications in file
                    CheckDuplicateRowsInWorksheet(job, batchUpload, ref batchDataTable);

                    // Check validation errors
                    var validationOk = CheckValidationFailures(job, batchUpload);
                    if (!validationOk) return;

                    UpdateJobProgress(job, ProgressStage.CheckingDuplicationInDb);

                    // Check for duplications database rows
                    var haveDuplicates = CheckDuplicateRowsInDatabase(job, batchUpload);
                    if (haveDuplicates) return;

                    UploadDataToCoreDataSet(job, processor, batchDataTable);
                }
            }
            catch (Exception ex)
            {
                jobStatus.UnexpectedError(job);
                _logger.Error(ex);
            }
        }
 public void FailedValidation(UploadJob job)
 {
     job.Status = UploadJobStatus.FailedValidation;
     _jobRepository.UpdateJob(job);
     _logger.Info("Job# {0} status changed to {1} ", job.Guid, job.Status);
 }
 public void ConfirmationAwaited(UploadJob job)
 {
     job.Status = UploadJobStatus.ConfirmationAwaited;
     _jobRepository.UpdateJob(job);
     _logger.Info("Job# {0} status changed to {1} ", job.Guid, job.Status);
 }
 public void UnexpectedError(UploadJob job)
 {
     job.Status = UploadJobStatus.UnexpectedError;
     _jobRepository.UpdateJob(job);
     _logger.Info("Job# {0} status changed to {1} ", job.Guid, job.Status);
 }
 public static string GetActualFilePath(UploadJob job)
 {
     var ext = Path.GetExtension(job.Filename);
     var fileOnDisk = Path.Combine(AppConfig.GetUploadFolder(), job.Guid + ext);
     return fileOnDisk;
 }
 private void SetUsername(UploadJob job)
 {
     var fpmUser = _profilesReader.GetUserByUserId(job.UserId);
     job.Username = fpmUser.UserName;
 }
        public void ProcessJob(UploadJob job, IWorksheetNameValidator nameValidator,
            ISimpleWorksheetDataProcessor processor, IExcelFileReader excelFileReader)
        {
            try
            {
                _jobRepository = new UploadJobRepository();
                _jobErrorRepository = new UploadJobErrorRepository();
                // Create SimpleUpload object from job
                var simpleUpload = ToSimpleUpload(job);
                jobStatus = new StatusHelper(_jobRepository, _logger);
                _logger.Info("Job# {0} current status is {1} ", job.Guid, job.Status);

                if (job.Status == UploadJobStatus.ConfirmationGiven)
                {
                    jobStatus.InProgress(job);
                    // Get indicator details worksheet as data table
                    var indicatorDetails = excelFileReader.GetIndicatorDetails();
                    // Get pholio data worksheet as data table
                    var pholioData = excelFileReader.GetPholioData();
                    // Save the total number of rows in file
                    WorkerHelper.UpdateNumberOfRowsInFile(job, pholioData, _jobRepository, true);
                    // Validate the Data            
                    processor.Validate(indicatorDetails, pholioData, simpleUpload);
                    // Remove duplicate rows
                    CheckDuplicateRowsInWorksheet(job, simpleUpload, ref pholioData);
                    // Archive rows
                    processor.ArchiveDuplicates(simpleUpload.DuplicateRowInDatabaseErrors, job);
                    // Upload data to core data set
                    UploadDataToCoreDataSet(job, processor, indicatorDetails, pholioData);
                }
                else
                {
                    // Update the job status to in progress            
                    jobStatus.InProgress(job);
                    // Get worksheets from file
                    var worksheets = excelFileReader.GetWorksheets();
                    // Check worksheet names are correct
                    var worksheetsOk = CheckWorksheets(job, worksheets, nameValidator);
                    if (!worksheetsOk) return;
                    //  Get indicator details worksheet as data table
                    var indicatorDetails = excelFileReader.GetIndicatorDetails();
                    // Get pholio data worksheet as data table
                    var pholioData = excelFileReader.GetPholioData();
                    // Save the total number of rows in file
                    WorkerHelper.UpdateNumberOfRowsInFile(job, pholioData, _jobRepository, true);
                    // Validate the Data            
                    processor.Validate(indicatorDetails, pholioData, simpleUpload);

                    // Check user permission for indicator
                    var indicatorIds = new List<int> { simpleUpload.IndicatorId };
                    var permissionsOk = CheckPermission(indicatorIds, job);
                    if (!permissionsOk) return;

                    // Check for duplications in file, if there will be any duplicate rows
                    // we will remove them.
                    CheckDuplicateRowsInWorksheet(job, simpleUpload, ref pholioData);

                    // Check validation errors
                    var validationOk = CheckValidationFailures(job, simpleUpload);
                    if (!validationOk) return;

                    // Check for duplications database rows
                    var haveDuplicates = CheckDuplicateRowsInDatabase(job, simpleUpload);
                    if (haveDuplicates) return;

                    // Upload to DB
                    UploadDataToCoreDataSet(job, processor, indicatorDetails, pholioData);
                }
            }
            catch (Exception ex)
            {
                jobStatus.UnexpectedError(job);
                _logger.Error(ex);
            }
        }
 private void UploadDataToCoreDataSet(UploadJob job, IBatchWorksheetDataProcessor processor, DataTable batchDataTable)
 {
     // Upload to DB
     var rowsUploaded = processor.UploadData(batchDataTable, job).DataToUpload.Count;
     // All good job completed
     jobStatus.SuccessfulUpload(job, rowsUploaded);
     UpdateJobProgress(job, ProgressStage.WrittingToDb);
 }
 private BatchUpload ToBatchUpload(UploadJob job)
 {
     var batchUpload = new BatchUpload
     {
         ShortFileName = job.Filename,
         FileName = job.Filename,
         DataToUpload = new List<UploadDataModel>(),
         DuplicateRowInDatabaseErrors = new List<DuplicateRowInDatabaseError>(),
         DuplicateRowInSpreadsheetErrors = new List<DuplicateRowInSpreadsheetError>(),
         ExcelDataSheets = new List<UploadExcelSheet>(),
         UploadValidationFailures = new List<UploadValidationFailure>()
     };
     return batchUpload;
 }
        private bool CheckDuplicateRowsInDatabase(UploadJob job, BatchUpload batchUpload)
        {
            if (batchUpload.DuplicateRowInDatabaseErrors.Count <= 0) return false;

            jobStatus.ConfirmationAwaited(job);

            var error = ErrorBuilder.GetDuplicateRowInDatabaseError(job.Guid,
                batchUpload.DuplicateRowInDatabaseErrors);
            _jobErrorRepository.Log(error);
            _logger.Info("Job# {0}, There are duplicate rows in database", job.Guid);

            return true;
        }
 public void ArchiveDuplicates(IEnumerable<DuplicateRowInDatabaseError> duplicateRows, UploadJob job)
 {
     _coreDataRepository.InsertCoreDataArchive(duplicateRows, job.Guid);
 }
        private bool CheckValidationFailures(UploadJob job, BatchUpload batchUpload)
        {
            if (batchUpload.UploadValidationFailures.Count <= 0) return true;

            jobStatus.FailedValidation(job);

            var error = ErrorBuilder.GetConversionError(job.Guid, batchUpload.UploadValidationFailures);
            _jobErrorRepository.Log(error);
            _logger.Info("Job# {0}, Data type conversion errors occurred ", job.Guid);
            return false;
        }
 private void AddNewJob(UploadJob newJob)
 {
     _uploadJobRepository.SaveJob(newJob);
 }
 private void CheckDuplicateRowsInWorksheet(UploadJob job, BatchUpload batchUpload, ref DataTable batchDataTable)
 {
     if (batchUpload.DuplicateRowInSpreadsheetErrors.Count > 0)
     {
         var dataWithoutDuplicates = new FileDuplicationHandler().RemoveDuplicatesInBatch(batchDataTable);
         batchDataTable = dataWithoutDuplicates;
         _logger.Info("Job# {0}, There are duplicate rows in spreadsheet", job.Guid);
         _logger.Info("Job# {0}, Dupllicate rows removed", job.Guid);
     }
 }
 public static UploadJobError GetWorkSheetNameValidationError(UploadJob job)
 {
     const string simpleWorksheetName = @" ""Pholio"" worksheet.";
     const string batchWorksheetName = @" ""IndicatorDetails"" and ""PholioData"" worksheets.";
     var simpleOrBatchText = job.JobType == UploadJobType.Simple ? simpleWorksheetName : batchWorksheetName;
     var errorText = new StringBuilder();
     errorText.Append("The uploaded spreadsheet must contain");
     errorText.Append(simpleOrBatchText);
     var error = CreateError(job.Guid, UploadJobErrorType.WorkSheetValidationError,
         errorText.ToString(), null);
     return error;
 }
 private void CheckDuplicateRowsInWorksheet(UploadJob job, SimpleUpload simpleUpload, ref DataTable pholioData)
 {
     if (simpleUpload.DuplicateRowInSpreadsheetErrors.Count > 0)
     {
         var dataWithoutDuplicates = new FileDuplicationHandler().RemoveDuplicatesInSimple(pholioData);
         pholioData = dataWithoutDuplicates;
         _logger.Info("Job# {0}, There are duplicate rows in spreadsheet", job.Guid);
         _logger.Info("Job# {0}, Dupllicate rows removed", job.Guid);
     }
 }