protected override Releaseability GetSpecificAssessment(IExtractionResults extractionResults) { var _extractDir = Configuration.GetProject().ExtractionDirectory; ExtractDirectory = new ExtractionDirectory(_extractDir, Configuration).GetDirectoryForDataset(DataSet); var externalServerId = int.Parse(extractionResults.DestinationDescription.Split('|')[0]); var externalServer = _repositoryLocator.CatalogueRepository.GetObjectByID <ExternalDatabaseServer>(externalServerId); var dbName = extractionResults.DestinationDescription.Split('|')[1]; var tblName = extractionResults.DestinationDescription.Split('|')[2]; var server = DataAccessPortal.GetInstance().ExpectServer(externalServer, DataAccessContext.DataExport, setInitialDatabase: false); var database = server.ExpectDatabase(dbName); if (!database.Exists()) { return(Releaseability.ExtractFilesMissing); } var foundTable = database.ExpectTable(tblName); if (!foundTable.Exists()) { return(Releaseability.ExtractFilesMissing); } return(Releaseability.Undefined); }
public void Extract(IExtractionArguments args, IExtractionResults extractionResults) { if (extractionResults == null) throw new ArgumentNullException("extractionResults"); _log.Debug("Extraction starting"); extractionResults.Start(); var firstPageUrl = _pageScraper.GetFirstPageUrl(args); HtmlDocument firstPage = _htmlWebWrapper.Load(firstPageUrl.OriginalString); extractionResults.Vehicles.AddRange(_pageScraper.Scrape(args, firstPage)); List<string> remainingUrls = _pageScraper.GetRemainingUrls(firstPage); Parallel.ForEach(remainingUrls, pagedUrl => { var resultsPage = _htmlWebWrapper.Load(pagedUrl); lock (_lockObject) { extractionResults.Vehicles.AddRange(_pageScraper.Scrape(args, resultsPage)); } }); extractionResults.Stop(); }
protected override Releaseability GetSupplementalSpecificAssessment(IExtractionResults supplementalExtractionResults) { if (File.Exists(supplementalExtractionResults.DestinationDescription)) { return(Releaseability.Undefined); } return(Releaseability.ExtractFilesMissing); }
private bool SqlOutOfSyncWithDataExportManagerConfiguration(IExtractionResults extractionResults) { if (extractionResults.SQLExecuted == null) { throw new Exception("Cumulative Extraction Results for the extraction in which this dataset was involved in does not have any SQLExecuted recorded for it."); } //if the SQL today is different to the SQL that was run when the user last extracted the data then there is a desync in the SQL (someone has changed something in the catalogue/data export manager configuration since the data was extracted) return(!SqlCurrentConfiguration.Equals(extractionResults.SQLExecuted)); }
protected override Releaseability GetSpecificAssessment(IExtractionResults extractionResults) { ExtractDirectory = new FileInfo(extractionResults.DestinationDescription).Directory; if (FilesAreMissing(extractionResults)) { return(Releaseability.ExtractFilesMissing); } ThrowIfPollutionFoundInConfigurationRootExtractionFolder(); return(Releaseability.Undefined);// Assesment = SqlDifferencesVsLiveCatalogue() ? Releaseability.ColumnDifferencesVsCatalogue : Releaseability.Releaseable; }
public void Init() { _htmlWrapper = MockRepository.GenerateMock<IHtmlWebWrapper>(); _log = MockRepository.GenerateMock<ILog>(); _extractionArgs = MockRepository.GenerateMock<IExtractionArguments>(); _extractionResults = MockRepository.GenerateMock<IExtractionResults>(); _pageScraper = MockRepository.GenerateMock<IAutoTraderZaPageScraper>(); _pageScraper.Stub(p => p.Scrape(Arg<IExtractionArguments>.Is.Anything, Arg<HtmlDocument>.Is.Anything)).Return(new List<IVehicle>()); _extractionResults.Stub(p => p.Vehicles).Return(new List<IVehicle>()); _dummyUri = new Uri("http://dummyUrl.com"); _dummyFirstDocument = new HtmlDocument(); _extractorEngine = new AutoTraderExtractionEngine(_htmlWrapper, _log, _pageScraper); }
private bool SqlOutOfSyncWithDataExportManagerConfiguration(IExtractionResults extractionResults) { if (extractionResults.SQLExecuted == null) { throw new Exception("Cumulative Extraction Results for the extraction in which this dataset was involved in does not have any SQLExecuted recorded for it."); } // When using extraction progress the SQL can be whatever you want // if the progress date says End then we blindly assume that whatever you // executed was legit if (SelectedDataSet.ExtractionProgressIfAny != null) { return(false); } //if the SQL today is different to the SQL that was run when the user last extracted the data then there is a desync in the SQL (someone has changed something in the catalogue/data export manager configuration since the data was extracted) return(!SqlCurrentConfiguration.Equals(extractionResults.SQLExecuted)); }
protected override Releaseability GetSupplementalSpecificAssessment(IExtractionResults supplementalExtractionResults) { if (supplementalExtractionResults.IsReferenceTo(typeof(SupportingDocument))) { if (File.Exists(supplementalExtractionResults.DestinationDescription)) { return(Releaseability.Undefined); } else { return(Releaseability.ExtractFilesMissing); } } if (supplementalExtractionResults.IsReferenceTo(typeof(SupportingSQLTable)) || supplementalExtractionResults.IsReferenceTo(typeof(TableInfo))) { return(GetSpecificAssessment(supplementalExtractionResults)); } return(Releaseability.Undefined); }
private bool FilesAreMissing(IExtractionResults extractionResults) { ExtractFile = new FileInfo(extractionResults.DestinationDescription); var metadataFile = new FileInfo(extractionResults.DestinationDescription.Replace(".csv", ".docx")); if (!ExtractFile.Exists) { return(true);//extract is missing } if (!ExtractFile.Extension.Equals(".csv")) { throw new Exception("Extraction file had extension '" + ExtractFile.Extension + "' (expected .csv)"); } if (!metadataFile.Exists) { return(true); } //see if there is any other polution in the extract directory FileInfo unexpectedFile = ExtractFile.Directory.EnumerateFiles().FirstOrDefault(f => !(f.Name.Equals(ExtractFile.Name) || f.Name.Equals(metadataFile.Name))); if (unexpectedFile != null) { throw new Exception("Unexpected file found in extract directory " + unexpectedFile.FullName + " (pollution of extract directory is not permitted)"); } DirectoryInfo unexpectedDirectory = ExtractFile.Directory.EnumerateDirectories().FirstOrDefault(d => !(d.Name.Equals("Lookups") || d.Name.Equals("SupportingDocuments") || d.Name.Equals(SupportingSQLTable.ExtractionFolderName))); if (unexpectedDirectory != null) { throw new Exception("Unexpected directory found in extraction directory " + unexpectedDirectory.FullName + " (pollution of extract directory is not permitted)"); } return(false); }
protected abstract Releaseability GetSpecificAssessment(IExtractionResults extractionResults);
protected abstract Releaseability GetSupplementalSpecificAssessment(IExtractionResults supplementalExtractionResults);
protected override Releaseability GetSpecificAssessment(IExtractionResults extractionResults) { return(Releaseability.NeverBeenSuccessfullyExecuted); }
public void Write(IExtractionArguments arguments, IExtractionResults extractionResults, string sourceSystem) { _log.DebugFormat("Writing extractionResults for {0}", extractionResults); var sourceSystemEntity = _dataContext.SourceSystems.FirstOrDefault(p => p.Name == sourceSystem); if (sourceSystemEntity == null) throw new ArgumentException("SourceSystem does not exist in the database", "sourceSystem"); var makeEntity = _dataContext.Makes.FirstOrDefault(p => p.MakeName == arguments.Make && p.SourceSystem == sourceSystemEntity); if (makeEntity == null) throw new ArgumentException("Make does not exist in the database", "arguments"); var modelEntity = makeEntity.Models.FirstOrDefault(p => p.ModelName == arguments.Model); if (modelEntity == null) throw new ArgumentException("Model does not exist in the database", "arguments"); var resultEntity = new ExtractionResult() { Model = modelEntity, From = arguments.From, To = arguments.To, ExtractionDateTime = DateTime.Now }; _dataContext.ExtractionResults.InsertOnSubmit(resultEntity); _dataContext.SubmitChanges(); foreach (var vehicle in extractionResults.Vehicles) { _dataContext.Vehicles.InsertOnSubmit(new Vehicle() { ExtractionResult = resultEntity, Milage = SafeInt( vehicle.Milage), Year = vehicle.Year, Price = Convert.ToDecimal( vehicle.Price), Title = vehicle.Title }); } _dataContext.SubmitChanges(); }
public bool TryRead(IExtractionArguments arguments, out IExtractionResults results) { throw new NotImplementedException(); }