protected override Releaseability GetSpecificAssessment(IExtractionResults extractionResults)
{
var _extractDir = Configuration.GetProject().ExtractionDirectory;
ExtractDirectory = new ExtractionDirectory(_extractDir, Configuration).GetDirectoryForDataset(DataSet);
var externalServerId = int.Parse(extractionResults.DestinationDescription.Split('|')[0]);
var externalServer = _repositoryLocator.CatalogueRepository.GetObjectByID <ExternalDatabaseServer>(externalServerId);
var dbName = extractionResults.DestinationDescription.Split('|')[1];
var tblName = extractionResults.DestinationDescription.Split('|')[2];
var server = DataAccessPortal.GetInstance().ExpectServer(externalServer, DataAccessContext.DataExport, setInitialDatabase: false);
var database = server.ExpectDatabase(dbName);
if (!database.Exists())
{
return(Releaseability.ExtractFilesMissing);
}
var foundTable = database.ExpectTable(tblName);
if (!foundTable.Exists())
{
return(Releaseability.ExtractFilesMissing);
}
return(Releaseability.Undefined);
}
public void Extract(IExtractionArguments args, IExtractionResults extractionResults)
{
if (extractionResults == null) throw new ArgumentNullException("extractionResults");
_log.Debug("Extraction starting");
extractionResults.Start();
var firstPageUrl = _pageScraper.GetFirstPageUrl(args);
HtmlDocument firstPage = _htmlWebWrapper.Load(firstPageUrl.OriginalString);
extractionResults.Vehicles.AddRange(_pageScraper.Scrape(args, firstPage));
List<string> remainingUrls = _pageScraper.GetRemainingUrls(firstPage);
Parallel.ForEach(remainingUrls, pagedUrl =>
{
var resultsPage = _htmlWebWrapper.Load(pagedUrl);
lock (_lockObject)
{
extractionResults.Vehicles.AddRange(_pageScraper.Scrape(args, resultsPage));
}
});
extractionResults.Stop();
}
protected override Releaseability GetSupplementalSpecificAssessment(IExtractionResults supplementalExtractionResults)
{
if (File.Exists(supplementalExtractionResults.DestinationDescription))
{
return(Releaseability.Undefined);
}
return(Releaseability.ExtractFilesMissing);
}
private bool SqlOutOfSyncWithDataExportManagerConfiguration(IExtractionResults extractionResults)
{
if (extractionResults.SQLExecuted == null)
{
throw new Exception("Cumulative Extraction Results for the extraction in which this dataset was involved in does not have any SQLExecuted recorded for it.");
}
//if the SQL today is different to the SQL that was run when the user last extracted the data then there is a desync in the SQL (someone has changed something in the catalogue/data export manager configuration since the data was extracted)
return(!SqlCurrentConfiguration.Equals(extractionResults.SQLExecuted));
}
protected override Releaseability GetSpecificAssessment(IExtractionResults extractionResults)
{
ExtractDirectory = new FileInfo(extractionResults.DestinationDescription).Directory;
if (FilesAreMissing(extractionResults))
{
return(Releaseability.ExtractFilesMissing);
}
ThrowIfPollutionFoundInConfigurationRootExtractionFolder();
return(Releaseability.Undefined);// Assesment = SqlDifferencesVsLiveCatalogue() ? Releaseability.ColumnDifferencesVsCatalogue : Releaseability.Releaseable;
}
public void Init()
{
_htmlWrapper = MockRepository.GenerateMock<IHtmlWebWrapper>();
_log = MockRepository.GenerateMock<ILog>();
_extractionArgs = MockRepository.GenerateMock<IExtractionArguments>();
_extractionResults = MockRepository.GenerateMock<IExtractionResults>();
_pageScraper = MockRepository.GenerateMock<IAutoTraderZaPageScraper>();
_pageScraper.Stub(p => p.Scrape(Arg<IExtractionArguments>.Is.Anything, Arg<HtmlDocument>.Is.Anything)).Return(new List<IVehicle>());
_extractionResults.Stub(p => p.Vehicles).Return(new List<IVehicle>());
_dummyUri = new Uri("http://dummyUrl.com");
_dummyFirstDocument = new HtmlDocument();
_extractorEngine = new AutoTraderExtractionEngine(_htmlWrapper, _log, _pageScraper);
}
private bool SqlOutOfSyncWithDataExportManagerConfiguration(IExtractionResults extractionResults)
{
if (extractionResults.SQLExecuted == null)
{
throw new Exception("Cumulative Extraction Results for the extraction in which this dataset was involved in does not have any SQLExecuted recorded for it.");
}
// When using extraction progress the SQL can be whatever you want
// if the progress date says End then we blindly assume that whatever you
// executed was legit
if (SelectedDataSet.ExtractionProgressIfAny != null)
{
return(false);
}
//if the SQL today is different to the SQL that was run when the user last extracted the data then there is a desync in the SQL (someone has changed something in the catalogue/data export manager configuration since the data was extracted)
return(!SqlCurrentConfiguration.Equals(extractionResults.SQLExecuted));
}
protected override Releaseability GetSupplementalSpecificAssessment(IExtractionResults supplementalExtractionResults)
{
if (supplementalExtractionResults.IsReferenceTo(typeof(SupportingDocument)))
{
if (File.Exists(supplementalExtractionResults.DestinationDescription))
{
return(Releaseability.Undefined);
}
else
{
return(Releaseability.ExtractFilesMissing);
}
}
if (supplementalExtractionResults.IsReferenceTo(typeof(SupportingSQLTable)) ||
supplementalExtractionResults.IsReferenceTo(typeof(TableInfo)))
{
return(GetSpecificAssessment(supplementalExtractionResults));
}
return(Releaseability.Undefined);
}
private bool FilesAreMissing(IExtractionResults extractionResults)
{
ExtractFile = new FileInfo(extractionResults.DestinationDescription);
var metadataFile = new FileInfo(extractionResults.DestinationDescription.Replace(".csv", ".docx"));
if (!ExtractFile.Exists)
{
return(true);//extract is missing
}
if (!ExtractFile.Extension.Equals(".csv"))
{
throw new Exception("Extraction file had extension '" + ExtractFile.Extension + "' (expected .csv)");
}
if (!metadataFile.Exists)
{
return(true);
}
//see if there is any other polution in the extract directory
FileInfo unexpectedFile = ExtractFile.Directory.EnumerateFiles().FirstOrDefault(f =>
!(f.Name.Equals(ExtractFile.Name) || f.Name.Equals(metadataFile.Name)));
if (unexpectedFile != null)
{
throw new Exception("Unexpected file found in extract directory " + unexpectedFile.FullName + " (pollution of extract directory is not permitted)");
}
DirectoryInfo unexpectedDirectory = ExtractFile.Directory.EnumerateDirectories().FirstOrDefault(d =>
!(d.Name.Equals("Lookups") || d.Name.Equals("SupportingDocuments") || d.Name.Equals(SupportingSQLTable.ExtractionFolderName)));
if (unexpectedDirectory != null)
{
throw new Exception("Unexpected directory found in extraction directory " + unexpectedDirectory.FullName + " (pollution of extract directory is not permitted)");
}
return(false);
}
protected abstract Releaseability GetSpecificAssessment(IExtractionResults extractionResults);
protected abstract Releaseability GetSupplementalSpecificAssessment(IExtractionResults supplementalExtractionResults);
protected override Releaseability GetSpecificAssessment(IExtractionResults extractionResults)
{
return(Releaseability.NeverBeenSuccessfullyExecuted);
}
public void Write(IExtractionArguments arguments, IExtractionResults extractionResults, string sourceSystem)
{
_log.DebugFormat("Writing extractionResults for {0}", extractionResults);
var sourceSystemEntity = _dataContext.SourceSystems.FirstOrDefault(p => p.Name == sourceSystem);
if (sourceSystemEntity == null)
throw new ArgumentException("SourceSystem does not exist in the database", "sourceSystem");
var makeEntity = _dataContext.Makes.FirstOrDefault(p => p.MakeName == arguments.Make && p.SourceSystem == sourceSystemEntity);
if (makeEntity == null)
throw new ArgumentException("Make does not exist in the database", "arguments");
var modelEntity = makeEntity.Models.FirstOrDefault(p => p.ModelName == arguments.Model);
if (modelEntity == null)
throw new ArgumentException("Model does not exist in the database", "arguments");
var resultEntity = new ExtractionResult() { Model = modelEntity, From = arguments.From, To = arguments.To, ExtractionDateTime = DateTime.Now };
_dataContext.ExtractionResults.InsertOnSubmit(resultEntity);
_dataContext.SubmitChanges();
foreach (var vehicle in extractionResults.Vehicles)
{
_dataContext.Vehicles.InsertOnSubmit(new Vehicle()
{
ExtractionResult = resultEntity,
Milage = SafeInt( vehicle.Milage),
Year = vehicle.Year,
Price = Convert.ToDecimal( vehicle.Price),
Title = vehicle.Title
});
}
_dataContext.SubmitChanges();
}
public bool TryRead(IExtractionArguments arguments, out IExtractionResults results)
{
throw new NotImplementedException();
}
