public IExtractionResults Read(IExtractionArguments arguments, string sourceSystem) { IExtractionResults results = new ExtractionResults(); var matchingExtractionResult = _dataContext.ExtractionResultsViews.Where(sqlResult => sqlResult.MakeName == arguments.Make && sqlResult.ModelName == arguments.Model && sqlResult.From == arguments.From && sqlResult.To == arguments.To && sqlResult.SourceSystem == sourceSystem); if (!matchingExtractionResult.Any()) return null; var mostRecentExtractionResult = matchingExtractionResult.OrderByDescending(p => p.Id).First(); var vehicles = _dataContext.Vehicles.Where(p => p.ExtractionResults_FK == mostRecentExtractionResult.Id); foreach (var vehicle in vehicles) { results.Vehicles.Add(new Core.Extraction.Vehicle() { Make = mostRecentExtractionResult.MakeName, Model = mostRecentExtractionResult.ModelName, Milage = vehicle.Milage.ToString(), Price = Convert.ToDouble(vehicle.Price), Title = vehicle.Title, Year = vehicle.Year }); } return results; }
private static void ExtractDataSequence() { bool more = true; while (more) { Console.WriteLine("ENTER MAKE"); var make = Console.ReadLine(); Console.WriteLine("ENTER MODEL"); var model = Console.ReadLine(); var from = 2001; var to = 2014; var mm = _sqlExtractRepository.ReadVehicleMakeModel(_sourceSystem).FirstOrDefault(p => p.Make == make && p.Models.Contains(model)); if (mm == null) { Console.WriteLine("Make and Model does not exist"); continue; } IExtractionArguments args = ExtractionArguments.Create(make, model, from, to); IExtractionResults results = _sqlExtractRepository.Read(args, _sourceSystem); if (results == null) { results = new ExtractionResults(); IExtractionEngine engine; if (_sourceSystem == GooNetConst) engine = new GooExtractionEngine(_japaneseHtmlWebWrapper, _log, _gooNetPageScraper); else { engine = new AutoTraderExtractionEngine(_htmlWebWrapper, _log, _autoTraderScraper); } engine.Extract(args, results); _sqlExtractRepository.Write(args, results, _sourceSystem); } StatisticsPrinter.Print(StatisticsFactory.GetStatistics(results.Vehicles)); StatisticsPrinter.SaveToCsv(StatisticsFactory.GetStatistics(results.Vehicles),_sourceSystem, "C:\\temp\\VehicleStatsApp"); Console.WriteLine("MORE? Y\\N"); if (Console.ReadLine().ToUpper() == "N") more = false; } }
public void test_extractionResults_operation_duration_works() { _pageScraper.Stub(p => p.GetFirstPageUrl(_extractionArgs)).Return(_dummyUri); _htmlWrapper.Stub(p => p.Load(_dummyUri.OriginalString)).Return(_dummyFirstDocument); _pageScraper.Stub(p => p.GetRemainingUrls(_dummyFirstDocument)).Return(new List<string>()); var extractionResults = new ExtractionResults(); _extractorEngine.Extract(_extractionArgs, extractionResults); Assert.AreNotEqual(extractionResults.OperationDuration.TotalMilliseconds, 0); }