public async Task ParseTest(string url, string searchPhrase)
{
var price = await PriceParser.Parse(new Item
{
Name = searchPhrase,
Url = url
});
Assert.True(price > 0, "Price can't be below zero.");
}
public void Parse_Num_dash_Num_Test()
{
var str = "100-345";
var nums = PriceParser.Parse(str);
foreach (var item in nums)
{
Trace.WriteLine(item);
}
}
/// <summary>
/// Finds all days from the calendar which have the Lowest Price
/// </summary>
/// <returns> List<IWebElement> of the same lowest prices </returns>
private static List <IWebElement> FindDaysWithLowestPrice(List <IWebElement> calendar, double lowestPrice)
{
foreach (var dayPrice in calendar)
{
if (PriceParser.ParsePrice(dayPrice.Text) != lowestPrice)
{
calendar.Remove(dayPrice);
}
}
return(calendar);
}
public ElectronicBookDataParser(ElectronicBookExcelData fileData, IDataRepository repository) : base(fileData)
{
_nameParser = new StringParser <ElectronicBookColumnNames>(ElectronicBookColumnNames.Name);
_priceParser = new PriceParser <ElectronicBookColumnNames>(ElectronicBookColumnNames.Price);
_descriptionParser = new StringParser <ElectronicBookColumnNames>(ElectronicBookColumnNames.Description);
_manufacturerParser = new ManufacturerParser <ElectronicBookColumnNames>(ElectronicBookColumnNames.Manufacturer, repository);
_screenDiagonalParser = new DoubleParser <ElectronicBookColumnNames>(ElectronicBookColumnNames.ScreenDiagonal);
_screenTypeParser = new ScreenTypeParser <ElectronicBookColumnNames>(ElectronicBookColumnNames.ScreenType, repository);
_batteryCapacityParser = new IntParser <ElectronicBookColumnNames>(ElectronicBookColumnNames.BatteryCapacity);
_workingTimeParser = new StringParser <ElectronicBookColumnNames>(ElectronicBookColumnNames.WorkingTime);
}
public SmartWatchDataParser(SmartWatchExcelData fileData, IDataRepository repository) : base(fileData)
{
_nameParser = new StringParser <SmartWatchColumnNames>(SmartWatchColumnNames.Name);
_priceParser = new PriceParser <SmartWatchColumnNames>(SmartWatchColumnNames.Price);
_descriptionParser = new StringParser <SmartWatchColumnNames>(SmartWatchColumnNames.Description);
_manufacturerParser = new ManufacturerParser <SmartWatchColumnNames>(SmartWatchColumnNames.Manufacturer, repository);
_connectionTypeParser = new ConnectionTypeParser <SmartWatchColumnNames>(SmartWatchColumnNames.InterfaceForConnecting, repository);
_screenDiagonalParser = new DoubleParser <SmartWatchColumnNames>(SmartWatchColumnNames.ScreenDiagonal);
_pulsometerParser = new BoolParser <SmartWatchColumnNames>(SmartWatchColumnNames.Pulsometer);
_simCardParser = new BoolParser <SmartWatchColumnNames>(SmartWatchColumnNames.SimCard);
}
public PhoneDataParser(PhoneExcelData fileData, IDataRepository repository) : base(fileData)
{
_nameParser = new StringParser <PhoneColumnNames>(PhoneColumnNames.Name);
_priceParser = new PriceParser <PhoneColumnNames>(PhoneColumnNames.Price);
_descriptionParser = new StringParser <PhoneColumnNames>(PhoneColumnNames.Description);
_manufacturerParser = new ManufacturerParser <PhoneColumnNames>(PhoneColumnNames.Manufacturer, repository);
_ramParser = new IntParser <PhoneColumnNames>(PhoneColumnNames.RAM);
_romParser = new IntParser <PhoneColumnNames>(PhoneColumnNames.ROM);
_cpuParser = new StringParser <PhoneColumnNames>(PhoneColumnNames.CPU);
_batteryCapacityParser = new IntParser <PhoneColumnNames>(PhoneColumnNames.BatteryCapacity);
_screenDiagonalParser = new DoubleParser <PhoneColumnNames>(PhoneColumnNames.ScreenDiagonal);
_cameraParser = new DoubleParser <PhoneColumnNames>(PhoneColumnNames.Camera);
}
public void ShouldGetPrices()
{
const double meatPrice = 12.3;
const double fruitPrice = 2.3;
var priceString = $@"Meat={meatPrice}
Fruit={fruitPrice}";
var reader = new StringReader(priceString);
var parser = new PriceParser();
var prices = parser.ParsePrices(reader);
Assert.Equal(fruitPrice, prices.Fruits);
Assert.Equal(meatPrice, prices.Meat);
}
/// <summary>
/// Finds the lowest price of the month
/// </summary>
/// <returns> double, minPrice </returns>
public static double GetMinPrice()
{
List <IWebElement> allPrices = CheapFlightPage.CalendarPrices;
var validListOfPrices = GetDaysWithPrices(allPrices);
var minPrice = PriceParser.ParsePrice(validListOfPrices[0].Text);
foreach (var pricePerOneDay in validListOfPrices)
{
if (PriceParser.ParsePrice(pricePerOneDay.Text) < minPrice)
{
var lowestPrice = pricePerOneDay;
minPrice = PriceParser.ParsePrice(lowestPrice.Text);
}
}
return(minPrice);
}
/// <summary>
/// Gets all days wich have any price higher than zero
/// </summary>
/// <returns> List<IWebElement> of prices of these days </returns>
public static List <IWebElement> GetDaysWithPrices(List <IWebElement> allDays)
{
var counter = 0;
indexesOfEmptyDays = new List <int>();
foreach (var dayPrice in allDays.ToList())
{
counter++;
if (PriceParser.ParsePrice(dayPrice.Text) == 0)
{
indexesOfEmptyDays.Add(counter);
allDays.Remove(dayPrice);
}
}
return(allDays);
}
public void Parse_Num_comma_Num_comma_Num_Test()
{
var str = "100,456,1234";
var numsActual = PriceParser.Parse(str);
var numsExpect = new List <int>()
{
100, 456, 1234
};
for (int i = 0; i < 3; i++)
{
Trace.WriteLine("actual: " + numsActual[i]);
Trace.WriteLine("expect: " + numsExpect[i]);
Trace.WriteLine("-----------");
Assert.AreEqual(numsActual[i], numsExpect[i]);
}
}
public override async Task ProcessDataAsync()
{
var scrapedSaleAnnounces = new List <TransportSaleAnnounce>();
while (ItemsCountForStep > scrapedSaleAnnounces.Count)
{
try
{
var urlForScrapping = UrlForScrapping + $"&start={ScrappingPage}";
var htmlDocument = await DataLoader.LoadHtmlDocumentAsync(urlForScrapping);
var previewOffers = StructureExtractor.GetPreviewOfferStructure(htmlDocument, ScrappingPage).ToList();
for (int i = 0; i < previewOffers.Count; i++)
{
tryAgain:
try
{
var sourceLink = SourceLinkParser.GetLink(previewOffers[i], BaseUrl);
var detailedHtmlDocument = await DataLoader.LoadHtmlDocumentAsync(sourceLink);
var detailedOfferNode = StructureExtractor.GetDetailedOfferStructure(detailedHtmlDocument);
var(brandId, modelId) = BrandModelParser.ParseForDetailed(detailedOfferNode);
var saleAnnounce = new TransportSaleAnnounce()
{
SourceLink = sourceLink,
TransmissionTypeId = TransmissionTypeParser.ParseForDetailed(detailedOfferNode),
AdNumber = OfferNumberParser.ParseForDetailed(detailedOfferNode),
BodyTypeId = BodyTypeParser.ParseForDetailed(detailedOfferNode),
CityId = RegionParser.ParseForDetailed(detailedOfferNode).Id,
Description = DescriptionParser.ParseForDetailed(detailedOfferNode),
DriveUnitId = DriveUnitParser.ParseForDetailed(detailedOfferNode),
EngineVolumetric = EngineVolumetricParser.ParseForDetailed(detailedOfferNode),
FuelTypeId = FuelTypeParser.ParseForDetailed(detailedOfferNode),
PreviewImageLink = ImageLinkParser.ParseForPreview(previewOffers[i]),
Mileage = MileageParser.ParseForDetailed(detailedOfferNode),
BrandId = brandId,
VehicleTypeId = VehicleTypeParser.ParseForDetailed(detailedOfferNode),
ModelId = modelId,
PriceInDollars = PriceParser.ParseForDetailed(detailedOfferNode),
UpdateOfferTime = PublishDateParser.ParseForDetailed(detailedOfferNode),
Year = YearParser.ParseForDetailed(detailedOfferNode),
CreatedAt = DateTime.Now,
SourceProviderId = (int)SourceProviderEnum.RST
};
var carConditionIds = CarConditionParser.ParseForPreview(previewOffers[i]);
foreach (var carConditionId in carConditionIds)
{
saleAnnounce.TransportConditions.Add(new TransportConditionInSaleAnnounce()
{
TransportConditionId = carConditionId
});
}
scrapedSaleAnnounces.Add(saleAnnounce);
UnitOfWork.TransportSaleAnnouncesRepository.Add(saleAnnounce);
await UnitOfWork.SaveChangesAsync();
ScrappingPage++;
}
catch (WebException)
{
goto tryAgain;
}
}
}
catch (Exception ex)
{
_logger.LogError($"{ex.Message}\n{ex.StackTrace}");
}
}
//await UnitOfWork.TransportSaleAnnouncesRepository.InsertRangeSaleAnnounces(scrapedSaleAnnounces);
}
public void Setup()
{
_mock = AutoMock.GetStrict();
_mock.VerifyAll = true;
_parser = new PriceParser(_mock.Mock <INumberParser>().Object);
}
public static double GetPriceOfDayLabeledLowestFare()
{
var dateOdElementWithLowestFare = DriverFactory.Driver.FindElement(By.XPath(LowestFareXPath));
return(PriceParser.ParsePrice(dateOdElementWithLowestFare.Text));
}
public dynamic Search(float lat = -1, float lon = -1, int hours = 1, double speed = 0.0)
{
Response.Headers.Add("Access-Control-Allow-Origin", new string[] { "*" });
// constant parameters
int TAKE = 5;
double PARKING_RADIUS = 5.0;
double SLOW_SPEED = 3.0;
// validity check: are lat and lon specified?
if (lat < 0 || lon < 0)
{
Response.StatusCode = 400;
Dictionary <string, string> err = new Dictionary <string, string>();
err.Add("error", "either 'lat' or 'lon' not defined as parameters");
return(err);
}
Console.WriteLine("----------------");
// create coordinate model form user given parameter
CoordinateModel currentPosition = new CoordinateModel();
currentPosition.Latitude = lat;
currentPosition.Longitude = lon;
// use connection to mongodb
var server = StaticGlobal.MongoDBClient.GetServer();
var database = server.GetDatabase("parkeasy");
var collectionParking = database.GetCollection <ParkingModel>("parking");
var collectionStatus = database.GetCollection <StatusModel>("status");
// load all parking options from mongodb
List <ParkingModel> parkingModels = new List <ParkingModel>();
StringBuilder osrmTableQueryBuilder = new StringBuilder("http://router.project-osrm.org/table?");
osrmTableQueryBuilder.AppendFormat("loc={0},{1}&", Math.Round(lat, 5), Math.Round(lon, 5));
// fetch all parking options sorted by nearest
var query = Query.Near("Coordinates", currentPosition.Longitude, currentPosition.Latitude, 1.5 / 111.12);
foreach (ParkingModel model in collectionParking.Find(query).SetLimit(100))
{
model.CalcFreeLikelihood();
// calculate the distance to the user and add to working list
//model.DistanceToUser = model.Coordinate.DistanceTo(currentPosition);
osrmTableQueryBuilder.AppendFormat("loc={0},{1}&", Math.Round(model.Coordinate.Latitude, 5), Math.Round(model.Coordinate.Longitude, 5));
parkingModels.Add(model);
}
String osrmTableQuery = osrmTableQueryBuilder.ToString();
osrmTableQuery = osrmTableQuery.Remove(osrmTableQuery.Length - 1);
// query osrm for distance table
using (var client = new System.Net.WebClient())
{
client.Headers.Add("User-Agent", "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.2.12) Gecko/20101026 Firefox/3.6.12");
client.Headers.Add("Accept", "*/*");
String body = client.DownloadString(osrmTableQuery);
dynamic table = JsonConvert.DeserializeObject(body);
int i = 0;
foreach (int distance in table.distance_table[0])
{
if (i == 0)
{
i++;
continue;
}
Console.WriteLine(distance);
parkingModels.ElementAt(i - 1).DistanceToUser = Convert.ToDouble(distance);
i++;
}
}
Console.WriteLine("{0} parking options", parkingModels.Count);
// filter places that have maximum parking hours lower than needed or
// opening hours that exceed the amount of time the user wants to park
parkingModels = parkingModels.Where(delegate(ParkingModel a)
{
if (a.FreeLikelihood == 0.0 || a.Price == null)
{
return(false);
}
// are there limitations on the amount of parking hours?
if (a.MaximumParkingHours.HasValue)
{
// does this exceed the amount of hours the user wants to park?
if (a.MaximumParkingHours.Value > 0.0 && a.MaximumParkingHours.Value > hours)
{
return(false);
}
}
// are there opening hours?
if (a.OpeningHours != null)
{
// find opening hours of current day
int day = (int)DateTime.Now.DayOfWeek;
OpeningHoursModel openingModel = a.OpeningHours[day];
// is the parking closed today?
if (openingModel.Closed == true)
{
return(false);
}
// check if current time + parking duration in limits of opening hours
int parkingDurationAbsolute = Convert.ToInt32(string.Format("{0:HHmm}", DateTime.Now.AddHours(hours)));
if (openingModel.Open > parkingDurationAbsolute && parkingDurationAbsolute > openingModel.Close)
{
return(false);
}
}
return(true);
}).ToList <ParkingModel>();
Console.WriteLine("{0} parking options that are open/available", parkingModels.Count);
// dictionary to store return values that translate to JSON
Dictionary <string, object> returnValues = new Dictionary <string, object>();
// there are no available parking options in the area
if (parkingModels.Count == 0)
{
// set appropriate state
returnValues.Add("state", "driving");
returnValues.Add("parking", parkingModels);
return(returnValues);
}
// store the current first one in the list since it will be the closest
// due to mongodb's geosearch
ParkingModel closestModel = parkingModels.First();
// sort by closeness to current position
parkingModels.Sort(delegate(ParkingModel a, ParkingModel b)
{
double scoreA = a.DistanceToUser * PriceParser.Interpret(a.Price, hours) * a.FreeLikelihood;
double scoreB = b.DistanceToUser * PriceParser.Interpret(b.Price, hours) * b.FreeLikelihood;
if (scoreA > scoreB)
{
return(1);
}
else if (scoreA < scoreB)
{
return(-1);
}
else
{
return(0);
}
});
// check if a user is right on the closest parkingspot,
// that would mean, that the STATE 'parked' would be induced
if (closestModel.DistanceToUser <= PARKING_RADIUS / 1000.0 && speed <= SLOW_SPEED)
{
// set appropriate state
returnValues.Add("state", "parking");
returnValues.Add("distance", Math.Round(closestModel.DistanceToUser, 2));
// append information about the parking spot we are on right now
Dictionary <string, object> data = new Dictionary <string, object>();
data.Add("id", closestModel.Id);
data.Add("name", closestModel.Name);
data.Add("price", closestModel.Price.PerHour.Price);
data.Add("type", closestModel.Type);
data.Add("redpoint", closestModel.RedPointText);
data.Add("description", closestModel.Description);
returnValues.Add("parking", data);
}
// user is still driving, we will offer him $(TAKE) of the best
// parking spots sorrounding him
else
{
// prepare the parking models for displaying on
// the enduser device
List <Dictionary <string, object> > parking = new List <Dictionary <string, object> >();
// append good parking options around user
foreach (ParkingModel model in parkingModels.Take(TAKE))
{
Console.WriteLine(PriceParser.Interpret(model.Price, hours));
Dictionary <string, object> data = new Dictionary <string, object>();
data.Add("id", model.Id);
data.Add("name", model.Name);
data.Add("price", PriceParser.Interpret(model.Price, hours));
data.Add("type", model.Type);
data.Add("coord", model.Coordinates);
data.Add("free", model.FreeLikelihood);
parking.Add(data);
}
// set appropriate state
returnValues.Add("state", "driving");
returnValues.Add("parking", parking);
}
return(returnValues);
}
private static async Task CheckItemPrice(Item item)
{
if (item.SubscribersEmails.Length == 0)
{
await DeleteUnusedItem(item);
return;
}
decimal currentPrice = 0;
try
{
currentPrice = await PriceParser.Parse(item, cache);
}
catch (Exception ex)
{
log.LogError($"Can't parse url {item.Url}: {ex.Message}.");
Errors += $"Can't parse url {item.Url}: {ex.Message}.<br>";
return;
}
log.LogInformation($"Item {item.Name} price: {currentPrice} rub.");
Price prevPrice;
try
{
prevPrice = await priceDb.GetLastItemPriceAsync(item);
}
catch
{
try
{
await priceDb.CreateItemPriceAsync(item, currentPrice);
}catch (Exception ex)
{
log.LogError($"Can't write item price to db: {ex.Message}.");
}
return;
}
if (currentPrice < prevPrice.ItemPrice)
{
await NotifyOfPriceReduction(item, currentPrice, prevPrice);
}
if (currentPrice == prevPrice.ItemPrice)
{
return;
}
Price newPrice;
try
{
newPrice = await priceDb.CreateItemPriceAsync(item, currentPrice);
}
catch (Exception ex)
{
log.LogError($"Can't save item {item.Name} price {currentPrice} to db: {ex.Message}.");
}
}
public dynamic Scrape()
{
List <ParkingModel> parkingModels = new List <ParkingModel>();
// open connection to mongodb
var server = StaticGlobal.MongoDBClient.GetServer();
var database = server.GetDatabase("parkeasy");
var collection = database.GetCollection <ParkingModel>("parking");
// GARAGE DATA //
Dictionary <string, dynamic> jsonGarage;
// load data from remote source
Console.WriteLine("Garages from WWW");
using (var client = new System.Net.WebClient())
{
var body = client.DownloadString("http://www.koeln.de/apps/parken/json/current");
jsonGarage = JsonConvert.DeserializeObject <Dictionary <string, dynamic> >(body);
}
List <string> prices = new List <string>();
// parse parking garages into our parking model
foreach (string key in jsonGarage.Keys)
{
dynamic obj = jsonGarage[key];
ParkingModel model = new ParkingModel();
// parse garage data
model.Type = ParkingType.Garage;
model.Name = obj.title;
model.Id = obj.shortname;
model.Capacity = obj.capacity;
model.Free = obj.free;
try
{
model.CapacityWomen = obj.womens_parking;
}
catch (Exception e) {}
try
{
model.Price = PriceParser.Parse(Convert.ToString(obj.price));
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
}
model.Description = obj.fulltext;
var collectionStatus = database.GetCollection <ParkingModel>("status");
// store the "free" parking spaces information
StatusModel status = new StatusModel();
status.ParkingId = model.Id;
status.Amount = obj.free;
status.Time = DateTime.UtcNow;
status.Id = ObjectId.GenerateNewId();
status.HighQualitySample = true;
collectionStatus.Insert(status);
// parse coordinates
CoordinateModel coordinateModel = new CoordinateModel();
coordinateModel.Latitude = Convert.ToDouble(obj.lat);
coordinateModel.Longitude = Convert.ToDouble(obj.lng);
model.Coordinate = coordinateModel;
model.Coordinates = new double[2] {
coordinateModel.Longitude, coordinateModel.Latitude
};
try
{
model.OpeningHours = new OpeningHoursModel[7];
// loop a whole week
for (int i = 0; i <= 6; i++)
{
// extract opening times for every day of the week
OpeningHoursModel hoursModel = new OpeningHoursModel();
hoursModel.Open = Convert.ToInt32(Convert.ToString(obj.open_time).Replace(":", ""));
hoursModel.Close = Convert.ToInt32(Convert.ToString(obj.close_time).Replace(":", ""));
model.OpeningHours[i] = hoursModel;
}
}
catch (Exception e) {
Console.WriteLine(e.ToString());
}
parkingModels.Add(model);
}
// MACHINE DATA //
dynamic jsonMachine;
// load data from remote source
Console.WriteLine("Machine from WWW");
using (var client = new System.Net.WebClient())
{
var body = client.DownloadString("http://geoportal1.stadt-koeln.de/ArcGIS/rest/services/66/Parkscheinautomaten/MapServer/0/query?text=&geometry=&geometryType=esriGeometryPoint&inSR=&spatialRel=esriSpatialRelIntersects&relationParam=&objectIds=&where=id%20is%20not%20null&time=&returnCountOnly=false&returnIdsOnly=false&returnGeometry=true&maxAllowableOffset=&outSR=4326&outFields=%2A&f=json");
jsonMachine = JsonConvert.DeserializeObject(body);
}
// parse parking machines into our parking model
foreach (dynamic machine in jsonMachine.features)
{
ParkingModel model = new ParkingModel();
// parse garage data
model.Type = ParkingType.TicketMachine;
model.Name = machine.attributes.Aufstellort;
model.Id = Convert.ToString(machine.attributes.ID);
model.Capacity = machine.attributes.Stellplaetze;
model.Price = new PriceModel(Convert.ToDouble(machine.attributes.Gebuehr));
model.MaximumParkingHours = Convert.ToDouble(machine.attributes.Hoechstparkdauer);
model.RedPointText = machine.attributes.Roter_Punkt_Text;
model.SectionFrom = machine.attributes.Abschnitt_Von;
model.SectionTo = machine.attributes.Abschnitt_Bis;
// parse coordinates
CoordinateModel coordinateModel = new CoordinateModel();
coordinateModel.Latitude = machine.geometry.y;
coordinateModel.Longitude = machine.geometry.x;
model.Coordinate = coordinateModel;
model.Coordinates = new double[2] {
coordinateModel.Longitude, coordinateModel.Latitude
};
parkingModels.Add(model);
}
// UNI DATA //
dynamic jsonUni;
// load data from remote source
Console.WriteLine("Uni from WWW");
using (var client = new System.Net.WebClient())
{
var body = client.DownloadString("https://github.com/ParkEasy/tools/releases/download/v1/uni.json");
jsonUni = JsonConvert.DeserializeObject(body);
}
// loop university parking spaces
foreach (dynamic uniparking in jsonUni)
{
ParkingModel model = new ParkingModel();
model.Id = uniparking.name;
model.Type = ParkingType.University;
model.Name = uniparking.name;
model.Description = string.Join(", ", uniparking.descriptions);
model.Capacity = uniparking.num_spaces;
model.CapacityDisabled = uniparking.num_disabled;
model.CapacityService = uniparking.num_service;
model.Gates = uniparking.gates;
model.Price = new PriceModel(1.3);
// parse opening hours
if (DynamicExist(uniparking, "hours"))
{
model.OpeningHours = new OpeningHoursModel[7];
int i = 0;
foreach (dynamic hour in uniparking.hours)
{
OpeningHoursModel hoursModel = new OpeningHoursModel();
if (hour != null)
{
hoursModel.Open = hour.open;
hoursModel.Close = hour.close;
}
else
{
hoursModel.Closed = true;
}
model.OpeningHours[i] = hoursModel;
i++;
}
}
// parse coordinates
CoordinateModel coordinateModel = new CoordinateModel();
coordinateModel.Latitude = uniparking.coordinates.latitude;
coordinateModel.Longitude = uniparking.coordinates.longitude;
model.Coordinate = coordinateModel;
model.Coordinates = new double[2] {
coordinateModel.Longitude, coordinateModel.Latitude
};
parkingModels.Add(model);
}
// upsert each of the available parking options
foreach (ParkingModel model in parkingModels)
{
collection.Save(model);
}
return(parkingModels.Count);
}
