public async Task PopulateDbWithHikesAsync()
{
var concurentBag = new ConcurrentBag <HikeDto>();
for (int i = 0; i <= 35; i++)
{
var url = new Url($"https://tripsjournal.com/marshruti/page/{i}");
var hikeDocument = this.context.OpenAsync(url).GetAwaiter().GetResult();
var allHikes = hikeDocument.QuerySelectorAll("div.list-item");
foreach (var hike in allHikes)
{
var hikeUrl = hike.QuerySelector("a").GetAttribute("href");
try
{
var hikeDto = this.GetHike(hikeUrl);
concurentBag.Add(hikeDto);
}
catch (Exception)
{
}
}
}
foreach (var hike in concurentBag)
{
var categoryId = await this.GetOrCreateCategoryAsync(hike.CategoryName);
var regionId = await this.GetOrCreateRegionAsync(hike.RegionName);
var mountainId = await this.GetOrCreateMountainAsync(hike.MountainName);
var startPointId = await this.GetOrCreateStartPointAsync(
hike.HikeStartPointName,
hike.HikeStartPointAltitude,
hike.HikeStartPointLatitude,
hike.HikeStartPointLongitude);
var endPointId = await this.GetOrCreateEndPointAsync(
hike.HikeEndPointName,
hike.HikeEndPointAltitude,
hike.HikeEndPointLatitude,
hike.HikeEndPointLongitude);
var landmarkExists = this.hikesRepository.AllAsNoTracking().Any(x => x.Name == hike.Name);
if (landmarkExists)
{
continue;
}
var newHike = new Hike()
{
Name = hike.Name,
CategoryId = categoryId,
RegionId = regionId,
MountainId = mountainId,
HikeStartPointId = startPointId,
HikeEndPointId = endPointId,
Length = hike.Length,
Denivelation = hike.Denivelation,
Duration = hike.Duration,
Marking = hike.Marking,
Difficulty = hike.Difficulty,
Description = hike.Description,
};
var imagesCollection = new List <HikeImage>();
foreach (var image in hike.HikeImages)
{
var imageUrl = this.imagesRepository
.AllAsNoTracking()
.FirstOrDefault(x => x.RemoteImageUrl == image.RemoteImageUrl);
if (imageUrl == null)
{
imageUrl = new HikeImage
{
RemoteImageUrl = image.RemoteImageUrl,
Extension = image.RemoteImageUrl.Split('.').Last(),
Hike = newHike,
};
await this.imagesRepository.AddAsync(imageUrl);
imagesCollection.Add(imageUrl);
}
}
newHike.HikeImages = imagesCollection;
await this.hikesRepository.AddAsync(newHike);
await this.hikesRepository.SaveChangesAsync();
}
}
public async Task CreateAsync(CreateHikeInputModel input, string userId, string imagePath)
{
var startPoint = this.hikeStartPointsRepository.All()
.FirstOrDefault(x => x.Name == input.HikeStartPoint.Name);
if (startPoint == null)
{
startPoint = new HikeStartPoint
{
Name = input.HikeStartPoint.Name,
Altitude = input.HikeStartPoint.Altitude,
Latitude = input.HikeStartPoint.Longitute,
Longitute = input.HikeStartPoint.Longitute,
};
await this.hikeStartPointsRepository.AddAsync(startPoint);
await this.hikeStartPointsRepository.SaveChangesAsync();
}
var endPoint = this.hikeEndPointsRepository.All()
.FirstOrDefault(x => x.Name == input.HikeEndPoint.Name);
if (endPoint == null)
{
endPoint = new HikeEndPoint
{
Name = input.HikeEndPoint.Name,
Altitude = input.HikeEndPoint.Altitude,
Latitude = input.HikeEndPoint.Latitude,
Longitute = input.HikeEndPoint.Longitute,
};
await this.hikeEndPointsRepository.AddAsync(endPoint);
await this.hikeEndPointsRepository.SaveChangesAsync();
}
var hike = new Hike
{
Name = input.Name,
Length = input.Length,
Duration = TimeSpan.FromMinutes(input.Duration),
Description = input.Description,
Marking = input.Marking,
Difficulty = input.Difficulty,
CategoryId = input.CategoryId,
RegionId = input.RegionId,
MountainId = input.RegionId,
HikeStartPointId = startPoint.Id,
HikeEndPointId = endPoint.Id,
Denivelation = startPoint.Altitude - endPoint.Altitude,
UserId = userId,
};
Directory.CreateDirectory($"{imagePath}/hikes/");
foreach (var image in input.HikeImages)
{
var extension = Path.GetExtension(image.FileName).TrimStart('.');
if (!this.allowedExtensions.Any(x => extension.EndsWith(x)))
{
throw new Exception($"Invalid image extension {extension}");
}
var dbImage = new HikeImage
{
UserId = userId,
Extension = extension,
};
var physicalPath = $"{imagePath}/hikes/{dbImage.Id}.{extension}";
string localImgUrl = physicalPath.Split("wwwroot")[1];
dbImage.RemoteImageUrl = localImgUrl;
hike.HikeImages.Add(dbImage);
using Stream fileStream = new FileStream(physicalPath, FileMode.Create);
await image.CopyToAsync(fileStream);
}
await this.hikesRepository.AddAsync(hike);
await this.hikesRepository.SaveChangesAsync();
}
private HikeDto GetHike(string url)
{
var document = this.context.OpenAsync(url).GetAwaiter().GetResult();
if (document.StatusCode == System.Net.HttpStatusCode.NotFound)
{
Console.WriteLine($"Not found url({url})");
throw new InvalidOperationException();
}
var hike = new HikeDto();
// Get name
var hikeNames = document.QuerySelectorAll(".title > h1");
foreach (var item in hikeNames)
{
var hikeName = item.TextContent;
hike.Name = hikeName;
}
// Get landmark model elements
var hikeModelElements = document.QuerySelectorAll(".list-table > ul > li > span");
var categoryName = hikeModelElements[0].TextContent;
hike.CategoryName = categoryName;
var regionName = hikeModelElements[2].TextContent;
hike.RegionName = regionName;
var mountainName = hikeModelElements[3].TextContent;
hike.MountainName = mountainName;
var startPointName = hikeModelElements[4].TextContent.Split("( ");
hike.HikeStartPointName = startPointName[0];
var pointCoordinates = document.QuerySelectorAll(".list-table > ul > li > span > small > em");
var latitudeStartPoint = pointCoordinates[0].TextContent;
var longitudeStartPoint = pointCoordinates[1].TextContent;
hike.HikeStartPointLatitude = double.Parse(latitudeStartPoint);
hike.HikeStartPointLongitude = double.Parse(longitudeStartPoint);
var endPointName = hikeModelElements[5].TextContent.Split("( ");
hike.HikeEndPointName = endPointName[0];
var latitudeEndPoint = pointCoordinates[2].TextContent;
var longitudeEndPoint = pointCoordinates[3].TextContent;
hike.HikeEndPointLatitude = double.Parse(latitudeEndPoint);
hike.HikeEndPointLongitude = double.Parse(longitudeEndPoint);
var fullDenivelationInfo = hikeModelElements[6].TextContent;
var denivelationAsString = fullDenivelationInfo.Split(" м");
int denivelationHike = int.Parse(denivelationAsString[0]);
hike.Denivelation = denivelationHike;
int denivelationStartpoint = int.Parse(denivelationAsString[1]);
var endPointDenivelation = denivelationAsString[2].Split('-');
int denivelationEndPoint = int.Parse(endPointDenivelation[1].TrimStart());
hike.HikeStartPointAltitude = denivelationStartpoint;
hike.HikeEndPointAltitude = denivelationEndPoint;
var lenght = hikeModelElements[7].TextContent.Split();
hike.Length = double.Parse(lenght[0]);
var time = hikeModelElements[8].TextContent.Split();
var duration = time[0].Split(":");
int timeInMinutes = (int.Parse(duration[0]) * 60) + int.Parse(duration[1]);
hike.Duration = TimeSpan.FromMinutes(timeInMinutes);
var markingAndDificulty = document.QuerySelectorAll(".list-table > ul > li > span > em.rating");
hike.Marking = int.Parse(markingAndDificulty[0].GetAttribute("data-score"));
hike.Difficulty = int.Parse(markingAndDificulty[1].GetAttribute("data-score"));
// Get description
var hikeDescription = document.QuerySelectorAll(".post > .entry > p");
StringBuilder sb = new StringBuilder();
foreach (var item in hikeDescription)
{
sb.AppendLine(item.TextContent);
}
var allDescription = sb.ToString().TrimEnd();
hike.Description = allDescription;
// Get image
// var imagesUrl = document.QuerySelectorAll(".gallery-small > div.gallery-small-images > a > img");
var imagesUrl = document.QuerySelectorAll(".gallery-small > div.gallery-small-images > a");
foreach (var imageUrl in imagesUrl)
{
var curentImageUrl = imageUrl.GetAttribute("href");
HikeImage image = new HikeImage
{
RemoteImageUrl = curentImageUrl,
};
hike.HikeImages.Add(image);
}
return(hike);
}