public void CanImportSampleData()
{
// Arrange
string httpClientFactoryContract = AttributedModelServices.GetContractName(typeof(IHttpClientFactory));
string routeServiceFactoryContract = AttributedModelServices.GetContractName(typeof(IRouteServiceFactory));
using (ApplicationCatalog initialCatalog = Program.BuildCompositionCatalog())
using (FilteredCatalog filteredCatalog = initialCatalog.Filter(d => !d.Exports(httpClientFactoryContract) && !d.Exports(routeServiceFactoryContract)))
using (TypeCatalog httpClientFactoryCatalog = new TypeCatalog(typeof(WtaResultFromEmbeddedResourceFactory)))
using (TypeCatalog routeServiceFactoryCatalog = new TypeCatalog(typeof(RouteServiceFromCalculatedDistanceFactory)))
using (AggregateCatalog aggregateCatalog = new AggregateCatalog(filteredCatalog, httpClientFactoryCatalog, routeServiceFactoryCatalog))
using (CompositionContainer container = new CompositionContainer(aggregateCatalog))
{
// Act
ITrailsImporter importer = container.GetExportedValue<ITrailsImporter>();
importer.Run();
}
using (MyTrailsContext context = new MyTrailsContext())
{
// Assert
ImportLogEntry logEntry = context.ImportLog
.OrderByDescending(le => le.Id)
.FirstOrDefault();
Assert.IsNotNull(logEntry);
Assert.IsNull(logEntry.ErrorString, message: logEntry.ErrorString);
Assert.AreEqual(0, logEntry.ErrorsCount);
Assert.IsTrue(context.Trails.Any());
Assert.IsTrue(context.Guidebooks.Any());
Assert.IsTrue(context.Passes.Any());
Assert.IsTrue(context.TripReports.Any());
}
}
public void TestInitialize()
{
using (MyTrailsContext context = new MyTrailsContext())
{
context.ClearDatabase();
context.SaveChanges();
}
}
/// <summary>
/// Add additional context to the trail.
/// </summary>
/// <param name="trail">The trail to extend.</param>
/// <param name="context">Datastore context.</param>
/// <returns>Task for asynchronous completion.</returns>
/// <seealso cref="ITrailExtender.Extend"/>
public async Task Extend(Trail trail, MyTrailsContext context)
{
if (trail.Location != null)
{
this.Logger.InfoFormat("Looking up driving directions for trail: {0}", trail.Name);
List<Address> addresses = context.Addresses
.Where(a => a.Directions.All(d => d.TrailId != trail.Id))
.ToList(); // Force EF query to avoid multiple active results sets on enumeration.
foreach (Address address in addresses)
{
await this.AddDrivingDirections(address, trail);
}
}
}
public void TestInitialize()
{
using (MyTrailsContext context = new MyTrailsContext())
{
context.ClearDatabase();
context.SaveChanges();
TripType tripType = context.TripTypes.First();
this._anyTripTypeId = tripType.Id;
this._anyTripTypeWtaId = tripType.WtaId;
}
this._anyTrail = new Trail
{
Name = "Any Trail Name",
WtaId = "any-wta-id",
Url = new Uri("http://any/trail/uri"),
};
this._wtaClientMock = new Mock<IWtaClient>(MockBehavior.Strict);
this._wtaClientMock
.Setup(wc => wc.FetchTripReports(It.IsAny<string>()))
.Returns((string ti) => TaskExt.WrapInTask<IList<WtaTripReport>>(() => new List<WtaTripReport>
{
new WtaTripReport
{
Title = "Any title",
Author = "Any author",
Date = DateTime.Now,
FullReportUrl = new Uri(new Uri("http://any.base.url"), AnyWtaTripReportId),
HikeType = this._anyTripTypeWtaId,
Photos =
{
new Uri("http://any/domain/photoUrl.jpg"),
},
}
}));
this._wtaClientMock
.Setup(wc => wc.BuildRetryPolicy(It.IsAny<ILog>()))
.Returns(new RetryPolicy(new StubErrorDetectionStrategy(), retryCount: 0));
this._extender = new TripReportExtender
{
WtaClient = this._wtaClientMock.Object,
Logger = new StubLog(),
};
}
/// <summary>
/// Add trip reports to the trail.
/// </summary>
/// <param name="trail">The trail to extend.</param>
/// <param name="context">Registered context.</param>
/// <returns>Task for asynchronous completion.</returns>
/// <seealso cref="ITrailExtender.Extend"/>
public async Task Extend(Trail trail, MyTrailsContext context)
{
this.Initialize(context);
string wtaTrailId = trail.WtaId;
RetryPolicy policy = this.WtaClient.BuildRetryPolicy(this.Logger);
IList<WtaTripReport> reports = await policy.ExecuteAsync(() => this.WtaClient.FetchTripReports(wtaTrailId));
foreach (WtaTripReport wtaReport in reports)
{
string wtaReportId = this.ParseWtaReportId(wtaReport);
Lazy<bool> firstToAdd = new Lazy<bool>(() => this._tripReportDictionary.TryAdd(wtaReportId, null));
TripReport report;
do
{
report = context.TripReports
.Where(tr => tr.WtaId == wtaReportId)
.FirstOrDefault();
if (report == null)
{
if (firstToAdd.Value)
{
// First thread to access new trip report, create it.
this.Logger.InfoFormat("Found new trip report: {0}", wtaReportId);
report = this.CreateReport(wtaReportId, wtaReport);
}
else
{
this.Logger.DebugFormat("Waiting for other thread to create trip report: {0}.", wtaReportId);
await Task.Delay(ConcurrentTripReportDelay);
}
}
}
while (report == null);
trail.TripReports.Add(report);
}
}
/// <summary>
/// Run the importer.
/// </summary>
/// <param name="logEntryId">The associated <see cref="ImportLogEntry"/> ID for the import run.</param>
/// <returns>Task for asynchronous completion.</returns>
private async Task RunInternal(int logEntryId)
{
// Verify there's not an execution already in progress.
this.CheckRecentExecutions(logEntryId);
using (CancellationTokenSource heartbeatTokenSource = new CancellationTokenSource())
{
Task heartbeatTask = this.SendHeartbeats(logEntryId, heartbeatTokenSource.Token);
RetryPolicy policy = this.WtaClient.BuildRetryPolicy(this.Logger);
Task<IList<WtaTrail>> fetchTrailTask = policy.ExecuteAsync(() => this.WtaClient.FetchTrails());
this.Logger.Info("Fetching existing trail IDs.");
List<string> existingTrailIds;
using (MyTrailsContext context = new MyTrailsContext())
{
existingTrailIds = context.Trails
.Select(t => t.WtaId)
.ToList();
}
IList<WtaTrail> wtaTrails = await fetchTrailTask;
this.DeDupeWtaTrails(wtaTrails);
IEnumerable<Tuple<WtaTrail, bool>> wtaTrailTuples = this.MatchExistingTrails(wtaTrails, existingTrailIds);
this.Logger.Debug("Creating new trail entries.");
Task[] trailTasks = wtaTrailTuples
.Select(tt => this.ImportOrUpdateTrail(tt.Item1, tt.Item2))
.ToArray();
await Task.WhenAll(trailTasks);
heartbeatTokenSource.Cancel();
await heartbeatTask;
}
}
/// <summary>
/// Register test data with the datastore context.
/// </summary>
/// <param name="trail">Test trail to register.</param>
/// <param name="addresses">The test addresses to register.</param>
/// <returns>The ID of the trail in the datastore.</returns>
private int RegisterTestData(Trail trail, params Address[] addresses)
{
int trailId;
using (MyTrailsContext context = new MyTrailsContext())
{
context.Trails.Add(trail);
foreach (Address address in addresses)
{
context.Addresses.Add(address);
}
context.SaveChanges();
trailId = trail.Id;
}
return trailId;
}
/// <summary>
/// Initialize caches and the maximum date of previously stored trip reports.
/// </summary>
/// <param name="context">Datastore context..</param>
private void Initialize(MyTrailsContext context)
{
if (!this._initialized)
{
lock (this._initSyncObject)
{
if (!this._initialized)
{
this.Logger.Debug("Initializing trip type dictionary");
this._tripTypeDictionary = context.TripTypes.ToDictionary(tt => tt.WtaId, tt => tt.Id);
this._initialized = true;
}
}
}
}
/// <summary>
/// Run the extender for the given trail ID.
/// </summary>
/// <param name="trailId">The ID of the trail to run for.</param>
/// <returns>Task for asynchronous completion.</returns>
private async Task RunExtender(int trailId)
{
using (MyTrailsContext context = new MyTrailsContext())
{
Trail trail = context.Trails.Find(trailId);
await this._extender.Extend(trail, context);
try
{
context.SaveChanges();
}
catch (DbEntityValidationException ex)
{
Assert.Fail("Datastore save failed with validation error: {0}", ex);
}
}
}
/// <summary>
/// Find driving directions associated with the given trail ID and address.
/// </summary>
/// <param name="trailId">The ID of the trail in the data store.</param>
/// <param name="address">The address to look up.</param>
/// <returns>The associated driving directions, or null if none exist.</returns>
private DrivingDirections FindDrivingDirections(int trailId, Address address)
{
DbGeographyPointComparer comparer = new DbGeographyPointComparer();
DrivingDirections directions;
using (MyTrailsContext context = new MyTrailsContext())
{
Address innerAddress = context.Addresses
.ToList()
.Where(a => comparer.Equals(a.Coordinate, address.Coordinate))
.FirstOrDefault();
directions = innerAddress == null ?
null :
innerAddress.Directions
.Where(d => d.TrailId == trailId)
.FirstOrDefault();
}
return directions;
}
/// <summary>
/// Initialize the database context and seed contents.
/// </summary>
private void InitializeDatabase()
{
this._dataContext = new MyTrailsContext();
// Clear any existing contents.
this._dataContext.ClearDatabase();
// Seed test data
foreach (Trail existingTrail in ExistingTrails)
{
this._dataContext.Trails.Add(existingTrail);
}
this._dataContext.SaveChanges();
}
public void SkipsIfAlreadyAdded()
{
// Arrange
this._anyTrail.TripReports.Add(new TripReport
{
WtaId = AnyWtaTripReportId,
Title = "Any trip title",
Author = "any author",
Url = new Uri("http://any/url"),
Date = DateTime.Now,
TripTypeId = this._anyTripTypeId,
});
int trailId = this.AddTestData(this._anyTrail);
// Act
this.RunExtender(trailId).Wait();
// Assert
int numReports;
using (MyTrailsContext context = new MyTrailsContext())
{
numReports = context.TripReports
.Where(tr => tr.Trails.Any(t => t.Id == trailId))
.Count();
}
Assert.AreEqual(1, numReports);
}
public void ChecksRecentHeartbeat()
{
// Arrange
using (MyTrailsContext context = new MyTrailsContext())
{
context.ImportLog.Add(new ImportLogEntry
{
StartTime = DateTimeOffset.Now - TimeSpan.FromMinutes(1234),
LastHeartbeat = DateTimeOffset.Now,
});
context.SaveChanges();
}
// Act
try
{
this._importer.Run().Wait();
}
catch (AggregateException ex)
{
// Assert - Expect InvalidOperationException
ex.Handle(e => e is InvalidOperationException);
}
}
/// <summary>
/// Add completion statistics to the import log and save it to the datastore.
/// </summary>
/// <param name="logEntryId">The ID log entry to finalize.</param>
/// <param name="errorString">An error string to associate, or null if no error was found.</param>
private void FinalizeAndCommitLog(int logEntryId, string errorString)
{
using (MyTrailsContext context = new MyTrailsContext())
{
ImportLogEntry logEntry = context.ImportLog.Find(logEntryId);
logEntry.CompletedTrailsCount = context.Trails.Count();
logEntry.CompletedTripReportsCount = context.TripReports.Count();
logEntry.CompletedTime = DateTimeOffset.Now;
logEntry.ErrorString = errorString;
logEntry.ErrorsCount = this._numImportErrors;
context.SaveChanges(this.Logger);
}
}
/// <summary>
/// Create a new <see cref="ImportLogEntry"/> for the import run.
/// </summary>
/// <returns>The ID of the newly created <see cref="ImportLogEntry"/>.</returns>
private int CreateImportLog()
{
int importLogId;
using (MyTrailsContext context = new MyTrailsContext())
{
ImportLogEntry logEntry = new ImportLogEntry
{
StartTime = DateTimeOffset.Now,
LastHeartbeat = DateTimeOffset.Now,
StartTrailsCount = context.Trails.Count(),
StartTripReportsCount = context.TripReports.Count(),
};
context.ImportLog.Add(logEntry);
context.SaveChanges();
importLogId = logEntry.Id;
}
return importLogId;
}
/// <summary>
/// Log the connection string for debugging.
/// </summary>
private void LogConnectionString()
{
using (MyTrailsContext context = new MyTrailsContext())
{
string connectionString = context.Database.Connection.ConnectionString;
this.Logger.DebugFormat("Using connection string: {0}", connectionString);
}
}
/// <summary>
/// Run <see cref="DrivingDistanceExtender.Extend"/> and save database context changes.
/// </summary>
/// <param name="trailId">The ID of the trail to add driving directions for.</param>
private void RunExtender(int trailId)
{
using (MyTrailsContext context = new MyTrailsContext())
{
Trail trail = context.Trails.Find(trailId);
this._extender.Extend(trail, context).Wait();
context.SaveChanges();
}
}
/// <summary>
/// Dispose of object resources.
/// </summary>
/// <param name="disposing">Whether it is safe to reference managed objects.</param>
protected virtual void Dispose(bool disposing)
{
if (!this._disposed)
{
if (disposing)
{
if (this._dataContext != null)
{
this._dataContext.Dispose();
this._dataContext = null;
}
}
this._disposed = true;
}
}
/// <summary>
/// Check for recent executions and throw is there is still one in progress. Enforces singleton
/// access.
/// </summary>
/// <param name="currentLogEntryId">The log entry ID for the current execution.</param>
private void CheckRecentExecutions(int currentLogEntryId)
{
const double heartbeatCheckMultiplier = 2.5;
this.Logger.Info("Checking for recent executions.");
DateTimeOffset? recentHeartbeat;
using (MyTrailsContext context = new MyTrailsContext())
{
recentHeartbeat = context.ImportLog
.Where(i => i.Id != currentLogEntryId && !i.CompletedTime.HasValue)
.OrderByDescending(i => i.LastHeartbeat)
.Select(i => i.LastHeartbeat)
.FirstOrDefault();
}
if (recentHeartbeat.HasValue)
{
TimeSpan heartbeatCheckInterval = TimeSpan.FromTicks((int)(this.Configuration.HeartbeatInterval.Ticks * heartbeatCheckMultiplier));
if ((DateTimeOffset.Now - heartbeatCheckInterval) < recentHeartbeat.Value)
{
throw new InvalidOperationException(string.Format(CultureInfo.InvariantCulture,
"Detected previous execution, with latest heartbeat: {0}", recentHeartbeat.Value));
}
else
{
this.Logger.WarnFormat("Detected previous execution, but with stale heartbeat: {0}", recentHeartbeat.Value);
}
}
}
public void AddsTripReportPhotos()
{
// Arrange
int trailId = this.AddTestData(this._anyTrail);
// Act
this.RunExtender(trailId).Wait();
// Assert
TripReportPhoto photo;
using (MyTrailsContext context = new MyTrailsContext())
{
photo = context.TripReports
.Where(tr => tr.Trails.Any(t => t.Id == trailId))
.SelectMany(tr => tr.Photos)
.FirstOrDefault();
}
Assert.IsNotNull(photo);
}
/// <summary>
/// Import a new <see cref="WtaTrail"/>, or update an existing one.
/// </summary>
/// <param name="wtaTrail">The <see cref="WtaTrail"/> to import or update.</param>
/// <param name="exists">Whether the trail already exists in the database.</param>
/// <returns>Task for asynchronous completion.</returns>
private async Task ImportOrUpdateTrail(WtaTrail wtaTrail, bool exists)
{
try
{
int trailId;
using (MyTrailsContext trailContext = new MyTrailsContext())
{
Trail trail;
if (exists)
{
trail = trailContext.Trails
.Where(t => t.WtaId == wtaTrail.Uid)
.First();
this.TrailFactory.UpdateTrail(trail, wtaTrail, trailContext);
}
else
{
trail = this.TrailFactory.CreateTrail(wtaTrail, trailContext);
trailContext.Trails.Add(trail);
}
trailContext.SaveChanges(this.Logger);
trailId = trail.Id;
}
IEnumerable<Task> extenderTasks = this.TrailExtenders
.Select(te => this.RunExtender(te, trailId));
await Task.WhenAll(extenderTasks);
}
catch (Exception ex)
{
Interlocked.Increment(ref this._numImportErrors);
this.Logger.ErrorFormat("Error importing trail '{0}': {1}", wtaTrail, ex);
throw;
}
}
/// <summary>
/// Add trail data to the datastore and return the trail id.
/// </summary>
/// <param name="trail">The trail to add to the datastore.</param>
/// <returns>The ID of the added trail.</returns>
private int AddTestData(Trail trail)
{
int id;
using (MyTrailsContext context = new MyTrailsContext())
{
context.Trails.Add(trail);
context.SaveChanges();
id = trail.Id;
}
return id;
}
/// <summary>
/// Execute a trail extender for a new trail.
/// </summary>
/// <param name="extender">The extender to run.</param>
/// <param name="trailId">The trail ID to extend.</param>
/// <returns>Task for asyncrhonous execution.</returns>
private async Task RunExtender(ITrailExtender extender, int trailId)
{
using (MyTrailsContext trailContext = new MyTrailsContext())
{
Trail trail = trailContext.Trails.Find(trailId);
await extender.Extend(trail, trailContext);
trailContext.SaveChanges(this.Logger);
}
}
/// <summary>
/// Attach heartbeats to the import log entry that the importer is running until cancellation is requested.
/// </summary>
/// <param name="entryId">Log entry ID to append heartbeats to.</param>
/// <param name="token">Cancellation token to stop heartbeats.</param>
/// <returns>Task for asynchronous runtime.</returns>
private async Task SendHeartbeats(int entryId, CancellationToken token)
{
TimeSpan interval = this.Configuration.HeartbeatInterval;
this.Logger.DebugFormat("Sending heartbeats at interval: {0}", interval);
try
{
while (!token.IsCancellationRequested)
{
await Task.Delay(interval, token);
using (MyTrailsContext context = new MyTrailsContext())
{
ImportLogEntry logEntry = context.ImportLog.Find(entryId);
logEntry.LastHeartbeat = DateTimeOffset.Now;
context.SaveChanges();
}
}
}
catch (OperationCanceledException)
{
// Cancellation requested.
}
this.Logger.Debug("Finished heartbeating.");
}
/// <summary>
/// Clear test data from the database.
/// </summary>
private void ClearDatabase()
{
using (MyTrailsContext trailContext = new MyTrailsContext())
{
trailContext.ClearDatabase();
trailContext.Addresses.Truncate();
trailContext.SaveChanges();
}
}