public void CanImportSampleData() { // Arrange string httpClientFactoryContract = AttributedModelServices.GetContractName(typeof(IHttpClientFactory)); string routeServiceFactoryContract = AttributedModelServices.GetContractName(typeof(IRouteServiceFactory)); using (ApplicationCatalog initialCatalog = Program.BuildCompositionCatalog()) using (FilteredCatalog filteredCatalog = initialCatalog.Filter(d => !d.Exports(httpClientFactoryContract) && !d.Exports(routeServiceFactoryContract))) using (TypeCatalog httpClientFactoryCatalog = new TypeCatalog(typeof(WtaResultFromEmbeddedResourceFactory))) using (TypeCatalog routeServiceFactoryCatalog = new TypeCatalog(typeof(RouteServiceFromCalculatedDistanceFactory))) using (AggregateCatalog aggregateCatalog = new AggregateCatalog(filteredCatalog, httpClientFactoryCatalog, routeServiceFactoryCatalog)) using (CompositionContainer container = new CompositionContainer(aggregateCatalog)) { // Act ITrailsImporter importer = container.GetExportedValue<ITrailsImporter>(); importer.Run(); } using (MyTrailsContext context = new MyTrailsContext()) { // Assert ImportLogEntry logEntry = context.ImportLog .OrderByDescending(le => le.Id) .FirstOrDefault(); Assert.IsNotNull(logEntry); Assert.IsNull(logEntry.ErrorString, message: logEntry.ErrorString); Assert.AreEqual(0, logEntry.ErrorsCount); Assert.IsTrue(context.Trails.Any()); Assert.IsTrue(context.Guidebooks.Any()); Assert.IsTrue(context.Passes.Any()); Assert.IsTrue(context.TripReports.Any()); } }
public void TestInitialize() { using (MyTrailsContext context = new MyTrailsContext()) { context.ClearDatabase(); context.SaveChanges(); } }
/// <summary> /// Add additional context to the trail. /// </summary> /// <param name="trail">The trail to extend.</param> /// <param name="context">Datastore context.</param> /// <returns>Task for asynchronous completion.</returns> /// <seealso cref="ITrailExtender.Extend"/> public async Task Extend(Trail trail, MyTrailsContext context) { if (trail.Location != null) { this.Logger.InfoFormat("Looking up driving directions for trail: {0}", trail.Name); List<Address> addresses = context.Addresses .Where(a => a.Directions.All(d => d.TrailId != trail.Id)) .ToList(); // Force EF query to avoid multiple active results sets on enumeration. foreach (Address address in addresses) { await this.AddDrivingDirections(address, trail); } } }
public void TestInitialize() { using (MyTrailsContext context = new MyTrailsContext()) { context.ClearDatabase(); context.SaveChanges(); TripType tripType = context.TripTypes.First(); this._anyTripTypeId = tripType.Id; this._anyTripTypeWtaId = tripType.WtaId; } this._anyTrail = new Trail { Name = "Any Trail Name", WtaId = "any-wta-id", Url = new Uri("http://any/trail/uri"), }; this._wtaClientMock = new Mock<IWtaClient>(MockBehavior.Strict); this._wtaClientMock .Setup(wc => wc.FetchTripReports(It.IsAny<string>())) .Returns((string ti) => TaskExt.WrapInTask<IList<WtaTripReport>>(() => new List<WtaTripReport> { new WtaTripReport { Title = "Any title", Author = "Any author", Date = DateTime.Now, FullReportUrl = new Uri(new Uri("http://any.base.url"), AnyWtaTripReportId), HikeType = this._anyTripTypeWtaId, Photos = { new Uri("http://any/domain/photoUrl.jpg"), }, } })); this._wtaClientMock .Setup(wc => wc.BuildRetryPolicy(It.IsAny<ILog>())) .Returns(new RetryPolicy(new StubErrorDetectionStrategy(), retryCount: 0)); this._extender = new TripReportExtender { WtaClient = this._wtaClientMock.Object, Logger = new StubLog(), }; }
/// <summary> /// Add trip reports to the trail. /// </summary> /// <param name="trail">The trail to extend.</param> /// <param name="context">Registered context.</param> /// <returns>Task for asynchronous completion.</returns> /// <seealso cref="ITrailExtender.Extend"/> public async Task Extend(Trail trail, MyTrailsContext context) { this.Initialize(context); string wtaTrailId = trail.WtaId; RetryPolicy policy = this.WtaClient.BuildRetryPolicy(this.Logger); IList<WtaTripReport> reports = await policy.ExecuteAsync(() => this.WtaClient.FetchTripReports(wtaTrailId)); foreach (WtaTripReport wtaReport in reports) { string wtaReportId = this.ParseWtaReportId(wtaReport); Lazy<bool> firstToAdd = new Lazy<bool>(() => this._tripReportDictionary.TryAdd(wtaReportId, null)); TripReport report; do { report = context.TripReports .Where(tr => tr.WtaId == wtaReportId) .FirstOrDefault(); if (report == null) { if (firstToAdd.Value) { // First thread to access new trip report, create it. this.Logger.InfoFormat("Found new trip report: {0}", wtaReportId); report = this.CreateReport(wtaReportId, wtaReport); } else { this.Logger.DebugFormat("Waiting for other thread to create trip report: {0}.", wtaReportId); await Task.Delay(ConcurrentTripReportDelay); } } } while (report == null); trail.TripReports.Add(report); } }
/// <summary> /// Run the importer. /// </summary> /// <param name="logEntryId">The associated <see cref="ImportLogEntry"/> ID for the import run.</param> /// <returns>Task for asynchronous completion.</returns> private async Task RunInternal(int logEntryId) { // Verify there's not an execution already in progress. this.CheckRecentExecutions(logEntryId); using (CancellationTokenSource heartbeatTokenSource = new CancellationTokenSource()) { Task heartbeatTask = this.SendHeartbeats(logEntryId, heartbeatTokenSource.Token); RetryPolicy policy = this.WtaClient.BuildRetryPolicy(this.Logger); Task<IList<WtaTrail>> fetchTrailTask = policy.ExecuteAsync(() => this.WtaClient.FetchTrails()); this.Logger.Info("Fetching existing trail IDs."); List<string> existingTrailIds; using (MyTrailsContext context = new MyTrailsContext()) { existingTrailIds = context.Trails .Select(t => t.WtaId) .ToList(); } IList<WtaTrail> wtaTrails = await fetchTrailTask; this.DeDupeWtaTrails(wtaTrails); IEnumerable<Tuple<WtaTrail, bool>> wtaTrailTuples = this.MatchExistingTrails(wtaTrails, existingTrailIds); this.Logger.Debug("Creating new trail entries."); Task[] trailTasks = wtaTrailTuples .Select(tt => this.ImportOrUpdateTrail(tt.Item1, tt.Item2)) .ToArray(); await Task.WhenAll(trailTasks); heartbeatTokenSource.Cancel(); await heartbeatTask; } }
/// <summary> /// Register test data with the datastore context. /// </summary> /// <param name="trail">Test trail to register.</param> /// <param name="addresses">The test addresses to register.</param> /// <returns>The ID of the trail in the datastore.</returns> private int RegisterTestData(Trail trail, params Address[] addresses) { int trailId; using (MyTrailsContext context = new MyTrailsContext()) { context.Trails.Add(trail); foreach (Address address in addresses) { context.Addresses.Add(address); } context.SaveChanges(); trailId = trail.Id; } return trailId; }
/// <summary> /// Initialize caches and the maximum date of previously stored trip reports. /// </summary> /// <param name="context">Datastore context..</param> private void Initialize(MyTrailsContext context) { if (!this._initialized) { lock (this._initSyncObject) { if (!this._initialized) { this.Logger.Debug("Initializing trip type dictionary"); this._tripTypeDictionary = context.TripTypes.ToDictionary(tt => tt.WtaId, tt => tt.Id); this._initialized = true; } } } }
/// <summary> /// Run the extender for the given trail ID. /// </summary> /// <param name="trailId">The ID of the trail to run for.</param> /// <returns>Task for asynchronous completion.</returns> private async Task RunExtender(int trailId) { using (MyTrailsContext context = new MyTrailsContext()) { Trail trail = context.Trails.Find(trailId); await this._extender.Extend(trail, context); try { context.SaveChanges(); } catch (DbEntityValidationException ex) { Assert.Fail("Datastore save failed with validation error: {0}", ex); } } }
/// <summary> /// Find driving directions associated with the given trail ID and address. /// </summary> /// <param name="trailId">The ID of the trail in the data store.</param> /// <param name="address">The address to look up.</param> /// <returns>The associated driving directions, or null if none exist.</returns> private DrivingDirections FindDrivingDirections(int trailId, Address address) { DbGeographyPointComparer comparer = new DbGeographyPointComparer(); DrivingDirections directions; using (MyTrailsContext context = new MyTrailsContext()) { Address innerAddress = context.Addresses .ToList() .Where(a => comparer.Equals(a.Coordinate, address.Coordinate)) .FirstOrDefault(); directions = innerAddress == null ? null : innerAddress.Directions .Where(d => d.TrailId == trailId) .FirstOrDefault(); } return directions; }
/// <summary> /// Initialize the database context and seed contents. /// </summary> private void InitializeDatabase() { this._dataContext = new MyTrailsContext(); // Clear any existing contents. this._dataContext.ClearDatabase(); // Seed test data foreach (Trail existingTrail in ExistingTrails) { this._dataContext.Trails.Add(existingTrail); } this._dataContext.SaveChanges(); }
public void SkipsIfAlreadyAdded() { // Arrange this._anyTrail.TripReports.Add(new TripReport { WtaId = AnyWtaTripReportId, Title = "Any trip title", Author = "any author", Url = new Uri("http://any/url"), Date = DateTime.Now, TripTypeId = this._anyTripTypeId, }); int trailId = this.AddTestData(this._anyTrail); // Act this.RunExtender(trailId).Wait(); // Assert int numReports; using (MyTrailsContext context = new MyTrailsContext()) { numReports = context.TripReports .Where(tr => tr.Trails.Any(t => t.Id == trailId)) .Count(); } Assert.AreEqual(1, numReports); }
public void ChecksRecentHeartbeat() { // Arrange using (MyTrailsContext context = new MyTrailsContext()) { context.ImportLog.Add(new ImportLogEntry { StartTime = DateTimeOffset.Now - TimeSpan.FromMinutes(1234), LastHeartbeat = DateTimeOffset.Now, }); context.SaveChanges(); } // Act try { this._importer.Run().Wait(); } catch (AggregateException ex) { // Assert - Expect InvalidOperationException ex.Handle(e => e is InvalidOperationException); } }
/// <summary> /// Add completion statistics to the import log and save it to the datastore. /// </summary> /// <param name="logEntryId">The ID log entry to finalize.</param> /// <param name="errorString">An error string to associate, or null if no error was found.</param> private void FinalizeAndCommitLog(int logEntryId, string errorString) { using (MyTrailsContext context = new MyTrailsContext()) { ImportLogEntry logEntry = context.ImportLog.Find(logEntryId); logEntry.CompletedTrailsCount = context.Trails.Count(); logEntry.CompletedTripReportsCount = context.TripReports.Count(); logEntry.CompletedTime = DateTimeOffset.Now; logEntry.ErrorString = errorString; logEntry.ErrorsCount = this._numImportErrors; context.SaveChanges(this.Logger); } }
/// <summary> /// Create a new <see cref="ImportLogEntry"/> for the import run. /// </summary> /// <returns>The ID of the newly created <see cref="ImportLogEntry"/>.</returns> private int CreateImportLog() { int importLogId; using (MyTrailsContext context = new MyTrailsContext()) { ImportLogEntry logEntry = new ImportLogEntry { StartTime = DateTimeOffset.Now, LastHeartbeat = DateTimeOffset.Now, StartTrailsCount = context.Trails.Count(), StartTripReportsCount = context.TripReports.Count(), }; context.ImportLog.Add(logEntry); context.SaveChanges(); importLogId = logEntry.Id; } return importLogId; }
/// <summary> /// Log the connection string for debugging. /// </summary> private void LogConnectionString() { using (MyTrailsContext context = new MyTrailsContext()) { string connectionString = context.Database.Connection.ConnectionString; this.Logger.DebugFormat("Using connection string: {0}", connectionString); } }
/// <summary> /// Run <see cref="DrivingDistanceExtender.Extend"/> and save database context changes. /// </summary> /// <param name="trailId">The ID of the trail to add driving directions for.</param> private void RunExtender(int trailId) { using (MyTrailsContext context = new MyTrailsContext()) { Trail trail = context.Trails.Find(trailId); this._extender.Extend(trail, context).Wait(); context.SaveChanges(); } }
/// <summary> /// Dispose of object resources. /// </summary> /// <param name="disposing">Whether it is safe to reference managed objects.</param> protected virtual void Dispose(bool disposing) { if (!this._disposed) { if (disposing) { if (this._dataContext != null) { this._dataContext.Dispose(); this._dataContext = null; } } this._disposed = true; } }
/// <summary> /// Check for recent executions and throw is there is still one in progress. Enforces singleton /// access. /// </summary> /// <param name="currentLogEntryId">The log entry ID for the current execution.</param> private void CheckRecentExecutions(int currentLogEntryId) { const double heartbeatCheckMultiplier = 2.5; this.Logger.Info("Checking for recent executions."); DateTimeOffset? recentHeartbeat; using (MyTrailsContext context = new MyTrailsContext()) { recentHeartbeat = context.ImportLog .Where(i => i.Id != currentLogEntryId && !i.CompletedTime.HasValue) .OrderByDescending(i => i.LastHeartbeat) .Select(i => i.LastHeartbeat) .FirstOrDefault(); } if (recentHeartbeat.HasValue) { TimeSpan heartbeatCheckInterval = TimeSpan.FromTicks((int)(this.Configuration.HeartbeatInterval.Ticks * heartbeatCheckMultiplier)); if ((DateTimeOffset.Now - heartbeatCheckInterval) < recentHeartbeat.Value) { throw new InvalidOperationException(string.Format(CultureInfo.InvariantCulture, "Detected previous execution, with latest heartbeat: {0}", recentHeartbeat.Value)); } else { this.Logger.WarnFormat("Detected previous execution, but with stale heartbeat: {0}", recentHeartbeat.Value); } } }
public void AddsTripReportPhotos() { // Arrange int trailId = this.AddTestData(this._anyTrail); // Act this.RunExtender(trailId).Wait(); // Assert TripReportPhoto photo; using (MyTrailsContext context = new MyTrailsContext()) { photo = context.TripReports .Where(tr => tr.Trails.Any(t => t.Id == trailId)) .SelectMany(tr => tr.Photos) .FirstOrDefault(); } Assert.IsNotNull(photo); }
/// <summary> /// Import a new <see cref="WtaTrail"/>, or update an existing one. /// </summary> /// <param name="wtaTrail">The <see cref="WtaTrail"/> to import or update.</param> /// <param name="exists">Whether the trail already exists in the database.</param> /// <returns>Task for asynchronous completion.</returns> private async Task ImportOrUpdateTrail(WtaTrail wtaTrail, bool exists) { try { int trailId; using (MyTrailsContext trailContext = new MyTrailsContext()) { Trail trail; if (exists) { trail = trailContext.Trails .Where(t => t.WtaId == wtaTrail.Uid) .First(); this.TrailFactory.UpdateTrail(trail, wtaTrail, trailContext); } else { trail = this.TrailFactory.CreateTrail(wtaTrail, trailContext); trailContext.Trails.Add(trail); } trailContext.SaveChanges(this.Logger); trailId = trail.Id; } IEnumerable<Task> extenderTasks = this.TrailExtenders .Select(te => this.RunExtender(te, trailId)); await Task.WhenAll(extenderTasks); } catch (Exception ex) { Interlocked.Increment(ref this._numImportErrors); this.Logger.ErrorFormat("Error importing trail '{0}': {1}", wtaTrail, ex); throw; } }
/// <summary> /// Add trail data to the datastore and return the trail id. /// </summary> /// <param name="trail">The trail to add to the datastore.</param> /// <returns>The ID of the added trail.</returns> private int AddTestData(Trail trail) { int id; using (MyTrailsContext context = new MyTrailsContext()) { context.Trails.Add(trail); context.SaveChanges(); id = trail.Id; } return id; }
/// <summary> /// Execute a trail extender for a new trail. /// </summary> /// <param name="extender">The extender to run.</param> /// <param name="trailId">The trail ID to extend.</param> /// <returns>Task for asyncrhonous execution.</returns> private async Task RunExtender(ITrailExtender extender, int trailId) { using (MyTrailsContext trailContext = new MyTrailsContext()) { Trail trail = trailContext.Trails.Find(trailId); await extender.Extend(trail, trailContext); trailContext.SaveChanges(this.Logger); } }
/// <summary> /// Attach heartbeats to the import log entry that the importer is running until cancellation is requested. /// </summary> /// <param name="entryId">Log entry ID to append heartbeats to.</param> /// <param name="token">Cancellation token to stop heartbeats.</param> /// <returns>Task for asynchronous runtime.</returns> private async Task SendHeartbeats(int entryId, CancellationToken token) { TimeSpan interval = this.Configuration.HeartbeatInterval; this.Logger.DebugFormat("Sending heartbeats at interval: {0}", interval); try { while (!token.IsCancellationRequested) { await Task.Delay(interval, token); using (MyTrailsContext context = new MyTrailsContext()) { ImportLogEntry logEntry = context.ImportLog.Find(entryId); logEntry.LastHeartbeat = DateTimeOffset.Now; context.SaveChanges(); } } } catch (OperationCanceledException) { // Cancellation requested. } this.Logger.Debug("Finished heartbeating."); }
/// <summary> /// Clear test data from the database. /// </summary> private void ClearDatabase() { using (MyTrailsContext trailContext = new MyTrailsContext()) { trailContext.ClearDatabase(); trailContext.Addresses.Truncate(); trailContext.SaveChanges(); } }