public async Task SyncController_Post_Success_Basic_NoChanges_NoGroups() { var testApp = new Application { Id = new Guid("afd8db1e-73b8-4d5f-9cb1-6b49d205555a"), AccountId = new Guid("250c6f28-4611-4c28-902c-8464fabc510b"), AccessKey = new Guid("3d65a27c-9d1d-48a3-a888-89cc0f7851d0"), Name = "Integration Test App" }; SyncRequestViewModel request = new SyncRequestViewModel() { AppId = testApp.Id, AppApiAccessKey = testApp.AccessKey }; var jsonPayload = new StringContent(JsonConvert.SerializeObject(request), Encoding.UTF8, "application/json"); var response = await HttpClient.PostAsync(SyncRequestUrl, jsonPayload); Assert.AreEqual(HttpStatusCode.OK, response.StatusCode); Assert.NotNull(response.Content); var syncResponse = JsonConvert.DeserializeObject <SyncResponseViewModel>(await response.Content.ReadAsStringAsync()); Assert.NotNull(syncResponse); Assert.Null(syncResponse.Errors); Assert.True(syncResponse.Success); }
private async Task <SyncResponseViewModel> GetChanges(IApplication app, SyncRequestViewModel request) { var response = new SyncResponseViewModel() { Groups = new List <SyncResponseViewModel.GroupViewModel>() }; if (request.Groups != null) { foreach (var group in request.Groups) { List <IChange> results = await ChangeRepository.ListChangesAsync(app.Id, group.Group, group.Tidemark); if (results != null && results.Any()) { response.Groups.Add(new SyncResponseViewModel.GroupViewModel() { Group = group.Group, Tidemark = results.Last().Id, Changes = results.Select(r => new SyncResponseViewModel.ChangeViewModel() { Modified = r.ClientModified, Value = r.RecordValue, Entity = r.Entity, RecordId = r.RecordId, Property = r.Property }).ToList() }); } } } return(response); }
private async Task ProcessChanges(IApplication app, SyncRequestViewModel request) { if (request.Changes != null && request.Changes.Any()) { var dbChanges = new List <IChange>(); Stopwatch sw = new Stopwatch(); sw.Start(); // Dedup any changes with the same Record, Entity and Property var uniqueChanges = new Dictionary <string, SyncRequestViewModel.ChangeViewModel>(); foreach (var change in request.Changes) { string key = $"{change.RecordId}-{change.Entity}-{change.Property}"; if (uniqueChanges.TryGetValue(key, out var existingChange)) { if (change.MillisecondsAgo < existingChange.MillisecondsAgo) { uniqueChanges[key] = change; } } else { uniqueChanges.Add(key, change); } } Logger.LogInformation($"Deduped changes in {sw.ElapsedMilliseconds}ms count: {dbChanges.Count}"); sw.Restart(); foreach (var change in uniqueChanges.Values) { if (change != null) { DateTime modifiedUTC = requestStartTimeUTC.AddMilliseconds(-change.MillisecondsAgo); var dbChange = ChangeRepository.CreateChange(app.AccountId, app.Id, change.RecordId, change.Group, change.Entity, change.Property, modifiedUTC.Ticks, change.Value); dbChanges.Add(dbChange); } } Logger.LogInformation($"Generated changes in {sw.ElapsedMilliseconds}ms count: {dbChanges.Count}"); sw.Restart(); if (dbChanges.Any()) { sw.Restart(); await ChangeRepository.UpsertChangesAsync(app.Id, dbChanges); Logger.LogInformation($"Saved changes to database in {sw.ElapsedMilliseconds}ms count: {dbChanges.Count}"); } sw.Stop(); } }
private async Task ProcessChanges(Application app, Device device, SyncRequestViewModel request) { if (request.Changes != null && request.Changes.Any()) { Stopwatch sw = new Stopwatch(); sw.Start(); foreach (var batch in request.Changes.Batch(50)) { var changes = new List <SendContextModel <UpsetModel <Change> > >(); foreach (var change in batch) { if (change != null) { Guid recordId = Guid.Empty; string path = null; // Path should contain a / in format <guid>/property.name if (!string.IsNullOrWhiteSpace(change.Path) && change.Path.IndexOf("/") > -1) { recordId = Guid.Parse(change.Path.Substring(0, change.Path.IndexOf("/"))); path = change.Path.Substring(change.Path.IndexOf("/") + 1); } DateTime modifiedUTC = requestStartTimeUTC.AddSeconds(-change.SecondsAgo); var dbChange = new Change() { Id = Guid.NewGuid(), RecordId = recordId, Path = change.Path, DeviceId = device.Id, Modified = modifiedUTC, Tidemark = "%clustertime%", Value = change.Value }; string partition = $"{app.Id}-{change.Group}"; changes.Add(ScaleContext.MakeUpsertModel(partition, "change", dbChange)); } } Logger.LogInformation($"Generated changes in {sw.ElapsedMilliseconds}ms count: {changes.Count}"); sw.Restart(); await ScaleContext.UpsertBulk(changes); Logger.LogInformation($"Saved changes to database in {sw.ElapsedMilliseconds}ms count: {changes.Count}"); } sw.Stop(); } }
private async Task <Device> ValidateDevice(SyncRequestViewModel request) { var device = await Cache.GetByPrimaryKeyFromCacheOrQuery <Device>(ScaleContext.SystemPartition, "device", "device_id", request.DeviceId, defaultCacheDuration); if (device == null) { ModelState.AddModelError("device_id", "No device found for device_id"); } return(device); }
public async Task <JsonResult> Post([FromBody] SyncRequestViewModel request) { requestStartTimeUTC = DateTime.UtcNow; var response = new SyncResponseViewModel(); try { Stopwatch sw = new Stopwatch(); sw.Start(); var app = await ValidateApplication(request); Logger.LogInformation($"ValidateApplication in {sw.ElapsedMilliseconds}ms"); if (!ModelState.IsValid) { return(JsonResultWithValidationErrors(response)); } sw.Restart(); var device = await ValidateDevice(request); Logger.LogInformation($"ValidateDevice in {sw.ElapsedMilliseconds}ms"); if (!ModelState.IsValid) { return(JsonResultWithValidationErrors(response)); } sw.Restart(); await ProcessChanges(app, device, request); Logger.LogInformation($"ProcessChanges in {sw.ElapsedMilliseconds}ms"); sw.Restart(); response = await GetChanges(app, device, request); Logger.LogInformation($"GetChanges in {sw.ElapsedMilliseconds}ms"); sw.Stop(); } catch (Exception ex) { Logger.LogError($"Failed to complete Sync Post: {ex.ToString()}"); ModelState.AddModelError("", $"Unhandled exception: {ex.ToString()}"); } return(JsonResultWithValidationErrors(response)); }
public async Task SyncController_Post_Fail_Empty_Request() { SyncRequestViewModel request = new SyncRequestViewModel(); var jsonPayload = new StringContent(JsonConvert.SerializeObject(request), Encoding.UTF8, "application/json"); var response = await HttpClient.PostAsync(SyncRequestUrl, jsonPayload); Assert.AreEqual(HttpStatusCode.BadRequest, response.StatusCode); Assert.NotNull(response.Content); var syncResponse = JsonConvert.DeserializeObject <SyncResponseViewModel>(await response.Content.ReadAsStringAsync()); Assert.NotNull(syncResponse); Assert.NotNull(syncResponse.Errors); Assert.AreEqual(1, syncResponse.Errors.Count()); Assert.AreEqual("app_id missing or invalid request", syncResponse.Errors.First()); Assert.False(syncResponse.Success); }
public async Task SyncController_Post_Fail_Missing_AppApiAccessKey() { SyncRequestViewModel request = new SyncRequestViewModel() { AppId = new Guid("afd8db1e-73b8-4d5f-9cb1-6b49d205555a") }; var jsonPayload = new StringContent(JsonConvert.SerializeObject(request), Encoding.UTF8, "application/json"); var response = await HttpClient.PostAsync(SyncRequestUrl, jsonPayload); Assert.AreEqual(HttpStatusCode.BadRequest, response.StatusCode); Assert.NotNull(response.Content); var syncResponse = JsonConvert.DeserializeObject <SyncResponseViewModel>(await response.Content.ReadAsStringAsync()); Assert.NotNull(syncResponse); Assert.NotNull(syncResponse.Errors); Assert.AreEqual(1, syncResponse.Errors.Count()); Assert.AreEqual("app_api_access_key incorrect for app_id", syncResponse.Errors.First()); Assert.False(syncResponse.Success); }
public async Task Success_Basic_EmptyChange() { var request = new SyncRequestViewModel() { AppId = app.Id, AppApiAccessKey = app.AccessKey, DeviceId = device.Id, Changes = new List <SyncRequestViewModel.ChangeViewModel> { new SyncRequestViewModel.ChangeViewModel { } } }; var controller = new SyncController(logger.Object, queryCache.Object, scaleContext.Object); var response = await controller.Post(request); var syncResponse = response.Value as SyncResponseViewModel; Assert.NotNull(syncResponse); Assert.Null(syncResponse.Errors); Assert.True(syncResponse.Success); }
private async Task <IApplication> ValidateApplication(SyncRequestViewModel request) { if (request == null || request.AppId == Guid.Empty) { ModelState.AddModelError("app_id", "app_id missing or invalid request"); } else { var app = await ApplicationRepository.GetByIdAsync(request.AppId); if (app == null) { ModelState.AddModelError("app_id", "No application found for app_id"); } else if (app.AccessKey != request.AppApiAccessKey) { ModelState.AddModelError("app_api_access_key", "app_api_access_key incorrect for app_id"); } return(app); } return(null); }
private async Task <Application> ValidateApplication(SyncRequestViewModel request) { if (request == null || request.AppId == Guid.Empty) { ModelState.AddModelError("app_id", "app_id missing or invalid request"); } else { var app = await Cache.GetByPrimaryKeyFromCacheOrQuery <Application>(ScaleContext.SystemPartition, "application", "app_id", request.AppId, defaultCacheDuration); if (app == null) { ModelState.AddModelError("app_id", "No application found for app_id"); } else if (app.AccessKey != request.AppApiAccessKey) { ModelState.AddModelError("app_api_access_key", "app_api_access_key incorrect for app_id"); } return(app); } return(null); }
public async Task Success_Basic_Badly_Formatted_Path() { string partition = null; string table = null; Change value = null; scaleContext .Setup(x => x.MakeUpsertModel(It.IsAny <string>(), "change", It.IsAny <Change>())) .Callback <string, string, Change>((p, t, v) => { partition = p; table = t; value = v; }); var request = new SyncRequestViewModel() { AppId = app.Id, AppApiAccessKey = app.AccessKey, DeviceId = device.Id, Changes = new List <SyncRequestViewModel.ChangeViewModel> { new SyncRequestViewModel.ChangeViewModel { Group = "Group", Path = "bad format" } } }; var controller = new SyncController(logger.Object, queryCache.Object, scaleContext.Object); var response = await controller.Post(request); var syncResponse = response.Value as SyncResponseViewModel; //var dbChange = new Change() //{ // Id = Guid.NewGuid(), // RecordId = recordId, // Path = change.Path, // DeviceId = device.Id, // Modified = modifiedUTC, // Tidemark = "%clustertime%", // Value = change.Value //}; Assert.NotNull(syncResponse); Assert.Null(syncResponse.Errors); Assert.True(syncResponse.Success); Assert.AreEqual($"{app.Id}-Group", partition); Assert.AreEqual($"change", table); Assert.NotNull(value); //// Path should contain a / in format <guid>/property.name //if (!string.IsNullOrWhiteSpace(change.Path) && change.Path.IndexOf("/") > -1) //{ // recordId = Guid.Parse(change.Path.Substring(0, change.Path.IndexOf("/"))); // path = change.Path.Substring(change.Path.IndexOf("/") + 1); //} scaleContext.Verify(t => t.MakeUpsertModel(It.IsAny <string>(), "change", It.IsAny <Change>())); }
public async Task SyncController_Post_Success_Send_Two_Changes_FilterOutChangesForTheSameRecordEntityAndProperty() { string propertyName = "name"; string group = "group"; string entity = "person"; Guid recordId = Guid.NewGuid(); long modifiedMillisecondsAgo = 10000; long modifiedMillisecondsAgo2 = 20000; string value = "Neil"; string value2 = "Adrian"; List <IChange> returnChanges = null; DateTime requestStart = DateTime.UtcNow; timeService = new Mock <ITimeService>(); timeService.Setup(x => x.GetUtcNow()).Returns(requestStart); changeRepository .Setup(x => x.UpsertChangesAsync(It.IsAny <Guid>(), It.IsAny <IEnumerable <IChange> >())) .Returns(() => Task.FromResult((string)null)) .Callback <Guid, IEnumerable <IChange> >((a, l) => { returnChanges = l.ToList(); }); var request = new SyncRequestViewModel() { AppId = app.Object.Id, AppApiAccessKey = app.Object.AccessKey, Changes = new List <SyncRequestViewModel.ChangeViewModel> { new SyncRequestViewModel.ChangeViewModel { Group = group, RecordId = recordId, Entity = entity, Property = propertyName, MillisecondsAgo = modifiedMillisecondsAgo, Value = value }, new SyncRequestViewModel.ChangeViewModel { Group = group, RecordId = recordId, Entity = entity, Property = propertyName, MillisecondsAgo = modifiedMillisecondsAgo2, Value = value2 } } }; SyncController controller = CreateSyncController(); var response = await controller.Post(request) as JsonResult; var syncResponse = response.Value as SyncResponseViewModel; Assert.NotNull(syncResponse); Assert.Null(syncResponse.Errors); Assert.True(syncResponse.Success); Assert.NotNull(syncResponse.Groups); Assert.AreEqual(0, syncResponse.Groups.Count); Assert.NotNull(returnChanges); Assert.AreEqual(1, returnChanges.Count); Assert.AreEqual(group, returnChanges[0].GroupId); Assert.AreEqual(propertyName, returnChanges[0].Property); Assert.AreEqual(entity, returnChanges[0].Entity); Assert.AreEqual(recordId, returnChanges[0].RecordId); Assert.AreEqual(value, returnChanges[0].RecordValue); IEnumerable <IChange> changes = new List <IChange>() { change.Object }; changeRepository.Verify(t => t.UpsertChangesAsync(app.Object.Id, changes), Times.Once); changeRepository.Verify(t => t.CreateChange(app.Object.AccountId, app.Object.Id, recordId, group, entity, propertyName, requestStart.AddMilliseconds(-modifiedMillisecondsAgo).Ticks, value), Times.Once); changeRepository.Verify(t => t.ListChangesAsync(It.IsAny <Guid>(), It.IsAny <string>(), It.IsAny <long?>()), Times.Never); }
private async Task <SyncResponseViewModel> GetChanges(Application app, Device device, SyncRequestViewModel request) { var response = new SyncResponseViewModel() { Groups = new List <SyncResponseViewModel.GroupViewModel>() }; Stopwatch sw = new Stopwatch(); sw.Start(); if (request.Groups != null) { foreach (var group in request.Groups) { sw.Restart(); string partition = $"{app.Id}-{group.Group}"; var queryParams = new List <object>(); string whereClause = null; if (!string.IsNullOrWhiteSpace(group.Tidemark)) { Logger.LogInformation($"Getting changes for group: {group.Group} after tidemark: {group.Tidemark}"); queryParams.Add(group.Tidemark); whereClause = "tidemark > ?"; } else { Logger.LogInformation($"Getting all changes for group: {group.Group}"); } List <Change> results = await ScaleContext.Query <Change>(partition, "change", whereClause, queryParams, orderBy : "tidemark", limit : 50); Logger.LogInformation($"Retrieved changes from database in {sw.ElapsedMilliseconds}ms count: {results.Count}"); if (results != null && results.Any()) { response.Groups.Add(new SyncResponseViewModel.GroupViewModel() { Group = group.Group, Tidemark = results.Last().Tidemark, Changes = results.Select(r => new SyncResponseViewModel.ChangeViewModel() { Modified = r.Modified, Value = r.Value, Path = r.Path }).ToList() }); } } } sw.Stop(); return(response); }
public async Task SyncController_Post_Success_TwoChangesToSamePropertyOverTwoRequests_EnsureOnlyLatestIsReturned() { var testApp = new Application { Id = new Guid("19d8856c-a439-46ae-9932-c81fd0fe5556"), AccountId = new Guid("250c6f28-4611-4c28-902c-8464fabc510b"), AccessKey = new Guid("0f458ce8-1a0e-450c-a2c4-2b50b3c4f41d"), Name = "Integration Test App 4" }; string propertyName = "name"; string group = "group"; string entity = "Person"; Guid recordId = Guid.NewGuid(); int modifiedMillisecondsAgo = 10000; int modifiedMillisecondsAgo2 = 20000; string value = "Neil"; string value2 = "Adrian"; await DeleteChangeRows(testApp.Id, group); var request = new SyncRequestViewModel() { AppId = testApp.Id, AppApiAccessKey = testApp.AccessKey, Changes = new List <SyncRequestViewModel.ChangeViewModel> { new SyncRequestViewModel.ChangeViewModel { Group = group, Entity = entity, Property = propertyName, RecordId = recordId, MillisecondsAgo = modifiedMillisecondsAgo, Value = value } } }; var jsonPayload = JsonConvert.SerializeObject(request); var requestContent = new StringContent(jsonPayload, Encoding.UTF8, "application/json"); var response = await HttpClient.PostAsync(SyncRequestUrl, requestContent); var responsePayload = await response.Content?.ReadAsStringAsync(); Assert.AreEqual(HttpStatusCode.OK, response.StatusCode); Assert.NotNull(responsePayload); var syncResponse = JsonConvert.DeserializeObject <SyncResponseViewModel>(responsePayload); Assert.NotNull(syncResponse); Assert.Null(syncResponse.Errors); Assert.True(syncResponse.Success); var request2 = new SyncRequestViewModel() { AppId = testApp.Id, AppApiAccessKey = testApp.AccessKey, Changes = new List <SyncRequestViewModel.ChangeViewModel> { new SyncRequestViewModel.ChangeViewModel { Group = group, Entity = entity, Property = propertyName, RecordId = recordId, MillisecondsAgo = modifiedMillisecondsAgo2, Value = value2 } }, Groups = new List <SyncRequestViewModel.GroupViewModel> { new SyncRequestViewModel.GroupViewModel { Group = group, Tidemark = null } } }; var jsonPayload2 = JsonConvert.SerializeObject(request2); var requestContent2 = new StringContent(jsonPayload2, Encoding.UTF8, "application/json"); var response2 = await HttpClient.PostAsync(SyncRequestUrl, requestContent2); var responsePayload2 = await response2.Content?.ReadAsStringAsync(); Assert.AreEqual(HttpStatusCode.OK, response2.StatusCode); Assert.NotNull(responsePayload2); var syncResponse2 = JsonConvert.DeserializeObject <SyncResponseViewModel>(responsePayload2); Assert.NotNull(syncResponse2); Assert.Null(syncResponse2.Errors); Assert.True(syncResponse2.Success); var dbRows = await GetChangeRows(testApp.Id, group); Assert.NotNull(dbRows); Assert.AreEqual(1, dbRows.Count); Assert.AreEqual(group, dbRows[0].GroupId); Assert.AreEqual(recordId, dbRows[0].RecordId); Assert.AreEqual(entity, dbRows[0].Entity); Assert.AreEqual(propertyName, dbRows[0].Property); Assert.AreEqual(recordId, dbRows[0].RecordId); Assert.AreEqual(value, dbRows[0].RecordValue); Assert.NotNull(syncResponse2.Groups); Assert.AreEqual(1, syncResponse2.Groups.Count); Assert.AreEqual(dbRows.Last().Id, syncResponse2.Groups[0].Tidemark); Assert.AreEqual(group, syncResponse2.Groups[0].Group); Assert.NotNull(syncResponse2.Groups[0].Changes); Assert.AreEqual(1, syncResponse2.Groups[0].Changes.Count); Assert.AreEqual(dbRows[0].ClientModified, syncResponse2.Groups[0].Changes[0].Modified); Assert.AreEqual(entity, syncResponse2.Groups[0].Changes[0].Entity); Assert.AreEqual(propertyName, syncResponse2.Groups[0].Changes[0].Property); Assert.AreEqual(recordId, syncResponse2.Groups[0].Changes[0].RecordId); Assert.AreEqual(value, syncResponse2.Groups[0].Changes[0].Value); }
public async Task SyncController_Post_Success_Single_Change() { var testApp = new Application { Id = new Guid("59eadf1b-c4bf-4ded-8a2b-b80305b960fe"), AccountId = new Guid("250c6f28-4611-4c28-902c-8464fabc510b"), AccessKey = new Guid("e7b40cf0-2781-4dc7-9545-91fd812fc506"), Name = "Integration Test App 2" }; string propertyName = "name"; string group = "group"; string entity = "Person"; Guid recordId = Guid.NewGuid(); long modifiedMillisecondsAgo = 10000; string value = "Neil"; await DeleteChangeRows(testApp.Id, group); SyncRequestViewModel request = new SyncRequestViewModel() { AppId = testApp.Id, AppApiAccessKey = testApp.AccessKey, Changes = new List <SyncRequestViewModel.ChangeViewModel> { new SyncRequestViewModel.ChangeViewModel { Group = group, RecordId = recordId, Entity = entity, Property = propertyName, MillisecondsAgo = modifiedMillisecondsAgo, Value = value } }, Groups = new List <SyncRequestViewModel.GroupViewModel> { new SyncRequestViewModel.GroupViewModel { Group = group, Tidemark = null } } }; var jsonPayload = JsonConvert.SerializeObject(request); var requestContent = new StringContent(jsonPayload, Encoding.UTF8, "application/json"); var response = await HttpClient.PostAsync(SyncRequestUrl, requestContent); var responsePayload = await response.Content?.ReadAsStringAsync(); Assert.AreEqual(HttpStatusCode.OK, response.StatusCode, $"Status: {response.StatusCode} Body: {await response.Content.ReadAsStringAsync()}"); Assert.NotNull(responsePayload); var syncResponse = JsonConvert.DeserializeObject <SyncResponseViewModel>(responsePayload); Assert.NotNull(syncResponse); Assert.Null(syncResponse.Errors); Assert.True(syncResponse.Success); Assert.NotNull(syncResponse.Groups); Assert.AreEqual(1, syncResponse.Groups.Count); Assert.NotNull(syncResponse.Groups[0].Changes); Assert.AreEqual(1, syncResponse.Groups[0].Changes.Count); var dbRows = await GetChangeRows(testApp.Id, group); Assert.NotNull(dbRows); Assert.AreEqual(1, dbRows.Count); Assert.AreEqual(group, dbRows[0].GroupId); Assert.AreEqual(propertyName, dbRows[0].Property); Assert.AreEqual(recordId, dbRows[0].RecordId); Assert.AreEqual(value, dbRows[0].RecordValue); Assert.AreEqual(dbRows[0].ClientModified, syncResponse.Groups[0].Changes[0].Modified); Assert.AreEqual(propertyName, syncResponse.Groups[0].Changes[0].Property); Assert.AreEqual(recordId, syncResponse.Groups[0].Changes[0].RecordId); Assert.AreEqual(entity, syncResponse.Groups[0].Changes[0].Entity); Assert.AreEqual(value, syncResponse.Groups[0].Changes[0].Value); }