public async Task SyncController_Post_Success_Basic_NoChanges_NoGroups()
{
var testApp = new Application
{
Id = new Guid("afd8db1e-73b8-4d5f-9cb1-6b49d205555a"),
AccountId = new Guid("250c6f28-4611-4c28-902c-8464fabc510b"),
AccessKey = new Guid("3d65a27c-9d1d-48a3-a888-89cc0f7851d0"),
Name = "Integration Test App"
};
SyncRequestViewModel request = new SyncRequestViewModel()
{
AppId = testApp.Id,
AppApiAccessKey = testApp.AccessKey
};
var jsonPayload = new StringContent(JsonConvert.SerializeObject(request), Encoding.UTF8, "application/json");
var response = await HttpClient.PostAsync(SyncRequestUrl, jsonPayload);
Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
Assert.NotNull(response.Content);
var syncResponse = JsonConvert.DeserializeObject <SyncResponseViewModel>(await response.Content.ReadAsStringAsync());
Assert.NotNull(syncResponse);
Assert.Null(syncResponse.Errors);
Assert.True(syncResponse.Success);
}
private async Task <SyncResponseViewModel> GetChanges(IApplication app, SyncRequestViewModel request)
{
var response = new SyncResponseViewModel()
{
Groups = new List <SyncResponseViewModel.GroupViewModel>()
};
if (request.Groups != null)
{
foreach (var group in request.Groups)
{
List <IChange> results = await ChangeRepository.ListChangesAsync(app.Id, group.Group, group.Tidemark);
if (results != null && results.Any())
{
response.Groups.Add(new SyncResponseViewModel.GroupViewModel()
{
Group = group.Group,
Tidemark = results.Last().Id,
Changes = results.Select(r => new SyncResponseViewModel.ChangeViewModel()
{
Modified = r.ClientModified,
Value = r.RecordValue,
Entity = r.Entity,
RecordId = r.RecordId,
Property = r.Property
}).ToList()
});
}
}
}
return(response);
}
private async Task ProcessChanges(IApplication app, SyncRequestViewModel request)
{
if (request.Changes != null && request.Changes.Any())
{
var dbChanges = new List <IChange>();
Stopwatch sw = new Stopwatch();
sw.Start();
// Dedup any changes with the same Record, Entity and Property
var uniqueChanges = new Dictionary <string, SyncRequestViewModel.ChangeViewModel>();
foreach (var change in request.Changes)
{
string key = $"{change.RecordId}-{change.Entity}-{change.Property}";
if (uniqueChanges.TryGetValue(key, out var existingChange))
{
if (change.MillisecondsAgo < existingChange.MillisecondsAgo)
{
uniqueChanges[key] = change;
}
}
else
{
uniqueChanges.Add(key, change);
}
}
Logger.LogInformation($"Deduped changes in {sw.ElapsedMilliseconds}ms count: {dbChanges.Count}");
sw.Restart();
foreach (var change in uniqueChanges.Values)
{
if (change != null)
{
DateTime modifiedUTC = requestStartTimeUTC.AddMilliseconds(-change.MillisecondsAgo);
var dbChange = ChangeRepository.CreateChange(app.AccountId, app.Id, change.RecordId, change.Group, change.Entity, change.Property, modifiedUTC.Ticks, change.Value);
dbChanges.Add(dbChange);
}
}
Logger.LogInformation($"Generated changes in {sw.ElapsedMilliseconds}ms count: {dbChanges.Count}");
sw.Restart();
if (dbChanges.Any())
{
sw.Restart();
await ChangeRepository.UpsertChangesAsync(app.Id, dbChanges);
Logger.LogInformation($"Saved changes to database in {sw.ElapsedMilliseconds}ms count: {dbChanges.Count}");
}
sw.Stop();
}
}
private async Task ProcessChanges(Application app, Device device, SyncRequestViewModel request)
{
if (request.Changes != null && request.Changes.Any())
{
Stopwatch sw = new Stopwatch();
sw.Start();
foreach (var batch in request.Changes.Batch(50))
{
var changes = new List <SendContextModel <UpsetModel <Change> > >();
foreach (var change in batch)
{
if (change != null)
{
Guid recordId = Guid.Empty;
string path = null;
// Path should contain a / in format <guid>/property.name
if (!string.IsNullOrWhiteSpace(change.Path) && change.Path.IndexOf("/") > -1)
{
recordId = Guid.Parse(change.Path.Substring(0, change.Path.IndexOf("/")));
path = change.Path.Substring(change.Path.IndexOf("/") + 1);
}
DateTime modifiedUTC = requestStartTimeUTC.AddSeconds(-change.SecondsAgo);
var dbChange = new Change()
{
Id = Guid.NewGuid(),
RecordId = recordId,
Path = change.Path,
DeviceId = device.Id,
Modified = modifiedUTC,
Tidemark = "%clustertime%",
Value = change.Value
};
string partition = $"{app.Id}-{change.Group}";
changes.Add(ScaleContext.MakeUpsertModel(partition, "change", dbChange));
}
}
Logger.LogInformation($"Generated changes in {sw.ElapsedMilliseconds}ms count: {changes.Count}");
sw.Restart();
await ScaleContext.UpsertBulk(changes);
Logger.LogInformation($"Saved changes to database in {sw.ElapsedMilliseconds}ms count: {changes.Count}");
}
sw.Stop();
}
}
private async Task <Device> ValidateDevice(SyncRequestViewModel request)
{
var device = await Cache.GetByPrimaryKeyFromCacheOrQuery <Device>(ScaleContext.SystemPartition, "device", "device_id", request.DeviceId, defaultCacheDuration);
if (device == null)
{
ModelState.AddModelError("device_id", "No device found for device_id");
}
return(device);
}
public async Task <JsonResult> Post([FromBody] SyncRequestViewModel request)
{
requestStartTimeUTC = DateTime.UtcNow;
var response = new SyncResponseViewModel();
try
{
Stopwatch sw = new Stopwatch();
sw.Start();
var app = await ValidateApplication(request);
Logger.LogInformation($"ValidateApplication in {sw.ElapsedMilliseconds}ms");
if (!ModelState.IsValid)
{
return(JsonResultWithValidationErrors(response));
}
sw.Restart();
var device = await ValidateDevice(request);
Logger.LogInformation($"ValidateDevice in {sw.ElapsedMilliseconds}ms");
if (!ModelState.IsValid)
{
return(JsonResultWithValidationErrors(response));
}
sw.Restart();
await ProcessChanges(app, device, request);
Logger.LogInformation($"ProcessChanges in {sw.ElapsedMilliseconds}ms");
sw.Restart();
response = await GetChanges(app, device, request);
Logger.LogInformation($"GetChanges in {sw.ElapsedMilliseconds}ms");
sw.Stop();
}
catch (Exception ex)
{
Logger.LogError($"Failed to complete Sync Post: {ex.ToString()}");
ModelState.AddModelError("", $"Unhandled exception: {ex.ToString()}");
}
return(JsonResultWithValidationErrors(response));
}
public async Task SyncController_Post_Fail_Empty_Request()
{
SyncRequestViewModel request = new SyncRequestViewModel();
var jsonPayload = new StringContent(JsonConvert.SerializeObject(request), Encoding.UTF8, "application/json");
var response = await HttpClient.PostAsync(SyncRequestUrl, jsonPayload);
Assert.AreEqual(HttpStatusCode.BadRequest, response.StatusCode);
Assert.NotNull(response.Content);
var syncResponse = JsonConvert.DeserializeObject <SyncResponseViewModel>(await response.Content.ReadAsStringAsync());
Assert.NotNull(syncResponse);
Assert.NotNull(syncResponse.Errors);
Assert.AreEqual(1, syncResponse.Errors.Count());
Assert.AreEqual("app_id missing or invalid request", syncResponse.Errors.First());
Assert.False(syncResponse.Success);
}
public async Task SyncController_Post_Fail_Missing_AppApiAccessKey()
{
SyncRequestViewModel request = new SyncRequestViewModel()
{
AppId = new Guid("afd8db1e-73b8-4d5f-9cb1-6b49d205555a")
};
var jsonPayload = new StringContent(JsonConvert.SerializeObject(request), Encoding.UTF8, "application/json");
var response = await HttpClient.PostAsync(SyncRequestUrl, jsonPayload);
Assert.AreEqual(HttpStatusCode.BadRequest, response.StatusCode);
Assert.NotNull(response.Content);
var syncResponse = JsonConvert.DeserializeObject <SyncResponseViewModel>(await response.Content.ReadAsStringAsync());
Assert.NotNull(syncResponse);
Assert.NotNull(syncResponse.Errors);
Assert.AreEqual(1, syncResponse.Errors.Count());
Assert.AreEqual("app_api_access_key incorrect for app_id", syncResponse.Errors.First());
Assert.False(syncResponse.Success);
}
public async Task Success_Basic_EmptyChange()
{
var request = new SyncRequestViewModel()
{
AppId = app.Id,
AppApiAccessKey = app.AccessKey,
DeviceId = device.Id,
Changes = new List <SyncRequestViewModel.ChangeViewModel>
{
new SyncRequestViewModel.ChangeViewModel
{
}
}
};
var controller = new SyncController(logger.Object, queryCache.Object, scaleContext.Object);
var response = await controller.Post(request);
var syncResponse = response.Value as SyncResponseViewModel;
Assert.NotNull(syncResponse);
Assert.Null(syncResponse.Errors);
Assert.True(syncResponse.Success);
}
private async Task <IApplication> ValidateApplication(SyncRequestViewModel request)
{
if (request == null || request.AppId == Guid.Empty)
{
ModelState.AddModelError("app_id", "app_id missing or invalid request");
}
else
{
var app = await ApplicationRepository.GetByIdAsync(request.AppId);
if (app == null)
{
ModelState.AddModelError("app_id", "No application found for app_id");
}
else if (app.AccessKey != request.AppApiAccessKey)
{
ModelState.AddModelError("app_api_access_key", "app_api_access_key incorrect for app_id");
}
return(app);
}
return(null);
}
private async Task <Application> ValidateApplication(SyncRequestViewModel request)
{
if (request == null || request.AppId == Guid.Empty)
{
ModelState.AddModelError("app_id", "app_id missing or invalid request");
}
else
{
var app = await Cache.GetByPrimaryKeyFromCacheOrQuery <Application>(ScaleContext.SystemPartition, "application", "app_id", request.AppId, defaultCacheDuration);
if (app == null)
{
ModelState.AddModelError("app_id", "No application found for app_id");
}
else if (app.AccessKey != request.AppApiAccessKey)
{
ModelState.AddModelError("app_api_access_key", "app_api_access_key incorrect for app_id");
}
return(app);
}
return(null);
}
public async Task Success_Basic_Badly_Formatted_Path()
{
string partition = null;
string table = null;
Change value = null;
scaleContext
.Setup(x => x.MakeUpsertModel(It.IsAny <string>(), "change", It.IsAny <Change>()))
.Callback <string, string, Change>((p, t, v) =>
{
partition = p;
table = t;
value = v;
});
var request = new SyncRequestViewModel()
{
AppId = app.Id,
AppApiAccessKey = app.AccessKey,
DeviceId = device.Id,
Changes = new List <SyncRequestViewModel.ChangeViewModel>
{
new SyncRequestViewModel.ChangeViewModel
{
Group = "Group",
Path = "bad format"
}
}
};
var controller = new SyncController(logger.Object, queryCache.Object, scaleContext.Object);
var response = await controller.Post(request);
var syncResponse = response.Value as SyncResponseViewModel;
//var dbChange = new Change()
//{
// Id = Guid.NewGuid(),
// RecordId = recordId,
// Path = change.Path,
// DeviceId = device.Id,
// Modified = modifiedUTC,
// Tidemark = "%clustertime%",
// Value = change.Value
//};
Assert.NotNull(syncResponse);
Assert.Null(syncResponse.Errors);
Assert.True(syncResponse.Success);
Assert.AreEqual($"{app.Id}-Group", partition);
Assert.AreEqual($"change", table);
Assert.NotNull(value);
//// Path should contain a / in format <guid>/property.name
//if (!string.IsNullOrWhiteSpace(change.Path) && change.Path.IndexOf("/") > -1)
//{
// recordId = Guid.Parse(change.Path.Substring(0, change.Path.IndexOf("/")));
// path = change.Path.Substring(change.Path.IndexOf("/") + 1);
//}
scaleContext.Verify(t => t.MakeUpsertModel(It.IsAny <string>(), "change", It.IsAny <Change>()));
}
public async Task SyncController_Post_Success_Send_Two_Changes_FilterOutChangesForTheSameRecordEntityAndProperty()
{
string propertyName = "name";
string group = "group";
string entity = "person";
Guid recordId = Guid.NewGuid();
long modifiedMillisecondsAgo = 10000;
long modifiedMillisecondsAgo2 = 20000;
string value = "Neil";
string value2 = "Adrian";
List <IChange> returnChanges = null;
DateTime requestStart = DateTime.UtcNow;
timeService = new Mock <ITimeService>();
timeService.Setup(x => x.GetUtcNow()).Returns(requestStart);
changeRepository
.Setup(x => x.UpsertChangesAsync(It.IsAny <Guid>(), It.IsAny <IEnumerable <IChange> >()))
.Returns(() => Task.FromResult((string)null))
.Callback <Guid, IEnumerable <IChange> >((a, l) =>
{
returnChanges = l.ToList();
});
var request = new SyncRequestViewModel()
{
AppId = app.Object.Id,
AppApiAccessKey = app.Object.AccessKey,
Changes = new List <SyncRequestViewModel.ChangeViewModel>
{
new SyncRequestViewModel.ChangeViewModel
{
Group = group,
RecordId = recordId,
Entity = entity,
Property = propertyName,
MillisecondsAgo = modifiedMillisecondsAgo,
Value = value
},
new SyncRequestViewModel.ChangeViewModel
{
Group = group,
RecordId = recordId,
Entity = entity,
Property = propertyName,
MillisecondsAgo = modifiedMillisecondsAgo2,
Value = value2
}
}
};
SyncController controller = CreateSyncController();
var response = await controller.Post(request) as JsonResult;
var syncResponse = response.Value as SyncResponseViewModel;
Assert.NotNull(syncResponse);
Assert.Null(syncResponse.Errors);
Assert.True(syncResponse.Success);
Assert.NotNull(syncResponse.Groups);
Assert.AreEqual(0, syncResponse.Groups.Count);
Assert.NotNull(returnChanges);
Assert.AreEqual(1, returnChanges.Count);
Assert.AreEqual(group, returnChanges[0].GroupId);
Assert.AreEqual(propertyName, returnChanges[0].Property);
Assert.AreEqual(entity, returnChanges[0].Entity);
Assert.AreEqual(recordId, returnChanges[0].RecordId);
Assert.AreEqual(value, returnChanges[0].RecordValue);
IEnumerable <IChange> changes = new List <IChange>()
{
change.Object
};
changeRepository.Verify(t => t.UpsertChangesAsync(app.Object.Id, changes), Times.Once);
changeRepository.Verify(t => t.CreateChange(app.Object.AccountId, app.Object.Id, recordId, group, entity, propertyName, requestStart.AddMilliseconds(-modifiedMillisecondsAgo).Ticks, value), Times.Once);
changeRepository.Verify(t => t.ListChangesAsync(It.IsAny <Guid>(), It.IsAny <string>(), It.IsAny <long?>()), Times.Never);
}
private async Task <SyncResponseViewModel> GetChanges(Application app, Device device, SyncRequestViewModel request)
{
var response = new SyncResponseViewModel()
{
Groups = new List <SyncResponseViewModel.GroupViewModel>()
};
Stopwatch sw = new Stopwatch();
sw.Start();
if (request.Groups != null)
{
foreach (var group in request.Groups)
{
sw.Restart();
string partition = $"{app.Id}-{group.Group}";
var queryParams = new List <object>();
string whereClause = null;
if (!string.IsNullOrWhiteSpace(group.Tidemark))
{
Logger.LogInformation($"Getting changes for group: {group.Group} after tidemark: {group.Tidemark}");
queryParams.Add(group.Tidemark);
whereClause = "tidemark > ?";
}
else
{
Logger.LogInformation($"Getting all changes for group: {group.Group}");
}
List <Change> results = await ScaleContext.Query <Change>(partition, "change", whereClause, queryParams, orderBy : "tidemark", limit : 50);
Logger.LogInformation($"Retrieved changes from database in {sw.ElapsedMilliseconds}ms count: {results.Count}");
if (results != null && results.Any())
{
response.Groups.Add(new SyncResponseViewModel.GroupViewModel()
{
Group = group.Group,
Tidemark = results.Last().Tidemark,
Changes = results.Select(r => new SyncResponseViewModel.ChangeViewModel()
{
Modified = r.Modified,
Value = r.Value,
Path = r.Path
}).ToList()
});
}
}
}
sw.Stop();
return(response);
}
public async Task SyncController_Post_Success_TwoChangesToSamePropertyOverTwoRequests_EnsureOnlyLatestIsReturned()
{
var testApp = new Application
{
Id = new Guid("19d8856c-a439-46ae-9932-c81fd0fe5556"),
AccountId = new Guid("250c6f28-4611-4c28-902c-8464fabc510b"),
AccessKey = new Guid("0f458ce8-1a0e-450c-a2c4-2b50b3c4f41d"),
Name = "Integration Test App 4"
};
string propertyName = "name";
string group = "group";
string entity = "Person";
Guid recordId = Guid.NewGuid();
int modifiedMillisecondsAgo = 10000;
int modifiedMillisecondsAgo2 = 20000;
string value = "Neil";
string value2 = "Adrian";
await DeleteChangeRows(testApp.Id, group);
var request = new SyncRequestViewModel()
{
AppId = testApp.Id,
AppApiAccessKey = testApp.AccessKey,
Changes = new List <SyncRequestViewModel.ChangeViewModel>
{
new SyncRequestViewModel.ChangeViewModel
{
Group = group,
Entity = entity,
Property = propertyName,
RecordId = recordId,
MillisecondsAgo = modifiedMillisecondsAgo,
Value = value
}
}
};
var jsonPayload = JsonConvert.SerializeObject(request);
var requestContent = new StringContent(jsonPayload, Encoding.UTF8, "application/json");
var response = await HttpClient.PostAsync(SyncRequestUrl, requestContent);
var responsePayload = await response.Content?.ReadAsStringAsync();
Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
Assert.NotNull(responsePayload);
var syncResponse = JsonConvert.DeserializeObject <SyncResponseViewModel>(responsePayload);
Assert.NotNull(syncResponse);
Assert.Null(syncResponse.Errors);
Assert.True(syncResponse.Success);
var request2 = new SyncRequestViewModel()
{
AppId = testApp.Id,
AppApiAccessKey = testApp.AccessKey,
Changes = new List <SyncRequestViewModel.ChangeViewModel>
{
new SyncRequestViewModel.ChangeViewModel
{
Group = group,
Entity = entity,
Property = propertyName,
RecordId = recordId,
MillisecondsAgo = modifiedMillisecondsAgo2,
Value = value2
}
},
Groups = new List <SyncRequestViewModel.GroupViewModel>
{
new SyncRequestViewModel.GroupViewModel
{
Group = group,
Tidemark = null
}
}
};
var jsonPayload2 = JsonConvert.SerializeObject(request2);
var requestContent2 = new StringContent(jsonPayload2, Encoding.UTF8, "application/json");
var response2 = await HttpClient.PostAsync(SyncRequestUrl, requestContent2);
var responsePayload2 = await response2.Content?.ReadAsStringAsync();
Assert.AreEqual(HttpStatusCode.OK, response2.StatusCode);
Assert.NotNull(responsePayload2);
var syncResponse2 = JsonConvert.DeserializeObject <SyncResponseViewModel>(responsePayload2);
Assert.NotNull(syncResponse2);
Assert.Null(syncResponse2.Errors);
Assert.True(syncResponse2.Success);
var dbRows = await GetChangeRows(testApp.Id, group);
Assert.NotNull(dbRows);
Assert.AreEqual(1, dbRows.Count);
Assert.AreEqual(group, dbRows[0].GroupId);
Assert.AreEqual(recordId, dbRows[0].RecordId);
Assert.AreEqual(entity, dbRows[0].Entity);
Assert.AreEqual(propertyName, dbRows[0].Property);
Assert.AreEqual(recordId, dbRows[0].RecordId);
Assert.AreEqual(value, dbRows[0].RecordValue);
Assert.NotNull(syncResponse2.Groups);
Assert.AreEqual(1, syncResponse2.Groups.Count);
Assert.AreEqual(dbRows.Last().Id, syncResponse2.Groups[0].Tidemark);
Assert.AreEqual(group, syncResponse2.Groups[0].Group);
Assert.NotNull(syncResponse2.Groups[0].Changes);
Assert.AreEqual(1, syncResponse2.Groups[0].Changes.Count);
Assert.AreEqual(dbRows[0].ClientModified, syncResponse2.Groups[0].Changes[0].Modified);
Assert.AreEqual(entity, syncResponse2.Groups[0].Changes[0].Entity);
Assert.AreEqual(propertyName, syncResponse2.Groups[0].Changes[0].Property);
Assert.AreEqual(recordId, syncResponse2.Groups[0].Changes[0].RecordId);
Assert.AreEqual(value, syncResponse2.Groups[0].Changes[0].Value);
}
public async Task SyncController_Post_Success_Single_Change()
{
var testApp = new Application
{
Id = new Guid("59eadf1b-c4bf-4ded-8a2b-b80305b960fe"),
AccountId = new Guid("250c6f28-4611-4c28-902c-8464fabc510b"),
AccessKey = new Guid("e7b40cf0-2781-4dc7-9545-91fd812fc506"),
Name = "Integration Test App 2"
};
string propertyName = "name";
string group = "group";
string entity = "Person";
Guid recordId = Guid.NewGuid();
long modifiedMillisecondsAgo = 10000;
string value = "Neil";
await DeleteChangeRows(testApp.Id, group);
SyncRequestViewModel request = new SyncRequestViewModel()
{
AppId = testApp.Id,
AppApiAccessKey = testApp.AccessKey,
Changes = new List <SyncRequestViewModel.ChangeViewModel>
{
new SyncRequestViewModel.ChangeViewModel
{
Group = group,
RecordId = recordId,
Entity = entity,
Property = propertyName,
MillisecondsAgo = modifiedMillisecondsAgo,
Value = value
}
},
Groups = new List <SyncRequestViewModel.GroupViewModel>
{
new SyncRequestViewModel.GroupViewModel
{
Group = group,
Tidemark = null
}
}
};
var jsonPayload = JsonConvert.SerializeObject(request);
var requestContent = new StringContent(jsonPayload, Encoding.UTF8, "application/json");
var response = await HttpClient.PostAsync(SyncRequestUrl, requestContent);
var responsePayload = await response.Content?.ReadAsStringAsync();
Assert.AreEqual(HttpStatusCode.OK, response.StatusCode, $"Status: {response.StatusCode} Body: {await response.Content.ReadAsStringAsync()}");
Assert.NotNull(responsePayload);
var syncResponse = JsonConvert.DeserializeObject <SyncResponseViewModel>(responsePayload);
Assert.NotNull(syncResponse);
Assert.Null(syncResponse.Errors);
Assert.True(syncResponse.Success);
Assert.NotNull(syncResponse.Groups);
Assert.AreEqual(1, syncResponse.Groups.Count);
Assert.NotNull(syncResponse.Groups[0].Changes);
Assert.AreEqual(1, syncResponse.Groups[0].Changes.Count);
var dbRows = await GetChangeRows(testApp.Id, group);
Assert.NotNull(dbRows);
Assert.AreEqual(1, dbRows.Count);
Assert.AreEqual(group, dbRows[0].GroupId);
Assert.AreEqual(propertyName, dbRows[0].Property);
Assert.AreEqual(recordId, dbRows[0].RecordId);
Assert.AreEqual(value, dbRows[0].RecordValue);
Assert.AreEqual(dbRows[0].ClientModified, syncResponse.Groups[0].Changes[0].Modified);
Assert.AreEqual(propertyName, syncResponse.Groups[0].Changes[0].Property);
Assert.AreEqual(recordId, syncResponse.Groups[0].Changes[0].RecordId);
Assert.AreEqual(entity, syncResponse.Groups[0].Changes[0].Entity);
Assert.AreEqual(value, syncResponse.Groups[0].Changes[0].Value);
}
