public async Task ConvertAsyncThrowsOnEmptyDevices()
{
string tenantId = this.rand.NextString();
List <ValueApiModel> deviceGroupsList = new List <ValueApiModel>
{
this.entityHelper.CreateDeviceGroup(),
this.entityHelper.CreateDeviceGroup(),
};
ValueListApiModel deviceGroups = new ValueListApiModel
{
Items = deviceGroupsList,
};
this.mockStorageAdapterClient
.Setup(c => c.GetAllAsync(
It.Is <string>(s => s == this.converter.Entity),
It.IsAny <string>()))
.ReturnsAsync(deviceGroups);
this.mockIotHubManagerClient
.Setup(c => c.GetListAsync(
It.IsAny <IEnumerable <DeviceGroupConditionModel> >(),
It.Is <string>(s => s == tenantId)))
.ReturnsAsync(new DeviceListModel {
Items = new List <DeviceModel> {
}
}); // return empty device lists, should cause the exception
Func <Task> conversion = async() => await this.converter.ConvertAsync(tenantId);
await Assert.ThrowsAsync <ResourceNotFoundException>(conversion);
}
public async Task VerifyGetBehaviorIfDoIncludeDeleted()
{
// Arrange
Rule test = new Rule
{
Enabled = false,
Deleted = true,
Id = "id",
ETag = "123"
};
string ruleString = JsonConvert.SerializeObject(test);
ValueApiModel model = new ValueApiModel
{
Data = ruleString,
ETag = "123",
Key = "id"
};
ValueListApiModel result = new ValueListApiModel();
result.Items = new List <ValueApiModel> {
model
};
this.storageAdapter.Setup(x => x.GetAllAsync(It.IsAny <string>()))
.Returns(Task.FromResult(result));
// Act
List <Rule> rulesList = await this.rules.GetListAsync("asc", 0, LIMIT, null, true);
// Assert
Assert.Single(rulesList);
this.storageAdapter.Verify(x => x.GetAllAsync(It.IsAny <string>()), Times.Once);
}
public async Task DeleteRule_RetriesLogOnError()
{
// Arrange
ValueListApiModel fakeRules = new ValueListApiModel();
fakeRules.Items.Add(this.CreateFakeRule("rule1"));
fakeRules.Items.Add(this.CreateFakeRule("rule2"));
this.storageAdapter.Setup(x => x.GetAllAsync(It.IsAny <string>())).Returns(Task.FromResult(fakeRules));
this.httpClientMock.SetupSequence(x => x.PostAsync(It.IsAny <HttpRequest>()))
.Throws <Exception>()
.ReturnsAsync(new HttpResponse(HttpStatusCode.ServiceUnavailable, "", null))
.ReturnsAsync(new HttpResponse(HttpStatusCode.OK, "", null))
.ReturnsAsync(new HttpResponse(HttpStatusCode.OK, "", null));
Rule test = new Rule
{
Enabled = true,
Deleted = false
};
this.SetUpStorageAdapterGet(test);
// Act
await this.rules.DeleteAsync("id");
// Assert
this.storageAdapter.Verify(x => x.GetAllAsync(It.IsAny <string>()), Times.Once);
this.httpClientMock.Verify(x => x.PostAsync(It.IsAny <HttpRequest>()), Times.Exactly(4));
}
public async Task DeleteRule_QueriesRuleCountAndLogs()
{
// Arrange
ValueListApiModel fakeRules = new ValueListApiModel();
fakeRules.Items.Add(this.CreateFakeRule("rule1"));
fakeRules.Items.Add(this.CreateFakeRule("rule2"));
this.storageAdapter.Setup(x => x.GetAllAsync(It.IsAny <string>())).Returns(Task.FromResult(fakeRules));
IHttpResponse fakeOkResponse = new HttpResponse(HttpStatusCode.OK, "", null);
this.httpClientMock.Setup(x => x.PostAsync(It.IsAny <HttpRequest>())).ReturnsAsync(fakeOkResponse);
Rule test = new Rule
{
Enabled = true,
Deleted = false
};
this.SetUpStorageAdapterGet(test);
// Act
await this.rules.DeleteAsync("id");
// Assert
this.storageAdapter.Verify(x => x.GetAllAsync(It.IsAny <string>()), Times.Once);
this.httpClientMock.Verify(x => x.PostAsync(It.IsAny <HttpRequest>()), Times.Exactly(2));
}
public async Task ConvertAsyncReturnsExpectedModel()
{
string tenantId = this.rand.NextString();
List <ValueApiModel> deviceGroupsList = new List <ValueApiModel>
{
this.entityHelper.CreateDeviceGroup(),
this.entityHelper.CreateDeviceGroup(),
};
ValueListApiModel deviceGroups = new ValueListApiModel
{
Items = deviceGroupsList,
};
this.mockStorageAdapterClient
.Setup(c => c.GetAllAsync(
It.Is <string>(s => s == this.converter.Entity)))
.ReturnsAsync(deviceGroups);
this.mockBlobStorageClient
.Setup(c => c.CreateBlobAsync(
It.IsAny <string>(),
It.IsAny <string>(),
It.IsAny <string>()))
.Returns(Task.CompletedTask);
this.mockIotHubManagerClient
.Setup(c => c.GetListAsync(
It.IsAny <IEnumerable <DeviceGroupConditionModel> >(),
It.Is <string>(s => s == tenantId)))
.ReturnsAsync(new DeviceListModel {
Items = new List <DeviceModel> {
this.entityHelper.CreateDevice(), this.entityHelper.CreateDevice()
}
}); // return a device for each device group
ConversionApiModel conversionResponse = await this.converter.ConvertAsync(tenantId);
this.mockStorageAdapterClient
.Verify(
c => c.GetAllAsync(
It.Is <string>(s => s == this.converter.Entity)),
Times.Once);
this.mockBlobStorageClient
.Verify(
c => c.CreateBlobAsync(
It.IsAny <string>(),
It.IsAny <string>(),
It.IsAny <string>()),
Times.Once);
this.mockIotHubManagerClient
.Verify(
c => c.GetListAsync(
It.IsAny <IEnumerable <DeviceGroupConditionModel> >(),
It.Is <string>(s => s == tenantId)),
Times.Exactly(deviceGroups.Items.Count));
Assert.Equal(conversionResponse.Entities, deviceGroups);
Assert.Equal(conversionResponse.TenantId, tenantId);
}
public void UpsertUsesOptimisticConcurrency()
{
// Arrange
const string ETAG1 = "001";
const string ETAG2 = "002";
// Initial simulation
var simulation1 = new SimulationModel {
Id = SIMULATION_ID, ETag = ETAG1
};
var storageRecord1 = new ValueApiModel
{
Key = SIMULATION_ID,
Data = JsonConvert.SerializeObject(simulation1),
ETag = simulation1.ETag
};
var storageList1 = new ValueListApiModel();
storageList1.Items.Add(storageRecord1);
// Simulation after update
var simulation2 = new SimulationModel {
Id = SIMULATION_ID, ETag = ETAG2
};
var storageRecord2 = new ValueApiModel
{
Key = SIMULATION_ID,
Data = JsonConvert.SerializeObject(simulation2),
ETag = simulation2.ETag
};
var storageList2 = new ValueListApiModel();
storageList2.Items.Add(storageRecord2);
// Initial setup - the ETag matches
this.storage.Setup(x => x.GetAllAsync(STORAGE_COLLECTION)).ReturnsAsync(storageList1);
this.storage.Setup(x => x.UpdateAsync(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>()))
.ReturnsAsync(storageRecord2);
// Act - No exception because ETag matches
// Note: the call to UpsertAsync modifies the object, don't reuse the variable later
this.target.UpsertAsync(simulation1).Wait(Constants.TEST_TIMEOUT);
// Arrange - the ETag won't match
this.storage.Setup(x => x.GetAllAsync(STORAGE_COLLECTION)).ReturnsAsync(storageList2);
// Act + Assert
var simulationOutOfDate = new SimulationModel {
Id = SIMULATION_ID, ETag = ETAG1
};
Assert.ThrowsAsync <ResourceOutOfDateException>(
async() => await this.target.UpsertAsync(simulationOutOfDate))
.Wait(Constants.TEST_TIMEOUT);
}
private async Task<int> GetRuleCountAsync()
{
ValueListApiModel rules = await this.storage.GetAllAsync(StorageCollection);
int ruleCount = 0;
foreach (var item in rules.Items)
{
var rule = this.Deserialize(item.Data);
if (!rule.Deleted)
{
ruleCount++;
}
}
return ruleCount;
}
private void SetupAListOfInvalidDeviceModelScriptsInStorage()
{
var list = new ValueListApiModel();
var value = new ValueApiModel
{
Key = "key1",
Data = "{ 'invalid': json",
ETag = "etag"
};
list.Items.Add(value);
this.storage
.Setup(x => x.GetAllAsync(It.IsAny <string>()))
.ReturnsAsync(list);
}
public async Task ConvertAsyncThrowsOnEmptyRules()
{
string tenantId = this.rand.NextString();
ValueListApiModel rules = new ValueListApiModel
{
Items = new List <ValueApiModel>(),
};
this.mockStorageAdapterClient
.Setup(c => c.GetAllAsync(
It.Is <string>(s => s == this.converter.Entity)))
.ReturnsAsync(rules);
Func <Task> conversion = async() => await this.converter.ConvertAsync(tenantId);
await Assert.ThrowsAsync <ResourceNotFoundException>(conversion);
}
public async Task ConvertAsyncReturnsExpectedModel()
{
string tenantId = this.rand.NextString();
List <ValueApiModel> rulesList = new List <ValueApiModel>
{
this.entityHelper.CreateRule(),
this.entityHelper.CreateRule(),
};
ValueListApiModel rules = new ValueListApiModel
{
Items = rulesList,
};
this.mockStorageAdapterClient
.Setup(c => c.GetAllAsync(
It.Is <string>(s => s == this.converter.Entity),
It.IsAny <string>()))
.ReturnsAsync(rules);
this.mockBlobStorageClient
.Setup(c => c.CreateBlobAsync(
It.IsAny <string>(),
It.IsAny <string>(),
It.IsAny <string>()))
.Returns(Task.CompletedTask);
ConversionApiModel conversionResponse = await this.converter.ConvertAsync(tenantId);
this.mockStorageAdapterClient
.Verify(
c => c.GetAllAsync(
It.Is <string>(s => s == this.converter.Entity),
It.IsAny <string>()),
Times.Once);
this.mockBlobStorageClient
.Verify(
c => c.CreateBlobAsync(
It.IsAny <string>(),
It.IsAny <string>(),
It.IsAny <string>()),
Times.Once);
Assert.Equal(conversionResponse.Entities, rules);
Assert.Equal(conversionResponse.TenantId, tenantId);
}
private void SetupAListOfDeviceModelsInStorage(string etag)
{
var deviceModel = new DeviceModel {
Id = "id", ETag = etag
};
var list = new ValueListApiModel();
var value = new ValueApiModel
{
Key = "key",
Data = JsonConvert.SerializeObject(deviceModel),
ETag = deviceModel.ETag
};
list.Items.Add(value);
this.storage
.Setup(x => x.GetAllAsync(It.IsAny <string>()))
.ReturnsAsync(list);
}
/// <summary>
/// Get List of deviceProperties from cache
/// </summary>
public async Task <List <string> > GetListAsync()
{
ValueListApiModel response = new ValueListApiModel();
try
{
response = await this.storageClient.GetAllAsync(CACHE_COLLECTION_ID);
}
catch (ResourceNotFoundException)
{
this.log.Debug($"Cache get: cache {CACHE_COLLECTION_ID}:{CACHE_KEY} was not found",
() => { });
}
catch (Exception e)
{
throw new ExternalDependencyException(
$"Cache get: unable to get device-twin-properties cache", e);
}
DevicePropertyServiceModel properties = new DevicePropertyServiceModel();
try
{
properties = JsonConvert.DeserializeObject <DevicePropertyServiceModel>(response.Items.FirstOrDefault().Data);
}
catch (Exception e)
{
throw new InvalidInputException("Unable to deserialize deviceProperties from CosmosDB", e);
}
List <string> result = new List <string>();
foreach (string tag in properties.Tags)
{
result.Add(TAG_PREFIX + tag);
}
foreach (string reported in properties.Reported)
{
result.Add(REPORTED_PREFIX + reported);
}
return(result);
}
private void ThereIsAnEnabledSimulationInTheStorage()
{
var simulation = new SimulationModel
{
Id = SIMULATION_ID,
Created = DateTimeOffset.UtcNow.Subtract(TimeSpan.FromDays(10)),
Modified = DateTimeOffset.UtcNow.Subtract(TimeSpan.FromDays(10)),
ETag = "ETag0",
Enabled = true
};
var list = new ValueListApiModel();
var value = new ValueApiModel
{
Key = SIMULATION_ID,
Data = JsonConvert.SerializeObject(simulation),
ETag = simulation.ETag
};
list.Items.Add(value);
this.mockStorageAdapterClient.Setup(x => x.GetAllAsync(STORAGE_COLLECTION)).ReturnsAsync(list);
}
public override async Task <ConversionApiModel> ConvertAsync(string tenantId, string operationId = null)
{
ValueListApiModel rules = null;
try
{
rules = await this.StorageAdapterClient.GetAllAsync(this.Entity);
}
catch (Exception e)
{
this.Logger.LogError(e, "Unable to query {entity} using storage adapter. OperationId: {operationId}. TenantId: {tenantId}", this.Entity, operationId, tenantId);
throw e;
}
if (rules.Items.Count() == 0 || rules == null)
{
string errorMessage = $"No entities were receieved from storage adapter to convert to {this.Entity}. OperationId: {operationId}. TenantId: {tenantId}";
this.Logger.LogError(new Exception(errorMessage), errorMessage);
throw new ResourceNotFoundException("No entities were receieved from storage adapter to convert to rules.");
}
List <RuleReferenceDataModel> jsonRulesList = new List <RuleReferenceDataModel>();
try
{
foreach (ValueApiModel item in rules.Items)
{
try
{
RuleDataModel dataModel = JsonConvert.DeserializeObject <RuleDataModel>(item.Data);
RuleModel ruleModel = new RuleModel(item.Key, dataModel);
// return a RuleReferenceModel which is a conversion from the RuleModel into a SAjob readable format with additional metadata
RuleReferenceDataModel referenceModel = new RuleReferenceDataModel(ruleModel);
jsonRulesList.Add(referenceModel);
}
catch (Exception)
{
this.Logger.LogInformation("Unable to convert a rule to the proper reference data model for {entity}. OperationId: {operationId}. TenantId: {tenantId}", this.Entity, operationId, tenantId);
}
}
if (jsonRulesList.Count() == 0)
{
throw new ResourceNotSupportedException("No rules were able to be converted to the proper rule reference data model.");
}
}
catch (Exception e)
{
this.Logger.LogError(e, "Unable to convert {entity} queried from storage adapter to appropriate data model. OperationId: {operationId}. TenantId: {tenantId}", this.Entity, operationId, tenantId);
throw e;
}
string fileContent = null;
try
{
fileContent = JsonConvert.SerializeObject(jsonRulesList, Formatting.Indented);
}
catch (Exception e)
{
this.Logger.LogError(e, "Unable to serialize the IEnumerable of {entity} data models for the temporary file content. OperationId: {operationId}. TenantId: {tenantId}", this.Entity, operationId, tenantId);
throw e;
}
string blobFilePath = await this.WriteFileContentToBlobAsync(fileContent, tenantId, operationId);
ConversionApiModel conversionResponse = new ConversionApiModel
{
TenantId = tenantId,
BlobFilePath = blobFilePath,
Entities = rules,
OperationId = operationId,
};
this.Logger.LogInformation("Successfully Completed {entity} conversion\n{model}", this.Entity, JsonConvert.SerializeObject(conversionResponse));
return(conversionResponse);
}
/// <summary>
/// Update Cache when devices are modified/created
/// </summary>
public async Task <DevicePropertyServiceModel> UpdateListAsync(
DevicePropertyServiceModel deviceProperties)
{
// To simplify code, use empty set to replace null set
deviceProperties.Tags = deviceProperties.Tags ?? new HashSet <string>();
deviceProperties.Reported = deviceProperties.Reported ?? new HashSet <string>();
string etag = null;
while (true)
{
ValueListApiModel model = null;
try
{
model = await this.storageClient.GetAllAsync(CACHE_COLLECTION_ID);
}
catch (ResourceNotFoundException)
{
this.log.Info($"Cache updating: cache {CACHE_COLLECTION_ID}:{CACHE_KEY} was not found",
() => { });
}
if (model != null)
{
DevicePropertyServiceModel devicePropertiesFromStorage;
try
{
devicePropertiesFromStorage = JsonConvert.
DeserializeObject <DevicePropertyServiceModel>(model.Items.FirstOrDefault().Data);
}
catch
{
devicePropertiesFromStorage = new DevicePropertyServiceModel();
}
devicePropertiesFromStorage.Tags = devicePropertiesFromStorage.Tags ??
new HashSet <string>();
devicePropertiesFromStorage.Reported = devicePropertiesFromStorage.Reported ??
new HashSet <string>();
deviceProperties.Tags.UnionWith(devicePropertiesFromStorage.Tags);
deviceProperties.Reported.UnionWith(devicePropertiesFromStorage.Reported);
etag = model.Items.FirstOrDefault().ETag;
// If the new set of deviceProperties are already there in cache, return
if (deviceProperties.Tags.Count == devicePropertiesFromStorage.Tags.Count &&
deviceProperties.Reported.Count == devicePropertiesFromStorage.Reported.Count)
{
return(deviceProperties);
}
}
var value = JsonConvert.SerializeObject(deviceProperties);
try
{
//var checkcacheList = await this.storageClient.GetAllAsync(CACHE_COLLECTION_ID);
//var updatedata = checkcacheList.Items.FirstOrDefault();
var response = await this.storageClient.UpdateAsync(
CACHE_COLLECTION_ID, model.Items.FirstOrDefault().objectid, value, etag);
return(JsonConvert.DeserializeObject <DevicePropertyServiceModel>(response.Data));
}
catch (ConflictingResourceException)
{
this.log.Info("Cache updating: failed due to conflict. Retry soon", () => { });
}
catch (Exception e)
{
this.log.Info("Cache updating: failed", () => e);
throw new Exception("Cache updating: failed");
}
}
}
public override async Task <ConversionApiModel> ConvertAsync(string tenantId, string operationId = null)
{
ValueListApiModel deviceGroups = null;
try
{
deviceGroups = await this.StorageAdapterClient.GetAllAsync(this.Entity, tenantId);
}
catch (Exception e)
{
this.Logger.LogError(e, "Unable to query {entity} using storage adapter. OperationId: {operationId}. TenantId: {tenantId}", this.Entity, operationId, tenantId);
throw e;
}
if (deviceGroups.Items.Count() == 0 || deviceGroups == null)
{
string errorMessage = $"No entities were receieved from storage adapter to convert to {this.Entity}. OperationId: {operationId}. TenantId: {tenantId}";
this.Logger.LogError(new Exception(errorMessage), errorMessage);
throw new ResourceNotFoundException("No entities were receieved from storage adapter to convert to rules.");
}
DeviceGroupListModel deviceGroupModels = new DeviceGroupListModel();
try
{
List <DeviceGroupModel> items = new List <DeviceGroupModel>();
foreach (ValueApiModel group in deviceGroups.Items)
{
try
{
DeviceGroupDataModel dataModel = JsonConvert.DeserializeObject <DeviceGroupDataModel>(group.Data);
DeviceGroupModel individualModel = new DeviceGroupModel(group.Key, group.ETag, dataModel);
items.Add(individualModel);
}
catch (Exception)
{
this.Logger.LogInformation("Unable to convert a device group to the proper reference data model for {entity}. OperationId: {operationId}. TenantId: {tenantId}", this.Entity, operationId, tenantId);
}
}
if (items.Count() == 0)
{
throw new ResourceNotSupportedException("No device groups were able to be converted to the proper rule reference data model.");
}
deviceGroupModels.Items = items;
}
catch (Exception e)
{
this.Logger.LogError(e, "Unable to convert {entity} queried from storage adapter to appropriate data model. OperationId: {operationId}. TenantId: {tenantId}", this.Entity, operationId, tenantId);
throw e;
}
Dictionary <DeviceGroupModel, DeviceListModel> deviceMapping = new Dictionary <DeviceGroupModel, DeviceListModel>();
foreach (DeviceGroupModel deviceGroup in deviceGroupModels.Items)
{
try
{
DeviceListModel devicesList = await this.iotHubManager.GetListAsync(deviceGroup.Conditions, tenantId);
if (devicesList.Items.Count() > 0)
{
deviceMapping.Add(deviceGroup, devicesList);
}
}
catch (Exception e)
{
// Do not throw an exception here, attempt to query other device groups instead to get as much data as possible
// Log all device groups that could not be retreived
this.Logger.LogError(e, "Unable to get list of devices for devicegroup {deviceGroup} from IotHubManager. OperationId: {operationId}. TenantId: {tenantId}", deviceGroup.Id, operationId, tenantId);
}
}
if (deviceMapping.Count() == 0)
{
string groups = $"[{string.Join(", ", deviceGroupModels.Items.Select(group => group.Id))}]";
string errorMessage = $"No Devices were found for any {this.Entity}. OperationId: {operationId}. TenantId: {tenantId}\n{groups}";
this.Logger.LogError(new Exception(errorMessage), errorMessage);
throw new ResourceNotFoundException($"No Devices were found for any {this.Entity}.");
}
string fileContent = null;
try
{
// Write a file in csv format:
// deviceId,groupId
// mapping contains devices groups, and a list model of all devices within each device group
// create a new csv row for each device and device group combination
string fileContentRows = string.Join("\n", deviceMapping.Select(mapping =>
{
return(string.Join("\n", mapping.Value.Items.Select(device => $"{device.Id},{mapping.Key.Id}")));
}));
// Add the rows and the header together to complete the csv file content
fileContent = $"{CsvHeader}\n{fileContentRows}";
}
catch (Exception e)
{
this.Logger.LogError(e, "Unable to serialize the {entity} data models for the temporary file content. OperationId: {operationId}. TenantId: {tenantId}", this.Entity, operationId, tenantId);
throw e;
}
string blobFilePath = await this.WriteFileContentToBlobAsync(fileContent, tenantId, operationId);
ConversionApiModel conversionResponse = new ConversionApiModel
{
TenantId = tenantId,
BlobFilePath = blobFilePath,
Entities = deviceGroups,
OperationId = operationId,
};
this.Logger.LogInformation("Successfully Completed {entity} conversion\n{model}", this.Entity, JsonConvert.SerializeObject(conversionResponse));
return(conversionResponse);
}
