public async Task <JobServiceModel> ScheduleTwinUpdateAsync(
string jobId,
string queryCondition,
DeviceTwinServiceModel twin,
DateTimeOffset startTimeUtc,
long maxExecutionTimeInSeconds)
{
var result = await this.jobClient.ScheduleTwinUpdateAsync(
jobId,
queryCondition,
twin.ToAzureModel(),
startTimeUtc.DateTime,
maxExecutionTimeInSeconds);
// Update the deviceProperties cache, no need to wait
var model = new DevicePropertyServiceModel();
var tagRoot = JsonConvert.DeserializeObject(JsonConvert.SerializeObject(twin.Tags)) as JToken;
if (tagRoot != null)
{
model.Tags = new HashSet <string>(tagRoot.GetAllLeavesPath());
}
var reportedRoot = JsonConvert.DeserializeObject(JsonConvert.SerializeObject(twin.ReportedProperties)) as JToken;
if (reportedRoot != null)
{
model.Reported = new HashSet <string>(reportedRoot.GetAllLeavesPath());
}
var unused = deviceProperties.UpdateListAsync(model);
return(new JobServiceModel(result));
}
public async Task UpdateListAsyncTest()
{
var mockStorageAdapterClient = new Mock <IStorageAdapterClient>();
var mockDevices = new Mock <IDevices>();
var cache = new DeviceProperties(
mockStorageAdapterClient.Object,
new ServicesConfig(),
new Logger("UnitTest", LogLevel.Debug),
mockDevices.Object);
var oldCacheValue = new DevicePropertyServiceModel
{
Rebuilding = false,
Tags = new HashSet <string> {
"c", "a", "y", "z"
},
Reported = new HashSet <string> {
"1", "9", "2", "3"
}
};
var cachePatch = new DevicePropertyServiceModel
{
Tags = new HashSet <string> {
"a", "y", "z", "@", "#"
},
Reported = new HashSet <string> {
"9", "2", "3", "11", "12"
}
};
var newCacheValue = new DevicePropertyServiceModel
{
Tags = new HashSet <string> {
"c", "a", "y", "z", "@", "#"
},
Reported = new HashSet <string> {
"1", "9", "2", "3", "12", "11"
}
};
mockStorageAdapterClient
.Setup(m => m.GetAsync(It.IsAny <string>(), It.IsAny <string>()))
.Returns(() => Task.FromResult(new ValueApiModel {
Data = JsonConvert.SerializeObject(oldCacheValue)
}));
mockStorageAdapterClient
.Setup(m => m.UpdateAsync(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>()))
.Returns(() => Task.FromResult(new ValueApiModel {
Data = JsonConvert.SerializeObject(newCacheValue)
}));
var result = await cache.UpdateListAsync(cachePatch);
Assert.True(result.Tags.SetEquals(newCacheValue.Tags));
Assert.True(result.Reported.SetEquals(newCacheValue.Reported));
}
public async Task <DevicePropertyServiceModel> UpdateListAsync(DevicePropertyServiceModel deviceProperties)
{
// To simplify code, use empty set to replace null set
deviceProperties.Tags = deviceProperties.Tags ?? new HashSet <string>();
deviceProperties.Reported = deviceProperties.Reported ?? new HashSet <string>();
string etag = null;
while (true)
{
ValueApiModel model = null;
try
{
model = await this.storageClient.GetAsync(CACHE_COLLECTION_ID, CACHE_KEY);
}
catch (ResourceNotFoundException)
{
this.log.Info($"Cache updating: cache {CACHE_COLLECTION_ID}:{CACHE_KEY} was not found", () => { });
}
if (model != null)
{
DevicePropertyServiceModel devicePropertiesFromStorage;
try
{
devicePropertiesFromStorage = JsonConvert.DeserializeObject <DevicePropertyServiceModel>(model.Data);
}
catch
{
devicePropertiesFromStorage = new DevicePropertyServiceModel();
}
devicePropertiesFromStorage.Tags = devicePropertiesFromStorage.Tags ?? new HashSet <string>();
devicePropertiesFromStorage.Reported = devicePropertiesFromStorage.Reported ?? new HashSet <string>();
deviceProperties.Tags.UnionWith(devicePropertiesFromStorage.Tags);
deviceProperties.Reported.UnionWith(devicePropertiesFromStorage.Reported);
etag = model.ETag;
if (deviceProperties.Tags.Count == devicePropertiesFromStorage.Tags.Count && deviceProperties.Reported.Count == devicePropertiesFromStorage.Reported.Count)
{
return(deviceProperties);
}
}
var value = JsonConvert.SerializeObject(deviceProperties);
try
{
var response = await this.storageClient.UpdateAsync(CACHE_COLLECTION_ID, CACHE_KEY, value, etag);
return(JsonConvert.DeserializeObject <DevicePropertyServiceModel>(response.Data));
}
catch (ConflictingResourceException)
{
this.log.Info("Cache updating: failed due to conflict. Retry soon", () => { });
}
}
}
public async Task <List <string> > GetListAsync()
{
ValueApiModel response = new ValueApiModel();
try
{
response = await this.storageClient.GetAsync(CacheCollectioId, CacheKey);
}
catch (ResourceNotFoundException)
{
this.logger.LogDebug($"Cache get: cache {CacheCollectioId}:{CacheKey} was not found");
}
catch (Exception e)
{
throw new ExternalDependencyException(
$"Cache get: unable to get device-twin-properties cache", e);
}
if (string.IsNullOrEmpty(response?.Data))
{
throw new Exception($"StorageAdapter did not return any data for {CacheCollectioId}:{CacheKey}. The DeviceProperties cache has not been created for this tenant yet.");
}
DevicePropertyServiceModel properties = new DevicePropertyServiceModel();
try
{
properties = JsonConvert.DeserializeObject <DevicePropertyServiceModel>(response.Data);
}
catch (Exception e)
{
throw new InvalidInputException("Unable to deserialize deviceProperties from CosmosDB", e);
}
List <string> result = new List <string>();
foreach (string tag in properties.Tags)
{
result.Add(TagPrefix + tag);
}
foreach (string reported in properties.Reported)
{
result.Add(ReportedPrefix + reported);
}
return(result);
}
public async Task GetListAsyncTest()
{
var mockStorageAdapterClient = new Mock <IStorageAdapterClient>();
var mockDevices = new Mock <IDevices>();
var cache = new DeviceProperties(
mockStorageAdapterClient.Object,
new Mock <AppConfig> {
DefaultValue = DefaultValue.Mock
}.Object,
new Mock <ILogger <DeviceProperties> >().Object,
mockDevices.Object);
var cacheValue = new DevicePropertyServiceModel
{
Rebuilding = false,
Tags = new HashSet <string> {
"ccc", "aaaa", "yyyy", "zzzz"
},
Reported = new HashSet <string> {
"1111", "9999", "2222", "3333"
},
};
mockStorageAdapterClient
.Setup(m => m.GetAsync(It.IsAny <string>(), It.IsAny <string>()))
.Returns(() => Task.FromResult(new ValueApiModel {
Data = JsonConvert.SerializeObject(cacheValue)
}));
var result = await cache.GetListAsync();
Assert.Equal(result.Count, cacheValue.Tags.Count + cacheValue.Reported.Count);
foreach (string tag in cacheValue.Tags)
{
Assert.Contains(result, s => s.Contains(tag));
}
foreach (string reported in cacheValue.Reported)
{
Assert.Contains(result, s => s.Contains(reported));
}
}
/// <summary>
/// Get List of deviceProperties from cache
/// </summary>
public async Task <List <string> > GetListAsync()
{
ValueApiModel response = new ValueApiModel();
try
{
response = await this.storageClient.GetAsync(CACHE_COLLECTION_ID, CACHE_KEY);
}
catch (ResourceNotFoundException)
{
this.log.Debug($"Cache get: cache {CACHE_COLLECTION_ID}:{CACHE_KEY} was not found",
() => { });
}
catch (Exception e)
{
throw new ExternalDependencyException(
$"Cache get: unable to get device-twin-properties cache", e);
}
DevicePropertyServiceModel properties = new DevicePropertyServiceModel();
try
{
properties = JsonConvert.DeserializeObject <DevicePropertyServiceModel>(response.Data);
}
catch (Exception e)
{
throw new InvalidInputException("Unable to deserialize deviceProperties from CosmosDB", e);
}
List <string> result = new List <string>();
foreach (string tag in properties.Tags)
{
result.Add(TAG_PREFIX + tag);
}
foreach (string reported in properties.Reported)
{
result.Add(REPORTED_PREFIX + reported);
}
return(result);
}
public async Task <DeviceServiceModel> UpdateAsync(DeviceServiceModel device, DevicePropertyDelegate devicePropertyDelegate)
{
// validate device module
var azureDevice = await this.tenantConnectionHelper.GetRegistry().GetDeviceAsync(device.Id);
if (azureDevice == null)
{
throw new ResourceNotFoundException($"Device {device.Id} could not be found on this tenant's IoT Hub. You must create the device first before calling the update method.");
}
Twin azureTwin;
if (device.Twin == null)
{
azureTwin = await this.tenantConnectionHelper.GetRegistry().GetTwinAsync(device.Id);
}
else
{
azureTwin = await this.tenantConnectionHelper.GetRegistry().UpdateTwinAsync(device.Id, device.Twin.ToAzureModel(), device.Twin.ETag);
// Update the deviceGroupFilter cache, no need to wait
var model = new DevicePropertyServiceModel();
if (JsonConvert.DeserializeObject(JsonConvert.SerializeObject(device.Twin.Tags)) is JToken tagRoot)
{
model.Tags = new HashSet <string>(tagRoot.GetAllLeavesPath());
}
if (JsonConvert.DeserializeObject(JsonConvert.SerializeObject(device.Twin.ReportedProperties)) is JToken reportedRoot)
{
model.Reported = new HashSet <string>(reportedRoot.GetAllLeavesPath());
}
_ = devicePropertyDelegate(model);
}
await this.asaManager.BeginDeviceGroupsConversionAsync();
return(new DeviceServiceModel(azureDevice, azureTwin, this.tenantConnectionHelper.GetIotHubName()));
}
public async Task <DeviceServiceModel> CreateOrUpdateAsync(DeviceServiceModel device, DevicePropertyDelegate devicePropertyDelegate)
{
// validate device module
var azureDevice = await this.tenantConnectionHelper.GetRegistry().GetDeviceAsync(device.Id);
if (azureDevice == null)
{
azureDevice = await this.tenantConnectionHelper.GetRegistry().AddDeviceAsync(device.ToAzureModel());
}
Twin azureTwin;
if (device.Twin == null)
{
azureTwin = await this.tenantConnectionHelper.GetRegistry().GetTwinAsync(device.Id);
}
else
{
azureTwin = await this.tenantConnectionHelper.GetRegistry().UpdateTwinAsync(device.Id, device.Twin.ToAzureModel(), device.Twin.ETag);
// Update the deviceGroupFilter cache, no need to wait
var model = new DevicePropertyServiceModel();
var tagRoot = JsonConvert.DeserializeObject(JsonConvert.SerializeObject(device.Twin.Tags)) as JToken;
if (tagRoot != null)
{
model.Tags = new HashSet <string>(tagRoot.GetAllLeavesPath());
}
var reportedRoot = JsonConvert.DeserializeObject(JsonConvert.SerializeObject(device.Twin.ReportedProperties)) as JToken;
if (reportedRoot != null)
{
model.Reported = new HashSet <string>(reportedRoot.GetAllLeavesPath());
}
var unused = devicePropertyDelegate(model);
}
return(new DeviceServiceModel(azureDevice, azureTwin, this.tenantConnectionHelper.GetIotHubName()));
}
/// <summary>
/// We only support update twin
/// </summary>
/// <param name="device"></param>
/// <param name="devicePropertyDelegate"></param>
/// <returns></returns>
public async Task <DeviceServiceModel> CreateOrUpdateAsync(DeviceServiceModel device, DevicePropertyDelegate devicePropertyDelegate)
{
// validate device module
Device azureDevice = await registry.GetDeviceAsync(device.Id);
if (azureDevice == null)
{
azureDevice = await registry.AddDeviceAsync(device.ToAzureModel());
}
Twin azureTwin;
if (device.Twin == null)
{
azureTwin = await registry.GetTwinAsync(device.Id);
}
else
{
azureTwin = await registry.UpdateTwinAsync(device.Id, device.Twin.ToAzureModel(), device.Twin.ETag);
// Update the deviceGroupFilter cache, no need to wait
DevicePropertyServiceModel model = new DevicePropertyServiceModel();
JToken tagRoot = JsonConvert.DeserializeObject(JsonConvert.SerializeObject(device.Twin.Tags)) as JToken;
if (tagRoot != null)
{
model.Tags = new HashSet <string>(tagRoot.GetAllLeavesPath());
}
JToken reportedRoot = JsonConvert.DeserializeObject(JsonConvert.SerializeObject(device.Twin.ReportedProperties)) as JToken;
if (reportedRoot != null)
{
model.Reported = new HashSet <string>(reportedRoot.GetAllLeavesPath());
}
Task <DevicePropertyServiceModel> unused = devicePropertyDelegate(model);
}
return(new DeviceServiceModel(azureDevice, azureTwin, ioTHubHostName));
}
/// <summary>
/// A function to decide whether or not cache needs to be rebuilt based on force flag and existing
/// cache's validity
/// </summary>
/// <param name="force">A boolean flag to decide if cache needs to be rebuilt.</param>
/// <param name="valueApiModel">An existing valueApiModel to check whether or not cache
/// has expired</param>
private bool ShouldCacheRebuild(bool force, ValueApiModel valueApiModel)
{
if (force)
{
this.log.Info("Cache will be rebuilt due to the force flag", () => { });
return(true);
}
if (valueApiModel == null)
{
this.log.Info("Cache will be rebuilt since no cache was found", () => { });
return(true);
}
DevicePropertyServiceModel cacheValue = new DevicePropertyServiceModel();
DateTimeOffset timstamp = new DateTimeOffset();
try
{
cacheValue = JsonConvert.DeserializeObject <DevicePropertyServiceModel>(valueApiModel.Data);
timstamp = DateTimeOffset.Parse(valueApiModel.Metadata["$modified"]);
}
catch
{
this.log.Info("DeviceProperties will be rebuilt because the last one is broken.", () => { });
return(true);
}
if (cacheValue.Rebuilding)
{
if (timstamp.AddSeconds(this.rebuildTimeout) < DateTimeOffset.UtcNow)
{
this.log.Debug("Cache will be rebuilt because last rebuilding had timedout", () => { });
return(true);
}
else
{
this.log.Debug
("Cache rebuilding skipped because it is being rebuilt by other instance", () => { });
return(false);
}
}
else
{
if (cacheValue.IsNullOrEmpty())
{
this.log.Info("Cache will be rebuilt since it is empty", () => { });
return(true);
}
if (timstamp.AddSeconds(this.ttl) < DateTimeOffset.UtcNow)
{
this.log.Info("Cache will be rebuilt because it has expired", () => { });
return(true);
}
else
{
this.log.Debug("Cache rebuilding skipped because it has not expired", () => { });
return(false);
}
}
}
/// <summary>
/// Update Cache when devices are modified/created
/// </summary>
public async Task <DevicePropertyServiceModel> UpdateListAsync(
DevicePropertyServiceModel deviceProperties)
{
// To simplify code, use empty set to replace null set
deviceProperties.Tags = deviceProperties.Tags ?? new HashSet <string>();
deviceProperties.Reported = deviceProperties.Reported ?? new HashSet <string>();
string etag = null;
while (true)
{
ValueListApiModel model = null;
try
{
model = await this.storageClient.GetAllAsync(CACHE_COLLECTION_ID);
}
catch (ResourceNotFoundException)
{
this.log.Info($"Cache updating: cache {CACHE_COLLECTION_ID}:{CACHE_KEY} was not found",
() => { });
}
if (model != null)
{
DevicePropertyServiceModel devicePropertiesFromStorage;
try
{
devicePropertiesFromStorage = JsonConvert.
DeserializeObject <DevicePropertyServiceModel>(model.Items.FirstOrDefault().Data);
}
catch
{
devicePropertiesFromStorage = new DevicePropertyServiceModel();
}
devicePropertiesFromStorage.Tags = devicePropertiesFromStorage.Tags ??
new HashSet <string>();
devicePropertiesFromStorage.Reported = devicePropertiesFromStorage.Reported ??
new HashSet <string>();
deviceProperties.Tags.UnionWith(devicePropertiesFromStorage.Tags);
deviceProperties.Reported.UnionWith(devicePropertiesFromStorage.Reported);
etag = model.Items.FirstOrDefault().ETag;
// If the new set of deviceProperties are already there in cache, return
if (deviceProperties.Tags.Count == devicePropertiesFromStorage.Tags.Count &&
deviceProperties.Reported.Count == devicePropertiesFromStorage.Reported.Count)
{
return(deviceProperties);
}
}
var value = JsonConvert.SerializeObject(deviceProperties);
try
{
//var checkcacheList = await this.storageClient.GetAllAsync(CACHE_COLLECTION_ID);
//var updatedata = checkcacheList.Items.FirstOrDefault();
var response = await this.storageClient.UpdateAsync(
CACHE_COLLECTION_ID, model.Items.FirstOrDefault().objectid, value, etag);
return(JsonConvert.DeserializeObject <DevicePropertyServiceModel>(response.Data));
}
catch (ConflictingResourceException)
{
this.log.Info("Cache updating: failed due to conflict. Retry soon", () => { });
}
catch (Exception e)
{
this.log.Info("Cache updating: failed", () => e);
throw new Exception("Cache updating: failed");
}
}
}
/// <summary>
/// Try to create cache of deviceProperties if lock failed retry after 10 seconds
/// </summary>
public async Task <bool> TryRecreateListAsync(bool force = false)
{
var @lock = new StorageWriteLock <DevicePropertyServiceModel>(
this.storageClient,
CACHE_COLLECTION_ID,
CACHE_KEY,
(c, b) => c.Rebuilding = b,
m => this.ShouldCacheRebuild(force, m));
while (true)
{
//var locked = await @lock.TryLockAsync();
//if (locked == null)
//{
// this.log.Warn("Cache rebuilding: lock failed due to conflict. Retry soon", () => { });
// continue;
//}
//if (!locked.Value)
//{
// return false;
//}
// Build the cache content
var twinNamesTask = this.GetValidNamesAsync();
try
{
Task.WaitAll(twinNamesTask);
}
catch (Exception)
{
this.log.Warn(
$"Some underlying service is not ready. Retry after {this.serviceQueryInterval}",
() => { });
try
{
//await @lock.ReleaseAsync();
}
catch (Exception e)
{
log.Error("Cache rebuilding: Unable to release lock", () => e);
}
await Task.Delay(this.serviceQueryInterval);
continue;
}
var twinNames = twinNamesTask.Result;
try
{
var model = await this.storageClient.GetAllAsync(CACHE_COLLECTION_ID);
foreach (var item in model.Items)
{
await this.storageClient.DeleteAsync(CACHE_COLLECTION_ID, item.objectid);
}
DevicePropertyServiceModel devicepropertites = new DevicePropertyServiceModel();
devicepropertites.Tags = twinNames.Tags;
devicepropertites.Reported = twinNames.ReportedProperties;
var value = JsonConvert.SerializeObject(devicepropertites, Formatting.Indented, new JsonSerializerSettings {
NullValueHandling = NullValueHandling.Ignore
});
var Createdevicepropertites = await this.storageClient.CreateAsync(CACHE_COLLECTION_ID, value);
//var updated = await @lock.WriteAndReleaseAsync(
// new DevicePropertyServiceModel
// {
// Tags = twinNames.Tags,
// Reported = twinNames.ReportedProperties
// });
if (Createdevicepropertites.objectid != null)
{
this.DevicePropertiesLastUpdated = DateTime.Now;
return(true);
}
}
catch (Exception e)
{
log.Error("Cache rebuilding: Unable to write and release lock", () => e);
}
this.log.Warn("Cache rebuilding: write failed due to conflict. Retry soon", () => { });
}
}
public async Task <JobServiceModel> ScheduleTwinUpdateAsync(
string jobId,
string queryCondition,
TwinServiceModel twin,
DateTimeOffset startTimeUtc,
long maxExecutionTimeInSeconds)
{
//var result = await this.jobClient.ScheduleTwinUpdateAsync(
// jobId,
// queryCondition,
// twin.ToAzureModel(),
// startTimeUtc.DateTime,
// maxExecutionTimeInSeconds);
var devicelistString = queryCondition.Replace("deviceId in", "").Trim();
var devicelist = JsonConvert.DeserializeObject <List <dynamic> >(devicelistString);
List <DeviceJobServiceModel> devicemodellist = new List <DeviceJobServiceModel>();
foreach (var item in devicelist)
{
DeviceJobServiceModel data = new DeviceJobServiceModel();
data.DeviceId = item;
data.Status = DeviceJobStatus.Scheduled;
data.CreatedDateTimeUtc = DateTime.UtcNow;
devicemodellist.Add(data);
}
var devicecount = devicemodellist.Count();
JobServiceModel json = new JobServiceModel();
json.CreatedTimeUtc = DateTime.UtcNow;
json.Devices = devicemodellist.ToList();
json.Status = JobStatus.Scheduled;
json.UpdateTwin = twin;
json.Type = JobType.ScheduleUpdateTwin;
JobStatistics ResultStatistics = new JobStatistics();
ResultStatistics.DeviceCount = devicecount;
ResultStatistics.SucceededCount = 0;
ResultStatistics.FailedCount = 0;
ResultStatistics.PendingCount = 0;
ResultStatistics.RunningCount = 0;
json.ResultStatistics = ResultStatistics;
var value = JsonConvert.SerializeObject(json, Formatting.Indented, new JsonSerializerSettings {
NullValueHandling = NullValueHandling.Ignore
});
var result = await this.client.CreateAsync(DEVICE_JOBS_COLLECTION_ID, value);
var Job = this.CreatejobServiceModel(result);
// Update the deviceProperties cache, no need to wait
var model = new DevicePropertyServiceModel();
var tagRoot = JsonConvert.DeserializeObject(JsonConvert.SerializeObject(twin.Tags)) as JToken;
if (tagRoot != null)
{
model.Tags = new HashSet <string>(tagRoot.GetAllLeavesPath());
}
var reportedRoot = JsonConvert.DeserializeObject(JsonConvert.SerializeObject(twin.ReportedProperties)) as JToken;
if (reportedRoot != null)
{
model.Reported = new HashSet <string>(reportedRoot.GetAllLeavesPath());
}
var unused = deviceProperties.UpdateListAsync(model);
return(Job);
}
public async Task <DevicePropertyServiceModel> UpdateListAsync(
DevicePropertyServiceModel deviceProperties)
{
// To simplify code, use empty set to replace null set
deviceProperties.Tags = deviceProperties.Tags ?? new HashSet <string>();
deviceProperties.Reported = deviceProperties.Reported ?? new HashSet <string>();
string etag = null;
while (true)
{
ValueApiModel model = null;
try
{
model = await this.storageClient.GetAsync(CacheCollectioId, CacheKey);
}
catch (ResourceNotFoundException)
{
this.logger.LogInformation($"Cache updating: cache {CacheCollectioId}:{CacheKey} was not found");
}
if (model != null)
{
DevicePropertyServiceModel devicePropertiesFromStorage;
try
{
devicePropertiesFromStorage = JsonConvert.
DeserializeObject <DevicePropertyServiceModel>(model.Data);
}
catch
{
devicePropertiesFromStorage = new DevicePropertyServiceModel();
}
devicePropertiesFromStorage.Tags = devicePropertiesFromStorage.Tags ??
new HashSet <string>();
devicePropertiesFromStorage.Reported = devicePropertiesFromStorage.Reported ??
new HashSet <string>();
deviceProperties.Tags.UnionWith(devicePropertiesFromStorage.Tags);
deviceProperties.Reported.UnionWith(devicePropertiesFromStorage.Reported);
etag = model.ETag;
// If the new set of deviceProperties are already there in cache, return
if (deviceProperties.Tags.Count == devicePropertiesFromStorage.Tags.Count &&
deviceProperties.Reported.Count == devicePropertiesFromStorage.Reported.Count)
{
return(deviceProperties);
}
}
var value = JsonConvert.SerializeObject(deviceProperties);
try
{
var response = await this.storageClient.UpdateAsync(
CacheCollectioId, CacheKey, value, etag);
return(JsonConvert.DeserializeObject <DevicePropertyServiceModel>(response.Data));
}
catch (ConflictingResourceException)
{
this.logger.LogInformation("Cache updating: failed due to conflict. Retry soon");
}
catch (Exception e)
{
this.logger.LogInformation(e, "Cache updating: failed");
throw new Exception("Cache updating: failed");
}
}
}
