public async Task <JobServiceModel> ScheduleTwinUpdateAsync( string jobId, string queryCondition, DeviceTwinServiceModel twin, DateTimeOffset startTimeUtc, long maxExecutionTimeInSeconds) { var result = await this.jobClient.ScheduleTwinUpdateAsync( jobId, queryCondition, twin.ToAzureModel(), startTimeUtc.DateTime, maxExecutionTimeInSeconds); // Update the deviceProperties cache, no need to wait var model = new DevicePropertyServiceModel(); var tagRoot = JsonConvert.DeserializeObject(JsonConvert.SerializeObject(twin.Tags)) as JToken; if (tagRoot != null) { model.Tags = new HashSet <string>(tagRoot.GetAllLeavesPath()); } var reportedRoot = JsonConvert.DeserializeObject(JsonConvert.SerializeObject(twin.ReportedProperties)) as JToken; if (reportedRoot != null) { model.Reported = new HashSet <string>(reportedRoot.GetAllLeavesPath()); } var unused = deviceProperties.UpdateListAsync(model); return(new JobServiceModel(result)); }
public async Task UpdateListAsyncTest() { var mockStorageAdapterClient = new Mock <IStorageAdapterClient>(); var mockDevices = new Mock <IDevices>(); var cache = new DeviceProperties( mockStorageAdapterClient.Object, new ServicesConfig(), new Logger("UnitTest", LogLevel.Debug), mockDevices.Object); var oldCacheValue = new DevicePropertyServiceModel { Rebuilding = false, Tags = new HashSet <string> { "c", "a", "y", "z" }, Reported = new HashSet <string> { "1", "9", "2", "3" } }; var cachePatch = new DevicePropertyServiceModel { Tags = new HashSet <string> { "a", "y", "z", "@", "#" }, Reported = new HashSet <string> { "9", "2", "3", "11", "12" } }; var newCacheValue = new DevicePropertyServiceModel { Tags = new HashSet <string> { "c", "a", "y", "z", "@", "#" }, Reported = new HashSet <string> { "1", "9", "2", "3", "12", "11" } }; mockStorageAdapterClient .Setup(m => m.GetAsync(It.IsAny <string>(), It.IsAny <string>())) .Returns(() => Task.FromResult(new ValueApiModel { Data = JsonConvert.SerializeObject(oldCacheValue) })); mockStorageAdapterClient .Setup(m => m.UpdateAsync(It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <string>())) .Returns(() => Task.FromResult(new ValueApiModel { Data = JsonConvert.SerializeObject(newCacheValue) })); var result = await cache.UpdateListAsync(cachePatch); Assert.True(result.Tags.SetEquals(newCacheValue.Tags)); Assert.True(result.Reported.SetEquals(newCacheValue.Reported)); }
public async Task <DevicePropertyServiceModel> UpdateListAsync(DevicePropertyServiceModel deviceProperties) { // To simplify code, use empty set to replace null set deviceProperties.Tags = deviceProperties.Tags ?? new HashSet <string>(); deviceProperties.Reported = deviceProperties.Reported ?? new HashSet <string>(); string etag = null; while (true) { ValueApiModel model = null; try { model = await this.storageClient.GetAsync(CACHE_COLLECTION_ID, CACHE_KEY); } catch (ResourceNotFoundException) { this.log.Info($"Cache updating: cache {CACHE_COLLECTION_ID}:{CACHE_KEY} was not found", () => { }); } if (model != null) { DevicePropertyServiceModel devicePropertiesFromStorage; try { devicePropertiesFromStorage = JsonConvert.DeserializeObject <DevicePropertyServiceModel>(model.Data); } catch { devicePropertiesFromStorage = new DevicePropertyServiceModel(); } devicePropertiesFromStorage.Tags = devicePropertiesFromStorage.Tags ?? new HashSet <string>(); devicePropertiesFromStorage.Reported = devicePropertiesFromStorage.Reported ?? new HashSet <string>(); deviceProperties.Tags.UnionWith(devicePropertiesFromStorage.Tags); deviceProperties.Reported.UnionWith(devicePropertiesFromStorage.Reported); etag = model.ETag; if (deviceProperties.Tags.Count == devicePropertiesFromStorage.Tags.Count && deviceProperties.Reported.Count == devicePropertiesFromStorage.Reported.Count) { return(deviceProperties); } } var value = JsonConvert.SerializeObject(deviceProperties); try { var response = await this.storageClient.UpdateAsync(CACHE_COLLECTION_ID, CACHE_KEY, value, etag); return(JsonConvert.DeserializeObject <DevicePropertyServiceModel>(response.Data)); } catch (ConflictingResourceException) { this.log.Info("Cache updating: failed due to conflict. Retry soon", () => { }); } } }
public async Task <List <string> > GetListAsync() { ValueApiModel response = new ValueApiModel(); try { response = await this.storageClient.GetAsync(CacheCollectioId, CacheKey); } catch (ResourceNotFoundException) { this.logger.LogDebug($"Cache get: cache {CacheCollectioId}:{CacheKey} was not found"); } catch (Exception e) { throw new ExternalDependencyException( $"Cache get: unable to get device-twin-properties cache", e); } if (string.IsNullOrEmpty(response?.Data)) { throw new Exception($"StorageAdapter did not return any data for {CacheCollectioId}:{CacheKey}. The DeviceProperties cache has not been created for this tenant yet."); } DevicePropertyServiceModel properties = new DevicePropertyServiceModel(); try { properties = JsonConvert.DeserializeObject <DevicePropertyServiceModel>(response.Data); } catch (Exception e) { throw new InvalidInputException("Unable to deserialize deviceProperties from CosmosDB", e); } List <string> result = new List <string>(); foreach (string tag in properties.Tags) { result.Add(TagPrefix + tag); } foreach (string reported in properties.Reported) { result.Add(ReportedPrefix + reported); } return(result); }
public async Task GetListAsyncTest() { var mockStorageAdapterClient = new Mock <IStorageAdapterClient>(); var mockDevices = new Mock <IDevices>(); var cache = new DeviceProperties( mockStorageAdapterClient.Object, new Mock <AppConfig> { DefaultValue = DefaultValue.Mock }.Object, new Mock <ILogger <DeviceProperties> >().Object, mockDevices.Object); var cacheValue = new DevicePropertyServiceModel { Rebuilding = false, Tags = new HashSet <string> { "ccc", "aaaa", "yyyy", "zzzz" }, Reported = new HashSet <string> { "1111", "9999", "2222", "3333" }, }; mockStorageAdapterClient .Setup(m => m.GetAsync(It.IsAny <string>(), It.IsAny <string>())) .Returns(() => Task.FromResult(new ValueApiModel { Data = JsonConvert.SerializeObject(cacheValue) })); var result = await cache.GetListAsync(); Assert.Equal(result.Count, cacheValue.Tags.Count + cacheValue.Reported.Count); foreach (string tag in cacheValue.Tags) { Assert.Contains(result, s => s.Contains(tag)); } foreach (string reported in cacheValue.Reported) { Assert.Contains(result, s => s.Contains(reported)); } }
/// <summary> /// Get List of deviceProperties from cache /// </summary> public async Task <List <string> > GetListAsync() { ValueApiModel response = new ValueApiModel(); try { response = await this.storageClient.GetAsync(CACHE_COLLECTION_ID, CACHE_KEY); } catch (ResourceNotFoundException) { this.log.Debug($"Cache get: cache {CACHE_COLLECTION_ID}:{CACHE_KEY} was not found", () => { }); } catch (Exception e) { throw new ExternalDependencyException( $"Cache get: unable to get device-twin-properties cache", e); } DevicePropertyServiceModel properties = new DevicePropertyServiceModel(); try { properties = JsonConvert.DeserializeObject <DevicePropertyServiceModel>(response.Data); } catch (Exception e) { throw new InvalidInputException("Unable to deserialize deviceProperties from CosmosDB", e); } List <string> result = new List <string>(); foreach (string tag in properties.Tags) { result.Add(TAG_PREFIX + tag); } foreach (string reported in properties.Reported) { result.Add(REPORTED_PREFIX + reported); } return(result); }
public async Task <DeviceServiceModel> UpdateAsync(DeviceServiceModel device, DevicePropertyDelegate devicePropertyDelegate) { // validate device module var azureDevice = await this.tenantConnectionHelper.GetRegistry().GetDeviceAsync(device.Id); if (azureDevice == null) { throw new ResourceNotFoundException($"Device {device.Id} could not be found on this tenant's IoT Hub. You must create the device first before calling the update method."); } Twin azureTwin; if (device.Twin == null) { azureTwin = await this.tenantConnectionHelper.GetRegistry().GetTwinAsync(device.Id); } else { azureTwin = await this.tenantConnectionHelper.GetRegistry().UpdateTwinAsync(device.Id, device.Twin.ToAzureModel(), device.Twin.ETag); // Update the deviceGroupFilter cache, no need to wait var model = new DevicePropertyServiceModel(); if (JsonConvert.DeserializeObject(JsonConvert.SerializeObject(device.Twin.Tags)) is JToken tagRoot) { model.Tags = new HashSet <string>(tagRoot.GetAllLeavesPath()); } if (JsonConvert.DeserializeObject(JsonConvert.SerializeObject(device.Twin.ReportedProperties)) is JToken reportedRoot) { model.Reported = new HashSet <string>(reportedRoot.GetAllLeavesPath()); } _ = devicePropertyDelegate(model); } await this.asaManager.BeginDeviceGroupsConversionAsync(); return(new DeviceServiceModel(azureDevice, azureTwin, this.tenantConnectionHelper.GetIotHubName())); }
public async Task <DeviceServiceModel> CreateOrUpdateAsync(DeviceServiceModel device, DevicePropertyDelegate devicePropertyDelegate) { // validate device module var azureDevice = await this.tenantConnectionHelper.GetRegistry().GetDeviceAsync(device.Id); if (azureDevice == null) { azureDevice = await this.tenantConnectionHelper.GetRegistry().AddDeviceAsync(device.ToAzureModel()); } Twin azureTwin; if (device.Twin == null) { azureTwin = await this.tenantConnectionHelper.GetRegistry().GetTwinAsync(device.Id); } else { azureTwin = await this.tenantConnectionHelper.GetRegistry().UpdateTwinAsync(device.Id, device.Twin.ToAzureModel(), device.Twin.ETag); // Update the deviceGroupFilter cache, no need to wait var model = new DevicePropertyServiceModel(); var tagRoot = JsonConvert.DeserializeObject(JsonConvert.SerializeObject(device.Twin.Tags)) as JToken; if (tagRoot != null) { model.Tags = new HashSet <string>(tagRoot.GetAllLeavesPath()); } var reportedRoot = JsonConvert.DeserializeObject(JsonConvert.SerializeObject(device.Twin.ReportedProperties)) as JToken; if (reportedRoot != null) { model.Reported = new HashSet <string>(reportedRoot.GetAllLeavesPath()); } var unused = devicePropertyDelegate(model); } return(new DeviceServiceModel(azureDevice, azureTwin, this.tenantConnectionHelper.GetIotHubName())); }
/// <summary> /// We only support update twin /// </summary> /// <param name="device"></param> /// <param name="devicePropertyDelegate"></param> /// <returns></returns> public async Task <DeviceServiceModel> CreateOrUpdateAsync(DeviceServiceModel device, DevicePropertyDelegate devicePropertyDelegate) { // validate device module Device azureDevice = await registry.GetDeviceAsync(device.Id); if (azureDevice == null) { azureDevice = await registry.AddDeviceAsync(device.ToAzureModel()); } Twin azureTwin; if (device.Twin == null) { azureTwin = await registry.GetTwinAsync(device.Id); } else { azureTwin = await registry.UpdateTwinAsync(device.Id, device.Twin.ToAzureModel(), device.Twin.ETag); // Update the deviceGroupFilter cache, no need to wait DevicePropertyServiceModel model = new DevicePropertyServiceModel(); JToken tagRoot = JsonConvert.DeserializeObject(JsonConvert.SerializeObject(device.Twin.Tags)) as JToken; if (tagRoot != null) { model.Tags = new HashSet <string>(tagRoot.GetAllLeavesPath()); } JToken reportedRoot = JsonConvert.DeserializeObject(JsonConvert.SerializeObject(device.Twin.ReportedProperties)) as JToken; if (reportedRoot != null) { model.Reported = new HashSet <string>(reportedRoot.GetAllLeavesPath()); } Task <DevicePropertyServiceModel> unused = devicePropertyDelegate(model); } return(new DeviceServiceModel(azureDevice, azureTwin, ioTHubHostName)); }
/// <summary> /// A function to decide whether or not cache needs to be rebuilt based on force flag and existing /// cache's validity /// </summary> /// <param name="force">A boolean flag to decide if cache needs to be rebuilt.</param> /// <param name="valueApiModel">An existing valueApiModel to check whether or not cache /// has expired</param> private bool ShouldCacheRebuild(bool force, ValueApiModel valueApiModel) { if (force) { this.log.Info("Cache will be rebuilt due to the force flag", () => { }); return(true); } if (valueApiModel == null) { this.log.Info("Cache will be rebuilt since no cache was found", () => { }); return(true); } DevicePropertyServiceModel cacheValue = new DevicePropertyServiceModel(); DateTimeOffset timstamp = new DateTimeOffset(); try { cacheValue = JsonConvert.DeserializeObject <DevicePropertyServiceModel>(valueApiModel.Data); timstamp = DateTimeOffset.Parse(valueApiModel.Metadata["$modified"]); } catch { this.log.Info("DeviceProperties will be rebuilt because the last one is broken.", () => { }); return(true); } if (cacheValue.Rebuilding) { if (timstamp.AddSeconds(this.rebuildTimeout) < DateTimeOffset.UtcNow) { this.log.Debug("Cache will be rebuilt because last rebuilding had timedout", () => { }); return(true); } else { this.log.Debug ("Cache rebuilding skipped because it is being rebuilt by other instance", () => { }); return(false); } } else { if (cacheValue.IsNullOrEmpty()) { this.log.Info("Cache will be rebuilt since it is empty", () => { }); return(true); } if (timstamp.AddSeconds(this.ttl) < DateTimeOffset.UtcNow) { this.log.Info("Cache will be rebuilt because it has expired", () => { }); return(true); } else { this.log.Debug("Cache rebuilding skipped because it has not expired", () => { }); return(false); } } }
/// <summary> /// Update Cache when devices are modified/created /// </summary> public async Task <DevicePropertyServiceModel> UpdateListAsync( DevicePropertyServiceModel deviceProperties) { // To simplify code, use empty set to replace null set deviceProperties.Tags = deviceProperties.Tags ?? new HashSet <string>(); deviceProperties.Reported = deviceProperties.Reported ?? new HashSet <string>(); string etag = null; while (true) { ValueListApiModel model = null; try { model = await this.storageClient.GetAllAsync(CACHE_COLLECTION_ID); } catch (ResourceNotFoundException) { this.log.Info($"Cache updating: cache {CACHE_COLLECTION_ID}:{CACHE_KEY} was not found", () => { }); } if (model != null) { DevicePropertyServiceModel devicePropertiesFromStorage; try { devicePropertiesFromStorage = JsonConvert. DeserializeObject <DevicePropertyServiceModel>(model.Items.FirstOrDefault().Data); } catch { devicePropertiesFromStorage = new DevicePropertyServiceModel(); } devicePropertiesFromStorage.Tags = devicePropertiesFromStorage.Tags ?? new HashSet <string>(); devicePropertiesFromStorage.Reported = devicePropertiesFromStorage.Reported ?? new HashSet <string>(); deviceProperties.Tags.UnionWith(devicePropertiesFromStorage.Tags); deviceProperties.Reported.UnionWith(devicePropertiesFromStorage.Reported); etag = model.Items.FirstOrDefault().ETag; // If the new set of deviceProperties are already there in cache, return if (deviceProperties.Tags.Count == devicePropertiesFromStorage.Tags.Count && deviceProperties.Reported.Count == devicePropertiesFromStorage.Reported.Count) { return(deviceProperties); } } var value = JsonConvert.SerializeObject(deviceProperties); try { //var checkcacheList = await this.storageClient.GetAllAsync(CACHE_COLLECTION_ID); //var updatedata = checkcacheList.Items.FirstOrDefault(); var response = await this.storageClient.UpdateAsync( CACHE_COLLECTION_ID, model.Items.FirstOrDefault().objectid, value, etag); return(JsonConvert.DeserializeObject <DevicePropertyServiceModel>(response.Data)); } catch (ConflictingResourceException) { this.log.Info("Cache updating: failed due to conflict. Retry soon", () => { }); } catch (Exception e) { this.log.Info("Cache updating: failed", () => e); throw new Exception("Cache updating: failed"); } } }
/// <summary> /// Try to create cache of deviceProperties if lock failed retry after 10 seconds /// </summary> public async Task <bool> TryRecreateListAsync(bool force = false) { var @lock = new StorageWriteLock <DevicePropertyServiceModel>( this.storageClient, CACHE_COLLECTION_ID, CACHE_KEY, (c, b) => c.Rebuilding = b, m => this.ShouldCacheRebuild(force, m)); while (true) { //var locked = await @lock.TryLockAsync(); //if (locked == null) //{ // this.log.Warn("Cache rebuilding: lock failed due to conflict. Retry soon", () => { }); // continue; //} //if (!locked.Value) //{ // return false; //} // Build the cache content var twinNamesTask = this.GetValidNamesAsync(); try { Task.WaitAll(twinNamesTask); } catch (Exception) { this.log.Warn( $"Some underlying service is not ready. Retry after {this.serviceQueryInterval}", () => { }); try { //await @lock.ReleaseAsync(); } catch (Exception e) { log.Error("Cache rebuilding: Unable to release lock", () => e); } await Task.Delay(this.serviceQueryInterval); continue; } var twinNames = twinNamesTask.Result; try { var model = await this.storageClient.GetAllAsync(CACHE_COLLECTION_ID); foreach (var item in model.Items) { await this.storageClient.DeleteAsync(CACHE_COLLECTION_ID, item.objectid); } DevicePropertyServiceModel devicepropertites = new DevicePropertyServiceModel(); devicepropertites.Tags = twinNames.Tags; devicepropertites.Reported = twinNames.ReportedProperties; var value = JsonConvert.SerializeObject(devicepropertites, Formatting.Indented, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); var Createdevicepropertites = await this.storageClient.CreateAsync(CACHE_COLLECTION_ID, value); //var updated = await @lock.WriteAndReleaseAsync( // new DevicePropertyServiceModel // { // Tags = twinNames.Tags, // Reported = twinNames.ReportedProperties // }); if (Createdevicepropertites.objectid != null) { this.DevicePropertiesLastUpdated = DateTime.Now; return(true); } } catch (Exception e) { log.Error("Cache rebuilding: Unable to write and release lock", () => e); } this.log.Warn("Cache rebuilding: write failed due to conflict. Retry soon", () => { }); } }
public async Task <JobServiceModel> ScheduleTwinUpdateAsync( string jobId, string queryCondition, TwinServiceModel twin, DateTimeOffset startTimeUtc, long maxExecutionTimeInSeconds) { //var result = await this.jobClient.ScheduleTwinUpdateAsync( // jobId, // queryCondition, // twin.ToAzureModel(), // startTimeUtc.DateTime, // maxExecutionTimeInSeconds); var devicelistString = queryCondition.Replace("deviceId in", "").Trim(); var devicelist = JsonConvert.DeserializeObject <List <dynamic> >(devicelistString); List <DeviceJobServiceModel> devicemodellist = new List <DeviceJobServiceModel>(); foreach (var item in devicelist) { DeviceJobServiceModel data = new DeviceJobServiceModel(); data.DeviceId = item; data.Status = DeviceJobStatus.Scheduled; data.CreatedDateTimeUtc = DateTime.UtcNow; devicemodellist.Add(data); } var devicecount = devicemodellist.Count(); JobServiceModel json = new JobServiceModel(); json.CreatedTimeUtc = DateTime.UtcNow; json.Devices = devicemodellist.ToList(); json.Status = JobStatus.Scheduled; json.UpdateTwin = twin; json.Type = JobType.ScheduleUpdateTwin; JobStatistics ResultStatistics = new JobStatistics(); ResultStatistics.DeviceCount = devicecount; ResultStatistics.SucceededCount = 0; ResultStatistics.FailedCount = 0; ResultStatistics.PendingCount = 0; ResultStatistics.RunningCount = 0; json.ResultStatistics = ResultStatistics; var value = JsonConvert.SerializeObject(json, Formatting.Indented, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); var result = await this.client.CreateAsync(DEVICE_JOBS_COLLECTION_ID, value); var Job = this.CreatejobServiceModel(result); // Update the deviceProperties cache, no need to wait var model = new DevicePropertyServiceModel(); var tagRoot = JsonConvert.DeserializeObject(JsonConvert.SerializeObject(twin.Tags)) as JToken; if (tagRoot != null) { model.Tags = new HashSet <string>(tagRoot.GetAllLeavesPath()); } var reportedRoot = JsonConvert.DeserializeObject(JsonConvert.SerializeObject(twin.ReportedProperties)) as JToken; if (reportedRoot != null) { model.Reported = new HashSet <string>(reportedRoot.GetAllLeavesPath()); } var unused = deviceProperties.UpdateListAsync(model); return(Job); }
public async Task <DevicePropertyServiceModel> UpdateListAsync( DevicePropertyServiceModel deviceProperties) { // To simplify code, use empty set to replace null set deviceProperties.Tags = deviceProperties.Tags ?? new HashSet <string>(); deviceProperties.Reported = deviceProperties.Reported ?? new HashSet <string>(); string etag = null; while (true) { ValueApiModel model = null; try { model = await this.storageClient.GetAsync(CacheCollectioId, CacheKey); } catch (ResourceNotFoundException) { this.logger.LogInformation($"Cache updating: cache {CacheCollectioId}:{CacheKey} was not found"); } if (model != null) { DevicePropertyServiceModel devicePropertiesFromStorage; try { devicePropertiesFromStorage = JsonConvert. DeserializeObject <DevicePropertyServiceModel>(model.Data); } catch { devicePropertiesFromStorage = new DevicePropertyServiceModel(); } devicePropertiesFromStorage.Tags = devicePropertiesFromStorage.Tags ?? new HashSet <string>(); devicePropertiesFromStorage.Reported = devicePropertiesFromStorage.Reported ?? new HashSet <string>(); deviceProperties.Tags.UnionWith(devicePropertiesFromStorage.Tags); deviceProperties.Reported.UnionWith(devicePropertiesFromStorage.Reported); etag = model.ETag; // If the new set of deviceProperties are already there in cache, return if (deviceProperties.Tags.Count == devicePropertiesFromStorage.Tags.Count && deviceProperties.Reported.Count == devicePropertiesFromStorage.Reported.Count) { return(deviceProperties); } } var value = JsonConvert.SerializeObject(deviceProperties); try { var response = await this.storageClient.UpdateAsync( CacheCollectioId, CacheKey, value, etag); return(JsonConvert.DeserializeObject <DevicePropertyServiceModel>(response.Data)); } catch (ConflictingResourceException) { this.logger.LogInformation("Cache updating: failed due to conflict. Retry soon"); } catch (Exception e) { this.logger.LogInformation(e, "Cache updating: failed"); throw new Exception("Cache updating: failed"); } } }