/// <summary>
/// Get list of DeviceTwinNames from IOT-hub and whitelist it.
/// </summary>
/// <remarks>
/// List of Twin Names to be whitelisted is hardcoded in appsettings.ini
/// </remarks>
private async Task <DeviceTwinName> GetValidNamesAsync()
{
ParseWhitelist(this.whitelist, out var fullNameWhitelist, out var prefixWhitelist);
var validNames = new DeviceTwinName
{
Tags = fullNameWhitelist.Tags,
ReportedProperties = fullNameWhitelist.ReportedProperties
};
if (prefixWhitelist.Tags.Any() || prefixWhitelist.ReportedProperties.Any())
{
DeviceTwinName allNames = new DeviceTwinName();
try
{
/// Get list of DeviceTwinNames from IOT-hub
allNames = await this.devices.GetDeviceTwinNamesAsync();
}
catch (Exception e)
{
throw new ExternalDependencyException("Unable to fetch IoT devices", e);
}
validNames.Tags.UnionWith(allNames.Tags.
Where(s => prefixWhitelist.Tags.Any(s.StartsWith)));
validNames.ReportedProperties.UnionWith(
allNames.ReportedProperties.Where(
s => prefixWhitelist.ReportedProperties.Any(s.StartsWith)));
}
return(validNames);
}
private static bool CheckNames(string data, DeviceTwinName expiredNames)
{
var cacheValue = JsonConvert.DeserializeObject <DevicePropertyServiceModel>(data);
return(cacheValue.Tags.SetEquals(expiredNames.Tags) &&
cacheValue.Reported.SetEquals(expiredNames.ReportedProperties));
}
private static void ParseWhitelist(string whitelist, out DeviceTwinName fullNameWhitelist, out DeviceTwinName prefixWhitelist)
{
var whitelistItems = whitelist.Split(',').Select(s => s.Trim());
var tags = whitelistItems
.Where(s => s.StartsWith(WHITELIST_TAG_PREFIX, StringComparison.OrdinalIgnoreCase))
.Select(s => s.Substring(WHITELIST_TAG_PREFIX.Length));
var reported = whitelistItems
.Where(s => s.StartsWith(WHITELIST_REPORTED_PREFIX, StringComparison.OrdinalIgnoreCase))
.Select(s => s.Substring(WHITELIST_REPORTED_PREFIX.Length));
var fixedTags = tags.Where(s => !s.EndsWith("*"));
var fixedReported = reported.Where(s => !s.EndsWith("*"));
var regexTags = tags.Where(s => s.EndsWith("*")).Select(s => s.Substring(0, s.Length - 1));
var regexReported = reported.Where(s => s.EndsWith("*")).Select(s => s.Substring(0, s.Length - 1));
fullNameWhitelist = new DeviceTwinName
{
Tags = new HashSet <string>(fixedTags),
ReportedProperties = new HashSet <string>(fixedReported)
};
prefixWhitelist = new DeviceTwinName
{
Tags = new HashSet <string>(regexTags),
ReportedProperties = new HashSet <string>(regexReported)
};
}
private async Task <DeviceTwinName> GetValidNamesAsync()
{
ParseWhitelist(this.cacheWhitelist, out var fullNameWhitelist, out var prefixWhitelist);
var validNames = new DeviceTwinName
{
Tags = fullNameWhitelist.Tags,
ReportedProperties = fullNameWhitelist.ReportedProperties
};
if (prefixWhitelist.Tags.Any() || prefixWhitelist.ReportedProperties.Any())
{
var allNames = await this.iotHubClient.GetDeviceTwinNamesAsync();
validNames.Tags.UnionWith(
allNames.Tags
.Where(s => prefixWhitelist.Tags.Any(s.StartsWith)));
validNames.ReportedProperties.UnionWith(
allNames.ReportedProperties
.Where(s => prefixWhitelist.ReportedProperties.Any(s.StartsWith)));
}
return(validNames);
}
public async Task TryRecreateListAsyncTest()
{
var mockStorageAdapterClient = new Mock <IStorageAdapterClient>();
var mockDevices = new Mock <IDevices>();
var cache = new DeviceProperties(
mockStorageAdapterClient.Object,
new AppConfig
{
Global = new GlobalConfig(),
IotHubManagerService = new IotHubManagerServiceConfig
{
DevicePropertiesCache = new DevicePropertiesCacheConfig
{
Whitelist = "tags.*, reported.Type, reported.Config.*",
Ttl = 3600,
},
},
},
new Mock <ILogger <DeviceProperties> >().Object,
mockDevices.Object);
var etagOld = this.rand.NextString();
var etagLock = this.rand.NextString();
var etagNew = this.rand.NextString();
mockStorageAdapterClient
.Setup(x => x.GetAsync(
It.Is <string>(s => s == DeviceProperties.CacheCollectioId),
It.Is <string>(s => s == DeviceProperties.CacheKey)))
.ReturnsAsync(new ValueApiModel
{
ETag = etagOld,
Data = JsonConvert.SerializeObject(new DevicePropertyServiceModel
{
Rebuilding = false,
}),
Metadata = new Dictionary <string, string>
{
{ "$modified", (DateTimeOffset.UtcNow - TimeSpan.FromDays(1)).ToString(CultureInfo.InvariantCulture) },
},
});
mockDevices.Setup(x => x.GetDeviceTwinNamesAsync())
.ReturnsAsync(new DeviceTwinName
{
Tags = new HashSet <string> {
"Building", "Group"
},
ReportedProperties = new HashSet <string> {
"Config.Interval", "otherProperty"
},
});
mockStorageAdapterClient
.Setup(x => x.UpdateAsync(
It.Is <string>(s => s == DeviceProperties.CacheCollectioId),
It.Is <string>(s => s == DeviceProperties.CacheKey),
It.Is <string>(s => Rebuilding(s)),
It.Is <string>(s => s == etagOld)))
.ReturnsAsync(new ValueApiModel
{
ETag = etagLock,
});
mockStorageAdapterClient
.Setup(x => x.UpdateAsync(
It.Is <string>(s => s == DeviceProperties.CacheCollectioId),
It.Is <string>(s => s == DeviceProperties.CacheKey),
It.Is <string>(s => !Rebuilding(s)),
It.Is <string>(s => s == etagLock)))
.ReturnsAsync(new ValueApiModel
{
ETag = etagNew,
});
var expiredNames = new DeviceTwinName
{
Tags = new HashSet <string>
{
"Building", "Group",
},
ReportedProperties = new HashSet <string>
{
"Type", "Config.Interval", "MethodStatus", "UpdateStatus",
},
};
var result = await cache.TryRecreateListAsync();
Assert.True(result);
mockStorageAdapterClient
.Verify(
x => x.GetAsync(
It.Is <string>(s => s == DeviceProperties.CacheCollectioId),
It.Is <string>(s => s == DeviceProperties.CacheKey)),
Times.Once);
mockDevices
.Verify(x => x.GetDeviceTwinNamesAsync(), Times.Once);
mockStorageAdapterClient
.Verify(
x => x.UpdateAsync(
It.Is <string>(s => s == DeviceProperties.CacheCollectioId),
It.Is <string>(s => s == DeviceProperties.CacheKey),
It.Is <string>(s => Rebuilding(s)),
It.Is <string>(s => s == etagOld)),
Times.Once);
}
/// <summary>
/// Parse the comma seperated string "whitelist" and create two separate list
/// One with regex(*) and one without regex(*)
/// </summary>
/// <param name="whitelist">Comma seperated list of deviceTwinName to be
/// whitlisted which is hardcoded in appsettings.ini.</param>
/// <param name="fullNameWhitelist">An out paramenter which is a list of
/// deviceTwinName to be whitlisted without regex.</param>
/// <param name="prefixWhitelist">An out paramenter which is a list of
/// deviceTwinName to be whitlisted with regex.</param>
private static void ParseWhitelist(string whitelist,
out DeviceTwinName fullNameWhitelist,
out DeviceTwinName prefixWhitelist)
{
/// <example>
/// whitelist = "tags.*, reported.Protocol, reported.SupportedMethods,
/// reported.DeviceMethodStatus, reported.FirmwareUpdateStatus"
/// whitelistItems = [tags.*,
/// reported.Protocol,
/// reported.SupportedMethods,
/// reported.DeviceMethodStatus,
/// reported.FirmwareUpdateStatus]
/// </example>
var whitelistItems = whitelist.Split(',').Select(s => s.Trim());
/// <example>
/// tags = [tags.*]
/// </example>
var tags = whitelistItems
.Where(s => s.StartsWith(WHITELIST_TAG_PREFIX, StringComparison.OrdinalIgnoreCase))
.Select(s => s.Substring(WHITELIST_TAG_PREFIX.Length));
/// <example>
/// reported = [reported.Protocol,
/// reported.SupportedMethods,
/// reported.DeviceMethodStatus,
/// reported.FirmwareUpdateStatus]
/// </example>
var reported = whitelistItems
.Where(s => s.StartsWith(WHITELIST_REPORTED_PREFIX, StringComparison.OrdinalIgnoreCase))
.Select(s => s.Substring(WHITELIST_REPORTED_PREFIX.Length));
/// <example>
/// fixedTags = []
/// </example>
var fixedTags = tags.Where(s => !s.EndsWith("*"));
/// <example>
/// fixedReported = [reported.Protocol,
/// reported.SupportedMethods,
/// reported.DeviceMethodStatus,
/// reported.FirmwareUpdateStatus]
/// </example>
var fixedReported = reported.Where(s => !s.EndsWith("*"));
/// <example>
/// regexTags = [tags.]
/// </example>
var regexTags = tags.Where(s => s.EndsWith("*")).Select(s => s.Substring(0, s.Length - 1));
/// <example>
/// regexReported = []
/// </example>
var regexReported = reported.
Where(s => s.EndsWith("*")).
Select(s => s.Substring(0, s.Length - 1));
/// <example>
/// fullNameWhitelist = {Tags = [],
/// ReportedProperties = [
/// reported.Protocol,
/// reported.SupportedMethods,
/// reported.DeviceMethodStatus,
/// reported.FirmwareUpdateStatus]
/// }
/// </example>
fullNameWhitelist = new DeviceTwinName
{
Tags = new HashSet <string>(fixedTags),
ReportedProperties = new HashSet <string>(fixedReported)
};
/// <example>
/// prefixWhitelist = {Tags = [tags.],
/// ReportedProperties = []}
/// </example>
prefixWhitelist = new DeviceTwinName
{
Tags = new HashSet <string>(regexTags),
ReportedProperties = new HashSet <string>(regexReported)
};
}
public async Task TryRecreateListAsyncTest()
{
var mockStorageAdapterClient = new Mock <IStorageAdapterClient>();
var mockDevices = new Mock <IDevices>();
var cache = new DeviceProperties(
mockStorageAdapterClient.Object,
new ServicesConfig
{
DevicePropertiesWhiteList = "tags.*, reported.Type, reported.Config.*",
DevicePropertiesTTL = 3600
},
new Logger("UnitTest", LogLevel.Debug),
mockDevices.Object);
var etagOld = this.rand.NextString();
var etagLock = this.rand.NextString();
var etagNew = this.rand.NextString();
mockStorageAdapterClient
.Setup(x => x.GetAsync(
It.Is <string>(s => s == DeviceProperties.CACHE_COLLECTION_ID),
It.Is <string>(s => s == DeviceProperties.CACHE_KEY)))
.ReturnsAsync(new ValueApiModel
{
ETag = etagOld,
Data = JsonConvert.SerializeObject(new DevicePropertyServiceModel
{
Rebuilding = false
}),
Metadata = new Dictionary <string, string>
{
{ "$modified", (DateTimeOffset.UtcNow - TimeSpan.FromDays(1)).ToString(CultureInfo.InvariantCulture) }
}
});
mockDevices.Setup(x => x.GetDeviceTwinNamesAsync())
.ReturnsAsync(new DeviceTwinName
{
Tags = new HashSet <string> {
"Building", "Group"
},
ReportedProperties = new HashSet <string> {
"Config.Interval", "otherProperty"
}
});
mockStorageAdapterClient
.Setup(x => x.UpdateAsync(
It.Is <string>(s => s == DeviceProperties.CACHE_COLLECTION_ID),
It.Is <string>(s => s == DeviceProperties.CACHE_KEY),
It.Is <string>(s => Rebuilding(s)),
It.Is <string>(s => s == etagOld)))
.ReturnsAsync(new ValueApiModel
{
ETag = etagLock
});
mockStorageAdapterClient
.Setup(x => x.UpdateAsync(
It.Is <string>(s => s == DeviceProperties.CACHE_COLLECTION_ID),
It.Is <string>(s => s == DeviceProperties.CACHE_KEY),
It.Is <string>(s => !Rebuilding(s)),
It.Is <string>(s => s == etagLock)))
.ReturnsAsync(new ValueApiModel
{
ETag = etagNew
});
var expiredNames = new DeviceTwinName
{
Tags = new HashSet <string>
{
"Building", "Group"
},
ReportedProperties = new HashSet <string>
{
"Type", "Config.Interval", "MethodStatus", "UpdateStatus"
}
};
var result = await cache.TryRecreateListAsync();
Assert.True(result);
mockStorageAdapterClient
.Verify(x => x.GetAsync(
It.Is <string>(s => s == DeviceProperties.CACHE_COLLECTION_ID),
It.Is <string>(s => s == DeviceProperties.CACHE_KEY)),
Times.Once);
mockDevices
.Verify(x => x.GetDeviceTwinNamesAsync(), Times.Once);
mockStorageAdapterClient
.Verify(x => x.UpdateAsync(
It.Is <string>(s => s == DeviceProperties.CACHE_COLLECTION_ID),
It.Is <string>(s => s == DeviceProperties.CACHE_KEY),
It.Is <string>(s => Rebuilding(s)),
It.Is <string>(s => s == etagOld)),
Times.Once);
}