/// <inheritdoc/>
public async Task ProcessEventsAsync(PartitionContext context,
IEnumerable <EventData> messages)
{
if (messages == null || !messages.Any())
{
return;
}
var used = new HashSet <IEventHandler>();
foreach (var eventData in messages)
{
if (!eventData.Properties.TryGetValue(CommonProperties.kDeviceId,
out var deviceId) &&
!eventData.SystemProperties.TryGetValue(
SystemProperties.ConnectionDeviceId, out deviceId))
{
// Not our content to process
continue;
}
if (!eventData.Properties.TryGetValue(CommonProperties.kContentType,
out var contentType) &&
!eventData.Properties.TryGetValue(EventProperties.kContentType,
out contentType) &&
!eventData.SystemProperties.TryGetValue(
SystemProperties.ContentType, out contentType))
{
// Not our content to process
continue;
}
if (deviceId == null || contentType == null)
{
// Not our content to process
continue;
}
eventData.Properties.TryGetValue(CommonProperties.kModuleId,
out var moduleId);
if (moduleId == null)
{
// TODO: Try get from system properties
}
if (_factory._handlers.TryGetValue(contentType.ToString().ToLowerInvariant(),
out var handler))
{
await handler.HandleAsync(deviceId.ToString(),
moduleId?.ToString(), eventData.Body.Array,
() => Try.Async(() => context.CheckpointAsync(eventData)));
used.Add(handler);
}
}
foreach (var handler in used)
{
await Try.Async(handler.OnBatchCompleteAsync);
}
}
/// <summary>
/// Process items in queue
/// </summary>
/// <returns></returns>
private async Task WorkAsync()
{
while (true)
{
if (!Queue.TryTake(out var item, TimeSpan.FromSeconds(20)))
{
if (Queue.IsAddingCompleted)
{
return;
}
continue;
}
try {
await item.Task().ConfigureAwait(false);
}
catch (Exception ex) {
if (item.Retries == 0)
{
// Give up.
_processor._logger.Error("Exception thrown, give up on task!",
() => ex);
return;
}
_processor._logger.Error($"Processing task failed with exception.",
() => ex);
item.Retries--;
_processor.TrySchedule(item);
}
await Try.Async(item.Checkpoint);
}
}
/// <inheritdoc/>
public async Task UnregisterAsync(EndpointModel endpoint)
{
if (endpoint == null)
{
throw new ArgumentNullException(nameof(endpoint));
}
var id = new EndpointIdentifier(endpoint);
_callbacks.TryRemove(id, out _);
await _lock.WaitAsync();
try {
// Remove any session
if (_clients.TryGetValue(id, out var client))
{
await Try.Async(client.CloseAsync);
Try.Op(client.Dispose);
_clients.Remove(id);
_logger.Debug("Endpoint {id} ({endpoint}) closed.",
id, endpoint.Url);
}
else
{
_logger.Debug(
"Session for endpoint {id} ({endpoint}) not found.",
endpoint.Url, id);
}
}
finally {
_lock.Release();
}
}
/// <summary>
/// Delete all vm resources if possible
/// </summary>
/// <returns></returns>
public async Task TryDeleteStorageAsync(
IResourceGroupResource resourceGroup, string id)
{
var client = await CreateClientAsync(resourceGroup);
await Try.Async(() => client.StorageAccounts.DeleteByIdAsync(id));
}
/// <inheritdoc/>
public async Task <TrustGroupRegistrationResultModel> CreateRootAsync(
TrustGroupRootCreateRequestModel request, CancellationToken ct)
{
if (request == null)
{
throw new ArgumentNullException(nameof(request));
}
var result = await _groups.AddAsync(request.ToRegistration(), ct);
try {
// Issues new root certificate
var certificate = await RenewGroupCertificateAsync(result, null, ct);
_logger.Information("Root {name} {groupId} created.",
request.Name, result.Id);
return(new TrustGroupRegistrationResultModel {
Id = result.Id
});
}
catch {
// Attempt to remove group
await Try.Async(() => _groups.DeleteAsync(result.Id, r => true));
throw;
}
}
/// <summary>
/// Browse all references
/// </summary>
private static async Task <BrowseResponseApiModel> NodeBrowseAsync(
ITwinServiceApi service, string endpoint, BrowseRequestApiModel request)
{
while (true)
{
var result = await service.NodeBrowseFirstAsync(endpoint, request);
while (result.ContinuationToken != null)
{
try {
var next = await service.NodeBrowseNextAsync(endpoint,
new BrowseNextRequestApiModel {
ContinuationToken = result.ContinuationToken,
Header = request.Header,
ReadVariableValues = request.ReadVariableValues,
TargetNodesOnly = request.TargetNodesOnly
});
result.References.AddRange(next.References);
result.ContinuationToken = next.ContinuationToken;
}
catch (Exception) {
await Try.Async(() => service.NodeBrowseNextAsync(endpoint,
new BrowseNextRequestApiModel {
ContinuationToken = result.ContinuationToken,
Abort = true
}));
throw;
}
}
return(result);
}
}
/// <summary>
/// Delete all vm resources if possible
/// </summary>
/// <returns></returns>
public async Task TryDeleteServiceBusAsync(
IResourceGroupResource resourceGroup, string id)
{
var client = await CreateClientAsync(resourceGroup);
await Try.Async(() => client.ServiceBusNamespaces.DeleteByIdAsync(id));
}
/// <summary>
/// Delete all requests for the given entity
/// </summary>
/// <param name="entityId"></param>
/// <param name="context"></param>
/// <returns></returns>
private async Task RemoveAllRequestsForEntity(string entityId,
RegistryOperationContextModel context)
{
string nextPageLink = null;
var result = await _requests.QueryRequestsAsync(
new CertificateRequestQueryRequestModel {
EntityId = entityId
});
while (true)
{
nextPageLink = result.NextPageLink;
foreach (var request in result.Requests)
{
if (request.State != CertificateRequestState.Accepted)
{
await Try.Async(() => _requests.AcceptRequestAsync(
request.RequestId, new VaultOperationContextModel {
AuthorityId = context?.AuthorityId,
Time = context?.Time ?? DateTime.UtcNow
}));
}
}
if (result.NextPageLink == null)
{
break;
}
result = await _requests.ListRequestsAsync(result.NextPageLink);
}
}
/// <inheritdoc/>
public async Task <KeyHandle> GetKeyHandleAsync(string name, CancellationToken ct)
{
if (string.IsNullOrEmpty(name))
{
throw new ArgumentNullException(nameof(name));
}
// Get key first
var keyBundle = await Try.Async(() => _keyVaultClient.GetKeyAsync(name, ct));
if (keyBundle == null)
{
// If no key - then try getting cert bundle instead
var certBundle = await Try.Async(
() => _keyVaultClient.GetCertificateAsync(_vaultBaseUrl, name, ct));
if (certBundle != null)
{
return(KeyVaultKeyHandle.Create(certBundle));
}
throw new ResourceNotFoundException("Key with name not found");
}
var secretBundle = await Try.Async(() => _keyVaultClient.GetSecretAsync(name, ct));
return(KeyVaultKeyHandle.Create(keyBundle, secretBundle));
}
/// <inheritdoc/>
public async Task ProcessEventsAsync(PartitionContext context,
IEnumerable <EventData> messages)
{
if (messages == null || !messages.Any())
{
return;
}
foreach (var eventData in messages)
{
if (_outer._config.SkipEventsOlderThan != null &&
eventData.SystemProperties.TryGetValue("x-opt-enqueued-time", out var enqueued) &&
(DateTime)enqueued + _outer._config.SkipEventsOlderThan < DateTime.UtcNow)
{
continue;
}
var properties = new EventProperties(eventData.SystemProperties,
eventData.Properties);
if (eventData.Body.Array == null)
{
_logger.Verbose("WARNING: Received empty message with properties {@properties}",
properties);
continue;
}
await _handler.HandleAsync(eventData.Body.Array, properties,
() => CheckpointAsync(context, eventData));
if (context.CancellationToken.IsCancellationRequested)
{
// Checkpoint to the last processed event.
await CheckpointAsync(context, eventData);
context.CancellationToken.ThrowIfCancellationRequested();
}
}
// Checkpoint if needed
if (_sw.ElapsedMilliseconds >= _interval)
{
try {
_logger.Debug("Checkpointing EventProcessor {id} for partition {partitionId}...",
_processorId, context.PartitionId);
await context.CheckpointAsync();
_sw.Restart();
}
catch (Exception ex) {
_logger.Warning(ex, "Failed checkpointing EventProcessor {id} for partition {partitionId}...",
_processorId, context.PartitionId);
kEventProcessorDetails.WithLabels(_processorId, context.PartitionId, "checkpoint_failed").Inc();
if (_sw.ElapsedMilliseconds >= 2 * _interval)
{
// Give up checkpointing after trying a couple more times
_sw.Restart();
}
}
}
await Try.Async(_handler.OnBatchCompleteAsync);
}
/// <summary>
/// Dispose
/// </summary>
public void Dispose()
{
Try.Async(StopAsync).Wait();
// Dispose
_cts.Dispose();
_lock.Dispose();
}
/// <inheritdoc/>
public async Task OnBatchCompleteAsync()
{
foreach (var handler in _used.ToList())
{
await Try.Async(handler.OnBatchCompleteAsync);
}
_used.Clear();
}
/// <inheritdoc/>
public async Task OnBatchCompleteAsync()
{
foreach (var handler in _used)
{
await Try.Async(_handlers[handler].OnBatchCompleteAsync);
}
_used.Clear();
}
/// <summary>
/// Get or add crl to cache database
/// </summary>
/// <param name="serialNumber"></param>
/// <param name="validatyPeriod"></param>
/// <param name="ct"></param>
/// <returns></returns>
public async Task <CrlDocument> TryGetOrAddCrlAsync(SerialNumber serialNumber,
TimeSpan validatyPeriod, CancellationToken ct)
{
while (true)
{
var crl = await _crls.FindAsync <CrlDocument>(serialNumber.ToString(), ct);
if (crl != null &&
crl.Value.NextUpdate > (DateTime.UtcNow - validatyPeriod))
{
return(crl.Value);
}
// Find issuer certificate.
var issuer = await _certificates.FindCertificateAsync(serialNumber.Value, ct);
if (issuer?.IssuerPolicies == null || issuer.Revoked != null)
{
if (crl != null)
{
// Get rid of crl
await Try.Async(() => _crls.DeleteAsync(crl, ct));
}
if (issuer == null)
{
return(null); // Unknown certificate
}
// Not an issuer cert
return(new CrlDocument {
IssuerSerialNumber = issuer.GetIssuerSerialNumberAsString(),
SerialNumber = issuer.GetSerialNumberAsString(),
});
}
// Get all revoked but still valid certificates issued by issuer
var revoked = await _certificates.GetIssuedCertificatesAsync(
issuer, null, true, true, ct);
System.Diagnostics.Debug.Assert(revoked.All(r => r.Revoked != null));
// Build crl
var result = await _factory.CreateCrlAsync(issuer,
issuer.IssuerPolicies.SignatureType.Value, revoked, null, ct);
var document = result.ToDocument(
issuer.GetSerialNumberAsString(), issuer.GetIssuerSerialNumberAsString());
try {
// Add crl
crl = await _crls.UpsertAsync(document, ct, null, null, crl?.Etag);
return(crl.Value);
}
catch (ResourceOutOfDateException) {
continue;
}
}
}
/// <inheritdoc/>
public void Dispose()
{
Try.Async(StopDiscoveryRequestProcessingAsync).Wait();
// Dispose
_cts.Dispose();
_timer.Dispose();
_lock.Dispose();
}
/// <summary>
/// Close connection
/// </summary>
/// <returns></returns>
private async Task DisposeAsync(HubConnection connection)
{
if (connection == null)
{
return;
}
await Try.Async(() => connection?.StopAsync());
await Try.Async(() => connection?.DisposeAsync());
}
/// <inheritdoc/>
public void Dispose()
{
Try.Async(StopAsync).Wait();
System.Diagnostics.Debug.Assert(_jobProcess == null);
_jobProcess?.Dispose();
_cts?.Dispose();
_heartbeatTimer.Dispose();
_lock.Dispose();
}
/// <inheritdoc/>
public async Task <FinishNewKeyPairRequestResultModel> FinishNewKeyPairRequestAsync(
string requestId, VaultOperationContextModel context, CancellationToken ct)
{
if (string.IsNullOrEmpty(requestId))
{
throw new ArgumentNullException(nameof(requestId));
}
var request = await _repo.FindAsync(requestId, ct);
if (request == null)
{
throw new ResourceNotFoundException("Request not found");
}
try {
var entity = await _entities.FindEntityAsync(request.Entity.Id);
if (entity != null)
{
throw new ResourceInvalidStateException("Entity removed.");
}
var result = new FinishNewKeyPairRequestResultModel {
Request = request.Record
};
if (request.Record.State == CertificateRequestState.Completed)
{
result.Certificate = request.Certificate;
// get private key
if (request.KeyHandle != null)
{
var handle = Try.Op(
() => _serializer.DeserializeHandle(request.KeyHandle));
if (handle != null)
{
var privateKey = await Try.Async(
() => _keys.ExportKeyAsync(handle, ct));
result.PrivateKey = privateKey.ToServiceModel();
await Try.Async(
() => _keys.DeleteKeyAsync(handle, ct));
}
}
}
return(result);
}
finally {
if (request.Record.State == CertificateRequestState.Completed)
{
// Accept
await _broker.NotifyAllAsync(
l => l.OnCertificateRequestAcceptedAsync(request));
_logger.Information("Key pair response accepted and finished.");
}
}
}
/// <summary>
/// Unregister all applications and endpoints
/// </summary>
/// <param name="service"></param>
/// <param name="ct"></param>
/// <returns></returns>
public static async Task UnregisterAllApplicationsAsync(
this IApplicationRegistry service, CancellationToken ct = default)
{
var apps = await service.ListAllApplicationsAsync(ct);
foreach (var app in apps)
{
await Try.Async(() => service.UnregisterApplicationAsync(
app.ApplicationId, null, ct));
}
}
/// <inheritdoc/>
public async Task HandleAsync(string deviceId, string moduleId, byte[] payload,
IDictionary <string, string> properties, Func <Task> checkpoint)
{
var completed = await HandleEventAsync(deviceId, moduleId,
payload, properties);
if (completed)
{
await Try.Async(() => checkpoint?.Invoke());
}
}
/// <inheritdoc/>
public async Task <Certificate> NewIssuerCertificateAsync(string rootCertificate,
string certificateName, X500DistinguishedName subjectName, DateTime?notBefore,
CreateKeyParams keyParams, IssuerPolicies policies,
Func <byte[], IEnumerable <X509Extension> > extensions,
CancellationToken ct)
{
if (string.IsNullOrEmpty(certificateName))
{
throw new ArgumentNullException(nameof(certificateName));
}
// Get CA certificate
var caCertificate = await _store.GetLatestCertificateAsync(rootCertificate, ct);
if (caCertificate.IssuerPolicies == null)
{
throw new ArgumentException("root certificate is not an issuer");
}
// Validate policies
policies = policies.Validate(caCertificate.IssuerPolicies, keyParams);
// Create new key
var keyHandle = await _keys.CreateKeyAsync(Guid.NewGuid().ToString(),
keyParams, new KeyStoreProperties {
Exportable = false
}, ct);
try {
// Get public key for the key
var publicKey = await _keys.GetPublicKeyAsync(keyHandle, ct);
var signedcert = await _factory.CreateCertificateAsync(_keys, caCertificate,
subjectName, publicKey,
GetNotAfter(notBefore, caCertificate.IssuerPolicies.IssuedLifetime.Value,
caCertificate.NotAfterUtc, out var notAfter),
notAfter,
caCertificate.IssuerPolicies.SignatureType.Value, true, extensions, ct);
using (signedcert) {
// Import new issued certificate
var result = signedcert.ToCertificate(policies, keyHandle);
await _repo.AddCertificateAsync(certificateName, result, null, ct);
return(result);
}
}
catch (Exception ex) {
_logger.Verbose(ex, "Failed to add certificate, delete key");
await Try.Async(() => _keys.DeleteKeyAsync(keyHandle, ct));
throw;
}
}
/// <inheritdoc/>
public void Dispose()
{
if (_ensureWorkerRunningTimer != null)
{
_ensureWorkerRunningTimer.Enabled = false;
_ensureWorkerRunningTimer.Elapsed -= EnsureWorkerRunningTimer_ElapsedAsync;
_ensureWorkerRunningTimer.Dispose();
_ensureWorkerRunningTimer = null;
}
Try.Async(StopAsync).GetAwaiter().GetResult();
}
/// <summary>
/// Unregister all applications and endpoints
/// </summary>
/// <param name="service"></param>
/// <returns></returns>
public static async Task UnregisterAllApplicationsAsync(
this IApplicationRegistry service)
{
var apps = await service.ListAllApplicationsAsync();
foreach (var app in apps)
{
await Try.Async(() => service.UnregisterApplicationAsync(
app.ApplicationId));
}
}
/// <summary>
/// Delete all vm resources if possible
/// </summary>
/// <returns></returns>
public async Task TryDeleteResourcesAsync(
IResourceGroupResource resourceGroup, string id)
{
var client = await CreateClientAsync(resourceGroup);
await Try.Async(() => client.VirtualMachines.DeleteByIdAsync(id));
await Try.Async(() => client.Networks.DeleteByIdAsync(id));
await Try.Async(() => client.PublicIPAddresses.DeleteByIdAsync(id));
}
/// <summary>
/// Get or create new vm in a resource group.
/// </summary>
/// <param name="service"></param>
/// <param name="resourceGroup"></param>
/// <param name="name"></param>
/// <param name="network"></param>
/// <param name="image"></param>
/// <returns></returns>
public static async Task <IVirtualMachineResource> GetOrCreateAsync(
this IVirtualMachineFactory service, IResourceGroupResource resourceGroup,
string name, INetworkResource network = null, VirtualMachineImage image = null)
{
var resource = await Try.Async(() => service.GetAsync(resourceGroup, name));
if (resource == null)
{
resource = await service.CreateAsync(resourceGroup, name, network, image, null);
}
return(resource);
}
/// <inheritdoc/>
public async Task OnCertificateRequestApprovedAsync(CertificateRequestModel request)
{
try {
if (request.Record.Type == CertificateRequestType.KeyPairRequest)
{
await CreateNewKeyPairAsync(request);
}
}
catch (Exception ex) {
await Try.Async(() => _workflow.FailRequestAsync(request.Record.RequestId, ex));
}
}
/// <inheritdoc/>
public void Dispose()
{
Try.Async(StopAsync).Wait();
_updateTimer.Dispose();
// _cts might be null if StartAsync() was never called.
if (!(_cts is null))
{
_cts.Dispose();
}
}
/// <inheritdoc/>
public async Task OnCertificateRequestApprovedAsync(CertificateRequestModel request)
{
try {
if (request.Record.Type == CertificateRequestType.SigningRequest)
{
await ProcessSigningRequestAsync(request);
}
}
catch (Exception ex) {
await Try.Async(() => _workflow.FailRequestAsync(request.Record.RequestId, ex));
}
}
/// <summary>
/// Get or create new storage in a resource group.
/// </summary>
/// <param name="service"></param>
/// <param name="resourceGroup"></param>
/// <param name="name"></param>
/// <returns></returns>
public static async Task <IStorageResource> GetOrCreateAsync(
this IStorageFactory service, IResourceGroupResource resourceGroup,
string name)
{
var resource = await Try.Async(() => service.GetAsync(resourceGroup,
name));
if (resource == null)
{
resource = await service.CreateAsync(resourceGroup, name);
}
return(resource);
}
/// <summary>
/// Get or create new storage in a resource group.
/// </summary>
/// <param name="service"></param>
/// <param name="resourceGroup"></param>
/// <param name="name"></param>
/// <param name="addressSpace"></param>
/// <param name="secure"></param>
/// <returns></returns>
public static async Task <INetworkResource> GetOrCreateAsync(
this INetworkFactory service, IResourceGroupResource resourceGroup,
string name, string addressSpace = null, bool secure = false)
{
var resource = await Try.Async(() => service.GetAsync(resourceGroup,
name));
if (resource == null)
{
resource = await service.CreateAsync(resourceGroup, name,
addressSpace, secure);
}
return(resource);
}