public async Task PublishAsync(EventMessage message, List <KeyValuePair <string, string> > indexes = null)
{
AuditRecord record = null;
DateTime receiveTime = DateTime.UtcNow;
try
{
record = new MessageAuditRecord(message.MessageId, identity, channelType, protocolType.ToUpperInvariant(), message.Message.Length, MessageDirectionType.In, true, receiveTime);
if (indexes == null || indexes.Count == 0)
{
await GraphManager.PublishAsync(message.ResourceUri, message);
}
else
{
await GraphManager.PublishAsync(message.ResourceUri, message, indexes);
}
}
catch (Exception ex)
{
record = new MessageAuditRecord(message.MessageId, identity, channelType, protocolType.ToUpperInvariant(), message.Message.Length, MessageDirectionType.In, false, receiveTime, ex.Message);
}
finally
{
if (message.Audit)
{
await auditor?.WriteAuditRecordAsync(record);
}
}
}
private void Adapter_OnObserve(object sender, ObserveMessageEventArgs e)
{
MessageAuditRecord record = null;
int length = 0;
DateTime sendTime = DateTime.UtcNow;
try
{
byte[] message = ProtocolTransition.ConvertToMqtt(session, e.Message);
Send(message).LogExceptions();
MqttMessage mm = MqttMessage.DecodeMessage(message);
length = mm.Payload.Length;
record = new MessageAuditRecord(e.Message.MessageId, session.Identity, Channel.TypeId, "MQTT", length,
MessageDirectionType.Out, true, sendTime);
}
catch (Exception ex)
{
string msg = string.Format("{0} - MQTT adapter observe error on channel '{1}' with '{2}'",
DateTime.UtcNow.ToString("yyyy-MM-ddTHH-MM-ss.fffff"), Channel.Id, ex.Message);
logger?.LogErrorAsync(ex, $"MQTT adapter observe error on channel '{Channel.Id}'.").GetAwaiter();
record = new MessageAuditRecord(e.Message.MessageId, session.Identity, Channel.TypeId, "MQTT", length,
MessageDirectionType.Out, true, sendTime, msg);
}
finally
{
if (e.Message.Audit)
{
messageAuditor?.WriteAuditRecordAsync(record).Ignore();
}
}
}
private async Task FaultTask(string id, byte[] payload, string contentType, bool canAudit)
{
AuditRecord record = null;
try
{
ServiceClientCredentials credentials = new TopicCredentials(topicKey);
EventGridClient client = new EventGridClient(credentials);
EventGridEvent gridEvent = new EventGridEvent(id, resourceUriString, payload, resourceUriString, DateTime.UtcNow, "1.0");
IList <EventGridEvent> events = new List <EventGridEvent>(new EventGridEvent[] { gridEvent });
await clients[arrayIndex].PublishEventsAsync(topicHostname, events);
record = new MessageAuditRecord(id, uri.Query.Length > 0 ? uri.ToString().Replace(uri.Query, "") : uri.ToString(), "EventGrid", "EventGrid", payload.Length, MessageDirectionType.Out, true, DateTime.UtcNow);
}
catch (Exception ex)
{
Trace.TraceWarning("Retry EventGrid failed.");
Trace.TraceError(ex.Message);
record = new MessageAuditRecord(id, uri.Query.Length > 0 ? uri.ToString().Replace(uri.Query, "") : uri.ToString(), "EventGrid", "EventGrid", payload.Length, MessageDirectionType.Out, false, DateTime.UtcNow, ex.Message);
}
finally
{
if (canAudit)
{
await auditor?.WriteAuditRecordAsync(record);
}
}
}
private void Adapter_OnObserve(object sender, ObserveMessageEventArgs e)
{
MessageAuditRecord record = null;
int length = 0;
DateTime sendTime = DateTime.UtcNow;
try
{
byte[] message = ProtocolTransition.ConvertToHttp(e.Message);
Send(message).LogExceptions();
OnObserve?.Invoke(this,
new ChannelObserverEventArgs(Channel.Id, e.Message.ResourceUri, e.Message.ContentType,
e.Message.Message));
length = message.Length;
record = new MessageAuditRecord(e.Message.MessageId, identity, Channel.TypeId, "WSN", length,
MessageDirectionType.Out, true, sendTime);
}
catch (Exception ex)
{
string msg = string.Format("{0} - WSN adapter observe error on channel '{1}' with '{2}'",
DateTime.UtcNow.ToString("yyyy-MM-ddTHH-MM-ss.fffff"), Channel.Id, ex.Message);
logger?.LogError(ex, $"WSN adapter observe error on channel '{Channel.Id}'.");
record = new MessageAuditRecord(e.Message.MessageId, identity, Channel.TypeId, "WSN", length,
MessageDirectionType.Out, true, sendTime, msg);
}
finally
{
if (e.Message.Audit)
{
messageAuditor?.WriteAuditRecordAsync(record).Ignore();
}
}
}
private async Task FaultTask(EventMessage message, bool canAudit)
{
AuditRecord record = null;
IDatabase db = null;
byte[] payload = null;
ConnectionMultiplexer conn = null;
try
{
string cacheKey = GetKey(message);
conn = await NewConnection();
if (dbNumber < 1)
{
db = connection.GetDatabase();
}
else
{
db = connection.GetDatabase(dbNumber);
}
payload = GetPayload(message);
if (message.ContentType != "application/octet-stream")
{
await db.StringSetAsync(cacheKey, Encoding.UTF8.GetString(payload), expiry);
}
else
{
await db.StringSetAsync(cacheKey, payload, expiry);
}
record = new MessageAuditRecord(message.MessageId,
uri.Query.Length > 0 ? uri.ToString().Replace(uri.Query, "") : uri.ToString(),
$"Redis({db.Database})", $"Redis({db.Database})", payload.Length,
MessageDirectionType.Out, true, DateTime.UtcNow);
}
catch (Exception ex)
{
await logger.LogErrorAsync(ex, $"Redis cache sink '{metadata.SubscriptionUriString}'");
record = new MessageAuditRecord(message.MessageId,
uri.Query.Length > 0 ? uri.ToString().Replace(uri.Query, "") : uri.ToString(),
$"Redis({db.Database})", $"Redis({db.Database})", payload.Length,
MessageDirectionType.Out, false, DateTime.UtcNow, ex.Message);
}
finally
{
if (canAudit)
{
await auditor?.WriteAuditRecordAsync(record);
}
conn?.Dispose();
}
}
public override async Task SendAsync(EventMessage message)
{
AuditRecord record = null;
byte[] payload = null;
EventMessage msg = null;
loadQueue.Enqueue(message);
try
{
while (!loadQueue.IsEmpty)
{
bool isdequeued = loadQueue.TryDequeue(out msg);
if (!isdequeued)
{
continue;
}
payload = GetPayload(msg);
if (payload == null)
{
await logger?.LogWarningAsync(
$"Subscription '{metadata.SubscriptionUriString}' message not written to queue sink because message is null.");
return;
}
await storage.EnqueueAsync(queue, payload, ttl);
if (message.Audit)
{
record = new MessageAuditRecord(msg.MessageId,
uri.Query.Length > 0 ? uri.ToString().Replace(uri.Query, "") : uri.ToString(),
"AzureQueue", "AzureQueue", payload.Length, MessageDirectionType.Out, true,
DateTime.UtcNow);
}
}
}
catch (Exception ex)
{
await logger?.LogErrorAsync(ex,
$"Subscription '{metadata.SubscriptionUriString}' message not written to queue sink.");
record = new MessageAuditRecord(msg.MessageId,
uri.Query.Length > 0 ? uri.ToString().Replace(uri.Query, "") : uri.ToString(), "AzureQueue",
"AzureQueue", payload.Length, MessageDirectionType.Out, false, DateTime.UtcNow, ex.Message);
throw;
}
finally
{
if (record != null && msg.Audit)
{
await auditor?.WriteAuditRecordAsync(record);
}
}
}
private async Task PublishAsync(PublishMessage message)
{
MessageAuditRecord record = null;
EventMetadata metadata = null;
try
{
MqttUri mqttUri = new MqttUri(message.Topic);
metadata = await graphManager.GetPiSystemMetadataAsync(mqttUri.Resource);
if (EventValidator.Validate(true, metadata, Channel, graphManager, context).Validated)
{
EventMessage msg = new EventMessage(mqttUri.ContentType, mqttUri.Resource, ProtocolType.MQTT,
message.Encode(), DateTime.UtcNow, metadata.Audit);
if (!string.IsNullOrEmpty(mqttUri.CacheKey))
{
msg.CacheKey = mqttUri.CacheKey;
}
if (mqttUri.Indexes != null)
{
List <KeyValuePair <string, string> > list = GetIndexes(mqttUri);
await adapter.PublishAsync(msg, list);
}
else
{
await adapter.PublishAsync(msg);
}
}
else
{
if (metadata.Audit)
{
record = new MessageAuditRecord("XXXXXXXXXXXX", session.Identity, Channel.TypeId, "MQTT",
message.Payload.Length, MessageDirectionType.In, false, DateTime.UtcNow,
"Not authorized, missing resource metadata, or channel encryption requirements");
}
throw new SecurityException(string.Format("'{0}' not authorized to publish to '{1}'",
session.Identity, metadata.ResourceUriString));
}
}
catch (Exception ex)
{
await logger?.LogErrorAsync(ex, $"MQTT adapter PublishAsync error on channel '{Channel.Id}'.");
OnError?.Invoke(this, new ProtocolAdapterErrorEventArgs(Channel.Id, ex));
}
finally
{
if (metadata != null && metadata.Audit && record != null)
{
await messageAuditor?.WriteAuditRecordAsync(record);
}
}
}
public override async Task SendAsync(EventMessage message)
{
AuditRecord record = null;
byte[] payload = null;
try
{
byte[] msg = GetPayload(message);
queue.Enqueue(msg);
while (!queue.IsEmpty)
{
arrayIndex = arrayIndex.RangeIncrement(0, clientCount - 1);
queue.TryDequeue(out payload);
if (payload == null)
{
Trace.TraceWarning("Subscription {0} could not write to event hub sink because payload was either null or unknown protocol type.");
return;
}
EventData data = new EventData(payload);
data.Properties.Add("Content-Type", message.ContentType);
if (String.IsNullOrEmpty(partitionId))
{
await storageArray[arrayIndex].SendAsync(data);
}
else
{
await senderArray[arrayIndex].SendAsync(data);
}
if (message.Audit)
{
record = new MessageAuditRecord(message.MessageId, String.Format("sb://{0}/{1}", uri.Authority, hubName), "EventHub", "EventHub", payload.Length, MessageDirectionType.Out, true, DateTime.UtcNow);
}
}
}
catch (Exception ex)
{
record = new MessageAuditRecord(message.MessageId, String.Format("sb://{0}", uri.Authority, hubName), "EventHub", "EventHub", payload.Length, MessageDirectionType.Out, false, DateTime.UtcNow, ex.Message);
throw;
}
finally
{
if (message.Audit && record != null)
{
await auditor?.WriteAuditRecordAsync(record);
}
}
}
private void Adapter_OnObserve(object sender, ObserveMessageEventArgs e)
{
logger?.LogDebugAsync("REST adapter received observed message");
OnObserve?.Invoke(this, new ChannelObserverEventArgs(channel.Id, e.Message.ResourceUri, e.Message.ContentType, e.Message.Message));
AuditRecord record = new UserAuditRecord(channel.Id, identity, DateTime.UtcNow);
userAuditor?.UpdateAuditRecordAsync(record).Ignore();
AuditRecord messageRecord = new MessageAuditRecord(e.Message.MessageId, identity, channel.TypeId, protocolType.ToString(), e.Message.Message.Length, MessageDirectionType.Out, true, DateTime.UtcNow);
messageAuditor?.WriteAuditRecordAsync(messageRecord);
}
private async Task FaultTask(string id, string container, string filename, byte[] payload, string contentType,
bool canAudit)
{
AuditRecord record = null;
try
{
BlobStorage storage = BlobStorage.New(connectionString, 2048, 102400);
if (blobType != "block")
{
if (blobType == "page")
{
int pad = payload.Length % 512 != 0 ? 512 - payload.Length % 512 : 0;
byte[] buffer = new byte[payload.Length + pad];
Buffer.BlockCopy(payload, 0, buffer, 0, payload.Length);
await storage.WritePageBlobAsync(container, filename, buffer, contentType);
}
else
{
await storage.WriteAppendBlobAsync(container, filename, payload);
}
}
else
{
string[] parts = filename.Split(new[] { '.' });
string path2 = parts.Length == 2
? $"{parts[0]}-R.{parts[1]}"
: $"{filename}-R";
await storage.WriteBlockBlobAsync(container, path2, payload, contentType);
}
record = new MessageAuditRecord(id,
uri.Query.Length > 0 ? uri.ToString().Replace(uri.Query, "") : uri.ToString(), "AzureBlob",
"AzureBlob", payload.Length, MessageDirectionType.Out, true, DateTime.UtcNow);
}
catch (Exception ex)
{
await logger?.LogErrorAsync(
$"Subscription '{metadata.SubscriptionUriString}' message not written to blob sink in fault task.");
record = new MessageAuditRecord(id,
uri.Query.Length > 0 ? uri.ToString().Replace(uri.Query, "") : uri.ToString(), "AzureBlob",
"AzureBlob", payload.Length, MessageDirectionType.Out, false, DateTime.UtcNow, ex.Message);
}
finally
{
if (canAudit && record != null)
{
await auditor?.WriteAuditRecordAsync(record);
}
}
}
public override async Task SendAsync(EventMessage message)
{
AuditRecord record = null;
try
{
byte[] payload = GetPayload(message);
if (payload == null)
{
Trace.TraceWarning(
"Subscription {0} could not write to service bus sink because payload was either null or unknown protocol type.");
return;
}
if (client == null)
{
client = new TopicClient(connectionString, topic);
}
Message brokerMessage = new Message(payload)
{
ContentType = message.ContentType,
MessageId = message.MessageId
};
await client.SendAsync(brokerMessage);
record = new MessageAuditRecord(message.MessageId, string.Format("sb://{0}/{1}", uri.Authority, topic),
"ServiceBus", "ServiceBus", message.Message.Length, MessageDirectionType.Out, true,
DateTime.UtcNow);
}
catch (Exception ex)
{
Trace.TraceError("Service bus failed to send to topic with error {0}", ex.Message);
record = new MessageAuditRecord(message.MessageId, $"sb://{uri.Authority}/{topic}",
"ServiceBus", "ServiceBus", message.Message.Length, MessageDirectionType.Out, false,
DateTime.UtcNow, ex.Message);
}
finally
{
if (message.Audit && record != null)
{
await auditor?.WriteAuditRecordAsync(record);
}
}
}
public override async Task SendAsync(EventMessage message)
{
AuditRecord record = null;
byte[] payload = null;
EventMessage msg = null;
loadQueue.Enqueue(message);
try
{
while (!loadQueue.IsEmpty)
{
bool isdequeued = loadQueue.TryDequeue(out msg);
if (isdequeued)
{
payload = GetPayload(msg);
if (payload == null)
{
Trace.TraceWarning("Subscription {0} could not write to queue storage sink because payload was either null or unknown protocol type.");
return;
}
await storage.EnqueueAsync(queue, payload, ttl);
if (message.Audit)
{
record = new MessageAuditRecord(msg.MessageId, uri.Query.Length > 0 ? uri.ToString().Replace(uri.Query, "") : uri.ToString(), "AzureQueue", "AzureQueue", payload.Length, MessageDirectionType.Out, true, DateTime.UtcNow);
}
}
}
}
catch (Exception ex)
{
record = new MessageAuditRecord(msg.MessageId, uri.Query.Length > 0 ? uri.ToString().Replace(uri.Query, "") : uri.ToString(), "AzureQueue", "AzureQueue", payload.Length, MessageDirectionType.Out, false, DateTime.UtcNow, ex.Message);
throw;
}
finally
{
if (record != null && msg.Audit)
{
await auditor?.WriteAuditRecordAsync(record);
}
}
}
public override async Task SendAsync(EventMessage message)
{
AuditRecord record = null;
HttpClient client = new HttpClient();
try
{
client.DefaultRequestHeaders.Add("Authorization", $"Bearer {token}");
HttpContent content = new StringContent(Encoding.UTF8.GetString(message.Message), Encoding.UTF8,
"application/json");
HttpResponseMessage response = await client.PostAsync(uri, content);
if (response.IsSuccessStatusCode)
{
byte[] outMessage = await response.Content.ReadAsByteArrayAsync();
EventMessage output = new EventMessage("application/json", outputPiSystem, message.Protocol,
outMessage, DateTime.UtcNow, message.Audit);
RaiseOnResponse(new EventSinkResponseArgs(output));
record = new MessageAuditRecord(message.MessageId, $"ai://{uri.Authority}", "MachineLearning",
"MachineLearning", message.Message.Length, MessageDirectionType.Out, true, DateTime.UtcNow);
}
else
{
record = new MessageAuditRecord(message.MessageId, $"ai://{uri.Authority}", "MachineLearning",
"MachineLearning", message.Message.Length, MessageDirectionType.Out, false, DateTime.UtcNow);
}
}
catch (Exception ex)
{
await logger?.LogErrorAsync(ex,
$"Subscription {metadata.SubscriptionUriString} machine learning sink.");
record = new MessageAuditRecord(message.MessageId, $"ai://{uri.Authority}", "MachineLearning",
"MachineLearning", message.Message.Length, MessageDirectionType.Out, false, DateTime.UtcNow);
}
finally
{
if (record != null && message.Audit)
{
await auditor?.WriteAuditRecordAsync(record);
}
}
}
private async Task FaultTask(string id, string container, string filename, byte[] payload, string contentType, bool canAudit)
{
AuditRecord record = null;
try
{
BlobStorage storage = BlobStorage.New(connectionString, 2048, 102400);
if (blobType == "block")
{
string[] parts = filename.Split(new char[] { '.' });
string path2 = parts.Length == 2 ? String.Format("{0}-R.{1}", parts[0], parts[1]) : String.Format("{0}-R", filename);
await storage.WriteBlockBlobAsync(container, path2, payload, contentType);
}
else if (blobType == "page")
{
int pad = payload.Length % 512 != 0 ? 512 - payload.Length % 512 : 0;
byte[] buffer = new byte[payload.Length + pad];
Buffer.BlockCopy(payload, 0, buffer, 0, payload.Length);
await storage.WritePageBlobAsync(container, filename, buffer, contentType);
}
else
{
await storage.WriteAppendBlobAsync(container, filename, payload, contentType);
}
record = new MessageAuditRecord(id, uri.Query.Length > 0 ? uri.ToString().Replace(uri.Query, "") : uri.ToString(), "AzureBlob", "AzureBlob", payload.Length, MessageDirectionType.Out, true, DateTime.UtcNow);
}
catch (Exception ex)
{
Trace.TraceWarning("Retry Blob failed.");
Trace.TraceError(ex.Message);
record = new MessageAuditRecord(id, uri.Query.Length > 0 ? uri.ToString().Replace(uri.Query, "") : uri.ToString(), "AzureBlob", "AzureBlob", payload.Length, MessageDirectionType.Out, false, DateTime.UtcNow, ex.Message);
}
finally
{
if (canAudit)
{
await auditor?.WriteAuditRecordAsync(record);
}
}
}
private async Task Send(byte[] message, ObserveMessageEventArgs e)
{
AuditRecord record = null;
try
{
await channel.SendAsync(message);
record = new MessageAuditRecord(e.Message.MessageId, session.Identity, this.channel.TypeId, "COAP", e.Message.Message.Length, MessageDirectionType.Out, true, DateTime.UtcNow);
}
catch (Exception ex)
{
record = new MessageAuditRecord(e.Message.MessageId, session.Identity, this.channel.TypeId, "COAP", e.Message.Message.Length, MessageDirectionType.Out, false, DateTime.UtcNow, ex.Message);
}
finally
{
if (e.Message.Audit)
{
await auditor?.WriteAuditRecordAsync(record);
}
}
}
public override async Task SendAsync(EventMessage message)
{
AuditRecord record = null;
byte[] payload = null;
try
{
arrayIndex = arrayIndex.RangeIncrement(0, clientCount - 1);
payload = GetPayload(message);
if (payload == null)
{
Trace.TraceWarning("Subscription {0} could not write to blob storage sink because payload was either null or unknown protocol type.");
return;
}
EventGridEvent gridEvent = new EventGridEvent(message.MessageId, resourceUriString, payload, resourceUriString, DateTime.UtcNow, "1.0");
IList <EventGridEvent> events = new List <EventGridEvent>(new EventGridEvent[] { gridEvent });
Task task = clients[arrayIndex].PublishEventsAsync(topicHostname, events);
Task innerTask = task.ContinueWith(async(a) => { await FaultTask(message.MessageId, payload, message.ContentType, message.Audit); }, TaskContinuationOptions.OnlyOnFaulted);
await Task.WhenAll(task);
record = new MessageAuditRecord(message.MessageId, uri.Query.Length > 0 ? uri.ToString().Replace(uri.Query, "") : uri.ToString(), "EventGrid", "EventGrid", payload.Length, MessageDirectionType.Out, true, DateTime.UtcNow);
}
catch (Exception ex)
{
Trace.TraceWarning("Initial event grid write error {0}", ex.Message);
record = new MessageAuditRecord(message.MessageId, uri.Query.Length > 0 ? uri.ToString().Replace(uri.Query, "") : uri.ToString(), "EventGrid", "EventGrid", payload.Length, MessageDirectionType.Out, false, DateTime.UtcNow, ex.Message);
}
finally
{
if (message.Audit && record != null)
{
await auditor?.WriteAuditRecordAsync(record);
}
}
}
public override async Task SendAsync(EventMessage message)
{
AuditRecord record = null;
byte[] payload = null;
try
{
byte[] msg = GetPayload(message);
queue.Enqueue(msg);
while (!queue.IsEmpty)
{
arrayIndex = arrayIndex.RangeIncrement(0, clientCount - 1);
queue.TryDequeue(out payload);
if (payload == null)
{
await logger?.LogWarningAsync(
$"Subscription '{metadata.SubscriptionUriString}' message not written to event hub sink because message is null.");
return;
}
EventData data = new EventData(payload);
data.Properties.Add("Content-Type", message.ContentType);
if (string.IsNullOrEmpty(partitionId))
{
await storageArray[arrayIndex].SendAsync(data);
}
else
{
await senderArray[arrayIndex].SendAsync(data);
}
if (message.Audit && record != null)
{
record = new MessageAuditRecord(message.MessageId,
$"sb://{uri.Authority}/{hubName}", "EventHub", "EventHub",
payload.Length, MessageDirectionType.Out, true, DateTime.UtcNow);
}
}
}
catch (Exception ex)
{
await logger?.LogErrorAsync(ex,
$"Subscription '{metadata.SubscriptionUriString}' message not written to event grid hub sink.");
record = new MessageAuditRecord(message.MessageId, string.Format("sb://{0}", uri.Authority, hubName),
"EventHub", "EventHub", payload.Length, MessageDirectionType.Out, false, DateTime.UtcNow,
ex.Message);
throw;
}
finally
{
if (message.Audit && record != null)
{
await auditor?.WriteAuditRecordAsync(record);
}
}
}
public override async Task SendAsync(EventMessage message)
{
AuditRecord record = null;
byte[] payload = null;
try
{
payload = GetPayload(message);
if (payload == null)
{
await logger?.LogWarningAsync(
"Subscription {0} could not write to web hook sink because payload was null.");
return;
}
HttpWebRequest request = WebRequest.Create(address) as HttpWebRequest;
request.ContentType = message.ContentType;
request.Method = "POST";
byte[] signature = SignPayload(payload);
request.Headers.Add("Authorization", $"Bearer {Convert.ToBase64String(signature)}");
request.ContentLength = payload.Length;
Stream stream = await request.GetRequestStreamAsync();
await stream.WriteAsync(payload, 0, payload.Length);
using HttpWebResponse response = await request.GetResponseAsync() as HttpWebResponse;
if (response.StatusCode == HttpStatusCode.Accepted ||
response.StatusCode == HttpStatusCode.OK ||
response.StatusCode == HttpStatusCode.NoContent ||
response.StatusCode == HttpStatusCode.Created)
{
await logger?.LogInformationAsync(
$"Subscription {metadata.SubscriptionUriString} web hook request is success.");
record = new MessageAuditRecord(message.MessageId, address, "WebHook", "HTTP", payload.Length,
MessageDirectionType.Out, true, DateTime.UtcNow);
}
else
{
await logger?.LogWarningAsync(
$"Subscription {metadata.SubscriptionUriString} web hook request returned an expected status code {response.StatusCode}");
record = new MessageAuditRecord(message.MessageId, address, "WebHook", "HTTP", payload.Length,
MessageDirectionType.Out, false, DateTime.UtcNow,
$"Rest request returned an expected status code {response.StatusCode}");
}
}
catch (WebException we)
{
await logger?.LogErrorAsync(we,
$"Subscription {metadata.SubscriptionUriString} web hook request sink.");
record = new MessageAuditRecord(message.MessageId, address, "WebHook", "HTTP", payload.Length,
MessageDirectionType.Out, false, DateTime.UtcNow, we.Message);
}
catch (Exception ex)
{
await logger?.LogErrorAsync(ex, $"Subscription {metadata.SubscriptionUriString} Web hook event sink.");
record = new MessageAuditRecord(message.MessageId, address, "WebHook", "HTTP", payload.Length,
MessageDirectionType.Out, false, DateTime.UtcNow, ex.Message);
}
finally
{
if (message.Audit && record != null)
{
await auditor?.WriteAuditRecordAsync(record);
}
}
}
public override async Task SendAsync(EventMessage message)
{
AuditRecord record = null;
HttpWebRequest request = null;
byte[] payload = null;
try
{
payload = GetPayload(message);
if (payload == null)
{
await logger.LogWarningAsync($"Rest request '{metadata.SubscriptionUriString}' null payload.");
return;
}
try
{
request = WebRequest.Create(address) as HttpWebRequest;
request.ContentType = message.ContentType;
request.Method = "POST";
SetSecurityToken(request);
request.ContentLength = payload.Length;
Stream stream = await request.GetRequestStreamAsync();
await stream.WriteAsync(payload, 0, payload.Length);
}
catch (Exception ex)
{
Trace.TraceError("REST event sink subscription {0} could set request; error {1} ",
metadata.SubscriptionUriString, ex.Message);
record = new MessageAuditRecord(message.MessageId, address, "WebService", "HTTP", payload.Length,
MessageDirectionType.Out, false, DateTime.UtcNow, ex.Message);
}
try
{
using HttpWebResponse response = await request.GetResponseAsync() as HttpWebResponse;
if (response.StatusCode == HttpStatusCode.Accepted || response.StatusCode == HttpStatusCode.OK ||
response.StatusCode == HttpStatusCode.NoContent)
{
await logger.LogInformationAsync($"Rest request success {response.StatusCode}");
record = new MessageAuditRecord(message.MessageId, address, "WebService", "HTTP",
payload.Length, MessageDirectionType.Out, true, DateTime.UtcNow);
}
else
{
await logger.LogWarningAsync($"Rest request warning {response.StatusCode}");
record = new MessageAuditRecord(message.MessageId, address, "WebService", "HTTP",
payload.Length, MessageDirectionType.Out, false, DateTime.UtcNow,
string.Format("Rest request returned an expected status code {0}", response.StatusCode));
}
}
catch (WebException we)
{
string faultMessage =
$"subscription '{metadata.SubscriptionUriString}' with status code '{we.Status}' and error message '{we.Message}'";
await logger.LogErrorAsync(we, $"Rest request success.");
record = new MessageAuditRecord(message.MessageId, address, "WebService", "HTTP", payload.Length,
MessageDirectionType.Out, false, DateTime.UtcNow, we.Message);
}
}
catch (Exception ex)
{
await logger.LogErrorAsync(ex, $"Rest request success.");
record = new MessageAuditRecord(message.MessageId, address, "WebService", "HTTP", payload.Length,
MessageDirectionType.Out, false, DateTime.UtcNow, ex.Message);
}
finally
{
if (message.Audit && record != null)
{
await auditor?.WriteAuditRecordAsync(record);
}
}
}
public override async Task SendAsync(EventMessage message)
{
AuditRecord record = null;
byte[] payload = null;
EventMessage msg = null;
queue.Enqueue(message);
try
{
while (!queue.IsEmpty)
{
bool isdequeued = queue.TryDequeue(out msg);
if (!isdequeued)
{
continue;
}
arrayIndex = arrayIndex.RangeIncrement(0, clientCount - 1);
payload = GetPayload(msg);
if (payload == null)
{
await logger?.LogWarningAsync(
$"Subscription '{metadata.SubscriptionUriString}' message not written to blob sink because message is null.");
return;
}
string filename = GetBlobName(msg.ContentType);
if (blobType != "block")
{
if (blobType == "page")
{
int pad = payload.Length % 512 != 0 ? 512 - payload.Length % 512 : 0;
byte[] buffer = new byte[payload.Length + pad];
Buffer.BlockCopy(payload, 0, buffer, 0, payload.Length);
Task task = storageArray[arrayIndex]
.WritePageBlobAsync(container, filename, buffer, msg.ContentType);
Task innerTask =
task.ContinueWith(
async a =>
{
await FaultTask(msg.MessageId, container, filename, payload, msg.ContentType,
msg.Audit);
}, TaskContinuationOptions.OnlyOnFaulted);
await Task.WhenAll(task);
}
else
{
appendFilename ??= GetAppendFilename(msg.ContentType);
byte[] suffix = Encoding.UTF8.GetBytes(Environment.NewLine);
byte[] buffer = new byte[payload.Length + suffix.Length];
Buffer.BlockCopy(payload, 0, buffer, 0, payload.Length);
Buffer.BlockCopy(suffix, 0, buffer, payload.Length, suffix.Length);
Task task = storageArray[arrayIndex]
.WriteAppendBlobAsync(container, appendFilename, buffer);
Task innerTask =
task.ContinueWith(
async a =>
{
await FaultTask(msg.MessageId, container, appendFilename, buffer,
msg.ContentType, msg.Audit);
}, TaskContinuationOptions.OnlyOnFaulted);
await Task.WhenAll(task);
}
}
else
{
Task task = storageArray[arrayIndex]
.WriteBlockBlobAsync(container, filename, payload, msg.ContentType);
Task innerTask =
task.ContinueWith(
async a =>
{
await FaultTask(msg.MessageId, container, filename, payload, msg.ContentType,
msg.Audit);
}, TaskContinuationOptions.OnlyOnFaulted);
await Task.WhenAll(task);
}
record = new MessageAuditRecord(msg.MessageId,
uri.Query.Length > 0 ? uri.ToString().Replace(uri.Query, "") : uri.ToString(), "AzureBlob",
"AzureBlob", payload.Length, MessageDirectionType.Out, true, DateTime.UtcNow);
}
}
catch (Exception ex)
{
await logger?.LogErrorAsync(ex,
$"Subscription '{metadata.SubscriptionUriString}' message not written to blob sink.");
record = new MessageAuditRecord(msg.MessageId,
uri.Query.Length > 0 ? uri.ToString().Replace(uri.Query, "") : uri.ToString(), "AzureBlob",
"AzureBlob", payload.Length, MessageDirectionType.Out, false, DateTime.UtcNow, ex.Message);
}
finally
{
if (msg.Audit && record != null)
{
await auditor?.WriteAuditRecordAsync(record);
}
}
}
public override async Task SendAsync(EventMessage message)
{
AuditRecord record = null;
byte[] payload = null;
queue.Enqueue(message);
try
{
while (!queue.IsEmpty)
{
arrayIndex = arrayIndex.RangeIncrement(0, clientCount - 1);
bool isdequeued = queue.TryDequeue(out EventMessage msg);
if (!isdequeued)
{
continue;
}
payload = GetPayload(message);
if (payload == null)
{
await logger?.LogWarningAsync(
$"Subscription '{metadata.SubscriptionUriString}' message not written to cosmos db sink because message is null.");
continue;
}
await using MemoryStream stream = new MemoryStream(payload)
{
Position = 0
};
if (message.ContentType.Contains("json"))
{
await storageArray[arrayIndex].CreateDocumentAsync(collection.SelfLink,
JsonSerializable.LoadFrom <Document>(stream));
}
else
{
dynamic documentWithAttachment = new {
Id = Guid.NewGuid().ToString(),
Timestamp = DateTime.UtcNow
};
Document doc = await storageArray[arrayIndex]
.CreateDocumentAsync(collection.SelfLink, documentWithAttachment);
string slug = GetSlug(documentWithAttachment.Id, message.ContentType);
await storageArray[arrayIndex].CreateAttachmentAsync(doc.AttachmentsLink, stream,
new MediaOptions {
ContentType = message.ContentType, Slug = slug
});
}
if (message.Audit)
{
record = new MessageAuditRecord(message.MessageId,
uri.Query.Length > 0 ? uri.ToString().Replace(uri.Query, "") : uri.ToString(),
"CosmosDB", "CosmoDB", payload.Length, MessageDirectionType.Out, true, DateTime.UtcNow);
}
}
}
catch (Exception ex)
{
await logger?.LogErrorAsync(ex,
$"Subscription '{metadata.SubscriptionUriString}' message not written to cosmos db sink.");
record = new MessageAuditRecord(message.MessageId,
uri.Query.Length > 0 ? uri.ToString().Replace(uri.Query, "") : uri.ToString(), "CosmosDB",
"CosmosDB", payload.Length, MessageDirectionType.Out, false, DateTime.UtcNow, ex.Message);
}
finally
{
if (record != null && message.Audit)
{
await auditor?.WriteAuditRecordAsync(record);
}
}
}
public override async Task SendAsync(EventMessage message)
{
AuditRecord record = null;
byte[] payload = null;
try
{
payload = GetPayload(message);
if (payload == null)
{
await logger?.LogWarningAsync(
$"Subscription '{metadata.SubscriptionUriString}' message not written to iot hub sink because message is null.");
return;
}
if (serviceClient != null)
{
if (!string.IsNullOrEmpty(methodName))
{
if (message.ContentType == "application/json")
{
CloudToDeviceMethod method = new CloudToDeviceMethod(methodName);
method.SetPayloadJson(Encoding.UTF8.GetString(payload));
await serviceClient.InvokeDeviceMethodAsync(deviceId, method);
record = new MessageAuditRecord(message.MessageId,
$"iothub://{uri.Authority}", "IoTHub", "IoTHub", payload.Length,
MessageDirectionType.Out, true, DateTime.UtcNow);
}
else
{
await logger?.LogWarningAsync(
$"Subscription '{metadata.SubscriptionUriString}' cannot send IoTHub direct method sink because content-type is not JSON.");
record = new MessageAuditRecord(message.MessageId,
string.Format("iothub://{0}", uri.Authority), "IoTHub", "IoTHub", payload.Length,
MessageDirectionType.Out, false, DateTime.UtcNow,
string.Format(
"Cannot send IoTHub device {0} direct message because content-type is not JSON.",
deviceId));
}
}
else
{
Message serviceMessage = new Message(payload)
{
ContentType = message.ContentType,
MessageId = message.MessageId
};
if (!string.IsNullOrEmpty(propertyName))
{
serviceMessage.Properties.Add(propertyName, propertyValue);
}
await serviceClient.SendAsync(deviceId, serviceMessage);
record = new MessageAuditRecord(message.MessageId, string.Format("iothub://{0}", uri.Authority),
"IoTHub", "IoTHub", payload.Length, MessageDirectionType.Out, true, DateTime.UtcNow);
}
}
else if (deviceClient != null)
{
Microsoft.Azure.Devices.Client.Message msg = new Microsoft.Azure.Devices.Client.Message(payload)
{
ContentType = message.ContentType,
MessageId = message.MessageId
};
if (!string.IsNullOrEmpty(propertyName))
{
msg.Properties.Add(propertyName, propertyValue);
}
await deviceClient.SendEventAsync(msg);
record = new MessageAuditRecord(message.MessageId, $"iothub://{uri.Authority}",
"IoTHub", "IoTHub", payload.Length, MessageDirectionType.Out, true, DateTime.UtcNow);
}
else
{
await logger?.LogWarningAsync(
$"Subscription '{metadata.SubscriptionUriString}' IoTHub sink has neither service or device client.");
record = new MessageAuditRecord(message.MessageId, $"iothub://{uri.Authority}",
"IoTHub", "IoTHub", payload.Length, MessageDirectionType.Out, false, DateTime.UtcNow,
"IoTHub subscription has neither service or device client");
}
}
catch (Exception ex)
{
await logger?.LogErrorAsync(ex,
$"Subscription '{metadata.SubscriptionUriString}' message not written to IoTHub sink.");
record = new MessageAuditRecord(message.MessageId, $"iothub://{uri.Authority}",
"IoTHub", "IoTHub", payload.Length, MessageDirectionType.Out, false, DateTime.UtcNow, ex.Message);
}
finally
{
if (record != null && message.Audit)
{
await auditor?.WriteAuditRecordAsync(record);
}
}
}
private void Channel_OnReceive(object sender, ChannelReceivedEventArgs e)
{
Exception error = null;
if (method == "POST" && string.IsNullOrEmpty(resource))
{
error = new Exception("REST adapter cannot send message without resource.");
}
if (method == "POST" && string.IsNullOrEmpty(contentType))
{
error = new Exception("REST adapter cannot send message without content-type.");
}
if (method == "POST" && (e.Message == null || e.Message.Length == 0))
{
error = new Exception("REST adapter cannot send empty message.");
}
if (method == "GET" && (subscriptions == null || subscriptions.Count() == 0))
{
error = new Exception("REST adapter cannot subscribe to '0' subscriptions.");
}
if (error != null)
{
logger?.LogWarningAsync(error.Message).GetAwaiter();
OnError?.Invoke(this, new ProtocolAdapterErrorEventArgs(channel.Id, error));
return;
}
try
{
if (method == "POST")
{
EventMessage message = new EventMessage(contentType, resource, protocolType, e.Message);
if (!string.IsNullOrEmpty(messageUri.CacheKey))
{
message.CacheKey = messageUri.CacheKey;
}
adapter.PublishAsync(message, indexes).GetAwaiter();
logger?.LogDebugAsync("REST adapter published message");
MessageAuditRecord record = new MessageAuditRecord(message.MessageId, identity, channel.TypeId, protocolType.ToString(), e.Message.Length, MessageDirectionType.In, true, DateTime.UtcNow);
messageAuditor?.WriteAuditRecordAsync(record).Ignore();
OnClose?.Invoke(this, new ProtocolAdapterCloseEventArgs(Channel.Id));
}
if (method == "GET")
{
foreach (var subscription in subscriptions)
{
SubscriptionMetadata metadata = new SubscriptionMetadata()
{
Identity = identity,
Indexes = indexes,
IsEphemeral = true
};
adapter.SubscribeAsync(subscription, metadata).GetAwaiter();
}
}
}
catch (Exception ex)
{
logger?.LogErrorAsync($"REST adapter processing error on receive - {ex.Message}");
OnError?.Invoke(this, new ProtocolAdapterErrorEventArgs(channel.Id, ex));
}
}
public override async Task SendAsync(EventMessage message)
{
AuditRecord record = null;
byte[] payload = null;
EventMessage msg = null;
if (connection == null || !connection.IsConnected)
{
connection = await ConnectionMultiplexer.ConnectAsync(connectionString);
}
await tqueue.Enqueue(() => cqm.EnqueueAsync(message));
try
{
while (!cqm.IsEmpty)
{
msg = await cqm.DequeueAsync();
string cacheKey = GetKey(msg);
if (cacheKey == null)
{
Trace.TraceWarning("Redis sink has no cache key for subscription '{0}'.", this.metadata.SubscriptionUriString);
Trace.TraceError("No cache key found.");
}
payload = GetPayload(msg);
if (payload.Length == 0)
{
throw new InvalidOperationException("Payload length is 0.");
}
if (msg.ContentType != "application/octet-stream")
{
Task task = database.StringSetAsync(cacheKey, Encoding.UTF8.GetString(payload), expiry);
Task innerTask = task.ContinueWith(async(a) => { await FaultTask(msg, message.Audit); }, TaskContinuationOptions.OnlyOnFaulted);
await Task.WhenAll(task);
}
else
{
Task task = database.StringSetAsync(cacheKey, payload, expiry);
Task innerTask = task.ContinueWith(async(a) => { await FaultTask(msg, message.Audit); }, TaskContinuationOptions.OnlyOnFaulted);
await Task.WhenAll(task);
}
record = new MessageAuditRecord(msg.MessageId, uri.Query.Length > 0 ? uri.ToString().Replace(uri.Query, "") : uri.ToString(), String.Format("Redis({0})", dbNumber), String.Format("Redis({0})", dbNumber), payload.Length, MessageDirectionType.Out, true, DateTime.UtcNow);
}
}
catch (Exception ex)
{
Trace.TraceWarning("Initial Redis write error {0}", ex.Message);
record = new MessageAuditRecord(msg.MessageId, uri.Query.Length > 0 ? uri.ToString().Replace(uri.Query, "") : uri.ToString(), String.Format("Redis({0})", dbNumber), String.Format("Redis({0})", dbNumber), payload.Length, MessageDirectionType.Out, false, DateTime.UtcNow, ex.Message);
}
finally
{
if (message.Audit && record != null)
{
await auditor?.WriteAuditRecordAsync(record);
}
}
}
public override async Task SendAsync(EventMessage message)
{
AuditRecord record = null;
byte[] payload = null;
try
{
payload = GetPayload(message);
if (payload == null)
{
Trace.TraceWarning("Subscription {0} could not write to iot hub sink because payload was either null or unknown protocol type.");
return;
}
if (serviceClient != null) //send message to device
{
if (!String.IsNullOrEmpty(methodName)) //direct method to device
{
if (message.ContentType == "application/json")
{
CloudToDeviceMethod method = new CloudToDeviceMethod(methodName);
method.SetPayloadJson(Encoding.UTF8.GetString(payload));
await serviceClient.InvokeDeviceMethodAsync(deviceId, method);
record = new MessageAuditRecord(message.MessageId, String.Format("iothub://{0}", uri.Authority), "IoTHub", "IoTHub", payload.Length, MessageDirectionType.Out, true, DateTime.UtcNow);
}
else
{
Trace.TraceWarning("Cannot send IoTHub device {0} direct message because content-type is not JSON.", deviceId);
record = new MessageAuditRecord(message.MessageId, String.Format("iothub://{0}", uri.Authority), "IoTHub", "IoTHub", payload.Length, MessageDirectionType.Out, false, DateTime.UtcNow, String.Format("Cannot send IoTHub device {0} direct message because content-type is not JSON.", deviceId));
}
}
else //command to device
{
Microsoft.Azure.Devices.Message serviceMessage = new Microsoft.Azure.Devices.Message(payload);
serviceMessage.ContentType = message.ContentType;
serviceMessage.MessageId = message.MessageId;
if (!String.IsNullOrEmpty(propertyName))
{
serviceMessage.Properties.Add(propertyName, propertyValue);
}
await serviceClient.SendAsync(deviceId, serviceMessage);
record = new MessageAuditRecord(message.MessageId, String.Format("iothub://{0}", uri.Authority), "IoTHub", "IoTHub", payload.Length, MessageDirectionType.Out, true, DateTime.UtcNow);
}
}
else if (deviceClient != null) //this subscription is a device and will send to IoTHub
{
Microsoft.Azure.Devices.Client.Message msg = new Microsoft.Azure.Devices.Client.Message(payload);
msg.ContentType = message.ContentType;
msg.MessageId = message.MessageId;
if (!String.IsNullOrEmpty(propertyName))
{
msg.Properties.Add(propertyName, propertyValue);
}
await deviceClient.SendEventAsync(msg);
record = new MessageAuditRecord(message.MessageId, String.Format("iothub://{0}", uri.Authority), "IoTHub", "IoTHub", payload.Length, MessageDirectionType.Out, true, DateTime.UtcNow);
}
else
{
Trace.TraceWarning("IoTHub subscription has neither Service or Device client");
record = new MessageAuditRecord(message.MessageId, String.Format("iothub://{0}", uri.Authority), "IoTHub", "IoTHub", payload.Length, MessageDirectionType.Out, false, DateTime.UtcNow, "IoTHub subscription has neither service or device client");
}
}
catch (Exception ex)
{
record = new MessageAuditRecord(message.MessageId, String.Format("iothub://{0}", uri.Authority), "IoTHub", "IoTHub", payload.Length, MessageDirectionType.Out, false, DateTime.UtcNow, ex.Message);
}
finally
{
if (record != null && message.Audit)
{
await auditor?.WriteAuditRecordAsync(record);
}
}
}
public override async Task SendAsync(EventMessage message)
{
AuditRecord record = null;
byte[] payload = null;
EventMessage msg = null;
queue.Enqueue(message);
//byte[] doc = GetPayload(message);
//if (doc != null)
//{
// queue.Enqueue(doc);
//}
try
{
while (!queue.IsEmpty)
{
arrayIndex = arrayIndex.RangeIncrement(0, clientCount - 1);
bool isdequeued = queue.TryDequeue(out msg);
if (isdequeued)
{
payload = GetPayload(message);
if (payload == null)
{
Trace.TraceWarning("Subscription {0} could not write to CosmosDB sink because payload was either null or unknown protocol type.");
continue;
}
using (MemoryStream stream = new MemoryStream(payload))
{
stream.Position = 0;
if (message.ContentType.Contains("json"))
{
await storageArray[arrayIndex].CreateDocumentAsync(collection.SelfLink, Microsoft.Azure.Documents.Resource.LoadFrom <Document>(stream));
}
else
{
dynamic documentWithAttachment = new
{
Id = Guid.NewGuid().ToString(),
Timestamp = DateTime.UtcNow
};
Document doc = await storageArray[arrayIndex].CreateDocumentAsync(collection.SelfLink, documentWithAttachment);
string slug = GetSlug(documentWithAttachment.Id, message.ContentType);
await storageArray[arrayIndex].CreateAttachmentAsync(doc.AttachmentsLink, stream, new MediaOptions {
ContentType = message.ContentType, Slug = slug
});
}
}
if (message.Audit)
{
record = new MessageAuditRecord(message.MessageId, uri.Query.Length > 0 ? uri.ToString().Replace(uri.Query, "") : uri.ToString(), "CosmosDB", "CosmoDB", payload.Length, MessageDirectionType.Out, true, DateTime.UtcNow);
}
}
}
}
catch (Exception ex)
{
record = new MessageAuditRecord(message.MessageId, uri.Query.Length > 0 ? uri.ToString().Replace(uri.Query, "") : uri.ToString(), "CosmosDB", "CosmosDB", payload.Length, MessageDirectionType.Out, false, DateTime.UtcNow, ex.Message);
}
finally
{
if (record != null && message.Audit)
{
await auditor?.WriteAuditRecordAsync(record);
}
}
}
public override async Task SendAsync(EventMessage message)
{
AuditRecord record = null;
HttpWebRequest request = null;
byte[] payload = null;
try
{
payload = GetPayload(message);
if (payload == null)
{
Trace.TraceWarning("Subscription {0} could not write to web service sink because payload was either null or unknown protocol type.");
return;
}
try
{
request = HttpWebRequest.Create(address) as HttpWebRequest;
request.ContentType = message.ContentType;
request.Method = "POST";
SetSecurityToken(request);
request.ContentLength = payload.Length;
Stream stream = await request.GetRequestStreamAsync();
await stream.WriteAsync(payload, 0, payload.Length);
}
catch (Exception ex)
{
Trace.TraceError("REST event sink subscription {0} could set request; error {1} ", metadata.SubscriptionUriString, ex.Message);
record = new MessageAuditRecord(message.MessageId, address, "WebService", "HTTP", payload.Length, MessageDirectionType.Out, false, DateTime.UtcNow, ex.Message);
}
try
{
using (HttpWebResponse response = await request.GetResponseAsync() as HttpWebResponse)
{
if (response.StatusCode == HttpStatusCode.Accepted || response.StatusCode == HttpStatusCode.OK || response.StatusCode == HttpStatusCode.NoContent)
{
Trace.TraceInformation("Rest request is success.");
record = new MessageAuditRecord(message.MessageId, address, "WebService", "HTTP", payload.Length, MessageDirectionType.Out, true, DateTime.UtcNow);
}
else
{
Trace.TraceInformation("Rest request returned an expected status code.");
record = new MessageAuditRecord(message.MessageId, address, "WebService", "HTTP", payload.Length, MessageDirectionType.Out, false, DateTime.UtcNow, String.Format("Rest request returned an expected status code {0}", response.StatusCode));
}
}
}
catch (WebException we)
{
string faultMessage = String.Format("subscription '{0}' with status code '{1}' and error message '{2}'", metadata.SubscriptionUriString, we.Status.ToString(), we.Message);
Trace.TraceError(faultMessage);
record = new MessageAuditRecord(message.MessageId, address, "WebService", "HTTP", payload.Length, MessageDirectionType.Out, false, DateTime.UtcNow, we.Message);
}
}
catch (Exception ex)
{
record = new MessageAuditRecord(message.MessageId, address, "WebService", "HTTP", payload.Length, MessageDirectionType.Out, false, DateTime.UtcNow, ex.Message);
}
finally
{
if (message.Audit && record != null)
{
await auditor?.WriteAuditRecordAsync(record);
}
}
}
