public static void Run([QueueTrigger("%ControlQueueName%", Connection = "IngestControlQueueConnection")] IngestControlMessage myQueueItem, Microsoft.Azure.WebJobs.ExecutionContext ExContext, ILogger log,
[EventHub("%EventHubName%",
Connection = "EventHubConnection")]
ICollector <EventData> outData)
{
if (AdobeAppID == default)
{
AdobeAppID = SecureStringHelper.ConvertToSecureString(Environment.GetEnvironmentVariable("AdobeAppID", EnvironmentVariableTarget.Process));
}
if (AdobeAppSecret == default)
{
AdobeAppSecret = SecureStringHelper.ConvertToSecureString(Environment.GetEnvironmentVariable("AdobeAppSecret", EnvironmentVariableTarget.Process));
}
SecureString authToken = RetrieveAuthToken(log);
RetrieveData(log, authToken, myQueueItem, outData);
}
private static void RetrieveData(ILogger log, SecureString authToken, IngestControlMessage controlMessage, ICollector <EventData> outData)
{
HttpClientHandler handler = new HttpClientHandler()
{
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
};
using (HttpClient httpClient = new HttpClient(handler))
{
//TODO: Move AdobeURI to KeyVault
string requestUri = Environment.GetEnvironmentVariable("AdobeURI", EnvironmentVariableTarget.Process);
int maxConnections = controlMessage.MaxConnections < 8 ? controlMessage.MaxConnections + 1 : 8;
//going to tell Adobe we need 1 more connection than we plan to use in case there is lag in shutdown that causes overlap
requestUri += "?maxConnections=" + maxConnections.ToString();
var request = new HttpRequestMessage(HttpMethod.Get, requestUri);
request.Headers.Add("Authorization", "Bearer " + SecureStringHelper.ConvertToUnsecureString(authToken));
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("*/*"));
request.Headers.AcceptEncoding.Add(new StringWithQualityHeaderValue("gzip"));
var response = httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead).Result;
var requestStream = response.Content.ReadAsStreamAsync().Result;
using (var reader = new StreamReader(requestStream))
{
DateTimeOffset loopExpiryOffset = DateTimeOffset.Now.AddSeconds(controlMessage.ExecutionSeconds);
ClickStreamEventHubSerializer ehSerializer = new ClickStreamEventHubSerializer();
ehSerializer.Initialize(log);
// Add records to the outData ICollector<> for the defined duration.
// Once completed the binder will handle serialization to EventHub upon Function exit.
//WARNING: This will cause memory growth. So, the volume over the duration needs to be kept manageable for the Function.
while (!reader.EndOfStream && loopExpiryOffset > DateTimeOffset.Now)
{
var currentLine = reader.ReadLine();
ehSerializer.Serialize(currentLine, outData);
}
}
//TODO: Log telemetry information about processing for AppInsights and debug/reconciliation
}
}
public static void Run([TimerTrigger("%TimerSchedule%")] TimerInfo myTimer, ExecutionContext ExContext, ILogger log, [Queue("%ControlQueueName%",
Connection = "IngestControlQueueConnection")]
CloudQueue outputQueue)
{
//Get configured number of messages to populate in the queue to spawn readers
int maxControlMessages = default;
string maxReaders = Environment.GetEnvironmentVariable("MaxReaders", EnvironmentVariableTarget.Process);
if (!int.TryParse(maxReaders, out maxControlMessages))
{
string errorMsg = "Invalid value for MaxReaders setting. Assigned value = " + maxReaders;
log.LogCritical(errorMsg);
throw new System.Exception(errorMsg);
}
//ensure that we don't queue more readers than concurrent max (8) for Adobe and > 0
if (maxControlMessages > 8)
{
string errorMsg = "Invalid value for maxControlMessages setting. Assigned value = " + maxControlMessages + ". Value assigned 8.";
log.LogInformation(errorMsg);
maxControlMessages = 8;
}
else if (maxControlMessages < 0)
{
string errorMsg = "Invalid value for maxControlMessages setting. Assigned value = " + maxControlMessages + ". Value assigned 1.";
log.LogInformation(errorMsg);
maxControlMessages = 1;
}
//retrieve setting value passed to ingestion function to dictate how long to execute the data fetch
int executionSeconds = default;
string execDurationSetting = Environment.GetEnvironmentVariable("ExecutionSeconds", EnvironmentVariableTarget.Process);
if (!int.TryParse(execDurationSetting, out executionSeconds))
{
string errorMsg = "Invalid value for ExecutionSeconds setting. Assigned value = " + execDurationSetting;
log.LogCritical(errorMsg);
throw new System.Exception(errorMsg);
}
//ensuring that there is a least 1 second of run time
if (executionSeconds < 1)
{
string errorMsg = "Invalid value for executionSeconds setting. Assigned value = " + executionSeconds + ". Value assigned 1.";
log.LogInformation(errorMsg);
executionSeconds = 1;
}
//use this to control message visibility
int gapIntervalSeconds = default;
string gapIntervalSetting = Environment.GetEnvironmentVariable("GapIntervalSeconds", EnvironmentVariableTarget.Process);
if (!int.TryParse(gapIntervalSetting, out gapIntervalSeconds))
{
string errorMsg = "Invalid value for GapIntervalSeconds setting. Assigned value = " + gapIntervalSetting;
log.LogCritical(errorMsg);
throw new System.Exception(errorMsg);
}
//ensure that the value is greater than
log.LogInformation("ID: " + ExContext.InvocationId.ToString() + " executing with maxControlMessages = " + maxControlMessages.ToString());
//create messages for control queue
for (int msgCount = 0; msgCount < maxControlMessages; msgCount++)
{
int visibilityDelay = msgCount * gapIntervalSeconds;
var newMessage = new IngestControlMessage {
ServiceLabel = "ClickstreamIngest", ExecutionSeconds = executionSeconds, MaxConnections = maxControlMessages
};
outputQueue.AddMessageAsync(new CloudQueueMessage(JsonConvert.SerializeObject(newMessage)), null, TimeSpan.FromSeconds(visibilityDelay), null, null);
}
log.LogInformation("ID: " + ExContext.InvocationId.ToString() + $"completed at: {DateTime.Now}");
}
