public static void Run([QueueTrigger("%ControlQueueName%", Connection = "IngestControlQueueConnection")] IngestControlMessage myQueueItem, Microsoft.Azure.WebJobs.ExecutionContext ExContext, ILogger log, [EventHub("%EventHubName%", Connection = "EventHubConnection")] ICollector <EventData> outData) { if (AdobeAppID == default) { AdobeAppID = SecureStringHelper.ConvertToSecureString(Environment.GetEnvironmentVariable("AdobeAppID", EnvironmentVariableTarget.Process)); } if (AdobeAppSecret == default) { AdobeAppSecret = SecureStringHelper.ConvertToSecureString(Environment.GetEnvironmentVariable("AdobeAppSecret", EnvironmentVariableTarget.Process)); } SecureString authToken = RetrieveAuthToken(log); RetrieveData(log, authToken, myQueueItem, outData); }
private static void RetrieveData(ILogger log, SecureString authToken, IngestControlMessage controlMessage, ICollector <EventData> outData) { HttpClientHandler handler = new HttpClientHandler() { AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate }; using (HttpClient httpClient = new HttpClient(handler)) { //TODO: Move AdobeURI to KeyVault string requestUri = Environment.GetEnvironmentVariable("AdobeURI", EnvironmentVariableTarget.Process); int maxConnections = controlMessage.MaxConnections < 8 ? controlMessage.MaxConnections + 1 : 8; //going to tell Adobe we need 1 more connection than we plan to use in case there is lag in shutdown that causes overlap requestUri += "?maxConnections=" + maxConnections.ToString(); var request = new HttpRequestMessage(HttpMethod.Get, requestUri); request.Headers.Add("Authorization", "Bearer " + SecureStringHelper.ConvertToUnsecureString(authToken)); request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("*/*")); request.Headers.AcceptEncoding.Add(new StringWithQualityHeaderValue("gzip")); var response = httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead).Result; var requestStream = response.Content.ReadAsStreamAsync().Result; using (var reader = new StreamReader(requestStream)) { DateTimeOffset loopExpiryOffset = DateTimeOffset.Now.AddSeconds(controlMessage.ExecutionSeconds); ClickStreamEventHubSerializer ehSerializer = new ClickStreamEventHubSerializer(); ehSerializer.Initialize(log); // Add records to the outData ICollector<> for the defined duration. // Once completed the binder will handle serialization to EventHub upon Function exit. //WARNING: This will cause memory growth. So, the volume over the duration needs to be kept manageable for the Function. while (!reader.EndOfStream && loopExpiryOffset > DateTimeOffset.Now) { var currentLine = reader.ReadLine(); ehSerializer.Serialize(currentLine, outData); } } //TODO: Log telemetry information about processing for AppInsights and debug/reconciliation } }
public static void Run([TimerTrigger("%TimerSchedule%")] TimerInfo myTimer, ExecutionContext ExContext, ILogger log, [Queue("%ControlQueueName%", Connection = "IngestControlQueueConnection")] CloudQueue outputQueue) { //Get configured number of messages to populate in the queue to spawn readers int maxControlMessages = default; string maxReaders = Environment.GetEnvironmentVariable("MaxReaders", EnvironmentVariableTarget.Process); if (!int.TryParse(maxReaders, out maxControlMessages)) { string errorMsg = "Invalid value for MaxReaders setting. Assigned value = " + maxReaders; log.LogCritical(errorMsg); throw new System.Exception(errorMsg); } //ensure that we don't queue more readers than concurrent max (8) for Adobe and > 0 if (maxControlMessages > 8) { string errorMsg = "Invalid value for maxControlMessages setting. Assigned value = " + maxControlMessages + ". Value assigned 8."; log.LogInformation(errorMsg); maxControlMessages = 8; } else if (maxControlMessages < 0) { string errorMsg = "Invalid value for maxControlMessages setting. Assigned value = " + maxControlMessages + ". Value assigned 1."; log.LogInformation(errorMsg); maxControlMessages = 1; } //retrieve setting value passed to ingestion function to dictate how long to execute the data fetch int executionSeconds = default; string execDurationSetting = Environment.GetEnvironmentVariable("ExecutionSeconds", EnvironmentVariableTarget.Process); if (!int.TryParse(execDurationSetting, out executionSeconds)) { string errorMsg = "Invalid value for ExecutionSeconds setting. Assigned value = " + execDurationSetting; log.LogCritical(errorMsg); throw new System.Exception(errorMsg); } //ensuring that there is a least 1 second of run time if (executionSeconds < 1) { string errorMsg = "Invalid value for executionSeconds setting. Assigned value = " + executionSeconds + ". Value assigned 1."; log.LogInformation(errorMsg); executionSeconds = 1; } //use this to control message visibility int gapIntervalSeconds = default; string gapIntervalSetting = Environment.GetEnvironmentVariable("GapIntervalSeconds", EnvironmentVariableTarget.Process); if (!int.TryParse(gapIntervalSetting, out gapIntervalSeconds)) { string errorMsg = "Invalid value for GapIntervalSeconds setting. Assigned value = " + gapIntervalSetting; log.LogCritical(errorMsg); throw new System.Exception(errorMsg); } //ensure that the value is greater than log.LogInformation("ID: " + ExContext.InvocationId.ToString() + " executing with maxControlMessages = " + maxControlMessages.ToString()); //create messages for control queue for (int msgCount = 0; msgCount < maxControlMessages; msgCount++) { int visibilityDelay = msgCount * gapIntervalSeconds; var newMessage = new IngestControlMessage { ServiceLabel = "ClickstreamIngest", ExecutionSeconds = executionSeconds, MaxConnections = maxControlMessages }; outputQueue.AddMessageAsync(new CloudQueueMessage(JsonConvert.SerializeObject(newMessage)), null, TimeSpan.FromSeconds(visibilityDelay), null, null); } log.LogInformation("ID: " + ExContext.InvocationId.ToString() + $"completed at: {DateTime.Now}"); }