//TODO: Clean-up function dependencies and scope
// Specifically access to log and AuthToken
private static SecureString RetrieveAuthToken(ILogger log)
{
// If the token or expiry time is missing or the expiry time is passed we'll refetch
// This will only help on single instances of the function and providing the same worker is running.
if (AuthToken == default || ExpiryOffset == default || DateTimeOffset.Now > ExpiryOffset)
{
//TODO: need a validity check for this setting
AuthUrl = Environment.GetEnvironmentVariable("AdobeAuthURI", EnvironmentVariableTarget.Process);
using (var httpClient = new HttpClient())
{
httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/x-www-form-urlencoded"));
var authPostBody = new FormUrlEncodedContent(new[]
{
new KeyValuePair <string, string>(AdobeGrantTypeKey, AdobeGrantType),
new KeyValuePair <string, string>(AdobeClientIdKey, SecureStringHelper.ConvertToUnsecureString(AdobeAppID)),
new KeyValuePair <string, string>(AdobeClientSecretKey, SecureStringHelper.ConvertToUnsecureString(AdobeAppSecret))
});
var response = httpClient.PostAsync(new Uri(AuthUrl), authPostBody).Result;
//TODO: Replace with custom deserializer so that the access_token member is never in plain text in memory.
// This code encrypts right after deserialization and clears the object which leaves exposure window
// once the response is received and after deserialization up until the object is GC.
// A custom deserializer to SecureString for the access_token will shorten the window to only the amount of time
// that the response is held in RAM.
var jsonContent = JsonConvert.DeserializeObject <AdobeAuthResponse>(response.Content.ReadAsStringAsync().Result);
//assign class vars
AuthToken = SecureStringHelper.ConvertToSecureString(jsonContent.access_token);
//set expiry time
int seconds = int.Parse(jsonContent.expires_in);
ExpiryOffset = DateTimeOffset.Now.AddSeconds(seconds);
//try to hurry up GC
jsonContent = null;
}
}
return(AuthToken);
}
public static void Run([QueueTrigger("%ControlQueueName%", Connection = "IngestControlQueueConnection")] IngestControlMessage myQueueItem, Microsoft.Azure.WebJobs.ExecutionContext ExContext, ILogger log,
[EventHub("%EventHubName%",
Connection = "EventHubConnection")]
ICollector <EventData> outData)
{
if (AdobeAppID == default)
{
AdobeAppID = SecureStringHelper.ConvertToSecureString(Environment.GetEnvironmentVariable("AdobeAppID", EnvironmentVariableTarget.Process));
}
if (AdobeAppSecret == default)
{
AdobeAppSecret = SecureStringHelper.ConvertToSecureString(Environment.GetEnvironmentVariable("AdobeAppSecret", EnvironmentVariableTarget.Process));
}
SecureString authToken = RetrieveAuthToken(log);
RetrieveData(log, authToken, myQueueItem, outData);
}
private static void RetrieveData(ILogger log, SecureString authToken, IngestControlMessage controlMessage, ICollector <EventData> outData)
{
HttpClientHandler handler = new HttpClientHandler()
{
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
};
using (HttpClient httpClient = new HttpClient(handler))
{
//TODO: Move AdobeURI to KeyVault
string requestUri = Environment.GetEnvironmentVariable("AdobeURI", EnvironmentVariableTarget.Process);
int maxConnections = controlMessage.MaxConnections < 8 ? controlMessage.MaxConnections + 1 : 8;
//going to tell Adobe we need 1 more connection than we plan to use in case there is lag in shutdown that causes overlap
requestUri += "?maxConnections=" + maxConnections.ToString();
var request = new HttpRequestMessage(HttpMethod.Get, requestUri);
request.Headers.Add("Authorization", "Bearer " + SecureStringHelper.ConvertToUnsecureString(authToken));
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("*/*"));
request.Headers.AcceptEncoding.Add(new StringWithQualityHeaderValue("gzip"));
var response = httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead).Result;
var requestStream = response.Content.ReadAsStreamAsync().Result;
using (var reader = new StreamReader(requestStream))
{
DateTimeOffset loopExpiryOffset = DateTimeOffset.Now.AddSeconds(controlMessage.ExecutionSeconds);
ClickStreamEventHubSerializer ehSerializer = new ClickStreamEventHubSerializer();
ehSerializer.Initialize(log);
// Add records to the outData ICollector<> for the defined duration.
// Once completed the binder will handle serialization to EventHub upon Function exit.
//WARNING: This will cause memory growth. So, the volume over the duration needs to be kept manageable for the Function.
while (!reader.EndOfStream && loopExpiryOffset > DateTimeOffset.Now)
{
var currentLine = reader.ReadLine();
ehSerializer.Serialize(currentLine, outData);
}
}
//TODO: Log telemetry information about processing for AppInsights and debug/reconciliation
}
}
