public void GetStorage() { StorageManager storageManager = this.configManager.GetStorageManager("Main", this.telemetryClient); string identifier = "identifier"; FunctionContext functionContext = new FunctionContext(); FunctionContextWriter <FunctionContext> contextWriter = new FunctionContextWriter <FunctionContext>(); AdlsClientWrapper adlsClientWrapper = new AdlsClientWrapper(); List <IRecordWriter> recordWriters = storageManager.InitializeRecordWriters(identifier, functionContext, contextWriter, adlsClientWrapper.AdlsClient); Assert.AreEqual(2, recordWriters.Count); Assert.IsTrue(recordWriters[0] is AdlsBulkRecordWriter <FunctionContext>); Assert.IsTrue(recordWriters[1] is AzureBlobRecordWriter <FunctionContext>); }
public async Task Traffic([QueueTrigger("traffic")] string queueItem, ExecutionContext executionContext, ILogger logger) { DateTime functionStartDate = DateTime.UtcNow; string sessionId = Guid.NewGuid().ToString(); JsonSerializerSettings serializerSettings = new JsonSerializerSettings() { TypeNameHandling = TypeNameHandling.None }; Repository repositoryDetails = JsonConvert.DeserializeObject <Repository>(queueItem, serializerSettings); FunctionContextWriter <FunctionContext> contextWriter = new FunctionContextWriter <FunctionContext>(); string identifier = $"Traffic"; FunctionContext context = new FunctionContext() { CollectorType = CollectorType.Traffic.ToString(), FunctionStartDate = functionStartDate, SessionId = sessionId, InvocationId = executionContext.InvocationId.ToString(), }; StatsTracker statsTracker = null; string outputPaths = string.Empty; bool success = false; ITelemetryClient telemetryClient = new GitHubApplicationInsightsTelemetryClient(this.telemetryClient, context, logger); try { telemetryClient.TrackEvent("SessionStart", GetRepositoryCollectorSessionStartEventProperties(context, identifier, repositoryDetails)); ICache <RateLimitTableEntity> rateLimiterCache = new AzureTableCache <RateLimitTableEntity>(telemetryClient, "ratelimiter"); await rateLimiterCache.InitializeAsync().ConfigureAwait(false); IRateLimiter rateLimiter = new GitHubRateLimiter(this.configManager.UsesGitHubAuth(context.CollectorType) ? repositoryDetails.OrganizationLogin : "******", rateLimiterCache, this.httpClient, telemetryClient, maxUsageBeforeDelayStarts: 50.0, this.apiDomain); ICache <ConditionalRequestTableEntity> requestsCache = new AzureTableCache <ConditionalRequestTableEntity>(telemetryClient, "requests"); await requestsCache.InitializeAsync().ConfigureAwait(false); GitHubHttpClient httpClient = new GitHubHttpClient(this.httpClient, rateLimiter, requestsCache, telemetryClient); IAuthentication authentication = this.configManager.GetAuthentication(CollectorType.Traffic, httpClient, repositoryDetails.OrganizationLogin, this.apiDomain); StorageManager storageManager; List <IRecordWriter> recordWriters; using (storageManager = this.configManager.GetStorageManager(context.CollectorType, telemetryClient)) { recordWriters = storageManager.InitializeRecordWriters(identifier, context, contextWriter, this.adlsClient.AdlsClient); IRecordStatsTracker recordStatsTracker = null; foreach (IRecordWriter recordWriter in recordWriters) { recordWriter.SetOutputPathPrefix($"{repositoryDetails.OrganizationId}/{repositoryDetails.RepositoryId}"); if (recordStatsTracker == null) { recordStatsTracker = recordWriter; } } statsTracker = new StatsTracker(telemetryClient, httpClient, recordStatsTracker, StatsTrackerRefreshFrequency); TrafficProcessor processor = new TrafficProcessor(authentication, recordWriters, httpClient, telemetryClient, this.apiDomain); await processor.ProcessAsync(repositoryDetails).ConfigureAwait(false); } await storageManager.FinalizeRecordWritersAsync().ConfigureAwait(false); outputPaths = RecordWriterExtensions.GetOutputPaths(recordWriters); success = true; } catch (Exception exception) when(!(exception is FatalException)) { telemetryClient.TrackException(exception, "Traffic failed."); throw exception; } finally { SendSessionEndEvent(telemetryClient, context.FunctionStartDate, outputPaths, GetRepositoryCollectorSessionStartEventProperties(context, identifier, repositoryDetails), success); statsTracker?.Stop(); } }