public async Task ProcessAsync_Success_JObject()
{
string url = "SomeUrl";
string response = @"
{
""ProjectId"": ""ProjectId1""
}
";
this.httpClient.AddResponse(url, HttpStatusCode.OK, response);
GitHubCollectionNode collectionNode = new GitHubCollectionNode()
{
ApiName = "ApiName",
RecordType = "RecordType",
GetInitialUrl = metadata => url,
ResponseType = typeof(JObject)
};
await this.collector.ProcessAsync(collectionNode).ConfigureAwait(false);
List <Tuple <JObject, RecordContext> > records = this.recordWriter.GetRecords();
Assert.AreEqual(1, records.Count);
Assert.AreEqual("ProjectId1", records[0].Item1.SelectToken("$.ProjectId").Value <string>());
RecordContext recordContext1 = records[0].Item2;
Assert.AreEqual(url, recordContext1.AdditionalMetadata["OriginatingUrl"]);
Assert.AreEqual("RecordType", recordContext1.RecordType);
}
public async Task ProcessAsync(Repository repository)
{
Dictionary <string, JToken> additionalMetadata = new Dictionary <string, JToken>()
{
{ "OrganizationId", repository.OrganizationId },
{ "OrganizationLogin", repository.OrganizationLogin },
{ "RepositoryId", repository.RepositoryId },
{ "RepositoryName", repository.RepositoryName },
};
// Referrers
GitHubCollectionNode referrersNode = new GitHubCollectionNode()
{
RecordType = DataContract.ReferrerInstanceRecordType,
ApiName = DataContract.ReferrersApiName,
GetInitialUrl = additionalMetadata => InitialReferrersUrl(repository, this.apiDomain),
AdditionalMetadata = additionalMetadata,
};
foreach (IRecordWriter recordWriter in this.recordWriters)
{
await recordWriter.NewOutputAsync(DataContract.ReferrerInstanceRecordType).ConfigureAwait(false);
}
await this.collector.ProcessAsync(referrersNode).ConfigureAwait(false);
// Views
GitHubCollectionNode viewsNode = new GitHubCollectionNode()
{
RecordType = DataContract.ViewInstanceRecordType,
ApiName = DataContract.ViewsApiName,
GetInitialUrl = additionalMetadata => InitialViewsUrl(repository, this.apiDomain),
AdditionalMetadata = additionalMetadata,
ResponseType = typeof(JObject),
};
foreach (IRecordWriter recordWriter in this.recordWriters)
{
await recordWriter.NewOutputAsync(DataContract.ViewInstanceRecordType).ConfigureAwait(false);
}
await this.collector.ProcessAsync(viewsNode).ConfigureAwait(false);
// Clones
GitHubCollectionNode clonesNode = new GitHubCollectionNode()
{
RecordType = DataContract.CloneInstanceRecordType,
ApiName = DataContract.ClonesApiName,
GetInitialUrl = additionalMetadata => InitialClonesUrl(repository, this.apiDomain),
AdditionalMetadata = additionalMetadata,
ResponseType = typeof(JObject),
};
foreach (IRecordWriter recordWriter in this.recordWriters)
{
await recordWriter.NewOutputAsync(DataContract.CloneInstanceRecordType).ConfigureAwait(false);
}
await this.collector.ProcessAsync(clonesNode).ConfigureAwait(false);
// Paths
GitHubCollectionNode pathsNode = new GitHubCollectionNode()
{
RecordType = DataContract.PathInstanceRecordType,
ApiName = DataContract.PathsApiName,
GetInitialUrl = additionalMetadata => InitialPathsUrl(repository, this.apiDomain),
AdditionalMetadata = additionalMetadata,
};
foreach (IRecordWriter recordWriter in this.recordWriters)
{
await recordWriter.NewOutputAsync(DataContract.PathInstanceRecordType).ConfigureAwait(false);
}
await this.collector.ProcessAsync(pathsNode).ConfigureAwait(false);
}
private async Task ExecuteTrafficCollector(ExecutionContext executionContext, ILogger logger)
{
DateTime functionStartDate = DateTime.UtcNow;
string sessionId = Guid.NewGuid().ToString();
string identifier = "TrafficTimer";
CloudQueue trafficCloudQueue = await AzureHelpers.GetStorageQueueAsync("traffic").ConfigureAwait(false);
IQueue trafficQueue = new CloudQueueWrapper(trafficCloudQueue);
FunctionContext context = new FunctionContext()
{
CollectorType = CollectorType.TrafficTimer.ToString(),
FunctionStartDate = functionStartDate,
SessionId = sessionId,
InvocationId = executionContext.InvocationId.ToString(),
};
StatsTracker statsTracker = null;
bool success = false;
ITelemetryClient telemetryClient = new GitHubApplicationInsightsTelemetryClient(this.telemetryClient, context, logger);
try
{
telemetryClient.TrackEvent("SessionStart", GetCollectorCommonSessionStartEventProperties(context, identifier));
ICache <RateLimitTableEntity> rateLimiterCache = new AzureTableCache <RateLimitTableEntity>(telemetryClient, "ratelimiter");
await rateLimiterCache.InitializeAsync().ConfigureAwait(false);
ICache <ConditionalRequestTableEntity> requestsCache = new AzureTableCache <ConditionalRequestTableEntity>(telemetryClient, "requests");
await requestsCache.InitializeAsync().ConfigureAwait(false);
string organizations = await AzureHelpers.GetBlobContentAsync("github-settings", "organizations.json").ConfigureAwait(false);
JArray organizationsArray = JArray.Parse(organizations);
foreach (JToken organizationToken in organizationsArray)
{
JObject organization = (JObject)organizationToken;
string organizationLogin = organization.SelectToken("$.OrganizationLogin").Value <string>();
long organizationId = organization.SelectToken("$.OrganizationId").Value <long>();
IRateLimiter rateLimiter = new GitHubRateLimiter(this.configManager.UsesGitHubAuth(context.CollectorType) ? organizationLogin : "******", rateLimiterCache, this.httpClient, telemetryClient, maxUsageBeforeDelayStarts: 80.0, this.apiDomain);
GitHubHttpClient httpClient = new GitHubHttpClient(this.httpClient, rateLimiter, requestsCache, telemetryClient);
statsTracker = new StatsTracker(telemetryClient, httpClient, StatsTrackerRefreshFrequency);
IAuthentication authentication = this.configManager.GetAuthentication(CollectorType.TrafficTimer, httpClient, organizationLogin, this.apiDomain);
CollectorBase <GitHubCollectionNode> collector = new GitHubCollector(httpClient, authentication, telemetryClient, new List <IRecordWriter>());
GitHubCollectionNode repositoriesNode = new GitHubCollectionNode()
{
RecordType = DataContract.RepositoryInstanceRecordType,
ApiName = DataContract.RepositoriesApiName,
GetInitialUrl = additionalMetadata => OnboardingProcessor.InitialRepositoriesUrl(organizationLogin, this.apiDomain),
ProcessRecordAsync = async record =>
{
string repositoryName = record.SelectToken("$.name").Value <string>();
long repositoryId = record.SelectToken("$.id").Value <long>();
Repository repository = new Repository(organizationId, repositoryId, organizationLogin, repositoryName);
await trafficQueue.PutObjectAsJsonStringAsync(repository, TimeSpan.MaxValue).ConfigureAwait(false);
return(new List <RecordWithContext>());
},
};
await collector.ProcessAsync(repositoriesNode).ConfigureAwait(false);
}
success = true;
}
catch (Exception exception) when(!(exception is FatalException))
{
telemetryClient.TrackException(exception, "TrafficTimer failed.");
throw exception;
}
finally
{
SendSessionEndEvent(telemetryClient, context.FunctionStartDate, outputPaths: string.Empty, GetCollectorCommonSessionStartEventProperties(context, identifier), success);
statsTracker?.Stop();
}
}
