private static int RunDownloadCodeChurnFromCosmosDbToJsonFiles(DownloadFromCosmosDbCommandLineArgs a)
{
var logger = new ConsoleLoggerWithTimestamp();
var cosmosConnection = new CosmosConnection(new DatabaseFactory(a, JsonSerializerSettingsFactory.CreateDefaultSerializerSettingsForCosmosDB()), a.DatabaseId, Properties.Settings.Default.CosmosBulkBatchSize);
var dataDocumentRepository = new DataDocumentRepository(cosmosConnection, a.CodeChurnCosmosContainer);
var cosmosOutputProcessor = new CosmosDbOutputProcessor(logger, dataDocumentRepository, new DataConverter(), a.CosmosProjectName, Properties.Settings.Default.CosmosBulkBatchSize);
var jsonOutputProcessor = new JsonOutputProcessor(new DataConverter(), new FileStreamFactory(), logger);
var environment = new EnvironmentImpl();
var endDate = a.EndDate ?? (a.EndDate = environment.GetCurrentDateTime());
switch (a.DocumentType)
{
case DocumentType.BugDatabase:
{
var data = cosmosOutputProcessor.GetDocumentsInDateRange <WorkItem>(a.StartDate.Value, endDate.Value);
jsonOutputProcessor.ProcessOutput(a.OutputType, a.OutputFile, data);
break;
}
case DocumentType.CodeChurn:
{
var data = cosmosOutputProcessor.GetDocumentsInDateRange <DailyCodeChurn>(a.StartDate.Value, endDate.Value);
jsonOutputProcessor.ProcessOutput(a.OutputType, a.OutputFile, data);
break;
}
default:
throw new ArgumentOutOfRangeException();
}
return(0);
}
public async Task <IConnection <T> > GetConnection()
{
if (_connectionType == "Cosmos")
{
return(await CosmosConnection <T> .CreateAsync(_configuration));
}
return(null);
}
public void WhenBulkInsertByBatchSizeShouldCreateBatches(int numberOfDocumentsPerBatch, CosmosDocumentBase[] documents, int numberOfDocumentsInserted, int numberOfBatches)
{
sut = new CosmosConnection(databaseFactory.Object, someDatabaseId, numberOfDocumentsPerBatch);
var result = sut.BulkInsertByBatchSize(someCollectionId, documents).Result;
Assert.Equal(numberOfDocumentsInserted, result.NumberOfDocumentsInserted);
Assert.Equal(numberOfBatches, result.NumberOfBatches);
}
private static int RunSonarGenericMetricsFromCosmosDb(SonarGenericMetricsCosmosDbCommandLineArgs a)
{
var logger = new ConsoleLoggerWithTimestamp();
var jsonParser = new JsonListParser <DailyCodeChurn>(new FileStreamFactory());
var jsonExporter = new JsonExporter(new FileStreamFactory());
var cosmosConnection = new CosmosConnection(new DatabaseFactory(a, JsonSerializerSettingsFactory.CreateDefaultSerializerSettingsForCosmosDB()), a.DatabaseId, Properties.Settings.Default.CosmosBulkBatchSize);
var dataDocumentRepository = new DataDocumentRepository(cosmosConnection, a.CodeChurnCosmosContainer);
var cosmosOutputProcessor = new CosmosDbOutputProcessor(logger, dataDocumentRepository, new DataConverter(), a.CosmosProjectName, Properties.Settings.Default.CosmosBulkBatchSize);
var environment = new EnvironmentImpl();
var converters = new MeasureConverterListBuilder(environment).Build(a);
Dictionary <DateTime, Dictionary <string, DailyCodeChurn> > data;
if (a.StartDate == null && a.EndDate == null)
{
data = cosmosOutputProcessor.GetAllDocumentsByProjectNameAndDocumentType <DailyCodeChurn>();
}
else
{
if (a.StartDate == null)
{
throw new ArgumentNullException(nameof(a.StartDate));
}
if (a.EndDate == null)
{
a.EndDate = environment.GetCurrentDateTime();
}
data = cosmosOutputProcessor.GetDocumentsInDateRange <DailyCodeChurn>(a.StartDate.Value, a.EndDate.Value);
}
var processor = new SonarGenericMetricsProcessor(jsonParser, converters, jsonExporter,
new ConsoleLoggerWithTimestamp(), new DataConverter());
processor.Process(a, data);
return(0);
}
private static int RunPerforceToCosmosDbCodeChurnProcessor(P4ExtractToCosmosDbCommandLineArgs a)
{
var processWrapper = new ProcessWrapper();
var changesParser = new ChangesParser();
var describeParser = new DescribeParser();
var commandLineParser = new CommandLineParser();
var logger = new ConsoleLoggerWithTimestamp();
var stopWatch = new StopWatchWrapper();
var bugDatabaseFactory = new BugDatabaseFactory();
var bugDatabaseDllLoader = new BugDatabaseDllLoader(logger, bugDatabaseFactory);
var webRequest = new WebRequest(new HttpClientWrapperFactory(bugDatabaseFactory));
var fileSystem = new FileSystem();
var cosmosConnection = new CosmosConnection(new DatabaseFactory(a, JsonSerializerSettingsFactory.CreateDefaultSerializerSettingsForCosmosDB()), a.DatabaseId, Properties.Settings.Default.CosmosBulkBatchSize);
var dataDocumentRepository = new DataDocumentRepository(cosmosConnection, a.CodeChurnCosmosContainer);
var cosmosOutputProcessor = new CosmosDbOutputProcessor(logger, dataDocumentRepository, new DataConverter(), a.CosmosProjectName, Properties.Settings.Default.CosmosBulkBatchSize);
var bugDatabaseProcessor = new CosmosDbBugDatabaseProcessor(bugDatabaseDllLoader, fileSystem, webRequest, logger, dataDocumentRepository, a.CosmosProjectName);
var processor = new PerforceCodeChurnProcessor(processWrapper, changesParser, describeParser, commandLineParser, bugDatabaseProcessor, logger, stopWatch, cosmosOutputProcessor, a);
processor.QueryBugDatabase();
return(processor.Extract());
}
private static int RunCodeChurnFromFiles(DataFromFilesToCosmosDbCommandLineArgs a)
{
var fileSystem = new FileSystem();
var logger = new ConsoleLoggerWithTimestamp();
var cosmosConnection = new CosmosConnection(new DatabaseFactory(a, JsonSerializerSettingsFactory.CreateDefaultSerializerSettingsForCosmosDB()), a.DatabaseId, Properties.Settings.Default.CosmosBulkBatchSize);
var dataDocumentRepository = new DataDocumentRepository(cosmosConnection, a.CodeChurnCosmosContainer);
var cosmosOutputProcessor = new CosmosDbOutputProcessor(logger, dataDocumentRepository, new DataConverter(), a.CosmosProjectName, Properties.Settings.Default.CosmosBulkBatchSize);
IDataFromFileToCosmosDb codeChurnFromFileToCosmosDb;
if (a.DocumentType == DocumentType.CodeChurn)
{
codeChurnFromFileToCosmosDb = new DataFromFileToCosmosDb <DailyCodeChurn>(logger, fileSystem, cosmosOutputProcessor, new JsonListParser <DailyCodeChurn>(new FileStreamFactory()), a.Path, a.CosmosProjectName);
}
else
{
codeChurnFromFileToCosmosDb = new DataFromFileToCosmosDb <WorkItem>(logger, fileSystem, cosmosOutputProcessor, new JsonListParser <WorkItem>(new FileStreamFactory()), a.Path, a.CosmosProjectName);
}
var output = codeChurnFromFileToCosmosDb.Extract();
return(output);
}
private static int RunGitToCosmosDbCodeChurnProcessor(GitExtractToCosmosDbCommandLineArgs a)
{
var processWrapper = new ProcessWrapper();
var commandLineParser = new CommandLineParser();
var gitLogParser = new GitLogParser();
var logger = new ConsoleLoggerWithTimestamp();
var cosmosConnection = new CosmosConnection(new DatabaseFactory(a, JsonSerializerSettingsFactory.CreateDefaultSerializerSettingsForCosmosDB()), a.DatabaseId, Properties.Settings.Default.CosmosBulkBatchSize);
var dataDocumentRepository = new DataDocumentRepository(cosmosConnection, a.CodeChurnCosmosContainer);
var cosmosOutputProcessor = new CosmosDbOutputProcessor(logger, dataDocumentRepository, new DataConverter(), a.CosmosProjectName, Properties.Settings.Default.CosmosBulkBatchSize);
var bugDatabaseFactory = new BugDatabaseFactory();
var bugDatabaseDllLoader = new BugDatabaseDllLoader(logger, bugDatabaseFactory);
var webRequest = new WebRequest(new HttpClientWrapperFactory(bugDatabaseFactory));
var fileSystem = new FileSystem();
var jsonParser = new JsonListParser <WorkItem>(new FileStreamFactory());
var bugDatabaseProcessor = new BugDatabaseProcessor(bugDatabaseDllLoader, webRequest, fileSystem, jsonParser, logger, string.Empty);
//, a.BugDatabaseOutputFile
var processor = new GitCodeChurnProcessor(commandLineParser, processWrapper, gitLogParser, cosmosOutputProcessor, bugDatabaseProcessor, logger, a);
processor.QueryBugDatabase();
return(processor.Extract());
}
public static Task <CosmosConnection <T> > CreateAsync(IConfiguration configuration)
{
var ret = new CosmosConnection <T>(configuration);
return(ret.InitialiseAsync());
}
public GivenACosmosConnection()
{
someDummyDocument = new DummyCosmosDocument
{
Id = someDocumentId,
Data = someDocumentData,
ETag = someDocumentETag,
SelfLink = someSelfLink
};
jsonSerializerSettings = new JsonSerializerSettings();
someRequestOptions = new RequestOptions();
someFeedOptions = new FeedOptions();
someDocument = new Document();
someDocument.LoadFrom(new JsonTextReader(new StringReader(JsonConvert.SerializeObject(someDummyDocument))), jsonSerializerSettings);
someCollectionUri = UriFactory.CreateDocumentCollectionUri(someDatabaseId, someCollectionId);
someDocumentUri = UriFactory.CreateDocumentUri(someDatabaseId, someCollectionId, someDocumentId);
connectionPolicy = new ConnectionPolicy();
retryOptions = new Mock <RetryOptions>();
retryOptions.Object.MaxRetryAttemptsOnThrottledRequests = 9;
retryOptions.Object.MaxRetryWaitTimeInSeconds = 30;
connectionPolicy.RetryOptions = retryOptions.Object;
documentClient = new Mock <IDocumentClient>();
documentClient.Setup(x => x.ConnectionPolicy).Returns(connectionPolicy);
documentClient.Setup(x => x.CreateDocumentAsync(someCollectionUri, someDummyDocument, It.IsAny <RequestOptions>(), It.IsAny <bool>(), It.IsAny <CancellationToken>()))
.Returns(Task.FromResult(new ResourceResponse <Document>(someDocument)));
documentClient.Setup(x => x.CreateDocumentQuery <DummyCosmosDocument>(someCollectionUri, It.IsAny <FeedOptions>()))
.Returns(new List <DummyCosmosDocument> {
someDummyDocument
}.AsQueryable().OrderBy(d => d.Id));
documentClient.Setup(x => x.CreateDocumentQuery <DummyCosmosDocument>(someCollectionUri, It.IsAny <SqlQuerySpec>(), It.IsAny <FeedOptions>()))
.Returns(new List <DummyCosmosDocument> {
someDummyDocument
}.AsQueryable().OrderBy(d => d.Id));
databaseFactory = new Mock <IDatabaseFactory>();
databaseFactory.Setup(x => x.DocumentClient()).Returns(documentClient.Object);
someDocumentCollection = new DocumentCollection
{
Id = someCollectionId
};
bulkExecutor = new Mock <IBulkExecutor>();
bulkExecutorWrapper = new Mock <IBulkExecutorWrapper>();
someBulkImportResponse = new BulkImportResponse();
bulkExecutor
.Setup(b => b.BulkImportAsync(It.Is <IEnumerable <CosmosDocumentBase> >(l => l.Count() == 1), true, false, null, null, default))
.Callback(() =>
{
someBulkImportResponse.GetType().GetProperty("NumberOfDocumentsImported").SetValue(someBulkImportResponse, 1);
someBulkImportResponse.GetType().GetProperty("TotalRequestUnitsConsumed").SetValue(someBulkImportResponse, 1);
someBulkImportResponse.GetType().GetProperty("TotalTimeTaken").SetValue(someBulkImportResponse, TimeSpan.FromSeconds(10));
})
.Returns(Task.FromResult(someBulkImportResponse));
bulkExecutor
.Setup(b => b.BulkImportAsync(It.Is <IEnumerable <CosmosDocumentBase> >(l => l.Count() == 2), true, false, null, null, default))
.Callback(() =>
{
someBulkImportResponse.GetType().GetProperty("NumberOfDocumentsImported").SetValue(someBulkImportResponse, 2);
someBulkImportResponse.GetType().GetProperty("TotalRequestUnitsConsumed").SetValue(someBulkImportResponse, 2);
someBulkImportResponse.GetType().GetProperty("TotalTimeTaken").SetValue(someBulkImportResponse, TimeSpan.FromSeconds(20));
})
.Returns(Task.FromResult(someBulkImportResponse));
someDatabaseUri = UriFactory.CreateDatabaseUri(someDatabaseId);
documentClient.Setup(x => x.CreateDocumentCollectionQuery(someDatabaseUri, It.IsAny <FeedOptions>()))
.Returns(new List <DocumentCollection> {
someDocumentCollection
}.AsQueryable().OrderBy(x => x.PartitionKey));
databaseFactory.Setup(x => x.DocumentClient()).Returns(documentClient.Object);
databaseFactory.Setup(x => x.BulkExecutor(documentClient.Object, someDocumentCollection)).Returns(bulkExecutor.Object);
databaseFactory.Setup(x => x.BulkExecutorWrapper(bulkExecutor.Object)).Returns(bulkExecutorWrapper.Object);
sut = new CosmosConnection(databaseFactory.Object, someDatabaseId, someBatchBulkSize);
}
