private static int RunDownloadCodeChurnFromCosmosDbToJsonFiles(DownloadFromCosmosDbCommandLineArgs a) { var logger = new ConsoleLoggerWithTimestamp(); var cosmosConnection = new CosmosConnection(new DatabaseFactory(a, JsonSerializerSettingsFactory.CreateDefaultSerializerSettingsForCosmosDB()), a.DatabaseId, Properties.Settings.Default.CosmosBulkBatchSize); var dataDocumentRepository = new DataDocumentRepository(cosmosConnection, a.CodeChurnCosmosContainer); var cosmosOutputProcessor = new CosmosDbOutputProcessor(logger, dataDocumentRepository, new DataConverter(), a.CosmosProjectName, Properties.Settings.Default.CosmosBulkBatchSize); var jsonOutputProcessor = new JsonOutputProcessor(new DataConverter(), new FileStreamFactory(), logger); var environment = new EnvironmentImpl(); var endDate = a.EndDate ?? (a.EndDate = environment.GetCurrentDateTime()); switch (a.DocumentType) { case DocumentType.BugDatabase: { var data = cosmosOutputProcessor.GetDocumentsInDateRange <WorkItem>(a.StartDate.Value, endDate.Value); jsonOutputProcessor.ProcessOutput(a.OutputType, a.OutputFile, data); break; } case DocumentType.CodeChurn: { var data = cosmosOutputProcessor.GetDocumentsInDateRange <DailyCodeChurn>(a.StartDate.Value, endDate.Value); jsonOutputProcessor.ProcessOutput(a.OutputType, a.OutputFile, data); break; } default: throw new ArgumentOutOfRangeException(); } return(0); }
public async Task <IConnection <T> > GetConnection() { if (_connectionType == "Cosmos") { return(await CosmosConnection <T> .CreateAsync(_configuration)); } return(null); }
public void WhenBulkInsertByBatchSizeShouldCreateBatches(int numberOfDocumentsPerBatch, CosmosDocumentBase[] documents, int numberOfDocumentsInserted, int numberOfBatches) { sut = new CosmosConnection(databaseFactory.Object, someDatabaseId, numberOfDocumentsPerBatch); var result = sut.BulkInsertByBatchSize(someCollectionId, documents).Result; Assert.Equal(numberOfDocumentsInserted, result.NumberOfDocumentsInserted); Assert.Equal(numberOfBatches, result.NumberOfBatches); }
private static int RunSonarGenericMetricsFromCosmosDb(SonarGenericMetricsCosmosDbCommandLineArgs a) { var logger = new ConsoleLoggerWithTimestamp(); var jsonParser = new JsonListParser <DailyCodeChurn>(new FileStreamFactory()); var jsonExporter = new JsonExporter(new FileStreamFactory()); var cosmosConnection = new CosmosConnection(new DatabaseFactory(a, JsonSerializerSettingsFactory.CreateDefaultSerializerSettingsForCosmosDB()), a.DatabaseId, Properties.Settings.Default.CosmosBulkBatchSize); var dataDocumentRepository = new DataDocumentRepository(cosmosConnection, a.CodeChurnCosmosContainer); var cosmosOutputProcessor = new CosmosDbOutputProcessor(logger, dataDocumentRepository, new DataConverter(), a.CosmosProjectName, Properties.Settings.Default.CosmosBulkBatchSize); var environment = new EnvironmentImpl(); var converters = new MeasureConverterListBuilder(environment).Build(a); Dictionary <DateTime, Dictionary <string, DailyCodeChurn> > data; if (a.StartDate == null && a.EndDate == null) { data = cosmosOutputProcessor.GetAllDocumentsByProjectNameAndDocumentType <DailyCodeChurn>(); } else { if (a.StartDate == null) { throw new ArgumentNullException(nameof(a.StartDate)); } if (a.EndDate == null) { a.EndDate = environment.GetCurrentDateTime(); } data = cosmosOutputProcessor.GetDocumentsInDateRange <DailyCodeChurn>(a.StartDate.Value, a.EndDate.Value); } var processor = new SonarGenericMetricsProcessor(jsonParser, converters, jsonExporter, new ConsoleLoggerWithTimestamp(), new DataConverter()); processor.Process(a, data); return(0); }
private static int RunPerforceToCosmosDbCodeChurnProcessor(P4ExtractToCosmosDbCommandLineArgs a) { var processWrapper = new ProcessWrapper(); var changesParser = new ChangesParser(); var describeParser = new DescribeParser(); var commandLineParser = new CommandLineParser(); var logger = new ConsoleLoggerWithTimestamp(); var stopWatch = new StopWatchWrapper(); var bugDatabaseFactory = new BugDatabaseFactory(); var bugDatabaseDllLoader = new BugDatabaseDllLoader(logger, bugDatabaseFactory); var webRequest = new WebRequest(new HttpClientWrapperFactory(bugDatabaseFactory)); var fileSystem = new FileSystem(); var cosmosConnection = new CosmosConnection(new DatabaseFactory(a, JsonSerializerSettingsFactory.CreateDefaultSerializerSettingsForCosmosDB()), a.DatabaseId, Properties.Settings.Default.CosmosBulkBatchSize); var dataDocumentRepository = new DataDocumentRepository(cosmosConnection, a.CodeChurnCosmosContainer); var cosmosOutputProcessor = new CosmosDbOutputProcessor(logger, dataDocumentRepository, new DataConverter(), a.CosmosProjectName, Properties.Settings.Default.CosmosBulkBatchSize); var bugDatabaseProcessor = new CosmosDbBugDatabaseProcessor(bugDatabaseDllLoader, fileSystem, webRequest, logger, dataDocumentRepository, a.CosmosProjectName); var processor = new PerforceCodeChurnProcessor(processWrapper, changesParser, describeParser, commandLineParser, bugDatabaseProcessor, logger, stopWatch, cosmosOutputProcessor, a); processor.QueryBugDatabase(); return(processor.Extract()); }
private static int RunCodeChurnFromFiles(DataFromFilesToCosmosDbCommandLineArgs a) { var fileSystem = new FileSystem(); var logger = new ConsoleLoggerWithTimestamp(); var cosmosConnection = new CosmosConnection(new DatabaseFactory(a, JsonSerializerSettingsFactory.CreateDefaultSerializerSettingsForCosmosDB()), a.DatabaseId, Properties.Settings.Default.CosmosBulkBatchSize); var dataDocumentRepository = new DataDocumentRepository(cosmosConnection, a.CodeChurnCosmosContainer); var cosmosOutputProcessor = new CosmosDbOutputProcessor(logger, dataDocumentRepository, new DataConverter(), a.CosmosProjectName, Properties.Settings.Default.CosmosBulkBatchSize); IDataFromFileToCosmosDb codeChurnFromFileToCosmosDb; if (a.DocumentType == DocumentType.CodeChurn) { codeChurnFromFileToCosmosDb = new DataFromFileToCosmosDb <DailyCodeChurn>(logger, fileSystem, cosmosOutputProcessor, new JsonListParser <DailyCodeChurn>(new FileStreamFactory()), a.Path, a.CosmosProjectName); } else { codeChurnFromFileToCosmosDb = new DataFromFileToCosmosDb <WorkItem>(logger, fileSystem, cosmosOutputProcessor, new JsonListParser <WorkItem>(new FileStreamFactory()), a.Path, a.CosmosProjectName); } var output = codeChurnFromFileToCosmosDb.Extract(); return(output); }
private static int RunGitToCosmosDbCodeChurnProcessor(GitExtractToCosmosDbCommandLineArgs a) { var processWrapper = new ProcessWrapper(); var commandLineParser = new CommandLineParser(); var gitLogParser = new GitLogParser(); var logger = new ConsoleLoggerWithTimestamp(); var cosmosConnection = new CosmosConnection(new DatabaseFactory(a, JsonSerializerSettingsFactory.CreateDefaultSerializerSettingsForCosmosDB()), a.DatabaseId, Properties.Settings.Default.CosmosBulkBatchSize); var dataDocumentRepository = new DataDocumentRepository(cosmosConnection, a.CodeChurnCosmosContainer); var cosmosOutputProcessor = new CosmosDbOutputProcessor(logger, dataDocumentRepository, new DataConverter(), a.CosmosProjectName, Properties.Settings.Default.CosmosBulkBatchSize); var bugDatabaseFactory = new BugDatabaseFactory(); var bugDatabaseDllLoader = new BugDatabaseDllLoader(logger, bugDatabaseFactory); var webRequest = new WebRequest(new HttpClientWrapperFactory(bugDatabaseFactory)); var fileSystem = new FileSystem(); var jsonParser = new JsonListParser <WorkItem>(new FileStreamFactory()); var bugDatabaseProcessor = new BugDatabaseProcessor(bugDatabaseDllLoader, webRequest, fileSystem, jsonParser, logger, string.Empty); //, a.BugDatabaseOutputFile var processor = new GitCodeChurnProcessor(commandLineParser, processWrapper, gitLogParser, cosmosOutputProcessor, bugDatabaseProcessor, logger, a); processor.QueryBugDatabase(); return(processor.Extract()); }
public static Task <CosmosConnection <T> > CreateAsync(IConfiguration configuration) { var ret = new CosmosConnection <T>(configuration); return(ret.InitialiseAsync()); }
public GivenACosmosConnection() { someDummyDocument = new DummyCosmosDocument { Id = someDocumentId, Data = someDocumentData, ETag = someDocumentETag, SelfLink = someSelfLink }; jsonSerializerSettings = new JsonSerializerSettings(); someRequestOptions = new RequestOptions(); someFeedOptions = new FeedOptions(); someDocument = new Document(); someDocument.LoadFrom(new JsonTextReader(new StringReader(JsonConvert.SerializeObject(someDummyDocument))), jsonSerializerSettings); someCollectionUri = UriFactory.CreateDocumentCollectionUri(someDatabaseId, someCollectionId); someDocumentUri = UriFactory.CreateDocumentUri(someDatabaseId, someCollectionId, someDocumentId); connectionPolicy = new ConnectionPolicy(); retryOptions = new Mock <RetryOptions>(); retryOptions.Object.MaxRetryAttemptsOnThrottledRequests = 9; retryOptions.Object.MaxRetryWaitTimeInSeconds = 30; connectionPolicy.RetryOptions = retryOptions.Object; documentClient = new Mock <IDocumentClient>(); documentClient.Setup(x => x.ConnectionPolicy).Returns(connectionPolicy); documentClient.Setup(x => x.CreateDocumentAsync(someCollectionUri, someDummyDocument, It.IsAny <RequestOptions>(), It.IsAny <bool>(), It.IsAny <CancellationToken>())) .Returns(Task.FromResult(new ResourceResponse <Document>(someDocument))); documentClient.Setup(x => x.CreateDocumentQuery <DummyCosmosDocument>(someCollectionUri, It.IsAny <FeedOptions>())) .Returns(new List <DummyCosmosDocument> { someDummyDocument }.AsQueryable().OrderBy(d => d.Id)); documentClient.Setup(x => x.CreateDocumentQuery <DummyCosmosDocument>(someCollectionUri, It.IsAny <SqlQuerySpec>(), It.IsAny <FeedOptions>())) .Returns(new List <DummyCosmosDocument> { someDummyDocument }.AsQueryable().OrderBy(d => d.Id)); databaseFactory = new Mock <IDatabaseFactory>(); databaseFactory.Setup(x => x.DocumentClient()).Returns(documentClient.Object); someDocumentCollection = new DocumentCollection { Id = someCollectionId }; bulkExecutor = new Mock <IBulkExecutor>(); bulkExecutorWrapper = new Mock <IBulkExecutorWrapper>(); someBulkImportResponse = new BulkImportResponse(); bulkExecutor .Setup(b => b.BulkImportAsync(It.Is <IEnumerable <CosmosDocumentBase> >(l => l.Count() == 1), true, false, null, null, default)) .Callback(() => { someBulkImportResponse.GetType().GetProperty("NumberOfDocumentsImported").SetValue(someBulkImportResponse, 1); someBulkImportResponse.GetType().GetProperty("TotalRequestUnitsConsumed").SetValue(someBulkImportResponse, 1); someBulkImportResponse.GetType().GetProperty("TotalTimeTaken").SetValue(someBulkImportResponse, TimeSpan.FromSeconds(10)); }) .Returns(Task.FromResult(someBulkImportResponse)); bulkExecutor .Setup(b => b.BulkImportAsync(It.Is <IEnumerable <CosmosDocumentBase> >(l => l.Count() == 2), true, false, null, null, default)) .Callback(() => { someBulkImportResponse.GetType().GetProperty("NumberOfDocumentsImported").SetValue(someBulkImportResponse, 2); someBulkImportResponse.GetType().GetProperty("TotalRequestUnitsConsumed").SetValue(someBulkImportResponse, 2); someBulkImportResponse.GetType().GetProperty("TotalTimeTaken").SetValue(someBulkImportResponse, TimeSpan.FromSeconds(20)); }) .Returns(Task.FromResult(someBulkImportResponse)); someDatabaseUri = UriFactory.CreateDatabaseUri(someDatabaseId); documentClient.Setup(x => x.CreateDocumentCollectionQuery(someDatabaseUri, It.IsAny <FeedOptions>())) .Returns(new List <DocumentCollection> { someDocumentCollection }.AsQueryable().OrderBy(x => x.PartitionKey)); databaseFactory.Setup(x => x.DocumentClient()).Returns(documentClient.Object); databaseFactory.Setup(x => x.BulkExecutor(documentClient.Object, someDocumentCollection)).Returns(bulkExecutor.Object); databaseFactory.Setup(x => x.BulkExecutorWrapper(bulkExecutor.Object)).Returns(bulkExecutorWrapper.Object); sut = new CosmosConnection(databaseFactory.Object, someDatabaseId, someBatchBulkSize); }