public void IngestData(AzDevopsWaterMark azureAzDevopsWaterMark) { try { var waterMark = azureAzDevopsWaterMark.ReadWaterMark(this.table); int continuationToken; DateTime minCreatedDateTime; int count = 0; int totalCount; do { using (var memStream = new MemoryStream()) using (var writer = new StreamWriter(memStream)) { // Write data to table WriteData(writer, waterMark, out continuationToken, out minCreatedDateTime, out totalCount); writer.Flush(); memStream.Seek(0, SeekOrigin.Begin); this.IngestData(table, mappingName, memStream); } waterMark = string.Format("{0},{1}", continuationToken, minCreatedDateTime); azureAzDevopsWaterMark.UpdateWaterMark(table, waterMark); count++; } while (totalCount > 0); } catch (Exception ex) { string error = $"Not able to ingest Releasetimelinerecord entity due to {ex}"; Logger.Error(error); } }
public void IngestData(AzDevopsWaterMark azureAzDevopsWaterMark) { try { var waterMark = azureAzDevopsWaterMark.ReadWaterMark(this._table); int continuationToken; using (var memStream = new MemoryStream()) using (var writer = new StreamWriter(memStream)) { // Write data to table WriteData(writer, waterMark, out continuationToken); writer.Flush(); memStream.Seek(0, SeekOrigin.Begin); this.IngestData(_table, _mappingName, memStream); } waterMark = $"{continuationToken}"; azureAzDevopsWaterMark.UpdateWaterMark(_table, waterMark); } catch (Exception ex) { string error = $"Not able to ingest ReleaseDefinition entity due to {ex}"; Logger.Error(error); } }
static void Main(string[] args) { try { string alias = args[0]; // "*****@*****.**"; string token = args[1]; //"PAT_TOKEN"; string kustoConnectionString = args[2]; //"axexperiments.southeastasia.kusto.windows.net"; string aadTenantIdOrTenantName = args[3]; //"microsoft.com" or tenant GUID; string organizationName = args[4]; //"mseng"; string projectName = args[5]; //"AzureDevops"; ProjectHttpClient projectHttpClient = new ProjectHttpClient( new Uri($"https://dev.azure.com/{organizationName}"), new VssCredentials(new VssBasicCredential(alias, token))); IKustoClientFactory kustoClientFactory = new KustoClientFactory(aadTenantIdOrTenantName, kustoConnectionString); var azDevopsReleaseProvider = new ReleaseRestAPIProvider(alias, token, organizationName, projectName); var azDevopsProjectsProvider = new ProjectRestAPIProvider(projectHttpClient, projectName); var projectId = azDevopsProjectsProvider.GetProjectInfo(projectName).Id.ToString(); var axAzDevopsWaterMark = new AzDevopsWaterMark(kustoClientFactory, organizationName, projectId); var azDevopsDeploymentIngestor = new AzDevopsReleaseDeployment(azDevopsReleaseProvider, kustoClientFactory, organizationName, projectId); var azDevopsArtifactIngestor = new AzDevopsReleaseArtifact(azDevopsReleaseProvider, kustoClientFactory, organizationName, projectId); var azDevopReleaseDefinitionIngestor = new AzDevopsReleaseDefinition(azDevopsReleaseProvider, kustoClientFactory, organizationName, projectId); var azDevopsReleaseIngestor = new AzDevopsRelease(azDevopsReleaseProvider, kustoClientFactory, organizationName, projectId); var azDevopsReleaseEnvironmentIngestor = new AzDevopsReleaseEnvironment(azDevopsReleaseProvider, kustoClientFactory, organizationName, projectId); //var azDevopsReleaseTimelineRecordIngestor = new AzDevopsReleaseTimelineRecord(azDevopsReleaseProvider, kustoClientFactory, organizationName, projectId); Logger.Info("Ingestion started for Release Entities"); Parallel.Invoke( () => azDevopReleaseDefinitionIngestor.IngestData(axAzDevopsWaterMark), () => azDevopsDeploymentIngestor.IngestData(axAzDevopsWaterMark), () => azDevopsReleaseIngestor.IngestData(axAzDevopsWaterMark), () => azDevopsArtifactIngestor.IngestData(axAzDevopsWaterMark), () => azDevopsReleaseEnvironmentIngestor.IngestData(axAzDevopsWaterMark) //() => azDevopsReleaseTimelineRecordIngestor.IngestData(axAzDevopsWaterMark) ); Logger.Info("Ingestion completed. It may take around 5 minutes to reflect whole data in Azure Data explorer"); Console.ReadKey(); } catch (Exception ex) { Logger.Error($"Not able to complete ingesting due to error {ex}"); } }