private async Task <bool> ExecuteAsync() { using (var client = new GrafanaClient(Host, AccessToken)) using (var deploy = new DeployPublisher( grafanaClient: client, keyVaultName: KeyVaultName, keyVaultConnectionString: KeyVaultConnectionString, sourceTagValue: Tag, dashboardDirectory: DashboardDirectory, datasourceDirectory: DataSourceDirectory, notificationDirectory: NotificationDirectory, environment: Environment, parametersFile: ParametersFile, log: Log)) { try { await deploy.PostToGrafanaAsync(); } catch (HttpRequestException e) { Log.LogErrorFromException(e, showStackTrace: false, showDetail: false, file: "MonitoringPublish"); return(false); } } return(true); }
private async Task <bool> ExecuteAsync() { using (var client = new GrafanaClient(Host, AccessToken)) { var deploy = new DeployImporter( client, Tag, DashboardDirectory, DataSourceDirectory, NotificationDirectory, Environments.Select(e => e.ItemSpec).ToArray(), Log); try { await deploy.ImportFromGrafana(DashboardId); } catch (HttpRequestException e) { Log.LogErrorFromException(e, showStackTrace: false, showDetail: false, file: "MonitoringImport"); return(false); } } return(true); }
private async Task <bool> ExecuteAsync() { using (var client = new GrafanaClient(Host, AccessToken)) { var deploy = new DeployImporter( grafanaClient: client, sourceTagValue: Tag, dashboardDirectory: DashboardDirectory, datasourceDirectory: DataSourceDirectory, notificationDirectory: NotificationDirectory, environments: Environments.Select(e => e.ItemSpec).ToArray(), parametersFilePath: ParametersFile, environment: Environment, log: Log); try { await deploy.ImportFromGrafana(DashboardId); } catch (HttpRequestException e) { Log.LogErrorFromException(e, showStackTrace: false, showDetail: false, file: "MonitoringImport"); return(false); } } return(true); }
public DeployImporter( GrafanaClient grafanaClient, string sourceTagValue, string dashboardDirectory, string datasourceDirectory, string notificationDirectory, string[] environments, TaskLoggingHelper log) : base( grafanaClient, sourceTagValue, dashboardDirectory, datasourceDirectory, notificationDirectory, log) { _environments = environments; }
private async Task ClearExtraneousDashboardsAsync(HashSet <string> knownUids) { JArray allTagged = await GrafanaClient.SearchDashboardsByTagAsync(SourceTag).ConfigureAwait(false); HashSet <string> toRemove = new HashSet <string>(allTagged.Where(IsManagedDashboard).Select(d => d.Value <string>("uid"))); // We shouldn't remove the ones we just deployed toRemove.ExceptWith(knownUids); foreach (string uid in toRemove) { Log.LogMessage(MessageImportance.Normal, "Deleting extra dashboard {0}...", uid); await GrafanaClient.DeleteDashboardAsync(uid).ConfigureAwait(false); } }
protected DeployToolBase( GrafanaClient grafanaClient, string sourceTagValue, string dashboardDirectory, string datasourceDirectory, string notificationDirectory, TaskLoggingHelper log) { GrafanaClient = grafanaClient; _sourceTagValue = sourceTagValue; DashboardDirectory = dashboardDirectory; DatasourceDirectory = datasourceDirectory; NotificationDirectory = notificationDirectory; Log = log; }
public DeployPublisher( GrafanaClient grafanaClient, string keyVaultName, string keyVaultConnectionString, string sourceTagValue, string dashboardDirectory, string datasourceDirectory, string notificationDirectory, string environment, TaskLoggingHelper log) : base( grafanaClient, sourceTagValue, dashboardDirectory, datasourceDirectory, notificationDirectory, log) { _keyVaultName = keyVaultName; _keyVaultConnectionString = keyVaultConnectionString; _environment = environment; _keyVault = new Lazy <KeyVaultClient>(GetKeyVaultClient); }
private async Task <bool> ExecuteAsync() { using (var client = new GrafanaClient(Host, AccessToken)) using (var deploy = new DeployPublisher(client, KeyVaultName, KeyVaultConnectionString, Tag, DashboardDirectory, DataSourceDirectory, NotificationDirectory, Environment, Log)) { try { await deploy.PostToGrafanaAsync(); } catch (HttpRequestException e) { Log.LogErrorFromException(e, showStackTrace: false, showDetail: false, file: "MonitoringPublish"); return(false); } } return(true); }
private async Task PostNotificationsAsync() { foreach (string notificationPath in Directory.GetFiles(EnvironmentNotificationDirectory, "*" + NotificationExtension, SearchOption.AllDirectories)) { string uid = GetUidFromNotificationFile(Path.GetFileName(notificationPath)); JObject data; using (var sr = new StreamReader(notificationPath)) using (var jr = new JsonTextReader(sr)) { data = await JObject.LoadAsync(jr).ConfigureAwait(false); } data["uid"] = uid; Log.LogMessage(MessageImportance.Normal, "Posting notification {0}...", uid); await ReplaceVaultAsync(data); await GrafanaClient.CreateNotificationChannelAsync(data).ConfigureAwait(false); } }
private async Task PostDatasourcesAsync() { foreach (string datasourcePath in Directory.GetFiles(EnvironmentDatasourceDirectory, "*" + DatasourceExtension, SearchOption.AllDirectories)) { var name = GetNameFromDatasourceFile(Path.GetFileName(datasourcePath)); JObject data; using (var sr = new StreamReader(datasourcePath)) using (var jr = new JsonTextReader(sr)) { data = await JObject.LoadAsync(jr).ConfigureAwait(false); } data["name"] = name; Log.LogMessage(MessageImportance.Normal, "Posting datasource {0}...", name); await ReplaceVaultAsync(data); await GrafanaClient.CreateDatasourceAsync(data).ConfigureAwait(false); } }
private async Task PostDashboardsAsync() { JArray folderArray = await GrafanaClient.ListFoldersAsync().ConfigureAwait(false); List <FolderData> folders = folderArray.Select(f => new FolderData(f.Value <string>("uid"), f.Value <string>("title"))) .ToList(); var knownUids = new HashSet <string>(); foreach (string dashboardPath in GetAllDashboardPaths()) { string folderName = Path.GetFileName(Path.GetDirectoryName(dashboardPath)); string dashboardFileName = Path.GetFileName(dashboardPath); string uid = GetUidFromDashboardFile(dashboardFileName); knownUids.Add(uid); FolderData folder = folders.FirstOrDefault(f => f.Title == folderName); JObject result = await GrafanaClient.CreateFolderAsync(folderName, folderName).ConfigureAwait(false); string folderUid = result["uid"].Value <string>(); int folderId = result["id"].Value <int>(); if (folder == null) { folder = new FolderData(folderUid, folderName); } folder.Id = folderId; JObject data; using (var sr = new StreamReader(dashboardPath)) using (var jr = new JsonTextReader(sr)) { data = await JObject.LoadAsync(jr).ConfigureAwait(false); } JArray tagArray = null; if (data.TryGetValue("tags", out JToken tagToken)) { tagArray = tagToken as JArray; } if (tagArray == null) { tagArray = new JArray(); } var newTags = new JArray(); foreach (JToken tag in tagArray) { if (tag.Value <string>().StartsWith(BaseUidTagPrefix) || tag.Value <string>().StartsWith(SourceTagPrefix)) { continue; } newTags.Add(tag); } tagArray.Add(GetUidTag(uid)); tagArray.Add(SourceTag); data["tags"] = newTags; data["uid"] = uid; Log.LogMessage(MessageImportance.Normal, "Posting dashboard {0}...", uid); await GrafanaClient.CreateDashboardAsync(data, folderId).ConfigureAwait(false); } await ClearExtraneousDashboardsAsync(knownUids); }
public async Task ImportFromGrafana(IEnumerable <string> dashboardUids) { HashSet <string> usedNotificationIds = new HashSet <string>(); foreach (var dashboardPath in GetAllDashboardPaths()) { if (dashboardUids.Any(d => d.Contains(GetUidFromDashboardFile(dashboardPath)))) { // This is a new dashboard, don't assume it alert tags JObject data; using (var sr = new StreamReader(dashboardPath)) using (var jr = new JsonTextReader(sr)) { data = await JObject.LoadAsync(jr).ConfigureAwait(false); } IEnumerable <string> notificationIds = data.SelectTokens("..alertRuleTags.NotificationId") .Select(d => d.Value <string>()); foreach (var alertId in notificationIds) { usedNotificationIds.Add(alertId); } } } List <Parameter> parameters; if (File.Exists(_parametersFilePath)) { using (var sr = new StreamReader(_parametersFilePath)) using (var jr = new JsonTextReader(sr)) { parameters = _serializer.Deserialize <List <Parameter> >(jr); } } else { parameters = new List <Parameter>(); } foreach (string dashboardUid in dashboardUids) { // Get a dashboard JObject dashboard = await GrafanaClient.GetDashboardAsync(dashboardUid).ConfigureAwait(false); // Cache dashboard data needed for post string targetUid = dashboard.Value <JObject>("dashboard").Value <string>("uid"); // SanitizeDashboard makes structural changes to the JSON and ParameterizeDashboard // expects a certain structure, so order matters when calling these methods. JObject parameterizedDashboard = GrafanaSerialization.ParameterizeDashboard(dashboard, parameters, _environments, _environment); JObject slimmedDashboard = GrafanaSerialization.SanitizeDashboard(parameterizedDashboard); UpdateNotificationIds(slimmedDashboard, usedNotificationIds); JArray tags = slimmedDashboard.Value <JArray>("tags"); JToken uidTag = tags.FirstOrDefault(t => t.Value <string>().StartsWith(BaseUidTagPrefix)); if (uidTag != null) { targetUid = uidTag.Value <string>().Substring(BaseUidTagPrefix.Length); } // Get the dashboard folder data and cache if not already present int folderId = GrafanaSerialization.ExtractFolderId(dashboard); JObject folder = await GrafanaClient.GetFolderAsync(folderId).ConfigureAwait(false); FolderData folderData = GrafanaSerialization.SanitizeFolder(folder); // Get datasources used in the dashboard IEnumerable <string> dataSourceIdentifiers = GrafanaSerialization.ExtractDataSourceIdentifiers(dashboard); foreach (string dataSourceIdentifier in dataSourceIdentifiers) { JObject datasource = await GrafanaClient.GetDataSourceByUidAsync(dataSourceIdentifier).ConfigureAwait(false) ?? await GrafanaClient.GetDataSourceByNameAsync(dataSourceIdentifier).ConfigureAwait(false); string datasourceName = datasource.Value <string>("name"); datasource = GrafanaSerialization.SanitizeDataSource(datasource); // Create the data source for each environment foreach (string env in _environments) { string datasourcePath = GetDatasourcePath(env, datasourceName); if (File.Exists(datasourcePath)) { // If we already have that datasource, don't overwrite it's useful values with empty ones continue; } Log.LogMessage(MessageImportance.Normal, "Importing datasource {0}...", datasourceName); Directory.CreateDirectory(Path.GetDirectoryName(datasourcePath)); using (var datasourceStreamWriter = new StreamWriter(datasourcePath)) using (var datasourceJsonWriter = new JsonTextWriter(datasourceStreamWriter)) { _serializer.Serialize(datasourceJsonWriter, datasource); } } } HashSet <string> usedNotifications = new HashSet <string>(slimmedDashboard .SelectTokens("panels.[*].alert.notifications.[*].uid") .Select(t => t.Value <string>()) .Where(uid => !string.IsNullOrEmpty(uid))); foreach (string notificationUid in usedNotifications) { JObject notificationChannel = await GrafanaClient.GetNotificationChannelAsync(notificationUid); notificationChannel = GrafanaSerialization.SanitizeNotificationChannel(notificationChannel); // Create the data source for each environment foreach (string env in _environments) { string notificationPath = GetNotificationPath(env, notificationUid); if (File.Exists(notificationPath)) { // If we already have that notification, don't overwrite it's useful values with empty ones continue; } Log.LogMessage(MessageImportance.Normal, "Importing notification channel {0}...", notificationUid); if (notificationChannel.ContainsKey("password")) { var password = notificationChannel.GetValue("password").Value <string>(); if (!password.Contains("vault")) { Log.LogWarning($"Please replace the password token with a key vault reference inside {notificationPath}"); } } Directory.CreateDirectory(Path.GetDirectoryName(notificationPath)); using (var datasourceStreamWriter = new StreamWriter(notificationPath)) using (var datasourceJsonWriter = new JsonTextWriter(datasourceStreamWriter)) { _serializer.Serialize(datasourceJsonWriter, notificationChannel); } } } Log.LogMessage(MessageImportance.Normal, "Importing dashboard {0}...", targetUid); string dashboardPath = GetDashboardFilePath(folderData.Title, targetUid); Directory.CreateDirectory(Path.GetDirectoryName(dashboardPath)); using (var dashboardStreamWriter = new StreamWriter(dashboardPath)) using (var dashboardJsonWriter = new JsonTextWriter(dashboardStreamWriter)) { _serializer.Serialize(dashboardJsonWriter, slimmedDashboard); } } // Save parameters back to disk Log.LogMessage(MessageImportance.Normal, "Saving parameters {0}...", _parametersFilePath); using (var sr = new StreamWriter(_parametersFilePath)) using (var jr = new JsonTextWriter(sr)) { _serializer.Serialize(jr, parameters); } }
public async Task ImportFromGrafana(IEnumerable <string> dashboardUids) { HashSet <string> usedNotificationIds = new HashSet <string>(); foreach (var dashboardPath in GetAllDashboardPaths()) { if (dashboardUids.Any(d => d.Contains(GetUidFromDashboardFile(dashboardPath)))) { // This is a new dashboard, don't assume it alert tags JObject data; using (var sr = new StreamReader(dashboardPath)) using (var jr = new JsonTextReader(sr)) { data = await JObject.LoadAsync(jr).ConfigureAwait(false); } IEnumerable <string> notificationIds = data.SelectTokens("..alertRuleTags.NotificationId") .Select(d => d.Value <string>()); foreach (var alertId in notificationIds) { usedNotificationIds.Add(alertId); } } } foreach (string dashboardUid in dashboardUids) { // Get a dashboard JObject dashboard = await GrafanaClient.GetDashboardAsync(dashboardUid).ConfigureAwait(false); // Cache dashboard data needed for post string targetUid = dashboard.Value <JObject>("dashboard").Value <string>("uid"); JObject slimmedDashboard = GrafanaSerialization.SanitizeDashboard(dashboard); UpdateNotificationIds(slimmedDashboard, usedNotificationIds); JArray tags = slimmedDashboard.Value <JArray>("tags"); JToken uidTag = tags.FirstOrDefault(t => t.Value <string>().StartsWith(BaseUidTagPrefix)); if (uidTag != null) { targetUid = uidTag.Value <string>().Substring(BaseUidTagPrefix.Length); } // Get the dashboard folder data and cache if not already present int folderId = GrafanaSerialization.ExtractFolderId(dashboard); JObject folder = await GrafanaClient.GetFolderAsync(folderId).ConfigureAwait(false); FolderData folderData = GrafanaSerialization.SanitizeFolder(folder); // Get datasources used in the dashboard IEnumerable <string> dataSourceNames = GrafanaSerialization.ExtractDataSourceNames(dashboard); foreach (string datasourceName in dataSourceNames) { JObject datasource = await GrafanaClient.GetDataSourceAsync(datasourceName).ConfigureAwait(false); datasource = GrafanaSerialization.SanitizeDataSource(datasource); // Create the data source for each environment foreach (string env in _environments) { string datasourcePath = GetDatasourcePath(env, datasourceName); if (File.Exists(datasourcePath)) { // If we already have that datasource, don't overwrite it's useful values with empty ones continue; } Log.LogMessage(MessageImportance.Normal, "Importing datasource {0}...", datasourceName); Directory.CreateDirectory(Path.GetDirectoryName(datasourcePath)); using (var datasourceStreamWriter = new StreamWriter(datasourcePath)) using (var datasourceJsonWriter = new JsonTextWriter(datasourceStreamWriter)) { _serializer.Serialize(datasourceJsonWriter, datasource); } } } HashSet <string> usedNotifications = new HashSet <string>(slimmedDashboard .SelectTokens("panels.[*].alert.notifications.[*].uid") .Select(t => t.Value <string>()) .Where(uid => !string.IsNullOrEmpty(uid))); foreach (string notificationUid in usedNotifications) { JObject notificationChannel = await GrafanaClient.GetNotificationChannelAsync(notificationUid); notificationChannel = GrafanaSerialization.SanitizeNotificationChannel(notificationChannel); // Create the data source for each environment foreach (string env in _environments) { string notificationPath = GetNotificationPath(env, notificationUid); if (File.Exists(notificationPath)) { // If we already have that notification, don't overwrite it's useful values with empty ones continue; } Log.LogMessage(MessageImportance.Normal, "Importing notification channel {0}...", notificationUid); Directory.CreateDirectory(Path.GetDirectoryName(notificationPath)); using (var datasourceStreamWriter = new StreamWriter(notificationPath)) using (var datasourceJsonWriter = new JsonTextWriter(datasourceStreamWriter)) { _serializer.Serialize(datasourceJsonWriter, notificationChannel); } } } Log.LogMessage(MessageImportance.Normal, "Importing dashboard {0}...", targetUid); string dashboardPath = GetDashboardFilePath(folderData.Title, targetUid); Directory.CreateDirectory(Path.GetDirectoryName(dashboardPath)); using (var dashboardStreamWriter = new StreamWriter(dashboardPath)) using (var dashboardJsonWriter = new JsonTextWriter(dashboardStreamWriter)) { _serializer.Serialize(dashboardJsonWriter, slimmedDashboard); } } }