private async Task <bool> ExecuteAsync()
{
using (var client = new GrafanaClient(Host, AccessToken))
using (var deploy = new DeployPublisher(
grafanaClient: client,
keyVaultName: KeyVaultName,
keyVaultConnectionString: KeyVaultConnectionString,
sourceTagValue: Tag,
dashboardDirectory: DashboardDirectory,
datasourceDirectory: DataSourceDirectory,
notificationDirectory: NotificationDirectory,
environment: Environment,
parametersFile: ParametersFile,
log: Log))
{
try
{
await deploy.PostToGrafanaAsync();
}
catch (HttpRequestException e)
{
Log.LogErrorFromException(e, showStackTrace: false, showDetail: false, file: "MonitoringPublish");
return(false);
}
}
return(true);
}
private async Task <bool> ExecuteAsync()
{
using (var client = new GrafanaClient(Host, AccessToken))
{
var deploy = new DeployImporter(
client,
Tag,
DashboardDirectory,
DataSourceDirectory,
NotificationDirectory,
Environments.Select(e => e.ItemSpec).ToArray(),
Log);
try
{
await deploy.ImportFromGrafana(DashboardId);
}
catch (HttpRequestException e)
{
Log.LogErrorFromException(e,
showStackTrace: false,
showDetail: false,
file: "MonitoringImport");
return(false);
}
}
return(true);
}
private async Task <bool> ExecuteAsync()
{
using (var client = new GrafanaClient(Host, AccessToken))
{
var deploy = new DeployImporter(
grafanaClient: client,
sourceTagValue: Tag,
dashboardDirectory: DashboardDirectory,
datasourceDirectory: DataSourceDirectory,
notificationDirectory: NotificationDirectory,
environments: Environments.Select(e => e.ItemSpec).ToArray(),
parametersFilePath: ParametersFile,
environment: Environment,
log: Log);
try
{
await deploy.ImportFromGrafana(DashboardId);
}
catch (HttpRequestException e)
{
Log.LogErrorFromException(e,
showStackTrace: false,
showDetail: false,
file: "MonitoringImport");
return(false);
}
}
return(true);
}
public DeployImporter(
GrafanaClient grafanaClient,
string sourceTagValue,
string dashboardDirectory,
string datasourceDirectory,
string notificationDirectory,
string[] environments,
TaskLoggingHelper log) : base(
grafanaClient, sourceTagValue, dashboardDirectory, datasourceDirectory, notificationDirectory, log)
{
_environments = environments;
}
private async Task ClearExtraneousDashboardsAsync(HashSet <string> knownUids)
{
JArray allTagged = await GrafanaClient.SearchDashboardsByTagAsync(SourceTag).ConfigureAwait(false);
HashSet <string> toRemove = new HashSet <string>(allTagged.Where(IsManagedDashboard).Select(d => d.Value <string>("uid")));
// We shouldn't remove the ones we just deployed
toRemove.ExceptWith(knownUids);
foreach (string uid in toRemove)
{
Log.LogMessage(MessageImportance.Normal, "Deleting extra dashboard {0}...", uid);
await GrafanaClient.DeleteDashboardAsync(uid).ConfigureAwait(false);
}
}
protected DeployToolBase(
GrafanaClient grafanaClient,
string sourceTagValue,
string dashboardDirectory,
string datasourceDirectory,
string notificationDirectory,
TaskLoggingHelper log)
{
GrafanaClient = grafanaClient;
_sourceTagValue = sourceTagValue;
DashboardDirectory = dashboardDirectory;
DatasourceDirectory = datasourceDirectory;
NotificationDirectory = notificationDirectory;
Log = log;
}
public DeployPublisher(
GrafanaClient grafanaClient,
string keyVaultName,
string keyVaultConnectionString,
string sourceTagValue,
string dashboardDirectory,
string datasourceDirectory,
string notificationDirectory,
string environment,
TaskLoggingHelper log) : base(
grafanaClient, sourceTagValue, dashboardDirectory, datasourceDirectory, notificationDirectory, log)
{
_keyVaultName = keyVaultName;
_keyVaultConnectionString = keyVaultConnectionString;
_environment = environment;
_keyVault = new Lazy <KeyVaultClient>(GetKeyVaultClient);
}
private async Task <bool> ExecuteAsync()
{
using (var client = new GrafanaClient(Host, AccessToken))
using (var deploy = new DeployPublisher(client, KeyVaultName, KeyVaultConnectionString, Tag, DashboardDirectory, DataSourceDirectory, NotificationDirectory, Environment, Log))
{
try
{
await deploy.PostToGrafanaAsync();
}
catch (HttpRequestException e)
{
Log.LogErrorFromException(e, showStackTrace: false, showDetail: false, file: "MonitoringPublish");
return(false);
}
}
return(true);
}
private async Task PostNotificationsAsync()
{
foreach (string notificationPath in Directory.GetFiles(EnvironmentNotificationDirectory,
"*" + NotificationExtension,
SearchOption.AllDirectories))
{
string uid = GetUidFromNotificationFile(Path.GetFileName(notificationPath));
JObject data;
using (var sr = new StreamReader(notificationPath))
using (var jr = new JsonTextReader(sr))
{
data = await JObject.LoadAsync(jr).ConfigureAwait(false);
}
data["uid"] = uid;
Log.LogMessage(MessageImportance.Normal, "Posting notification {0}...", uid);
await ReplaceVaultAsync(data);
await GrafanaClient.CreateNotificationChannelAsync(data).ConfigureAwait(false);
}
}
private async Task PostDatasourcesAsync()
{
foreach (string datasourcePath in Directory.GetFiles(EnvironmentDatasourceDirectory,
"*" + DatasourceExtension,
SearchOption.AllDirectories))
{
var name = GetNameFromDatasourceFile(Path.GetFileName(datasourcePath));
JObject data;
using (var sr = new StreamReader(datasourcePath))
using (var jr = new JsonTextReader(sr))
{
data = await JObject.LoadAsync(jr).ConfigureAwait(false);
}
data["name"] = name;
Log.LogMessage(MessageImportance.Normal, "Posting datasource {0}...", name);
await ReplaceVaultAsync(data);
await GrafanaClient.CreateDatasourceAsync(data).ConfigureAwait(false);
}
}
private async Task PostDashboardsAsync()
{
JArray folderArray = await GrafanaClient.ListFoldersAsync().ConfigureAwait(false);
List <FolderData> folders = folderArray.Select(f => new FolderData(f.Value <string>("uid"), f.Value <string>("title")))
.ToList();
var knownUids = new HashSet <string>();
foreach (string dashboardPath in GetAllDashboardPaths())
{
string folderName = Path.GetFileName(Path.GetDirectoryName(dashboardPath));
string dashboardFileName = Path.GetFileName(dashboardPath);
string uid = GetUidFromDashboardFile(dashboardFileName);
knownUids.Add(uid);
FolderData folder = folders.FirstOrDefault(f => f.Title == folderName);
JObject result = await GrafanaClient.CreateFolderAsync(folderName, folderName).ConfigureAwait(false);
string folderUid = result["uid"].Value <string>();
int folderId = result["id"].Value <int>();
if (folder == null)
{
folder = new FolderData(folderUid, folderName);
}
folder.Id = folderId;
JObject data;
using (var sr = new StreamReader(dashboardPath))
using (var jr = new JsonTextReader(sr))
{
data = await JObject.LoadAsync(jr).ConfigureAwait(false);
}
JArray tagArray = null;
if (data.TryGetValue("tags", out JToken tagToken))
{
tagArray = tagToken as JArray;
}
if (tagArray == null)
{
tagArray = new JArray();
}
var newTags = new JArray();
foreach (JToken tag in tagArray)
{
if (tag.Value <string>().StartsWith(BaseUidTagPrefix) ||
tag.Value <string>().StartsWith(SourceTagPrefix))
{
continue;
}
newTags.Add(tag);
}
tagArray.Add(GetUidTag(uid));
tagArray.Add(SourceTag);
data["tags"] = newTags;
data["uid"] = uid;
Log.LogMessage(MessageImportance.Normal, "Posting dashboard {0}...", uid);
await GrafanaClient.CreateDashboardAsync(data, folderId).ConfigureAwait(false);
}
await ClearExtraneousDashboardsAsync(knownUids);
}
public async Task ImportFromGrafana(IEnumerable <string> dashboardUids)
{
HashSet <string> usedNotificationIds = new HashSet <string>();
foreach (var dashboardPath in GetAllDashboardPaths())
{
if (dashboardUids.Any(d => d.Contains(GetUidFromDashboardFile(dashboardPath))))
{
// This is a new dashboard, don't assume it alert tags
JObject data;
using (var sr = new StreamReader(dashboardPath))
using (var jr = new JsonTextReader(sr))
{
data = await JObject.LoadAsync(jr).ConfigureAwait(false);
}
IEnumerable <string> notificationIds = data.SelectTokens("..alertRuleTags.NotificationId")
.Select(d => d.Value <string>());
foreach (var alertId in notificationIds)
{
usedNotificationIds.Add(alertId);
}
}
}
List <Parameter> parameters;
if (File.Exists(_parametersFilePath))
{
using (var sr = new StreamReader(_parametersFilePath))
using (var jr = new JsonTextReader(sr))
{
parameters = _serializer.Deserialize <List <Parameter> >(jr);
}
}
else
{
parameters = new List <Parameter>();
}
foreach (string dashboardUid in dashboardUids)
{
// Get a dashboard
JObject dashboard = await GrafanaClient.GetDashboardAsync(dashboardUid).ConfigureAwait(false);
// Cache dashboard data needed for post
string targetUid = dashboard.Value <JObject>("dashboard").Value <string>("uid");
// SanitizeDashboard makes structural changes to the JSON and ParameterizeDashboard
// expects a certain structure, so order matters when calling these methods.
JObject parameterizedDashboard = GrafanaSerialization.ParameterizeDashboard(dashboard, parameters, _environments, _environment);
JObject slimmedDashboard = GrafanaSerialization.SanitizeDashboard(parameterizedDashboard);
UpdateNotificationIds(slimmedDashboard, usedNotificationIds);
JArray tags = slimmedDashboard.Value <JArray>("tags");
JToken uidTag = tags.FirstOrDefault(t => t.Value <string>().StartsWith(BaseUidTagPrefix));
if (uidTag != null)
{
targetUid = uidTag.Value <string>().Substring(BaseUidTagPrefix.Length);
}
// Get the dashboard folder data and cache if not already present
int folderId = GrafanaSerialization.ExtractFolderId(dashboard);
JObject folder = await GrafanaClient.GetFolderAsync(folderId).ConfigureAwait(false);
FolderData folderData = GrafanaSerialization.SanitizeFolder(folder);
// Get datasources used in the dashboard
IEnumerable <string> dataSourceIdentifiers = GrafanaSerialization.ExtractDataSourceIdentifiers(dashboard);
foreach (string dataSourceIdentifier in dataSourceIdentifiers)
{
JObject datasource = await GrafanaClient.GetDataSourceByUidAsync(dataSourceIdentifier).ConfigureAwait(false) ??
await GrafanaClient.GetDataSourceByNameAsync(dataSourceIdentifier).ConfigureAwait(false);
string datasourceName = datasource.Value <string>("name");
datasource = GrafanaSerialization.SanitizeDataSource(datasource);
// Create the data source for each environment
foreach (string env in _environments)
{
string datasourcePath = GetDatasourcePath(env, datasourceName);
if (File.Exists(datasourcePath))
{
// If we already have that datasource, don't overwrite it's useful values with empty ones
continue;
}
Log.LogMessage(MessageImportance.Normal, "Importing datasource {0}...", datasourceName);
Directory.CreateDirectory(Path.GetDirectoryName(datasourcePath));
using (var datasourceStreamWriter = new StreamWriter(datasourcePath))
using (var datasourceJsonWriter = new JsonTextWriter(datasourceStreamWriter))
{
_serializer.Serialize(datasourceJsonWriter, datasource);
}
}
}
HashSet <string> usedNotifications = new HashSet <string>(slimmedDashboard
.SelectTokens("panels.[*].alert.notifications.[*].uid")
.Select(t => t.Value <string>())
.Where(uid => !string.IsNullOrEmpty(uid)));
foreach (string notificationUid in usedNotifications)
{
JObject notificationChannel = await GrafanaClient.GetNotificationChannelAsync(notificationUid);
notificationChannel = GrafanaSerialization.SanitizeNotificationChannel(notificationChannel);
// Create the data source for each environment
foreach (string env in _environments)
{
string notificationPath = GetNotificationPath(env, notificationUid);
if (File.Exists(notificationPath))
{
// If we already have that notification, don't overwrite it's useful values with empty ones
continue;
}
Log.LogMessage(MessageImportance.Normal, "Importing notification channel {0}...", notificationUid);
if (notificationChannel.ContainsKey("password"))
{
var password = notificationChannel.GetValue("password").Value <string>();
if (!password.Contains("vault"))
{
Log.LogWarning($"Please replace the password token with a key vault reference inside {notificationPath}");
}
}
Directory.CreateDirectory(Path.GetDirectoryName(notificationPath));
using (var datasourceStreamWriter = new StreamWriter(notificationPath))
using (var datasourceJsonWriter = new JsonTextWriter(datasourceStreamWriter))
{
_serializer.Serialize(datasourceJsonWriter, notificationChannel);
}
}
}
Log.LogMessage(MessageImportance.Normal, "Importing dashboard {0}...", targetUid);
string dashboardPath = GetDashboardFilePath(folderData.Title, targetUid);
Directory.CreateDirectory(Path.GetDirectoryName(dashboardPath));
using (var dashboardStreamWriter = new StreamWriter(dashboardPath))
using (var dashboardJsonWriter = new JsonTextWriter(dashboardStreamWriter))
{
_serializer.Serialize(dashboardJsonWriter, slimmedDashboard);
}
}
// Save parameters back to disk
Log.LogMessage(MessageImportance.Normal, "Saving parameters {0}...", _parametersFilePath);
using (var sr = new StreamWriter(_parametersFilePath))
using (var jr = new JsonTextWriter(sr))
{
_serializer.Serialize(jr, parameters);
}
}
public async Task ImportFromGrafana(IEnumerable <string> dashboardUids)
{
HashSet <string> usedNotificationIds = new HashSet <string>();
foreach (var dashboardPath in GetAllDashboardPaths())
{
if (dashboardUids.Any(d => d.Contains(GetUidFromDashboardFile(dashboardPath))))
{
// This is a new dashboard, don't assume it alert tags
JObject data;
using (var sr = new StreamReader(dashboardPath))
using (var jr = new JsonTextReader(sr))
{
data = await JObject.LoadAsync(jr).ConfigureAwait(false);
}
IEnumerable <string> notificationIds = data.SelectTokens("..alertRuleTags.NotificationId")
.Select(d => d.Value <string>());
foreach (var alertId in notificationIds)
{
usedNotificationIds.Add(alertId);
}
}
}
foreach (string dashboardUid in dashboardUids)
{
// Get a dashboard
JObject dashboard = await GrafanaClient.GetDashboardAsync(dashboardUid).ConfigureAwait(false);
// Cache dashboard data needed for post
string targetUid = dashboard.Value <JObject>("dashboard").Value <string>("uid");
JObject slimmedDashboard = GrafanaSerialization.SanitizeDashboard(dashboard);
UpdateNotificationIds(slimmedDashboard, usedNotificationIds);
JArray tags = slimmedDashboard.Value <JArray>("tags");
JToken uidTag = tags.FirstOrDefault(t => t.Value <string>().StartsWith(BaseUidTagPrefix));
if (uidTag != null)
{
targetUid = uidTag.Value <string>().Substring(BaseUidTagPrefix.Length);
}
// Get the dashboard folder data and cache if not already present
int folderId = GrafanaSerialization.ExtractFolderId(dashboard);
JObject folder = await GrafanaClient.GetFolderAsync(folderId).ConfigureAwait(false);
FolderData folderData = GrafanaSerialization.SanitizeFolder(folder);
// Get datasources used in the dashboard
IEnumerable <string> dataSourceNames = GrafanaSerialization.ExtractDataSourceNames(dashboard);
foreach (string datasourceName in dataSourceNames)
{
JObject datasource = await GrafanaClient.GetDataSourceAsync(datasourceName).ConfigureAwait(false);
datasource = GrafanaSerialization.SanitizeDataSource(datasource);
// Create the data source for each environment
foreach (string env in _environments)
{
string datasourcePath = GetDatasourcePath(env, datasourceName);
if (File.Exists(datasourcePath))
{
// If we already have that datasource, don't overwrite it's useful values with empty ones
continue;
}
Log.LogMessage(MessageImportance.Normal, "Importing datasource {0}...", datasourceName);
Directory.CreateDirectory(Path.GetDirectoryName(datasourcePath));
using (var datasourceStreamWriter = new StreamWriter(datasourcePath))
using (var datasourceJsonWriter = new JsonTextWriter(datasourceStreamWriter))
{
_serializer.Serialize(datasourceJsonWriter, datasource);
}
}
}
HashSet <string> usedNotifications = new HashSet <string>(slimmedDashboard
.SelectTokens("panels.[*].alert.notifications.[*].uid")
.Select(t => t.Value <string>())
.Where(uid => !string.IsNullOrEmpty(uid)));
foreach (string notificationUid in usedNotifications)
{
JObject notificationChannel = await GrafanaClient.GetNotificationChannelAsync(notificationUid);
notificationChannel = GrafanaSerialization.SanitizeNotificationChannel(notificationChannel);
// Create the data source for each environment
foreach (string env in _environments)
{
string notificationPath = GetNotificationPath(env, notificationUid);
if (File.Exists(notificationPath))
{
// If we already have that notification, don't overwrite it's useful values with empty ones
continue;
}
Log.LogMessage(MessageImportance.Normal, "Importing notification channel {0}...", notificationUid);
Directory.CreateDirectory(Path.GetDirectoryName(notificationPath));
using (var datasourceStreamWriter = new StreamWriter(notificationPath))
using (var datasourceJsonWriter = new JsonTextWriter(datasourceStreamWriter))
{
_serializer.Serialize(datasourceJsonWriter, notificationChannel);
}
}
}
Log.LogMessage(MessageImportance.Normal, "Importing dashboard {0}...", targetUid);
string dashboardPath = GetDashboardFilePath(folderData.Title, targetUid);
Directory.CreateDirectory(Path.GetDirectoryName(dashboardPath));
using (var dashboardStreamWriter = new StreamWriter(dashboardPath))
using (var dashboardJsonWriter = new JsonTextWriter(dashboardStreamWriter))
{
_serializer.Serialize(dashboardJsonWriter, slimmedDashboard);
}
}
}
