GetMappedDashboards(this MetabaseApi api, IReadOnlyDictionary <CardId, CardId> cardMapping)
{
var dashboards = await api.GetAllDashboards();
var nonArchivedDashboards = dashboards.Where(x => x.Archived == false).OrderBy(x => x.Id).ToArray();
var dashboardMapping = Renumber(nonArchivedDashboards.Select(x => x.Id).ToList());
foreach (var dashboard in nonArchivedDashboards)
{
dashboard.Id = dashboardMapping[dashboard.Id];
var dashboardCardMapping = Renumber(dashboard.Cards.Select(x => x.Id).ToList());
foreach (var card in dashboard.Cards.OrderBy(x => x.Id))
{
card.Id = dashboardCardMapping[card.Id];
if (card.CardId.HasValue)
{
card.CardId = cardMapping[card.CardId.Value];
}
foreach (var parameter in card.ParameterMappings)
{
parameter.CardId = cardMapping[parameter.CardId];
}
foreach (var seriesCard in card.Series)
{
seriesCard.Id = cardMapping[seriesCard.Id];
}
}
}
return(nonArchivedDashboards, dashboardMapping);
}
GetMappedCards(this MetabaseApi api, IReadOnlyDictionary <CollectionId, CollectionId> collectionMapping)
{
var cards = await api.GetAllCards();
var cardsToExport = cards
.Where(x => x.Archived == false)
.Where(x => x.CollectionId == null || collectionMapping.ContainsKey(x.CollectionId.Value))
.OrderBy(x => x.Id)
.ToArray();
var cardMapping = Renumber(cardsToExport.Select(x => x.Id).ToList());
foreach (var card in cardsToExport)
{
var newId = cardMapping.GetOrThrow(card.Id, "Card not found in card mapping");
var oldId = card.Id;
Console.WriteLine($"Mapping card {oldId} to {newId} ({card.Name})");
card.Id = newId;
if (card.CollectionId.HasValue)
{
card.CollectionId = collectionMapping.GetOrThrow(card.CollectionId.Value, $"Collection not found in collection mapping for card {card.Id}");
}
if (card.DatasetQuery.Native == null)
{
Console.WriteLine($"WARNING: card {oldId} has a non-SQL definition. Its state might not be exported/imported correctly. ({card.Name})");
}
card.Description = string.IsNullOrEmpty(card.Description) ? null : card.Description;
}
return(cardsToExport, cardMapping);
}
static async Task Import(this MetabaseApi api, Config.Import import)
{
var rawState = File.ReadAllText(import.InputFilename);
var state = JsonConvert.DeserializeObject <MetabaseState>(rawState);
await api.Import(state, import.DatabaseMapping);
Console.WriteLine($"Done importing from {import.InputFilename} into {import.MetabaseApiSettings.MetabaseApiUrl}");
}
static async Task DeleteAllCards(this MetabaseApi api)
{
var cards = await api.GetAllCards();
foreach (var card in cards)
{
await api.DeleteCard(card.Id);
}
}
static async Task MapAndCreateDashboard(this MetabaseApi api, Dashboard stateDashboard, IReadOnlyList <Mapping <CardId> > cardMapping)
{
var mappedCards = MapDashboardCards(stateDashboard.Cards, cardMapping).ToList();
Console.WriteLine($"Creating dashboard '{stateDashboard.Name}'");
await api.CreateDashboard(stateDashboard);
await api.AddCardsToDashboard(stateDashboard.Id, mappedCards);
}
static async Task Export(this MetabaseApi api, Config.Export export)
{
var state = await api.Export();
var stateJson = JsonConvert.SerializeObject(state, Formatting.Indented);
File.WriteAllText(path: export.OutputFilename, contents: stateJson);
Console.WriteLine($"Exported current state for {export.MetabaseApiSettings.MetabaseApiUrl} to {export.OutputFilename}");
}
static async Task DeleteAllDashboards(this MetabaseApi api)
{
var dashboards = await api.GetAllDashboards();
foreach (var dashboard in dashboards)
{
await api.DeleteDashboard(dashboard.Id);
}
}
public static async Task TestQuestions(this MetabaseApi api)
{
var cards = await api.GetAllCards();
var testResults = await cards.Traverse(api.TestCard);
Console.WriteLine();
Console.WriteLine("Passed: " + testResults.Count(x => x == TestResult.Passed));
Console.WriteLine("Failed: " + testResults.Count(x => x == TestResult.Failed));
Console.WriteLine("Crashed: " + testResults.Count(x => x == TestResult.Crashed));
}
static async Task ValidateTargetDatabaseMapping(this MetabaseApi api, IReadOnlyDictionary <DatabaseId, DatabaseId> databaseMapping)
{
var databaseIds = await api.GetAllDatabaseIds();
var incorrectMappings = databaseMapping.Where(kv => databaseIds.Contains(kv.Value) == false).ToList();
if (incorrectMappings.Count > 0)
{
throw new Exception("Mappings referencing invalid databases: " +
string.Join(", ", incorrectMappings.Select(kv => $"{kv.Key} -> {kv.Value}")));
}
}
GetMappedCollections(this MetabaseApi api)
{
var collections = await api.GetAllCollections();
var nonArchivedCollections = collections.Where(x => x.Archived == false).OrderBy(x => x.Id).ToArray();
var collectionMapping = Renumber(nonArchivedCollections.Select(x => x.Id).ToList());
foreach (var collection in nonArchivedCollections)
{
collection.Id = collectionMapping[collection.Id];
}
return(nonArchivedCollections, collectionMapping);
}
static async Task MapAndCreateDashboard(this MetabaseApi api, Dashboard stateDashboard,
IReadOnlyList <Mapping <CardId> > cardMapping, IReadOnlyList <Mapping <Collection> > collectionMapping)
{
var mappedCards = MapDashboardCards(stateDashboard.Cards, cardMapping).ToList();
if (stateDashboard.CollectionId.HasValue)
{
stateDashboard.CollectionId = collectionMapping
.Where(x => x.Source.Id == stateDashboard.CollectionId.Value)
.Select(x => x.Target.Id)
.First();
}
Console.WriteLine($"Creating dashboard '{stateDashboard.Name}'");
await api.CreateDashboard(stateDashboard);
await api.AddCardsToDashboard(stateDashboard.Id, mappedCards);
}
GetMappedCollections(this MetabaseApi api, bool excludePersonalCollections)
{
var collections = await api.GetAllCollections();
var collectionsToExport = collections
.Where(x => x.Archived == false)
.Where(x => excludePersonalCollections == false || x.IsPersonal() == false)
.OrderBy(x => x.Id)
.ToArray();
var collectionMapping = Renumber(collectionsToExport.Select(x => x.Id).ToList());
foreach (var collection in collectionsToExport)
{
collection.Id = collectionMapping.GetOrThrow(collection.Id, "Collection not found in collection mapping");
}
return(collectionsToExport, collectionMapping);
}
/// <summary>
/// Export Metabase data
/// </summary>
public static async Task <MetabaseState> Export(this MetabaseApi api, bool excludePersonalCollections)
{
var mappedCollections = await api.GetMappedCollections(excludePersonalCollections);
var mappedCards = await api.GetMappedCards(mappedCollections.CollectionMapping);
var mappedDashboards = await api.GetMappedDashboards(mappedCards.CardMapping, mappedCollections.Collections);
var state = new MetabaseState
{
Cards = mappedCards.Cards.ToArray(),
Dashboards = mappedDashboards.Dashboards.ToArray(),
Collections = mappedCollections.Collections.ToArray(),
};
return(state);
}
static async Task <MetabaseApi> InitApi(MetabaseApiSettings apiSettings)
{
const string filename = "metabase-token.txt";
string GetInitialToken()
{
try
{
return(File.ReadAllText(filename));
}
catch (Exception e)
{
return(null);
}
}
// get an existing token if available to work around Metabase throttling
// https://github.com/metabase/metabase/issues/4979
var MetabaseInitialToken = GetInitialToken();
var metabaseSession = new MetabaseSessionTokenManager(apiSettings, MetabaseInitialToken);
var api = new MetabaseApi(metabaseSession);
try
{
await api.GetAllDashboards(); // attempt an API call to either validate or renew the session token
var token = await metabaseSession.CurrentToken();
File.WriteAllText(filename, token);
return(api);
}
catch (Exception e)
{
throw new Exception("Error initialising Metabase API for " + apiSettings.MetabaseApiUrl, e);
}
}
static async Task <Card> MapAndCreateCard(this MetabaseApi api, Card cardFromState, IReadOnlyList <Mapping <Collection> > collectionMapping, IReadOnlyDictionary <DatabaseId, DatabaseId> databaseMapping)
{
if (cardFromState.DatasetQuery.Native == null)
{
Console.WriteLine("WARNING: skipping card because it does not have a SQL definition: " + cardFromState.Name);
return(null);
}
Console.WriteLine($"Creating card '{cardFromState.Name}'");
if (cardFromState.CollectionId.HasValue)
{
cardFromState.CollectionId = collectionMapping
.Where(x => x.Source.Id == cardFromState.CollectionId.Value)
.Select(x => x.Target.Id)
.First();
}
cardFromState.Description = string.IsNullOrEmpty(cardFromState.Description) ? null : cardFromState.Description;
cardFromState.DatabaseId = databaseMapping[cardFromState.DatabaseId];
cardFromState.DatasetQuery.DatabaseId = databaseMapping[cardFromState.DatasetQuery.DatabaseId];
await api.CreateCard(cardFromState);
return(cardFromState);
}
GetMappedDashboards(this MetabaseApi api, IReadOnlyDictionary <CardId, CardId> cardMapping, IReadOnlyCollection <Collection> exportedCollections)
{
var dashboards = await api.GetAllDashboards();
var nonArchivedDashboards = dashboards
.Where(x => x.Archived == false)
.Where(dashboard => dashboard.CollectionId.HasValue == false || exportedCollections.Any(collection => collection.Id == dashboard.CollectionId))
.OrderBy(x => x.Id)
.ToArray();
var dashboardMapping = Renumber(nonArchivedDashboards.Select(x => x.Id).ToList());
foreach (var dashboard in nonArchivedDashboards)
{
var oldDashboardId = dashboard.Id;
dashboard.Id = dashboardMapping.GetOrThrow(dashboard.Id, "Dashboard not found in mapping");
var dashboardCardMapping = Renumber(dashboard.Cards.Select(x => x.Id).ToList());
foreach (var card in dashboard.Cards.OrderBy(x => x.Id))
{
card.Id = dashboardCardMapping.GetOrThrow(card.Id, $"Card not found in dashboard card mapping for dashboard {oldDashboardId}");
if (card.CardId.HasValue)
{
card.CardId = cardMapping.GetOrThrow(card.CardId.Value, $"Card not found in card mapping for dashboard {oldDashboardId}");
}
foreach (var parameter in card.ParameterMappings)
{
parameter.CardId = cardMapping.GetOrThrow(parameter.CardId, $"Card not found in card mapping for parameter {parameter.ParameterId}, dashboard {oldDashboardId}");
}
foreach (var seriesCard in card.Series)
{
seriesCard.Id = cardMapping.GetOrThrow(seriesCard.Id, $"Card not found in card mapping for series {seriesCard.Name}, dashboard {oldDashboardId}");
}
}
}
return(nonArchivedDashboards, dashboardMapping);
}
/// <summary>
/// Imports Metabase data. DELETES all current dashboards/questions/etc.
/// </summary>
/// <param name="api"></param>
/// <param name="state"></param>
/// <param name="databaseMapping"></param>
/// <returns></returns>
public static async Task Import(this MetabaseApi api, MetabaseState state, IReadOnlyDictionary <DatabaseId, DatabaseId> databaseMapping)
{
// firstly check that the database mapping is complete and correct
await api.ValidateDatabaseMapping(state, databaseMapping);
// now map/create collections then cards then dashboards
Console.WriteLine("Creating collections...");
var collectionMapping = await api.MapAndCreateCollections(state.Collections);
Console.WriteLine("Deleting all dashboards...");
await api.DeleteAllDashboards();
Console.WriteLine("Deleting all cards...");
await api.DeleteAllCards();
Console.WriteLine("Creating cards...");
var partialCardMapping = await state.Cards
.Traverse(async cardFromState => {
var source = cardFromState.Id;
var target = await api.MapAndCreateCard(cardFromState, collectionMapping, databaseMapping);
var mapping = new Mapping <CardId?>(source: source, target: target?.Id);
return(mapping);
});
var cardMapping = partialCardMapping
.Where(x => x.Source.HasValue && x.Target.HasValue)
.Select(x => new Mapping <CardId>(x.Source.Value, x.Target.Value))
.ToList();
Console.WriteLine("Creating dashboards...");
foreach (var dashboard in state.Dashboards)
{
await api.MapAndCreateDashboard(dashboard, cardMapping);
}
Console.WriteLine("Done importing");
}
static async Task <TestResult> TestCard(this MetabaseApi api, Card card)
{
try
{
var result = await api.RunCard(card.Id);
Console.Write($"'{card.Name}'...");
if (result.Status == "completed")
{
Console.WriteLine("Passed");
return(TestResult.Passed);
}
else
{
Console.WriteLine($"Failed:\n{result.Error}\n");
return(TestResult.Failed);
}
}
catch (MetabaseApiException e)
{
Console.WriteLine($"ERROR querying '{card.Name}': {e.Message}");
return(TestResult.Crashed);
}
}
static async Task <IReadOnlyList <Mapping <Collection> > > MapAndCreateCollections(this MetabaseApi api, IReadOnlyList <Collection> stateCollections)
{
// collections can't be deleted so we have to match existing collections or create new ones
var allExistingCollections = await api.GetAllCollections();
var nonArchivedExistingCollections = allExistingCollections.Where(x => x.Archived == false).ToList();
var collectionsToCreate = stateCollections
.Where(c => nonArchivedExistingCollections.Select(x => x.Name).Contains(c.Name) == false)
.ToList();
var createdCollections = await collectionsToCreate
.Traverse(async collectionFromState => {
Console.WriteLine($"Creating collection '{collectionFromState.Name}'");
var mapping = new Mapping <Collection>(
source: collectionFromState,
target: await api.CreateCollection(collectionFromState)
);
return(mapping);
});
var mappedExistingCollections = stateCollections
.Select(collectionFromState =>
new Mapping <Collection>(
source: collectionFromState,
target: nonArchivedExistingCollections.Where(x => x.Name == collectionFromState.Name).FirstOrDefault()
)
)
.Where(x => x.Target != null)
.ToList();
var collectionMapping = createdCollections.Concat(mappedExistingCollections).ToList();
return(collectionMapping);
}
static async Task ValidateDatabaseMapping(this MetabaseApi api, MetabaseState state, IReadOnlyDictionary <DatabaseId, DatabaseId> databaseMapping)
{
ValidateSourceDatabaseMapping(state, databaseMapping);
await api.ValidateTargetDatabaseMapping(databaseMapping);
}
