GetMappedDashboards(this MetabaseApi api, IReadOnlyDictionary <CardId, CardId> cardMapping) { var dashboards = await api.GetAllDashboards(); var nonArchivedDashboards = dashboards.Where(x => x.Archived == false).OrderBy(x => x.Id).ToArray(); var dashboardMapping = Renumber(nonArchivedDashboards.Select(x => x.Id).ToList()); foreach (var dashboard in nonArchivedDashboards) { dashboard.Id = dashboardMapping[dashboard.Id]; var dashboardCardMapping = Renumber(dashboard.Cards.Select(x => x.Id).ToList()); foreach (var card in dashboard.Cards.OrderBy(x => x.Id)) { card.Id = dashboardCardMapping[card.Id]; if (card.CardId.HasValue) { card.CardId = cardMapping[card.CardId.Value]; } foreach (var parameter in card.ParameterMappings) { parameter.CardId = cardMapping[parameter.CardId]; } foreach (var seriesCard in card.Series) { seriesCard.Id = cardMapping[seriesCard.Id]; } } } return(nonArchivedDashboards, dashboardMapping); }
GetMappedCards(this MetabaseApi api, IReadOnlyDictionary <CollectionId, CollectionId> collectionMapping) { var cards = await api.GetAllCards(); var cardsToExport = cards .Where(x => x.Archived == false) .Where(x => x.CollectionId == null || collectionMapping.ContainsKey(x.CollectionId.Value)) .OrderBy(x => x.Id) .ToArray(); var cardMapping = Renumber(cardsToExport.Select(x => x.Id).ToList()); foreach (var card in cardsToExport) { var newId = cardMapping.GetOrThrow(card.Id, "Card not found in card mapping"); var oldId = card.Id; Console.WriteLine($"Mapping card {oldId} to {newId} ({card.Name})"); card.Id = newId; if (card.CollectionId.HasValue) { card.CollectionId = collectionMapping.GetOrThrow(card.CollectionId.Value, $"Collection not found in collection mapping for card {card.Id}"); } if (card.DatasetQuery.Native == null) { Console.WriteLine($"WARNING: card {oldId} has a non-SQL definition. Its state might not be exported/imported correctly. ({card.Name})"); } card.Description = string.IsNullOrEmpty(card.Description) ? null : card.Description; } return(cardsToExport, cardMapping); }
static async Task Import(this MetabaseApi api, Config.Import import) { var rawState = File.ReadAllText(import.InputFilename); var state = JsonConvert.DeserializeObject <MetabaseState>(rawState); await api.Import(state, import.DatabaseMapping); Console.WriteLine($"Done importing from {import.InputFilename} into {import.MetabaseApiSettings.MetabaseApiUrl}"); }
static async Task DeleteAllCards(this MetabaseApi api) { var cards = await api.GetAllCards(); foreach (var card in cards) { await api.DeleteCard(card.Id); } }
static async Task MapAndCreateDashboard(this MetabaseApi api, Dashboard stateDashboard, IReadOnlyList <Mapping <CardId> > cardMapping) { var mappedCards = MapDashboardCards(stateDashboard.Cards, cardMapping).ToList(); Console.WriteLine($"Creating dashboard '{stateDashboard.Name}'"); await api.CreateDashboard(stateDashboard); await api.AddCardsToDashboard(stateDashboard.Id, mappedCards); }
static async Task Export(this MetabaseApi api, Config.Export export) { var state = await api.Export(); var stateJson = JsonConvert.SerializeObject(state, Formatting.Indented); File.WriteAllText(path: export.OutputFilename, contents: stateJson); Console.WriteLine($"Exported current state for {export.MetabaseApiSettings.MetabaseApiUrl} to {export.OutputFilename}"); }
static async Task DeleteAllDashboards(this MetabaseApi api) { var dashboards = await api.GetAllDashboards(); foreach (var dashboard in dashboards) { await api.DeleteDashboard(dashboard.Id); } }
public static async Task TestQuestions(this MetabaseApi api) { var cards = await api.GetAllCards(); var testResults = await cards.Traverse(api.TestCard); Console.WriteLine(); Console.WriteLine("Passed: " + testResults.Count(x => x == TestResult.Passed)); Console.WriteLine("Failed: " + testResults.Count(x => x == TestResult.Failed)); Console.WriteLine("Crashed: " + testResults.Count(x => x == TestResult.Crashed)); }
static async Task ValidateTargetDatabaseMapping(this MetabaseApi api, IReadOnlyDictionary <DatabaseId, DatabaseId> databaseMapping) { var databaseIds = await api.GetAllDatabaseIds(); var incorrectMappings = databaseMapping.Where(kv => databaseIds.Contains(kv.Value) == false).ToList(); if (incorrectMappings.Count > 0) { throw new Exception("Mappings referencing invalid databases: " + string.Join(", ", incorrectMappings.Select(kv => $"{kv.Key} -> {kv.Value}"))); } }
GetMappedCollections(this MetabaseApi api) { var collections = await api.GetAllCollections(); var nonArchivedCollections = collections.Where(x => x.Archived == false).OrderBy(x => x.Id).ToArray(); var collectionMapping = Renumber(nonArchivedCollections.Select(x => x.Id).ToList()); foreach (var collection in nonArchivedCollections) { collection.Id = collectionMapping[collection.Id]; } return(nonArchivedCollections, collectionMapping); }
static async Task MapAndCreateDashboard(this MetabaseApi api, Dashboard stateDashboard, IReadOnlyList <Mapping <CardId> > cardMapping, IReadOnlyList <Mapping <Collection> > collectionMapping) { var mappedCards = MapDashboardCards(stateDashboard.Cards, cardMapping).ToList(); if (stateDashboard.CollectionId.HasValue) { stateDashboard.CollectionId = collectionMapping .Where(x => x.Source.Id == stateDashboard.CollectionId.Value) .Select(x => x.Target.Id) .First(); } Console.WriteLine($"Creating dashboard '{stateDashboard.Name}'"); await api.CreateDashboard(stateDashboard); await api.AddCardsToDashboard(stateDashboard.Id, mappedCards); }
GetMappedCollections(this MetabaseApi api, bool excludePersonalCollections) { var collections = await api.GetAllCollections(); var collectionsToExport = collections .Where(x => x.Archived == false) .Where(x => excludePersonalCollections == false || x.IsPersonal() == false) .OrderBy(x => x.Id) .ToArray(); var collectionMapping = Renumber(collectionsToExport.Select(x => x.Id).ToList()); foreach (var collection in collectionsToExport) { collection.Id = collectionMapping.GetOrThrow(collection.Id, "Collection not found in collection mapping"); } return(collectionsToExport, collectionMapping); }
/// <summary> /// Export Metabase data /// </summary> public static async Task <MetabaseState> Export(this MetabaseApi api, bool excludePersonalCollections) { var mappedCollections = await api.GetMappedCollections(excludePersonalCollections); var mappedCards = await api.GetMappedCards(mappedCollections.CollectionMapping); var mappedDashboards = await api.GetMappedDashboards(mappedCards.CardMapping, mappedCollections.Collections); var state = new MetabaseState { Cards = mappedCards.Cards.ToArray(), Dashboards = mappedDashboards.Dashboards.ToArray(), Collections = mappedCollections.Collections.ToArray(), }; return(state); }
static async Task <MetabaseApi> InitApi(MetabaseApiSettings apiSettings) { const string filename = "metabase-token.txt"; string GetInitialToken() { try { return(File.ReadAllText(filename)); } catch (Exception e) { return(null); } } // get an existing token if available to work around Metabase throttling // https://github.com/metabase/metabase/issues/4979 var MetabaseInitialToken = GetInitialToken(); var metabaseSession = new MetabaseSessionTokenManager(apiSettings, MetabaseInitialToken); var api = new MetabaseApi(metabaseSession); try { await api.GetAllDashboards(); // attempt an API call to either validate or renew the session token var token = await metabaseSession.CurrentToken(); File.WriteAllText(filename, token); return(api); } catch (Exception e) { throw new Exception("Error initialising Metabase API for " + apiSettings.MetabaseApiUrl, e); } }
static async Task <Card> MapAndCreateCard(this MetabaseApi api, Card cardFromState, IReadOnlyList <Mapping <Collection> > collectionMapping, IReadOnlyDictionary <DatabaseId, DatabaseId> databaseMapping) { if (cardFromState.DatasetQuery.Native == null) { Console.WriteLine("WARNING: skipping card because it does not have a SQL definition: " + cardFromState.Name); return(null); } Console.WriteLine($"Creating card '{cardFromState.Name}'"); if (cardFromState.CollectionId.HasValue) { cardFromState.CollectionId = collectionMapping .Where(x => x.Source.Id == cardFromState.CollectionId.Value) .Select(x => x.Target.Id) .First(); } cardFromState.Description = string.IsNullOrEmpty(cardFromState.Description) ? null : cardFromState.Description; cardFromState.DatabaseId = databaseMapping[cardFromState.DatabaseId]; cardFromState.DatasetQuery.DatabaseId = databaseMapping[cardFromState.DatasetQuery.DatabaseId]; await api.CreateCard(cardFromState); return(cardFromState); }
GetMappedDashboards(this MetabaseApi api, IReadOnlyDictionary <CardId, CardId> cardMapping, IReadOnlyCollection <Collection> exportedCollections) { var dashboards = await api.GetAllDashboards(); var nonArchivedDashboards = dashboards .Where(x => x.Archived == false) .Where(dashboard => dashboard.CollectionId.HasValue == false || exportedCollections.Any(collection => collection.Id == dashboard.CollectionId)) .OrderBy(x => x.Id) .ToArray(); var dashboardMapping = Renumber(nonArchivedDashboards.Select(x => x.Id).ToList()); foreach (var dashboard in nonArchivedDashboards) { var oldDashboardId = dashboard.Id; dashboard.Id = dashboardMapping.GetOrThrow(dashboard.Id, "Dashboard not found in mapping"); var dashboardCardMapping = Renumber(dashboard.Cards.Select(x => x.Id).ToList()); foreach (var card in dashboard.Cards.OrderBy(x => x.Id)) { card.Id = dashboardCardMapping.GetOrThrow(card.Id, $"Card not found in dashboard card mapping for dashboard {oldDashboardId}"); if (card.CardId.HasValue) { card.CardId = cardMapping.GetOrThrow(card.CardId.Value, $"Card not found in card mapping for dashboard {oldDashboardId}"); } foreach (var parameter in card.ParameterMappings) { parameter.CardId = cardMapping.GetOrThrow(parameter.CardId, $"Card not found in card mapping for parameter {parameter.ParameterId}, dashboard {oldDashboardId}"); } foreach (var seriesCard in card.Series) { seriesCard.Id = cardMapping.GetOrThrow(seriesCard.Id, $"Card not found in card mapping for series {seriesCard.Name}, dashboard {oldDashboardId}"); } } } return(nonArchivedDashboards, dashboardMapping); }
/// <summary> /// Imports Metabase data. DELETES all current dashboards/questions/etc. /// </summary> /// <param name="api"></param> /// <param name="state"></param> /// <param name="databaseMapping"></param> /// <returns></returns> public static async Task Import(this MetabaseApi api, MetabaseState state, IReadOnlyDictionary <DatabaseId, DatabaseId> databaseMapping) { // firstly check that the database mapping is complete and correct await api.ValidateDatabaseMapping(state, databaseMapping); // now map/create collections then cards then dashboards Console.WriteLine("Creating collections..."); var collectionMapping = await api.MapAndCreateCollections(state.Collections); Console.WriteLine("Deleting all dashboards..."); await api.DeleteAllDashboards(); Console.WriteLine("Deleting all cards..."); await api.DeleteAllCards(); Console.WriteLine("Creating cards..."); var partialCardMapping = await state.Cards .Traverse(async cardFromState => { var source = cardFromState.Id; var target = await api.MapAndCreateCard(cardFromState, collectionMapping, databaseMapping); var mapping = new Mapping <CardId?>(source: source, target: target?.Id); return(mapping); }); var cardMapping = partialCardMapping .Where(x => x.Source.HasValue && x.Target.HasValue) .Select(x => new Mapping <CardId>(x.Source.Value, x.Target.Value)) .ToList(); Console.WriteLine("Creating dashboards..."); foreach (var dashboard in state.Dashboards) { await api.MapAndCreateDashboard(dashboard, cardMapping); } Console.WriteLine("Done importing"); }
static async Task <TestResult> TestCard(this MetabaseApi api, Card card) { try { var result = await api.RunCard(card.Id); Console.Write($"'{card.Name}'..."); if (result.Status == "completed") { Console.WriteLine("Passed"); return(TestResult.Passed); } else { Console.WriteLine($"Failed:\n{result.Error}\n"); return(TestResult.Failed); } } catch (MetabaseApiException e) { Console.WriteLine($"ERROR querying '{card.Name}': {e.Message}"); return(TestResult.Crashed); } }
static async Task <IReadOnlyList <Mapping <Collection> > > MapAndCreateCollections(this MetabaseApi api, IReadOnlyList <Collection> stateCollections) { // collections can't be deleted so we have to match existing collections or create new ones var allExistingCollections = await api.GetAllCollections(); var nonArchivedExistingCollections = allExistingCollections.Where(x => x.Archived == false).ToList(); var collectionsToCreate = stateCollections .Where(c => nonArchivedExistingCollections.Select(x => x.Name).Contains(c.Name) == false) .ToList(); var createdCollections = await collectionsToCreate .Traverse(async collectionFromState => { Console.WriteLine($"Creating collection '{collectionFromState.Name}'"); var mapping = new Mapping <Collection>( source: collectionFromState, target: await api.CreateCollection(collectionFromState) ); return(mapping); }); var mappedExistingCollections = stateCollections .Select(collectionFromState => new Mapping <Collection>( source: collectionFromState, target: nonArchivedExistingCollections.Where(x => x.Name == collectionFromState.Name).FirstOrDefault() ) ) .Where(x => x.Target != null) .ToList(); var collectionMapping = createdCollections.Concat(mappedExistingCollections).ToList(); return(collectionMapping); }
static async Task ValidateDatabaseMapping(this MetabaseApi api, MetabaseState state, IReadOnlyDictionary <DatabaseId, DatabaseId> databaseMapping) { ValidateSourceDatabaseMapping(state, databaseMapping); await api.ValidateTargetDatabaseMapping(databaseMapping); }