internal static void WriteMetadataToFile(TelemetryMetadata meta) { TelemetryDashboardMain.Logger.Log("Writing metadata files to disk.", LogLevel.Info); // check to see if folder exists TelemetryDashboardMain.Logger.Log("Checking to see if output dir exists.", LogLevel.Debug); string outputDir = Path.Combine(FileLocationManager.GetTelemetrySharePath(), FileLocationManager.TELEMETRY_OUTPUT_FOLDER_NAME); if (!Directory.Exists(outputDir)) { TelemetryDashboardMain.Logger.Log("Output dir didn't exist. Creating.", LogLevel.Debug); Directory.CreateDirectory(outputDir); } TelemetryDashboardMain.Logger.Log("Writing System Info file to disk.", LogLevel.Info); WriteSystemInfoFile(meta); TelemetryDashboardMain.Logger.Log("Writing Extensions file to disk.", LogLevel.Info); WriteExtensionsMetadataFile(meta); TelemetryDashboardMain.Logger.Log("Writing Extension Schemas file to disk.", LogLevel.Info); WriteExtensionSchemasMetadataFile(meta); TelemetryDashboardMain.Logger.Log("Writing Engine Infos file to disk.", LogLevel.Info); WriteEngineInfosMetadataFile(meta); TelemetryDashboardMain.Logger.Log("Writing User file to disk.", LogLevel.Info); WriteUsersMetadataFile(meta); TelemetryDashboardMain.Logger.Log("Writing all App metadata files to disk.", LogLevel.Info); WriteAppsMetadataFiles(meta); }
private static void GetRepositoryUsers(TelemetryMetadata metadataObject) { string userBody = @" { 'columns': [{ 'columnType': 'Property', 'definition': 'id', 'name': 'id' }, { 'columnType': 'Property', 'definition': 'userid', 'name': 'userid' }, { 'columnType': 'Property', 'definition': 'userdirectory', 'name': 'userdirectory' }, { 'columnType': 'Property', 'definition': 'name', 'name': 'name' }], 'entity': 'User' }" ; Action <JArray> addAction = (user) => metadataObject.Users.Add(new User(user[0].ToObject <Guid>(), user[1].ToString(), user[2].ToString(), user[3].ToString())); GetRepositoryPagedObjects("user", userBody, addAction); }
internal static int Run(int engineRequestTimeoutMS) { // check to see if metadata binary exists string telemetryMetadataFile = Path.Combine(FileLocationManager.GetTelemetrySharePath(), FileLocationManager.METADATA_BINARY_FILE_NAME); TelemetryDashboardMain.Logger.Log("Trying to get metadata binary file: " + telemetryMetadataFile, LogLevel.Debug); TelemetryMetadata oldMeta, newMetadata; if (File.Exists(telemetryMetadataFile)) { TelemetryDashboardMain.Logger.Log("Found metadata file, will load contents.", LogLevel.Info); Stream openFileStream = File.OpenRead(telemetryMetadataFile); BinaryFormatter deserializer = new BinaryFormatter(); oldMeta = (TelemetryMetadata)deserializer.Deserialize(openFileStream); openFileStream.Close(); } else { oldMeta = new TelemetryMetadata(); } newMetadata = new TelemetryMetadata(true); GetAboutServiceInfo(newMetadata); GetRepositoryExtensions(newMetadata); GetRepositoryExtensionSchemas(newMetadata); GetRepositoryEngineInfos(newMetadata); GetRepositoryUsers(newMetadata); GetRepositoryApps(newMetadata); IList <UnparsedSheet> unparsedSheets = GetRepositorySheets(); newMetadata.ParseSheets(unparsedSheets); newMetadata.PopulateFromCachedMetadata(oldMeta); string centralNodeHost; if (!TelemetryDashboardMain.ArgsManager.UseLocalEngine) { centralNodeHost = GetCentralNodeHostname(); TelemetryDashboardMain.Logger.Log("Got central node hostname for engine calls: " + centralNodeHost, LogLevel.Info); } else { centralNodeHost = CertificateConfigHelpers.Hostname; TelemetryDashboardMain.Logger.Log("Arg '-uselocalengine' was used. Using hostname '" + centralNodeHost + "' for all engine calls.", LogLevel.Info); } GetEngineObjects(centralNodeHost, newMetadata, engineRequestTimeoutMS); Stream SaveFileStream = File.Create(telemetryMetadataFile); BinaryFormatter serializer = new BinaryFormatter(); serializer.Serialize(SaveFileStream, newMetadata); SaveFileStream.Close(); MetadataWriter.DeleteMetadataFiles(); MetadataWriter.WriteMetadataToFile(newMetadata); return(0); }
private static void GetAboutServiceInfo(TelemetryMetadata telemetryMetadata) { TelemetryDashboardMain.Logger.Log("Getting system info from About Service on port 9032.", LogLevel.Info); Tuple <HttpStatusCode, string> aboutResponse = TelemetryDashboardMain.QRSRequest.MakeRequest("/v1/systemInfo", HttpMethod.Get, HTTPContentType.json, null, false, "9032"); JObject aboutJSON = JObject.Parse(aboutResponse.Item2); telemetryMetadata.Version = aboutJSON["version"].ToString(); telemetryMetadata.ReleaseLabel = aboutJSON["releaseLabel"].ToString(); }
private static void GetEngineObjects(string centralNodeHostname, TelemetryMetadata metadata, int engineRequestTimeoutMS) { TelemetryDashboardMain.Logger.Log(string.Format("Engine request timeout set to: {0} ms (default is: 30000 ms)", engineRequestTimeoutMS.ToString()), LogLevel.Info); Qlik.Sense.JsonRpc.RpcConnection.Timeout = engineRequestTimeoutMS; string wssPath = "https://" + centralNodeHostname + ":4747"; ILocation location = Location.FromUri(new Uri(wssPath)); X509Certificate2Collection certificateCollection = new X509Certificate2Collection(CertificateConfigHelpers.Certificate); // Defining the location as a direct connection to Qlik Sense Server location.AsDirectConnection("INTERNAL", "sa_api", certificateCollection: certificateCollection); int totalApps = metadata.Apps.Count; int currentApp = 0; TelemetryDashboardMain.Logger.Log("Will start to fetch all app objects from the engine.", LogLevel.Info); foreach (KeyValuePair <Guid, QRSApp> appTuple in metadata.Apps) { currentApp++; TelemetryDashboardMain.Logger.Log(string.Format("App {0} of {1} - Checking to see if visualaizations fetch is needed for app '{2}' with ID '{3}' ", currentApp, totalApps, appTuple.Value.Name, appTuple.Key.ToString()), LogLevel.Debug); if (appTuple.Value.VisualizationUpdateNeeded) { TelemetryDashboardMain.Logger.Log(string.Format("Getting visualizations for app '{0}' with ID '{1}' ", appTuple.Value.Name, appTuple.Key.ToString()), LogLevel.Info); try { IAppIdentifier appIdentifier = new AppIdentifier() { AppId = appTuple.Key.ToString() }; using (IApp app = location.App(appIdentifier, null, true)) { IEnumerable <ISheet> sheetList = app.GetSheets(); foreach (ISheet sheet in sheetList) { ISheetLayout sheetObject = (SheetLayout)sheet.GetLayout(); IList <Visualization> vizs = new List <Visualization>(); sheetObject.Cells.ToList().ForEach(c => vizs.Add(new Visualization(c.Name, c.Type))); metadata.Apps[appTuple.Key].Sheets.FirstOrDefault(s => s.Value.EngineObjectID == sheetObject.Info.Id).Value.SetSheetsList(vizs); } } } catch (Exception e) { TelemetryDashboardMain.Logger.Log("Failed to get engine objects from App: " + appTuple.Key.ToString() + ". Message: " + e.Message, LogLevel.Error); TelemetryDashboardMain.Logger.Log("Skipping app: " + appTuple.Key.ToString(), LogLevel.Error); } } } TelemetryDashboardMain.Logger.Log("Done getting all app objects from the engine.", LogLevel.Info); }
public List <TelemetryMetadata> GetMetadata(int hours) { List <TelemetryMetadata> results = new List <TelemetryMetadata>(); var matches = this.repositoryWrapper.TelemetryRepository.GetAll <TelemetryContainer>(f => f.AddedAtUtc >= DateTime.UtcNow.AddHours(-1 * hours)).ToList(); foreach (var item in matches.OrderByDescending(o => o.UtcDate)) { var result = new TelemetryMetadata(); result.ApplicationId = item.ApplicationId; result.DateTimeOffset = item.UtcDate; result.TelemetryLength = item.TelemetryData.Length; results.Add(result); } return(results); }
private static void WriteSystemInfoFile(TelemetryMetadata meta) { StringBuilder sb = new StringBuilder(); WriteHeaders(sb, HEADERS_SYSTEMINFO); foreach (Extension extension in meta.Extensions) { sb.Append(meta.Version); sb.Append(CSV_SEPARATOR); sb.Append(meta.ReleaseLabel); sb.Append('\n'); } WriteFile(sb, Path.Combine(FileLocationManager.GetTelemetrySharePath(), FileLocationManager.TELEMETRY_OUTPUT_FOLDER_NAME, FileLocationManager.METADATA_SYSTEMINFO_FILE_NAME)); }
private static void WriteEngineInfosMetadataFile(TelemetryMetadata meta) { StringBuilder sb = new StringBuilder(); WriteHeaders(sb, HEADERS_ENGINEINFOS); foreach (EngineInfo engine in meta.EngineInfos) { sb.Append(engine.Hostname); sb.Append(CSV_SEPARATOR); sb.Append("WorkingSetMin"); sb.Append(CSV_SEPARATOR); sb.Append(engine.WorkingSetMin); sb.Append('\n'); sb.Append(engine.Hostname); sb.Append(CSV_SEPARATOR); sb.Append("WorkingSetMax"); sb.Append(CSV_SEPARATOR); sb.Append(engine.WorkingSetMax); sb.Append('\n'); sb.Append(engine.Hostname); sb.Append(CSV_SEPARATOR); sb.Append("PerformanceLogLevel"); sb.Append(CSV_SEPARATOR); sb.Append(engine.PerformanceLogLevel); sb.Append('\n'); sb.Append(engine.Hostname); sb.Append(CSV_SEPARATOR); sb.Append("QIXPerformanceLogLevel"); sb.Append(CSV_SEPARATOR); sb.Append(engine.QIXPerformanceLogLevel); sb.Append('\n'); sb.Append(engine.Hostname); sb.Append(CSV_SEPARATOR); sb.Append("SessionLogLevel"); sb.Append(CSV_SEPARATOR); sb.Append(engine.SessionLogLevel); sb.Append('\n'); } WriteFile(sb, Path.Combine(FileLocationManager.GetTelemetrySharePath(), FileLocationManager.TELEMETRY_OUTPUT_FOLDER_NAME, FileLocationManager.METADATA_ENGINEINFOS_FILE_NAME)); }
private static void WriteExtensionSchemasMetadataFile(TelemetryMetadata meta) { StringBuilder sb = new StringBuilder(); WriteHeaders(sb, HEADERS_EXTENSIONSCHEMAS); foreach (ExtensionSchema schema in meta.ExtensionSchemas) { sb.Append(schema.ID); sb.Append(CSV_SEPARATOR); sb.Append(schema.Name); sb.Append(CSV_SEPARATOR); sb.Append(schema.Type); sb.Append('\n'); } WriteFile(sb, Path.Combine(FileLocationManager.GetTelemetrySharePath(), FileLocationManager.TELEMETRY_OUTPUT_FOLDER_NAME, FileLocationManager.METADATA_EXTENSIONSCHEMAS_FILE_NAME)); }
private static void GetRepositoryEngineInfos(TelemetryMetadata metadataObject) { string engineBody = @" { 'columns': [{ 'columnType': 'Property', 'definition': 'servernodeconfiguration.hostname', 'name': 'servernodeconfiguration.hostname' }, { 'columnType': 'Property', 'definition': 'settings.workingSetSizeLoPct', 'name': 'settings.workingSetSizeLoPct' }, { 'columnType': 'Property', 'definition': 'settings.workingSetSizeHiPct', 'name': 'settings.workingSetSizeHiPct' }, { 'columnType': 'Property', 'definition': 'settings.performanceLogVerbosity', 'name': 'settings.performanceLogVerbosity' }, { 'columnType': 'Property', 'definition': 'settings.qixPerformanceLogVerbosity', 'name': 'settings.qixPerformanceLogVerbosity' }, { 'columnType': 'Property', 'definition': 'settings.sessionLogVerbosity', 'name': 'settings.sessionLogVerbosity' }], 'entity': 'EngineService' }" ; Action <JArray> addAction = (engine) => metadataObject.EngineInfos.Add(new EngineInfo(engine[0].ToString(), engine[1].ToObject <int>(), engine[2].ToObject <int>(), (EngineLogLevel)engine[3].ToObject <int>(), (EngineLogLevel)engine[4].ToObject <int>(), (EngineLogLevel)engine[5].ToObject <int>())); GetRepositoryPagedObjects("engineservice", engineBody, addAction); }
private static void WriteUsersMetadataFile(TelemetryMetadata meta) { StringBuilder sb = new StringBuilder(); WriteHeaders(sb, HEADERS_USERS); foreach (User user in meta.Users) { sb.Append(user.ID.ToString()); sb.Append(CSV_SEPARATOR); sb.Append(user.UserID); sb.Append(CSV_SEPARATOR); sb.Append(user.UserDirectory); sb.Append(CSV_SEPARATOR); sb.Append(user.Username); sb.Append('\n'); } WriteFile(sb, Path.Combine(FileLocationManager.GetTelemetrySharePath(), FileLocationManager.TELEMETRY_OUTPUT_FOLDER_NAME, FileLocationManager.METADATA_USERS_FILE_NAME)); }
private static void GetRepositoryExtensionSchemas(TelemetryMetadata metadataObject) { TelemetryDashboardMain.Logger.Log("Fetching all extension schemas.", LogLevel.Info); Tuple <HttpStatusCode, string> extensionSchemas = TelemetryDashboardMain.QRSRequest.MakeRequest("/extension/schema", HttpMethod.Get); if (extensionSchemas.Item1 != HttpStatusCode.OK) { throw new InvalidResponseException(extensionSchemas.Item1.ToString() + " returned when trying to get all extension schemas. Request failed."); } TelemetryDashboardMain.Logger.Log("Got some extension schemas.", LogLevel.Debug); JObject parsedExtensionSchemas = JObject.Parse(extensionSchemas.Item2); foreach (JProperty schema in parsedExtensionSchemas.Children()) { JToken type = schema.Value["type"]; if (type != null) { string typeString = type.ToString(); if (typeString == "visualization") { string name; JToken jTokenName = schema.Value["name"]; if (jTokenName != null) { name = jTokenName.ToString(); } else { name = schema.Name; TelemetryDashboardMain.Logger.Log("No 'name' property found for extension schema object '" + name + "', using object name instead", LogLevel.Debug); } metadataObject.ExtensionSchemas.Add(new ExtensionSchema(schema.Name, name, typeString)); } } } }
private static void WriteExtensionsMetadataFile(TelemetryMetadata meta) { StringBuilder sb = new StringBuilder(); WriteHeaders(sb, HEADERS_EXTENSIONS); foreach (Extension extension in meta.Extensions) { sb.Append(extension.ID.ToString()); sb.Append(CSV_SEPARATOR); sb.Append(extension.CreatedDate.ToString("O")); sb.Append(CSV_SEPARATOR); sb.Append(extension.Name); sb.Append(CSV_SEPARATOR); sb.Append(extension.OwnerID.ToString()); sb.Append(CSV_SEPARATOR); sb.Append(extension.DashboardBundle.ToString()); sb.Append(CSV_SEPARATOR); sb.Append(extension.VisualizationBundle.ToString()); sb.Append('\n'); } WriteFile(sb, Path.Combine(FileLocationManager.GetTelemetrySharePath(), FileLocationManager.TELEMETRY_OUTPUT_FOLDER_NAME, FileLocationManager.METADATA_EXTENSIONS_FILE_NAME)); }
private static void WriteAppsMetadataFiles(TelemetryMetadata meta) { StringBuilder appsSB = new StringBuilder(); StringBuilder sheetsSB = new StringBuilder(); StringBuilder visualizationsSB = new StringBuilder(); WriteHeaders(appsSB, HEADERS_APPS); WriteHeaders(sheetsSB, HEADERS_SHEETS); WriteHeaders(visualizationsSB, HEADERS_VISUALIZATIONS); foreach (KeyValuePair <Guid, QRSApp> app in meta.Apps) { appsSB.Append(app.Key.ToString()); appsSB.Append(CSV_SEPARATOR); appsSB.Append(app.Value.Name); appsSB.Append(CSV_SEPARATOR); appsSB.Append(app.Value.AppOwnerID.ToString()); appsSB.Append(CSV_SEPARATOR); appsSB.Append(app.Value.Published); if (app.Value.Published) { appsSB.Append(CSV_SEPARATOR); appsSB.Append(app.Value.PublishedDateTime.ToString("o")); appsSB.Append(CSV_SEPARATOR); appsSB.Append(app.Value.StreamID.ToString()); appsSB.Append(CSV_SEPARATOR); appsSB.Append(app.Value.StreamName); } appsSB.Append('\n'); foreach (KeyValuePair <Guid, QRSSheet> sheet in app.Value.Sheets) { sheetsSB.Append(app.Key.ToString()); sheetsSB.Append(CSV_SEPARATOR); sheetsSB.Append(sheet.Value.EngineObjectID); sheetsSB.Append(CSV_SEPARATOR); sheetsSB.Append(sheet.Value.Name); sheetsSB.Append(CSV_SEPARATOR); sheetsSB.Append(sheet.Value.OwnerID); sheetsSB.Append(CSV_SEPARATOR); sheetsSB.Append(sheet.Value.Published); sheetsSB.Append(CSV_SEPARATOR); sheetsSB.Append(sheet.Value.Approved); sheetsSB.Append('\n'); foreach (Visualization viz in sheet.Value.Visualizations) { visualizationsSB.Append(app.Key.ToString() + '|' + sheet.Value.EngineObjectID); visualizationsSB.Append(CSV_SEPARATOR); visualizationsSB.Append(viz.ObjectName); visualizationsSB.Append(CSV_SEPARATOR); visualizationsSB.Append(viz.ObjectType); visualizationsSB.Append('\n'); } } } WriteFile(appsSB, Path.Combine(FileLocationManager.GetTelemetrySharePath(), FileLocationManager.TELEMETRY_OUTPUT_FOLDER_NAME, FileLocationManager.METADATA_APPS_FILE_NAME)); WriteFile(sheetsSB, Path.Combine(FileLocationManager.GetTelemetrySharePath(), FileLocationManager.TELEMETRY_OUTPUT_FOLDER_NAME, FileLocationManager.METADATA_SHEETS_FILE_NAME)); WriteFile(visualizationsSB, Path.Combine(FileLocationManager.GetTelemetrySharePath(), FileLocationManager.TELEMETRY_OUTPUT_FOLDER_NAME, FileLocationManager.METADATA_VISUALIZATIONS_FILE_NAME)); }
private static void GetRepositoryApps(TelemetryMetadata metadataObject) { string appBody = @" { 'columns': [{ 'columnType': 'Property', 'definition': 'id', 'name': 'id' }, { 'columnType': 'Property', 'definition': 'name', 'name': 'name' }, { 'columnType': 'Property', 'definition': 'modifiedDate', 'name': 'modifiedDate' }, { 'columnType': 'Property', 'definition': 'owner.id', 'name': 'owner.id' }, { 'columnType': 'Property', 'definition': 'published', 'name': 'published' }, { 'columnType': 'Property', 'definition': 'publishtime', 'name': 'publishtime' }, { 'columnType': 'Property', 'definition': 'stream.id', 'name': 'stream.id' }, { 'columnType': 'Property', 'definition': 'stream.name', 'name': 'stream.name' }], 'entity': 'App' }" ; Action <JArray> addAction = (app) => { Guid appID = app[0].ToObject <Guid>(); string appName = app[1].ToString(); TelemetryDashboardMain.Logger.Log(string.Format("Processing app '{1}' with ID '{0}'", appID, appName), LogLevel.Debug); bool published = app[4].ToObject <bool>(); QRSApp newApp; if (!published) { newApp = new QRSApp(appName, app[2].ToObject <DateTime>(), app[3].ToObject <Guid>(), published); } else { newApp = new QRSApp(appName, app[2].ToObject <DateTime>(), app[3].ToObject <Guid>(), published, app[5].ToObject <DateTime>(), app[6].ToObject <Guid>(), app[7].ToString()); } try { metadataObject.Apps.Add(appID, newApp); } catch (Exception e) { TelemetryDashboardMain.Logger.Log(string.Format("App '{1}' with ID '{0}' has already been added. This is probably due to an app being added while fetching the metadata. This error can likely be ignored. Internal error: {2}", appID, appName, e.Message), LogLevel.Error); } }; GetRepositoryPagedObjects("app", appBody, addAction); }
private static void GetRepositoryExtensions(TelemetryMetadata metadataObject) { string extensionBody = @" { 'columns': [{ 'columnType': 'Property', 'definition': 'id', 'name': 'id' }, { 'columnType': 'Property', 'definition': 'createdDate', 'name': 'createdDate' }, { 'columnType': 'Property', 'definition': 'name', 'name': 'name' }, { 'columnType': 'Property', 'definition': 'owner.id', 'name': 'owner.id' }, { 'name':'customProperties', 'columnType':'List', 'definition':'CustomPropertyValue', 'list':[ { 'name':'definition', 'columnType':'Property', 'definition':'definition' }, { 'name':'value', 'columnType':'Property', 'definition':'value' } ] } ], 'entity': 'Extension' }" ; Action <JArray> addAction = (extension) => { bool dashboardBundle = false; bool visualizationBundle = false; foreach (JArray cp in extension[4]["rows"]) { if (cp[0]["name"].ToString() == "ExtensionBundle") { string cpValue = cp[1].ToString(); if (cpValue == "Dashboard-bundle") { dashboardBundle = true; } else if (cpValue == "Visualization-bundle") { visualizationBundle = true; } else { TelemetryDashboardMain.Logger.Log("Found invalid custom property for 'ExtensionBundle'. Value was: " + cpValue, LogLevel.Error); } } } metadataObject.Extensions.Add(new Extension(extension[0].ToObject <Guid>(), extension[1].ToObject <DateTime>(), extension[2].ToString(), extension[3].ToObject <Guid>(), dashboardBundle, visualizationBundle)); }; GetRepositoryPagedObjects("extension", extensionBody, addAction); }