internal static void WriteMetadataToFile(TelemetryMetadata meta)
{
TelemetryDashboardMain.Logger.Log("Writing metadata files to disk.", LogLevel.Info);
// check to see if folder exists
TelemetryDashboardMain.Logger.Log("Checking to see if output dir exists.", LogLevel.Debug);
string outputDir = Path.Combine(FileLocationManager.GetTelemetrySharePath(), FileLocationManager.TELEMETRY_OUTPUT_FOLDER_NAME);
if (!Directory.Exists(outputDir))
{
TelemetryDashboardMain.Logger.Log("Output dir didn't exist. Creating.", LogLevel.Debug);
Directory.CreateDirectory(outputDir);
}
TelemetryDashboardMain.Logger.Log("Writing System Info file to disk.", LogLevel.Info);
WriteSystemInfoFile(meta);
TelemetryDashboardMain.Logger.Log("Writing Extensions file to disk.", LogLevel.Info);
WriteExtensionsMetadataFile(meta);
TelemetryDashboardMain.Logger.Log("Writing Extension Schemas file to disk.", LogLevel.Info);
WriteExtensionSchemasMetadataFile(meta);
TelemetryDashboardMain.Logger.Log("Writing Engine Infos file to disk.", LogLevel.Info);
WriteEngineInfosMetadataFile(meta);
TelemetryDashboardMain.Logger.Log("Writing User file to disk.", LogLevel.Info);
WriteUsersMetadataFile(meta);
TelemetryDashboardMain.Logger.Log("Writing all App metadata files to disk.", LogLevel.Info);
WriteAppsMetadataFiles(meta);
}
private static void GetRepositoryUsers(TelemetryMetadata metadataObject)
{
string userBody = @"
{
'columns':
[{
'columnType': 'Property',
'definition': 'id',
'name': 'id'
},
{
'columnType': 'Property',
'definition': 'userid',
'name': 'userid'
},
{
'columnType': 'Property',
'definition': 'userdirectory',
'name': 'userdirectory'
},
{
'columnType': 'Property',
'definition': 'name',
'name': 'name'
}],
'entity': 'User'
}" ;
Action <JArray> addAction = (user) => metadataObject.Users.Add(new User(user[0].ToObject <Guid>(), user[1].ToString(), user[2].ToString(), user[3].ToString()));
GetRepositoryPagedObjects("user", userBody, addAction);
}
internal static int Run(int engineRequestTimeoutMS)
{
// check to see if metadata binary exists
string telemetryMetadataFile = Path.Combine(FileLocationManager.GetTelemetrySharePath(), FileLocationManager.METADATA_BINARY_FILE_NAME);
TelemetryDashboardMain.Logger.Log("Trying to get metadata binary file: " + telemetryMetadataFile, LogLevel.Debug);
TelemetryMetadata oldMeta, newMetadata;
if (File.Exists(telemetryMetadataFile))
{
TelemetryDashboardMain.Logger.Log("Found metadata file, will load contents.", LogLevel.Info);
Stream openFileStream = File.OpenRead(telemetryMetadataFile);
BinaryFormatter deserializer = new BinaryFormatter();
oldMeta = (TelemetryMetadata)deserializer.Deserialize(openFileStream);
openFileStream.Close();
}
else
{
oldMeta = new TelemetryMetadata();
}
newMetadata = new TelemetryMetadata(true);
GetAboutServiceInfo(newMetadata);
GetRepositoryExtensions(newMetadata);
GetRepositoryExtensionSchemas(newMetadata);
GetRepositoryEngineInfos(newMetadata);
GetRepositoryUsers(newMetadata);
GetRepositoryApps(newMetadata);
IList <UnparsedSheet> unparsedSheets = GetRepositorySheets();
newMetadata.ParseSheets(unparsedSheets);
newMetadata.PopulateFromCachedMetadata(oldMeta);
string centralNodeHost;
if (!TelemetryDashboardMain.ArgsManager.UseLocalEngine)
{
centralNodeHost = GetCentralNodeHostname();
TelemetryDashboardMain.Logger.Log("Got central node hostname for engine calls: " + centralNodeHost, LogLevel.Info);
}
else
{
centralNodeHost = CertificateConfigHelpers.Hostname;
TelemetryDashboardMain.Logger.Log("Arg '-uselocalengine' was used. Using hostname '" + centralNodeHost + "' for all engine calls.", LogLevel.Info);
}
GetEngineObjects(centralNodeHost, newMetadata, engineRequestTimeoutMS);
Stream SaveFileStream = File.Create(telemetryMetadataFile);
BinaryFormatter serializer = new BinaryFormatter();
serializer.Serialize(SaveFileStream, newMetadata);
SaveFileStream.Close();
MetadataWriter.DeleteMetadataFiles();
MetadataWriter.WriteMetadataToFile(newMetadata);
return(0);
}
private static void GetAboutServiceInfo(TelemetryMetadata telemetryMetadata)
{
TelemetryDashboardMain.Logger.Log("Getting system info from About Service on port 9032.", LogLevel.Info);
Tuple <HttpStatusCode, string> aboutResponse = TelemetryDashboardMain.QRSRequest.MakeRequest("/v1/systemInfo", HttpMethod.Get, HTTPContentType.json, null, false, "9032");
JObject aboutJSON = JObject.Parse(aboutResponse.Item2);
telemetryMetadata.Version = aboutJSON["version"].ToString();
telemetryMetadata.ReleaseLabel = aboutJSON["releaseLabel"].ToString();
}
private static void GetEngineObjects(string centralNodeHostname, TelemetryMetadata metadata, int engineRequestTimeoutMS)
{
TelemetryDashboardMain.Logger.Log(string.Format("Engine request timeout set to: {0} ms (default is: 30000 ms)", engineRequestTimeoutMS.ToString()), LogLevel.Info);
Qlik.Sense.JsonRpc.RpcConnection.Timeout = engineRequestTimeoutMS;
string wssPath = "https://" + centralNodeHostname + ":4747";
ILocation location = Location.FromUri(new Uri(wssPath));
X509Certificate2Collection certificateCollection = new X509Certificate2Collection(CertificateConfigHelpers.Certificate);
// Defining the location as a direct connection to Qlik Sense Server
location.AsDirectConnection("INTERNAL", "sa_api", certificateCollection: certificateCollection);
int totalApps = metadata.Apps.Count;
int currentApp = 0;
TelemetryDashboardMain.Logger.Log("Will start to fetch all app objects from the engine.", LogLevel.Info);
foreach (KeyValuePair <Guid, QRSApp> appTuple in metadata.Apps)
{
currentApp++;
TelemetryDashboardMain.Logger.Log(string.Format("App {0} of {1} - Checking to see if visualaizations fetch is needed for app '{2}' with ID '{3}' ", currentApp, totalApps, appTuple.Value.Name, appTuple.Key.ToString()), LogLevel.Debug);
if (appTuple.Value.VisualizationUpdateNeeded)
{
TelemetryDashboardMain.Logger.Log(string.Format("Getting visualizations for app '{0}' with ID '{1}' ", appTuple.Value.Name, appTuple.Key.ToString()), LogLevel.Info);
try
{
IAppIdentifier appIdentifier = new AppIdentifier()
{
AppId = appTuple.Key.ToString()
};
using (IApp app = location.App(appIdentifier, null, true))
{
IEnumerable <ISheet> sheetList = app.GetSheets();
foreach (ISheet sheet in sheetList)
{
ISheetLayout sheetObject = (SheetLayout)sheet.GetLayout();
IList <Visualization> vizs = new List <Visualization>();
sheetObject.Cells.ToList().ForEach(c => vizs.Add(new Visualization(c.Name, c.Type)));
metadata.Apps[appTuple.Key].Sheets.FirstOrDefault(s => s.Value.EngineObjectID == sheetObject.Info.Id).Value.SetSheetsList(vizs);
}
}
}
catch (Exception e)
{
TelemetryDashboardMain.Logger.Log("Failed to get engine objects from App: " + appTuple.Key.ToString() + ". Message: " + e.Message, LogLevel.Error);
TelemetryDashboardMain.Logger.Log("Skipping app: " + appTuple.Key.ToString(), LogLevel.Error);
}
}
}
TelemetryDashboardMain.Logger.Log("Done getting all app objects from the engine.", LogLevel.Info);
}
public List <TelemetryMetadata> GetMetadata(int hours)
{
List <TelemetryMetadata> results = new List <TelemetryMetadata>();
var matches = this.repositoryWrapper.TelemetryRepository.GetAll <TelemetryContainer>(f => f.AddedAtUtc >= DateTime.UtcNow.AddHours(-1 * hours)).ToList();
foreach (var item in matches.OrderByDescending(o => o.UtcDate))
{
var result = new TelemetryMetadata();
result.ApplicationId = item.ApplicationId;
result.DateTimeOffset = item.UtcDate;
result.TelemetryLength = item.TelemetryData.Length;
results.Add(result);
}
return(results);
}
private static void WriteSystemInfoFile(TelemetryMetadata meta)
{
StringBuilder sb = new StringBuilder();
WriteHeaders(sb, HEADERS_SYSTEMINFO);
foreach (Extension extension in meta.Extensions)
{
sb.Append(meta.Version);
sb.Append(CSV_SEPARATOR);
sb.Append(meta.ReleaseLabel);
sb.Append('\n');
}
WriteFile(sb, Path.Combine(FileLocationManager.GetTelemetrySharePath(), FileLocationManager.TELEMETRY_OUTPUT_FOLDER_NAME, FileLocationManager.METADATA_SYSTEMINFO_FILE_NAME));
}
private static void WriteEngineInfosMetadataFile(TelemetryMetadata meta)
{
StringBuilder sb = new StringBuilder();
WriteHeaders(sb, HEADERS_ENGINEINFOS);
foreach (EngineInfo engine in meta.EngineInfos)
{
sb.Append(engine.Hostname);
sb.Append(CSV_SEPARATOR);
sb.Append("WorkingSetMin");
sb.Append(CSV_SEPARATOR);
sb.Append(engine.WorkingSetMin);
sb.Append('\n');
sb.Append(engine.Hostname);
sb.Append(CSV_SEPARATOR);
sb.Append("WorkingSetMax");
sb.Append(CSV_SEPARATOR);
sb.Append(engine.WorkingSetMax);
sb.Append('\n');
sb.Append(engine.Hostname);
sb.Append(CSV_SEPARATOR);
sb.Append("PerformanceLogLevel");
sb.Append(CSV_SEPARATOR);
sb.Append(engine.PerformanceLogLevel);
sb.Append('\n');
sb.Append(engine.Hostname);
sb.Append(CSV_SEPARATOR);
sb.Append("QIXPerformanceLogLevel");
sb.Append(CSV_SEPARATOR);
sb.Append(engine.QIXPerformanceLogLevel);
sb.Append('\n');
sb.Append(engine.Hostname);
sb.Append(CSV_SEPARATOR);
sb.Append("SessionLogLevel");
sb.Append(CSV_SEPARATOR);
sb.Append(engine.SessionLogLevel);
sb.Append('\n');
}
WriteFile(sb, Path.Combine(FileLocationManager.GetTelemetrySharePath(), FileLocationManager.TELEMETRY_OUTPUT_FOLDER_NAME, FileLocationManager.METADATA_ENGINEINFOS_FILE_NAME));
}
private static void WriteExtensionSchemasMetadataFile(TelemetryMetadata meta)
{
StringBuilder sb = new StringBuilder();
WriteHeaders(sb, HEADERS_EXTENSIONSCHEMAS);
foreach (ExtensionSchema schema in meta.ExtensionSchemas)
{
sb.Append(schema.ID);
sb.Append(CSV_SEPARATOR);
sb.Append(schema.Name);
sb.Append(CSV_SEPARATOR);
sb.Append(schema.Type);
sb.Append('\n');
}
WriteFile(sb, Path.Combine(FileLocationManager.GetTelemetrySharePath(), FileLocationManager.TELEMETRY_OUTPUT_FOLDER_NAME, FileLocationManager.METADATA_EXTENSIONSCHEMAS_FILE_NAME));
}
private static void GetRepositoryEngineInfos(TelemetryMetadata metadataObject)
{
string engineBody = @"
{
'columns':
[{
'columnType': 'Property',
'definition': 'servernodeconfiguration.hostname',
'name': 'servernodeconfiguration.hostname'
},
{
'columnType': 'Property',
'definition': 'settings.workingSetSizeLoPct',
'name': 'settings.workingSetSizeLoPct'
},
{
'columnType': 'Property',
'definition': 'settings.workingSetSizeHiPct',
'name': 'settings.workingSetSizeHiPct'
},
{
'columnType': 'Property',
'definition': 'settings.performanceLogVerbosity',
'name': 'settings.performanceLogVerbosity'
},
{
'columnType': 'Property',
'definition': 'settings.qixPerformanceLogVerbosity',
'name': 'settings.qixPerformanceLogVerbosity'
},
{
'columnType': 'Property',
'definition': 'settings.sessionLogVerbosity',
'name': 'settings.sessionLogVerbosity'
}],
'entity': 'EngineService'
}" ;
Action <JArray> addAction = (engine) => metadataObject.EngineInfos.Add(new EngineInfo(engine[0].ToString(), engine[1].ToObject <int>(), engine[2].ToObject <int>(), (EngineLogLevel)engine[3].ToObject <int>(), (EngineLogLevel)engine[4].ToObject <int>(), (EngineLogLevel)engine[5].ToObject <int>()));
GetRepositoryPagedObjects("engineservice", engineBody, addAction);
}
private static void WriteUsersMetadataFile(TelemetryMetadata meta)
{
StringBuilder sb = new StringBuilder();
WriteHeaders(sb, HEADERS_USERS);
foreach (User user in meta.Users)
{
sb.Append(user.ID.ToString());
sb.Append(CSV_SEPARATOR);
sb.Append(user.UserID);
sb.Append(CSV_SEPARATOR);
sb.Append(user.UserDirectory);
sb.Append(CSV_SEPARATOR);
sb.Append(user.Username);
sb.Append('\n');
}
WriteFile(sb, Path.Combine(FileLocationManager.GetTelemetrySharePath(), FileLocationManager.TELEMETRY_OUTPUT_FOLDER_NAME, FileLocationManager.METADATA_USERS_FILE_NAME));
}
private static void GetRepositoryExtensionSchemas(TelemetryMetadata metadataObject)
{
TelemetryDashboardMain.Logger.Log("Fetching all extension schemas.", LogLevel.Info);
Tuple <HttpStatusCode, string> extensionSchemas = TelemetryDashboardMain.QRSRequest.MakeRequest("/extension/schema", HttpMethod.Get);
if (extensionSchemas.Item1 != HttpStatusCode.OK)
{
throw new InvalidResponseException(extensionSchemas.Item1.ToString() + " returned when trying to get all extension schemas. Request failed.");
}
TelemetryDashboardMain.Logger.Log("Got some extension schemas.", LogLevel.Debug);
JObject parsedExtensionSchemas = JObject.Parse(extensionSchemas.Item2);
foreach (JProperty schema in parsedExtensionSchemas.Children())
{
JToken type = schema.Value["type"];
if (type != null)
{
string typeString = type.ToString();
if (typeString == "visualization")
{
string name;
JToken jTokenName = schema.Value["name"];
if (jTokenName != null)
{
name = jTokenName.ToString();
}
else
{
name = schema.Name;
TelemetryDashboardMain.Logger.Log("No 'name' property found for extension schema object '" + name + "', using object name instead", LogLevel.Debug);
}
metadataObject.ExtensionSchemas.Add(new ExtensionSchema(schema.Name, name, typeString));
}
}
}
}
private static void WriteExtensionsMetadataFile(TelemetryMetadata meta)
{
StringBuilder sb = new StringBuilder();
WriteHeaders(sb, HEADERS_EXTENSIONS);
foreach (Extension extension in meta.Extensions)
{
sb.Append(extension.ID.ToString());
sb.Append(CSV_SEPARATOR);
sb.Append(extension.CreatedDate.ToString("O"));
sb.Append(CSV_SEPARATOR);
sb.Append(extension.Name);
sb.Append(CSV_SEPARATOR);
sb.Append(extension.OwnerID.ToString());
sb.Append(CSV_SEPARATOR);
sb.Append(extension.DashboardBundle.ToString());
sb.Append(CSV_SEPARATOR);
sb.Append(extension.VisualizationBundle.ToString());
sb.Append('\n');
}
WriteFile(sb, Path.Combine(FileLocationManager.GetTelemetrySharePath(), FileLocationManager.TELEMETRY_OUTPUT_FOLDER_NAME, FileLocationManager.METADATA_EXTENSIONS_FILE_NAME));
}
private static void WriteAppsMetadataFiles(TelemetryMetadata meta)
{
StringBuilder appsSB = new StringBuilder();
StringBuilder sheetsSB = new StringBuilder();
StringBuilder visualizationsSB = new StringBuilder();
WriteHeaders(appsSB, HEADERS_APPS);
WriteHeaders(sheetsSB, HEADERS_SHEETS);
WriteHeaders(visualizationsSB, HEADERS_VISUALIZATIONS);
foreach (KeyValuePair <Guid, QRSApp> app in meta.Apps)
{
appsSB.Append(app.Key.ToString());
appsSB.Append(CSV_SEPARATOR);
appsSB.Append(app.Value.Name);
appsSB.Append(CSV_SEPARATOR);
appsSB.Append(app.Value.AppOwnerID.ToString());
appsSB.Append(CSV_SEPARATOR);
appsSB.Append(app.Value.Published);
if (app.Value.Published)
{
appsSB.Append(CSV_SEPARATOR);
appsSB.Append(app.Value.PublishedDateTime.ToString("o"));
appsSB.Append(CSV_SEPARATOR);
appsSB.Append(app.Value.StreamID.ToString());
appsSB.Append(CSV_SEPARATOR);
appsSB.Append(app.Value.StreamName);
}
appsSB.Append('\n');
foreach (KeyValuePair <Guid, QRSSheet> sheet in app.Value.Sheets)
{
sheetsSB.Append(app.Key.ToString());
sheetsSB.Append(CSV_SEPARATOR);
sheetsSB.Append(sheet.Value.EngineObjectID);
sheetsSB.Append(CSV_SEPARATOR);
sheetsSB.Append(sheet.Value.Name);
sheetsSB.Append(CSV_SEPARATOR);
sheetsSB.Append(sheet.Value.OwnerID);
sheetsSB.Append(CSV_SEPARATOR);
sheetsSB.Append(sheet.Value.Published);
sheetsSB.Append(CSV_SEPARATOR);
sheetsSB.Append(sheet.Value.Approved);
sheetsSB.Append('\n');
foreach (Visualization viz in sheet.Value.Visualizations)
{
visualizationsSB.Append(app.Key.ToString() + '|' + sheet.Value.EngineObjectID);
visualizationsSB.Append(CSV_SEPARATOR);
visualizationsSB.Append(viz.ObjectName);
visualizationsSB.Append(CSV_SEPARATOR);
visualizationsSB.Append(viz.ObjectType);
visualizationsSB.Append('\n');
}
}
}
WriteFile(appsSB, Path.Combine(FileLocationManager.GetTelemetrySharePath(), FileLocationManager.TELEMETRY_OUTPUT_FOLDER_NAME, FileLocationManager.METADATA_APPS_FILE_NAME));
WriteFile(sheetsSB, Path.Combine(FileLocationManager.GetTelemetrySharePath(), FileLocationManager.TELEMETRY_OUTPUT_FOLDER_NAME, FileLocationManager.METADATA_SHEETS_FILE_NAME));
WriteFile(visualizationsSB, Path.Combine(FileLocationManager.GetTelemetrySharePath(), FileLocationManager.TELEMETRY_OUTPUT_FOLDER_NAME, FileLocationManager.METADATA_VISUALIZATIONS_FILE_NAME));
}
private static void GetRepositoryApps(TelemetryMetadata metadataObject)
{
string appBody = @"
{
'columns':
[{
'columnType': 'Property',
'definition': 'id',
'name': 'id'
},
{
'columnType': 'Property',
'definition': 'name',
'name': 'name'
},
{
'columnType': 'Property',
'definition': 'modifiedDate',
'name': 'modifiedDate'
},
{
'columnType': 'Property',
'definition': 'owner.id',
'name': 'owner.id'
},
{
'columnType': 'Property',
'definition': 'published',
'name': 'published'
},
{
'columnType': 'Property',
'definition': 'publishtime',
'name': 'publishtime'
},
{
'columnType': 'Property',
'definition': 'stream.id',
'name': 'stream.id'
},
{
'columnType': 'Property',
'definition': 'stream.name',
'name': 'stream.name'
}],
'entity': 'App'
}" ;
Action <JArray> addAction = (app) =>
{
Guid appID = app[0].ToObject <Guid>();
string appName = app[1].ToString();
TelemetryDashboardMain.Logger.Log(string.Format("Processing app '{1}' with ID '{0}'", appID, appName), LogLevel.Debug);
bool published = app[4].ToObject <bool>();
QRSApp newApp;
if (!published)
{
newApp = new QRSApp(appName, app[2].ToObject <DateTime>(), app[3].ToObject <Guid>(), published);
}
else
{
newApp = new QRSApp(appName, app[2].ToObject <DateTime>(), app[3].ToObject <Guid>(), published, app[5].ToObject <DateTime>(), app[6].ToObject <Guid>(), app[7].ToString());
}
try
{
metadataObject.Apps.Add(appID, newApp);
}
catch (Exception e)
{
TelemetryDashboardMain.Logger.Log(string.Format("App '{1}' with ID '{0}' has already been added. This is probably due to an app being added while fetching the metadata. This error can likely be ignored. Internal error: {2}", appID, appName, e.Message), LogLevel.Error);
}
};
GetRepositoryPagedObjects("app", appBody, addAction);
}
private static void GetRepositoryExtensions(TelemetryMetadata metadataObject)
{
string extensionBody = @"
{
'columns':
[{
'columnType': 'Property',
'definition': 'id',
'name': 'id'
},
{
'columnType': 'Property',
'definition': 'createdDate',
'name': 'createdDate'
},
{
'columnType': 'Property',
'definition': 'name',
'name': 'name'
},
{
'columnType': 'Property',
'definition': 'owner.id',
'name': 'owner.id'
},
{
'name':'customProperties',
'columnType':'List',
'definition':'CustomPropertyValue',
'list':[
{
'name':'definition',
'columnType':'Property',
'definition':'definition'
},
{
'name':'value',
'columnType':'Property',
'definition':'value'
}
]
}
],
'entity': 'Extension'
}" ;
Action <JArray> addAction = (extension) =>
{
bool dashboardBundle = false;
bool visualizationBundle = false;
foreach (JArray cp in extension[4]["rows"])
{
if (cp[0]["name"].ToString() == "ExtensionBundle")
{
string cpValue = cp[1].ToString();
if (cpValue == "Dashboard-bundle")
{
dashboardBundle = true;
}
else if (cpValue == "Visualization-bundle")
{
visualizationBundle = true;
}
else
{
TelemetryDashboardMain.Logger.Log("Found invalid custom property for 'ExtensionBundle'. Value was: " + cpValue, LogLevel.Error);
}
}
}
metadataObject.Extensions.Add(new Extension(extension[0].ToObject <Guid>(), extension[1].ToObject <DateTime>(), extension[2].ToString(), extension[3].ToObject <Guid>(), dashboardBundle, visualizationBundle));
};
GetRepositoryPagedObjects("extension", extensionBody, addAction);
}