public void Configure(IWriterFactory writerFactory, IConfiguration pluginConfig, IProcessingNotificationsCollector processingNotificationsCollector, ILoggerFactory loggerFactory)
{
_processingNotificationsCollector = processingNotificationsCollector;
_desktopWriter = writerFactory.GetWriter <FlattenedArtEvent>(DesktopOutputInfo);
_serverWriter = writerFactory.GetWriter <FlattenedArtEvent>(ServerOutputInfo);
_jsonSerializer = JsonSerializer.Create();
}
public void Configure(IWriterFactory writerFactory, IConfiguration pluginConfig, IProcessingNotificationsCollector processingNotificationsCollector, ILoggerFactory loggerFactory)
{
_logger = loggerFactory.CreateLogger <ReplayerPlugin>();
_processingNotificationsCollector = processingNotificationsCollector;
PopulateTimeZoneDictionary();
CheckForMissingSystemTimeZones();
// get the relevant vizqlserver events from config if specified
_relevantVizqlServerEvents = pluginConfig?.GetConfigurationValueOrDefault(ReplayRelevantEventsKey, _relevantVizqlServerEvents.ToArray(), _logger)
.ToHashSet();
// get the relevant vizqlserver events from config if specified
_skippableCommands = pluginConfig?.GetConfigurationValueOrDefault(ReplaySkippableCommandsKey, _skippableCommands.ToArray(), _logger)
.ToHashSet();
// get path of the Json file from config if specified
_replayerOutputDirectory = pluginConfig?.GetConfigurationValueOrDefault(ReplayerOutputDirectoryKey, _replayerOutputDirectory, _logger);
// get the file containing the time zones dictionary
_timeZonesDictionaryFile = pluginConfig?.GetConfigurationValueOrDefault(ReplayerTimeZonesDictionaryFileKey, _timeZonesDictionaryFile, _logger);
// get replay file name of the Json file from config if specified
_jsonReplayFile = pluginConfig?.GetConfigurationValueOrDefault(ReplayFileNameKey, string.Format($"Playback{DateTime.Now:_dd_MM_-HH-mm-ss}.json"), _logger);
if (string.IsNullOrEmpty(_jsonReplayFile))
{
_jsonReplayFile = string.Format($"Playback{DateTime.Now:_dd_MM_-HH-mm-ss}.json");
}
}
public ProcessInfoExtractor(ConfigFile workgroupYml, ConfigFile tabsvcYml, IProcessingNotificationsCollector processingNotificationsCollector)
{
_workgroupYml = workgroupYml;
_tabsvcYml = tabsvcYml;
_hostnameMap = GetWorkerHostnameMap(workgroupYml, processingNotificationsCollector);
_processingNotificationsCollector = processingNotificationsCollector;
}
public IEnumerable <IPlugin> CreatePlugins(IWriterFactory writerFactory, IProcessingNotificationsCollector processingNotificationsCollector)
{
var pluginNamesToLoad = GetPluginNamesToLoad().ToList();
_logger.LogInformation($"Loading plugins: {string.Join(';', pluginNamesToLoad)}");
foreach (var pluginName in pluginNamesToLoad)
{
IPlugin plugin;
try
{
plugin = _pluginFactory.CreatePlugin(pluginName);
}
catch (Exception ex)
{
_logger.LogError($"Exception occurred when creating plugin '{pluginName}'. See inner exception for details.", ex);
throw;
}
try
{
var pluginConfiguration = _config.GetPluginConfiguration(plugin.Name);
plugin.Configure(writerFactory, pluginConfiguration, processingNotificationsCollector, _loggerFactory);
}
catch (Exception ex)
{
_logger.LogError(ex, $"Exception occurred when configuring plugin '{pluginName}'. See inner exception for details.");
throw;
}
_plugins.Add(plugin);
}
return(_plugins);
}
public void Configure(IWriterFactory writerFactory, IConfiguration pluginConfig, IProcessingNotificationsCollector processingNotificationsCollector, ILoggerFactory loggerFactory)
{
_configEntriesWriter = writerFactory.GetWriter <ConfigEntry>(ConfigEntriesOutputInfo);
_processTopologyWriter = writerFactory.GetWriter <ConfigProcessInfo>(ProcessTopologyOutputInfo);
_processedConfigFiles = new List <ConfigFile>();
_logger = loggerFactory.CreateLogger <ConfigPlugin>();
_processingNotificationsCollector = processingNotificationsCollector;
}
public void Configure(IWriterFactory writerFactory, IConfiguration pluginConfig, IProcessingNotificationsCollector processingNotificationsCollector, ILoggerFactory loggerFactory)
{
_processingNotificationsCollector = processingNotificationsCollector;
_tabadminControllerEventParser = new TabadminControllerEventParser(
new BuildTracker(processingNotificationsCollector),
_processingNotificationsCollector);
_writer = writerFactory.GetWriter <TabadminControllerEvent>(OutputInfo);
}
public void Configure(IWriterFactory writerFactory, IConfiguration pluginConfig, IProcessingNotificationsCollector processingNotificationsCollector, ILoggerFactory loggerFactory)
{
_tabadminActionWriter = writerFactory.GetWriter <TabadminAction>(TabadminActionOutputInfo);
_tableauServerVersionWriter = writerFactory.GetWriter <TableauServerVersion>(TableauServerVersionOutputInfo);
_tabadminErrorWriter = writerFactory.GetWriter <TabadminError>(TabadminErrorOutputInfo);
_versionsDict = new Dictionary <string, List <TableauServerVersion> >();
_processingNotificationsCollector = processingNotificationsCollector;
}
public ResourceManagerEventsProcessor(IWriterFactory writerFactory, IProcessingNotificationsCollector processingNotificationsCollector)
{
_processingNotificationsCollector = processingNotificationsCollector;
_actionsWriter = writerFactory.GetWriter <ResourceManagerAction>(ActionsDataSetInfo);
_cpuSamplesWriter = writerFactory.GetWriter <ResourceManagerCpuSample>(CpuSamplesDataSetInfo);
_memorySamplesWriter = writerFactory.GetWriter <ResourceManagerMemorySample>(MemorySamplesDataSetInfo);
_thresholdsWriters = writerFactory.GetWriter <ResourceManagerThreshold>(ThresholdsDataSetInfo);
_cpuUsageWriter = writerFactory.GetWriter <ResourceManagerHighCpuUsage>(HighCpuUsageDataSetInfo);
}
public void Configure(IWriterFactory writerFactory, IConfiguration pluginConfig, IProcessingNotificationsCollector processingNotificationsCollector, ILoggerFactory loggerFactory)
{
_errorWriter = writerFactory.GetWriter <ClusterControllerError>(_errorDSI);
_postgresActionWriter = writerFactory.GetWriter <ClusterControllerPostgresAction>(_postgresActionDSI);
_diskIOSampleWriter = writerFactory.GetWriter <ClusterControllerDiskIoSample>(_diskIOSampleDSI);
_zkErrorWriter = writerFactory.GetWriter <ZookeeperError>(_zkErrorDSI);
_zkFsyncLatencyWriter = writerFactory.GetWriter <ZookeeperFsyncLatency>(_zkFsyncLatencyDSI);
_processingNotificationsCollector = processingNotificationsCollector;
}
public NativeJsonLogsReader(Stream stream, string filePath, IProcessingNotificationsCollector processingNotificationsCollector)
{
_filePath = filePath;
_processingNotificationsCollector = processingNotificationsCollector;;
_stream = stream;
_serializerSettings = new JsonSerializerSettings
{
DateTimeZoneHandling = DateTimeZoneHandling.Unspecified
};
}
public CsvLogReader(Stream stream, string filePath, IProcessingNotificationsCollector processingNotificationsCollector)
{
_stream = stream;
_filePath = filePath;
_processingNotificationsCollector = processingNotificationsCollector;
_csvConfig = new CsvConfiguration(CultureInfo.InvariantCulture)
{
BufferSize = 1048576, // 1 MB, this drastically speeds up reading long lines
BadDataFound = BadDataFound,
MissingFieldFound = MissingFieldFound,
HasHeaderRecord = false
};
}
public void Configure(IWriterFactory writerFactory, IConfiguration pluginConfig, IProcessingNotificationsCollector processingNotificationsCollector, ILoggerFactory loggerFactory)
{
_writer = writerFactory.GetWriter <ApacheEvent>(OutputInfo);
var section = pluginConfig?.GetSection("IncludeGatewayChecks");
var sectionExists = section != null && section.Exists();
if (!sectionExists)
{
var logger = loggerFactory.CreateLogger <ApachePlugin>();
logger.LogWarning("{pluginName} config was missing from Plugins Configuration. Using default value of {defaultValue} for {missingParameterName} parameter", nameof(ApachePlugin), DefaultIncludeGatewayHealthChecksValue, "IncludeGatewayChecks");
}
_includeGatewayHealthChecks = sectionExists ? section.Get <bool>() : DefaultIncludeGatewayHealthChecksValue;
_processingNotificationsCollector = processingNotificationsCollector;
}
public TableauLogsProcessor(
LogSharkConfiguration config,
IPluginManager pluginManager,
IWriterFactory writerFactory,
ILogTypeDetails logTypeDetails,
IProcessingNotificationsCollector processingNotificationsCollector,
ILoggerFactory loggerFactory)
{
_config = config;
_loggerFactory = loggerFactory;
_logger = _loggerFactory.CreateLogger <TableauLogsProcessor>();
_logTypeDetails = logTypeDetails;
_pluginManager = pluginManager;
_processingNotificationsCollector = processingNotificationsCollector;
_writerFactory = writerFactory;
}
public void Configure(IWriterFactory writerFactory, IConfiguration pluginConfig, IProcessingNotificationsCollector processingNotificationsCollector, ILoggerFactory loggerFactory)
{
var section = pluginConfig?.GetSection("MaxQueryLength");
var sectionExists = section != null && section.Exists();
var parsedMaxQueryLength = 0;
var parsedSuccessfully = sectionExists && int.TryParse(section.Value, out parsedMaxQueryLength);
if (!parsedSuccessfully)
{
var logger = loggerFactory.CreateLogger <VizqlDesktopPlugin>();
logger.LogWarning("{pluginName} config was missing from Plugins Configuration or contained incorrect value. Using default value of {defaultValue} for {missingParameterName} parameter", nameof(VizqlDesktopPlugin), DefaultMaxQueryLength, "MaxQueryLength");
}
var maxQueryLength = parsedSuccessfully ? parsedMaxQueryLength : DefaultMaxQueryLength;
_eventProcessor = new VizqlDesktopEventProcessor(writerFactory, maxQueryLength);
_processingNotificationsCollector = processingNotificationsCollector;
}
public TableauLogsExtractor(string logSetPath, string tempDir, IProcessingNotificationsCollector processingNotificationsCollector, ILogger logger)
{
_logger = logger;
_logger.LogInformation("Starting {extractorName} for log set `{logSetPath}`", nameof(TableauLogsExtractor), logSetPath ?? "(null)");
if (!File.Exists(logSetPath) && !Directory.Exists(logSetPath))
{
var message = $"`{logSetPath}` does not exist";
_logger.LogError(message);
throw new ArgumentException(message);
}
_rootPath = logSetPath;
_tempFileTracker = new TempFileTracker();
LogSetParts = EvaluateLogSetStructure(logSetPath, tempDir, processingNotificationsCollector);
_logger.LogInformation("Completed {extractorName} for log set `{logSetPath}`", nameof(TableauLogsExtractor), logSetPath ?? "(null)");
}
private static IEnumerable <LogTypeInfo> LoadDetails(IProcessingNotificationsCollector processingNotificationsCollector)
{
return(new List <LogTypeInfo>
{
new LogTypeInfo(
logType: LogType.Apache,
logReaderProvider: (stream, filePath) => new SimpleLinePerLineReader(stream),
fileLocations: new List <Regex>
{
TabadminLog("httpd", "access"), // pre-TSM - httpd/access.*.log (access.2015_05_18_00_00_00.log)
TsmV0Log("httpd", "access"), // TSMv0 - localhost\tabadminagent_0.20181.18.0404.16052600117725665315795\logs\httpd\access.2018_08_08_00_00_00.log
TsmLog("gateway", "access"), // TSM - node2\gateway_0.20182.18.0627.22308342643935754496180\logs\access.2018_08_08_00_00_00.log
}),
new LogTypeInfo(
logType: LogType.BackgrounderCpp,
logReaderProvider: (stream, filePath) => new NativeJsonLogsReader(stream, filePath, processingNotificationsCollector),
fileLocations: new List <Regex>
{
TabadminNativeLog("backgrounder"), // pre-TSM - vizqlserver\Logs\backgrounder-0_2018_07_28_00_00_00.txt
TsmV0NativeLog("backgrounder"), // TSMv0 - localhost\tabadminagent_0.20181.18.0404.16052600117725665315795\logs\backgrounder\nativeapi_backgrounder_1-0_2018_08_07_00_00_00.txt
TsmNativeLog("backgrounder"), // TSM - node2\backgrounder_0.20182.18.0627.22306436150448756480580\logs\nativeapi_backgrounder_2-1_2018_08_08_00_00_00.txt
}),
new LogTypeInfo(
logType: LogType.BackgrounderJava,
logReaderProvider: (stream, _) => new MultilineJavaLogReader(stream),
fileLocations: new List <Regex>
{
TabadminLog("backgrounder"), // pre-TSM - backgrounder/backgrounder-*.log with optional date at the end (backgrounder-0.log.2015-05-18)
TsmV0Log("backgrounder"), // TSMv0 - localhost\tabadminagent_0.20181.18.0404.16052600117725665315795\logs\backgrounder\backgrounder_node1-1.log
TsmLog("backgrounder"), // TSM - node2\backgrounder_0.20182.18.0627.22306436150448756480580\logs\backgrounder_node2-1.log.2018-08-08
}),
new LogTypeInfo(
logType: LogType.ClusterController,
logReaderProvider: (stream, _) => new MultilineJavaLogReader(stream),
fileLocations: new List <Regex>
{
TabadminLog("clustercontroller"), // pre-TSM - clustercontroller/clustercontroller.log.2015-05-18
TsmV0Log("clustercontroller"), // TSMv0 - localhost\tabadminagent_0.20181.18.0404.16052600117725665315795\logs\clustercontroller\clustercontroller.log
TsmLog("clustercontroller"), // TSM - node2\clustercontroller_0.20182.18.0627.22301467407848617992908\logs\clustercontroller.log
}),
new LogTypeInfo(
logType: LogType.ControlLogsJava,
logReaderProvider: (stream, _) => new MultilineJavaLogReader(stream),
fileLocations: new List <Regex>
{
// Control logs are placed in the logs folder of the component they control, so dir can be anything
TsmLog("[^/]+", "control"), // node1/tabadminagent_0.20202.20.0818.08576654979113587254208/logs/control_tabadminagent_node1-0.log.2020-09-27
}),
new LogTypeInfo(
logType: LogType.CrashPackageLog,
logReaderProvider: (stream, filePath) => new NativeJsonLogsReader(stream, filePath, processingNotificationsCollector),
fileLocations: new List <Regex>
{
Regex(@"^[^/]*\.log$"),
Regex(@"^[^/]*\.txt$")
}),
new LogTypeInfo(
logType: LogType.CrashPackageManifest,
logReaderProvider: (stream, filePath) => new SimpleLinePerLineReader(stream),
fileLocations: new List <Regex>
{
Regex(@"^[^/]*\.manifest$")
}),
new LogTypeInfo(
logType: LogType.DataserverCpp,
logReaderProvider: (stream, filePath) => new NativeJsonLogsReader(stream, filePath, processingNotificationsCollector),
fileLocations: new List <Regex>
{
TabadminNativeLog("dataserver"), // pre-TSM - vizqlserver\Logs\dataserver-0_2018_07_28_00_00_00.txt
TsmV0NativeLog("dataserver"), // TSMv0 - localhost\tabadminagent_0.20181.18.0404.16052600117725665315795\logs\dataserver\nativeapi_dataserver_1-0_2018_08_07_00_00_00.txt
TsmNativeLog("dataserver"), // TSM - node2\dataserver_0.20182.18.0627.22301765668120146669553\logs\nativeapi_dataserver_2-1_2018_08_08_00_00_00.txt
}),
new LogTypeInfo(
logType: LogType.DataserverJava,
logReaderProvider: (stream, _) => new MultilineJavaLogReader(stream),
fileLocations: new List <Regex>
{
TabadminLog("dataserver"),
TsmV0Log("dataserver"),
TsmLog("dataserver")
}),
new LogTypeInfo(
logType: LogType.Filestore,
logReaderProvider: (stream, _) => new MultilineJavaLogReader(stream),
fileLocations: new List <Regex>
{
TabadminLog("filestore"), // pre-TSM - filestore/filestore.log.2018-07-31
TsmV0Log("filestore"), // TSMv0 - localhost\tabadminagent_0.20181.18.0404.16052600117725665315795\logs\filestore\filestore.log
TsmLog("filestore"), // TSM - node1\filestore_0.20182.18.0627.22302895224363938766334\logs\filestore.log
}),
new LogTypeInfo(
logType: LogType.Hyper,
logReaderProvider: (stream, filePath) => new NativeJsonLogsReader(stream, filePath, processingNotificationsCollector),
fileLocations: new List <Regex>
{
TabadminLog("hyper"), // pre-TSM - hyper/hyper_2018_07_19_22_24_32.log
TsmV0Log("hyper"), // TSMv0 - localhost\tabadminagent_0.20181.18.0510.1418770265691097820228\logs\hyper\hyper_0_2018_07_30_08_08_24.log
TsmLog("hyper"), // TSM - node2\hyper_0.20182.18.0627.22308540150062437331610\logs\hyper_0_2018_08_08_15_07_26.log
}),
new LogTypeInfo(
logType: LogType.NetstatLinux,
logReaderProvider: (stream, filePath) => new SimpleLinePerLineReader(stream),
fileLocations: new List <Regex>
{
Regex(@".+/netstat-anp\.txt$") // TSMv0 and TSM - (TSMv0 example: localhost\tabadminagent_0.20181.18.0510.1418770265691097820228\sysinfo\netstat-anp.txt)
}),
new LogTypeInfo(
logType: LogType.NetstatWindows,
logReaderProvider: (stream, filePath) => new NetstatWindowsReader(stream, filePath, processingNotificationsCollector),
fileLocations: new List <Regex>
{
Regex(@"netstat-info\.txt$"), // pre-TSM - netstat-info.txt in the root
Regex(@"/netstat-info\.txt$") // TSM - node1\tabadminagent_0.20182.18.1001.21153436271280456730793\netstat-info.txt
}),
new LogTypeInfo(
logType: LogType.PostgresCsv,
logReaderProvider: (stream, filePath) => new CsvLogReader <PostgresCsvMapping>(stream, filePath, processingNotificationsCollector),
fileLocations: new List <Regex>
{
TabadminLog("pgsql", "postgresql", "csv"), // pre-TSM - pgsql/postgresql-Sat.csv
TsmV0Log("pgsql", "postgresql", "csv"), // TSMv0 - localhost\tabadminagent_0.20181.18.0510.1418770265691097820228\logs\pgsql\postgresql-Mon.csv
TsmLog("pgsql", "postgresql", "csv"), // TSM - node2\pgsql_0.20182.18.0627.22303045353787439845635\logs\postgresql-Wed.csv
}),
new LogTypeInfo(
logType: LogType.ProtocolServer,
logReaderProvider: (stream, filePath) => new NativeJsonLogsReader(stream, filePath, processingNotificationsCollector),
fileLocations: new List <Regex>
{
Regex(@"^tabprotosrv.*\.txt"), // Desktop, zipped files
Regex(@"^[Ll]ogs\tabprotosrv.*\.txt"), // Desktop, zipped Logs folder
TabadminLog("vizqlserver", "tabprotosrv", "txt"), // pre-TSM - vizqlserver\tabprotosrv_vizqlserver_0-0_1.txt
TsmV0ProtocolLog("backgrounder"), // TSMv0 - Backgrounder - localhost\tabadminagent_0.20181.18.0404.16052600117725665315795\logs\backgrounder\tabprotosrv_backgrounder_1-0.txt
TsmV0ProtocolLog("dataserver"), // TSMv0 - Dataserver
TsmV0ProtocolLog("vizportal"), // TSMv0 - Vizportal - localhost\tabadminagent_0.20181.18.0510.14183743094502915100234\logs\vizportal\tabprotosrv_vizportal_2-0.txt
TsmV0ProtocolLog("vizqlserver"), // TSMv0 - Vizqlserver - locahost\tabadminagent_0.20181.18.0510.14183743094502915100234\logs\vizqlserver\tabprotosrv_vizqlserver_2-0.txt
TsmProtocolLog("backgrounder"), // TSM - Backgrounder - node2\backgrounder_0.20182.18.0627.22306436150448756480580\logs\tabprotosrv_backgrounder_2-0.txt
TsmProtocolLog("dataserver"), // TSM - Dataserver
TsmProtocolLog("vizportal"), // TSM - Vizportal - node1\vizportal_0.20182.18.0627.22304211226147125020104\logs\tabprotosrv_vizportal_1-0.txt
TsmProtocolLog("vizqlserver"), // TSM - Vizqlserver - node1\vizqlserver_0.20182.18.0627.22305268092790927660381\logs\tabprotosrv_vizqlserver_1-1.txt
}),
new LogTypeInfo(
logType: LogType.SearchServer,
logReaderProvider: (stream, _) => new MultilineJavaLogReader(stream),
fileLocations: new List <Regex>
{
TabadminLog("searchserver"), // pre-TSM - searchserver/searchserver-0.log.2018-07-19
TsmV0Log("searchserver"), // TSMv0 - localhost\tabadminagent_0.20181.18.0510.1418770265691097820228\logs\searchserver\searchserver_node1-0.log.2018-07-30
TsmLog("searchserver"), // TSM - node1\searchserver_0.20182.18.0627.22308531537836253176995\logs\searchserver_node1-0.log
}),
new LogTypeInfo(
logType: LogType.Tabadmin,
logReaderProvider: (stream, _) => new MultilineJavaLogReader(stream),
fileLocations: new List <Regex>
{
Regex(@"logs/tabadmin\.log"), // pre-TSM - logs/tabadmin.log
Regex(@"tabadmin/tabadmin.*\.log.*") // pre-TSM
}),
new LogTypeInfo(
logType: LogType.TabadminAgentJava,
logReaderProvider: (stream, _) => new MultilineJavaLogReader(stream),
fileLocations: new List <Regex>
{
TsmLog("tabadminagent", "tabadminagent_node") // node1/tabadminagent_0.20202.20.0818.08576654979113587254208/logs/tabadminagent_node1-0.log.2020-09-26
}),
new LogTypeInfo(
logType: LogType.TabadminControllerJava,
logReaderProvider: (stream, _) => new MultilineJavaLogReader(stream),
fileLocations: new List <Regex>
{
TsmLog("tabadmincontroller") // node1/tabadmincontroller_0.20201.20.0913.21097164108816806990865/logs/tabadmincontroller_node1-0.log
}),
new LogTypeInfo(
logType: LogType.TabsvcYml,
logReaderProvider: (stream, filePath) => new YamlConfigLogReader(stream),
fileLocations: new List <Regex>
{
Regex(@"^tabsvc\.yml$"), // pre-TSM - tabsvc.yml in the root of the archive
// TSMv0 doesn't include config info
Regex(@"config/tabadminagent[^/]+/tabsvc\.yml$") // TSM - node1\tabadminagent_0.20182.18.1001.21153436271280456730793\config\tabadminagent_0.20182.18.1001.2115\tabsvc.yml
}),
new LogTypeInfo(
logType: LogType.VizportalCpp,
logReaderProvider: (stream, filePath) => new NativeJsonLogsReader(stream, filePath, processingNotificationsCollector),
fileLocations: new List <Regex>
{
TabadminNativeLog("vizportal"), // pre-TSM - vizqlserver\Logs\vizportal_0-0_2018_08_01_00_00_00.txt
TsmV0NativeLog("vizportal"), // TSMv0 - localhost\tabadminagent_0.20181.18.0510.14183743094502915100234\logs\vizportal\nativeapi_vizportal_2-0_2018_07_31_00_00_00.txt
TsmNativeLog("vizportal"), // TSM - node1\vizportal_0.20182.18.0627.22304211226147125020104\logs\nativeapi_vizportal_1-0_2018_08_08_00_00_00.txt
}),
new LogTypeInfo(
logType: LogType.VizportalJava,
logReaderProvider: (stream, _) => new MultilineJavaLogReader(stream),
fileLocations: new List <Regex>
{
TabadminLog("vizportal"), // pre-TSM - vizportal/vizportal-0.log.2018-07-22
TsmV0Log("vizportal"), // TSMv0 - localhost\tabadminagent_0.20181.18.0510.14183743094502915100234\logs\vizportal\vizportal_node2-0.log.2018-07-30
TsmLog("vizportal"), // TSM - node1\vizportal_0.20182.18.0627.22304211226147125020104\logs\vizportal_node1-0.log
}),
new LogTypeInfo(
logType: LogType.VizqlserverCpp,
logReaderProvider: (stream, filePath) => new NativeJsonLogsReader(stream, filePath, processingNotificationsCollector),
fileLocations: new List <Regex>
{
TabadminNativeLog("vizqlserver"), // pre-TSM
TsmV0NativeLog("vizqlserver"), // TSMv0 - locahost\tabadminagent_0.20181.18.0510.14183743094502915100234\logs\vizqlserver\nativeapi_vizqlserver_2-1_2018_07_30_00_00_00.txt
TsmNativeLog("vizqlserver"), // TSM - node2\vizqlserver_0.20182.18.0627.22306727828712986214311\logs\nativeapi_vizqlserver_2-0_2018_08_08_00_00_00.txt
}),
new LogTypeInfo(
logType: LogType.VizqlDesktop,
logReaderProvider: (stream, filePath) => new NativeJsonLogsReader(stream, filePath, processingNotificationsCollector),
fileLocations: new List <Regex>
{
Regex(@"^log[^/]*\.txt"), // If somebody zipped files, log files are in root
Regex(@"^[Ll]ogs/log.*\.txt"), // If somebody zipped whole Logs folder
}),
new LogTypeInfo(
logType: LogType.WorkgroupYml,
logReaderProvider: (stream, filePath) => new YamlConfigLogReader(stream),
fileLocations: new List <Regex>
{
Regex(@"config/workgroup\.yml$"), // pre-TSM & TSM. Pre TSM - config folder is at the root. TSM - node1\tabadminagent_0.20182.18.1001.21153436271280456730793\config\tabadminagent_0.20182.18.1001.2115\workgroup.yml
// TSMv0 doesn't include config info
Regex(@"config/tabadminagent[^/]+/workgroup\.yml$") // TSM - node1\tabadminagent_0.20182.18.1001.21153436271280456730793\config\tabadminagent_0.20182.18.1001.2115\workgroup.yml
}),
new LogTypeInfo(
logType: LogType.Zookeeper,
logReaderProvider: (stream, _) => new MultilineJavaLogReader(stream),
fileLocations: new List <Regex>
{
TabadminLog("zookeeper"), // pre-TSM - zookeeper/zookeeper-0.log.2018-07-19
TsmV0Log("appzookeeper"), // TSMv0 - localhost\tabadminagent_0.20181.18.0404.16052600117725665315795\logs\appzookeeper\appzookeeper_node1-0.log
TsmLog("appzookeeper"), // TSM - node1\appzookeeper_1.20182.18.0627.22308155521326766729002\logs\appzookeeper_node1-0.log.2018-08-08
}),
});
}
public LogTypeDetails(IProcessingNotificationsCollector processingNotificationsCollector)
{
_logFileInfoDictionary = LoadDetails(processingNotificationsCollector)
.ToDictionary(logTypeInfo => logTypeInfo.LogType, logTypeInfo => logTypeInfo);
}
public BackgrounderEventParser(IBackgrounderEventPersister backgrounderEventPersister, IProcessingNotificationsCollector processingNotificationsCollector)
{
_backgrounderEventPersister = backgrounderEventPersister;
_processingNotificationsCollector = processingNotificationsCollector;
}
private IEnumerable <LogSetInfo> HandleNestedZips(string zippedLogSetPath, string tempDirRoot, IProcessingNotificationsCollector processingNotificationsCollector)
{
_logger.LogInformation("Looking for known nested zip files inside zip file '{logSetPath}'", zippedLogSetPath);
using var zip = ZipFile.Open(zippedLogSetPath, ZipArchiveMode.Read);
var nestedZips = zip.Entries.Where(IsNestedZip).ToList();
var nestedZipFilesInfo = new List <LogSetInfo>();
if (nestedZips.Count == 0)
{
_logger.LogInformation("No known nested zip files found in '{logSetPath}", zippedLogSetPath);
return(nestedZipFilesInfo);
}
var tempDir = Path.Combine(tempDirRoot, NestedZipTempDirName);
Directory.CreateDirectory(tempDir);
_tempFileTracker.AddDirectory(tempDir);
foreach (var nestedZip in nestedZips)
{
var extractPath = Path.Combine(tempDir, nestedZip.Name);
_logger.LogInformation("Extracting nested archive {nestedZipName} into '{nestedZipExtractPath}'", nestedZip.Name, extractPath);
nestedZip.ExtractToFile(extractPath);
_logger.LogInformation("Successfully extracted {nestedZipName} into '{nestedZipExtractPath}'", nestedZip.Name, extractPath);
var checkResult = FileCanBeOpened(extractPath, _logger);
if (checkResult.FileCanBeOpened)
{
nestedZipFilesInfo.Add(new LogSetInfo(extractPath.NormalizeSeparatorsToUnix(), nestedZip.FullName.RemoveZipFromTail(), true, _rootPath));
}
else
{
var error = $"Nested zip \"{nestedZip.FullName}\" appears to be corrupt. It will not be processed. Underlying error: {checkResult.ErrorMessage ?? "(null)"}";
processingNotificationsCollector.ReportError(error, nestedZip.FullName, 0, nameof(TableauLogsExtractor));
}
}
_logger.LogInformation("Found and extracted {numberOfNestedZips} known nested zip files inside `{logSetPath}`", nestedZipFilesInfo.Count, zippedLogSetPath);
return(nestedZipFilesInfo);
}
private IList <LogSetInfo> EvaluateLogSetStructure(string logSetPath, string tempDirRoot, IProcessingNotificationsCollector processingNotificationsCollector)
{
IsDirectory = IsPathADirectory(logSetPath);
_logger.LogInformation("Provided log set appears to be {logSetType}", IsDirectory ? "directory" : "zip file");
var rootSetInfo = new LogSetInfo(logSetPath, string.Empty, !IsDirectory, _rootPath);
var zipInfoList = new List <LogSetInfo> {
rootSetInfo
};
zipInfoList.AddRange(IsDirectory
? LookForZippedParts(logSetPath)
: HandleNestedZips(logSetPath, tempDirRoot, processingNotificationsCollector)
);
LogSetSizeBytes = IsDirectory
? GetDirSize(_rootPath)
: new FileInfo(_rootPath).Length;
return(zipInfoList);
}
public NetstatWindowsReader(Stream stream, string filePath, IProcessingNotificationsCollector processingNotificationsCollector)
{
_filePath = filePath;
_stream = stream;
_processingNotificationsCollector = processingNotificationsCollector;
}
public void Configure(IWriterFactory writerFactory, IConfiguration pluginConfig, IProcessingNotificationsCollector processingNotificationsCollector, ILoggerFactory loggerFactory)
{
_hyperErrorWriter = writerFactory.GetWriter <HyperError>(HyperErrorOutputInfo);
_hyperQueryWriter = writerFactory.GetWriter <HyperEvent>(HyperQueryOutputInfo);
_processingNotificationsCollector = processingNotificationsCollector;
}
private IDictionary <int, string> GetWorkerHostnameMap(ConfigFile configFile, IProcessingNotificationsCollector processingNotificationsCollector)
{
if (configFile == null || !configFile.Values.ContainsKey(ConfigKeyWithHostMapping))
{
var error = $"Config file is null or doesn't contain {ConfigKeyWithHostMapping} key. Cannot generate hostname map for workers";
processingNotificationsCollector.ReportError(error, nameof(ConfigPlugin));
return(new Dictionary <int, string>());
}
var workerHostsLine = configFile.Values[ConfigKeyWithHostMapping];
return(workerHostsLine
.Split(',')
.Select((hostName, ind) => new
{
Index = ind,
HostName = hostName.Trim()
})
.ToDictionary(obj => obj.Index, obj => obj.HostName));
}
public void Configure(IWriterFactory writerFactory, IConfiguration pluginConfig, IProcessingNotificationsCollector processingNotificationsCollector, ILoggerFactory loggerFactory)
{
_writer = writerFactory.GetWriter <PostgresEvent>(OutputInfo);
_processingNotificationsCollector = processingNotificationsCollector;
}
public void Configure(IWriterFactory writerFactory, IConfiguration pluginConfig, IProcessingNotificationsCollector processingNotificationsCollector, ILoggerFactory loggerFactory)
{
_eventsProcessor = new ResourceManagerEventsProcessor(writerFactory, processingNotificationsCollector);
_processingNotificationsCollector = processingNotificationsCollector;
}
public void Configure(IWriterFactory writerFactory, IConfiguration pluginConfig,
IProcessingNotificationsCollector processingNotificationsCollector, ILoggerFactory loggerFactory)
{
throw new NotImplementedException();
}
public void Configure(IWriterFactory writerFactory, IConfiguration pluginConfig, IProcessingNotificationsCollector processingNotificationsCollector, ILoggerFactory loggerFactory)
{
_eventWriter = writerFactory.GetWriter <ServerTelemetryEvent>(EventsDsi);
_metricWriter = writerFactory.GetWriter <ServerTelemetryMetric>(MetricsDsi);
_processingNotificationsCollector = processingNotificationsCollector;
}
public void Configure(IWriterFactory writerFactory, IConfiguration pluginConfig, IProcessingNotificationsCollector processingNotificationsCollector, ILoggerFactory loggerFactory)
{
_backgrounderEventPersister = new BackgrounderEventPersister(writerFactory);
_backgrounderEventParser = new BackgrounderEventParser(_backgrounderEventPersister, processingNotificationsCollector);
_processingNotificationsCollector = processingNotificationsCollector;
}
public void Configure(IWriterFactory writerFactory, IConfiguration pluginConfig, IProcessingNotificationsCollector processingNotificationsCollector, ILoggerFactory loggerFactory)
{
_writer = writerFactory.GetWriter <NetstatActiveConnection>(OutputInfo);
_processingNotificationsCollector = processingNotificationsCollector;
}
public BuildTracker(IProcessingNotificationsCollector processingNotificationsCollector)
{
_processingNotificationsCollector = processingNotificationsCollector;
_timestampsForBuilds = new Dictionary <long, string>();
}
