public void PollFullTables(ITableauRepoConn connection, ICollection <RepoTable> tables)
{
if (connection == null)
{
Log.Error("Missing Tableau Repo connection while polling full tables!");
return;
}
Log.Info("Polling Tableau repository tables.");
tables.Where(t => t.Full)
.Select(t => t.Name)
.ToList()
.ForEach(t =>
{
try
{
DataTable table = connection.GetTable(t);
OutputSerializer.Write(table, true);
}
catch (InvalidOperationException ioe)
{
if (ioe.Message.Contains("Connection property has not been initialized"))
{
// This might also mean that the connection to Tableau is down
Log.Warn(ioe, "Temporarily unable to poll full table: '{0}'! Exception: ", t);
}
else
{
Log.Error(ioe, "Invalid operation exception while polling full table: '{0}'! Exception: ", t);
}
}
});
}
public PaletteInsightAgent(bool loadOptionsFromConfig = true)
{
// Set the working directory
Assembly assembly = Assembly.GetExecutingAssembly();
Directory.SetCurrentDirectory(Path.GetDirectoryName(assembly.Location));
APIClient.SetTrustSSL();
// Load PaletteInsightAgentOptions. In certain use cases we may not want to load options from the config, but provide them another way (such as via a UI).
options = PaletteInsightAgentOptions.Instance;
// Locate the Tableau data folder
tableauDataFolder = Loader.FindTableauDataFolder();
var configuration = Loader.LoadConfigFile("config/Config.yml");
Loader.LoadConfigTo(configuration, tableauDataFolder, options);
// Make sure that our HTTP client is initialized, because Splunk logger might be enabled
// and it is using HTTP to send log messages to Splunk.
APIClient.Init(options.WebserviceConfig);
// Add the webservice username/auth token from the license
Configuration.Loader.UpdateWebserviceConfigFromLicense(options);
// NOTE: License check disabled as this project became open-source
//// check the license after the configuration has been loaded.
//licenseGuard = new LicenseGuard();
//if (!licenseGuard.CheckLicense(options.LicenseKey))
//{
// Log.Fatal("Invalid license! Exiting...");
// Environment.Exit(-1);
//}
// Showing the current version in the log
FileVersionInfo fvi = FileVersionInfo.GetVersionInfo(assembly.Location);
string version = fvi.FileVersion;
Log.Info("Palette Insight Agent version: " + version);
USE_LOGPOLLER = options.UseLogPolling;
USE_THREADINFO = options.UseThreadInfo;
USE_COUNTERSAMPLES = options.UseCounterSamples;
USE_TABLEAU_REPO = options.UseRepoPolling;
USE_STREAMING_TABLES = options.UseStreamingTables;
if (USE_TABLEAU_REPO != USE_STREAMING_TABLES)
{
// Having different setup for Tableau repo and streaming tables is not welcome
Log.Error("Invalid repo poll configuration! Either both Tableau repo poll and streaming tables poll should be enabled, or neither! Repo poll: {0} vs. Streaming tables: {1}",
USE_TABLEAU_REPO, USE_STREAMING_TABLES);
}
if (USE_LOGPOLLER)
{
// Load the log poller config & start the agent
logPollerAgent = new LogPollerAgent(options.LogFolders, options.LogLinesPerBatch);
}
if (USE_THREADINFO)
{
// start the thread info agent
threadInfoAgent = new ThreadInfoAgent(options.ThreadInfoPollInterval);
}
Loader.AddRepoFromWorkgroupYaml(configuration, tableauDataFolder, options);
tableauRepo = new Tableau9RepoConn(options.RepositoryDatabase, options.StreamingTablesPollLimit);
repoPollAgent = new RepoPollAgent();
}
public void PollStreamingTables(ITableauRepoConn connection, ICollection <RepoTable> tables, IOutput output)
{
if (connection == null)
{
Log.Error("Missing Tableau Repo connection while polling streaming tables!");
return;
}
Log.Info("Polling Tableau Streaming tables.");
tables.Where(table => !table.Full)
.ToList()
.ForEach((table) =>
{
var tableName = table.Name;
try
{
// If we have a pending request for this table, then just skip this iteration
if (output.IsInProgress(tableName))
{
return;
}
// Get maxid from remote server
var maxId = this.GetMaxId(tableName);
// Get data from that max id
string newMax;
DataTable dataTable = connection.GetStreamingTable(tableName, table, maxId, out newMax);
Log.Info("Polled records of streaming table {0} from {1} to {2}", tableName, maxId, newMax);
if (dataTable != null)
{
RepoPollAgent.localMaxId[tableName] = newMax;
OutputSerializer.Write(dataTable, false, newMax);
}
}
catch (AggregateException ae)
{
ae.Handle((x) =>
{
if (x is HttpRequestException || x is TaskCanceledException || x is TemporaryException)
{
// HttpRequestException is expected on network errors. TaskCanceledException is thrown if the async task (HTTP request) timed out.
Log.Warn(x, "Polling streaming table: '{0}' timed out! Exception: ", tableName);
return(true);
}
Log.Error(x, "Async exception caught while polling streaming table: {0}! Exception: ", tableName);
return(true);
});
}
catch (TemporaryException tex)
{
Log.Warn("Temporarily unable to poll streaming table: {0}! Exception: {1}", tableName, tex);
}
catch (TaskCanceledException tce)
{
// This should be only a temporary condition, it is only a problem if it occurs many times in a row.
Log.Warn("Polling streaming table: '{0}' timed out! Exception: {1}", tableName, tce);
}
catch (HttpRequestException hre)
{
Log.Warn("HTTP request exception while polling streaming table: '{0}'! Exception: {1}", tableName, hre);
}
catch (Exception e)
{
if (e is InvalidOperationException && e.Message.Contains("Connection property has not been initialized"))
{
// This might also mean that the connection to Tableau is down
Log.Warn(e, "Temporarily unable to poll streaming table: '{0}'! Exception: ", tableName);
return;
}
Log.Error(e, "Error while polling streaming table: '{0}'! Exception: ", tableName);
}
});
}
