public IEnumerable<string> GetMetrics(GraphiteClient client)
{
//Load the domain parser rule set
if ((File.Exists("effective_tld_names.dat")) && (parser == null))
{
var list = new RulesList(); // get a new rules list
var rules = list.FromFile(@"effective_tld_names.dat"); // parse a local copy of the publicsuffix.org file
parser = new Parser(rules); // create an instance of the parser
}
foreach (var site in siteSection.Sites.OfType<SiteConfigurationElement>())
{
string lastRead = cursor.GetLastRead(logName, site.GraphiteKey);
Parallel.ForEach(GetResultUrls(site.Url, lastRead), resultUrl =>
{
if (!String.IsNullOrWhiteSpace(resultUrl))
{
try
{
var metrics = XmlToMetrics(site, new XPathDocument(resultUrl));
if (metrics.Any())
{
client.SendMetrics(metrics);
lock (cursor)
{
lastRead += Environment.NewLine + resultUrl;
cursor.StoreLastRead(logName, site.GraphiteKey, lastRead);
}
}
}
catch
{
lock (cursor)
{
lastRead += Environment.NewLine + resultUrl + Environment.NewLine + "failed:" + resultUrl;
cursor.StoreLastRead(logName, site.GraphiteKey, lastRead);
}
}
}
});
}
return cursor.GetUsedOffsetFiles();
}
static void Main(string[] args)
{
var client = new GraphiteClient(ConfigurationManager.AppSettings["GraphiteHost"],
Int32.Parse(ConfigurationManager.AppSettings["GraphitePort"]),
ConfigurationManager.AppSettings["GraphiteKeyPrefix"]);
var filesUsed = new List<string>();
bool feedback = ((args != null) && (args.Contains("--show-progress")));
//Debug information
var sw = new Stopwatch();
sw.Start();
//Configuration based. Config file lists logs, sizes, regex, etc
var logParser = new LogTailParser();
if (feedback)
{
logParser.MessageReporter = new RawConsoleLogger();
}
filesUsed.AddRange(logParser.GetMetrics(client));
//WPT parsers
var wptParser = new WebPagetestParser();
filesUsed.AddRange(wptParser.GetMetrics(client));
sw.Stop();
client.SendQuickMetric("metrics.runTime.avg", (int)sw.ElapsedMilliseconds);
//Cleanup old offset files
foreach (var file in Directory.GetFiles(".", "*.offset"))
{
if (!filesUsed.Contains(Path.GetFileName(file)))
{
File.Delete(file);
}
}
}
public IEnumerable<string> GetMetrics(GraphiteClient client)
{
graphiteClient = client;
MessageReporter.ReportProgress("Calculating total...");
var totalFilesToProcess = CalculateTotalFilesToProcess();
MessageReporter.ReportProgress("DONE : Calculating total");
var count = 0;
foreach (var log in logs)
{
string workerId = log.Name;
MessageReporter.ReportWorkerStatus(workerId, String.Format("[{0}] Warming up", log.Name));
//Group files by their date/filename
var logFiles = new List<Log>();
foreach (var locationKey in log.Locations.AllKeys)
{
foreach (var file in GetValidLogfilesFromDirectory(log.Locations[locationKey].Value, log.Pattern,
log.MaxDaysToProcess))
{
logFiles.Add(new Log { Filename = file, LocationKey = locationKey });
}
}
//Go through files by their filename (which is the log date)
foreach (var logFileGroup in logFiles.GroupBy(logFile => Path.GetFileName(logFile.Filename)))
{
var boomerangValues = new ConcurrentBag<Metric>();
//Do all files with the same name at the same time
Parallel.ForEach(logFileGroup, file =>
{
MessageReporter.ReportWorkerStatus(workerId,
String.Format("[{0}] Processing file {1} ",
file.LocationKey,
file.Filename.Substring(
file.Filename.LastIndexOf("\\", System.StringComparison.Ordinal) + 1)));
var readNewLines = ReadTail(file.Filename, log, file.LocationKey, boomerangValues);
Interlocked.Increment(ref count);
MessageReporter.SetHeadline(String.Format("Processed {0}/{1} ({2:P}) files", count,
totalFilesToProcess,
(double)count / (double)totalFilesToProcess));
MessageReporter.ReportWorkerStatus(workerId,
String.Format("[{0}] {1} files ", file.LocationKey,
readNewLines.HasValue
? (readNewLines.Value
? "Finished"
: "Not Changed")
: "Skipped"));
});
FlushBoomerangMetrics(boomerangValues);
}
MessageReporter.ReportWorkerStatus(workerId, String.Format("DONE ALL"));
}
return cursor.GetUsedOffsetFiles();
}