private static void GeneralUsage() { Stdout.WriteLine(); Stdout.WriteLine("USAGE:"); Stdout.WriteLine("lat.exe [throttlinganalysis <args>] [download <args>]"); Stdout.WriteLine("run with empty <args> to get help on specific function"); Environment.Exit(1); }
internal static void DateUsage() { Stdout.WriteLine(); Stdout.WriteLine("Dates should be specified in format understandable to DateTime.Parse()."); Stdout.WriteLine("Examples:"); Stdout.WriteLine(" \"2013-11-22\" -- midnight of Nov22 in Local Time"); Stdout.WriteLine(" \"2013-11-22Z\" -- midnight of Nov22 in UTC Time"); Stdout.WriteLine(" \"2013-11-22 01:00:00Z\" -- 1AM Nov22 UTC"); Stdout.WriteLine(" \"2013-07-25T01:00:00Z\" -- 1AM Nov22 UTC"); }
private void EmitGlobalAggregates(List <AzureBlobLogRecordV1> records, List <PerSecondAggregate> perSecondAggregates, TextWriter w) { long totalDataBytes = records.Sum(r => r.DataSize); TimeSpan jobDuration = this.endTimeUTC.Subtract(this.startTimeUTC); var jobDurationSeconds = (long)jobDuration.TotalSeconds; long jobMbps = totalDataBytes * 8 / (1024 * 1024) / jobDurationSeconds; long throttleErrorCount = records.Count(r => r.IsThrottle); long throttleErrorBytes = records.Where(r => r.IsThrottle).Sum(r => r.DataSize); long failErrorCount = records.Count(r => r.IsError); long failErrorBytes = records.Where(r => r.IsError).Sum(r => r.DataSize); Stdout.TeeWriteLine(w, string.Format(CultureInfo.CurrentCulture, "Records,{0}", records.Count)); Stdout.TeeWriteLine( w, string.Format(CultureInfo.CurrentCulture, "JobDuration,{0}h {1}m {2}s", jobDuration.Hours, jobDuration.Minutes, jobDuration.Seconds)); Stdout.TeeWriteLine(w, string.Format(CultureInfo.CurrentCulture, "JobDuration(sec),{0}", +jobDurationSeconds)); Stdout.TeeWriteLine(w, string.Format(CultureInfo.CurrentCulture, "JobTotalGB,{0}", totalDataBytes / (1024 * 1024 * 1024))); Stdout.TeeWriteLine(w, string.Format(CultureInfo.CurrentCulture, "JobMbps,{0}", jobMbps)); Stdout.TeeWriteLine( w, string.Format(CultureInfo.CurrentCulture, "ThrottledPackets/TotalBytes,{0} / {1}", throttleErrorCount, throttleErrorBytes)); Stdout.TeeWriteLine(w, string.Format(CultureInfo.CurrentCulture, "FailedPackets/TotalBytes,{0} / {1}", failErrorCount, failErrorBytes)); Stdout.TeeWriteLine( w, string.Format(CultureInfo.CurrentCulture, "Peak ReadMbps (Instantaneous),{0}", (long)perSecondAggregates.Max(r => r.ReadMbps))); Stdout.TeeWriteLine( w, string.Format( CultureInfo.CurrentCulture, "Peak ReadMbps (Moving30SecAv),{0}", (long)perSecondAggregates.Max(r => r.ReadMbpsMovingAv30Secs))); Stdout.TeeWriteLine( w, string.Format( CultureInfo.CurrentCulture, "Peak ReadMbps (Moving60SecAv),{0}", (long)perSecondAggregates.Max(r => r.ReadMbpsMovingAv60Secs))); Stdout.TeeWriteLine( w, string.Format(CultureInfo.CurrentCulture, "Peak WriteMbps (Instantaneous),{0}", (long)perSecondAggregates.Max(r => r.WriteMbps))); Stdout.TeeWriteLine( w, string.Format( CultureInfo.CurrentCulture, "Peak WriteMbps (Moving30SecAv),{0}", (long)perSecondAggregates.Max(r => r.WriteMbpsMovingAv30Secs))); Stdout.TeeWriteLine( w, string.Format( CultureInfo.CurrentCulture, "Peak WriteMbps (Moving60SecAv),{0}", (long)perSecondAggregates.Max(r => r.WriteMbpsMovingAv60Secs))); Stdout.TeeWriteLine( w, string.Format(CultureInfo.CurrentCulture, "Peak E2E latency (ms),{0}", (long)perSecondAggregates.Max(r => r.MaxE2ELatency))); Stdout.TeeWriteLine( w, string.Format( CultureInfo.CurrentCulture, "Count E2E latency > 1000ms,{0}", (long)perSecondAggregates.Count(r => r.MaxE2ELatency > 1000))); Stdout.WriteLine(); }
private void Process(List <AzureBlobLogRecordV1> records) { //w.WriteLine("Records," + records.Count); if (records.Count() == 0) { Stdout.WriteLine("WARN: no records match."); return; } records.Sort((AzureBlobLogRecordV1 r1, AzureBlobLogRecordV1 r2) => r1.RequestStartTimeUTC.CompareTo(r2.RequestStartTimeUTC)); List <PerSecondAggregate> perSecondAggregates = this.CalculatePerSecondAggregates(records); this.GenerateMovingAverages(perSecondAggregates); this.EmitGlobalAggregates(records, perSecondAggregates, this.summaryWriter); this.EmitPerSecondResults(perSecondAggregates, this.detailsWriter); }
private void AnalyserUsage() { Stdout.WriteLine(); Stdout.WriteLine(); Stdout.WriteLine("USAGE:"); Stdout.WriteLine("AzureLogAnalysis.exe analyse <args>"); Stdout.WriteLine(" -logcache <folderPath>"); Stdout.WriteLine(" -account <storageAccountName>"); Stdout.WriteLine(" -start <datetime>"); Stdout.WriteLine(" -end <datetime>"); Stdout.WriteLine(" -name <analysisName> Used to name output files"); Stdout.WriteLine(" -note <noteText> A note to store in summary output"); Stdout.WriteLine(" -debug Launch debugger at process start"); Stdout.WriteLine(" -v,-verbose Verbose output"); Utils.DateUsage(); Environment.Exit(1); }
private void DownloaderUsage() { Stdout.WriteLine(); Stdout.WriteLine(); Stdout.WriteLine("USAGE:"); Stdout.WriteLine("AzureLogAnalysis.exe download <args>"); Stdout.WriteLine(" -start <datetime>"); Stdout.WriteLine(" -end <datetime>"); Stdout.WriteLine(" -logcache <folderPath>"); Stdout.WriteLine(" -account <storageAccountName>"); Stdout.WriteLine(" -key <storageKey>"); Stdout.WriteLine(" -f,-force Force creation of logcache folder area"); Stdout.WriteLine(" -debug Launches debugger at process start"); Stdout.WriteLine(" -v,-verbose Verbose output"); Utils.DateUsage(); Environment.Exit(1); }
internal void DoDownload() { string blobBaseUriString = null; string containerName = "$logs"; string blobPrefix = "blob/"; if (this.accountName.Contains(".")) { Stdout.WriteLine("ERROR: Fully qualifed account names are not supported. {name}.blob.core.windows.net is assumed"); this.DownloaderUsage(); } else { blobBaseUriString = string.Format(CultureInfo.CurrentCulture, "http://{0}.blob.core.windows.net/", this.accountName); } StorageCredentials creds = null; if (this.accountKey.StartsWith("?sv", StringComparison.Ordinal)) { Stdout.WriteLine("ERROR: SAS key is not supported. Use full key instead."); //incidentally, AzureManagementStudio doesn't want to create SAS keys for $logs container.. not sure if there is a WAS limitation. //need a way to create SAS keys to test it out. //creds = new StorageCredentials(accountKey); this.DownloaderUsage(); } else { creds = new StorageCredentials(this.accountName, this.accountKey); //It't not clear what to provide here if accountName is FQDN } var blobClient = new CloudBlobClient(new Uri(blobBaseUriString), creds); CloudBlobContainer logsContainer = blobClient.GetContainerReference(containerName); //CloudBlobDirectory rootDir = logsContainer.GetDirectoryReference("/"); List <string> blobList = this.EnumerateBlobsInContainer(logsContainer, blobPrefix); Stdout.WriteLine(string.Format(CultureInfo.CurrentCulture, "{0} files total in $log container", blobList.Count)); string cachefolder = Path.Combine(this.logCacheRootfolder, this.accountName + "\\" + containerName); List <DownloadListItem> downloadList = this.CalculateDownloadList(logsContainer, this.logCacheRootfolder, cachefolder, blobList); this.DownloadFiles(logsContainer, downloadList); }
private void DownloadFiles(CloudBlobContainer logsContainer, List <DownloadListItem> downloadList) { if (downloadList.Count == 0) { return; } long bytes = downloadList.Sum(x => x.Length); Stdout.WriteLine(string.Format(CultureInfo.CurrentCulture, "Downloading {0} files, {1} MB..", downloadList.Count, bytes / (1024 * 1024))); foreach (DownloadListItem item in downloadList) { ICloudBlob blobRef = null; try { blobRef = logsContainer.GetBlobReferenceFromServer(item.RemoteBlobPath); } catch (StorageException se) { Stdout.WriteLine("Error accessing blob on server. skipping. Exception=" + se.Message); } Utils.CreateLocalFolderIfNotExists(Path.GetDirectoryName(item.LocalFullPath)); Stdout.Write(string.Format(CultureInfo.CurrentCulture, item.RemoteBlobPath + " ({0} MB)..", item.Length / (1024 * 1024))); try { blobRef.DownloadToFile(item.LocalFullPath, FileMode.Create); Stdout.WriteLine("done."); } catch (StorageException se) { Stdout.WriteLine("Error: " + se.Message); } } }
internal void ReadArgsDownload(string[] args) { Console.WriteLine(); Exception e = null; bool unknownArg = false; int curr = 1; try { while (curr < args.Length) { if (args[curr].ToUpperInvariant() == "-LOGCACHE" && curr + 1 < args.Length) { curr++; this.logCacheRootfolder = args[curr]; curr++; } else if (args[curr].ToUpperInvariant() == "-START" && curr + 1 < args.Length) { curr++; this.startTime = DateTime.Parse(args[curr], CultureInfo.CurrentCulture).ToUniversalTime(); curr++; } else if (args[curr].ToUpperInvariant() == "-END" && curr + 1 < args.Length) { curr++; this.endTime = DateTime.Parse(args[curr], CultureInfo.CurrentCulture).ToUniversalTime(); curr++; } else if (args[curr].ToUpperInvariant() == "-DEBUG") { curr++; //ignore } else if (args[curr].ToUpperInvariant() == "-FORCE" || args[curr].ToUpperInvariant() == "-F") { curr++; this.forceFolderCreation = true; } else if (args[curr].ToUpperInvariant() == "-VERBOSE" || args[curr].ToUpperInvariant() == "-V") { curr++; Stdout.Verbose = true; } else if (args[curr].ToUpperInvariant() == "-ACCOUNT" && curr + 1 < args.Length) { curr++; this.accountName = args[curr]; curr++; } else if (args[curr].ToUpperInvariant() == "-KEY" && curr + 1 < args.Length) { curr++; this.accountKey = args[curr]; curr++; } else { unknownArg = true; break; } } } catch (FormatException ex) { e = ex; } if (e != null) { Stdout.WriteLine("Exception during arg processing:" + e.Message); this.DownloaderUsage(); } if (unknownArg) { Stdout.WriteLine("Unknown arg:" + args[curr]); this.DownloaderUsage(); } //sanity checks. bool fail = false; if (this.accountName == null) { Stdout.WriteLine("ERROR: account name not specified. Use -account"); fail = true; } if (this.accountKey == null) { Stdout.WriteLine("ERROR: account key not specified. Use -key."); fail = true; } if (this.startTime == DateTime.MinValue) { Stdout.WriteLine("WARN: startTime not specified. Defaulting to MinValue."); } if (this.endTime == DateTime.MaxValue) { Stdout.WriteLine("WARN: endTime not specified. Defaulting to MaxValue"); } if (this.logCacheRootfolder == null) { Stdout.WriteLine("WARN: logcache defaulting to current directory. Use -logcache to target a specific directory for caching log files."); this.logCacheRootfolder = Environment.CurrentDirectory; } if (fail) { this.DownloaderUsage(); } if (!Directory.Exists(Path.Combine(this.logCacheRootfolder, this.accountName)) && !this.forceFolderCreation) { Stdout.WriteLine( string.Format( CultureInfo.CurrentCulture, "ERROR: root folder {0} does not yet contain a folder for {1}. Either target existing log cache folder or use -f to force creation.", this.logCacheRootfolder, this.accountName)); this.DownloaderUsage(); } Stdout.WriteLine(); Stdout.WriteLine("Configuration:"); Stdout.WriteLine("-start: " + Utils.DateTimeToStandardizedStringFormat(this.startTime)); Stdout.WriteLine("-end: " + Utils.DateTimeToStandardizedStringFormat(this.endTime)); Stdout.VerboseWriteLine("-logCache: " + this.logCacheRootfolder); Stdout.VerboseWriteLine("-account: " + this.accountName); Stdout.VerboseWriteLine("-key: " + this.accountKey); Stdout.WriteLine(); }
private List <DownloadListItem> CalculateDownloadList( CloudBlobContainer logsContainer, string rootfolder, string cachefolder, List <string> blobList) { var downloadList = new List <DownloadListItem>(); long countFilesMatchingTimeRange = 0; Utils.CreateLocalFolderIfNotExists(rootfolder); foreach (string blobName in blobList) { int idx = 0; int year = int.Parse(blobName.Substring(idx, 4), CultureInfo.InvariantCulture); int month = int.Parse(blobName.Substring(idx + 5, 2), CultureInfo.InvariantCulture); int day = int.Parse(blobName.Substring(idx + 8, 2), CultureInfo.InvariantCulture); int hour = int.Parse(blobName.Substring(idx + 11, 2), CultureInfo.InvariantCulture); string fileName = blobName.Substring(idx + 16); var logfileDateMin = new DateTime(year, month, day, hour, 0, 0, DateTimeKind.Utc); // TODO: review UTC/Local conversions. DateTime logfileDateMax = logfileDateMin.AddHours(1); string pathComponents = string.Format( CultureInfo.InvariantCulture, "{0:00}\\{1:00}\\{2:00}\\{3:00}00\\{4}", year, month, day, hour, fileName); string localPath = Path.Combine(cachefolder, pathComponents); string uriComponents = string.Format( CultureInfo.InvariantCulture, "{0:00}/{1:00}/{2:00}/{3:00}00/{4}", year, month, day, hour, fileName); string blobPath = "blob/" + uriComponents; Stdout.VerboseWrite(blobPath + " .. "); ICloudBlob blobRef = null; try { blobRef = logsContainer.GetBlobReferenceFromServer(blobPath); } catch (StorageException se) { Stdout.WriteLine("Error accessing blob on server. skipping. Exception=" + se.Message); continue; } bool needsDownload = true; if (this.startTime < logfileDateMax && this.endTime > logfileDateMin) { countFilesMatchingTimeRange++; if (File.Exists(localPath)) { var info = new FileInfo(localPath); if (blobRef.Properties.Length == info.Length) { Stdout.VerboseWrite("cached OK. "); needsDownload = false; } else { Stdout.VerboseWrite("cached but doesn't match. "); } } if (needsDownload) { Stdout.VerboseWrite("requires download.."); downloadList.Add(new DownloadListItem(blobPath, localPath, blobRef.Properties.Length)); } } else { Stdout.VerboseWrite("not required"); } Stdout.VerboseWriteLine(); } Stdout.WriteLine(string.Format(CultureInfo.CurrentCulture, "{0} files in specified time range", countFilesMatchingTimeRange)); Stdout.WriteLine(string.Format(CultureInfo.CurrentCulture, "{0} files require download", downloadList.Count)); return(downloadList); }
internal void DoAnalysis() { Stdout.VerboseWriteLine("Reading log data from: " + this.logFolder); try { this.summaryWriter = new StreamWriter(this.summaryFilename); } catch (IOException ioe) { Stdout.WriteLine("ERROR: can't open output file for writing:" + this.summaryFilename + ". " + ioe.Message); Environment.Exit(1); } catch (UnauthorizedAccessException uae) { Stdout.WriteLine("ERROR: can't open output file for writing:" + this.summaryFilename + ". " + uae.Message); Environment.Exit(1); } try { this.detailsWriter = new StreamWriter(this.detailsFilename); } catch (IOException ioe) { Stdout.WriteLine("ERROR: can't open output file for writing:" + this.detailsFilename + ". " + ioe.Message); Environment.Exit(1); } catch (UnauthorizedAccessException uae) { Stdout.WriteLine("ERROR: can't open output file for writing:" + this.detailsFilename + ". " + uae.Message); Environment.Exit(1); } this.OutputSettings(this.summaryWriter); // -- !! determine input files List <string> logFilePaths = Directory.EnumerateFiles(this.logFolder, "*", SearchOption.AllDirectories).ToList(); int logFilesTotalCount = logFilePaths.Count; // filter out paths that have datestamps that do not match. if (logFilePaths[0].IndexOf("$logs\\20", StringComparison.Ordinal) >= 0) { int c = 0; bool skipped; while (c < logFilePaths.Count) { skipped = false; int idx = logFilePaths[c].IndexOf("$logs\\20", StringComparison.Ordinal) + 6; // set idx to point to first digit of the year component. if (idx >= 0) { try { int year = int.Parse(logFilePaths[c].Substring(idx, 4), CultureInfo.InvariantCulture); int month = int.Parse(logFilePaths[c].Substring(idx + 5, 2), CultureInfo.InvariantCulture); int day = int.Parse(logFilePaths[c].Substring(idx + 8, 2), CultureInfo.InvariantCulture); int hour = int.Parse(logFilePaths[c].Substring(idx + 11, 2), CultureInfo.InvariantCulture); DateTime logfileDateMinUTC = new DateTime(year, month, day, hour, 0, 0, DateTimeKind.Utc).ToUniversalTime(); DateTime logfileDateMaxUTC = logfileDateMinUTC.AddHours(1); if (this.endTimeUTC < logfileDateMinUTC || this.startTimeUTC > logfileDateMaxUTC) { Stdout.VerboseWriteLine("skipping: " + logFilePaths[c]); logFilePaths.RemoveAt(c); skipped = true; } } catch (FormatException) { //parsing the date failed. we will include the file in the processing list } } if (!skipped) { c++; } } } Stdout.WriteLine("Log files found: " + logFilesTotalCount); Stdout.WriteLine("Log files after date-filtering: " + logFilePaths.Count()); if (logFilePaths.Count() == 0) { Stdout.WriteLine("No log files."); Environment.Exit(1); } Stdout.VerboseWriteLine("Reading: "); var records = new List <AzureBlobLogRecordV1>(); int filesWithError = 0; foreach (string filename in logFilePaths) { Stdout.VerboseWriteLine("Reading: " + filename); try { this.ParseLogFile(filename, this.startTimeUTC, this.endTimeUTC, records); } catch (IndexOutOfRangeException e) { filesWithError++; Stdout.VerboseWriteLine(string.Format(CultureInfo.CurrentCulture, "File: {0}. Parsing exception:{1},", filename, e.Message)); } catch (IOException e) { filesWithError++; Stdout.VerboseWriteLine(string.Format(CultureInfo.CurrentCulture, "File: {0}. Parsing exception:{1},", filename, e.Message)); } catch (FormatException e) { filesWithError++; Stdout.VerboseWriteLine(string.Format(CultureInfo.CurrentCulture, "File: {0}. Parsing exception:{1},", filename, e.Message)); } } if (filesWithError > 0) { Stdout.WriteLine(string.Format(CultureInfo.CurrentCulture, "WARN: {0} logs files had errors. Was logcache set correctly? Use -verbose to see parsing exception messages", filesWithError)); } Stdout.WriteLine(); // -- !! do the analysis. this.Process(records); this.summaryWriter.Close(); this.detailsWriter.Close(); Stdout.WriteLine("Results written to: " + this.summaryFilename + " and " + this.detailsFilename); }
internal void ReadArgsThrottlingAnalysis(string[] args) { Stdout.WriteLine(); this.fullCmdLine = string.Join(" ", args); Exception e = null; bool unknownArg = false; int curr = 1; try { while (curr < args.Length) { if (args[curr].ToUpperInvariant() == "-LOGCACHE" && curr + 1 < args.Length) { curr++; this.logCacheRootfolder = args[curr]; curr++; } else if (args[curr].ToUpperInvariant() == "-ACCOUNT" && curr + 1 < args.Length) { curr++; this.accountName = args[curr]; curr++; } else if (args[curr].ToUpperInvariant() == "-START" && curr + 1 < args.Length) { curr++; this.startTimeUTC = DateTime.Parse(args[curr], CultureInfo.CurrentCulture).ToUniversalTime(); curr++; } else if (args[curr].ToUpperInvariant() == "-END" && curr + 1 < args.Length) { curr++; this.endTimeUTC = DateTime.Parse(args[curr], CultureInfo.CurrentCulture).ToUniversalTime(); curr++; } else if (args[curr].ToUpperInvariant() == "-NAME" && curr + 1 < args.Length) { curr++; this.jobName = args[curr]; curr++; this.summaryFilename = this.jobName + ".summary.csv"; this.detailsFilename = this.jobName + ".details.csv"; } else if (args[curr].ToUpperInvariant() == "-NOTE" && curr + 1 < args.Length) { curr++; this.jobNote = args[curr]; curr++; } else if (args[curr].ToUpperInvariant() == "-DEBUG") { curr++; //ignore } else if (args[curr].ToUpperInvariant() == "-VERBOSE" || args[curr].ToUpperInvariant() == "-V") { curr++; Stdout.Verbose = true; } else { unknownArg = true; break; } } } catch (FormatException ex) { e = ex; } if (e != null) { Stdout.WriteLine("Exception during arg processing:" + e.Message); this.AnalyserUsage(); } if (unknownArg) { Stdout.WriteLine("Unknown arg:" + args[curr]); this.AnalyserUsage(); } // input checks if (this.logCacheRootfolder == null) { Stdout.WriteLine("WARN: logcache defaulting to current directory. Use -logcache to target a specific directory for caching log files."); this.logCacheRootfolder = Environment.CurrentDirectory; } if (string.IsNullOrEmpty(this.accountName)) { Stdout.WriteLine("WARN: account not specified. Analysis will include all storage accounts in logcache directory."); } string path = Path.Combine(this.logCacheRootfolder, this.accountName); if (!Directory.Exists(this.logCacheRootfolder)) { Stdout.WriteLine("ERROR: logcache folder does not exist:" + path); } if (Directory.Exists(path)) { this.logFolder = path; } else { Stdout.WriteLine("WARN: account specific folder does not exist:" + path); Stdout.WriteLine(" using bare logcache folder:" + this.logCacheRootfolder); this.logFolder = this.logCacheRootfolder; } if (this.startTimeUTC == DateTime.MinValue) { Stdout.WriteLine("WARN: startTime not specified. Defaulting to MinValue. Some summary statistics will be affected."); } if (this.endTimeUTC == DateTime.MaxValue) { Stdout.WriteLine("WARN: endTime not specified. Defaulting to MaxValue. Some summary statistics will be affected."); } Stdout.WriteLine(); Stdout.WriteLine("Configuration:"); Stdout.WriteLine("-start " + Utils.DateTimeToStandardizedStringFormat(this.startTimeUTC)); Stdout.WriteLine("-end " + Utils.DateTimeToStandardizedStringFormat(this.endTimeUTC)); Stdout.VerboseWriteLine("-logCache " + this.logCacheRootfolder); Stdout.VerboseWriteLine("-account " + this.accountName); Stdout.WriteLine(); }