public static async Task<int> ExecuteAsync(RunOptions options) { // Create cancellation token CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); // Connect to Azure storage var storageAccount = CloudStorageAccount.Parse(options.ConnectionString); var container = storageAccount.CreateCloudBlobClient().GetContainerReference(options.LogContainer); if (!container.Exists()) { Trace.TraceError("No such container exists"); return -1; } // Determine data range var fromPrefix = options.FromPrefix.ToArray(); var untilPrefix = options.UntilPrefix.ToArray(); var fromDate = CreateDateTimeOffSet(fromPrefix); var untilDate = CreateDateTimeOffSet(untilPrefix); if (untilDate <= fromDate) { untilDate = untilDate.AddHours(1); } Trace.TraceInformation("Will replay requests in range: {0} - {1}", fromDate, untilDate); // Build container root var containerRoot = GetContainerRoot(options, fromPrefix, untilPrefix); // Get the log file blobs that should be replayed var blobsToReplay = await GetLogBlobsToReplay(container, containerRoot, fromDate, untilDate); if (!blobsToReplay.Any()) { Trace.TraceError("No log files have been found that correspond to the requested from-until range."); return -2; } Trace.TraceInformation("Found {0} log files to replay.", blobsToReplay.Count); // Run (and log to CSV) using (var logFileWriter = new StreamWriter(File.Open(options.LogFile, FileMode.Append, FileAccess.Write))) { // Write log CSV header if (logFileWriter.BaseStream.Position == 0) { await logFileWriter.WriteLineAsync("DateTime;Url;OriginalStatusCode;ReplayedStatusCode;OriginalTimeTaken;ReplayedTimeTaken;ReplayedStatusDescription"); } logFileWriter.AutoFlush = true; // Perform requests var tasks = new List<Task>(blobsToReplay.Count); foreach (var temp in blobsToReplay) { var blob = temp; var logCsv = logFileWriter; var task = await Task.Factory.StartNew(async () => { var httpClient = new HttpClient(); httpClient.Timeout = TimeSpan.FromMinutes(1); var stopwatch = new Stopwatch(); using (var streamReader = new StreamReader(blob.OpenRead())) { var previousRequestDateTime = DateTimeOffset.MinValue; do { var rawLogLine = streamReader.ReadLine(); if (rawLogLine != null) { var entry = W3CLogEntryParser.ParseLogEntryFromLine(rawLogLine); if (entry != null) { // Make sure the request rate is the same as the request rate in the log. if (previousRequestDateTime != DateTimeOffset.MinValue) { var difference = entry.RequestDateTime - previousRequestDateTime; await Task.Delay(difference, cancellationTokenSource.Token); } previousRequestDateTime = entry.RequestDateTime; // Run request var uriBuilder = new UriBuilder(options.Target); uriBuilder.Path = entry.RequestPath; uriBuilder.Query = entry.QueryString; Trace.WriteLine(string.Format("Current request: {0}", rawLogLine), "Verbose"); var referenceTime = DateTime.UtcNow; stopwatch.Start(); string csvLogLine; try { var response = await httpClient.GetAsync(uriBuilder.Uri, cancellationTokenSource.Token); stopwatch.Stop(); csvLogLine = string.Format("{0:O};\"{1}\";{2:0};{3:0};{4:0};{5:0};\"{6}\"", referenceTime, uriBuilder.Uri, entry.StatusCode ?? 0, (int)response.StatusCode, entry.TimeTaken ?? 0, (int)stopwatch.Elapsed.TotalMilliseconds, !response.IsSuccessStatusCode ? response.ReasonPhrase : string.Empty); Trace.TraceInformation("{0} {1} ({2:0.00}ms) - {3}", (int)response.StatusCode, response.ReasonPhrase, stopwatch.Elapsed.TotalMilliseconds, uriBuilder.Uri); } catch (TaskCanceledException) { stopwatch.Stop(); csvLogLine = string.Format("{0:O};\"{1}\";{2:0};{3:0};{4:0};{5:0};\"{6}\"", referenceTime, uriBuilder.Uri, entry.StatusCode ?? 0, 0, entry.TimeTaken ?? 0, (int)stopwatch.Elapsed.TotalMilliseconds, "Request timeout."); Trace.TraceError("Timeout ({0:0.00}ms) - {1}", stopwatch.Elapsed.TotalMilliseconds, uriBuilder.Uri); } finally { stopwatch.Reset(); } lock (logCsv) { logCsv.WriteLine(csvLogLine); } Trace.WriteLine(string.Format("Result: {0}", csvLogLine), "Verbose"); } } } while (!streamReader.EndOfStream); } }, cancellationTokenSource.Token, TaskCreationOptions.LongRunning, TaskScheduler.Default); tasks.Add(task); } // Wait for either a Console.ReadLine() or completion of all requests Task.WaitAny( Task.Factory.StartNew(() => { Console.ReadLine(); cancellationTokenSource.Cancel(false); }, cancellationTokenSource.Token), Task.Factory.StartNew(() => { try { Task.WaitAll(tasks.ToArray()); } catch (Exception) { if (!cancellationTokenSource.IsCancellationRequested) { throw; } } }, cancellationTokenSource.Token) ); } return 0; }
private static string GetContainerRoot(RunOptions options, string[] fromPrefix, string[] untilPrefix) { // We can be smart in listing blobs if the from - to range is relatively small var containerRoot = options.LogContainerRoot ?? string.Empty; if (!string.IsNullOrEmpty(containerRoot)) { containerRoot += "/"; } for (int i = 0; i < Math.Min(fromPrefix.Length, untilPrefix.Length); i++) { if (fromPrefix[i] == untilPrefix[i]) { containerRoot += string.Format("{0}/", fromPrefix[i]); } } return containerRoot; }
public static async Task <int> ExecuteAsync(RunOptions options) { // Create cancellation token CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); // Connect to Azure storage var storageAccount = CloudStorageAccount.Parse(options.ConnectionString); var container = storageAccount.CreateCloudBlobClient().GetContainerReference(options.LogContainer); if (!container.Exists()) { Trace.TraceError("No such container exists"); return(-1); } // Determine data range var fromPrefix = options.FromPrefix.ToArray(); var untilPrefix = options.UntilPrefix.ToArray(); var fromDate = CreateDateTimeOffSet(fromPrefix); var untilDate = CreateDateTimeOffSet(untilPrefix); if (untilDate <= fromDate) { untilDate = untilDate.AddHours(1); } Trace.TraceInformation("Will replay requests in range: {0} - {1}", fromDate, untilDate); // Build container root var containerRoot = GetContainerRoot(options, fromPrefix, untilPrefix); // Get the log file blobs that should be replayed var blobsToReplay = await GetLogBlobsToReplay(container, containerRoot, fromDate, untilDate); if (!blobsToReplay.Any()) { Trace.TraceError("No log files have been found that correspond to the requested from-until range."); return(-2); } Trace.TraceInformation("Found {0} log files to replay.", blobsToReplay.Count); // Run (and log to CSV) using (var logFileWriter = new StreamWriter(File.Open(options.LogFile, FileMode.Append, FileAccess.Write))) { // Write log CSV header if (logFileWriter.BaseStream.Position == 0) { await logFileWriter.WriteLineAsync("DateTime;Url;OriginalStatusCode;ReplayedStatusCode;OriginalTimeTaken;ReplayedTimeTaken;ReplayedStatusDescription"); } logFileWriter.AutoFlush = true; // Perform requests var tasks = new List <Task>(blobsToReplay.Count); foreach (var temp in blobsToReplay) { var blob = temp; var logCsv = logFileWriter; var task = await Task.Factory.StartNew(async() => { var httpClient = new HttpClient(); httpClient.Timeout = TimeSpan.FromMinutes(1); var stopwatch = new Stopwatch(); using (var streamReader = new StreamReader(blob.OpenRead())) { var previousRequestDateTime = DateTimeOffset.MinValue; do { var rawLogLine = streamReader.ReadLine(); if (rawLogLine != null) { var entry = W3CLogEntryParser.ParseLogEntryFromLine(rawLogLine); if (entry != null) { // Make sure the request rate is the same as the request rate in the log. if (previousRequestDateTime != DateTimeOffset.MinValue) { var difference = entry.RequestDateTime - previousRequestDateTime; await Task.Delay(difference, cancellationTokenSource.Token); } previousRequestDateTime = entry.RequestDateTime; // Run request var uriBuilder = new UriBuilder(options.Target); uriBuilder.Path = entry.RequestPath; uriBuilder.Query = entry.QueryString; Trace.WriteLine(string.Format("Current request: {0}", rawLogLine), "Verbose"); var referenceTime = DateTime.UtcNow; stopwatch.Start(); string csvLogLine; try { var response = await httpClient.GetAsync(uriBuilder.Uri, cancellationTokenSource.Token); stopwatch.Stop(); csvLogLine = string.Format("{0:O};\"{1}\";{2:0};{3:0};{4:0};{5:0};\"{6}\"", referenceTime, uriBuilder.Uri, entry.StatusCode ?? 0, (int)response.StatusCode, entry.TimeTaken ?? 0, (int)stopwatch.Elapsed.TotalMilliseconds, !response.IsSuccessStatusCode ? response.ReasonPhrase : string.Empty); Trace.TraceInformation("{0} {1} ({2:0.00}ms) - {3}", (int)response.StatusCode, response.ReasonPhrase, stopwatch.Elapsed.TotalMilliseconds, uriBuilder.Uri); } catch (TaskCanceledException) { stopwatch.Stop(); csvLogLine = string.Format("{0:O};\"{1}\";{2:0};{3:0};{4:0};{5:0};\"{6}\"", referenceTime, uriBuilder.Uri, entry.StatusCode ?? 0, 0, entry.TimeTaken ?? 0, (int)stopwatch.Elapsed.TotalMilliseconds, "Request timeout."); Trace.TraceError("Timeout ({0:0.00}ms) - {1}", stopwatch.Elapsed.TotalMilliseconds, uriBuilder.Uri); } finally { stopwatch.Reset(); } lock (logCsv) { logCsv.WriteLine(csvLogLine); } Trace.WriteLine(string.Format("Result: {0}", csvLogLine), "Verbose"); } } }while (!streamReader.EndOfStream); } }, cancellationTokenSource.Token, TaskCreationOptions.LongRunning, TaskScheduler.Default); tasks.Add(task); } // Wait for either a Console.ReadLine() or completion of all requests Task.WaitAny( Task.Factory.StartNew(() => { Console.ReadLine(); cancellationTokenSource.Cancel(false); }, cancellationTokenSource.Token), Task.Factory.StartNew(() => { try { Task.WaitAll(tasks.ToArray()); } catch (Exception) { if (!cancellationTokenSource.IsCancellationRequested) { throw; } } }, cancellationTokenSource.Token) ); } return(0); }