private static void PrintUsage() { Console.ForegroundColor = ConsoleColor.DarkMagenta; Console.WriteLine(@" Traffic Recording and Replaying utility for RavenDB ---------------------------------------------- Copyright (C) 2008 - {0} - Hibernating Rhinos ----------------------------------------------", SystemTime.UtcNow.Year); Console.ForegroundColor = ConsoleColor.White; Console.WriteLine(@" Usage: Raven.Traffic [Mode(rec/play)] [Url] [resource name] [recordingFile] [[option1], [option2] ...] Examples: - Record 'Northwind' database found on specified server: Raven.Traffic rec http://localhost:8080/ Northwind recording.json - Replay 'Northwind' database from specified server to the dump.raven file: Raven.Traffic play http://localhost:8080/ Northwind recording.json * Use ""<System>"" to state the system database" ); Console.ForegroundColor = ConsoleColor.Green; TrafficToolConfiguration.InitOptionsSetObject().WriteOptionDescriptions(Console.Out); Console.ForegroundColor = ConsoleColor.White; }
public static OptionSet InitOptionsSetObject(TrafficToolConfiguration config = null) { var options = new OptionSet(); options.OnWarning += s => ConsoleHelper.WriteLineWithColor(ConsoleColor.Yellow, s); options.Add("traceSeconds:", OptionCategory.TrafficRecordReplay, "Time to perform the traffic watch(seconds)", x => { var durationConstraint = Int32.Parse(x); config.DurationConstraint = TimeSpan.FromSeconds(durationConstraint); }); options.Add("traceRequests:", OptionCategory.TrafficRecordReplay, "Time to perform the traffic watch", x => { var amountConstraint = Int32.Parse(x); config.AmountConstraint = amountConstraint; }); options.Add("compressed", OptionCategory.TrafficRecordReplay, "Work with compressed json outpu/input", x => { config.IsCompressed = true; }); options.Add("noOutput", OptionCategory.TrafficRecordReplay, "Suppress console progress output", value => config.PrintOutput = false); options.Add("timeout:", OptionCategory.TrafficRecordReplay, "The timeout to use for requests(seconds)", s => config.Timeout = TimeSpan.FromSeconds(int.Parse(s))); options.Add("u|user|username:"******"The username to use when the database requires the client to authenticate.", value => GetCredentials(config.ConnectionString).UserName = value); options.Add("p|pass|password:"******"The password to use when the database requires the client to authenticate.", value => GetCredentials(config.ConnectionString).Password = value); options.Add("domain:", OptionCategory.TrafficRecordReplay, "The domain to use when the database requires the client to authenticate.", value => GetCredentials(config.ConnectionString).Domain = value); options.Add("key|api-key|apikey:", OptionCategory.TrafficRecordReplay, "The API-key to use, when using OAuth.", value => config.ApiKey = value); return(options); }
private static void Main(string[] args) { TrafficToolConfiguration config; var parseStatus = TrafficToolConfiguration.ProcessArgs(args, out config); switch (parseStatus) { case TrafficToolConfiguration.TrafficArgsProcessStatus.InvalidMode: PrintUsage(); break; case TrafficToolConfiguration.TrafficArgsProcessStatus.NoArguments: PrintUsage(); break; case TrafficToolConfiguration.TrafficArgsProcessStatus.NotEnoughArguments: Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("Illegal arguments amount, see usage instructions:"); Console.ForegroundColor = ConsoleColor.White; break; case TrafficToolConfiguration.TrafficArgsProcessStatus.ValidConfig: IDocumentStore store; config.ResourceName = config.ResourceName == "<system>" ? null : config.ResourceName; try { store = new DocumentStore { Url = config.ConnectionString.Url, DefaultDatabase = config.ResourceName, Credentials = config.ConnectionString.Credentials, }.Initialize(); } catch (Exception e) { Console.WriteLine("Could not connect to server. Exception: {0}", e); return; } using (store) { try { store.DatabaseCommands.GetStatistics(); } catch (Exception e) { Console.WriteLine("Database does not exist"); return; } new TrafficRec(store, config).ExecuteTrafficCommand(); } break; } }
public static TrafficArgsProcessStatus ProcessArgs(string[] args, out TrafficToolConfiguration config) { if (args.Length == 0) { config = null; return(TrafficArgsProcessStatus.NoArguments); } if (args.Length < 4) { config = null; return(TrafficArgsProcessStatus.NoArguments); } new Url(args[1]); config = new TrafficToolConfiguration(); switch (args[0]) { case "rec": config.Mode = TrafficToolMode.Record; break; case "play": config.Mode = TrafficToolMode.Replay; break; default: config = null; return(TrafficArgsProcessStatus.InvalidMode); } config.ConnectionString.Url = args[1]; config.ConnectionString.DefaultDatabase = args[2] == "<system>"?null:args[2]; config.ResourceName = args[2] == "<system>" ? null : args[2]; config.RecordFilePath = args[3]; InitOptionsSetObject(config).Parse(args); if (config.AmountConstraint.HasValue == false && config.DurationConstraint.HasValue == false) { config.AmountConstraint = 1000; config.DurationConstraint = TimeSpan.FromSeconds(60); } return(TrafficArgsProcessStatus.ValidConfig); }
public static OptionSet InitOptionsSetObject(TrafficToolConfiguration config = null) { var options = new OptionSet(); options.OnWarning += s => ConsoleHelper.WriteLineWithColor(ConsoleColor.Yellow, s); options.Add("traceSeconds:", OptionCategory.TrafficRecordReplay, "Time to perform the traffic watch(seconds)", x => { var durationConstraint = Int32.Parse(x); config.DurationConstraint = TimeSpan.FromSeconds(durationConstraint); }); options.Add("traceRequests:", OptionCategory.TrafficRecordReplay, "Time to perform the traffic watch", x => { var amountConstraint = Int32.Parse(x); config.AmountConstraint = amountConstraint; }); options.Add("compressed", OptionCategory.TrafficRecordReplay, "Time to perform the traffic watch", x => { config.IsCompressed = true; }); options.Add("noOutput", OptionCategory.TrafficRecordReplay, "Suppress output", value => config.PrintOutput = false); options.Add("timeout:", OptionCategory.TrafficRecordReplay, "The timeout to use for requests(seconds)", s => config.Timeout = TimeSpan.FromSeconds(int.Parse(s))); options.Add("u|user|username:"******"The username to use when the database requires the client to authenticate.", value => GetCredentials(config.ConnectionString).UserName = value); options.Add("p|pass|password:"******"The password to use when the database requires the client to authenticate.", value => GetCredentials(config.ConnectionString).Password = value); options.Add("domain:", OptionCategory.TrafficRecordReplay, "The domain to use when the database requires the client to authenticate.", value => GetCredentials(config.ConnectionString).Domain = value); options.Add("key|api-key|apikey:", OptionCategory.TrafficRecordReplay, "The API-key to use, when using OAuth.", value => config.ApiKey = value); return options; }
private void ReplayRequests(TrafficToolConfiguration config, IDocumentStore store) { Stream finalStream; var requestsCounter = 0; var skippedRequestsCounter = 0; var totalCountOfLogicRequests = 0; var totalSp = Stopwatch.StartNew(); using (var stream = File.Open(config.RecordFilePath, FileMode.Open)) { if (config.IsCompressed) { finalStream = new GZipStream(stream, CompressionMode.Decompress, leaveOpen: true); } else { finalStream = stream; } var trafficLogs = JsonSerializer.Create().Deserialize<LogHttpRequestStatsParams[]>(new JsonTextReader(new StreamReader(finalStream))); ConcurrentQueue<string> queue = null; var cts = new CancellationTokenSource(); var ct = cts.Token; Task outputTask = null; if (config.PrintOutput) { queue = new ConcurrentQueue<string>(); outputTask = Task.Run(() => { while (!ct.IsCancellationRequested || queue.Count != 0) { string message; if (queue.TryDequeue(out message)) { Console.WriteLine(message); } else { Thread.Sleep(10); } } }); } const string postLineSeparatorRegex = "\\t\\d: databases\\/[\\w\\.]+"; const string endOfPostLineString = "\t\t\tQuery:"; const string uriCleanRegex = "http://[\\w\\.]+(:\\d*)?(\\/databases\\/[\\w\\.]+)?"; Parallel.ForEach(trafficLogs, new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }, trafficLog => { var sp = Stopwatch.StartNew(); GetRequest[] requestsArray = null; var uriString = Regex.Replace(trafficLog.RequestUri, uriCleanRegex, string.Empty); string trafficUrlPart; string trafficQueryPart; trafficUrlPart = ExtractUrlAndQuery(uriString, out trafficQueryPart); var curCount = Interlocked.Increment(ref requestsCounter); if (ValidateUrlString(trafficUrlPart)) { Interlocked.Increment(ref skippedRequestsCounter); if (queue != null) { queue.Enqueue(string.Format("{0} out of {1}, skipped whole message", curCount, trafficLogs.Length)); } return; } Interlocked.Increment(ref totalCountOfLogicRequests); if (trafficLog.HttpMethod.Equals("get", StringComparison.CurrentCultureIgnoreCase)) { requestsArray = new[] { new GetRequest { Url = trafficUrlPart, Query = trafficQueryPart } }; } else if (trafficLog.CustomInfo != null) { var subArray = Regex.Split(trafficLog.CustomInfo.Replace("\r", string.Empty), postLineSeparatorRegex).Where(x => !String.IsNullOrEmpty(x)).Select(x => { var endOfPostLastIndex = x.IndexOf(endOfPostLineString); if (endOfPostLastIndex < 0) return x; return x.Remove(endOfPostLastIndex); }).ToArray(); requestsArray = subArray.Select(customInfoLine => { trafficUrlPart = ExtractUrlAndQuery(customInfoLine, out trafficQueryPart); if (ValidateUrlString(trafficUrlPart)) { if (queue != null) { queue.Enqueue(string.Format("{0} out of {1}, skipped inner message", curCount, trafficLogs.Length)); } return null; } return new GetRequest { Url = trafficUrlPart, Query = trafficQueryPart, }; }).Where(x => x != null).ToArray(); } Interlocked.Add(ref totalCountOfLogicRequests, requestsArray.Length); try { store.DatabaseCommands.MultiGet(requestsArray); if (queue != null) { queue.Enqueue(string.Format("{0} out of {1}, took {2} ms. Total Time: {3} ms", curCount, trafficLogs.Length, sp.ElapsedMilliseconds, totalSp.ElapsedMilliseconds)); } } catch (Exception e) { Interlocked.Increment(ref skippedRequestsCounter); if (queue != null) { queue.Enqueue(string.Format("{0} out of {1}, failed", curCount, trafficLogs.Length, sp.ElapsedMilliseconds, totalSp.ElapsedMilliseconds)); } } }); if (outputTask != null) { cts.Cancel(); outputTask.Wait(); } } Console.WriteLine(@"Summary: Requests sent: {0} Requests skipped: {1} Nested and non nested request: {2} Total Time: {3}", requestsCounter, skippedRequestsCounter, totalCountOfLogicRequests, totalSp.ElapsedMilliseconds); }
public TrafficRec(IDocumentStore store, TrafficToolConfiguration config) { this.store = store; this.config = config; }
/// <summary> /// Connects to raven traffic event source and registers all the requests to the file defined in the config /// </summary> /// <param name="config">configuration conatining the connection, the file to write to, etc.</param> /// <param name="store">the store to work with</param> private void RecordRequests(TrafficToolConfiguration config, IDocumentStore store) { var requestsQueue = new ConcurrentQueue<RavenJObject>(); var messagesCount = 0; var mre = new ManualResetEvent(false); var trafficWatchObserver = new TrafficWatchObserver(store,config.ResourceName, mre, config.Timeout, x => { if (config.AmountConstraint.HasValue && Interlocked.Increment(ref messagesCount) > config.AmountConstraint.Value) mre.Set(); else requestsQueue.Enqueue(x); }); trafficWatchObserver.EstablishConnection(); var writingTask = Task.Run(() => { WriteRequestsFromQueueToFile(requestsQueue, config.RecordFilePath, config.IsCompressed,config.PrintOutput, mre); }); if (config.DurationConstraint.HasValue) mre.WaitOne(config.DurationConstraint.Value); else mre.WaitOne(); mre.Set(); writingTask.Wait(); trafficWatchObserver.OnCompleted(); }
public static TrafficArgsProcessStatus ProcessArgs(string[] args, out TrafficToolConfiguration config) { if (args.Length == 0) { config = null; return TrafficArgsProcessStatus.NoArguments; } if (args.Length < 4) { config = null; return TrafficArgsProcessStatus.NoArguments; } new Url(args[1]); config = new TrafficToolConfiguration(); switch (args[0]) { case "rec": config.Mode = TrafficToolMode.Record; break; case "play": config.Mode = TrafficToolMode.Replay; break; default: config = null; return TrafficArgsProcessStatus.InvalidMode; } config.ConnectionString.Url = args[1]; config.ConnectionString.DefaultDatabase = args[2]; config.ResourceName = args[2]; config.RecordFilePath = args[3]; InitOptionsSetObject(config).Parse(args); if (config.AmountConstraint.HasValue == false && config.DurationConstraint.HasValue == false) { config.AmountConstraint = 1000; config.DurationConstraint = TimeSpan.FromSeconds(60); } return TrafficArgsProcessStatus.ValidConfig; }