private static void PrintUsage()
{
Console.ForegroundColor = ConsoleColor.DarkMagenta;
Console.WriteLine(@"
Traffic Recording and Replaying utility for RavenDB
----------------------------------------------
Copyright (C) 2008 - {0} - Hibernating Rhinos
----------------------------------------------", SystemTime.UtcNow.Year);
Console.ForegroundColor = ConsoleColor.White;
Console.WriteLine(@"
Usage:
Raven.Traffic [Mode(rec/play)] [Url] [resource name] [recordingFile] [[option1], [option2] ...]
Examples:
- Record 'Northwind' database found on specified server:
Raven.Traffic rec http://localhost:8080/ Northwind recording.json
- Replay 'Northwind' database from specified server to the dump.raven file:
Raven.Traffic play http://localhost:8080/ Northwind recording.json
* Use ""<System>"" to state the system database"
);
Console.ForegroundColor = ConsoleColor.Green;
TrafficToolConfiguration.InitOptionsSetObject().WriteOptionDescriptions(Console.Out);
Console.ForegroundColor = ConsoleColor.White;
}
public static OptionSet InitOptionsSetObject(TrafficToolConfiguration config = null)
{
var options = new OptionSet();
options.OnWarning += s => ConsoleHelper.WriteLineWithColor(ConsoleColor.Yellow, s);
options.Add("traceSeconds:", OptionCategory.TrafficRecordReplay, "Time to perform the traffic watch(seconds)", x =>
{
var durationConstraint = Int32.Parse(x);
config.DurationConstraint = TimeSpan.FromSeconds(durationConstraint);
});
options.Add("traceRequests:", OptionCategory.TrafficRecordReplay, "Time to perform the traffic watch", x =>
{
var amountConstraint = Int32.Parse(x);
config.AmountConstraint = amountConstraint;
});
options.Add("compressed", OptionCategory.TrafficRecordReplay, "Work with compressed json outpu/input", x => { config.IsCompressed = true; });
options.Add("noOutput", OptionCategory.TrafficRecordReplay, "Suppress console progress output", value => config.PrintOutput = false);
options.Add("timeout:", OptionCategory.TrafficRecordReplay, "The timeout to use for requests(seconds)", s => config.Timeout = TimeSpan.FromSeconds(int.Parse(s)));
options.Add("u|user|username:"******"The username to use when the database requires the client to authenticate.", value => GetCredentials(config.ConnectionString).UserName = value);
options.Add("p|pass|password:"******"The password to use when the database requires the client to authenticate.", value => GetCredentials(config.ConnectionString).Password = value);
options.Add("domain:", OptionCategory.TrafficRecordReplay, "The domain to use when the database requires the client to authenticate.", value => GetCredentials(config.ConnectionString).Domain = value);
options.Add("key|api-key|apikey:", OptionCategory.TrafficRecordReplay, "The API-key to use, when using OAuth.", value => config.ApiKey = value);
return(options);
}
private static void Main(string[] args)
{
TrafficToolConfiguration config;
var parseStatus = TrafficToolConfiguration.ProcessArgs(args, out config);
switch (parseStatus)
{
case TrafficToolConfiguration.TrafficArgsProcessStatus.InvalidMode:
PrintUsage();
break;
case TrafficToolConfiguration.TrafficArgsProcessStatus.NoArguments:
PrintUsage();
break;
case TrafficToolConfiguration.TrafficArgsProcessStatus.NotEnoughArguments:
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("Illegal arguments amount, see usage instructions:");
Console.ForegroundColor = ConsoleColor.White;
break;
case TrafficToolConfiguration.TrafficArgsProcessStatus.ValidConfig:
IDocumentStore store;
config.ResourceName = config.ResourceName == "<system>" ? null : config.ResourceName;
try
{
store = new DocumentStore
{
Url = config.ConnectionString.Url,
DefaultDatabase = config.ResourceName,
Credentials = config.ConnectionString.Credentials,
}.Initialize();
}
catch (Exception e)
{
Console.WriteLine("Could not connect to server. Exception: {0}", e);
return;
}
using (store)
{
try
{
store.DatabaseCommands.GetStatistics();
}
catch (Exception e)
{
Console.WriteLine("Database does not exist");
return;
}
new TrafficRec(store, config).ExecuteTrafficCommand();
}
break;
}
}
public static TrafficArgsProcessStatus ProcessArgs(string[] args, out TrafficToolConfiguration config)
{
if (args.Length == 0)
{
config = null;
return(TrafficArgsProcessStatus.NoArguments);
}
if (args.Length < 4)
{
config = null;
return(TrafficArgsProcessStatus.NoArguments);
}
new Url(args[1]);
config = new TrafficToolConfiguration();
switch (args[0])
{
case "rec":
config.Mode = TrafficToolMode.Record;
break;
case "play":
config.Mode = TrafficToolMode.Replay;
break;
default:
config = null;
return(TrafficArgsProcessStatus.InvalidMode);
}
config.ConnectionString.Url = args[1];
config.ConnectionString.DefaultDatabase = args[2] == "<system>"?null:args[2];
config.ResourceName = args[2] == "<system>" ? null : args[2];
config.RecordFilePath = args[3];
InitOptionsSetObject(config).Parse(args);
if (config.AmountConstraint.HasValue == false && config.DurationConstraint.HasValue == false)
{
config.AmountConstraint = 1000;
config.DurationConstraint = TimeSpan.FromSeconds(60);
}
return(TrafficArgsProcessStatus.ValidConfig);
}
public static OptionSet InitOptionsSetObject(TrafficToolConfiguration config = null)
{
var options = new OptionSet();
options.OnWarning += s => ConsoleHelper.WriteLineWithColor(ConsoleColor.Yellow, s);
options.Add("traceSeconds:", OptionCategory.TrafficRecordReplay, "Time to perform the traffic watch(seconds)", x =>
{
var durationConstraint = Int32.Parse(x);
config.DurationConstraint = TimeSpan.FromSeconds(durationConstraint);
});
options.Add("traceRequests:", OptionCategory.TrafficRecordReplay, "Time to perform the traffic watch", x =>
{
var amountConstraint = Int32.Parse(x);
config.AmountConstraint = amountConstraint;
});
options.Add("compressed", OptionCategory.TrafficRecordReplay, "Time to perform the traffic watch", x => { config.IsCompressed = true; });
options.Add("noOutput", OptionCategory.TrafficRecordReplay, "Suppress output", value => config.PrintOutput = false);
options.Add("timeout:", OptionCategory.TrafficRecordReplay, "The timeout to use for requests(seconds)", s => config.Timeout = TimeSpan.FromSeconds(int.Parse(s)));
options.Add("u|user|username:"******"The username to use when the database requires the client to authenticate.", value => GetCredentials(config.ConnectionString).UserName = value);
options.Add("p|pass|password:"******"The password to use when the database requires the client to authenticate.", value => GetCredentials(config.ConnectionString).Password = value);
options.Add("domain:", OptionCategory.TrafficRecordReplay, "The domain to use when the database requires the client to authenticate.", value => GetCredentials(config.ConnectionString).Domain = value);
options.Add("key|api-key|apikey:", OptionCategory.TrafficRecordReplay, "The API-key to use, when using OAuth.", value => config.ApiKey = value);
return options;
}
private void ReplayRequests(TrafficToolConfiguration config, IDocumentStore store)
{
Stream finalStream;
var requestsCounter = 0;
var skippedRequestsCounter = 0;
var totalCountOfLogicRequests = 0;
var totalSp = Stopwatch.StartNew();
using (var stream = File.Open(config.RecordFilePath, FileMode.Open))
{
if (config.IsCompressed)
{
finalStream = new GZipStream(stream, CompressionMode.Decompress, leaveOpen: true);
}
else
{
finalStream = stream;
}
var trafficLogs =
JsonSerializer.Create().Deserialize<LogHttpRequestStatsParams[]>(new JsonTextReader(new StreamReader(finalStream)));
ConcurrentQueue<string> queue = null;
var cts = new CancellationTokenSource();
var ct = cts.Token;
Task outputTask = null;
if (config.PrintOutput)
{
queue = new ConcurrentQueue<string>();
outputTask = Task.Run(() =>
{
while (!ct.IsCancellationRequested || queue.Count != 0)
{
string message;
if (queue.TryDequeue(out message))
{
Console.WriteLine(message);
}
else
{
Thread.Sleep(10);
}
}
});
}
const string postLineSeparatorRegex = "\\t\\d: databases\\/[\\w\\.]+";
const string endOfPostLineString = "\t\t\tQuery:";
const string uriCleanRegex = "http://[\\w\\.]+(:\\d*)?(\\/databases\\/[\\w\\.]+)?";
Parallel.ForEach(trafficLogs, new ParallelOptions
{
MaxDegreeOfParallelism = Environment.ProcessorCount
}, trafficLog =>
{
var sp = Stopwatch.StartNew();
GetRequest[] requestsArray = null;
var uriString = Regex.Replace(trafficLog.RequestUri, uriCleanRegex, string.Empty);
string trafficUrlPart;
string trafficQueryPart;
trafficUrlPart = ExtractUrlAndQuery(uriString, out trafficQueryPart);
var curCount = Interlocked.Increment(ref requestsCounter);
if (ValidateUrlString(trafficUrlPart))
{
Interlocked.Increment(ref skippedRequestsCounter);
if (queue != null)
{
queue.Enqueue(string.Format("{0} out of {1}, skipped whole message",
curCount, trafficLogs.Length));
}
return;
}
Interlocked.Increment(ref totalCountOfLogicRequests);
if (trafficLog.HttpMethod.Equals("get", StringComparison.CurrentCultureIgnoreCase))
{
requestsArray = new[]
{
new GetRequest
{
Url = trafficUrlPart,
Query = trafficQueryPart
}
};
}
else if (trafficLog.CustomInfo != null)
{
var subArray = Regex.Split(trafficLog.CustomInfo.Replace("\r", string.Empty), postLineSeparatorRegex).Where(x => !String.IsNullOrEmpty(x)).Select(x =>
{
var endOfPostLastIndex = x.IndexOf(endOfPostLineString);
if (endOfPostLastIndex < 0)
return x;
return x.Remove(endOfPostLastIndex);
}).ToArray();
requestsArray =
subArray.Select(customInfoLine =>
{
trafficUrlPart = ExtractUrlAndQuery(customInfoLine, out trafficQueryPart);
if (ValidateUrlString(trafficUrlPart))
{
if (queue != null)
{
queue.Enqueue(string.Format("{0} out of {1}, skipped inner message",
curCount, trafficLogs.Length));
}
return null;
}
return new GetRequest
{
Url = trafficUrlPart,
Query = trafficQueryPart,
};
}).Where(x => x != null).ToArray();
}
Interlocked.Add(ref totalCountOfLogicRequests, requestsArray.Length);
try
{
store.DatabaseCommands.MultiGet(requestsArray);
if (queue != null)
{
queue.Enqueue(string.Format("{0} out of {1}, took {2} ms. Total Time: {3} ms",
curCount, trafficLogs.Length, sp.ElapsedMilliseconds, totalSp.ElapsedMilliseconds));
}
}
catch (Exception e)
{
Interlocked.Increment(ref skippedRequestsCounter);
if (queue != null)
{
queue.Enqueue(string.Format("{0} out of {1}, failed",
curCount, trafficLogs.Length, sp.ElapsedMilliseconds, totalSp.ElapsedMilliseconds));
}
}
});
if (outputTask != null)
{
cts.Cancel();
outputTask.Wait();
}
}
Console.WriteLine(@"Summary:
Requests sent: {0}
Requests skipped: {1}
Nested and non nested request: {2}
Total Time: {3}", requestsCounter, skippedRequestsCounter, totalCountOfLogicRequests, totalSp.ElapsedMilliseconds);
}
public TrafficRec(IDocumentStore store, TrafficToolConfiguration config)
{
this.store = store;
this.config = config;
}
/// <summary>
/// Connects to raven traffic event source and registers all the requests to the file defined in the config
/// </summary>
/// <param name="config">configuration conatining the connection, the file to write to, etc.</param>
/// <param name="store">the store to work with</param>
private void RecordRequests(TrafficToolConfiguration config, IDocumentStore store)
{
var requestsQueue = new ConcurrentQueue<RavenJObject>();
var messagesCount = 0;
var mre = new ManualResetEvent(false);
var trafficWatchObserver = new TrafficWatchObserver(store,config.ResourceName, mre, config.Timeout, x =>
{
if (config.AmountConstraint.HasValue && Interlocked.Increment(ref messagesCount) > config.AmountConstraint.Value)
mre.Set();
else
requestsQueue.Enqueue(x);
});
trafficWatchObserver.EstablishConnection();
var writingTask = Task.Run(() =>
{
WriteRequestsFromQueueToFile(requestsQueue, config.RecordFilePath, config.IsCompressed,config.PrintOutput, mre);
});
if (config.DurationConstraint.HasValue)
mre.WaitOne(config.DurationConstraint.Value);
else
mre.WaitOne();
mre.Set();
writingTask.Wait();
trafficWatchObserver.OnCompleted();
}
public static TrafficArgsProcessStatus ProcessArgs(string[] args, out TrafficToolConfiguration config)
{
if (args.Length == 0)
{
config = null;
return TrafficArgsProcessStatus.NoArguments;
}
if (args.Length < 4)
{
config = null;
return TrafficArgsProcessStatus.NoArguments;
}
new Url(args[1]);
config = new TrafficToolConfiguration();
switch (args[0])
{
case "rec":
config.Mode = TrafficToolMode.Record;
break;
case "play":
config.Mode = TrafficToolMode.Replay;
break;
default:
config = null;
return TrafficArgsProcessStatus.InvalidMode;
}
config.ConnectionString.Url = args[1];
config.ConnectionString.DefaultDatabase = args[2];
config.ResourceName = args[2];
config.RecordFilePath = args[3];
InitOptionsSetObject(config).Parse(args);
if (config.AmountConstraint.HasValue == false && config.DurationConstraint.HasValue == false)
{
config.AmountConstraint = 1000;
config.DurationConstraint = TimeSpan.FromSeconds(60);
}
return TrafficArgsProcessStatus.ValidConfig;
}
