public static OptionSet InitOptionsSetObject(TrafficToolConfiguration config = null)
{
var options = new OptionSet();
options.OnWarning += s => WriteLineWithColor(ConsoleColor.Yellow, s);
options.Add("traceSeconds:", OptionCategory.TrafficRecordReplay, "Time to perform the traffic watch(seconds)", x =>
{
var durationConstraint = Int32.Parse(x);
config.DurationConstraint = TimeSpan.FromSeconds(durationConstraint);
});
options.Add("traceRequests:", OptionCategory.TrafficRecordReplay, "Time to perform the traffic watch", x =>
{
var amountConstraint = Int32.Parse(x);
config.AmountConstraint = amountConstraint;
});
options.Add("compressed", OptionCategory.TrafficRecordReplay, "Work with compressed json outpu/input", x => { config.IsCompressed = true; });
options.Add("noOutput", OptionCategory.TrafficRecordReplay, "Suppress console progress output", value => config.PrintOutput = false);
options.Add("timeout:", OptionCategory.TrafficRecordReplay, "The timeout to use for requests(seconds)", s => config.Timeout = TimeSpan.FromSeconds(int.Parse(s)));
options.Add("u|user|username:"******"The username to use when the database requires the client to authenticate.", value => GetCredentials(config.ConnectionString).UserName = value);
options.Add("p|pass|password:"******"The password to use when the database requires the client to authenticate.", value => GetCredentials(config.ConnectionString).Password = value);
options.Add("domain:", OptionCategory.TrafficRecordReplay, "The domain to use when the database requires the client to authenticate.", value => GetCredentials(config.ConnectionString).Domain = value);
options.Add("key|api-key|apikey:", OptionCategory.TrafficRecordReplay, "The API-key to use, when using OAuth.", value => config.ApiKey = value);
return(options);
}
public static TrafficArgsProcessStatus ProcessArgs(string[] args, out TrafficToolConfiguration config)
{
if (args.Length == 0)
{
config = null;
return(TrafficArgsProcessStatus.NoArguments);
}
if (args.Length < 4)
{
config = null;
return(TrafficArgsProcessStatus.NoArguments);
}
try
{
// ReSharper disable once ObjectCreationAsStatement
new Uri(args[1]);
}
catch (UriFormatException)
{
Console.WriteLine("ERROR : Server's url provided isn't in valid format");
throw;
}
config = new TrafficToolConfiguration();
switch (args[0])
{
case "rec":
config.Mode = TrafficToolMode.Record;
break;
case "play":
config.Mode = TrafficToolMode.Replay;
break;
default:
config = null;
return(TrafficArgsProcessStatus.InvalidMode);
}
config.Urls = new List <string> {
args[1]
};
config.Database = args[2];
config.ResourceName = args[2];
config.RecordFilePath = args[3];
InitOptionsSetObject(config).Parse(args);
if (config.AmountConstraint.HasValue == false && config.DurationConstraint.HasValue == false)
{
config.AmountConstraint = 1000;
config.DurationConstraint = TimeSpan.FromSeconds(60);
}
return(TrafficArgsProcessStatus.ValidConfig);
}
private static void Main(string[] args)
{
TrafficToolConfiguration config;
var parseStatus = TrafficToolConfiguration.ProcessArgs(args, out config);
switch (parseStatus)
{
case TrafficToolConfiguration.TrafficArgsProcessStatus.InvalidMode:
PrintUsage();
break;
case TrafficToolConfiguration.TrafficArgsProcessStatus.NoArguments:
PrintUsage();
break;
case TrafficToolConfiguration.TrafficArgsProcessStatus.NotEnoughArguments:
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("Illegal arguments amount, see usage instructions:");
Console.ForegroundColor = ConsoleColor.White;
break;
case TrafficToolConfiguration.TrafficArgsProcessStatus.ValidConfig:
IDocumentStore store;
try
{
store = new DocumentStore
{
Urls = config.Urls.ToArray(),
Database = config.ResourceName
}.Initialize();
}
catch (Exception e)
{
Console.WriteLine("Could not connect to server. Exception: {0}", e);
return;
}
using (store)
{
try
{
store.Maintenance.Send(new GetStatisticsOperation());
}
catch (Exception)
{
Console.WriteLine("Database does not exist");
return;
}
new TrafficRec(store, config).ExecuteTrafficCommand();
}
break;
}
}
public static OptionSet InitOptionsSetObject(TrafficToolConfiguration config = null)
{
var options = new OptionSet();
options.OnWarning += s => WriteLineWithColor(ConsoleColor.Yellow, s);
options.Add("traceSeconds:", OptionCategory.TrafficRecordReplay, "Time to perform the traffic watch(seconds)", x =>
{
var durationConstraint = int.Parse(x);
config.DurationConstraint = TimeSpan.FromSeconds(durationConstraint);
});
options.Add("traceRequests:", OptionCategory.TrafficRecordReplay, "Time to perform the traffic watch", x =>
{
var amountConstraint = int.Parse(x);
config.AmountConstraint = amountConstraint;
});
options.Add("compressed", OptionCategory.TrafficRecordReplay, "Work with compressed json outpu/input", x => { config.IsCompressed = true; });
options.Add("noOutput", OptionCategory.TrafficRecordReplay, "Suppress console progress output", value => config.PrintOutput = false);
options.Add("timeout:", OptionCategory.TrafficRecordReplay, "The timeout to use for requests(seconds)", s => config.Timeout = TimeSpan.FromSeconds(int.Parse(s)));
return(options);
}
/// <summary>
/// Connects to raven traffic event source and registers all the requests to the file defined in the config
/// </summary>
/// <param name="config">configuration conatining the connection, the file to write to, etc.</param>
/// <param name="store">the store to work with</param>
private void RecordRequests(TrafficToolConfiguration config, IDocumentStore store)
{
var requestsQueue = new ConcurrentQueue <RavenJObject>();
var messagesCount = 0;
var mre = new ManualResetEvent(false);
var trafficWatchObserver = new TrafficWatchObserver(store, config.ResourceName, mre, config.Timeout, x =>
{
if (config.AmountConstraint.HasValue && Interlocked.Increment(ref messagesCount) > config.AmountConstraint.Value)
{
mre.Set();
}
else
{
requestsQueue.Enqueue(x);
}
});
trafficWatchObserver.EstablishConnection();
var writingTask = Task.Run(() =>
{
WriteRequestsFromQueueToFile(requestsQueue, config.RecordFilePath, config.IsCompressed, config.PrintOutput, mre);
});
if (config.DurationConstraint.HasValue)
{
mre.WaitOne(config.DurationConstraint.Value);
}
else
{
mre.WaitOne();
}
mre.Set();
writingTask.Wait();
trafficWatchObserver.OnCompleted();
}
private static void PrintUsage()
{
Console.ForegroundColor = ConsoleColor.DarkMagenta;
Console.WriteLine(@"
Traffic Recording and Replaying utility for RavenDB
----------------------------------------------
Copyright (C) 2008 - {0} - Hibernating Rhinos
----------------------------------------------", SystemTime.UtcNow.Year);
Console.ForegroundColor = ConsoleColor.White;
Console.WriteLine(@"
Usage:
Raven.Traffic [Mode(rec/play)] [Url] [resource name] [recordingFile] [[option1], [option2] ...]
Examples:
- Record 'Northwind' database found on specified server:
Raven.Traffic rec http://localhost:8080/ Northwind recording.json
- Replay 'Northwind' database from specified server to the dump.raven file:
Raven.Traffic play http://localhost:8080/ Northwind recording.json ");
Console.ForegroundColor = ConsoleColor.Green;
TrafficToolConfiguration.InitOptionsSetObject().WriteOptionDescriptions(Console.Out);
Console.ForegroundColor = ConsoleColor.White;
}
private void ReplayRequests(TrafficToolConfiguration config, IDocumentStore store)
{
throw new NotImplementedException();
}
public TrafficRec(IDocumentStore store, TrafficToolConfiguration config)
{
_store = store;
_config = config;
}
/// <summary>
/// Connects to raven traffic event source and registers all the requests to the file defined in the config
/// </summary>
/// <param name="config">configuration conatining the connection, the file to write to, etc.</param>
/// <param name="store">the store to work with</param>
private async Task RecordRequests(TrafficToolConfiguration config, IDocumentStore store)
{
var id = Guid.NewGuid().ToString();
using (var client = new ClientWebSocket())
{
var url = store.Urls.First() + "/admin/traffic-watch";
var uri = new Uri(url.ToWebSocketPath());
await client.ConnectAsync(uri, CancellationToken.None)
.ConfigureAwait(false);
// record traffic no more then 7 days
var day = 24 * 60 * 60;
var timeout = (int)config.Timeout.TotalMilliseconds / 1000;
timeout = Math.Min(timeout, 7 * day);
if (timeout <= 0)
{
timeout = 7 * day;
}
try
{
string resourceName = config.ResourceName ?? "N/A";
var connectMessage = new DynamicJsonValue
{
["Id"] = id,
["DatabaseName"] = resourceName,
["Timeout"] = timeout
};
var stream = new MemoryStream();
JsonOperationContext context;
using (_jsonContextPool.AllocateOperationContext(out context))
using (var writer = new BlittableJsonTextWriter(context, stream))
{
context.Write(writer, connectMessage);
writer.Flush();
ArraySegment <byte> bytes;
stream.TryGetBuffer(out bytes);
await client.SendAsync(bytes, WebSocketMessageType.Text, true, CancellationToken.None)
.ConfigureAwait(false);
}
var requestsCounter = 0;
using (var fileStream = File.Create(config.RecordFilePath))
{
Stream finalStream = fileStream;
if (config.IsCompressed)
{
finalStream = new GZipStream(fileStream, CompressionMode.Compress, leaveOpen: true);
}
using (var streamWriter = new StreamWriter(finalStream))
{
var jsonWriter = new JsonTextWriter(streamWriter)
{
Formatting = Formatting.Indented
};
jsonWriter.WriteStartArray();
var sp = Stopwatch.StartNew();
while (true)
{
using (var reader = await Receive(client, context))
{
if (reader == null)
{
// server asked to close connection
break;
}
string type;
if (reader.TryGet("Type", out type))
{
if (type.Equals("Heartbeat"))
{
continue;
}
}
string error;
if (reader.TryGet("Error", out error))
{
throw new InvalidOperationException("Server returned error: " + error);
}
var notification = new TrafficWatchChange();
notification.TimeStamp = GetDateTimeFromJson(reader, "TimeStamp");
notification.RequestId = GetIntFromJson(reader, "RequestId");
notification.HttpMethod = GetStringFromJson(reader, "HttpMethod");
notification.ElapsedMilliseconds = GetIntFromJson(reader, "ElapsedMilliseconds");
notification.ResponseStatusCode = GetIntFromJson(reader, "ResponseStatusCode");
notification.TenantName = GetStringFromJson(reader, "TenantName");
notification.CustomInfo = GetStringFromJson(reader, "CustomInfo");
notification.InnerRequestsCount = GetIntFromJson(reader, "InnerRequestsCount");
// notification.QueryTimings = GetRavenJObjectFromJson(reader, "QueryTimings"); // TODO (TrafficWatch) : Handle this both server and client sides
if (config.PrintOutput)
{
Console.Write("\rRequest #{0} Stored...\t\t ", ++requestsCounter);
}
var jobj = JObject.FromObject(notification);
jobj.WriteTo(jsonWriter);
if (sp.ElapsedMilliseconds > 5000)
{
streamWriter.Flush();
sp.Restart();
}
}
}
jsonWriter.WriteEndArray();
streamWriter.Flush();
if (config.IsCompressed)
{
finalStream.Dispose();
}
}
}
}
catch (Exception ex)
{
Console.WriteLine("\r\n\nError while reading messages from server : " + ex);
}
finally
{
Console.WriteLine("\r\n\nClosing connection to server...`");
try
{
await
client.CloseAsync(WebSocketCloseStatus.NormalClosure, "CLOSE_NORMAL", CancellationToken.None)
.ConfigureAwait(false);
}
catch
{
// ignored
}
}
}
}
private void ReplayRequests(TrafficToolConfiguration config, IDocumentStore store)
{
Stream finalStream;
var requestsCounter = 0;
var skippedRequestsCounter = 0;
var totalCountOfLogicRequests = 0;
var totalSp = Stopwatch.StartNew();
using (var stream = File.Open(config.RecordFilePath, FileMode.Open))
{
if (config.IsCompressed)
{
finalStream = new GZipStream(stream, CompressionMode.Decompress, leaveOpen: true);
}
else
{
finalStream = stream;
}
var trafficLogs =
JsonSerializer.Create().Deserialize <LogHttpRequestStatsParams[]>(new JsonTextReader(new StreamReader(finalStream)));
ConcurrentQueue <string> queue = null;
var cts = new CancellationTokenSource();
var ct = cts.Token;
Task outputTask = null;
if (config.PrintOutput)
{
queue = new ConcurrentQueue <string>();
outputTask = Task.Run(() =>
{
while (!ct.IsCancellationRequested || queue.Count != 0)
{
string message;
if (queue.TryDequeue(out message))
{
Console.WriteLine(message);
}
else
{
Thread.Sleep(10);
}
}
});
}
const string postLineSeparatorRegex = "\\t\\d: databases\\/[\\w\\.]+";
const string endOfPostLineString = "\t\t\tQuery:";
const string uriCleanRegex = "http://[\\w\\.-]+(:\\d*)?(\\/databases\\/[\\w\\.]+)?";
Parallel.ForEach(trafficLogs, new ParallelOptions
{
MaxDegreeOfParallelism = 60
}, trafficLog =>
{
var sp = Stopwatch.StartNew();
GetRequest[] requestsArray = null;
string uriString = Regex.Replace(trafficLog.RequestUri, uriCleanRegex, string.Empty);
string trafficQueryPart;
var trafficUrlPart = ExtractUrlAndQuery(uriString, out trafficQueryPart);
var curCount = Interlocked.Increment(ref requestsCounter);
if (ValidateUrlString(trafficUrlPart))
{
Interlocked.Increment(ref skippedRequestsCounter);
if (queue != null)
{
queue.Enqueue(string.Format("{0} out of {1}, skipped whole message",
curCount, trafficLogs.Length));
}
return;
}
Interlocked.Increment(ref totalCountOfLogicRequests);
if (trafficLog.HttpMethod.Equals("get", StringComparison.CurrentCultureIgnoreCase))
{
requestsArray = new[]
{
new GetRequest
{
Url = trafficUrlPart,
Query = trafficQueryPart
}
};
}
else if (trafficLog.CustomInfo != null)
{
var subArray = Regex.Split(trafficLog.CustomInfo.Replace("\r", string.Empty), postLineSeparatorRegex).Where(x => !String.IsNullOrEmpty(x)).Select(x =>
{
var endOfPostLastIndex = x.IndexOf(endOfPostLineString);
if (endOfPostLastIndex < 0)
{
return(x);
}
return(x.Remove(endOfPostLastIndex));
}).ToArray();
requestsArray =
subArray.Select(customInfoLine =>
{
trafficUrlPart = ExtractUrlAndQuery(customInfoLine, out trafficQueryPart);
if (ValidateUrlString(trafficUrlPart))
{
if (queue != null)
{
queue.Enqueue(string.Format("{0} out of {1}, skipped inner message",
curCount, trafficLogs.Length));
}
return(null);
}
return(new GetRequest
{
Url = trafficUrlPart,
Query = trafficQueryPart,
});
}).Where(x => x != null).ToArray();
}
Interlocked.Add(ref totalCountOfLogicRequests, requestsArray?.Length ?? 0);
if (requestsArray == null || requestsArray.Length == 0)
{
Interlocked.Increment(ref skippedRequestsCounter);
if (queue != null)
{
queue.Enqueue(string.Format("{0} out of {1}, skipped",
curCount, trafficLogs.Length, sp.ElapsedMilliseconds, totalSp.ElapsedMilliseconds));
}
return;
}
try
{
store.DatabaseCommands.MultiGet(requestsArray);
if (queue != null)
{
queue.Enqueue(string.Format("{0} out of {1}, took {2} ms. Total Time: {3} ms",
curCount, trafficLogs.Length, sp.ElapsedMilliseconds, totalSp.ElapsedMilliseconds));
}
}
catch (Exception e)
{
Interlocked.Increment(ref skippedRequestsCounter);
if (queue != null)
{
queue.Enqueue(string.Format("{0} out of {1}, failed, took {2} ms. Total Time: {3} ms",
curCount, trafficLogs.Length, sp.ElapsedMilliseconds, totalSp.ElapsedMilliseconds));
}
}
});
if (outputTask != null)
{
cts.Cancel();
outputTask.Wait();
}
}
Console.WriteLine(@"Summary:
Requests sent: {0}
Requests skipped: {1}
Nested and non nested request: {2}
Total Time: {3}", requestsCounter, skippedRequestsCounter, totalCountOfLogicRequests, totalSp.ElapsedMilliseconds);
}
public TrafficRec(IDocumentStore store, TrafficToolConfiguration config)
{
this.store = store;
this.config = config;
}
private static void RunCommand(CommandLineApplication cmd, TrafficToolConfiguration.TrafficToolMode mode)
{
cmd.HelpOption(HelpOptionString);
var urlArg = cmd.Argument("URL", "RavenDB Server URL");
var databaseArg = cmd.Argument("Database", "Database name");
var recordFilePathArg = cmd.Argument("RecordFilePath", "Record file path");
var durationConstraintArg = cmd.Option("--trace-seconds", "Time to perform the traffic watch(seconds)", CommandOptionType.SingleValue);
var amountConstraintArg = cmd.Option("--trace-requests", "Time to perform the traffic watch", CommandOptionType.SingleValue);
var compressedArg = cmd.Option("--compressed", "Work with compressed json outpu/input", CommandOptionType.NoValue);
var noOutputArg = cmd.Option("--no-output", "Suppress console progress output", CommandOptionType.NoValue);
var timeoutArg = cmd.Option("--timeout", "The timeout to use for requests(seconds)", CommandOptionType.SingleValue);
cmd.OnExecute(() =>
{
var url = urlArg.Value;
try
{
// ReSharper disable once ObjectCreationAsStatement
new Uri(url);
}
catch (UriFormatException e)
{
return(ExitWithError($"Server's url provided isn't in valid format. {e}", cmd));
}
IDocumentStore store;
try
{
store = new DocumentStore
{
Urls = new[] { url },
Database = databaseArg.Value
}.Initialize();
}
catch (Exception e)
{
return(ExitWithError($"Could not connect to server. Exception: {e}", cmd));
}
using (store)
{
try
{
store.Maintenance.Send(new GetStatisticsOperation());
}
catch (Exception)
{
return(ExitWithError("Database does not exist.", cmd));
}
var configuration = new TrafficToolConfiguration
{
Database = databaseArg.Value,
RecordFilePath = recordFilePathArg.Value,
Mode = mode,
DurationConstraint = TimeSpan.FromSeconds(durationConstraintArg.HasValue()
? int.Parse(durationConstraintArg.Value())
: 60),
AmountConstraint = amountConstraintArg.HasValue()
? int.Parse(amountConstraintArg.Value())
: 1000,
};
if (timeoutArg.HasValue())
{
configuration.Timeout = TimeSpan.FromSeconds(int.Parse(timeoutArg.Value()));
}
if (compressedArg.HasValue())
{
configuration.IsCompressed = true;
}
if (noOutputArg.HasValue())
{
configuration.PrintOutput = false;
}
new TrafficRec(store, configuration).ExecuteTrafficCommand();
}
return(0);
});
}
