private void HandleReceivingResponseLine(string line)
{
byte[] bytes;
bytes = ReadNextBytes(line, _receivingResponseRegex);
if (bytes != null)
{
if (_currentRequestData == null || _currentHeader == null)
{
//this is a situation where we have a response without a request
return;
}
_currentRequestData.AddToResponse(bytes);
if (String.IsNullOrEmpty(_currentHeader.ResponseStatus))
{
//check if we have the full response line
MemoryStream ms = new MemoryStream(_currentRequestData.RawResponse);
byte[] respLineBytes = Utils.ReadLine(ms, ESTIMATED_LINE_SIZE);
string respLine = Utils.ByteToString(respLineBytes);
if (_lineTypeSelector.GetLineType(respLine) == LineType.FirstResponseLine)
{
HandleFirstResponseLine(respLine, null);
}
}
}
}
public static void Main(string [] args) {
Interpreter.Console = new TextConsole();
Utils.Write(caption+"\n"+prompt);
Interpreter interp = new Interpreter();
string defs = args.Length > 0 ? args[0] : interp.DefaultIncludeFile();
interp.ReadIncludeFile(defs);
while (interp.ProcessLine(Utils.ReadLine()))
Utils.Write(interp.BlockLevel > 0 ? block_prompt : prompt);
}
public void TestReadLineEmptyLineFollowedByText()
{
byte[] bytes = Encoding.UTF8.GetBytes("\r\nline2");
MemoryStream stream = new MemoryStream(bytes);
byte[] line = Utils.ReadLine(stream, 1024);
Assert.IsNotNull(line);
Assert.AreEqual(0, line.Length);
}
public void TestReadLineLastLine()
{
byte[] bytes = Encoding.UTF8.GetBytes("\r\nline2");
MemoryStream stream = new MemoryStream(bytes);
Utils.ReadLine(stream, 1024);
byte[] line = Utils.ReadLine(stream, 1024);
Assert.IsNotNull(line);
Assert.AreEqual("line2", Encoding.UTF8.GetString(line));
}
/// <summary>
/// Optimized method to obtain the request line from a raw request bytes
/// </summary>
/// <returns></returns>
public static string GetRequestLine(byte[] rawRequest)
{
string reqLine = String.Empty;
MemoryStream stream = new MemoryStream(rawRequest);
byte [] reqLineBytes = Utils.ReadLine(stream, ESTIMATED_LINE_SIZE, LineEnding.Any);
if (reqLineBytes != null)
{
reqLine = Constants.DefaultEncoding.GetString(reqLineBytes);
}
return(reqLine);
}
public static void XMain(string [] args)
{
Interpreter.Console = new TextConsole();
Utils.Write(caption + "\n" + prompt);
Interpreter interp = new Interpreter();
string defs = args.Length > 0 ? args[0] : "csi.csi";
interp.ReadIncludeFile(defs);
while (interp.ProcessLine(Utils.ReadLine()))
{
Utils.Write(prompt);
}
}
/// <summary>
/// Searches in the specified context
/// </summary>
/// <param name="dataSource"></param>
/// <param name="header"></param>
/// <param name="criteria"></param>
/// <param name="context"></param>
/// <param name="matches"></param>
/// <returns></returns>
private bool SearchInContext(
ITrafficDataAccessor dataSource,
TVRequestInfo header,
ISearchCriteria criteria,
SearchContext context,
ref LineMatches matches)
{
bool found;
MemoryStream toBeSearched;
List <MatchCoordinates> matchCoordinates;
toBeSearched = GetMemoryStream(dataSource, header, context);
byte[] lineBytes;
found = false;
bool searchLine = criteria.Context != SearchContext.RequestBody;
int relativePosition = 0;
while ((lineBytes = Utils.ReadLine(toBeSearched, ESTIMATED_LINE_LEN, LineEnding.Any)) != null)
{
string line = Constants.DefaultEncoding.GetString(lineBytes);
if (searchLine)
{
if (criteria.Matches(line, relativePosition, out matchCoordinates))
{
found = true;
matches.Add(new LineMatch(header.Id, line, matchCoordinates, context));
}
}
else
{
if (line == String.Empty) //if a \r\n was hit in the request we're in the req body
{
searchLine = true;
}
}
//update the relative position with the current stream position, before reading a new line
relativePosition = (int)toBeSearched.Position;
}
return(found);
}
/// <summary>
/// Reads a chunked response
/// </summary>
/// <param name="ms"></param>
private void ProcessChunkedResponse(MemoryStream ms)
{
byte[] lineBytes;
string line;
do
{
lineBytes = Utils.ReadLine(ms, ESTIMATED_LINE_SIZE, LineEnding.Any);
if (lineBytes != null)
{
line = Constants.DefaultEncoding.GetString(lineBytes);
int scIndex = line.IndexOf(';');
if (scIndex > -1)
{
line = line.Substring(0, scIndex);
}
int chunkSize;
if (line != String.Empty && Uri.IsHexDigit(line[0]) && Int32.TryParse(line, NumberStyles.HexNumber, null, out chunkSize))
{
//read the specified chunk of bytes
byte[] chunk = new byte[chunkSize];
ms.Read(chunk, 0, (int)chunkSize);
//add the chunk to the body
_responseBody.AddChunkReference(chunk, chunkSize);
//advance one line after that
lineBytes = Utils.ReadLine(ms, ESTIMATED_LINE_SIZE, LineEnding.Any);
//the line read should be empty
if (lineBytes != null && lineBytes.Length > 0)
{
throw new Exception("Invalid chunk");
}
}
else
{
throw new Exception("Invalid chunk size");
}
}
}while (lineBytes != null);
}
/// <summary>
/// Parses the request
/// </summary>
/// <param name="requestBytes"></param>
/// <param name="parseVariables">Whether to parse parameters for the request</param>
private void ParseRequest(byte [] requestBytes, bool parseVariables)
{
MemoryStream ms = new MemoryStream(requestBytes);
byte [] lineBytes = Utils.ReadLine(ms, ESTIMATED_LINE_SIZE, LineEnding.Any);
string requestLine = Constants.DefaultEncoding.GetString(lineBytes);
if (!InitMethod(requestLine))
{
//the request line is not long enough to read the method
return;
}
//extract version
int lastSpace = requestLine.LastIndexOf(' ');
if (lastSpace > -1)
{
_httpVersion = requestLine.Substring(lastSpace + 1);
}
//extract query string
int firstQuestionMark = requestLine.LastIndexOf('?');
if (firstQuestionMark > -1)
{
_queryString = requestLine.Substring(firstQuestionMark + 1);
int endOfQuery = _queryString.LastIndexOf(' ');
if (endOfQuery > -1)
{
_queryString = _queryString.Substring(0, endOfQuery);
}
}
//iterate through the lines as long as there is no empty line which would separate post data
string line;
do
{
lineBytes = Utils.ReadLine(ms, ESTIMATED_LINE_SIZE, LineEnding.Any);
if (lineBytes != null && lineBytes.Length > 0)
{
line = Constants.DefaultEncoding.GetString(lineBytes);
string [] nameAndValue = line.Split(new char[] { ':' }, 3);
if (nameAndValue.Length == 2)
{
_headers.Add(nameAndValue[0], nameAndValue[1].Trim());
}
else if (nameAndValue.Length == 3)
{
if (String.IsNullOrWhiteSpace(nameAndValue[0]))
{
_headers.Add(String.Join(":", nameAndValue[0], nameAndValue[1]), nameAndValue[2].Trim());
}
else
{
_headers.Add(nameAndValue[0], String.Join(":", nameAndValue[1], nameAndValue[2]).Trim());
}
}
}
}while (lineBytes != null && lineBytes.Length > 0);
//initialize encoding
string contentTypeValue = _headers["Content-Type"];
if (contentTypeValue != null)
{
_contentEncoding = HttpUtil.GetEncoding(contentTypeValue);
}
if (lineBytes != null) //we read an empty line signaling that the headers are over
{
_isFullRequest = true;
string contentLenHeaderString = _headers["Content-Length"];
if (contentLenHeaderString != null || IsPost || IsPut)
{
int contentLenHeaderVal = 0;
int.TryParse(contentLenHeaderString, out contentLenHeaderVal);
long pos = ms.Position;
int contentLength = (int)(ms.Length - pos);
if (contentLenHeaderVal > contentLength)
{
_isFullRequest = false;
}
_contentData = new byte[contentLength];
ms.Read(_contentData, 0, contentLength);
}
}
else
{
_isFullRequest = false; //we didn't finish reading the headers
}
if (_headers["Host"] != null)
{
PopulateHostAndPort(_headers["Host"]);
}
else if (_headers[":authority"] != null)
{
PopulateHostAndPort(_headers[":authority"]);
}
int indexOfPath = _method.Length + 1;
string temp = requestLine.Substring(indexOfPath);
//remove the query and the http version
if (_httpVersion != String.Empty)
{
temp = temp.Replace(_httpVersion, String.Empty);
}
if (_queryString != String.Empty)
{
temp = temp.Replace(String.Format("?{0} ", _queryString), String.Empty);
}
string protocol = String.Empty;
//remove the protocol://host
if (temp.IndexOf("https://", StringComparison.OrdinalIgnoreCase) == 0)
{
_isSecure = true;
protocol = "https://";
}
else if (temp.IndexOf("http://", StringComparison.OrdinalIgnoreCase) == 0)
{
_isProxyHttp = true;
_isSecure = false;
protocol = "http://";
}
if (protocol != String.Empty) //this is a proxy request
{
// skip the protocol string and process whatever comes after that (hostAndPort, and then Path)
// we already checked that temp starts with protocol and protocol is not empty
temp = temp.Substring(protocol.Length);
//next extract host and port
//find the first fw slash
int indexOfFws = temp.IndexOf('/');
string hostAndPort = temp.Substring(0, indexOfFws);
PopulateHostAndPort(hostAndPort);
temp = temp.Substring(indexOfFws);
}
_path = temp.TrimEnd(' ');
//construct a search regex
string _regexPath = Regex.Escape(_path);
if (parseVariables)
{
ParseVariables();
}
//in the case of custom parameters in the path make sure to remove from the regex
if (_pathVariables.Count > 0)
{
_regexPath = NormalizePath(_regexPath, ".+");
}
_searchRegex = String.Format("{0} ({1}[^/]+)?{2}[\\s|\\?]", _method, _isSecure ? "https://" : "http://", _regexPath);
//replace any dynamic elements
_searchRegex = _searchRegex.Replace(Constants.DYNAMIC_ELEM_STRING, ".+");
_path = _path.TrimEnd(' ', '?');
}
/// <summary>
/// This is the method resposible for parsing the raw traffic log
/// </summary>
private void ParseFunction()
{
_parserStatus = TrafficParserStatus.Running;
byte[] bytes; //will store a line of bytes
string line; //will store the converted bytes to string
LineType lineType;
_thisSessionRequestCount = 0;
do
{
try
{
//read a line and handle end of file or stop requested
bytes = Utils.ReadLine(_rawLog, ESTIMATED_LINE_SIZE);
line = Utils.ByteToString(bytes);
lineType = _lineTypeSelector.GetLineType(line);
//if the end of file was reached or the user stopped the parse
if (lineType == LineType.EndOfFile || _stopRequested)
{
HandleHalt();
return;
}
//according to the line type perform the coresponding action
switch (lineType)
{
case LineType.BeginThread:
HandleBeginThread(line);
break;
case LineType.EndThread:
HandleEndThread();
break;
case LineType.FirstRequestLine:
HandleFirstRequestLine(line, bytes);
break;
case LineType.FirstResponseLine:
HandleFirstResponseLine(line, bytes);
break;
case LineType.HttpTraffic:
HandleHttpTraffic(line, bytes);
break;
case LineType.ResponseReceived:
HandleResponseReceivedMessage();
break;
case LineType.SendingRequest:
HandleSendingRequestLine(line);
break;
case LineType.ReceivingResponse:
HandleReceivingResponseLine(line);
break;
}
//handle tail (if is enabled _tailChunk > 0)
if (_tailChunk > 0 && _thisSessionRequestCount >= _tailChunk)
{
_stopRequested = true;
}
//extract custom fields
HandleCustomFields(line);
}
catch (OutOfMemoryException)
{
SdkSettings.Instance.Logger.Log(TraceLevel.Error, "Out of memory exception occured during parsing {0}. Calling the garbage collector.", _rawLog.Name);
GC.Collect();
}
catch (Exception ex)
{
SdkSettings.Instance.Logger.Log(TraceLevel.Error, "Error parsing {0}:{1}", _rawLog.Name, ex.Message);
}
}while (true);
}
/// <summary>
/// Parses the raw response
/// </summary>
/// <param name="fullResponse"></param>
public void ProcessResponse(byte[] fullResponse)
{
//reset all properties
_statusLine = String.Empty;
_headers = new HTTPHeaders();
_responseBody = null;
byte[] lineBytes;
string line;
_responseBody = new HttpResponseBody();
MemoryStream ms = new MemoryStream(fullResponse);
while (true)
{
lineBytes = Utils.ReadLine(ms, ESTIMATED_LINE_SIZE, LineEnding.Any);
if (lineBytes != null)
{
line = Constants.DefaultEncoding.GetString(lineBytes);
}
else
{
return; //end of response
}
if (line == String.Empty)
{
if (_status != 100)
{
break; //end of headers
}
else
{
//the response status was 100 Continue expect a new set of headers and a response line
//clear the current information
_headers = new HTTPHeaders();
_statusLine = String.Empty;
continue;
}
}
if (_statusLine == String.Empty)
{
_statusLine = line;
int j = STATUS_START, count = _statusLine.Length;
while (j < count && !Char.IsDigit(_statusLine[j]))
{
j++; //for some data coming from logs or files the file has some additional characters
}
StringBuilder statusBuilder = new StringBuilder();
//append the digits
while (j < count && Char.IsDigit(_statusLine[j]))
{
statusBuilder.Append(_statusLine[j]); j++;
}
int.TryParse(statusBuilder.ToString(), out _status);
}
else
{
//this is a header split it
string[] header = line.Split(new char[1] {
':'
}, 2);
if (header.Length > 1)
{
_headers.Add(header[0], header[1]);
}
}
}
long contentLength = ms.Length - ms.Position;
//find out if the response is chunked
List <HTTPHeader> transferEncoding = _headers.GetHeaders("Transfer-Encoding");
if (transferEncoding.Count > 0 && String.Compare(transferEncoding[0].Values[0], "chunked", true) == 0)
{
_responseBody.IsChunked = true;
//save the ms position in case something happens
long msPos = ms.Position;
try
{
ProcessChunkedResponse(ms);
}
catch
{
//reset the ms position to what it was before the process chunk failed
ms.Position = msPos;
ProcessChunkedResponseHeuristically(ms);
}
//finish by setting the content length to the total size of the chunks
contentLength = _responseBody.Length;
}
else
{
if (contentLength > 0)
{
//add one big chunk with the whole response
_responseBody.IsChunked = false;
byte[] fullArray = new byte[contentLength];
ms.Read(fullArray, 0, (int)contentLength);
_responseBody.AddChunkReference(fullArray, fullArray.Length);
}
//do not update the content length header with the current content length
//like this _headers["Content-Length"] = _contentLength.ToString();
//we don't know if the response body size will change
}
}
/// <summary>
/// Processes a chunked response by determining a chunk size line based on the line format and skipping those lines
/// This applies to responses that do not contain correct chunk sizes
/// </summary>
/// <param name="ms"></param>
private void ProcessChunkedResponseHeuristically(MemoryStream ms)
{
_responseBody.ClearChunks();
byte[] lineBytes = new byte[0];
string line;
long posBefRdLn;
long startChunkPos = ms.Position;
long chunkLen = 0;
while (true)
{
//save the position so we can revert to reading the full chunk
posBefRdLn = ms.Position;
lineBytes = Utils.ReadLine(ms, ESTIMATED_LINE_SIZE, LineEnding.Any);
if (lineBytes != null)
{
line = Constants.DefaultEncoding.GetString(lineBytes);
}
else
{
if (_responseBody.Length < ms.Position - startChunkPos) //end of stream but there are remaining bytes that were not added
{
ms.Position = startChunkPos;
//read the remaining response
byte[] chunk = new byte[chunkLen];
ms.Read(chunk, 0, (int)chunkLen);
//add the chunk to the response
_responseBody.AddChunkReference(chunk, (int)chunkLen);
}
break; //end of response
}
int scIndex = line.IndexOf(';');
if (scIndex > -1)
{
line = line.Substring(0, scIndex);
}
//this is done this way because some traffic files are malformed and
//do not correctly display the chunk sizes
int chunkSize;
if (line != String.Empty && Uri.IsHexDigit(line[0]) && Int32.TryParse(line, NumberStyles.HexNumber, null, out chunkSize))
{
if (chunkLen > 0)
{
long currPos = ms.Position;
ms.Position = startChunkPos;
byte[] chunk = new byte[chunkLen];
ms.Read(chunk, 0, (int)chunkLen);
//revert back to the last position
ms.Position = currPos;
//add the chunk to the response
_responseBody.AddChunkReference(chunk, (int)chunkLen);
}
//save the current position as the beggining of a new chunk
startChunkPos = ms.Position;
}
else
{
//calculate the current chunk size
//...which is the length of the current line
//plus the position in stream before reading the current line
//minus the position in stream when the last chunk header was read
chunkLen = posBefRdLn + lineBytes.Length - startChunkPos;
}
}
}
/// <summary>
/// Find all the entries in the specified range of the given timestamp
/// </summary>
/// <param name="timeStamp"></param>
/// <param name="range">Range in seconds</param>
/// <returns>List of timestamp snapshots in byte array format (as they were read from the stream)</returns>
public List <ByteArrayBuilder> Find(DateTime timeStamp, int range)
{
long rangeTicks = range * 1000 * 10000; //1ms = 10000ticks
DateTime startStamp = new DateTime(timeStamp.Ticks - rangeTicks);
DateTime endStamp = new DateTime(timeStamp.Ticks + rangeTicks);
_currentLog.Position = _cache.FindBestEntry(startStamp);
DateTime cacheEntry = default(DateTime);
byte[] bytes;
List <ByteArrayBuilder> result = new List <ByteArrayBuilder>();
ByteArrayBuilder snapShot = null;
DateTime currExtractedTime = startStamp; //last time extracted from the line read
long lastPos = _currentLog.Position; //used to keep track of the log position before read
while ((bytes = Utils.ReadLine(_currentLog, ESTIMATED_LINE_SIZE)) != null)
{
DateTime time = GetTimeStamp(bytes, timeStamp);
if (time != default(DateTime))
{
if (time.CompareTo(endStamp) <= 0)
{
//if we are inside the range
if (time.CompareTo(startStamp) >= 0)
{
//if the current search was not cached
if (cacheEntry == default(DateTime))
{
//we save the first match to the cache
_cache.Add(currExtractedTime, lastPos);
cacheEntry = currExtractedTime;
}
currExtractedTime = time;
if (snapShot != null)
{
//save the current snapshot
result.Add(snapShot);
}
snapShot = new ByteArrayBuilder();
}
}
else
{
//the current extracted time is higher than the upper range limit
if (result.Count > 0 || (snapShot != null && snapShot.Length > 0))
{
//if we have already extracted snapshots then we stop here
break;
//otherwise we keep looking;
}
}
}
//save the last read position
lastPos = _currentLog.Position;
//here we add bytes we read to the current snapshot
if (snapShot != null)
{
snapShot.AddChunkReference(bytes, bytes.Length);
snapShot.AddChunkReference(Constants.NewLineBytes, Constants.NewLineBytes.Length);
}
} //end read loop
//check if there is anything left to add
if (snapShot != null && snapShot.Length > 0)
{
result.Add(snapShot);
}
return(result);
}
