private void HandleReceivingResponseLine(string line) { byte[] bytes; bytes = ReadNextBytes(line, _receivingResponseRegex); if (bytes != null) { if (_currentRequestData == null || _currentHeader == null) { //this is a situation where we have a response without a request return; } _currentRequestData.AddToResponse(bytes); if (String.IsNullOrEmpty(_currentHeader.ResponseStatus)) { //check if we have the full response line MemoryStream ms = new MemoryStream(_currentRequestData.RawResponse); byte[] respLineBytes = Utils.ReadLine(ms, ESTIMATED_LINE_SIZE); string respLine = Utils.ByteToString(respLineBytes); if (_lineTypeSelector.GetLineType(respLine) == LineType.FirstResponseLine) { HandleFirstResponseLine(respLine, null); } } } }
public static void Main(string [] args) { Interpreter.Console = new TextConsole(); Utils.Write(caption+"\n"+prompt); Interpreter interp = new Interpreter(); string defs = args.Length > 0 ? args[0] : interp.DefaultIncludeFile(); interp.ReadIncludeFile(defs); while (interp.ProcessLine(Utils.ReadLine())) Utils.Write(interp.BlockLevel > 0 ? block_prompt : prompt); }
public void TestReadLineEmptyLineFollowedByText() { byte[] bytes = Encoding.UTF8.GetBytes("\r\nline2"); MemoryStream stream = new MemoryStream(bytes); byte[] line = Utils.ReadLine(stream, 1024); Assert.IsNotNull(line); Assert.AreEqual(0, line.Length); }
public void TestReadLineLastLine() { byte[] bytes = Encoding.UTF8.GetBytes("\r\nline2"); MemoryStream stream = new MemoryStream(bytes); Utils.ReadLine(stream, 1024); byte[] line = Utils.ReadLine(stream, 1024); Assert.IsNotNull(line); Assert.AreEqual("line2", Encoding.UTF8.GetString(line)); }
/// <summary> /// Optimized method to obtain the request line from a raw request bytes /// </summary> /// <returns></returns> public static string GetRequestLine(byte[] rawRequest) { string reqLine = String.Empty; MemoryStream stream = new MemoryStream(rawRequest); byte [] reqLineBytes = Utils.ReadLine(stream, ESTIMATED_LINE_SIZE, LineEnding.Any); if (reqLineBytes != null) { reqLine = Constants.DefaultEncoding.GetString(reqLineBytes); } return(reqLine); }
public static void XMain(string [] args) { Interpreter.Console = new TextConsole(); Utils.Write(caption + "\n" + prompt); Interpreter interp = new Interpreter(); string defs = args.Length > 0 ? args[0] : "csi.csi"; interp.ReadIncludeFile(defs); while (interp.ProcessLine(Utils.ReadLine())) { Utils.Write(prompt); } }
/// <summary> /// Searches in the specified context /// </summary> /// <param name="dataSource"></param> /// <param name="header"></param> /// <param name="criteria"></param> /// <param name="context"></param> /// <param name="matches"></param> /// <returns></returns> private bool SearchInContext( ITrafficDataAccessor dataSource, TVRequestInfo header, ISearchCriteria criteria, SearchContext context, ref LineMatches matches) { bool found; MemoryStream toBeSearched; List <MatchCoordinates> matchCoordinates; toBeSearched = GetMemoryStream(dataSource, header, context); byte[] lineBytes; found = false; bool searchLine = criteria.Context != SearchContext.RequestBody; int relativePosition = 0; while ((lineBytes = Utils.ReadLine(toBeSearched, ESTIMATED_LINE_LEN, LineEnding.Any)) != null) { string line = Constants.DefaultEncoding.GetString(lineBytes); if (searchLine) { if (criteria.Matches(line, relativePosition, out matchCoordinates)) { found = true; matches.Add(new LineMatch(header.Id, line, matchCoordinates, context)); } } else { if (line == String.Empty) //if a \r\n was hit in the request we're in the req body { searchLine = true; } } //update the relative position with the current stream position, before reading a new line relativePosition = (int)toBeSearched.Position; } return(found); }
/// <summary> /// Reads a chunked response /// </summary> /// <param name="ms"></param> private void ProcessChunkedResponse(MemoryStream ms) { byte[] lineBytes; string line; do { lineBytes = Utils.ReadLine(ms, ESTIMATED_LINE_SIZE, LineEnding.Any); if (lineBytes != null) { line = Constants.DefaultEncoding.GetString(lineBytes); int scIndex = line.IndexOf(';'); if (scIndex > -1) { line = line.Substring(0, scIndex); } int chunkSize; if (line != String.Empty && Uri.IsHexDigit(line[0]) && Int32.TryParse(line, NumberStyles.HexNumber, null, out chunkSize)) { //read the specified chunk of bytes byte[] chunk = new byte[chunkSize]; ms.Read(chunk, 0, (int)chunkSize); //add the chunk to the body _responseBody.AddChunkReference(chunk, chunkSize); //advance one line after that lineBytes = Utils.ReadLine(ms, ESTIMATED_LINE_SIZE, LineEnding.Any); //the line read should be empty if (lineBytes != null && lineBytes.Length > 0) { throw new Exception("Invalid chunk"); } } else { throw new Exception("Invalid chunk size"); } } }while (lineBytes != null); }
/// <summary> /// Parses the request /// </summary> /// <param name="requestBytes"></param> /// <param name="parseVariables">Whether to parse parameters for the request</param> private void ParseRequest(byte [] requestBytes, bool parseVariables) { MemoryStream ms = new MemoryStream(requestBytes); byte [] lineBytes = Utils.ReadLine(ms, ESTIMATED_LINE_SIZE, LineEnding.Any); string requestLine = Constants.DefaultEncoding.GetString(lineBytes); if (!InitMethod(requestLine)) { //the request line is not long enough to read the method return; } //extract version int lastSpace = requestLine.LastIndexOf(' '); if (lastSpace > -1) { _httpVersion = requestLine.Substring(lastSpace + 1); } //extract query string int firstQuestionMark = requestLine.LastIndexOf('?'); if (firstQuestionMark > -1) { _queryString = requestLine.Substring(firstQuestionMark + 1); int endOfQuery = _queryString.LastIndexOf(' '); if (endOfQuery > -1) { _queryString = _queryString.Substring(0, endOfQuery); } } //iterate through the lines as long as there is no empty line which would separate post data string line; do { lineBytes = Utils.ReadLine(ms, ESTIMATED_LINE_SIZE, LineEnding.Any); if (lineBytes != null && lineBytes.Length > 0) { line = Constants.DefaultEncoding.GetString(lineBytes); string [] nameAndValue = line.Split(new char[] { ':' }, 3); if (nameAndValue.Length == 2) { _headers.Add(nameAndValue[0], nameAndValue[1].Trim()); } else if (nameAndValue.Length == 3) { if (String.IsNullOrWhiteSpace(nameAndValue[0])) { _headers.Add(String.Join(":", nameAndValue[0], nameAndValue[1]), nameAndValue[2].Trim()); } else { _headers.Add(nameAndValue[0], String.Join(":", nameAndValue[1], nameAndValue[2]).Trim()); } } } }while (lineBytes != null && lineBytes.Length > 0); //initialize encoding string contentTypeValue = _headers["Content-Type"]; if (contentTypeValue != null) { _contentEncoding = HttpUtil.GetEncoding(contentTypeValue); } if (lineBytes != null) //we read an empty line signaling that the headers are over { _isFullRequest = true; string contentLenHeaderString = _headers["Content-Length"]; if (contentLenHeaderString != null || IsPost || IsPut) { int contentLenHeaderVal = 0; int.TryParse(contentLenHeaderString, out contentLenHeaderVal); long pos = ms.Position; int contentLength = (int)(ms.Length - pos); if (contentLenHeaderVal > contentLength) { _isFullRequest = false; } _contentData = new byte[contentLength]; ms.Read(_contentData, 0, contentLength); } } else { _isFullRequest = false; //we didn't finish reading the headers } if (_headers["Host"] != null) { PopulateHostAndPort(_headers["Host"]); } else if (_headers[":authority"] != null) { PopulateHostAndPort(_headers[":authority"]); } int indexOfPath = _method.Length + 1; string temp = requestLine.Substring(indexOfPath); //remove the query and the http version if (_httpVersion != String.Empty) { temp = temp.Replace(_httpVersion, String.Empty); } if (_queryString != String.Empty) { temp = temp.Replace(String.Format("?{0} ", _queryString), String.Empty); } string protocol = String.Empty; //remove the protocol://host if (temp.IndexOf("https://", StringComparison.OrdinalIgnoreCase) == 0) { _isSecure = true; protocol = "https://"; } else if (temp.IndexOf("http://", StringComparison.OrdinalIgnoreCase) == 0) { _isProxyHttp = true; _isSecure = false; protocol = "http://"; } if (protocol != String.Empty) //this is a proxy request { // skip the protocol string and process whatever comes after that (hostAndPort, and then Path) // we already checked that temp starts with protocol and protocol is not empty temp = temp.Substring(protocol.Length); //next extract host and port //find the first fw slash int indexOfFws = temp.IndexOf('/'); string hostAndPort = temp.Substring(0, indexOfFws); PopulateHostAndPort(hostAndPort); temp = temp.Substring(indexOfFws); } _path = temp.TrimEnd(' '); //construct a search regex string _regexPath = Regex.Escape(_path); if (parseVariables) { ParseVariables(); } //in the case of custom parameters in the path make sure to remove from the regex if (_pathVariables.Count > 0) { _regexPath = NormalizePath(_regexPath, ".+"); } _searchRegex = String.Format("{0} ({1}[^/]+)?{2}[\\s|\\?]", _method, _isSecure ? "https://" : "http://", _regexPath); //replace any dynamic elements _searchRegex = _searchRegex.Replace(Constants.DYNAMIC_ELEM_STRING, ".+"); _path = _path.TrimEnd(' ', '?'); }
/// <summary> /// This is the method resposible for parsing the raw traffic log /// </summary> private void ParseFunction() { _parserStatus = TrafficParserStatus.Running; byte[] bytes; //will store a line of bytes string line; //will store the converted bytes to string LineType lineType; _thisSessionRequestCount = 0; do { try { //read a line and handle end of file or stop requested bytes = Utils.ReadLine(_rawLog, ESTIMATED_LINE_SIZE); line = Utils.ByteToString(bytes); lineType = _lineTypeSelector.GetLineType(line); //if the end of file was reached or the user stopped the parse if (lineType == LineType.EndOfFile || _stopRequested) { HandleHalt(); return; } //according to the line type perform the coresponding action switch (lineType) { case LineType.BeginThread: HandleBeginThread(line); break; case LineType.EndThread: HandleEndThread(); break; case LineType.FirstRequestLine: HandleFirstRequestLine(line, bytes); break; case LineType.FirstResponseLine: HandleFirstResponseLine(line, bytes); break; case LineType.HttpTraffic: HandleHttpTraffic(line, bytes); break; case LineType.ResponseReceived: HandleResponseReceivedMessage(); break; case LineType.SendingRequest: HandleSendingRequestLine(line); break; case LineType.ReceivingResponse: HandleReceivingResponseLine(line); break; } //handle tail (if is enabled _tailChunk > 0) if (_tailChunk > 0 && _thisSessionRequestCount >= _tailChunk) { _stopRequested = true; } //extract custom fields HandleCustomFields(line); } catch (OutOfMemoryException) { SdkSettings.Instance.Logger.Log(TraceLevel.Error, "Out of memory exception occured during parsing {0}. Calling the garbage collector.", _rawLog.Name); GC.Collect(); } catch (Exception ex) { SdkSettings.Instance.Logger.Log(TraceLevel.Error, "Error parsing {0}:{1}", _rawLog.Name, ex.Message); } }while (true); }
/// <summary> /// Parses the raw response /// </summary> /// <param name="fullResponse"></param> public void ProcessResponse(byte[] fullResponse) { //reset all properties _statusLine = String.Empty; _headers = new HTTPHeaders(); _responseBody = null; byte[] lineBytes; string line; _responseBody = new HttpResponseBody(); MemoryStream ms = new MemoryStream(fullResponse); while (true) { lineBytes = Utils.ReadLine(ms, ESTIMATED_LINE_SIZE, LineEnding.Any); if (lineBytes != null) { line = Constants.DefaultEncoding.GetString(lineBytes); } else { return; //end of response } if (line == String.Empty) { if (_status != 100) { break; //end of headers } else { //the response status was 100 Continue expect a new set of headers and a response line //clear the current information _headers = new HTTPHeaders(); _statusLine = String.Empty; continue; } } if (_statusLine == String.Empty) { _statusLine = line; int j = STATUS_START, count = _statusLine.Length; while (j < count && !Char.IsDigit(_statusLine[j])) { j++; //for some data coming from logs or files the file has some additional characters } StringBuilder statusBuilder = new StringBuilder(); //append the digits while (j < count && Char.IsDigit(_statusLine[j])) { statusBuilder.Append(_statusLine[j]); j++; } int.TryParse(statusBuilder.ToString(), out _status); } else { //this is a header split it string[] header = line.Split(new char[1] { ':' }, 2); if (header.Length > 1) { _headers.Add(header[0], header[1]); } } } long contentLength = ms.Length - ms.Position; //find out if the response is chunked List <HTTPHeader> transferEncoding = _headers.GetHeaders("Transfer-Encoding"); if (transferEncoding.Count > 0 && String.Compare(transferEncoding[0].Values[0], "chunked", true) == 0) { _responseBody.IsChunked = true; //save the ms position in case something happens long msPos = ms.Position; try { ProcessChunkedResponse(ms); } catch { //reset the ms position to what it was before the process chunk failed ms.Position = msPos; ProcessChunkedResponseHeuristically(ms); } //finish by setting the content length to the total size of the chunks contentLength = _responseBody.Length; } else { if (contentLength > 0) { //add one big chunk with the whole response _responseBody.IsChunked = false; byte[] fullArray = new byte[contentLength]; ms.Read(fullArray, 0, (int)contentLength); _responseBody.AddChunkReference(fullArray, fullArray.Length); } //do not update the content length header with the current content length //like this _headers["Content-Length"] = _contentLength.ToString(); //we don't know if the response body size will change } }
/// <summary> /// Processes a chunked response by determining a chunk size line based on the line format and skipping those lines /// This applies to responses that do not contain correct chunk sizes /// </summary> /// <param name="ms"></param> private void ProcessChunkedResponseHeuristically(MemoryStream ms) { _responseBody.ClearChunks(); byte[] lineBytes = new byte[0]; string line; long posBefRdLn; long startChunkPos = ms.Position; long chunkLen = 0; while (true) { //save the position so we can revert to reading the full chunk posBefRdLn = ms.Position; lineBytes = Utils.ReadLine(ms, ESTIMATED_LINE_SIZE, LineEnding.Any); if (lineBytes != null) { line = Constants.DefaultEncoding.GetString(lineBytes); } else { if (_responseBody.Length < ms.Position - startChunkPos) //end of stream but there are remaining bytes that were not added { ms.Position = startChunkPos; //read the remaining response byte[] chunk = new byte[chunkLen]; ms.Read(chunk, 0, (int)chunkLen); //add the chunk to the response _responseBody.AddChunkReference(chunk, (int)chunkLen); } break; //end of response } int scIndex = line.IndexOf(';'); if (scIndex > -1) { line = line.Substring(0, scIndex); } //this is done this way because some traffic files are malformed and //do not correctly display the chunk sizes int chunkSize; if (line != String.Empty && Uri.IsHexDigit(line[0]) && Int32.TryParse(line, NumberStyles.HexNumber, null, out chunkSize)) { if (chunkLen > 0) { long currPos = ms.Position; ms.Position = startChunkPos; byte[] chunk = new byte[chunkLen]; ms.Read(chunk, 0, (int)chunkLen); //revert back to the last position ms.Position = currPos; //add the chunk to the response _responseBody.AddChunkReference(chunk, (int)chunkLen); } //save the current position as the beggining of a new chunk startChunkPos = ms.Position; } else { //calculate the current chunk size //...which is the length of the current line //plus the position in stream before reading the current line //minus the position in stream when the last chunk header was read chunkLen = posBefRdLn + lineBytes.Length - startChunkPos; } } }
/// <summary> /// Find all the entries in the specified range of the given timestamp /// </summary> /// <param name="timeStamp"></param> /// <param name="range">Range in seconds</param> /// <returns>List of timestamp snapshots in byte array format (as they were read from the stream)</returns> public List <ByteArrayBuilder> Find(DateTime timeStamp, int range) { long rangeTicks = range * 1000 * 10000; //1ms = 10000ticks DateTime startStamp = new DateTime(timeStamp.Ticks - rangeTicks); DateTime endStamp = new DateTime(timeStamp.Ticks + rangeTicks); _currentLog.Position = _cache.FindBestEntry(startStamp); DateTime cacheEntry = default(DateTime); byte[] bytes; List <ByteArrayBuilder> result = new List <ByteArrayBuilder>(); ByteArrayBuilder snapShot = null; DateTime currExtractedTime = startStamp; //last time extracted from the line read long lastPos = _currentLog.Position; //used to keep track of the log position before read while ((bytes = Utils.ReadLine(_currentLog, ESTIMATED_LINE_SIZE)) != null) { DateTime time = GetTimeStamp(bytes, timeStamp); if (time != default(DateTime)) { if (time.CompareTo(endStamp) <= 0) { //if we are inside the range if (time.CompareTo(startStamp) >= 0) { //if the current search was not cached if (cacheEntry == default(DateTime)) { //we save the first match to the cache _cache.Add(currExtractedTime, lastPos); cacheEntry = currExtractedTime; } currExtractedTime = time; if (snapShot != null) { //save the current snapshot result.Add(snapShot); } snapShot = new ByteArrayBuilder(); } } else { //the current extracted time is higher than the upper range limit if (result.Count > 0 || (snapShot != null && snapShot.Length > 0)) { //if we have already extracted snapshots then we stop here break; //otherwise we keep looking; } } } //save the last read position lastPos = _currentLog.Position; //here we add bytes we read to the current snapshot if (snapShot != null) { snapShot.AddChunkReference(bytes, bytes.Length); snapShot.AddChunkReference(Constants.NewLineBytes, Constants.NewLineBytes.Length); } } //end read loop //check if there is anything left to add if (snapShot != null && snapShot.Length > 0) { result.Add(snapShot); } return(result); }