public void TestIterator() { List <Token> tokens = new List <Token>(); Token tokenReference = new Token(TokenType.DollarIdentifier, "a", 5, 2); tokens.Add(tokenReference); tokens.Add(new Token(TokenType.Word, "hi", 5, 3)); tokens.Add(new Token(TokenType.AtIdentifier, "b", 12, 0)); tokens.Add(new Token(TokenType.Word, "yEs", 14, 42)); TokenIterator tokenIterator = new TokenIterator(tokens); Assert.IsTrue(tokenIterator.HasNext()); Assert.AreEqual(4, tokenIterator.Count); // Compare references to make sure we're getting back what we add. Token token = tokenIterator.Next(); Assert.AreEqual(tokenReference, token); token = tokenIterator.Next(); Assert.AreEqual(TokenType.Word, token.Type); Assert.AreEqual("hi", token.Text, false); Assert.AreEqual(5, token.LineNumber); Assert.AreEqual(3, token.CharOffset); int marker = tokenIterator.GetMarker(); Assert.AreEqual(2, marker); Assert.IsTrue(tokenIterator.HasNext()); token = tokenIterator.Next(); Assert.AreEqual(TokenType.AtIdentifier, token.Type); Assert.AreEqual("b", token.Text, false); Assert.AreEqual(12, token.LineNumber); Assert.AreEqual(0, token.CharOffset); Assert.IsTrue(tokenIterator.HasNext()); token = tokenIterator.Next(); Assert.AreEqual(TokenType.Word, token.Type); Assert.AreEqual("yEs", token.Text, false); Assert.AreEqual(14, token.LineNumber); Assert.AreEqual(42, token.CharOffset); Assert.IsFalse(tokenIterator.HasNext()); // Return to the second token. tokenIterator.SetToMarker(marker); Assert.IsTrue(tokenIterator.HasNext()); token = tokenIterator.Next(); Assert.AreEqual(TokenType.AtIdentifier, token.Type); Assert.AreEqual("b", token.Text, false); Assert.AreEqual(12, token.LineNumber); Assert.AreEqual(0, token.CharOffset); }
public void TestEmptyIterator() { TokenIterator tokenIterator = new TokenIterator(new List <Token>()); Assert.IsFalse(tokenIterator.HasNext()); Assert.AreEqual(0, tokenIterator.Count); int marker = tokenIterator.GetMarker(); Assert.AreEqual(0, marker); Assert.AreEqual(0, tokenIterator.GetMarker()); }
public void HasNextTest() { //Create list and add dummy tokens to it List <Token> tokenStream = new List <Token>(); Token token1 = new Token(); Token token2 = new Token(); tokenStream.Add(token1); tokenStream.Add(token2); //Create iterator with tokenstream and then test some things out TokenIterator iterator = new TokenIterator(tokenStream); //Check size Assert.IsTrue(iterator.GetSize() == tokenStream.Count); //Check if tokens are in list Assert.IsTrue(iterator.HasNext()); //Check also if there are 2 items in list Assert.IsTrue(iterator.HasNext(2)); //Get first token and check Token CurrentToken = iterator.NextToken(); Assert.AreEqual(token1, CurrentToken); //Get second token and check Assert.IsTrue(iterator.HasNext()); CurrentToken = iterator.NextToken(); Assert.AreEqual(token2, CurrentToken); //No items in list left Assert.IsFalse(iterator.HasNext()); }
public void TestTokenizeFileWithIterator() { Tokenizer tokenizer = Tokenizer.FromFile(RESOURCE_TEST_FILE); TokenIterator it = tokenizer.GetTokenIterator(); Token token; Token actualToken; int index = 0; Assert.IsTrue(FILE_EXPECTED_TOKEN_ELEMENTS.Length == it.Count * 4); while (it.HasNext()) { token = it.Next(); actualToken = new Token((TokenType)FILE_EXPECTED_TOKEN_ELEMENTS[index, 0], (string)FILE_EXPECTED_TOKEN_ELEMENTS[index, 1], (int)FILE_EXPECTED_TOKEN_ELEMENTS[index, 2], (int)FILE_EXPECTED_TOKEN_ELEMENTS[index, 3]); Assert.AreEqual(actualToken, token); index++; } }
// see interface ConnectionReuseStrategy public virtual bool KeepAlive(HttpResponse response, HttpContext context) { Args.NotNull(response, "HTTP response"); Args.NotNull(context, "HTTP context"); // Check for a self-terminating entity. If the end of the entity will // be indicated by closing the connection, there is no keep-alive. ProtocolVersion ver = response.GetStatusLine().GetProtocolVersion(); Header teh = response.GetFirstHeader(HTTP.TransferEncoding); if (teh != null) { if (!Sharpen.Runtime.EqualsIgnoreCase(HTTP.ChunkCoding, teh.GetValue())) { return(false); } } else { if (CanResponseHaveBody(response)) { Header[] clhs = response.GetHeaders(HTTP.ContentLen); // Do not reuse if not properly content-length delimited if (clhs.Length == 1) { Header clh = clhs[0]; try { int contentLen = System.Convert.ToInt32(clh.GetValue()); if (contentLen < 0) { return(false); } } catch (FormatException) { return(false); } } else { return(false); } } } // Check for the "Connection" header. If that is absent, check for // the "Proxy-Connection" header. The latter is an unspecified and // broken but unfortunately common extension of HTTP. HeaderIterator hit = response.HeaderIterator(HTTP.ConnDirective); if (!hit.HasNext()) { hit = response.HeaderIterator("Proxy-Connection"); } // Experimental usage of the "Connection" header in HTTP/1.0 is // documented in RFC 2068, section 19.7.1. A token "keep-alive" is // used to indicate that the connection should be persistent. // Note that the final specification of HTTP/1.1 in RFC 2616 does not // include this information. Neither is the "Connection" header // mentioned in RFC 1945, which informally describes HTTP/1.0. // // RFC 2616 specifies "close" as the only connection token with a // specific meaning: it disables persistent connections. // // The "Proxy-Connection" header is not formally specified anywhere, // but is commonly used to carry one token, "close" or "keep-alive". // The "Connection" header, on the other hand, is defined as a // sequence of tokens, where each token is a header name, and the // token "close" has the above-mentioned additional meaning. // // To get through this mess, we treat the "Proxy-Connection" header // in exactly the same way as the "Connection" header, but only if // the latter is missing. We scan the sequence of tokens for both // "close" and "keep-alive". As "close" is specified by RFC 2068, // it takes precedence and indicates a non-persistent connection. // If there is no "close" but a "keep-alive", we take the hint. if (hit.HasNext()) { try { TokenIterator ti = CreateTokenIterator(hit); bool keepalive = false; while (ti.HasNext()) { string token = ti.NextToken(); if (Sharpen.Runtime.EqualsIgnoreCase(HTTP.ConnClose, token)) { return(false); } else { if (Sharpen.Runtime.EqualsIgnoreCase(HTTP.ConnKeepAlive, token)) { // continue the loop, there may be a "close" afterwards keepalive = true; } } } if (keepalive) { return(true); } } catch (ParseException) { // neither "close" nor "keep-alive", use default policy // invalid connection header means no persistent connection // we don't have logging in HttpCore, so the exception is lost return(false); } } // default since HTTP/1.1 is persistent, before it was non-persistent return(!ver.LessEquals(HttpVersion.Http10)); }