public void TestValidHttpDateFormats()
{
DateTime parsedDate;
var httpDate = "Fri, 23 Jan 2015 23:55:58 GMT";
Assert.True(NScrapeUtility.TryParseHttpDate(httpDate, out parsedDate));
Assert.Equal(DateTimeKind.Utc, parsedDate.Kind);
Assert.Equal(httpDate, parsedDate.ToString("r"));
httpDate = "Tue, 15 Nov 1994 08:12:31 GMT";
Assert.True(NScrapeUtility.TryParseHttpDate(httpDate, out parsedDate));
Assert.Equal(DateTimeKind.Utc, parsedDate.Kind);
Assert.Equal(httpDate, parsedDate.ToString("r"));
httpDate = "Thu, 01 Jan 1970 00:00:00 GMT";
Assert.True(NScrapeUtility.TryParseHttpDate(httpDate, out parsedDate));
Assert.Equal(DateTimeKind.Utc, parsedDate.Kind);
Assert.Equal(httpDate, parsedDate.ToString("r"));
httpDate = "Mon, 15 Jun 2009 20:45:30 GMT";
Assert.True(NScrapeUtility.TryParseHttpDate(httpDate, out parsedDate));
Assert.Equal(DateTimeKind.Utc, parsedDate.Kind);
Assert.Equal(httpDate, parsedDate.ToString("r"));
}
// http://tools.ietf.org/html/rfc7231#section-7.1.1.1
public void TestPreferredAndObsoleteHttpDateFormats()
{
const string preferredFormatHttpDate = "Sun, 06 Nov 1994 08:49:37 GMT";
DateTime parsedDate;
// Preferred format
var httpDate = preferredFormatHttpDate;
Assert.True(NScrapeUtility.TryParseHttpDate(httpDate, out parsedDate));
Assert.Equal(DateTimeKind.Utc, parsedDate.Kind);
Assert.Equal(preferredFormatHttpDate, parsedDate.ToString("r"));
// Obsolete RFC 850 format
httpDate = "Sunday, 06-Nov-94 08:49:37 GMT";
Assert.True(NScrapeUtility.TryParseHttpDate(httpDate, out parsedDate));
Assert.Equal(DateTimeKind.Utc, parsedDate.Kind);
Assert.Equal(preferredFormatHttpDate, parsedDate.ToString("r"));
// Obsolete ANSI C's asctime() format
httpDate = "Sun Nov 6 08:49:37 1994";
Assert.True(NScrapeUtility.TryParseHttpDate(httpDate, out parsedDate));
Assert.Equal(DateTimeKind.Utc, parsedDate.Kind);
Assert.Equal(preferredFormatHttpDate, parsedDate.ToString("r"));
}
public void TestInvalidButParseableHttpDateFormats()
{
const string preferredFormatHttpDate = "Sun, 02 Jul 2017 22:07:33 GMT";
DateTime parsedDate;
var httpDate = "Sun, 02-Jul-17 22:07:33 GMT";
Assert.True(NScrapeUtility.TryParseHttpDate(httpDate, out parsedDate));
Assert.Equal(DateTimeKind.Utc, parsedDate.Kind);
Assert.Equal(preferredFormatHttpDate, parsedDate.ToString("r"));
httpDate = "Sunday, July 2, 2017 10:07:33pm";
Assert.True(NScrapeUtility.TryParseHttpDate(httpDate, out parsedDate));
Assert.Equal(DateTimeKind.Utc, parsedDate.Kind);
Assert.Equal(preferredFormatHttpDate, parsedDate.ToString("r"));
httpDate = "Sun, 02 Jul 2017 22:07:33 -0000";
Assert.True(NScrapeUtility.TryParseHttpDate(httpDate, out parsedDate));
Assert.Equal(DateTimeKind.Utc, parsedDate.Kind);
Assert.Equal(preferredFormatHttpDate, parsedDate.ToString("r"));
httpDate = "Sun, 02 Jul 2017 22:07:33 +0000";
Assert.True(NScrapeUtility.TryParseHttpDate(httpDate, out parsedDate));
Assert.Equal(DateTimeKind.Utc, parsedDate.Kind);
Assert.Equal(preferredFormatHttpDate, parsedDate.ToString("r"));
}
public void MultipleCookieTest()
{
var header = string.Join(",", CookieGithub1, CookieGithub2, CookieGithub3);
var cookies = NScrapeUtility.ParseSetCookieHeader(header, DefaultDomain).ToList();
Assert.Equal(3, cookies.Count);
CookieGithub1Test(cookies[0]);
CookieGithub2Test(cookies[1]);
CookieGithub3Test(cookies[2]);
}
public void OddExpiresDateFailingTests()
{
var cookies = NScrapeUtility.ParseSetCookieHeader(string.Join(",", CookieOddExpiresDate1, CookieOddExpiresDate2), DefaultDomain).ToList();
Assert.Equal(2, cookies.Count);
CookieOddExpiresDateTest(cookies[0]);
CookieOddExpiresDateTest(cookies[1]);
cookies = NScrapeUtility.ParseSetCookieHeader(string.Join(",", CookieOddExpiresDate2, CookieOddExpiresDate1), DefaultDomain).ToList();
Assert.Equal(2, cookies.Count);
CookieOddExpiresDateTest(cookies[0]);
CookieOddExpiresDateTest(cookies[1]);
}
public void NullAndEmptyArgumentsCornerCaseTests()
{
// Null arguments
var cookies = NScrapeUtility.ParseSetCookieHeader(null, DefaultDomain);
Assert.Equal(0, cookies.Count());
var ex = Assert.Throws <ArgumentException>(() => NScrapeUtility.ParseSetCookieHeader(null, null));
Assert.Equal(ex.ParamName, "hostName");
// Empty arguments
cookies = NScrapeUtility.ParseSetCookieHeader(string.Empty, DefaultDomain);
Assert.Equal(0, cookies.Count());
ex = Assert.Throws <ArgumentException>(() => NScrapeUtility.ParseSetCookieHeader(string.Empty, string.Empty));
Assert.Equal(ex.ParamName, "hostName");
}
public void OddExpiresDateTests()
{
var cookies = NScrapeUtility.ParseSetCookieHeader(CookieOddExpiresDate1, DefaultDomain).ToList();
Assert.Equal(1, cookies.Count);
CookieOddExpiresDateTest(cookies[0]);
cookies = NScrapeUtility.ParseSetCookieHeader(CookieOddExpiresDate2, DefaultDomain).ToList();
Assert.Equal(1, cookies.Count);
CookieOddExpiresDateTest(cookies[0]);
cookies = NScrapeUtility.ParseSetCookieHeader(CookieOddExpiresDateAnsiCasctime, DefaultDomain).ToList();
Assert.Equal(1, cookies.Count);
CookieOddExpiresDateTest(cookies[0]);
cookies = NScrapeUtility.ParseSetCookieHeader(CookieOddExpiresDateRfc850, DefaultDomain).ToList();
Assert.Equal(1, cookies.Count);
CookieOddExpiresDateTest(cookies[0]);
}
public void TestInvalidHttpDateFormats()
{
DateTime parsedDate;
string httpDate = null;
// ReSharper disable once ExpressionIsAlwaysNull
Assert.False(NScrapeUtility.TryParseHttpDate(httpDate, out parsedDate));
Assert.Equal(new DateTime(), parsedDate);
httpDate = string.Empty;
Assert.False(NScrapeUtility.TryParseHttpDate(httpDate, out parsedDate));
Assert.Equal(new DateTime(), parsedDate);
httpDate = "Xxx, 00-Xxx-00 00:00:00 GMT";
Assert.False(NScrapeUtility.TryParseHttpDate(httpDate, out parsedDate));
Assert.Equal(new DateTime(), parsedDate);
httpDate = "ks alkdjf alkd falk flka flakf";
Assert.False(NScrapeUtility.TryParseHttpDate(httpDate, out parsedDate));
Assert.Equal(new DateTime(), parsedDate);
}
public void SingleCookieTests()
{
var cookies = NScrapeUtility.ParseSetCookieHeader(CookieGithub1, DefaultDomain).ToList();
Assert.Equal(1, cookies.Count);
CookieGithub1Test(cookies[0]);
cookies = NScrapeUtility.ParseSetCookieHeader(CookieGithub2, DefaultDomain).ToList();
Assert.Equal(1, cookies.Count);
CookieGithub2Test(cookies[0]);
cookies = NScrapeUtility.ParseSetCookieHeader(CookieGithub3, DefaultDomain).ToList();
Assert.Equal(1, cookies.Count);
CookieGithub3Test(cookies[0]);
cookies = NScrapeUtility.ParseSetCookieHeader(CookieStackOverflow, DefaultDomain).ToList();
Assert.Equal(1, cookies.Count);
CookieStackOverflowTest(cookies[0]);
cookies = NScrapeUtility.ParseSetCookieHeader(CookieFrederik, DefaultDomain).ToList();
Assert.Equal(1, cookies.Count);
CookieFrederikTest(cookies[0]);
}
public void BadSetCookieHeaderTests()
{
// Bad header
var cookies = NScrapeUtility.ParseSetCookieHeader("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", DefaultDomain).ToList();
Assert.Equal(0, cookies.Count);
// Bad header with commas
cookies = NScrapeUtility.ParseSetCookieHeader("xxxxxxxxxxxxxxxxxx,xxxxxxxxxxxxxxxxxxx,xxxxxxxxxxxxxxx", DefaultDomain).ToList();
Assert.Equal(0, cookies.Count);
// Bad header with commas and semicolons
cookies = NScrapeUtility.ParseSetCookieHeader("xxxxxxxx; xxxxxxxxxx,xxxxxxx; xxxxxxxxxxxx,xxxxx; xxxxxxxxxx", DefaultDomain).ToList();
Assert.Equal(0, cookies.Count);
// Valid cookie but invalid attributes
cookies = NScrapeUtility.ParseSetCookieHeader("foo=bar; xxxxxxxxxx; xxxxxxx; xxxxx; xxxxxxxxxx", DefaultDomain).ToList();
Assert.Equal(1, cookies.Count);
var cookie = cookies[0];
Assert.Equal(string.Empty, cookie.Comment);
Assert.Null(cookie.CommentUri);
Assert.False(cookie.Discard);
Assert.Equal(DefaultDomain, cookie.Domain);
Assert.False(cookie.Expired);
Assert.Equal(new DateTime(1, 1, 1, 0, 0, 0), cookie.Expires);
Assert.False(cookie.HttpOnly);
Assert.Equal("foo", cookie.Name);
Assert.Equal("/", cookie.Path);
Assert.Equal(string.Empty, cookie.Port);
Assert.False(cookie.Secure);
var timeStamp = DateTime.Now;
Assert.True(timeStamp.Subtract(cookie.TimeStamp).TotalSeconds < 1.0); // just a few milliseconds between parse and test
Assert.Equal("bar", cookie.Value);
Assert.Equal(0, cookie.Version);
}
public void BadExpiresDateTest()
{
var cookies = NScrapeUtility.ParseSetCookieHeader("foo=bar; expires=Xxx, 00-Xxx-00 00:00:00 GMT; path=/", DefaultDomain).ToList();
Assert.Equal(1, cookies.Count);
var cookie = cookies[0];
Assert.Equal(string.Empty, cookie.Comment);
Assert.Null(cookie.CommentUri);
Assert.False(cookie.Discard);
Assert.Equal(DefaultDomain, cookie.Domain);
Assert.False(cookie.Expired);
Assert.Equal(new DateTime(1, 1, 1, 0, 0, 0), cookie.Expires);
Assert.False(cookie.HttpOnly);
Assert.Equal("foo", cookie.Name);
Assert.Equal("/", cookie.Path);
Assert.Equal(string.Empty, cookie.Port);
Assert.False(cookie.Secure);
var timeStamp = DateTime.Now;
Assert.True(timeStamp.Subtract(cookie.TimeStamp).TotalSeconds < 1.0); // just a few milliseconds between parse and test
Assert.Equal("bar", cookie.Value);
Assert.Equal(0, cookie.Version);
}
public void NewlinesTest()
{
var cookies = NScrapeUtility.ParseSetCookieHeader("foo=\rline1\nline2\rline3\r\nline4\r\rline5\n", DefaultDomain).ToList();
Assert.Equal(1, cookies.Count);
var cookie = cookies[0];
Assert.Equal(string.Empty, cookie.Comment);
Assert.Null(cookie.CommentUri);
Assert.False(cookie.Discard);
Assert.Equal(DefaultDomain, cookie.Domain);
Assert.False(cookie.Expired);
Assert.Equal(new DateTime(1, 1, 1, 0, 0, 0), cookie.Expires);
Assert.False(cookie.HttpOnly);
Assert.Equal("foo", cookie.Name);
Assert.Equal("/", cookie.Path);
Assert.Equal(string.Empty, cookie.Port);
Assert.False(cookie.Secure);
var timeStamp = DateTime.Now;
Assert.True(timeStamp.Subtract(cookie.TimeStamp).TotalSeconds < 1.0); // just a few milliseconds between parse and test
Assert.Equal("line1line2line3line4line5", cookie.Value);
Assert.Equal(0, cookie.Version);
}
