// GET: Go
public ActionResult Index(String password)
{
using (var db = new ErrorReportContext())
{
db.ErrorReports.Add(new ErrorReport {
reportNumber = 1, errorNumber = 15, errorType = "dumb", parentURL = "http://spsu.edu/", URL = "http://spsu.edu/broken"
});
db.SaveChanges();
}
return(View());
}
public ErrorController(ErrorReportContext context)
{
_context = context;
// Seed
if (_context.ErrorReports.Count() == 0)
{
context.ErrorReports.Add(new ErrorReport()
{
Created = DateTimeOffset.UtcNow, Message = "some message", Json = "{ stuff }"
});
_context.SaveChanges();
}
}
public void CreateReport(ErrorReportEntity report)
{
if (report == null)
{
throw new ArgumentNullException("report");
}
if (string.IsNullOrEmpty(report.Title) && report.Exception != null)
{
report.Title = report.Exception.Message;
if (report.Title.Length > 100)
{
report.Title = report.Title.Substring(0, 100);
}
}
var collections = new List <string>();
foreach (var context in report.ContextInfo)
{
var data = EntitySerializer.Serialize(context);
if (data.Length > MaxCollectionSize)
{
var tooLargeCtx = new ErrorReportContext(context.Name,
new Dictionary <string, string>()
{
{
"Error",
$"This collection was larger ({data.Length}bytes) than the threshold of {MaxCollectionSize}bytes"
}
});
data = EntitySerializer.Serialize(tooLargeCtx);
}
collections.Add(data);
}
_unitOfWork.Insert(report);
var cols = string.Join(", ", collections);
var inboound = new InboundCollection
{
JsonData = $"[{cols}]",
ReportId = report.Id
};
_unitOfWork.Insert(inboound);
}
private static DataRow CreateDataTableRow(DataTable dataTable, int reportId,
ErrorReportContext context,
KeyValuePair <string, string> property)
{
var contextName = context.Name.Length > 50
? context.Name.Substring(0, 47) + "..."
: context.Name;
var propertyName = property.Key.Length > 50
? property.Key.Substring(0, 47) + "..."
: property.Key;
var row = dataTable.NewRow();
row["ReportId"] = reportId;
row["Name"] = contextName;
row["PropertyName"] = propertyName;
row["Value"] = property.Value;
return(row);
}
public void Crawl()
{
string current_page = "";
string domain = "spsu.edu";
string start_page = "https://" + domain;
List <string> to_search = new List <string>();
List <string> searched = new List <string>();
List <string> parents = new List <string>();
string puppies = "";
string href = "";
string content = "";
HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(start_page);
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
StreamReader stream = new StreamReader(response.GetResponseStream());
string source_code = stream.ReadToEnd();
HtmlDocument doc = new HtmlDocument();
doc.OptionReadEncoding = false;
to_search.Add(start_page);
int errorNumber = 0;
ErrorReportContext db = new ErrorReportContext();
while (to_search.Count > 0)
{
current_page = to_search.ElementAt(0);
to_search.RemoveAt(0);
if (parents.Count() != 0)
{
parents.RemoveAt(0);
}
//foreach (string link in searched) // add things to searched
// if (link == current_page)
// continue;
if (searched.Contains(current_page))
{
continue;
}
//request = (HttpWebRequest)HttpWebRequest.Create(current_page);
//request.Method = "HEAD";
////response.Dispose();
//try
//{
// response = (HttpWebResponse)request.GetResponse();
// //request.EndGetResponse();
//}
//catch (System.Net.WebException ex)
//{
// var status = response.StatusCode;
// switch(status.ToString())
// {
// case "NotFound":
// Console.WriteLine("Huzzah!");
// break;
// }
// Console.WriteLine(ex);
// continue;
//}
//var status_code = response.StatusCode;
//Console.WriteLine(status_code.ToString());
try
{
request = (HttpWebRequest)HttpWebRequest.Create(current_page);
response = (HttpWebResponse)request.GetResponse();
}
catch (System.Net.WebException ex)
{
//insert error here
db.ErrorReports.Add(new ErrorReport {
reportNumber = 2, errorNumber = errorNumber++, URL = current_page, parentURL = parents.ElementAt(0), errorType = ex.Source.ToString()
});
db.SaveChanges();
Console.WriteLine(ex.Status);
continue;
}
var status = response.StatusCode;
Console.WriteLine(status);
searched.Add(current_page);
content = response.ContentType;
if (!(content.Contains("text/html") || content.Contains("application/xhtml+xml") || content.Contains("application/xml")))
{
Console.WriteLine("Invalid Content type " + current_page);
continue;
}
//if (!current_page.Contains(domain)) //use host?
//{
// Console.WriteLine("Off domain " + current_page);
// continue;
//}
if (!request.Host.Equals(domain))
{
Console.WriteLine("Off domain " + current_page);
continue;
}
// start to parse html
stream = new StreamReader(response.GetResponseStream());
source_code = stream.ReadToEnd();
doc.LoadHtml(source_code);
Console.WriteLine(current_page);
var links = doc.DocumentNode.SelectNodes("//a[@href]");
if (links != null)
{
foreach (HtmlNode link in links)
{
HtmlAttribute att = link.Attributes["href"];
href = att.Value;
if (href == null)
{
break;
}
else if (href == "")
{
break;
}
else if (href.Contains("mailto:"))
{
break;
}
else if (href.ElementAt(0) == '/')
{
puppies = start_page + href;
}
else if (href.ElementAt(0) == '?')
{
puppies = start_page + href;
}
else if (href.ElementAt(0) == '#')
{
break;
}
else if (href.Contains("http"))
{
puppies = href;
}
else if (href.Contains("javascript"))
{
break;
}
else
{
puppies = start_page + href;
}
to_search.Add(puppies);
parents.Add(current_page);
//Console.WriteLine(puppies);
}
}
response.Close();
}
}
