/// <summary>Processes and URL and gets the data for each mail item in a mail list</summary>
/// <param name="url">The Mail list URL to scan</param>
/// <returns>A list of mail items</returns>
private IEnumerable<Contribution> ProcessUrl(Uri url)
{
var returnValue = new List<Contribution>();
string responseData;
var webRequest = WebRequest.CreateHttp(url);
var httpResponse = (HttpWebResponse)webRequest.GetResponse();
var stream = httpResponse.GetResponseStream();
if (null == stream)
{
return returnValue;
}
using (var responseReader = new StreamReader(stream))
{
responseData = responseReader.ReadToEnd();
}
var mailListItem = MailListIdentifier.Match(responseData);
var dateMailMatches = DateMaildentifier.Matches(mailListItem.Groups["messages"].Value);
foreach (Match dateMails in dateMailMatches)
{
var itemsDateString = dateMails.Groups["date"].Value;
DateTime itemsDate;
if (!DateTime.TryParse(itemsDateString, out itemsDate))
{
continue;
}
var mailItems = dateMails.Groups["mails"].Value;
foreach (Match matchedRecord in RecordIdentifier.Matches(mailItems))
{
var r = new Contribution
{
Date = itemsDate,
Tracker = new Tracker() {Name = Tracker}
};
var mailUri = new Uri(url, matchedRecord.Groups["url"].Value);
r.Url = mailUri.AbsoluteUri;
r.Author = new Author()
{
Name = matchedRecord.Groups["author"].Value
};
r.Message = HttpUtility.HtmlDecode(matchedRecord.Groups["subject"].Value);
string[] emailFullMessage;
if(SpecIdentifier.IsMatch(r.Message))
{
foreach (var spec in from Match specRecord in SpecIdentifier.Matches(r.Message) select specRecord.Groups["spec"].Value.IsNullOrEmptyOrWhiteSpace("unknown").ToLowerInvariant())
{
r.Sha = (r.Url + spec).GetHashSha1();
if (CheckContributionsForSha(r.Sha)) continue;
var s = r.DeepClone();
//if it is not in DB then go here
emailFullMessage = GetEmailAndFullMessage(mailUri);
s.Email = emailFullMessage[0];
SetAuthorEmailOrganization(s);
s.Spec = new Spec()
{
Name = spec
};
returnValue.Add(s);//submit record to DB
}
}
else
{
r.Sha = r.Url.GetHashSha1();
if (CheckContributionsForSha(r.Sha))
{
continue;
}
//if it is not in DB then go here
emailFullMessage = GetEmailAndFullMessage(mailUri);
r.Email = emailFullMessage[0];
SetAuthorEmailOrganization(r);
r.Spec = new Spec()
{
Name = matchedRecord.Groups["spec"].Value.IsNullOrEmptyOrWhiteSpace("unknown").ToLowerInvariant()
};
if(r.Spec.Name.Equals("unknown", StringComparison.InvariantCultureIgnoreCase))
{
r.Spec = new Spec()
{
Name = TrackerData.DetermineIntent(emailFullMessage[1])
};
}
//if spec is still unknown don't add it to DB
if(!r.Spec.Name.Equals("unknown", StringComparison.InvariantCultureIgnoreCase))
{
returnValue.Add(r);//submit record to DB
}
}
}
}
return returnValue;
}