public IEnumerable <ProcessQueue> GetProcessQueueItems(int sessionId, int websiteId)
{
using ItsyBitsyDbContext context = new ItsyBitsyDbContext();
var nextQueueItems = (from queueItem in context.ProcessQueue
where queueItem.SessionId == sessionId && queueItem.WebsiteId == websiteId
orderby queueItem.TimeStamp
select queueItem)
.Take(400);
return(nextQueueItems.ToList());
}
public static async Task <Website> GetDomainWebsite(int websiteId)
{
using ItsyBitsyDbContext context = new ItsyBitsyDbContext();
var dbWebsite = await context.Website.FindAsync(websiteId);
if (dbWebsite != null)
{
return(new Website(dbWebsite));
}
return(null);
}
public static async Task <int> CreateNewSession()
{
using ItsyBitsyDbContext context = new ItsyBitsyDbContext();
var session = new Session()
{
StartTime = DateTime.Now
};
var dbWebsite = context.Session.Add(session);
await context.SaveChangesAsync();
return(session.Id);
}
public static async Task EndSession(int sessionId)
{
using ItsyBitsyDbContext context = new ItsyBitsyDbContext();
var session = new Session()
{
Id = sessionId,
EndTime = DateTime.Now
};
context.Entry(session).Property(x => x.EndTime).IsModified = true;
await context.SaveChangesAsync();
}
internal static async Task AddPageRelation(IEnumerable <string> existingLinks, int parentId)
{
using ItsyBitsyDbContext context = new ItsyBitsyDbContext();
var pageRelations = from page in context.Page
where existingLinks.Contains(page.Uri)
select new PageRelation()
{
ChildPageId = page.Id, ParentPageId = parentId
};
context.PageRelation.AddRange(pageRelations);
await context.SaveChangesAsync();
}
public void AddToProcessQueue(string link, int parentId, int websiteId, int sessionId)
{
using ItsyBitsyDbContext context = new ItsyBitsyDbContext();
var newItem = new ProcessQueue()
{
Link = link,
ParentId = parentId,
SessionId = sessionId,
WebsiteId = websiteId
};
context.ProcessQueue.Add(newItem);
context.SaveChanges();
}
public static async Task <Website> CreateWebsite(string seed)
{
if (!Uri.TryCreate(seed, UriKind.Absolute, out Uri uri))
{
throw new InvalidCastException($"{seed} is not a valid uri.");
}
if (uri.Scheme != "http" && uri.Scheme != "https")
{
throw new Exception("Only http and https are accepted.");
}
var host = new Uri($"{uri.Scheme}://{uri.Host}");
var webResuest = WebRequest.CreateHttp(host);
webResuest.AutomaticDecompression = DecompressionMethods.All;
webResuest.AllowAutoRedirect = true;
webResuest.Method = "GET";
var response = webResuest.GetResponse();
var websiteHomeUri = response.ResponseUri.ToString();
if (seed != websiteHomeUri)
{
throw new Exception($"{seed} redirects to {websiteHomeUri} please use this as the seed.");
}
using ItsyBitsyDbContext context = new ItsyBitsyDbContext();
if (context.Website.Any(x => x.Seed == websiteHomeUri))
{
throw new InvalidCastException($"{websiteHomeUri} already exists.");
}
var dbWebsite = new Data.Website()
{
Seed = seed
};
context.Website.Add(dbWebsite);
await context.SaveChangesAsync();
return(new Website(dbWebsite));
}
public int SaveLink(DownloadResult response, int websiteId, int sessionId)
{
try
{
using ItsyBitsyDbContext context = new ItsyBitsyDbContext();
var newPage = new Page()
{
SessionId = sessionId,
StatusCode = response.Status,
WebsiteId = websiteId,
Uri = response.Uri,
ContentType = (byte)response.ContentType,
DownloadTime = response.DownloadTime,
ContentLength = response.ContentLengthBytes
};
context.Page.Add(newPage);
if (response.ParentId.HasValue)
{
var newPageRelation = new PageRelation()
{
ParentPageId = response.ParentId.Value,
ChildPage = newPage,
};
context.PageRelation.Add(newPageRelation);
}
context.SaveChanges();
return(newPage.Id);
}
catch (Exception e)
{
throw e;
}
}
public static List <Website> GetWebsites()
{
using ItsyBitsyDbContext context = new ItsyBitsyDbContext();
return(context.Website.Select(x => new Website(x)).ToList());
}
public void RemoveQueuedItems(IEnumerable <ProcessQueue> successfullyQueued)
{
using ItsyBitsyDbContext context = new ItsyBitsyDbContext();
context.ProcessQueue.RemoveRange(successfullyQueued);
context.SaveChanges();
}
public bool PageExists(string newLinkFound, int sessionId)
{
using ItsyBitsyDbContext context = new ItsyBitsyDbContext();
return(context.Page.Any(x => x.SessionId == sessionId && x.Uri == newLinkFound));
}
