private void ClearDatabase()
{
db.Query <Person>().Remove().Execute();
db.Query <Follow>().Remove().Execute();
db.Query <Host>().Remove().Execute();
if (db.ListGraphs().Count(x => x.Key == "SocialGraph") != 0)
{
db.Graph("SocialGraph").Drop();
}
}
/// <summary>
/// upsert edge to arango collection (T) and ensure the to side is unique
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="arangoDatabase"></param>
/// <param name="item"></param>
public static void UpsertEdgeUniqueTo <T>(this IArangoDatabase arangoDatabase, T item) where T : EdgeBase
{
arangoDatabase.Query <T>()
.Filter(item_to_remove => item_to_remove._to == item._to)
.Remove(item_to_remove => new { _key = item_to_remove._key })
.Execute();
arangoDatabase.Query().Upsert <T>(
arangoDB_item_for_upsert => new { _key = item._key },
_ => item,
(_, old) => item)
.Execute();
}
/// <summary>
/// Search from the originId to min and max level via any direction of edges
/// </summary>
/// <param name="arangoDatabase"></param>
/// <param name="originId"></param>
/// <param name="minDepth"></param>
/// <param name="maxDepth"></param>
/// <param name="edges"></param>
/// <returns></returns>
public static IEnumerable <EdgeTrace <TVertex, TEdge> > TraverseEdgeFromOrigin <TVertex, TEdge>(
this IArangoDatabase arangoDatabase,
string originId,
int minDepth,
int maxDepth,
params Type[] edges) where TVertex : VertexBase where TEdge : EdgeBase
{
var query = arangoDatabase.Query()
.Traversal <TVertex, TEdge>(originId)
.Depth(minDepth, maxDepth);
if (edges != null)
{
foreach (var type in edges)
{
query = query.Edge(type.Name);
}
}
return(query.AnyDirection()
.Return(g => new EdgeTrace <TVertex, TEdge>()
{
Edge = g.Edge, Vertex = g.Vertex
})
.ToListAsync()
.GetAwaiter()
.GetResult());
}
/// <summary>
/// Search from the originId to min and max level via any direction of edges
/// </summary>
/// <param name="arangoDatabase"></param>
/// <param name="originId"></param>
/// <param name="minDepth"></param>
/// <param name="maxDepth"></param>
/// <param name="edges"></param>
/// <returns></returns>
public static IEnumerable <RawEdgeTrace> TraverseRawFromOrigin(
this IArangoDatabase arangoDatabase,
string originId,
int minDepth,
int maxDepth,
params Type[] edges)
{
var query = arangoDatabase.Query()
.Traversal <JObject, JObject>(originId)
.Depth(minDepth, maxDepth);
if (edges != null)
{
foreach (var type in edges)
{
query = query.Edge(type.Name);
}
}
return(query.AnyDirection()
.Return(g => new RawEdgeTrace()
{
Edge = g.Edge, Vertex = g.Vertex
})
.ToListAsync()
.GetAwaiter()
.GetResult());
}
/// <summary>
/// Search from the originId to min and max level via any direction of edges
/// </summary>
/// <typeparam name="TTarget"></typeparam>
/// <param name="arangoDatabase"></param>
/// <param name="originId"></param>
/// <param name="minDepth"></param>
/// <param name="maxDepth"></param>
/// <param name="edges"></param>
/// <returns></returns>
public static IEnumerable <TTarget> TraverseFromOrigin <TTarget>(
this IArangoDatabase arangoDatabase,
string originId,
int minDepth,
int maxDepth,
params Type[] edges)
where TTarget : VertexBase
{
var query = arangoDatabase.Query()
.Traversal <TTarget, EdgeBase>(originId)
.Depth(minDepth, maxDepth);
if (edges != null)
{
foreach (var type in edges)
{
query = query.Edge(type.Name);
}
}
return(query.AnyDirection()
.Filter(g => AQL.Split(g.Vertex._id, "/")[0] == typeof(TTarget).Name)
.Return(g => g.Vertex)
.ToListAsync()
.GetAwaiter()
.GetResult());
}
public async Task JoinGroup(JoinKanbanHubRequest request)
{
var group = await(from g in Database.Query <Group>()
where g.Key == request.GroupId
select g).FirstOrDefaultAsync();
if (group == null)
{
Context.Abort();
return;
}
var kanbanBoard = (from kb in @group.KanbanBoards where kb.Id == request.BoardId select kb).FirstOrDefault();
if (kanbanBoard == null)
{
Context.Abort();
return;
}
var memberData = kanbanBoard.Members.Find(km => km.UserId == Context.User.Identity.Name);
if (memberData == null)
{
Context.Abort();
return;
}
await Groups.AddToGroupAsync(Context.ConnectionId, $"Group/${request.GroupId}/Kanban/${request.BoardId}");
}
/// <summary>
/// upsert edge to arango edge collection (T)
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="arangoDatabase"></param>
/// <param name="item"></param>
public static void UpsertEdge <T>(this IArangoDatabase arangoDatabase, T item) where T : EdgeBase
{
arangoDatabase.Query().Upsert <T>(
arangoDB_item_for_upsert => new { _key = item._key },
_ => item,
(_, old) => item)
.Execute();
}
/// <summary>
/// upsert an item to arango collection and ignore the null properties
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="arangoDatabase"></param>
/// <param name="item"></param>
public static void UpsertIgnoreNull <T>(this IArangoDatabase arangoDatabase, T item) where T : VertexBase
{
JObject jObject = JObject.FromObject(item, IgnoreNullJsonSerializer);
arangoDatabase.Query().Upsert <T>(
arangoDB_item_for_upsert => new { _key = item._key },
_ => jObject,
(_, old) => jObject)
.Execute();
}
/// <summary>
/// get edge by key
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="arangoDatabase"></param>
/// <param name="key"></param>
/// <returns></returns>
public static T GetEdgeByKey <T>(this IArangoDatabase arangoDatabase, object key) where T : EdgeBase
{
return(arangoDatabase.Query <T>()
.Filter(item => item._key == key.ToString())
.Return(item => item)
.ToListAsync()
.GetAwaiter()
.GetResult()
.FirstOrDefault());
}
/// <summary>
/// test if the arango collection has this key
/// </summary>
/// <typeparam name="T"></typeparam>
/// <param name="arangoDatabase"></param>
/// <param name="key"></param>
/// <returns></returns>
public static bool HasKey <T>(this IArangoDatabase arangoDatabase, object key) where T : GraphBase
{
return(arangoDatabase.Query <T>()
.Filter((item) => item._id == key.ToString())
.Return(item => item._key)
.ToListAsync()
.GetAwaiter()
.GetResult()
.Count > 0);
}
/// <summary>
/// Search from the originId to min and max level via any direction of edges
/// </summary>
/// <param name="arangoDatabase"></param>
/// <param name="originId"></param>
/// <param name="minDepth"></param>
/// <param name="maxDepth"></param>
/// <param name="edges"></param>
/// <returns></returns>
public static IEnumerable <TraversalData <JObject, EdgeBase> > TraverseGraphFromOrigin(
this IArangoDatabase arangoDatabase,
string originId,
int minDepth,
int maxDepth,
params Type[] edges)
{
var query = arangoDatabase.Query()
.Traversal <JObject, EdgeBase>(originId)
.Depth(minDepth, maxDepth);
if (edges != null)
{
foreach (var type in edges)
{
query = query.Edge(type.Name);
}
}
return(query.AnyDirection()
.Return(g => g)
.ToListAsync()
.GetAwaiter()
.GetResult());
}
public async Task DownloadJob(string url, JobAnalysisEntry jobAnalysisEntry, IArangoDatabase client, Dictionary <string, Job> jobs)
{
var match = jobLink.Match(url);
var key = match.Groups[1].Value;
// if job has been saved in database, don't do it again.
bool exists = false;
{
bool success = false;
int retry = 0;
while (!success && retry < _searchOptions.MaxRetry)
{
try
{
retry++;
var jobsFound = await client.Query <Job>().Filter(j => j._key == key).ToListAsync();
var first = jobsFound.FirstOrDefault();
if (first != null && first.Description != null && first.Description != "")
{
jobs.Add(first._key, first);
exists = true;
}
success = true;
}
catch (Exception ex)
{
_logger.Error(ex, $"failed to access job({key}) from arango. {retry} of {_searchOptions.MaxRetry} attempts.");
}
}
}
if (!exists)
{
bool success = false;
int retry = 0;
HtmlDocument document = null;
while (!success && retry < _searchOptions.MaxRetry)
{
try
{
retry++;
HtmlNode jobDescription = null;
HtmlNode jobTitle = null;
HtmlNode workType = null;
HtmlNode publishDate = null;
HtmlNode location = null;
HtmlNode salary = null;
HtmlNode city = null;
int chromeRetry = 0;
do
{
try
{
chromeRetry++;
document = WebExtensions.LoadPage(url);
HtmlNode jobBox = document.DocumentNode.DescendantsAndSelf()
.Where(n => n.Name.ToLower() == "div" && n.GetAttributeValue("data-automation", null) == "jobDescription")
.FirstOrDefault();
jobDescription = jobBox.DescendantsAndSelf()
.Where(n => n.Name.ToLower() == "div" && n.HasClass("templatetext")).FirstOrDefault();
jobTitle = jobBox.DescendantsAndSelf()
.Where(n => n.HasClass("jobtitle")).FirstOrDefault();
HtmlNode infoHeader = document.DocumentNode.DescendantsAndSelf()
.Where(n => n.Name.ToLower() == "section" && n.GetAttributeValue("aria-labelledby", null) == "jobInfoHeader")
.FirstOrDefault();
publishDate = infoHeader.DescendantsAndSelf()
.Where(n => n.Name.ToLower() == "dd" && n.GetAttributeValue("data-automation", null) == "job-detail-date").FirstOrDefault();
workType = infoHeader.DescendantsAndSelf()
.Where(n => n.Name.ToLower() == "dd" && n.GetAttributeValue("data-automation", null) == "job-detail-work-type").FirstOrDefault();
HtmlNode dataList = infoHeader.DescendantsAndSelf()
.Where(n => n.Name.ToLower() == "dl").FirstOrDefault();
var dataTitles = infoHeader.DescendantsAndSelf()
.Where(n => n.Name.ToLower() == "dt").ToList();
var dataDetails = infoHeader.DescendantsAndSelf()
.Where(n => n.Name.ToLower() == "dd").ToList();
for (int i = 0; i < Math.Min(dataTitles.Count, dataDetails.Count); i++)
{
string innerText = dataTitles[i].InnerText;
if (innerText.Contains("Location"))
{
location = dataDetails[i];
city = location.DescendantsAndSelf().Where(n => n.Name.ToLower() == "strong").FirstOrDefault();
}
else if (innerText.Contains("Salary"))
{
salary = dataDetails[i];
}
}
}
catch (Exception ex)
{
_logger.Error(ex, $"failed to get jobDescription element from url {url}. {chromeRetry} of {_searchOptions.MaxRetry} attempts.");
}
} while (jobDescription == null && chromeRetry < _searchOptions.MaxRetry);
Job job = new Job()
{
_key = key,
Description = jobDescription.InnerText.HtmlDecode(),
TimeStamp = DateTime.Now
};
if (jobTitle != null)
{
job.Title = jobTitle.InnerText.HtmlDecode();
}
if (location != null)
{
job.Location = location.InnerText.HtmlDecode();
if (city != null)
{
job.City = city.InnerText.HtmlDecode();
}
}
if (salary != null)
{
job.RawSalary = salary.InnerText.HtmlDecode();
}
if (workType != null)
{
job.WorkType = workType.InnerText.HtmlDecode();
}
if (publishDate != null)
{
job.RawPublishDate = publishDate.InnerText.HtmlDecode();
DateTime publish;
if (DateTime.TryParse(job.RawPublishDate, out publish))
{
job.PublishDate = publish;
}
}
if (jobs.ContainsKey(job._key))
{
jobs[job._key] = job;
}
else
{
jobs.Add(job._key, job);
}
// save to arango db
client.UpsertIgnoreNull(job);
success = true;
}
catch (Exception ex)
{
_logger.Error(ex, $"failed to get job details from url {url}. {retry} of {_searchOptions.MaxRetry} attempts.");
}
}
}
}
