private void ClearDatabase() { db.Query <Person>().Remove().Execute(); db.Query <Follow>().Remove().Execute(); db.Query <Host>().Remove().Execute(); if (db.ListGraphs().Count(x => x.Key == "SocialGraph") != 0) { db.Graph("SocialGraph").Drop(); } }
/// <summary> /// upsert edge to arango collection (T) and ensure the to side is unique /// </summary> /// <typeparam name="T"></typeparam> /// <param name="arangoDatabase"></param> /// <param name="item"></param> public static void UpsertEdgeUniqueTo <T>(this IArangoDatabase arangoDatabase, T item) where T : EdgeBase { arangoDatabase.Query <T>() .Filter(item_to_remove => item_to_remove._to == item._to) .Remove(item_to_remove => new { _key = item_to_remove._key }) .Execute(); arangoDatabase.Query().Upsert <T>( arangoDB_item_for_upsert => new { _key = item._key }, _ => item, (_, old) => item) .Execute(); }
/// <summary> /// Search from the originId to min and max level via any direction of edges /// </summary> /// <param name="arangoDatabase"></param> /// <param name="originId"></param> /// <param name="minDepth"></param> /// <param name="maxDepth"></param> /// <param name="edges"></param> /// <returns></returns> public static IEnumerable <EdgeTrace <TVertex, TEdge> > TraverseEdgeFromOrigin <TVertex, TEdge>( this IArangoDatabase arangoDatabase, string originId, int minDepth, int maxDepth, params Type[] edges) where TVertex : VertexBase where TEdge : EdgeBase { var query = arangoDatabase.Query() .Traversal <TVertex, TEdge>(originId) .Depth(minDepth, maxDepth); if (edges != null) { foreach (var type in edges) { query = query.Edge(type.Name); } } return(query.AnyDirection() .Return(g => new EdgeTrace <TVertex, TEdge>() { Edge = g.Edge, Vertex = g.Vertex }) .ToListAsync() .GetAwaiter() .GetResult()); }
/// <summary> /// Search from the originId to min and max level via any direction of edges /// </summary> /// <param name="arangoDatabase"></param> /// <param name="originId"></param> /// <param name="minDepth"></param> /// <param name="maxDepth"></param> /// <param name="edges"></param> /// <returns></returns> public static IEnumerable <RawEdgeTrace> TraverseRawFromOrigin( this IArangoDatabase arangoDatabase, string originId, int minDepth, int maxDepth, params Type[] edges) { var query = arangoDatabase.Query() .Traversal <JObject, JObject>(originId) .Depth(minDepth, maxDepth); if (edges != null) { foreach (var type in edges) { query = query.Edge(type.Name); } } return(query.AnyDirection() .Return(g => new RawEdgeTrace() { Edge = g.Edge, Vertex = g.Vertex }) .ToListAsync() .GetAwaiter() .GetResult()); }
/// <summary> /// Search from the originId to min and max level via any direction of edges /// </summary> /// <typeparam name="TTarget"></typeparam> /// <param name="arangoDatabase"></param> /// <param name="originId"></param> /// <param name="minDepth"></param> /// <param name="maxDepth"></param> /// <param name="edges"></param> /// <returns></returns> public static IEnumerable <TTarget> TraverseFromOrigin <TTarget>( this IArangoDatabase arangoDatabase, string originId, int minDepth, int maxDepth, params Type[] edges) where TTarget : VertexBase { var query = arangoDatabase.Query() .Traversal <TTarget, EdgeBase>(originId) .Depth(minDepth, maxDepth); if (edges != null) { foreach (var type in edges) { query = query.Edge(type.Name); } } return(query.AnyDirection() .Filter(g => AQL.Split(g.Vertex._id, "/")[0] == typeof(TTarget).Name) .Return(g => g.Vertex) .ToListAsync() .GetAwaiter() .GetResult()); }
public async Task JoinGroup(JoinKanbanHubRequest request) { var group = await(from g in Database.Query <Group>() where g.Key == request.GroupId select g).FirstOrDefaultAsync(); if (group == null) { Context.Abort(); return; } var kanbanBoard = (from kb in @group.KanbanBoards where kb.Id == request.BoardId select kb).FirstOrDefault(); if (kanbanBoard == null) { Context.Abort(); return; } var memberData = kanbanBoard.Members.Find(km => km.UserId == Context.User.Identity.Name); if (memberData == null) { Context.Abort(); return; } await Groups.AddToGroupAsync(Context.ConnectionId, $"Group/${request.GroupId}/Kanban/${request.BoardId}"); }
/// <summary> /// upsert edge to arango edge collection (T) /// </summary> /// <typeparam name="T"></typeparam> /// <param name="arangoDatabase"></param> /// <param name="item"></param> public static void UpsertEdge <T>(this IArangoDatabase arangoDatabase, T item) where T : EdgeBase { arangoDatabase.Query().Upsert <T>( arangoDB_item_for_upsert => new { _key = item._key }, _ => item, (_, old) => item) .Execute(); }
/// <summary> /// upsert an item to arango collection and ignore the null properties /// </summary> /// <typeparam name="T"></typeparam> /// <param name="arangoDatabase"></param> /// <param name="item"></param> public static void UpsertIgnoreNull <T>(this IArangoDatabase arangoDatabase, T item) where T : VertexBase { JObject jObject = JObject.FromObject(item, IgnoreNullJsonSerializer); arangoDatabase.Query().Upsert <T>( arangoDB_item_for_upsert => new { _key = item._key }, _ => jObject, (_, old) => jObject) .Execute(); }
/// <summary> /// get edge by key /// </summary> /// <typeparam name="T"></typeparam> /// <param name="arangoDatabase"></param> /// <param name="key"></param> /// <returns></returns> public static T GetEdgeByKey <T>(this IArangoDatabase arangoDatabase, object key) where T : EdgeBase { return(arangoDatabase.Query <T>() .Filter(item => item._key == key.ToString()) .Return(item => item) .ToListAsync() .GetAwaiter() .GetResult() .FirstOrDefault()); }
/// <summary> /// test if the arango collection has this key /// </summary> /// <typeparam name="T"></typeparam> /// <param name="arangoDatabase"></param> /// <param name="key"></param> /// <returns></returns> public static bool HasKey <T>(this IArangoDatabase arangoDatabase, object key) where T : GraphBase { return(arangoDatabase.Query <T>() .Filter((item) => item._id == key.ToString()) .Return(item => item._key) .ToListAsync() .GetAwaiter() .GetResult() .Count > 0); }
/// <summary> /// Search from the originId to min and max level via any direction of edges /// </summary> /// <param name="arangoDatabase"></param> /// <param name="originId"></param> /// <param name="minDepth"></param> /// <param name="maxDepth"></param> /// <param name="edges"></param> /// <returns></returns> public static IEnumerable <TraversalData <JObject, EdgeBase> > TraverseGraphFromOrigin( this IArangoDatabase arangoDatabase, string originId, int minDepth, int maxDepth, params Type[] edges) { var query = arangoDatabase.Query() .Traversal <JObject, EdgeBase>(originId) .Depth(minDepth, maxDepth); if (edges != null) { foreach (var type in edges) { query = query.Edge(type.Name); } } return(query.AnyDirection() .Return(g => g) .ToListAsync() .GetAwaiter() .GetResult()); }
public async Task DownloadJob(string url, JobAnalysisEntry jobAnalysisEntry, IArangoDatabase client, Dictionary <string, Job> jobs) { var match = jobLink.Match(url); var key = match.Groups[1].Value; // if job has been saved in database, don't do it again. bool exists = false; { bool success = false; int retry = 0; while (!success && retry < _searchOptions.MaxRetry) { try { retry++; var jobsFound = await client.Query <Job>().Filter(j => j._key == key).ToListAsync(); var first = jobsFound.FirstOrDefault(); if (first != null && first.Description != null && first.Description != "") { jobs.Add(first._key, first); exists = true; } success = true; } catch (Exception ex) { _logger.Error(ex, $"failed to access job({key}) from arango. {retry} of {_searchOptions.MaxRetry} attempts."); } } } if (!exists) { bool success = false; int retry = 0; HtmlDocument document = null; while (!success && retry < _searchOptions.MaxRetry) { try { retry++; HtmlNode jobDescription = null; HtmlNode jobTitle = null; HtmlNode workType = null; HtmlNode publishDate = null; HtmlNode location = null; HtmlNode salary = null; HtmlNode city = null; int chromeRetry = 0; do { try { chromeRetry++; document = WebExtensions.LoadPage(url); HtmlNode jobBox = document.DocumentNode.DescendantsAndSelf() .Where(n => n.Name.ToLower() == "div" && n.GetAttributeValue("data-automation", null) == "jobDescription") .FirstOrDefault(); jobDescription = jobBox.DescendantsAndSelf() .Where(n => n.Name.ToLower() == "div" && n.HasClass("templatetext")).FirstOrDefault(); jobTitle = jobBox.DescendantsAndSelf() .Where(n => n.HasClass("jobtitle")).FirstOrDefault(); HtmlNode infoHeader = document.DocumentNode.DescendantsAndSelf() .Where(n => n.Name.ToLower() == "section" && n.GetAttributeValue("aria-labelledby", null) == "jobInfoHeader") .FirstOrDefault(); publishDate = infoHeader.DescendantsAndSelf() .Where(n => n.Name.ToLower() == "dd" && n.GetAttributeValue("data-automation", null) == "job-detail-date").FirstOrDefault(); workType = infoHeader.DescendantsAndSelf() .Where(n => n.Name.ToLower() == "dd" && n.GetAttributeValue("data-automation", null) == "job-detail-work-type").FirstOrDefault(); HtmlNode dataList = infoHeader.DescendantsAndSelf() .Where(n => n.Name.ToLower() == "dl").FirstOrDefault(); var dataTitles = infoHeader.DescendantsAndSelf() .Where(n => n.Name.ToLower() == "dt").ToList(); var dataDetails = infoHeader.DescendantsAndSelf() .Where(n => n.Name.ToLower() == "dd").ToList(); for (int i = 0; i < Math.Min(dataTitles.Count, dataDetails.Count); i++) { string innerText = dataTitles[i].InnerText; if (innerText.Contains("Location")) { location = dataDetails[i]; city = location.DescendantsAndSelf().Where(n => n.Name.ToLower() == "strong").FirstOrDefault(); } else if (innerText.Contains("Salary")) { salary = dataDetails[i]; } } } catch (Exception ex) { _logger.Error(ex, $"failed to get jobDescription element from url {url}. {chromeRetry} of {_searchOptions.MaxRetry} attempts."); } } while (jobDescription == null && chromeRetry < _searchOptions.MaxRetry); Job job = new Job() { _key = key, Description = jobDescription.InnerText.HtmlDecode(), TimeStamp = DateTime.Now }; if (jobTitle != null) { job.Title = jobTitle.InnerText.HtmlDecode(); } if (location != null) { job.Location = location.InnerText.HtmlDecode(); if (city != null) { job.City = city.InnerText.HtmlDecode(); } } if (salary != null) { job.RawSalary = salary.InnerText.HtmlDecode(); } if (workType != null) { job.WorkType = workType.InnerText.HtmlDecode(); } if (publishDate != null) { job.RawPublishDate = publishDate.InnerText.HtmlDecode(); DateTime publish; if (DateTime.TryParse(job.RawPublishDate, out publish)) { job.PublishDate = publish; } } if (jobs.ContainsKey(job._key)) { jobs[job._key] = job; } else { jobs.Add(job._key, job); } // save to arango db client.UpsertIgnoreNull(job); success = true; } catch (Exception ex) { _logger.Error(ex, $"failed to get job details from url {url}. {retry} of {_searchOptions.MaxRetry} attempts."); } } } }