Пример #1
0
        private void ClearDatabase()
        {
            db.Query <Person>().Remove().Execute();
            db.Query <Follow>().Remove().Execute();
            db.Query <Host>().Remove().Execute();

            if (db.ListGraphs().Count(x => x.Key == "SocialGraph") != 0)
            {
                db.Graph("SocialGraph").Drop();
            }
        }
Пример #2
0
 /// <summary>
 /// upsert edge to arango collection (T) and ensure the to side is unique
 /// </summary>
 /// <typeparam name="T"></typeparam>
 /// <param name="arangoDatabase"></param>
 /// <param name="item"></param>
 public static void UpsertEdgeUniqueTo <T>(this IArangoDatabase arangoDatabase, T item) where T : EdgeBase
 {
     arangoDatabase.Query <T>()
     .Filter(item_to_remove => item_to_remove._to == item._to)
     .Remove(item_to_remove => new { _key = item_to_remove._key })
     .Execute();
     arangoDatabase.Query().Upsert <T>(
         arangoDB_item_for_upsert => new { _key = item._key },
         _ => item,
         (_, old) => item)
     .Execute();
 }
Пример #3
0
        /// <summary>
        /// Search from the originId to min and max level via any direction of edges
        /// </summary>
        /// <param name="arangoDatabase"></param>
        /// <param name="originId"></param>
        /// <param name="minDepth"></param>
        /// <param name="maxDepth"></param>
        /// <param name="edges"></param>
        /// <returns></returns>
        public static IEnumerable <EdgeTrace <TVertex, TEdge> > TraverseEdgeFromOrigin <TVertex, TEdge>(
            this IArangoDatabase arangoDatabase,
            string originId,
            int minDepth,
            int maxDepth,
            params Type[] edges) where TVertex : VertexBase where TEdge : EdgeBase
        {
            var query = arangoDatabase.Query()
                        .Traversal <TVertex, TEdge>(originId)
                        .Depth(minDepth, maxDepth);

            if (edges != null)
            {
                foreach (var type in edges)
                {
                    query = query.Edge(type.Name);
                }
            }
            return(query.AnyDirection()
                   .Return(g => new EdgeTrace <TVertex, TEdge>()
            {
                Edge = g.Edge, Vertex = g.Vertex
            })
                   .ToListAsync()
                   .GetAwaiter()
                   .GetResult());
        }
Пример #4
0
        /// <summary>
        /// Search from the originId to min and max level via any direction of edges
        /// </summary>
        /// <param name="arangoDatabase"></param>
        /// <param name="originId"></param>
        /// <param name="minDepth"></param>
        /// <param name="maxDepth"></param>
        /// <param name="edges"></param>
        /// <returns></returns>
        public static IEnumerable <RawEdgeTrace> TraverseRawFromOrigin(
            this IArangoDatabase arangoDatabase,
            string originId,
            int minDepth,
            int maxDepth,
            params Type[] edges)
        {
            var query = arangoDatabase.Query()
                        .Traversal <JObject, JObject>(originId)
                        .Depth(minDepth, maxDepth);

            if (edges != null)
            {
                foreach (var type in edges)
                {
                    query = query.Edge(type.Name);
                }
            }
            return(query.AnyDirection()
                   .Return(g => new RawEdgeTrace()
            {
                Edge = g.Edge, Vertex = g.Vertex
            })
                   .ToListAsync()
                   .GetAwaiter()
                   .GetResult());
        }
Пример #5
0
        /// <summary>
        /// Search from the originId to min and max level via any direction of edges
        /// </summary>
        /// <typeparam name="TTarget"></typeparam>
        /// <param name="arangoDatabase"></param>
        /// <param name="originId"></param>
        /// <param name="minDepth"></param>
        /// <param name="maxDepth"></param>
        /// <param name="edges"></param>
        /// <returns></returns>
        public static IEnumerable <TTarget> TraverseFromOrigin <TTarget>(
            this IArangoDatabase arangoDatabase,
            string originId,
            int minDepth,
            int maxDepth,
            params Type[] edges)
            where TTarget : VertexBase
        {
            var query = arangoDatabase.Query()
                        .Traversal <TTarget, EdgeBase>(originId)
                        .Depth(minDepth, maxDepth);

            if (edges != null)
            {
                foreach (var type in edges)
                {
                    query = query.Edge(type.Name);
                }
            }
            return(query.AnyDirection()
                   .Filter(g => AQL.Split(g.Vertex._id, "/")[0] == typeof(TTarget).Name)
                   .Return(g => g.Vertex)
                   .ToListAsync()
                   .GetAwaiter()
                   .GetResult());
        }
Пример #6
0
        public async Task JoinGroup(JoinKanbanHubRequest request)
        {
            var group = await(from g in Database.Query <Group>()
                              where g.Key == request.GroupId
                              select g).FirstOrDefaultAsync();

            if (group == null)
            {
                Context.Abort();
                return;
            }

            var kanbanBoard = (from kb in @group.KanbanBoards where kb.Id == request.BoardId select kb).FirstOrDefault();

            if (kanbanBoard == null)
            {
                Context.Abort();
                return;
            }

            var memberData = kanbanBoard.Members.Find(km => km.UserId == Context.User.Identity.Name);

            if (memberData == null)
            {
                Context.Abort();
                return;
            }


            await Groups.AddToGroupAsync(Context.ConnectionId, $"Group/${request.GroupId}/Kanban/${request.BoardId}");
        }
Пример #7
0
 /// <summary>
 /// upsert edge to arango edge collection (T)
 /// </summary>
 /// <typeparam name="T"></typeparam>
 /// <param name="arangoDatabase"></param>
 /// <param name="item"></param>
 public static void UpsertEdge <T>(this IArangoDatabase arangoDatabase, T item) where T : EdgeBase
 {
     arangoDatabase.Query().Upsert <T>(
         arangoDB_item_for_upsert => new { _key = item._key },
         _ => item,
         (_, old) => item)
     .Execute();
 }
Пример #8
0
        /// <summary>
        /// upsert an item to arango collection and ignore the null properties
        /// </summary>
        /// <typeparam name="T"></typeparam>
        /// <param name="arangoDatabase"></param>
        /// <param name="item"></param>
        public static void UpsertIgnoreNull <T>(this IArangoDatabase arangoDatabase, T item) where T : VertexBase
        {
            JObject jObject = JObject.FromObject(item, IgnoreNullJsonSerializer);

            arangoDatabase.Query().Upsert <T>(
                arangoDB_item_for_upsert => new { _key = item._key },
                _ => jObject,
                (_, old) => jObject)
            .Execute();
        }
Пример #9
0
 /// <summary>
 /// get edge by key
 /// </summary>
 /// <typeparam name="T"></typeparam>
 /// <param name="arangoDatabase"></param>
 /// <param name="key"></param>
 /// <returns></returns>
 public static T GetEdgeByKey <T>(this IArangoDatabase arangoDatabase, object key) where T : EdgeBase
 {
     return(arangoDatabase.Query <T>()
            .Filter(item => item._key == key.ToString())
            .Return(item => item)
            .ToListAsync()
            .GetAwaiter()
            .GetResult()
            .FirstOrDefault());
 }
Пример #10
0
 /// <summary>
 /// test if the arango collection has this key
 /// </summary>
 /// <typeparam name="T"></typeparam>
 /// <param name="arangoDatabase"></param>
 /// <param name="key"></param>
 /// <returns></returns>
 public static bool HasKey <T>(this IArangoDatabase arangoDatabase, object key) where T : GraphBase
 {
     return(arangoDatabase.Query <T>()
            .Filter((item) => item._id == key.ToString())
            .Return(item => item._key)
            .ToListAsync()
            .GetAwaiter()
            .GetResult()
            .Count > 0);
 }
Пример #11
0
        /// <summary>
        /// Search from the originId to min and max level via any direction of edges
        /// </summary>
        /// <param name="arangoDatabase"></param>
        /// <param name="originId"></param>
        /// <param name="minDepth"></param>
        /// <param name="maxDepth"></param>
        /// <param name="edges"></param>
        /// <returns></returns>
        public static IEnumerable <TraversalData <JObject, EdgeBase> > TraverseGraphFromOrigin(
            this IArangoDatabase arangoDatabase,
            string originId,
            int minDepth,
            int maxDepth,
            params Type[] edges)
        {
            var query = arangoDatabase.Query()
                        .Traversal <JObject, EdgeBase>(originId)
                        .Depth(minDepth, maxDepth);

            if (edges != null)
            {
                foreach (var type in edges)
                {
                    query = query.Edge(type.Name);
                }
            }
            return(query.AnyDirection()
                   .Return(g => g)
                   .ToListAsync()
                   .GetAwaiter()
                   .GetResult());
        }
Пример #12
0
        public async Task DownloadJob(string url, JobAnalysisEntry jobAnalysisEntry, IArangoDatabase client, Dictionary <string, Job> jobs)
        {
            var match = jobLink.Match(url);
            var key   = match.Groups[1].Value;

            // if job has been saved in database, don't do it again.
            bool exists = false;

            {
                bool success = false;
                int  retry   = 0;
                while (!success && retry < _searchOptions.MaxRetry)
                {
                    try
                    {
                        retry++;
                        var jobsFound = await client.Query <Job>().Filter(j => j._key == key).ToListAsync();

                        var first = jobsFound.FirstOrDefault();
                        if (first != null && first.Description != null && first.Description != "")
                        {
                            jobs.Add(first._key, first);
                            exists = true;
                        }
                        success = true;
                    }
                    catch (Exception ex)
                    {
                        _logger.Error(ex, $"failed to access job({key}) from arango. {retry} of {_searchOptions.MaxRetry} attempts.");
                    }
                }
            }

            if (!exists)
            {
                bool success = false;
                int  retry   = 0;

                HtmlDocument document = null;

                while (!success && retry < _searchOptions.MaxRetry)
                {
                    try
                    {
                        retry++;

                        HtmlNode jobDescription = null;
                        HtmlNode jobTitle       = null;
                        HtmlNode workType       = null;
                        HtmlNode publishDate    = null;
                        HtmlNode location       = null;
                        HtmlNode salary         = null;
                        HtmlNode city           = null;
                        int      chromeRetry    = 0;

                        do
                        {
                            try
                            {
                                chromeRetry++;

                                document = WebExtensions.LoadPage(url);

                                HtmlNode jobBox = document.DocumentNode.DescendantsAndSelf()
                                                  .Where(n => n.Name.ToLower() == "div" && n.GetAttributeValue("data-automation", null) == "jobDescription")
                                                  .FirstOrDefault();
                                jobDescription = jobBox.DescendantsAndSelf()
                                                 .Where(n => n.Name.ToLower() == "div" && n.HasClass("templatetext")).FirstOrDefault();

                                jobTitle = jobBox.DescendantsAndSelf()
                                           .Where(n => n.HasClass("jobtitle")).FirstOrDefault();

                                HtmlNode infoHeader = document.DocumentNode.DescendantsAndSelf()
                                                      .Where(n => n.Name.ToLower() == "section" && n.GetAttributeValue("aria-labelledby", null) == "jobInfoHeader")
                                                      .FirstOrDefault();

                                publishDate = infoHeader.DescendantsAndSelf()
                                              .Where(n => n.Name.ToLower() == "dd" && n.GetAttributeValue("data-automation", null) == "job-detail-date").FirstOrDefault();

                                workType = infoHeader.DescendantsAndSelf()
                                           .Where(n => n.Name.ToLower() == "dd" && n.GetAttributeValue("data-automation", null) == "job-detail-work-type").FirstOrDefault();

                                HtmlNode dataList = infoHeader.DescendantsAndSelf()
                                                    .Where(n => n.Name.ToLower() == "dl").FirstOrDefault();

                                var dataTitles = infoHeader.DescendantsAndSelf()
                                                 .Where(n => n.Name.ToLower() == "dt").ToList();

                                var dataDetails = infoHeader.DescendantsAndSelf()
                                                  .Where(n => n.Name.ToLower() == "dd").ToList();


                                for (int i = 0; i < Math.Min(dataTitles.Count, dataDetails.Count); i++)
                                {
                                    string innerText = dataTitles[i].InnerText;
                                    if (innerText.Contains("Location"))
                                    {
                                        location = dataDetails[i];
                                        city     = location.DescendantsAndSelf().Where(n => n.Name.ToLower() == "strong").FirstOrDefault();
                                    }
                                    else if (innerText.Contains("Salary"))
                                    {
                                        salary = dataDetails[i];
                                    }
                                }
                            }
                            catch (Exception ex)
                            {
                                _logger.Error(ex, $"failed to get jobDescription element from url {url}. {chromeRetry} of {_searchOptions.MaxRetry} attempts.");
                            }
                        } while (jobDescription == null && chromeRetry < _searchOptions.MaxRetry);

                        Job job = new Job()
                        {
                            _key        = key,
                            Description = jobDescription.InnerText.HtmlDecode(),
                            TimeStamp   = DateTime.Now
                        };

                        if (jobTitle != null)
                        {
                            job.Title = jobTitle.InnerText.HtmlDecode();
                        }

                        if (location != null)
                        {
                            job.Location = location.InnerText.HtmlDecode();
                            if (city != null)
                            {
                                job.City = city.InnerText.HtmlDecode();
                            }
                        }

                        if (salary != null)
                        {
                            job.RawSalary = salary.InnerText.HtmlDecode();
                        }

                        if (workType != null)
                        {
                            job.WorkType = workType.InnerText.HtmlDecode();
                        }

                        if (publishDate != null)
                        {
                            job.RawPublishDate = publishDate.InnerText.HtmlDecode();
                            DateTime publish;
                            if (DateTime.TryParse(job.RawPublishDate, out publish))
                            {
                                job.PublishDate = publish;
                            }
                        }

                        if (jobs.ContainsKey(job._key))
                        {
                            jobs[job._key] = job;
                        }
                        else
                        {
                            jobs.Add(job._key, job);
                        }

                        // save to arango db

                        client.UpsertIgnoreNull(job);

                        success = true;
                    }
                    catch (Exception ex)
                    {
                        _logger.Error(ex, $"failed to get job details from url {url}. {retry} of {_searchOptions.MaxRetry} attempts.");
                    }
                }
            }
        }