Пример #1
0
        internal void GetPageInfoForAllCompetitors()
        {
            WriteToUsageLogFile("FacebookFanInfo.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Started GetPageInfoForAllCompetitors");

            FacebookDataSource fds = DroneDataSource as FacebookDataSource;

            if (!Object.Equals(null, fds))
            {
                foreach (Competitor item in fds.GetCompetitorAccounts())
                {
                    if (item.FacebookID != 0)
                    {
                        Page fpo = GetPageInfoByCompany(item.FacebookID);
                        if (!Object.Equals(fpo, null))
                        {
                            FacebookDataComponent fdc = new FacebookDataComponent();
                            fdc.FBPage = fpo;
                            DroneDataSource.Process(fdc);
                        }
                        else
                        {
                            Shared.Utility.WriteToLogFile(String.Format("Facebook_NoPage_{0:M_d_yyyy}", DateTime.Today) + ".log", string.Format("{0},{1}", fpo.Name, fpo.Id));
                        }
                    }
                }
            }

            WriteToUsageLogFile("FacebookFanInfo.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Completed GetPageInfoForAllCompetitors");
        }
Пример #2
0
        private void GetChannelData()
        {
            YouTubeDataSource yds = DroneDataSource as YouTubeDataSource;

            if (!Object.Equals(null, yds))
            {
                foreach (Competitor account in yds.GetCompetitorAccounts())
                {
                    if (!String.IsNullOrEmpty(account.YouTubeAccount))
                    {
                        ChannelManager cManager = new ChannelManager(new KeyValuePair <int, string>(account.ID, account.YouTubeAccount)
                                                                     , Utility.ApplicationName
                                                                     , XMLUtility.GetTextFromAccountNode(Xml, "token/devkey"));
                        Channel chan = cManager.GetUserChannel();

                        if (!Object.Equals(chan, null))
                        {
                            YouTubeDataComponent ydc = new YouTubeDataComponent();
                            ydc.YouTubeChannel = chan;
                            DroneDataSource.Process(ydc);
                        }
                        else
                        {
                            Utility.WriteToLogFile(String.Format("YouTube_ChannelNull_{0:M_d_yyyy}", DateTime.Today) + ".log", "No account info for: " + account.YouTubeAccount);
                        }
                    }
                }
            }
        }
Пример #3
0
        private void GetAllWebHosts()
        {
            MarketShareDataSource mds = DroneDataSource as MarketShareDataSource;

            if (!Object.Equals(null, mds))
            {
                using (Dig dig = new Dig())
                {
                    int maxParallel = XMLUtility.GetTextFromAccountNode(Xml, ProcessorName + "/maxparallel").ConvertStringToInt(1);

                    Parallel.ForEach(mds.GetAllCompanies(MarketShareTypeBitMaskEnum.WebHost, XMLUtility.GetIntFromAccountNode(Xml, ProcessorName + "/recordcount"))
                                     , new ParallelOptions {
                        MaxDegreeOfParallelism = maxParallel
                    }
                                     , (company, state) =>
                    {
                        try
                        {
                            if (Context.ShuttingDown)
                            {
                                state.Break();
                            }

                            MarketShareDataType marketType = new MarketShareDataType();
                            string url = Utility.CleanUrl(company.Uri.ToString());

                            if (!String.IsNullOrWhiteSpace(url))
                            {
                                WriteToUsageLogFile("Domain:" + url, string.Format("Executing {0}.{1}", ComponentTypeName, MethodInfo.GetCurrentMethod().Name), true);

                                marketType.Value      = dig.GetWebHostName(url);
                                marketType.DomainId   = company.DomainId;
                                marketType.SampleDate = company.DomainAttributes["SampleDate"];
                                marketType.TypeId     = (int)MarketShareDataTypeEnum.WebHost;
                                marketType.BitMaskId  = (int)MarketShareTypeBitMaskEnum.WebHost;
                                marketType.UniqueID   = g;

                                if (!string.IsNullOrWhiteSpace(marketType.Value))
                                {
                                    MarketShareDataComponent dc = new MarketShareDataComponent();
                                    dc.MarketShareType          = marketType;
                                    DroneDataSource.Process(dc);
                                }
                                else
                                {
                                    Utility.WriteToLogFile(String.Format("SmallBiz_NoWebHost_{0:M_d_yyyy}", DateTime.Today) + ".log", string.Format("Domain: ,{0}", url));
                                }
                            }
                        }
                        catch (Exception e)
                        {
                            ExceptionExtensions.LogError(e, "WebHost.GetAllWebHostsParallel", "Domain: " + company.Uri.ToString());
                        }
                    });
                }
            }
        }
Пример #4
0
        private void GetSquatterData()
        {
            ChannelManager cManager = new ChannelManager();
            Channel        chanVids = cManager.GetSquatterVideos(GetAllSquatterVideoIds());

            if (!Object.Equals(chanVids, null))
            {
                YouTubeDataComponent ydc = new YouTubeDataComponent();
                ydc.YouTubeChannel = chanVids;
                DroneDataSource.Process(ydc);
            }
            else
            {
                Utility.WriteToLogFile(String.Format("YouTube_SquattersNull_{0:M_d_yyyy}", DateTime.Today) + ".log", "No videos found");
            }
        }
Пример #5
0
        /// <summary>
        /// Gets registrars for all companies in our database
        /// </summary>
        internal void GetAllEmailHosts()
        {
            MarketShareDataSource mds = DroneDataSource as MarketShareDataSource;

            if (!Object.Equals(null, mds))
            {
                using (Dig dig = new Dig())
                {
                    var list = mds.GetAllCompanies(MarketShareTypeBitMaskEnum.EmailHost, XMLUtility.GetIntFromAccountNode(Xml, "emailhost/recordcount"));

                    foreach (var company in list)
                    {
                        try
                        {
                            MarketShareDataType marketType = new MarketShareDataType();
                            string url = Utility.CleanUrl(company.Uri.ToString());

                            if (!String.IsNullOrWhiteSpace(url))
                            {
                                WriteToUsageLogFile("Domain:" + url, string.Format("Executing {0}.{1}", ComponentTypeName, MethodInfo.GetCurrentMethod().Name), true);

                                marketType.Value      = dig.GetEmailHostName(url);
                                marketType.DomainId   = company.DomainId;
                                marketType.SampleDate = company.DomainAttributes["SampleDate"];
                                marketType.TypeId     = (int)MarketShareDataTypeEnum.EmailHost;
                                marketType.BitMaskId  = (int)MarketShareTypeBitMaskEnum.EmailHost;

                                if (!string.IsNullOrWhiteSpace(marketType.Value))
                                {
                                    MarketShareDataComponent dc = new MarketShareDataComponent();
                                    dc.MarketShareType = marketType;
                                    DroneDataSource.Process(dc);
                                }
                                else
                                {
                                    Utility.WriteToLogFile(String.Format("SmallBiz_NoEmailHost_{0:M_d_yyyy}", DateTime.Today) + ".log", string.Format("Domain: ,{0}", url));
                                }
                            }
                        }
                        catch (Exception e)
                        {
                            ExceptionExtensions.LogError(e, "EmailHost.GetAllEmailHosts", "Domain: " + company.Uri.ToString());
                        }
                    }
                }
            }
        }
Пример #6
0
        internal void Process(string fileName)
        {
            Dig           digMgr = new Dig();
            List <string> tempLines;

            while ((tempLines = Utility.ReadLinesFromFile(fileName, 100, true)).Count() > 0)
            {
                int maxParallel = XMLUtility.GetTextFromAccountNode(Xml, ProcessorName + "/maxparallel").ConvertStringToInt(1);

                Parallel.ForEach(tempLines, new ParallelOptions {
                    MaxDegreeOfParallelism = maxParallel
                }, (tempLine) =>
                {
                    try
                    {
                        if (!String.IsNullOrWhiteSpace(tempLine))
                        {
                            string[] companyEntry = tempLine.Split('|');
                            CompanyRoot cr        = GetFullCompany(companyEntry[1], digMgr);

                            //Add to MSMQ, add will call receive and then ProcessMessage below when to add to DB asynchronously
                            if (!Object.Equals(null, cr) && !string.IsNullOrEmpty(cr.homepage_url))
                            {
                                WriteToUsageLogFile("Domain:" + cr.homepage_url, string.Format("Executing {0}.{1}", ComponentTypeName, MethodInfo.GetCurrentMethod().Name), true);
                                CrunchbaseDataComponent cdc = new CrunchbaseDataComponent();
                                cdc.CompanyLocal            = cr;
                                DroneDataSource.Process(cdc);
                            }
                            else
                            {
                                if (Object.Equals(null, cr) && companyEntry.Length < 3)
                                {
                                    Utility.AddLineToFile(fileName, tempLine + "|retry");
                                }
                            }
                            cr = null;
                        }
                    }
                    catch (Exception e)
                    {
                        ExceptionExtensions.LogError(e, "Crunchbase.Process", "Permalink: " + tempLine);
                    }
                });
            }
        }
Пример #7
0
        /// <summary>
        /// Main method that gathers data
        /// </summary>
        /// <param name="context">iDroneContext</param>
        public override void GetData(object context)
        {
            try
            {
                BaseContext cont = context as BaseContext;

                if (!Object.Equals(cont, null))
                {
                    SetContextStatus("processing", cont);

                    DateTime tempNextRun = cont.NextRun;
                    SetNextRunIntervalByNode(ProcessorName, cont);

                    if (XMLUtility.IsEnabled(Xml) && XMLUtility.IsComponentEnabled(Xml, ProcessorName))
                    {
                        WriteToUsageLogFile("TwitterFollower.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Started GetTwitterFollowers");

                        //wait 5 min for other twitter processes to finish so we don't get rate limited
                        Thread.Sleep(300000);

                        //do work
                        TwitterDataComponent _dataComponent = DroneDataComponent as TwitterDataComponent;
                        UserManager          um             = new UserManager();
                        OAuthTokens          oat            = Utility.GetOAuthToken(Xml);

                        _dataComponent.TwitterUserList = um.GetAllFollowers(oat);

                        DroneDataSource.Process(_dataComponent);

                        WriteToUsageLogFile("TwitterFollower.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Completed GetTwitterFollowers");
                    }

                    SetContextStatus("waiting", cont);
                }

                RaiseProcessingCompletedEvent(new EventArgs());
            }
            catch (Exception e)
            {
                ExceptionExtensions.LogError(e, "Drone.Twitter.Components.TwitterFollower.GetData()");
            }
        }
Пример #8
0
        internal void GetFanDemographics()
        {
            WriteToUsageLogFile("FacebookFanInfo.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Started GetFanDemographics");

            Graph g = new Graph();
            FacebookDataComponent fdc;
            string accountId   = XMLUtility.GetTextFromAccountNode(Xml, "id");
            string accessToken = XMLUtility.GetTextFromAccountNode(Xml, "accesstoken");

            Demographic <Country> country = g.GetFanDemographics <Demographic <Country> >(accountId, accessToken, "page_fans_country");

            if (!Object.Equals(country, null))
            {
                fdc = new FacebookDataComponent();
                fdc.CountryDemographic = country;
                DroneDataSource.Process(fdc);
            }

            Demographic <Locale> locale = g.GetFanDemographics <Demographic <Locale> >(accountId, accessToken, "page_fans_locale");

            if (!Object.Equals(locale, null))
            {
                fdc = new FacebookDataComponent();
                fdc.LocaleDemographic = locale;
                DroneDataSource.Process(fdc);
            }

            Demographic <Gender> gender = g.GetFanDemographics <Demographic <Gender> >(accountId, accessToken, "page_fans_gender_age");

            if (!Object.Equals(gender, null))
            {
                fdc = new FacebookDataComponent();
                fdc.GenderDemographic = gender;
                DroneDataSource.Process(fdc);
            }

            WriteToUsageLogFile("FacebookFanInfo.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Completed GetFanDemographics");
        }
Пример #9
0
        /// <summary>
        /// Main method that gathers data
        /// </summary>
        /// <param name="context">iDroneContext</param>
        public override void GetData(object context)
        {
            try
            {
                BaseContext cont = context as BaseContext;

                if (!Object.Equals(cont, null))
                {
                    SetContextStatus("processing", cont);

                    DateTime tempNextRun = cont.NextRun;
                    SetNextRunIntervalByNode(ProcessorName, cont);

                    if (DateTime.Compare(tempNextRun, DateTime.MinValue) != 0 && XMLUtility.IsEnabled(Xml) && XMLUtility.IsComponentEnabled(Xml, ProcessorName))
                    {
                        WriteToUsageLogFile("TwitterUser.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Started GetTwitterUsers Competitors");

                        //do work
                        TwitterDataComponent _dataComponent = DroneDataComponent as TwitterDataComponent;
                        _dataComponent.TwitterUserList = GetTwitterUsers();

                        DroneDataSource.Process(_dataComponent);

                        WriteToUsageLogFile("TwitterUser.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Completed GetTwitterUsers Competitors");
                    }

                    SetContextStatus("waiting", cont);
                }

                RaiseProcessingCompletedEvent(new EventArgs());
            }
            catch (Exception e)
            {
                ExceptionExtensions.LogError(e, "Drone.Twitter.Components.TwitterUser.GetData()");
            }
        }
Пример #10
0
        /// <summary>
        /// Get all tweets with "GoDaddy" in them.
        /// 100 per page, up to 15 pages if we loop and make seperate calls for each page
        /// </summary>
        internal Dictionary <int, List <Status> > GetAllMentionsByQuery(TwitterContext cont)
        {
            Dictionary <int, List <Status> > allTwitterMentions = new Dictionary <int, List <Status> >();

            if (XMLUtility.IsComponentEnabled(Xml, ProcessorName))
            {
                int countPerPage; int pageCount;
                TwitterDataSource data = new TwitterDataSource();
                TweetManager      tm   = new TweetManager();

                XMLUtility.GetPageResultCounts(Xml, ProcessorName, out countPerPage, out pageCount, 100, 3);
                List <Keyword> queries    = data.GetTweetQueries();
                bool           useSinceId = XMLUtility.UseSinceId(Xml, ProcessorName);

                //create backup of current keys in case of failure at db level
                lock (cont) cont.prevRunLatestTweetIDs = cont.LatestTweetIDs.ToDictionary(entry => entry.Key, entry => entry.Value);

                foreach (Keyword item in queries)
                {
                    try
                    {
                        long sinceId = 0;
                        if (useSinceId)
                        {
                            //get the last recorded id for this query and use it
                            if (!Object.Equals(cont, null))
                            {
                                lock (cont)
                                {
                                    if (cont.LatestTweetIDs.ContainsKey(item.KeywordId))
                                    {
                                        cont.LatestTweetIDs.TryGetValue(item.KeywordId, out sinceId);
                                    }
                                }
                            }
                        }

                        //call the mention object in the API wrapper
                        TwitterDataComponent _dataComponent = DroneDataComponent as TwitterDataComponent;
                        KeywordStatus        ks             = new KeywordStatus();
                        ks.KeywordId = item.KeywordId;

                        ks.StatusList = tm.GetTweetsByQuery(countPerPage, item.KeywordValue + (sinceId > 0 ? "&since_id=" + sinceId : string.Empty), Utility.GetOAuthToken(Xml));
                        allTwitterMentions.Add(ks.KeywordId, ks.StatusList);

                        _dataComponent.KeywordStatus = ks;
                        DroneDataSource.Process(_dataComponent);

                        //if there was a failure saving to the db, reset the since id to gather and try again
                        if (_dataComponent.SaveFailure)
                        {
                            lock (cont) cont.LatestTweetIDs = cont.prevRunLatestTweetIDs.ToDictionary(entry => entry.Key, entry => entry.Value);
                        }

                        //get the last id for this query and store it
                        if (!Object.Equals(cont, null) && allTwitterMentions.ContainsKey(item.KeywordId) && allTwitterMentions[item.KeywordId].Count > 0)
                        {
                            long latestID;
                            long.TryParse(allTwitterMentions[item.KeywordId][0].id.ToString(), out latestID);

                            lock (cont)
                            {
                                if (cont.LatestTweetIDs.ContainsKey(item.KeywordId))
                                {
                                    cont.LatestTweetIDs[item.KeywordId] = latestID;
                                }
                                else
                                {
                                    cont.LatestTweetIDs.Add(item.KeywordId, latestID);
                                }
                            }
                        }
                    }
                    catch (Exception e)
                    {
                        ExceptionExtensions.LogError(e, "Twitter.GetAllMentionsByQuery()", "Keyword name: " + item.KeywordName);
                    }
                }
            }
            return(allTwitterMentions);
        }