public void Facebook_GetFanDemographicsAndInsertQueue() { FacebookFanInfo fg = new FacebookFanInfo(); string accountId = XMLUtility.GetTextFromAccountNode(fg.Xml, "id"); string accessToken = XMLUtility.GetTextFromAccountNode(fg.Xml, "accesstoken"); Demographic <Country> country = _graph.GetFanDemographics <Demographic <Country> >(accountId, accessToken, "page_fans_country"); if (!Object.Equals(country, null)) //fg.Context.MSMQManager.AddToQueue(country, "country"); { } Demographic <Locale> locale = _graph.GetFanDemographics <Demographic <Locale> >(accountId, accessToken, "page_fans_locale"); if (!Object.Equals(locale, null)) //fg.Context.MSMQManager.AddToQueue(locale, "locale"); { } Demographic <Gender> gender = _graph.GetFanDemographics <Demographic <Gender> >(accountId, accessToken, "page_fans_gender_age"); if (!Object.Equals(gender, null)) //fg.Context.MSMQManager.AddToQueue(gender, "gender"); { } Assert.IsNotNull(fg); }
private bool GetBoolFromXMLConfig(string node) { bool boolVal; bool.TryParse(XMLUtility.GetTextFromAccountNode(Xml, node), out boolVal); return(boolVal); }
public override void GetData(object context) { try { BaseContext cont = context as BaseContext; if (!Object.Equals(cont, null)) { SetContextStatus("processing", cont); SetNextRunIntervalByNode(ProcessorName, cont); if (XMLUtility.IsEnabled(Xml) && XMLUtility.IsComponentEnabled(Xml, ProcessorName)) { WriteToUsageLogFile("FacebookInsight.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Started GetInsightInfo"); string accountId = XMLUtility.GetTextFromAccountNode(Xml, "id"); string accessToken = XMLUtility.GetTextFromAccountNode(Xml, "accesstoken"); FBIO = new Graph().GetInsightInfo(accountId, accessToken); WriteToUsageLogFile("FacebookInsight.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Completed GetInsightInfo"); } SetContextStatus("waiting", cont); } RaiseProcessingCompletedEvent(new EventArgs()); } catch (Exception e) { ExceptionExtensions.LogError(e, "FacebookInsight.GetData()"); } }
private void GetChannelData() { YouTubeDataSource yds = DroneDataSource as YouTubeDataSource; if (!Object.Equals(null, yds)) { foreach (Competitor account in yds.GetCompetitorAccounts()) { if (!String.IsNullOrEmpty(account.YouTubeAccount)) { ChannelManager cManager = new ChannelManager(new KeyValuePair <int, string>(account.ID, account.YouTubeAccount) , Utility.ApplicationName , XMLUtility.GetTextFromAccountNode(Xml, "token/devkey")); Channel chan = cManager.GetUserChannel(); if (!Object.Equals(chan, null)) { YouTubeDataComponent ydc = new YouTubeDataComponent(); ydc.YouTubeChannel = chan; DroneDataSource.Process(ydc); } else { Utility.WriteToLogFile(String.Format("YouTube_ChannelNull_{0:M_d_yyyy}", DateTime.Today) + ".log", "No account info for: " + account.YouTubeAccount); } } } } }
public void Crunchbase_GetNodeText() { Crunch t = new Crunch(new MarketShareDataSource()); XmlDocument xmlDoc = t.Xml as XmlDocument; string text = XMLUtility.GetTextFromAccountNode(t.Xml, "crunch/interval"); Assert.AreEqual(text, "saturday"); }
private void GetAllWebHosts() { MarketShareDataSource mds = DroneDataSource as MarketShareDataSource; if (!Object.Equals(null, mds)) { using (Dig dig = new Dig()) { int maxParallel = XMLUtility.GetTextFromAccountNode(Xml, ProcessorName + "/maxparallel").ConvertStringToInt(1); Parallel.ForEach(mds.GetAllCompanies(MarketShareTypeBitMaskEnum.WebHost, XMLUtility.GetIntFromAccountNode(Xml, ProcessorName + "/recordcount")) , new ParallelOptions { MaxDegreeOfParallelism = maxParallel } , (company, state) => { try { if (Context.ShuttingDown) { state.Break(); } MarketShareDataType marketType = new MarketShareDataType(); string url = Utility.CleanUrl(company.Uri.ToString()); if (!String.IsNullOrWhiteSpace(url)) { WriteToUsageLogFile("Domain:" + url, string.Format("Executing {0}.{1}", ComponentTypeName, MethodInfo.GetCurrentMethod().Name), true); marketType.Value = dig.GetWebHostName(url); marketType.DomainId = company.DomainId; marketType.SampleDate = company.DomainAttributes["SampleDate"]; marketType.TypeId = (int)MarketShareDataTypeEnum.WebHost; marketType.BitMaskId = (int)MarketShareTypeBitMaskEnum.WebHost; marketType.UniqueID = g; if (!string.IsNullOrWhiteSpace(marketType.Value)) { MarketShareDataComponent dc = new MarketShareDataComponent(); dc.MarketShareType = marketType; DroneDataSource.Process(dc); } else { Utility.WriteToLogFile(String.Format("SmallBiz_NoWebHost_{0:M_d_yyyy}", DateTime.Today) + ".log", string.Format("Domain: ,{0}", url)); } } } catch (Exception e) { ExceptionExtensions.LogError(e, "WebHost.GetAllWebHostsParallel", "Domain: " + company.Uri.ToString()); } }); } } }
protected void CreateGetRequest() { Uri apiuri = new Uri(XMLUtility.GetTextFromAccountNode(Xml, "apiuri") + "/keywords"); _requestGet = (HttpWebRequest)WebRequest.Create(apiuri); _requestGet.Method = "GET"; _requestGet.ContentType = "application/json"; _requestGet.UseDefaultCredentials = true; _requestGet.PreAuthenticate = true; }
public MarketShareBulkDataSource() { _bulkThreshold = XMLUtility.GetTextFromAccountNode(Xml, "queuemarketshare/bulklimit").ConvertStringToInt(500); _bulkTimeLimit = XMLUtility.GetTextFromAccountNode(Xml, "queuemarketshare/bulktimer").ConvertStringToInt(1000); _isVerboseEnabled = XMLUtility.GetTextFromAccountNode(Xml, "queuemarketshare/verboselogging").ToLower() == "true"; _bulkTimer.Elapsed += _bulkTimer_Elapsed; _bulkTimer.Interval = _bulkTimeLimit; _bulkTimer.Start(); }
public void Facebook_GetWhoWeLike() { FacebookFanInfo ffi = new FacebookFanInfo(new FacebookTestDataSource()); string accountId = XMLUtility.GetTextFromAccountNode(ffi.Xml, "id"); string accessToken = XMLUtility.GetTextFromAccountNode(ffi.Xml, "accesstoken"); Graph g = new Graph(); //Page p = g.GetPageLikes(Convert.ToInt64(accountId), accessToken); }
private static Demographic <Country> GetCountryDemographic() { FacebookDataComponent fdc = new FacebookDataComponent(); FacebookFanInfo ffi = new FacebookFanInfo(new FacebookDataSource()); string accountId = XMLUtility.GetTextFromAccountNode(ffi.Xml, "id"); string accessToken = XMLUtility.GetTextFromAccountNode(ffi.Xml, "accesstoken"); Graph _graph = new Graph(); Demographic <Country> country = _graph.GetFanDemographics <Demographic <Country> >(accountId, accessToken, "page_fans_country"); return(country); }
private void SendData(object value, string typeName) { try { Uri apiuri = new Uri(XMLUtility.GetTextFromAccountNode(Xml, "apiuri") + "/" + typeName); HttpStatusCode code = SendRequest(apiuri, value, true); } catch (Exception e) { ExceptionExtensions.LogError(e, "FacebookDataSource.SendRequest", typeName); } }
private void SendFollowerData(List <User> list) { //CreatePostRequest Uri apiuri = new Uri(XMLUtility.GetTextFromAccountNode(Xml, "apiuri") + "/users"); //send try { HttpStatusCode code = SendRequest(apiuri, list, true); } catch (Exception e) { ExceptionExtensions.LogError(e, "TwitterDataSource.SendFollowerData"); } }
private void SendChannelInfo(YouTubeDataComponent ytdc) { //CreatePostRequest Uri apiuri = new Uri(XMLUtility.GetTextFromAccountNode(Xml, "apiuri")); //send try { HttpStatusCode code = SendRequest(apiuri, ytdc.YouTubeChannel, true); } catch (Exception e) { ExceptionExtensions.LogError(e, "YouTubeDataSource.SendRequest", ytdc.YouTubeChannel.Name); } }
private void SendAllMentions(KeywordStatus keywordStatus) { //CreatePostRequest Uri apiuri = new Uri(XMLUtility.GetTextFromAccountNode(Xml, "apiuri") + "/mentions"); //send try { HttpStatusCode code = SendRequest(apiuri, keywordStatus, true); } catch (Exception e) { ExceptionExtensions.LogError(e, "TwitterDataSource.SendAllMentions"); } }
private void SendCompany(CompanyRoot companyRoot) { //CreatePostRequest Uri apiuri = new Uri(XMLUtility.GetTextFromAccountNode(Xml, "apiuri")); //send try { HttpStatusCode code = SendRequest(apiuri, companyRoot, true); } catch (Exception e) { ExceptionExtensions.LogError(e, "CrunchbaseDataSource.SendRequest", companyRoot.permalink); } }
public void Facebook_GetFanDemographics() { FacebookDataComponent fdc = new FacebookDataComponent(); FacebookFanInfo ffi = new FacebookFanInfo(new FacebookTestDataSource()); string accountId = XMLUtility.GetTextFromAccountNode(ffi.Xml, "id"); string accessToken = XMLUtility.GetTextFromAccountNode(ffi.Xml, "accesstoken"); Demographic <Country> country = _graph.GetFanDemographics <Demographic <Country> >(accountId, accessToken, "page_fans_country"); Assert.IsNotNull(country.Data); Assert.AreNotEqual(0, country.Data.Count); Assert.IsNotNull(country.Data[0].Days); Assert.AreNotEqual(0, country.Data[0].Days.Count); Assert.IsNotNull(country.Data[0].Days[0].Country); Assert.AreNotEqual(0, country.Data[0].Days[0].Country.US); fdc.CountryDemographic = country; FanInfo.DroneDataSource.Process(fdc); Demographic <Locale> locale = _graph.GetFanDemographics <Demographic <Locale> >(accountId, accessToken, "page_fans_locale"); Assert.IsNotNull(locale.Data); Assert.AreNotEqual(0, locale.Data.Count); Assert.IsNotNull(locale.Data[0].Days); Assert.AreNotEqual(0, locale.Data[0].Days.Count); Assert.IsNotNull(locale.Data[0].Days[0].Locale); Assert.AreNotEqual(0, locale.Data[0].Days[0].Locale.en_US); fdc = new FacebookDataComponent(); fdc.LocaleDemographic = locale; FanInfo.DroneDataSource.Process(fdc); Demographic <Gender> gender = _graph.GetFanDemographics <Demographic <Gender> >(accountId, accessToken, "page_fans_gender_age"); Assert.IsNotNull(gender.Data); Assert.AreNotEqual(0, gender.Data.Count); Assert.IsNotNull(gender.Data[0].Days); Assert.AreNotEqual(0, gender.Data[0].Days.Count); Assert.IsNotNull(gender.Data[0].Days[0].Gender); Assert.AreNotEqual(0, gender.Data[0].Days[0].Gender.M_25to34); fdc = new FacebookDataComponent(); fdc.GenderDemographic = gender; FanInfo.DroneDataSource.Process(fdc); }
internal void Process(string fileName) { Dig digMgr = new Dig(); List <string> tempLines; while ((tempLines = Utility.ReadLinesFromFile(fileName, 100, true)).Count() > 0) { int maxParallel = XMLUtility.GetTextFromAccountNode(Xml, ProcessorName + "/maxparallel").ConvertStringToInt(1); Parallel.ForEach(tempLines, new ParallelOptions { MaxDegreeOfParallelism = maxParallel }, (tempLine) => { try { if (!String.IsNullOrWhiteSpace(tempLine)) { string[] companyEntry = tempLine.Split('|'); CompanyRoot cr = GetFullCompany(companyEntry[1], digMgr); //Add to MSMQ, add will call receive and then ProcessMessage below when to add to DB asynchronously if (!Object.Equals(null, cr) && !string.IsNullOrEmpty(cr.homepage_url)) { WriteToUsageLogFile("Domain:" + cr.homepage_url, string.Format("Executing {0}.{1}", ComponentTypeName, MethodInfo.GetCurrentMethod().Name), true); CrunchbaseDataComponent cdc = new CrunchbaseDataComponent(); cdc.CompanyLocal = cr; DroneDataSource.Process(cdc); } else { if (Object.Equals(null, cr) && companyEntry.Length < 3) { Utility.AddLineToFile(fileName, tempLine + "|retry"); } } cr = null; } } catch (Exception e) { ExceptionExtensions.LogError(e, "Crunchbase.Process", "Permalink: " + tempLine); } }); } }
private void SendDomainData(MarketShareDataType msType) { //CreatePostRequest Uri apiuri = new Uri(XMLUtility.GetTextFromAccountNode(Xml, "apiuri")); //send try { HttpStatusCode code = SendRequest(apiuri, msType, true); if (code != HttpStatusCode.Created) { ExceptionExtensions.LogInformation("MarketShareDataSource.SendDomainData", "Send failed with status code: " + code + ". Trying again."); SendDomainData(msType); } } catch (Exception e) { ExceptionExtensions.LogError(e, "MarketShareDataSource.SendRequest()", "DomainID: " + msType.DomainId); } }
public List <Competitor> GetCompetitorAccounts() { Uri apiuri = new Uri(XMLUtility.GetTextFromAccountNode(Xml, "commonapiuri") + "/competitors"); _requestGet = (HttpWebRequest)WebRequest.Create(apiuri); _requestGet.Method = "GET"; _requestGet.ContentType = "application/json"; _requestGet.UseDefaultCredentials = true; string result = string.Empty; // Get response using (HttpWebResponse response = _requestGet.GetResponse() as HttpWebResponse) { StreamReader reader = new StreamReader(response.GetResponseStream()); result = reader.ReadToEnd(); } List <Competitor> compList = new JavaScriptSerializer().Deserialize <List <Competitor> >(result); return(compList); }
internal void GetFanDemographics() { WriteToUsageLogFile("FacebookFanInfo.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Started GetFanDemographics"); Graph g = new Graph(); FacebookDataComponent fdc; string accountId = XMLUtility.GetTextFromAccountNode(Xml, "id"); string accessToken = XMLUtility.GetTextFromAccountNode(Xml, "accesstoken"); Demographic <Country> country = g.GetFanDemographics <Demographic <Country> >(accountId, accessToken, "page_fans_country"); if (!Object.Equals(country, null)) { fdc = new FacebookDataComponent(); fdc.CountryDemographic = country; DroneDataSource.Process(fdc); } Demographic <Locale> locale = g.GetFanDemographics <Demographic <Locale> >(accountId, accessToken, "page_fans_locale"); if (!Object.Equals(locale, null)) { fdc = new FacebookDataComponent(); fdc.LocaleDemographic = locale; DroneDataSource.Process(fdc); } Demographic <Gender> gender = g.GetFanDemographics <Demographic <Gender> >(accountId, accessToken, "page_fans_gender_age"); if (!Object.Equals(gender, null)) { fdc = new FacebookDataComponent(); fdc.GenderDemographic = gender; DroneDataSource.Process(fdc); } WriteToUsageLogFile("FacebookFanInfo.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Completed GetFanDemographics"); }
/// <summary> /// Main method that gathers data /// </summary> /// <param name="context">iDroneContext</param> public override void GetData(object context) { try { BaseContext cont = context as BaseContext; Context = cont; if (!Object.Equals(cont, null)) { SetContextStatus("processing", cont); SetNextRunIntervalByNode(ProcessorName, cont); if (XMLUtility.IsEnabled(Xml) && XMLUtility.IsComponentEnabled(Xml, ProcessorName) && (int)DateTime.Today.DayOfWeek == (int)Enum.Parse(typeof(DayOfWeek), XMLUtility.GetTextFromAccountNode(Xml, ProcessorName + "/interval"), true) || Utility.FileExists("Crunchbase_AllCompanies.txt")) //if this file exists, it means the service failed and is restarting. { //do work WriteToUsageLogFile("Crunchbase.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Started Crunchbase calls"); GetAllCompanies(); WriteToUsageLogFile("Crunchbase.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Completed Crunchbase calls"); } SetContextStatus("waiting", cont); } RaiseProcessingCompletedEvent(new EventArgs()); } catch (Exception e) { ExceptionExtensions.LogError(e, "Drone.SmallBusinessTracking.Components.Crunchbase.GetData()"); } }
public List <Domain> GetAllCompanies(MarketShareTypeBitMaskEnum marketTypeBitMaskEnum, int rows) { string result = string.Empty; HttpWebRequest requestGet = (HttpWebRequest)WebRequest.Create(string.Format("{0}?count={1}&mask={2}" , XMLUtility.GetTextFromAccountNode(Xml, "domainsapiuri") , rows , (int)marketTypeBitMaskEnum)); requestGet.UseDefaultCredentials = true; // Get response using (HttpWebResponse response = requestGet.GetResponse() as HttpWebResponse) { StreamReader reader = new StreamReader(response.GetResponseStream()); result = reader.ReadToEnd(); } List <Domain> domainList = new JavaScriptSerializer().Deserialize <List <Domain> >(result); return(domainList); }