public void Facebook_GetFanDemographicsAndInsertQueue()
{
FacebookFanInfo fg = new FacebookFanInfo();
string accountId = XMLUtility.GetTextFromAccountNode(fg.Xml, "id");
string accessToken = XMLUtility.GetTextFromAccountNode(fg.Xml, "accesstoken");
Demographic <Country> country = _graph.GetFanDemographics <Demographic <Country> >(accountId, accessToken, "page_fans_country");
if (!Object.Equals(country, null))
//fg.Context.MSMQManager.AddToQueue(country, "country");
{
}
Demographic <Locale> locale = _graph.GetFanDemographics <Demographic <Locale> >(accountId, accessToken, "page_fans_locale");
if (!Object.Equals(locale, null))
//fg.Context.MSMQManager.AddToQueue(locale, "locale");
{
}
Demographic <Gender> gender = _graph.GetFanDemographics <Demographic <Gender> >(accountId, accessToken, "page_fans_gender_age");
if (!Object.Equals(gender, null))
//fg.Context.MSMQManager.AddToQueue(gender, "gender");
{
}
Assert.IsNotNull(fg);
}
private bool GetBoolFromXMLConfig(string node)
{
bool boolVal;
bool.TryParse(XMLUtility.GetTextFromAccountNode(Xml, node), out boolVal);
return(boolVal);
}
public override void GetData(object context)
{
try
{
BaseContext cont = context as BaseContext;
if (!Object.Equals(cont, null))
{
SetContextStatus("processing", cont);
SetNextRunIntervalByNode(ProcessorName, cont);
if (XMLUtility.IsEnabled(Xml) && XMLUtility.IsComponentEnabled(Xml, ProcessorName))
{
WriteToUsageLogFile("FacebookInsight.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Started GetInsightInfo");
string accountId = XMLUtility.GetTextFromAccountNode(Xml, "id");
string accessToken = XMLUtility.GetTextFromAccountNode(Xml, "accesstoken");
FBIO = new Graph().GetInsightInfo(accountId, accessToken);
WriteToUsageLogFile("FacebookInsight.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Completed GetInsightInfo");
}
SetContextStatus("waiting", cont);
}
RaiseProcessingCompletedEvent(new EventArgs());
}
catch (Exception e)
{
ExceptionExtensions.LogError(e, "FacebookInsight.GetData()");
}
}
private void GetChannelData()
{
YouTubeDataSource yds = DroneDataSource as YouTubeDataSource;
if (!Object.Equals(null, yds))
{
foreach (Competitor account in yds.GetCompetitorAccounts())
{
if (!String.IsNullOrEmpty(account.YouTubeAccount))
{
ChannelManager cManager = new ChannelManager(new KeyValuePair <int, string>(account.ID, account.YouTubeAccount)
, Utility.ApplicationName
, XMLUtility.GetTextFromAccountNode(Xml, "token/devkey"));
Channel chan = cManager.GetUserChannel();
if (!Object.Equals(chan, null))
{
YouTubeDataComponent ydc = new YouTubeDataComponent();
ydc.YouTubeChannel = chan;
DroneDataSource.Process(ydc);
}
else
{
Utility.WriteToLogFile(String.Format("YouTube_ChannelNull_{0:M_d_yyyy}", DateTime.Today) + ".log", "No account info for: " + account.YouTubeAccount);
}
}
}
}
}
public void Crunchbase_GetNodeText()
{
Crunch t = new Crunch(new MarketShareDataSource());
XmlDocument xmlDoc = t.Xml as XmlDocument;
string text = XMLUtility.GetTextFromAccountNode(t.Xml, "crunch/interval");
Assert.AreEqual(text, "saturday");
}
private void GetAllWebHosts()
{
MarketShareDataSource mds = DroneDataSource as MarketShareDataSource;
if (!Object.Equals(null, mds))
{
using (Dig dig = new Dig())
{
int maxParallel = XMLUtility.GetTextFromAccountNode(Xml, ProcessorName + "/maxparallel").ConvertStringToInt(1);
Parallel.ForEach(mds.GetAllCompanies(MarketShareTypeBitMaskEnum.WebHost, XMLUtility.GetIntFromAccountNode(Xml, ProcessorName + "/recordcount"))
, new ParallelOptions {
MaxDegreeOfParallelism = maxParallel
}
, (company, state) =>
{
try
{
if (Context.ShuttingDown)
{
state.Break();
}
MarketShareDataType marketType = new MarketShareDataType();
string url = Utility.CleanUrl(company.Uri.ToString());
if (!String.IsNullOrWhiteSpace(url))
{
WriteToUsageLogFile("Domain:" + url, string.Format("Executing {0}.{1}", ComponentTypeName, MethodInfo.GetCurrentMethod().Name), true);
marketType.Value = dig.GetWebHostName(url);
marketType.DomainId = company.DomainId;
marketType.SampleDate = company.DomainAttributes["SampleDate"];
marketType.TypeId = (int)MarketShareDataTypeEnum.WebHost;
marketType.BitMaskId = (int)MarketShareTypeBitMaskEnum.WebHost;
marketType.UniqueID = g;
if (!string.IsNullOrWhiteSpace(marketType.Value))
{
MarketShareDataComponent dc = new MarketShareDataComponent();
dc.MarketShareType = marketType;
DroneDataSource.Process(dc);
}
else
{
Utility.WriteToLogFile(String.Format("SmallBiz_NoWebHost_{0:M_d_yyyy}", DateTime.Today) + ".log", string.Format("Domain: ,{0}", url));
}
}
}
catch (Exception e)
{
ExceptionExtensions.LogError(e, "WebHost.GetAllWebHostsParallel", "Domain: " + company.Uri.ToString());
}
});
}
}
}
protected void CreateGetRequest()
{
Uri apiuri = new Uri(XMLUtility.GetTextFromAccountNode(Xml, "apiuri") + "/keywords");
_requestGet = (HttpWebRequest)WebRequest.Create(apiuri);
_requestGet.Method = "GET";
_requestGet.ContentType = "application/json";
_requestGet.UseDefaultCredentials = true;
_requestGet.PreAuthenticate = true;
}
public MarketShareBulkDataSource()
{
_bulkThreshold = XMLUtility.GetTextFromAccountNode(Xml, "queuemarketshare/bulklimit").ConvertStringToInt(500);
_bulkTimeLimit = XMLUtility.GetTextFromAccountNode(Xml, "queuemarketshare/bulktimer").ConvertStringToInt(1000);
_isVerboseEnabled = XMLUtility.GetTextFromAccountNode(Xml, "queuemarketshare/verboselogging").ToLower() == "true";
_bulkTimer.Elapsed += _bulkTimer_Elapsed;
_bulkTimer.Interval = _bulkTimeLimit;
_bulkTimer.Start();
}
public void Facebook_GetWhoWeLike()
{
FacebookFanInfo ffi = new FacebookFanInfo(new FacebookTestDataSource());
string accountId = XMLUtility.GetTextFromAccountNode(ffi.Xml, "id");
string accessToken = XMLUtility.GetTextFromAccountNode(ffi.Xml, "accesstoken");
Graph g = new Graph();
//Page p = g.GetPageLikes(Convert.ToInt64(accountId), accessToken);
}
private static Demographic <Country> GetCountryDemographic()
{
FacebookDataComponent fdc = new FacebookDataComponent();
FacebookFanInfo ffi = new FacebookFanInfo(new FacebookDataSource());
string accountId = XMLUtility.GetTextFromAccountNode(ffi.Xml, "id");
string accessToken = XMLUtility.GetTextFromAccountNode(ffi.Xml, "accesstoken");
Graph _graph = new Graph();
Demographic <Country> country = _graph.GetFanDemographics <Demographic <Country> >(accountId, accessToken, "page_fans_country");
return(country);
}
private void SendData(object value, string typeName)
{
try
{
Uri apiuri = new Uri(XMLUtility.GetTextFromAccountNode(Xml, "apiuri") + "/" + typeName);
HttpStatusCode code = SendRequest(apiuri, value, true);
}
catch (Exception e)
{
ExceptionExtensions.LogError(e, "FacebookDataSource.SendRequest", typeName);
}
}
private void SendFollowerData(List <User> list)
{
//CreatePostRequest
Uri apiuri = new Uri(XMLUtility.GetTextFromAccountNode(Xml, "apiuri") + "/users");
//send
try
{
HttpStatusCode code = SendRequest(apiuri, list, true);
}
catch (Exception e)
{
ExceptionExtensions.LogError(e, "TwitterDataSource.SendFollowerData");
}
}
private void SendChannelInfo(YouTubeDataComponent ytdc)
{
//CreatePostRequest
Uri apiuri = new Uri(XMLUtility.GetTextFromAccountNode(Xml, "apiuri"));
//send
try
{
HttpStatusCode code = SendRequest(apiuri, ytdc.YouTubeChannel, true);
}
catch (Exception e)
{
ExceptionExtensions.LogError(e, "YouTubeDataSource.SendRequest", ytdc.YouTubeChannel.Name);
}
}
private void SendAllMentions(KeywordStatus keywordStatus)
{
//CreatePostRequest
Uri apiuri = new Uri(XMLUtility.GetTextFromAccountNode(Xml, "apiuri") + "/mentions");
//send
try
{
HttpStatusCode code = SendRequest(apiuri, keywordStatus, true);
}
catch (Exception e)
{
ExceptionExtensions.LogError(e, "TwitterDataSource.SendAllMentions");
}
}
private void SendCompany(CompanyRoot companyRoot)
{
//CreatePostRequest
Uri apiuri = new Uri(XMLUtility.GetTextFromAccountNode(Xml, "apiuri"));
//send
try
{
HttpStatusCode code = SendRequest(apiuri, companyRoot, true);
}
catch (Exception e)
{
ExceptionExtensions.LogError(e, "CrunchbaseDataSource.SendRequest", companyRoot.permalink);
}
}
public void Facebook_GetFanDemographics()
{
FacebookDataComponent fdc = new FacebookDataComponent();
FacebookFanInfo ffi = new FacebookFanInfo(new FacebookTestDataSource());
string accountId = XMLUtility.GetTextFromAccountNode(ffi.Xml, "id");
string accessToken = XMLUtility.GetTextFromAccountNode(ffi.Xml, "accesstoken");
Demographic <Country> country = _graph.GetFanDemographics <Demographic <Country> >(accountId, accessToken, "page_fans_country");
Assert.IsNotNull(country.Data);
Assert.AreNotEqual(0, country.Data.Count);
Assert.IsNotNull(country.Data[0].Days);
Assert.AreNotEqual(0, country.Data[0].Days.Count);
Assert.IsNotNull(country.Data[0].Days[0].Country);
Assert.AreNotEqual(0, country.Data[0].Days[0].Country.US);
fdc.CountryDemographic = country;
FanInfo.DroneDataSource.Process(fdc);
Demographic <Locale> locale = _graph.GetFanDemographics <Demographic <Locale> >(accountId, accessToken, "page_fans_locale");
Assert.IsNotNull(locale.Data);
Assert.AreNotEqual(0, locale.Data.Count);
Assert.IsNotNull(locale.Data[0].Days);
Assert.AreNotEqual(0, locale.Data[0].Days.Count);
Assert.IsNotNull(locale.Data[0].Days[0].Locale);
Assert.AreNotEqual(0, locale.Data[0].Days[0].Locale.en_US);
fdc = new FacebookDataComponent();
fdc.LocaleDemographic = locale;
FanInfo.DroneDataSource.Process(fdc);
Demographic <Gender> gender = _graph.GetFanDemographics <Demographic <Gender> >(accountId, accessToken, "page_fans_gender_age");
Assert.IsNotNull(gender.Data);
Assert.AreNotEqual(0, gender.Data.Count);
Assert.IsNotNull(gender.Data[0].Days);
Assert.AreNotEqual(0, gender.Data[0].Days.Count);
Assert.IsNotNull(gender.Data[0].Days[0].Gender);
Assert.AreNotEqual(0, gender.Data[0].Days[0].Gender.M_25to34);
fdc = new FacebookDataComponent();
fdc.GenderDemographic = gender;
FanInfo.DroneDataSource.Process(fdc);
}
internal void Process(string fileName)
{
Dig digMgr = new Dig();
List <string> tempLines;
while ((tempLines = Utility.ReadLinesFromFile(fileName, 100, true)).Count() > 0)
{
int maxParallel = XMLUtility.GetTextFromAccountNode(Xml, ProcessorName + "/maxparallel").ConvertStringToInt(1);
Parallel.ForEach(tempLines, new ParallelOptions {
MaxDegreeOfParallelism = maxParallel
}, (tempLine) =>
{
try
{
if (!String.IsNullOrWhiteSpace(tempLine))
{
string[] companyEntry = tempLine.Split('|');
CompanyRoot cr = GetFullCompany(companyEntry[1], digMgr);
//Add to MSMQ, add will call receive and then ProcessMessage below when to add to DB asynchronously
if (!Object.Equals(null, cr) && !string.IsNullOrEmpty(cr.homepage_url))
{
WriteToUsageLogFile("Domain:" + cr.homepage_url, string.Format("Executing {0}.{1}", ComponentTypeName, MethodInfo.GetCurrentMethod().Name), true);
CrunchbaseDataComponent cdc = new CrunchbaseDataComponent();
cdc.CompanyLocal = cr;
DroneDataSource.Process(cdc);
}
else
{
if (Object.Equals(null, cr) && companyEntry.Length < 3)
{
Utility.AddLineToFile(fileName, tempLine + "|retry");
}
}
cr = null;
}
}
catch (Exception e)
{
ExceptionExtensions.LogError(e, "Crunchbase.Process", "Permalink: " + tempLine);
}
});
}
}
private void SendDomainData(MarketShareDataType msType)
{
//CreatePostRequest
Uri apiuri = new Uri(XMLUtility.GetTextFromAccountNode(Xml, "apiuri"));
//send
try
{
HttpStatusCode code = SendRequest(apiuri, msType, true);
if (code != HttpStatusCode.Created)
{
ExceptionExtensions.LogInformation("MarketShareDataSource.SendDomainData", "Send failed with status code: " + code + ". Trying again.");
SendDomainData(msType);
}
}
catch (Exception e)
{
ExceptionExtensions.LogError(e, "MarketShareDataSource.SendRequest()", "DomainID: " + msType.DomainId);
}
}
public List <Competitor> GetCompetitorAccounts()
{
Uri apiuri = new Uri(XMLUtility.GetTextFromAccountNode(Xml, "commonapiuri") + "/competitors");
_requestGet = (HttpWebRequest)WebRequest.Create(apiuri);
_requestGet.Method = "GET";
_requestGet.ContentType = "application/json";
_requestGet.UseDefaultCredentials = true;
string result = string.Empty;
// Get response
using (HttpWebResponse response = _requestGet.GetResponse() as HttpWebResponse)
{
StreamReader reader = new StreamReader(response.GetResponseStream());
result = reader.ReadToEnd();
}
List <Competitor> compList = new JavaScriptSerializer().Deserialize <List <Competitor> >(result);
return(compList);
}
internal void GetFanDemographics()
{
WriteToUsageLogFile("FacebookFanInfo.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Started GetFanDemographics");
Graph g = new Graph();
FacebookDataComponent fdc;
string accountId = XMLUtility.GetTextFromAccountNode(Xml, "id");
string accessToken = XMLUtility.GetTextFromAccountNode(Xml, "accesstoken");
Demographic <Country> country = g.GetFanDemographics <Demographic <Country> >(accountId, accessToken, "page_fans_country");
if (!Object.Equals(country, null))
{
fdc = new FacebookDataComponent();
fdc.CountryDemographic = country;
DroneDataSource.Process(fdc);
}
Demographic <Locale> locale = g.GetFanDemographics <Demographic <Locale> >(accountId, accessToken, "page_fans_locale");
if (!Object.Equals(locale, null))
{
fdc = new FacebookDataComponent();
fdc.LocaleDemographic = locale;
DroneDataSource.Process(fdc);
}
Demographic <Gender> gender = g.GetFanDemographics <Demographic <Gender> >(accountId, accessToken, "page_fans_gender_age");
if (!Object.Equals(gender, null))
{
fdc = new FacebookDataComponent();
fdc.GenderDemographic = gender;
DroneDataSource.Process(fdc);
}
WriteToUsageLogFile("FacebookFanInfo.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Completed GetFanDemographics");
}
/// <summary>
/// Main method that gathers data
/// </summary>
/// <param name="context">iDroneContext</param>
public override void GetData(object context)
{
try
{
BaseContext cont = context as BaseContext;
Context = cont;
if (!Object.Equals(cont, null))
{
SetContextStatus("processing", cont);
SetNextRunIntervalByNode(ProcessorName, cont);
if (XMLUtility.IsEnabled(Xml) &&
XMLUtility.IsComponentEnabled(Xml, ProcessorName) &&
(int)DateTime.Today.DayOfWeek == (int)Enum.Parse(typeof(DayOfWeek), XMLUtility.GetTextFromAccountNode(Xml, ProcessorName + "/interval"), true) ||
Utility.FileExists("Crunchbase_AllCompanies.txt")) //if this file exists, it means the service failed and is restarting.
{
//do work
WriteToUsageLogFile("Crunchbase.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Started Crunchbase calls");
GetAllCompanies();
WriteToUsageLogFile("Crunchbase.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Completed Crunchbase calls");
}
SetContextStatus("waiting", cont);
}
RaiseProcessingCompletedEvent(new EventArgs());
}
catch (Exception e)
{
ExceptionExtensions.LogError(e, "Drone.SmallBusinessTracking.Components.Crunchbase.GetData()");
}
}
public List <Domain> GetAllCompanies(MarketShareTypeBitMaskEnum marketTypeBitMaskEnum, int rows)
{
string result = string.Empty;
HttpWebRequest requestGet = (HttpWebRequest)WebRequest.Create(string.Format("{0}?count={1}&mask={2}"
, XMLUtility.GetTextFromAccountNode(Xml, "domainsapiuri")
, rows
, (int)marketTypeBitMaskEnum));
requestGet.UseDefaultCredentials = true;
// Get response
using (HttpWebResponse response = requestGet.GetResponse() as HttpWebResponse)
{
StreamReader reader = new StreamReader(response.GetResponseStream());
result = reader.ReadToEnd();
}
List <Domain> domainList = new JavaScriptSerializer().Deserialize <List <Domain> >(result);
return(domainList);
}