internal void GetPageInfoForAllCompetitors()
{
WriteToUsageLogFile("FacebookFanInfo.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Started GetPageInfoForAllCompetitors");
FacebookDataSource fds = DroneDataSource as FacebookDataSource;
if (!Object.Equals(null, fds))
{
foreach (Competitor item in fds.GetCompetitorAccounts())
{
if (item.FacebookID != 0)
{
Page fpo = GetPageInfoByCompany(item.FacebookID);
if (!Object.Equals(fpo, null))
{
FacebookDataComponent fdc = new FacebookDataComponent();
fdc.FBPage = fpo;
DroneDataSource.Process(fdc);
}
else
{
Shared.Utility.WriteToLogFile(String.Format("Facebook_NoPage_{0:M_d_yyyy}", DateTime.Today) + ".log", string.Format("{0},{1}", fpo.Name, fpo.Id));
}
}
}
}
WriteToUsageLogFile("FacebookFanInfo.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Completed GetPageInfoForAllCompetitors");
}
private void GetChannelData()
{
YouTubeDataSource yds = DroneDataSource as YouTubeDataSource;
if (!Object.Equals(null, yds))
{
foreach (Competitor account in yds.GetCompetitorAccounts())
{
if (!String.IsNullOrEmpty(account.YouTubeAccount))
{
ChannelManager cManager = new ChannelManager(new KeyValuePair <int, string>(account.ID, account.YouTubeAccount)
, Utility.ApplicationName
, XMLUtility.GetTextFromAccountNode(Xml, "token/devkey"));
Channel chan = cManager.GetUserChannel();
if (!Object.Equals(chan, null))
{
YouTubeDataComponent ydc = new YouTubeDataComponent();
ydc.YouTubeChannel = chan;
DroneDataSource.Process(ydc);
}
else
{
Utility.WriteToLogFile(String.Format("YouTube_ChannelNull_{0:M_d_yyyy}", DateTime.Today) + ".log", "No account info for: " + account.YouTubeAccount);
}
}
}
}
}
private void GetAllWebHosts()
{
MarketShareDataSource mds = DroneDataSource as MarketShareDataSource;
if (!Object.Equals(null, mds))
{
using (Dig dig = new Dig())
{
int maxParallel = XMLUtility.GetTextFromAccountNode(Xml, ProcessorName + "/maxparallel").ConvertStringToInt(1);
Parallel.ForEach(mds.GetAllCompanies(MarketShareTypeBitMaskEnum.WebHost, XMLUtility.GetIntFromAccountNode(Xml, ProcessorName + "/recordcount"))
, new ParallelOptions {
MaxDegreeOfParallelism = maxParallel
}
, (company, state) =>
{
try
{
if (Context.ShuttingDown)
{
state.Break();
}
MarketShareDataType marketType = new MarketShareDataType();
string url = Utility.CleanUrl(company.Uri.ToString());
if (!String.IsNullOrWhiteSpace(url))
{
WriteToUsageLogFile("Domain:" + url, string.Format("Executing {0}.{1}", ComponentTypeName, MethodInfo.GetCurrentMethod().Name), true);
marketType.Value = dig.GetWebHostName(url);
marketType.DomainId = company.DomainId;
marketType.SampleDate = company.DomainAttributes["SampleDate"];
marketType.TypeId = (int)MarketShareDataTypeEnum.WebHost;
marketType.BitMaskId = (int)MarketShareTypeBitMaskEnum.WebHost;
marketType.UniqueID = g;
if (!string.IsNullOrWhiteSpace(marketType.Value))
{
MarketShareDataComponent dc = new MarketShareDataComponent();
dc.MarketShareType = marketType;
DroneDataSource.Process(dc);
}
else
{
Utility.WriteToLogFile(String.Format("SmallBiz_NoWebHost_{0:M_d_yyyy}", DateTime.Today) + ".log", string.Format("Domain: ,{0}", url));
}
}
}
catch (Exception e)
{
ExceptionExtensions.LogError(e, "WebHost.GetAllWebHostsParallel", "Domain: " + company.Uri.ToString());
}
});
}
}
}
private void GetSquatterData()
{
ChannelManager cManager = new ChannelManager();
Channel chanVids = cManager.GetSquatterVideos(GetAllSquatterVideoIds());
if (!Object.Equals(chanVids, null))
{
YouTubeDataComponent ydc = new YouTubeDataComponent();
ydc.YouTubeChannel = chanVids;
DroneDataSource.Process(ydc);
}
else
{
Utility.WriteToLogFile(String.Format("YouTube_SquattersNull_{0:M_d_yyyy}", DateTime.Today) + ".log", "No videos found");
}
}
/// <summary>
/// Gets registrars for all companies in our database
/// </summary>
internal void GetAllEmailHosts()
{
MarketShareDataSource mds = DroneDataSource as MarketShareDataSource;
if (!Object.Equals(null, mds))
{
using (Dig dig = new Dig())
{
var list = mds.GetAllCompanies(MarketShareTypeBitMaskEnum.EmailHost, XMLUtility.GetIntFromAccountNode(Xml, "emailhost/recordcount"));
foreach (var company in list)
{
try
{
MarketShareDataType marketType = new MarketShareDataType();
string url = Utility.CleanUrl(company.Uri.ToString());
if (!String.IsNullOrWhiteSpace(url))
{
WriteToUsageLogFile("Domain:" + url, string.Format("Executing {0}.{1}", ComponentTypeName, MethodInfo.GetCurrentMethod().Name), true);
marketType.Value = dig.GetEmailHostName(url);
marketType.DomainId = company.DomainId;
marketType.SampleDate = company.DomainAttributes["SampleDate"];
marketType.TypeId = (int)MarketShareDataTypeEnum.EmailHost;
marketType.BitMaskId = (int)MarketShareTypeBitMaskEnum.EmailHost;
if (!string.IsNullOrWhiteSpace(marketType.Value))
{
MarketShareDataComponent dc = new MarketShareDataComponent();
dc.MarketShareType = marketType;
DroneDataSource.Process(dc);
}
else
{
Utility.WriteToLogFile(String.Format("SmallBiz_NoEmailHost_{0:M_d_yyyy}", DateTime.Today) + ".log", string.Format("Domain: ,{0}", url));
}
}
}
catch (Exception e)
{
ExceptionExtensions.LogError(e, "EmailHost.GetAllEmailHosts", "Domain: " + company.Uri.ToString());
}
}
}
}
}
internal void Process(string fileName)
{
Dig digMgr = new Dig();
List <string> tempLines;
while ((tempLines = Utility.ReadLinesFromFile(fileName, 100, true)).Count() > 0)
{
int maxParallel = XMLUtility.GetTextFromAccountNode(Xml, ProcessorName + "/maxparallel").ConvertStringToInt(1);
Parallel.ForEach(tempLines, new ParallelOptions {
MaxDegreeOfParallelism = maxParallel
}, (tempLine) =>
{
try
{
if (!String.IsNullOrWhiteSpace(tempLine))
{
string[] companyEntry = tempLine.Split('|');
CompanyRoot cr = GetFullCompany(companyEntry[1], digMgr);
//Add to MSMQ, add will call receive and then ProcessMessage below when to add to DB asynchronously
if (!Object.Equals(null, cr) && !string.IsNullOrEmpty(cr.homepage_url))
{
WriteToUsageLogFile("Domain:" + cr.homepage_url, string.Format("Executing {0}.{1}", ComponentTypeName, MethodInfo.GetCurrentMethod().Name), true);
CrunchbaseDataComponent cdc = new CrunchbaseDataComponent();
cdc.CompanyLocal = cr;
DroneDataSource.Process(cdc);
}
else
{
if (Object.Equals(null, cr) && companyEntry.Length < 3)
{
Utility.AddLineToFile(fileName, tempLine + "|retry");
}
}
cr = null;
}
}
catch (Exception e)
{
ExceptionExtensions.LogError(e, "Crunchbase.Process", "Permalink: " + tempLine);
}
});
}
}
/// <summary>
/// Main method that gathers data
/// </summary>
/// <param name="context">iDroneContext</param>
public override void GetData(object context)
{
try
{
BaseContext cont = context as BaseContext;
if (!Object.Equals(cont, null))
{
SetContextStatus("processing", cont);
DateTime tempNextRun = cont.NextRun;
SetNextRunIntervalByNode(ProcessorName, cont);
if (XMLUtility.IsEnabled(Xml) && XMLUtility.IsComponentEnabled(Xml, ProcessorName))
{
WriteToUsageLogFile("TwitterFollower.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Started GetTwitterFollowers");
//wait 5 min for other twitter processes to finish so we don't get rate limited
Thread.Sleep(300000);
//do work
TwitterDataComponent _dataComponent = DroneDataComponent as TwitterDataComponent;
UserManager um = new UserManager();
OAuthTokens oat = Utility.GetOAuthToken(Xml);
_dataComponent.TwitterUserList = um.GetAllFollowers(oat);
DroneDataSource.Process(_dataComponent);
WriteToUsageLogFile("TwitterFollower.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Completed GetTwitterFollowers");
}
SetContextStatus("waiting", cont);
}
RaiseProcessingCompletedEvent(new EventArgs());
}
catch (Exception e)
{
ExceptionExtensions.LogError(e, "Drone.Twitter.Components.TwitterFollower.GetData()");
}
}
internal void GetFanDemographics()
{
WriteToUsageLogFile("FacebookFanInfo.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Started GetFanDemographics");
Graph g = new Graph();
FacebookDataComponent fdc;
string accountId = XMLUtility.GetTextFromAccountNode(Xml, "id");
string accessToken = XMLUtility.GetTextFromAccountNode(Xml, "accesstoken");
Demographic <Country> country = g.GetFanDemographics <Demographic <Country> >(accountId, accessToken, "page_fans_country");
if (!Object.Equals(country, null))
{
fdc = new FacebookDataComponent();
fdc.CountryDemographic = country;
DroneDataSource.Process(fdc);
}
Demographic <Locale> locale = g.GetFanDemographics <Demographic <Locale> >(accountId, accessToken, "page_fans_locale");
if (!Object.Equals(locale, null))
{
fdc = new FacebookDataComponent();
fdc.LocaleDemographic = locale;
DroneDataSource.Process(fdc);
}
Demographic <Gender> gender = g.GetFanDemographics <Demographic <Gender> >(accountId, accessToken, "page_fans_gender_age");
if (!Object.Equals(gender, null))
{
fdc = new FacebookDataComponent();
fdc.GenderDemographic = gender;
DroneDataSource.Process(fdc);
}
WriteToUsageLogFile("FacebookFanInfo.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Completed GetFanDemographics");
}
/// <summary>
/// Main method that gathers data
/// </summary>
/// <param name="context">iDroneContext</param>
public override void GetData(object context)
{
try
{
BaseContext cont = context as BaseContext;
if (!Object.Equals(cont, null))
{
SetContextStatus("processing", cont);
DateTime tempNextRun = cont.NextRun;
SetNextRunIntervalByNode(ProcessorName, cont);
if (DateTime.Compare(tempNextRun, DateTime.MinValue) != 0 && XMLUtility.IsEnabled(Xml) && XMLUtility.IsComponentEnabled(Xml, ProcessorName))
{
WriteToUsageLogFile("TwitterUser.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Started GetTwitterUsers Competitors");
//do work
TwitterDataComponent _dataComponent = DroneDataComponent as TwitterDataComponent;
_dataComponent.TwitterUserList = GetTwitterUsers();
DroneDataSource.Process(_dataComponent);
WriteToUsageLogFile("TwitterUser.GetData | Thread:" + Thread.CurrentThread.ManagedThreadId.ToString(), "Completed GetTwitterUsers Competitors");
}
SetContextStatus("waiting", cont);
}
RaiseProcessingCompletedEvent(new EventArgs());
}
catch (Exception e)
{
ExceptionExtensions.LogError(e, "Drone.Twitter.Components.TwitterUser.GetData()");
}
}
/// <summary>
/// Get all tweets with "GoDaddy" in them.
/// 100 per page, up to 15 pages if we loop and make seperate calls for each page
/// </summary>
internal Dictionary <int, List <Status> > GetAllMentionsByQuery(TwitterContext cont)
{
Dictionary <int, List <Status> > allTwitterMentions = new Dictionary <int, List <Status> >();
if (XMLUtility.IsComponentEnabled(Xml, ProcessorName))
{
int countPerPage; int pageCount;
TwitterDataSource data = new TwitterDataSource();
TweetManager tm = new TweetManager();
XMLUtility.GetPageResultCounts(Xml, ProcessorName, out countPerPage, out pageCount, 100, 3);
List <Keyword> queries = data.GetTweetQueries();
bool useSinceId = XMLUtility.UseSinceId(Xml, ProcessorName);
//create backup of current keys in case of failure at db level
lock (cont) cont.prevRunLatestTweetIDs = cont.LatestTweetIDs.ToDictionary(entry => entry.Key, entry => entry.Value);
foreach (Keyword item in queries)
{
try
{
long sinceId = 0;
if (useSinceId)
{
//get the last recorded id for this query and use it
if (!Object.Equals(cont, null))
{
lock (cont)
{
if (cont.LatestTweetIDs.ContainsKey(item.KeywordId))
{
cont.LatestTweetIDs.TryGetValue(item.KeywordId, out sinceId);
}
}
}
}
//call the mention object in the API wrapper
TwitterDataComponent _dataComponent = DroneDataComponent as TwitterDataComponent;
KeywordStatus ks = new KeywordStatus();
ks.KeywordId = item.KeywordId;
ks.StatusList = tm.GetTweetsByQuery(countPerPage, item.KeywordValue + (sinceId > 0 ? "&since_id=" + sinceId : string.Empty), Utility.GetOAuthToken(Xml));
allTwitterMentions.Add(ks.KeywordId, ks.StatusList);
_dataComponent.KeywordStatus = ks;
DroneDataSource.Process(_dataComponent);
//if there was a failure saving to the db, reset the since id to gather and try again
if (_dataComponent.SaveFailure)
{
lock (cont) cont.LatestTweetIDs = cont.prevRunLatestTweetIDs.ToDictionary(entry => entry.Key, entry => entry.Value);
}
//get the last id for this query and store it
if (!Object.Equals(cont, null) && allTwitterMentions.ContainsKey(item.KeywordId) && allTwitterMentions[item.KeywordId].Count > 0)
{
long latestID;
long.TryParse(allTwitterMentions[item.KeywordId][0].id.ToString(), out latestID);
lock (cont)
{
if (cont.LatestTweetIDs.ContainsKey(item.KeywordId))
{
cont.LatestTweetIDs[item.KeywordId] = latestID;
}
else
{
cont.LatestTweetIDs.Add(item.KeywordId, latestID);
}
}
}
}
catch (Exception e)
{
ExceptionExtensions.LogError(e, "Twitter.GetAllMentionsByQuery()", "Keyword name: " + item.KeywordName);
}
}
}
return(allTwitterMentions);
}
