private async Task SavePageJobEntity(string jobId, string accessToken)
{
string page_id = string.Empty;
SourceInfoTwitter sourceInfo = new SourceInfoTwitter();
sourceInfo.SinceId = "0";
foreach (var pair in accessToken.Split('&'))
{
var keys = pair.Split('=');
if (keys[0].Equals("oauth_token"))
{
sourceInfo.ClientToken = keys[1];
}
if (keys[0].Equals("oauth_token_secret"))
{
sourceInfo.ClientSecret = keys[1];
}
if (keys[0].Equals("user_id"))
{
page_id = keys[1];
}
}
var PageJobMappingTable = azureTableProvider.GetAzureTableReference(Settings.PageJobMappingTableName);
var pageJobEntity = new PageJobEntity(page_id, jobId)
{
SourceInfo = JsonConvert.SerializeObject(sourceInfo)
};
await azureTableProvider.InsertOrReplaceEntityAsync(PageJobMappingTable, pageJobEntity);
}
public async Task UpdateSourceInfo(ConnectorTask taskInfo, SourceInfoTwitter twitterSourceInfo)
{
PageJobMappingTable = azureTableProviderInstance.GetAzureTableReference(Settings.PageJobMappingTableName);
Expression <Func <PageJobEntity, bool> > filter = (entity => entity.RowKey == taskInfo.JobId);
CloudTable pageJobMappingTable = azureTableProviderInstance.GetAzureTableReference(Settings.PageJobMappingTableName);
List <PageJobEntity> pageJobEntityList = await azureTableProviderInstance.QueryEntitiesAsync <PageJobEntity>(pageJobMappingTable, filter);
PageJobEntity pageJobEntity = pageJobEntityList[0];
pageJobEntity.SourceInfo = JsonConvert.SerializeObject(twitterSourceInfo);
await azureTableProviderInstance.InsertOrReplaceEntityAsync(PageJobMappingTable, pageJobEntity);
}
/// <summary>
/// Fetches Data wrt given Tweet ID
/// </summary>
/// <param name="taskInfo">contains the TweetID for which data is to be fetched</param>
public override async Task <List <ItemMetadata> > FetchData(ConnectorTask taskInfo, string sourceInfo)
{
Trace.TraceInformation("Data fetch Started");
List <ItemMetadata> itemMetaData = new List <ItemMetadata>();
SourceInfoTwitter twitterSourceInfo = JsonConvert.DeserializeObject <SourceInfoTwitter>(sourceInfo);
OAuth1Token token = new OAuth1Token(SettingsTwitter.TwitterApiKey, SettingsTwitter.TwitterApiSecretKey, twitterSourceInfo.ClientToken, twitterSourceInfo.ClientSecret);
var filterTime = taskInfo.EndTime;
OAuth1Helper oAuth1Helper = new OAuth1Helper(url, token, HttpMethod.Get.ToString().ToUpperInvariant());
while (true)
{
Dictionary <string, string> param = getParams(taskInfo, twitterSourceInfo);
string queryString = oAuth1Helper.GetQueryString(param);
string authHeader = oAuth1Helper.GenerateAuthorizationHeader();
AuthenticationHeaderValue header = new AuthenticationHeaderValue("OAuth", authHeader);
List <Tweet> tweets = await downloader.GetWebContent <List <Tweet>, ErrorsTwitter>(queryString, header);
bool isScheduleCompleted = false;
if (tweets != null && tweets.Any())
{
var minId = tweets.Select(t => long.Parse(t.Tweetid)).ToList <long>().Min().ToString() ?? twitterSourceInfo.SinceId;
isScheduleCompleted = DateTime.Compare(DateTime.ParseExact(tweets.Where(t => t.Tweetid.Equals(minId)).First().CreatedAt, Const_TwitterDateTemplate, new System.Globalization.CultureInfo("en-US")), taskInfo.EndTime) > 0;
}
if (tweets == null || tweets.Count == 0 || isScheduleCompleted)
{
break; // When no new data to get since sinceID(last fetched tweet)
}
twitterSourceInfo.SinceId = tweets.Select(t => long.Parse(t.Tweetid)).ToList <long>().Max().ToString();
tweets.RemoveAll(t => DateTime.Compare(DateTime.ParseExact(t.CreatedAt, Const_TwitterDateTemplate, new System.Globalization.CultureInfo("en-US")), taskInfo.StartTime) < 0);
tweets.RemoveAll(t => DateTime.Compare(DateTime.ParseExact(t.CreatedAt, Const_TwitterDateTemplate, new System.Globalization.CultureInfo("en-US")), taskInfo.EndTime) > 0);
Trace.TraceInformation($"Tweets Fetched {tweets.Count}");
if (tweets.Any())
{
foreach (var tweet in tweets)
{
var enrichedTweet = await EnrichTweetWithAttachments(tweet);
itemMetaData.Add(await UploadTweet(twitterItemMapper, enrichedTweet, taskInfo));
}
twitterSourceInfo.SinceId = tweets.Select(t => long.Parse(t.Tweetid)).ToList <long>().Max().ToString();
}
}
return(itemMetaData);
}
/// <summary>
/// Set parameters accordingly
/// </summary>
/// <param name="sourceInfo"></param>
/// <returns>parameters list</returns>
private Dictionary <string, string> getParams(ConnectorTask taskInfo, SourceInfoTwitter sourceInfo)
{
Dictionary <string, string> parameters = new Dictionary <string, string>
{
{ "include_entities", "true" },
{ "count", "200" },
{ "include_rts", "true" },
};
if (Convert.ToInt64(sourceInfo.SinceId) > 0)
{
parameters.Add("since_id", sourceInfo.SinceId);
}
return(parameters);
}
public string GetUserTokenForjobId(PageJobEntity pageJobEntity)
{
var queryParams = new Dictionary <string, string>();
queryParams.Add("format", "json");
queryParams.Add("include_email", "true");
queryParams.Add("include_entities", "false");
SourceInfoTwitter SourceInfoTwitter = JsonConvert.DeserializeObject <SourceInfoTwitter>(pageJobEntity.SourceInfo);
string clientToken = SourceInfoTwitter.ClientToken;
string clientSecret = SourceInfoTwitter.ClientSecret;
string userToken = GetToken(SettingsTwitter.TwitterEndPoint + "/1.1/account/verify_credentials.json", HttpMethod.Get.ToString().ToUpperInvariant(), clientToken, clientSecret, queryParams);
return(userToken);
}
private async Task SavePageJobEntity(string jobId, string accessToken)
{
string page_id = string.Empty;
SourceInfoTwitter sourceInfo = new SourceInfoTwitter();
var pageJobMappingTable = azureTableProvider.GetAzureTableReference(Settings.PageJobMappingTableName);
Expression <Func <PageJobEntity, bool> > filter = (entity => entity.RowKey == $"{jobId}");
List <PageJobEntity> pageJobEntityList = await azureTableProvider.QueryEntitiesAsync <PageJobEntity>(pageJobMappingTable, filter);
PageJobEntity pageJobEntity;
if (pageJobEntityList.Count != 0)
{
pageJobEntity = pageJobEntityList[0];
SourceInfoTwitter SourceInfoTwitter = JsonConvert.DeserializeObject <SourceInfoTwitter>(pageJobEntity.SourceInfo);
sourceInfo.SinceId = SourceInfoTwitter.SinceId;
}
else
{
sourceInfo.SinceId = "0";
}
foreach (var pair in accessToken.Split('&'))
{
var keys = pair.Split('=');
if (keys[0].Equals("oauth_token"))
{
sourceInfo.ClientToken = keys[1];
}
if (keys[0].Equals("oauth_token_secret"))
{
sourceInfo.ClientSecret = keys[1];
}
if (keys[0].Equals("user_id"))
{
page_id = keys[1];
}
}
pageJobEntity = new PageJobEntity(page_id, jobId)
{
SourceInfo = JsonConvert.SerializeObject(sourceInfo)
};
await azureTableProvider.InsertOrReplaceEntityAsync(pageJobMappingTable, pageJobEntity);
}