public override async Task <CrawlJobData> GetCrawlJobData(
ProviderUpdateContext context,
IDictionary <string, object> configuration,
Guid organizationId,
Guid userId,
Guid providerDefinitionId)
{
if (configuration == null)
{
throw new ArgumentNullException(nameof(configuration));
}
var dynamics365CrawlJobData = new Dynamics365CrawlJobData();
if (configuration.ContainsKey(Dynamics365Constants.KeyName.Url))
{
dynamics365CrawlJobData.Url = configuration[Dynamics365Constants.KeyName.Url].ToString();
}
if (configuration.ContainsKey(Dynamics365Constants.KeyName.DeltaCrawlEnabled))
{
dynamics365CrawlJobData.DeltaCrawlEnabled = bool.Parse(configuration[Dynamics365Constants.KeyName.DeltaCrawlEnabled].ToString());
}
if (configuration.ContainsKey(Dynamics365Constants.KeyName.UserName))
{
dynamics365CrawlJobData.UserName = configuration[Dynamics365Constants.KeyName.UserName].ToString();
}
if (configuration.ContainsKey(Dynamics365Constants.KeyName.Password))
{
dynamics365CrawlJobData.Password = configuration[Dynamics365Constants.KeyName.Password].ToString();
}
dynamics365CrawlJobData.ClientId = ConfigurationManager.AppSettings.GetValue <string>("Providers.Dynamics365ClientId", null);
dynamics365CrawlJobData.ClientSecret = ConfigurationManager.AppSettings.GetValue <string>("Providers.Dynamics365ClientSecret", null);
string apiVersion = "9.1";
string webApiUrl = $"{dynamics365CrawlJobData.Url}/api/data/v{apiVersion}/";
if (dynamics365CrawlJobData.UserName != null && dynamics365CrawlJobData.Password != null)
{
Crawling.Dynamics365.Infrastructure.Dynamics365Client.RefreshToken(dynamics365CrawlJobData);
}
else
{
var clientCredential = new ClientCredential(dynamics365CrawlJobData.ClientId, dynamics365CrawlJobData.ClientSecret);
var authParameters = await AuthenticationParameters.CreateFromResourceUrlAsync(new Uri(webApiUrl));
var authContext = new AuthenticationContext(authParameters.Authority);
var authResult = authContext.AcquireTokenAsync(authParameters.Resource, clientCredential).Result;
dynamics365CrawlJobData.TargetApiKey = authResult.AccessToken;
}
return(await Task.FromResult(dynamics365CrawlJobData));
}
public static void RefreshToken(Dynamics365CrawlJobData dynamics365CrawlJobData)
{
string apiVersion = "9.1";
string webApiUrl = $"{dynamics365CrawlJobData.Url}/api/data/v{apiVersion}/";
var userCredential = new UserCredential(dynamics365CrawlJobData.UserName, dynamics365CrawlJobData.Password);
var authParameters = AuthenticationParameters.CreateFromResourceUrlAsync(new Uri(webApiUrl)).Result;
var authContext = new AuthenticationContext(authParameters.Authority, false);
var authResult = authContext.AcquireTokenAsync(authParameters.Resource, dynamics365CrawlJobData.ClientId, userCredential).Result;
var refreshToken = authContext.AcquireTokenByRefreshTokenAsync(authResult.RefreshToken, dynamics365CrawlJobData.ClientId).Result;
dynamics365CrawlJobData.ApiKey = refreshToken.AccessToken;
}
public Dynamics365Client(ILogger log, Dynamics365CrawlJobData dynamics365CrawlJobData, IRestClient client) // TODO: pass on any extra dependencies
{
if (dynamics365CrawlJobData == null)
{
throw new ArgumentNullException(nameof(dynamics365CrawlJobData));
}
if (client == null)
{
throw new ArgumentNullException(nameof(client));
}
this.log = log ?? throw new ArgumentNullException(nameof(log));
this.client = client ?? throw new ArgumentNullException(nameof(client));
_dynamics365CrawlJobData = dynamics365CrawlJobData ?? throw new ArgumentNullException(nameof(dynamics365CrawlJobData));
// TODO use info from dynamics365CrawlJobData to instantiate the connection
client.BaseUrl = new Uri(BaseUri);
client.AddDefaultParameter("api_key", dynamics365CrawlJobData.Url, ParameterType.QueryString);
}
public IEnumerable <T> Get <T>(string value, Dynamics365CrawlJobData dynamics365CrawlJobData)
{
DateTimeOffset lastCrawlFinishTime;
if (_dynamics365CrawlJobData.LastCrawlFinishTime == DateTimeOffset.Parse("1/1/0001 12:00:00 AM +00:00"))
{
lastCrawlFinishTime = DateTimeOffset.Parse("01/01/1753 00:00:00");
}
else
{
lastCrawlFinishTime = _dynamics365CrawlJobData.LastCrawlFinishTime;
}
var filter = $"(createdon ge {lastCrawlFinishTime:yyyy-MM-ddThh:mm:ssZ} or modifiedon ge {lastCrawlFinishTime:yyyy-MM-ddThh:mm:ssZ})";
var url = _dynamics365CrawlJobData.Url;
if (_dynamics365CrawlJobData.DeltaCrawlEnabled)
{
url = _dynamics365CrawlJobData.Url + string.Format("/api/data/v9.1/{0}?$filter={1}", value, filter);
}
ResultList <T> resultList = null;
while (true)
{
using (HttpClient httpClient = new HttpClient())
{
try
{
httpClient.Timeout = new TimeSpan(0, 2, 0);
httpClient.DefaultRequestHeaders.Add("Prefer", "odata.maxpagesize=100");
httpClient.DefaultRequestHeaders.Add("OData-MaxVersion", "4.0");
httpClient.DefaultRequestHeaders.Add("OData-Version", "4.0");
httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", dynamics365CrawlJobData.ApiKey);
HttpResponseMessage responseMessage = httpClient.GetAsync(url).Result;
var content = responseMessage.Content.ReadAsStringAsync().Result;
if (responseMessage.StatusCode == HttpStatusCode.Unauthorized)
{
RefreshToken(dynamics365CrawlJobData);
continue;
}
else if (responseMessage.StatusCode != HttpStatusCode.OK)
{
log.Error(() => "Connection failed " + responseMessage.StatusCode);
}
resultList = JsonConvert.DeserializeObject <ResultList <T> >(content, new JsonSerializerSettings()
{
NullValueHandling = NullValueHandling.Ignore
});
}
catch (Exception e)
{
log.Error(() => e.Message);
}
if (resultList?.Value != null)
{
foreach (var item in resultList.Value)
{
yield return(item);
}
}
else
{
break;
}
if (resultList.NextLink == null)
{
break;
}
}
}
}
