public override async Task <CrawlJobData> GetCrawlJobData(
ProviderUpdateContext context,
IDictionary <string, object> configuration,
Guid organizationId,
Guid userId,
Guid providerDefinitionId)
{
if (configuration == null)
{
throw new ArgumentNullException(nameof(configuration));
}
var sharepointCrawlJobData = new SharePointCrawlJobData();
if (configuration.ContainsKey(SharePointConstants.KeyName.Url))
{
sharepointCrawlJobData.Url = configuration[SharePointConstants.KeyName.Url].ToString();
}
if (configuration.ContainsKey(SharePointConstants.KeyName.DeltaCrawlEnabled))
{
sharepointCrawlJobData.DeltaCrawlEnabled = bool.Parse(configuration[SharePointConstants.KeyName.DeltaCrawlEnabled].ToString());
}
if (configuration.ContainsKey(SharePointConstants.KeyName.UserName))
{
sharepointCrawlJobData.UserName = configuration[SharePointConstants.KeyName.UserName].ToString();
}
if (configuration.ContainsKey(SharePointConstants.KeyName.Password))
{
sharepointCrawlJobData.Password = configuration[SharePointConstants.KeyName.Password].ToString();
}
sharepointCrawlJobData.ClientId = ConfigurationManager.AppSettings.GetValue <string>("Providers.SharePointClientID", null);
sharepointCrawlJobData.ClientSecret = ConfigurationManager.AppSettings.GetValue <string>("Providers.SharePointClientSecret", null);
string apiVersion = "9.1";
string webApiUrl = $"{sharepointCrawlJobData.Url}/api/data/v{apiVersion}/";
if (sharepointCrawlJobData.UserName != null && sharepointCrawlJobData.Password != null)
{
Crawling.SharePoint.Infrastructure.SharePointClient.RefreshToken(sharepointCrawlJobData);
}
else
{
var clientCredential = new ClientCredential(sharepointCrawlJobData.ClientId, sharepointCrawlJobData.ClientSecret);
var authParameters = await AuthenticationParameters.CreateFromResourceUrlAsync(new Uri(webApiUrl));
var authContext = new AuthenticationContext(authParameters.Authority);
var authResult = authContext.AcquireTokenAsync(authParameters.Resource, clientCredential).Result;
sharepointCrawlJobData.TargetApiKey = authResult.AccessToken;
}
return(await Task.FromResult(sharepointCrawlJobData));
}
public static void RefreshToken(SharePointCrawlJobData sharePointCrawlJobData)
{
string apiVersion = "9.1";
string webApiUrl = $"{sharePointCrawlJobData.Url}/api/data/v{apiVersion}/";
var userCredential = new UserCredential(sharePointCrawlJobData.UserName, sharePointCrawlJobData.Password);
var authParameters = AuthenticationParameters.CreateFromResourceUrlAsync(new Uri(webApiUrl)).Result;
var authContext = new AuthenticationContext(authParameters.Authority, false);
var authResult = authContext.AcquireTokenAsync(authParameters.Resource, sharePointCrawlJobData.ClientId, userCredential).Result;
var refreshToken = authContext.AcquireTokenByRefreshTokenAsync(authResult.RefreshToken, sharePointCrawlJobData.ClientId).Result;
sharePointCrawlJobData.ApiKey = refreshToken.AccessToken;
}
public SharePointClient(ILogger log, SharePointCrawlJobData sharepointCrawlJobData, IRestClient client) // TODO: pass on any extra dependencies
{
if (sharepointCrawlJobData == null)
{
throw new ArgumentNullException(nameof(sharepointCrawlJobData));
}
if (client == null)
{
throw new ArgumentNullException(nameof(client));
}
this.log = log ?? throw new ArgumentNullException(nameof(log));
this.client = client ?? throw new ArgumentNullException(nameof(client));
_sharePointCrawlJobData = sharepointCrawlJobData ?? throw new ArgumentNullException(nameof(sharepointCrawlJobData));
// TODO use info from sharepointCrawlJobData to instantiate the connection
client.BaseUrl = new Uri(BaseUri);
client.AddDefaultParameter("api_key", sharepointCrawlJobData.Url, ParameterType.QueryString);
}
public IEnumerable <T> Get <T>(string value, SharePointCrawlJobData sharePointCrawlJobData)
{
DateTimeOffset lastCrawlFinishTime;
if (_sharePointCrawlJobData.LastCrawlFinishTime == DateTimeOffset.Parse("1/1/0001 12:00:00 AM +00:00"))
{
lastCrawlFinishTime = DateTimeOffset.Parse("01/01/1753 00:00:00");
}
else
{
lastCrawlFinishTime = _sharePointCrawlJobData.LastCrawlFinishTime;
}
var filter = $"(createdon ge {lastCrawlFinishTime:yyyy-MM-ddThh:mm:ssZ} or modifiedon ge {lastCrawlFinishTime:yyyy-MM-ddThh:mm:ssZ})";
var url = _sharePointCrawlJobData.Url;
if (_sharePointCrawlJobData.DeltaCrawlEnabled)
{
url = _sharePointCrawlJobData.Url + string.Format("/api/data/v9.1/{0}?$filter={1}", value, filter);
}
ResultList <T> resultList = null;
while (true)
{
using (HttpClient httpClient = new HttpClient())
{
try
{
httpClient.Timeout = new TimeSpan(0, 2, 0);
httpClient.DefaultRequestHeaders.Add("Prefer", "odata.maxpagesize=100");
httpClient.DefaultRequestHeaders.Add("OData-MaxVersion", "4.0");
httpClient.DefaultRequestHeaders.Add("OData-Version", "4.0");
httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", sharePointCrawlJobData.ApiKey);
HttpResponseMessage responseMessage = httpClient.GetAsync(url).Result;
var content = responseMessage.Content.ReadAsStringAsync().Result;
if (responseMessage.StatusCode == HttpStatusCode.Unauthorized)
{
RefreshToken(sharePointCrawlJobData);
continue;
}
else if (responseMessage.StatusCode != HttpStatusCode.OK)
{
log.Error(() => "Connection failed " + responseMessage.StatusCode);
}
resultList = JsonConvert.DeserializeObject <ResultList <T> >(content, new JsonSerializerSettings()
{
NullValueHandling = NullValueHandling.Ignore
});
}
catch (Exception e)
{
log.Error(() => e.Message);
}
if (resultList?.Value != null)
{
foreach (var item in resultList.Value)
{
yield return(item);
}
}
else
{
break;
}
if (resultList.NextLink == null)
{
break;
}
}
}
}