public static IDrivesCollectionPage GetFacade(ISiteDrivesCollectionPage graphpage)
{
if (graphpage == null)
{
return(null);
}
else
{
FacadeSitesDrivesCollectionPage newpage = new FacadeSitesDrivesCollectionPage();
newpage._page = graphpage;
return(newpage);
}
}
/// <summary>
/// Retrieves an appropriate Drive URL from a SharePoint document library root URL
/// </summary>
/// <param name="rawDocLibUrl">Raw URL for SharePoint document library</param>
/// <returns>Drive URL</returns>
public async Task <string> GetDriveUrlFromSharePointUrlAsync(string rawDocLibUrl)
{
try
{
if (string.IsNullOrEmpty(rawDocLibUrl))
{
return(rawDocLibUrl);
}
rawDocLibUrl = WebUtility.UrlDecode(rawDocLibUrl);
Match match = Regex.Match(rawDocLibUrl, @"(https?://([^/]+)((/[^/?]+)*?)(/[^/?]+))(/(Forms/\w+.aspx)?)?(\?.*)?$", RegexOptions.IgnoreCase);
string docLibUrl = match.Groups[1].Value;
string hostName = match.Groups[2].Value;
string siteRelativePath = match.Groups[3].Value;
if (string.IsNullOrEmpty(siteRelativePath))
{
siteRelativePath = "/";
}
GraphServiceClient graphServiceClient = await GraphServiceHelper.GetGraphServiceClientAsync();
if (graphServiceClient != null)
{
Site site = await graphServiceClient.Sites.GetByPath(siteRelativePath, hostName).Request().GetAsync();
ISiteDrivesCollectionPage drives = await graphServiceClient.Sites[site.Id].Drives.Request().GetAsync();
Drive drive = drives.SingleOrDefault(o => WebUtility.UrlDecode(o.WebUrl).Equals(docLibUrl, StringComparison.CurrentCultureIgnoreCase));
if (drive == null)
{
throw new Exception("Drive not found");
}
return(graphServiceClient.Drives[drive.Id].RequestUrl);
}
}
catch (Exception exception)
{
MessageDialog messageDialog = new MessageDialog(exception.Message);
await messageDialog.ShowAsync();
}
return(rawDocLibUrl);
}
static void Main(string[] args)
{
string clientId = "e0cefc2xxxxxxxxxxxx1063112";
string clientSecret = @"1kQxxxxxxxxxxxxxm05-qZ";
string tenantID = "8a400dxxxxxxxxxxxxxxxxc6872cfeef";
var host = "abc.sharepoint.com";
IConfidentialClientApplication confidentialClientApplication = ConfidentialClientApplicationBuilder
.Create(clientId)
.WithTenantId(tenantID)
.WithClientSecret(clientSecret)
.Build();
ClientCredentialProvider authProvider = new ClientCredentialProvider(confidentialClientApplication);
GraphServiceClient graphClient = new GraphServiceClient(authProvider);
var siteid = graphClient.Sites[host].Request().GetAsync().Result.Id;
Console.WriteLine(siteid);
ISiteDrivesCollectionPage drives = graphClient.Sites[host].Drives.Request().GetAsync().Result;
Drive driveIT = drives.CurrentPage.Where(d => d.DriveType == "documentLibrary" && d.Name == "IT").FirstOrDefault();
DriveItem testfile = graphClient.Sites[host].Drives[driveIT.Id].Root.ItemWithPath("test.txt").Request().GetAsync().Result;
// this cause the error
Stream streamtest = graphClient.Sites[siteid]
.Drives[driveIT.Id].Root.ItemWithPath("test.txt").Content
.Request()
.GetAsync().Result;
StreamReader sr = new StreamReader(streamtest);
Console.WriteLine(sr.ReadToEnd());
Console.ReadKey();
}
static async Task Main(string[] args)
{
// Load appsettings.json
var config = LoadAppSettings();
if (null == config)
{
Console.WriteLine("Missing or invalid appsettings.json file. Please see README.md for configuration instructions.");
return;
}
SetGlobalConfig(config);
searchServiceHelper = new SearchServiceHelper(SearchServiceName, SearchServiceAdminKey);
System.Diagnostics.Trace.TraceWarning("Slow response - database01");
TimeSpan elapsedTime;
//Start stopwatch for timing telemtry
Stopwatch sw = new Stopwatch();
var timeStart = DateTime.Now;
sw.Start();
//Storage
var storageAccount = CloudStorageAccount.Parse(StorageConnectionString);
var storageClient = storageAccount.CreateCloudBlobClient();
AzureTableStorage azTableStorage = new AzureTableStorage(StorageConnectionString, StorageTableName);
AzureTableStorage azTableStorageSpoItems = new AzureTableStorage(StorageConnectionString, SpoItemStorageTableName);
CloudBlobContainer container = await AzureBLOBStorage.CreateAzureBLOBContainer(storageClient, BlobContainerName);
//Search
AzureSearchServiceHelper searchClient = new AzureSearchServiceHelper(SearchServiceName, SearchServiceAdminKey);
IDriveItemChildrenCollectionPage docLibItems;
IDriveItemDeltaCollectionPage docLibDeltaItems;
for (int i = 0; i < args.Length; i++)
{
if (args[i].ToLower() == "-incrementalcrawl")
{
IncrementalCrawl = true;
Console.WriteLine("Search Crawl mode set to Incremental");
container = await AzureBLOBStorage.CreateAzureBLOBContainer(storageClient, BlobContainerName);
}
if (args[i].ToLower() == "-fullcrawl")
{
IncrementalCrawl = false;
Console.WriteLine("Search Crawl mode set to Full");
await AzureBLOBStorage.DeleteContainerFromAzureBLOB(container);
container = await AzureBLOBStorage.CreateAzureBLOBContainer(storageClient, BlobContainerName);
}
if (args[i].ToLower() == "-includeacls")
{
IncludeAcls = true;
Console.WriteLine("Search Crawl mode set to Full");
}
}
SharePointOnlineHelper.metadataFieldsToIgnore = MetadataFieldsToIgnore;
SharePointOnlineHelper.metadataJSONStore = MetadataJSONStore;
SharePointOnlineHelper.acls = IncludeAcls;
SharePointOnlineHelper.azTableStorage = azTableStorageSpoItems;
foreach (var metadataFieldToIgnore in MetadataFieldsToIgnore)
{
Console.WriteLine("Removing key [{0}] from metadata fields to extract", metadataFieldToIgnore);
}
//Query using Graph SDK (preferred when possible)
GraphServiceClient graphClient = SharePointOnlineHelper.GetAuthenticatedGraphClient(config);
Site targetSite = await graphClient.Sites.GetByPath(SiteUrl, SPOHostName).Request().GetAsync();
ISiteDrivesCollectionPage drives = graphClient.Sites[targetSite.Id].Drives.Request().GetAsync().Result;
//Graph BETA supports site pages
//var sitePages = graphClient.Sites[targetSite.Id].Pages.Request().GetAsync().GetAwaiter().GetResult();
//var sitePages = graphClient.Sites[targetSite.Id].Pages.Request().GetAsync().Result;
//var a = 1;
foreach (var drive in drives)
{
var driveName = drive.Name;
var driveUrl = drive.WebUrl;
bool excludedDocLIb = Array.Exists(DocLibsToIgnore, element => element == driveName);
if (excludedDocLIb)
{
Console.WriteLine("Skipping [{0}] as its an excluded docLib", DocLibsToIgnore);
continue;
}
Console.WriteLine("Fetching items from drive [{0}]", driveName);
var driveId = drive.Id;
var driveContents = new List <DriveItem>();
//Full Crawl Logic
if (!IncrementalCrawl)
{
docLibItems = await graphClient
.Drives[driveId]
.Root
.Children
.Request()
.GetAsync();
driveContents.AddRange(docLibItems.CurrentPage);
if (docLibItems.NextPageRequest != null)
{
while (docLibItems.NextPageRequest != null)
{
docLibItems = await docLibItems.NextPageRequest.GetAsync();
driveContents.AddRange(docLibItems.CurrentPage);
await SharePointOnlineHelper.GetSpoDocumentItems(graphClient, driveContents, driveId, container, IncludeAcls);
}
}
else
{
await SharePointOnlineHelper.GetSpoDocumentItems(graphClient, driveContents, driveId, container, IncludeAcls);
}
}
//Incremental Crawl Logic
if (IncrementalCrawl)
{
//Retrieve the last known deltaToken from Table storage, if the value is null it will fetch all items for that drive
//Base64 encode the string to remove special characters
byte[] byt = System.Text.Encoding.UTF8.GetBytes(driveUrl);
var driveUrlEscpaed = Convert.ToBase64String(byt);
var lastDeltaToken = await azTableStorage.GetEntitiesInPartion(driveUrlEscpaed);
docLibDeltaItems = await graphClient
.Drives[driveId]
.Root
.Delta(lastDeltaToken)
.Request()
.GetAsync();
var deltaLink = docLibDeltaItems.AdditionalData["@odata.deltaLink"].ToString();
if (deltaLink != null)
{
var tokenindex = deltaLink.IndexOf("token=");
var token = deltaLink.Substring(tokenindex + 7, deltaLink.ToString().Length - tokenindex - 9);
driveContents.AddRange(docLibDeltaItems.CurrentPage);
if (docLibDeltaItems.NextPageRequest != null)
{
while (docLibDeltaItems.NextPageRequest != null)
{
var docLibItems2 = await docLibDeltaItems.NextPageRequest.GetAsync();
driveContents.AddRange(docLibItems2.CurrentPage);
await SharePointOnlineHelper.GetSpoDocumentItems(graphClient, driveContents, driveId, container, IncludeAcls);
}
}
else
{
await SharePointOnlineHelper.GetSpoDocumentItems(graphClient, driveContents, driveId, container, IncludeAcls);
//Lets persist the changeToken to storage so we can continue the next incrmental crawl from this point.
IndexCrawlEntity indexCrawlEntity = new IndexCrawlEntity(driveUrlEscpaed, token);
azTableStorage.InsertEntity(indexCrawlEntity);
}
//Console.WriteLine("Fetched total of {0} documents from [{1}] data source", DownloadFileCount, driveName);
}
}
}
if (!IncrementalCrawl)
{
//Now lets do a full crawl of all the fetched SPO documents from the BLOB store as the fetching of all documents into storage would have completed by now
//Warning this will perform an entire search index rebuild - so while this phase is running search resultset will be impacted
await IndexDocumentsAsync();
}
sw.Stop();
elapsedTime = sw.Elapsed;
var timeEnd = DateTime.Now;
Console.WriteLine("Fetched total of {0} documents during crawl", AzureBLOBStorage.DownloadFileCount);
Console.WriteLine("Crawl Start time: {0}", timeStart);
Console.WriteLine("Crawl Completed time: {0}", timeEnd);
Console.WriteLine("Total crawl duration time: {0}", elapsedTime);
}
