private int GetContentSourceIdForUserProfiles(DocumentCrawlLog crawlLog)
{
var hostName = GetHostName();
var peopleContent = crawlLog.GetCrawledUrls(false, 100, $"sps3s://{hostName}-my.sharepoint.com", true, -1, (int)LogLevel.All, -1, DateTime.Now.AddDays(-100), DateTime.Now.AddDays(1));
ClientContext.ExecuteQueryRetry();
return((int)peopleContent.Value.Rows.First()["ContentSourceID"]);
}
private int GetContentSourceIdForSites(DocumentCrawlLog crawlLog)
{
var hostName = GetHostName();
var spContent = crawlLog.GetCrawledUrls(false, 10, $"https://{hostName}.sharepoint.com/sites", true, -1, (int)LogLevel.All, -1, DateTime.Now.AddDays(-100), DateTime.Now.AddDays(1));
ClientContext.ExecuteQueryRetry();
return((int)spContent.Value.Rows.First()["ContentSourceID"]);
}
protected override void ExecuteCmdlet()
{
try
{
var crawlLog = new DocumentCrawlLog(ClientContext, ClientContext.Site);
ClientContext.Load(crawlLog);
int contentSourceId;
switch (ContentSource)
{
case ContentSource.Sites:
contentSourceId = GetContentSourceIdForSites(crawlLog);
break;
case ContentSource.UserProfiles:
contentSourceId = GetContentSourceIdForUserProfiles(crawlLog);
break;
default:
throw new ArgumentOutOfRangeException();
}
string postFilter = string.Empty;
if (string.IsNullOrWhiteSpace(Filter) && ContentSource == ContentSource.Sites)
{
Filter = $"https://{GetHostName()}.sharepoint.com";
}
int origLimit = RowLimit;
if (ContentSource == ContentSource.UserProfiles)
{
postFilter = Filter;
Filter = $"https://{GetHostName()}-my.sharepoint.com";
RowLimit = MaxRows;
}
var logEntries = crawlLog.GetCrawledUrls(false, RowLimit, Filter, true, contentSourceId, (int)LogLevel, -1, StartDate, EndDate);
ClientContext.ExecuteQueryRetry();
var entries = new List <CrawlEntry>(logEntries.Value.Rows.Count);
foreach (var dictionary in logEntries.Value.Rows)
{
var entry = MapCrawlLogEntry(dictionary);
if (string.IsNullOrWhiteSpace(postFilter))
{
entries.Add(entry);
}
else if (entry.Url.Contains(postFilter))
{
entries.Add(entry);
}
}
WriteObject(entries.Take(origLimit).OrderByDescending(i => i.CrawlTime).ToList());
}
catch (Exception e)
{
WriteError(new ErrorRecord(new Exception("Make sure you are granted access to the crawl log via the SharePoint search admin center at https://<tenant>-admin.sharepoint.com/_layouts/15/searchadmin/TA_searchadministration.aspx"), e.Message, ErrorCategory.AuthenticationError, null));
}
}
private int GetContentSourceIdForUserProfiles(DocumentCrawlLog crawlLog)
{
var hostName = GetHostName();
var peopleContent = crawlLog.GetCrawledUrls(false, 100, $"sps3s://{hostName}-my.sharepoint.{PnP.Framework.AuthenticationManager.GetSharePointDomainSuffix(PnPConnection.CurrentConnection.AzureEnvironment)}", true, -1, (int)LogLevel.All, -1, DateTime.Now.AddDays(-100), DateTime.Now.AddDays(1));
ClientContext.ExecuteQueryRetry();
if (peopleContent.Value.Rows.Count > 0)
{
return((int)peopleContent.Value.Rows.First()["ContentSourceID"]);
}
return(-1);
}
private int GetContentSourceIdForSites(DocumentCrawlLog crawlLog)
{
var hostName = GetHostName();
var spContent = crawlLog.GetCrawledUrls(false, 10, $"https://{hostName}.sharepoint.{OfficeDevPnP.Core.AuthenticationManager.GetSharePointDomainSuffix(PnPConnection.CurrentConnection.AzureEnvironment)}/sites", true, -1, (int)LogLevel.All, -1, DateTime.Now.AddDays(-100), DateTime.Now.AddDays(1));
ClientContext.ExecuteQueryRetry();
if (spContent.Value.Rows.Count > 0)
{
return((int)spContent.Value.Rows.First()["ContentSourceID"]);
}
return(-1);
}
protected override void ExecuteCmdlet()
{
try
{
var crawlLog = new DocumentCrawlLog(ClientContext, ClientContext.Site);
ClientContext.Load(crawlLog);
int contentSourceId;
switch (ContentSource)
{
case ContentSource.Sites:
contentSourceId = GetContentSourceIdForSites(crawlLog);
break;
case ContentSource.UserProfiles:
contentSourceId = GetContentSourceIdForUserProfiles(crawlLog);
break;
default:
throw new ArgumentOutOfRangeException();
}
string postFilter = string.Empty;
if (string.IsNullOrWhiteSpace(Filter) && ContentSource == ContentSource.Sites)
{
Filter = $"https://{GetHostName()}.sharepoint.{PnP.Framework.AuthenticationManager.GetSharePointDomainSuffix(PnPConnection.CurrentConnection.AzureEnvironment)}";
}
int origLimit = RowLimit;
if (ContentSource == ContentSource.UserProfiles)
{
postFilter = Filter;
Filter = $"https://{GetHostName()}-my.sharepoint.{PnP.Framework.AuthenticationManager.GetSharePointDomainSuffix(PnPConnection.CurrentConnection.AzureEnvironment)}";
RowLimit = MaxRows;
}
var logEntries = crawlLog.GetCrawledUrls(false, RowLimit, Filter, true, contentSourceId, (int)LogLevel, -1, StartDate, EndDate);
ClientContext.ExecuteQueryRetry();
if (RawFormat)
{
var entries = new List <object>();
foreach (var dictionary in logEntries.Value.Rows)
{
string url = System.Net.WebUtility.UrlDecode(dictionary["FullUrl"].ToString());
if (ContentSource == ContentSource.UserProfiles && contentSourceId == -1)
{
if (!url.Contains(":443/person"))
{
continue;
}
}
if (string.IsNullOrWhiteSpace(postFilter) || url.Contains(postFilter))
{
entries.Add(ConvertToPSObject(dictionary));
}
}
WriteObject(entries.Take(origLimit));
}
else
{
var entries = new List <CrawlEntry>(logEntries.Value.Rows.Count);
foreach (var dictionary in logEntries.Value.Rows)
{
var entry = MapCrawlLogEntry(dictionary);
if (string.IsNullOrWhiteSpace(postFilter) || entry.Url.Contains(postFilter))
{
entries.Add(entry);
}
}
if (ContentSource == ContentSource.UserProfiles && contentSourceId == -1)
{
// Crawling has changed and uses one content source
// Need to apply post-filter to pull out profile entries only
entries =
entries.Where(e => System.Net.WebUtility.UrlDecode(e.Url.ToString()).ToLower().Contains(":443/person"))
.ToList();
}
WriteObject(entries.Take(origLimit).OrderByDescending(i => i.CrawlTime).ToList());
}
}
catch (Exception e)
{
WriteError(new ErrorRecord(new Exception("Make sure you are granted access to the crawl log via the SharePoint search admin center at https://<tenant>-admin.sharepoint.com/_layouts/15/searchadmin/crawllogreadpermission.aspx"), e.Message, ErrorCategory.AuthenticationError, null));
}
}