public void LogScanError(Exception ex, ScanError error)
{
if (this.telemetryClient == null || ex == null)
{
return;
}
try
{
// Prepare event data
Dictionary <string, string> properties = new Dictionary <string, string>();
Dictionary <string, double> metrics = new Dictionary <string, double>();
if (error != null)
{
// Field 1 never contain PII data
if (!string.IsNullOrEmpty(error.Field1))
{
properties.Add(ScanCrash.StackTrace.ToString(), error.Field1);
}
}
this.telemetryClient.TrackException(ex, properties, metrics);
}
catch (Exception ex2)
{
// Eat all exceptions
}
}
/// <summary>
/// Analyses a web for it's blog page usage
/// </summary>
/// <param name="cc">ClientContext instance used to retrieve blog data</param>
/// <returns>Duration of the blog analysis</returns>
public override TimeSpan Analyze(ClientContext cc)
{
try
{
var web = cc.Web;
base.Analyze(cc);
// Load the customized forms per site collection
cc.Load(cc.Site, p => p.CustomizedFormsPages);
cc.ExecuteQueryRetry();
foreach (var formPage in cc.Site.CustomizedFormsPages)
{
CustomizedFormsScanResult customizedFormsScanResult = new CustomizedFormsScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
//WebRelativeUrl = this.SiteUrl.Replace(this.SiteCollectionUrl, ""),
FormType = formPage.formType,
Url = formPage.Url,
PageId = formPage.pageId,
WebpartId = formPage.webpartId,
};
if (!this.ScanJob.CustomizedFormsScanResults.TryAdd($"customizedFormsScanResult.WebURL.{Guid.NewGuid()}", customizedFormsScanResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add customized forms scan result for {customizedFormsScanResult.SiteColUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "CustomizedFormsAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
}
catch (Exception ex)
{
ScanError error = new ScanError()
{
Error = ex.Message,
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "CustomizedFormsAnalyzerLoop",
Field2 = ex.StackTrace
};
this.ScanJob.ScanErrors.Push(error);
}
finally
{
this.StopTime = DateTime.Now;
}
// return the duration of this scan
return(new TimeSpan((this.StopTime.Subtract(this.StartTime).Ticks)));
}
private async void buttonConvert_Click(object sender, RoutedEventArgs e)
{
buttonConvert.IsEnabled = false;
Cursor = Cursors.Wait;
textBlockScanStatus.Text = "scanning...";
progressBarScan.Visibility = Visibility.Visible;
taskBarProgressScan.ProgressState = System.Windows.Shell.TaskbarItemProgressState.Indeterminate;
monsterListDataTable.Rows.Clear();
string inputDir = textBoxInputPath.Text;
string outputDir = textBoxOutputPath.Text;
IMonsterConverter inputFormat = (IMonsterConverter)comboInputFormat.SelectedItem;
IMonsterConverter outputFormat = (IMonsterConverter)comboOutputFormat.SelectedItem;
ScanError result = ScanError.Success;
await Task.Run(() =>
{
result = fileProcessor.ConvertMonsterFiles(inputDir, inputFormat, outputDir, outputFormat, true);
});
switch (result)
{
case ScanError.Success:
textBlockScanStatus.Text = "Completed successfully.";
break;
case ScanError.NoMonstersFound:
textBlockScanStatus.Text = "Couldn't find any monster files.";
break;
case ScanError.InvalidMonsterDirectory:
textBlockScanStatus.Text = "The selected project directory is invald.";
break;
case ScanError.InvalidMonsterFormat:
textBlockScanStatus.Text = "The selected input or output monster format is invalid.";
break;
case ScanError.CouldNotCreateDirectory:
textBlockScanStatus.Text = "Couldn't create destination directory.";
break;
case ScanError.DirectoriesMatch:
textBlockScanStatus.Text = "Input and output directories can't be the same.";
break;
default:
break;
}
taskBarProgressScan.ProgressState = System.Windows.Shell.TaskbarItemProgressState.None;
progressBarScan.Visibility = Visibility.Hidden;
Cursor = Cursors.Arrow;
buttonConvert.IsEnabled = true;
}
public bool ScanFiles()
{
Console.WriteLine("Scanning...");
ScanError result = fileProcessor.ConvertMonsterFiles(inputDirectory, input, outputDirectory, output, mirrorFolderStructure);
switch (result)
{
case ScanError.Success:
Console.WriteLine("Completed.");
Console.WriteLine($"{ConvertSuccessCount} Monsters converted succesfully.");
Console.WriteLine($"{ConvertWarningCount} Monsters converted with warnings.");
Console.WriteLine($"{ConvertErrorCount} Monsters converted with errors.");
break;
case ScanError.NoMonstersFound:
Console.WriteLine("Couldn't find any monster files.");
break;
case ScanError.InvalidMonsterDirectory:
Console.WriteLine("The selected project directory is invald.");
break;
case ScanError.InvalidMonsterFormat:
Console.WriteLine("The selected input or output monster format is invalid.");
break;
case ScanError.CouldNotCreateDirectory:
Console.WriteLine("Couldn't create destination directory.");
break;
case ScanError.DirectoriesMatch:
Console.WriteLine("Input and output directories can't be the same.");
break;
default:
break;
}
return(result == ScanError.Success);
}
public void LogScanError(Exception ex, ScanError error)
{
if (this.telemetryClient == null || ex == null)
{
return;
}
try
{
// Prepare event data
Dictionary <string, string> properties = new Dictionary <string, string>();
Dictionary <string, double> metrics = new Dictionary <string, double>();
// Populate properties
// Page transformation engine version
properties.Add(EngineVersion, this.version);
if (error != null)
{
// Field 1 never contain PII data
if (!string.IsNullOrEmpty(error.Field1))
{
properties.Add(ScanCrash.StackTrace.ToString(), error.Field1);
}
}
// Azure AD tenant
properties.Add(AADTenantId, this.aadTenantId.ToString());
// Send the event
this.telemetryClient.TrackException(ex, properties, metrics);
}
catch
{
// Eat all exceptions
}
}
public override TimeSpan Analyze(ClientContext cc)
{
try
{
base.Analyze(cc);
// Only scan when it's a valid publishing portal
var pageCount = ContinueScanning(cc);
if (pageCount > 0 || pageCount == -1)
{
PublishingScanResult scanResult = new PublishingScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
WebRelativeUrl = this.SiteUrl.Replace(this.SiteCollectionUrl, ""),
WebTemplate = this.webScanResult.WebTemplate,
BrokenPermissionInheritance = this.webScanResult.BrokenPermissionInheritance,
PageCount = pageCount == -1 ? 0 : pageCount,
SiteMasterPage = this.webScanResult.CustomMasterPage,
SystemMasterPage = this.webScanResult.MasterPage,
AlternateCSS = this.webScanResult.AlternateCSS,
Admins = this.siteScanResult.Admins,
Owners = this.webScanResult.Owners,
};
Web web = cc.Web;
// Load additional web properties
web.EnsureProperties(p => p.Language);
scanResult.Language = web.Language;
// PageLayouts handling
var availablePageLayouts = web.GetPropertyBagValueString(AvailablePageLayouts, "");
var defaultPageLayout = web.GetPropertyBagValueString(DefaultPageLayout, "");
if (string.IsNullOrEmpty(availablePageLayouts))
{
scanResult.PageLayoutsConfiguration = "Any";
}
else if (availablePageLayouts.Equals("__inherit", StringComparison.InvariantCultureIgnoreCase))
{
scanResult.PageLayoutsConfiguration = "Inherit from parent";
}
else
{
scanResult.PageLayoutsConfiguration = "Defined list";
// Fill the defined list
var element = XElement.Parse(availablePageLayouts);
var nodes = element.Descendants("layout");
if (nodes != null && nodes.Count() > 0)
{
string allowedPageLayouts = "";
foreach (var node in nodes)
{
allowedPageLayouts = allowedPageLayouts + node.Attribute("url").Value.Replace("_catalogs/masterpage/", "") + ",";
}
allowedPageLayouts = allowedPageLayouts.TrimEnd(new char[] { ',' });
scanResult.AllowedPageLayouts = allowedPageLayouts;
}
}
if (!string.IsNullOrEmpty(defaultPageLayout))
{
var element = XElement.Parse(defaultPageLayout);
scanResult.DefaultPageLayout = element.Attribute("url").Value.Replace("_catalogs/masterpage/", "");
}
// Navigation
var navigationSettings = web.GetNavigationSettings();
if (navigationSettings != null)
{
if (navigationSettings.GlobalNavigation.ManagedNavigation)
{
scanResult.GlobalNavigationType = "Managed";
}
else
{
scanResult.GlobalNavigationType = "Structural";
scanResult.GlobalStruturalNavigationMaxCount = navigationSettings.GlobalNavigation.MaxDynamicItems;
scanResult.GlobalStruturalNavigationShowPages = navigationSettings.GlobalNavigation.ShowPages;
scanResult.GlobalStruturalNavigationShowSiblings = navigationSettings.GlobalNavigation.ShowSiblings;
scanResult.GlobalStruturalNavigationShowSubSites = navigationSettings.GlobalNavigation.ShowSubsites;
}
if (navigationSettings.CurrentNavigation.ManagedNavigation)
{
scanResult.CurrentNavigationType = "Managed";
}
else
{
scanResult.CurrentNavigationType = "Structural";
scanResult.CurrentStruturalNavigationMaxCount = navigationSettings.CurrentNavigation.MaxDynamicItems;
scanResult.CurrentStruturalNavigationShowPages = navigationSettings.CurrentNavigation.ShowPages;
scanResult.CurrentStruturalNavigationShowSiblings = navigationSettings.CurrentNavigation.ShowSiblings;
scanResult.CurrentStruturalNavigationShowSubSites = navigationSettings.CurrentNavigation.ShowSubsites;
}
if (navigationSettings.GlobalNavigation.ManagedNavigation || navigationSettings.CurrentNavigation.ManagedNavigation)
{
scanResult.ManagedNavigationAddNewPages = navigationSettings.AddNewPagesToNavigation;
scanResult.ManagedNavigationCreateFriendlyUrls = navigationSettings.CreateFriendlyUrlsForNewPages;
// get information about the managed nav term set configuration
var managedNavXml = web.GetPropertyBagValueString(WebNavigationSettings, "");
if (!string.IsNullOrEmpty(managedNavXml))
{
var managedNavSettings = XElement.Parse(managedNavXml);
IEnumerable <XElement> navNodes = managedNavSettings.XPathSelectElements("./SiteMapProviderSettings/TaxonomySiteMapProviderSettings");
foreach (var node in navNodes)
{
if (node.Attribute("Name").Value.Equals("CurrentNavigationTaxonomyProvider", StringComparison.InvariantCulture))
{
if (node.Attribute("TermSetId") != null)
{
scanResult.CurrentManagedNavigationTermSetId = node.Attribute("TermSetId").Value;
}
else if (node.Attribute("UseParentSiteMap") != null)
{
scanResult.CurrentManagedNavigationTermSetId = "Inherit from parent";
}
}
else if (node.Attribute("Name").Value.Equals("GlobalNavigationTaxonomyProvider", StringComparison.InvariantCulture))
{
if (node.Attribute("TermSetId") != null)
{
scanResult.GlobalManagedNavigationTermSetId = node.Attribute("TermSetId").Value;
}
else if (node.Attribute("UseParentSiteMap") != null)
{
scanResult.GlobalManagedNavigationTermSetId = "Inherit from parent";
}
}
}
}
}
}
// Pages library
var pagesLibrary = web.GetListsToScan().Where(p => p.BaseTemplate == 850).FirstOrDefault();
if (pagesLibrary != null)
{
pagesLibrary.EnsureProperties(p => p.EnableModeration, p => p.EnableVersioning, p => p.EnableMinorVersions, p => p.EventReceivers, p => p.Fields, p => p.DefaultContentApprovalWorkflowId);
scanResult.LibraryEnableModeration = pagesLibrary.EnableModeration;
scanResult.LibraryEnableVersioning = pagesLibrary.EnableVersioning;
scanResult.LibraryEnableMinorVersions = pagesLibrary.EnableMinorVersions;
scanResult.LibraryItemScheduling = pagesLibrary.ItemSchedulingEnabled();
scanResult.LibraryApprovalWorkflowDefined = pagesLibrary.DefaultContentApprovalWorkflowId != Guid.Empty;
}
// Variations
if (scanResult.Level == 0)
{
var variationLabels = cc.GetVariationLabels();
string labels = "";
string sourceLabel = "";
foreach (var label in variationLabels)
{
labels = labels + $"{label.Title} ({label.Language}),";
if (label.IsSource)
{
sourceLabel = label.Title;
}
}
scanResult.VariationLabels = labels.TrimEnd(new char[] { ',' });;
scanResult.VariationSourceLabel = sourceLabel;
}
// Persist publishing scan results
if (!this.ScanJob.PublishingScanResults.TryAdd(this.SiteUrl, scanResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add publishing scan result for {this.SiteUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "PublishingAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
}
finally
{
this.StopTime = DateTime.Now;
}
// return the duration of this scan
return(new TimeSpan((this.StopTime.Subtract(this.StartTime).Ticks)));
}
/// <summary>
/// Analyses a page
/// </summary>
/// <param name="cc">ClientContext instance used to retrieve page data</param>
/// <returns>Duration of the page analysis</returns>
public override TimeSpan Analyze(ClientContext cc)
{
try
{
base.Analyze(cc);
Web web = cc.Web;
cc.Web.EnsureProperties(p => p.WebTemplate, p => p.Configuration, p => p.Features);
var homePageUrl = web.WelcomePage;
if (string.IsNullOrEmpty(homePageUrl))
{
// Will be case when the site home page is a web part page
homePageUrl = "default.aspx";
}
var listsToScan = web.GetListsToScan();
var sitePagesLibraries = listsToScan.Where(p => p.BaseTemplate == (int)ListTemplateType.WebPageLibrary);
if (sitePagesLibraries.Count() > 0)
{
foreach (var sitePagesLibrary in sitePagesLibraries)
{
CamlQuery query = new CamlQuery
{
ViewXml = CAMLQueryByExtension
};
var pages = sitePagesLibrary.GetItems(query);
web.Context.Load(pages);
web.Context.ExecuteQueryRetry();
if (pages.FirstOrDefault() != null)
{
DateTime start;
bool forceCheckout = sitePagesLibrary.ForceCheckout;
foreach (var page in pages)
{
string pageUrl = null;
try
{
if (page.FieldValues.ContainsKey(Field_FileRefField) && !String.IsNullOrEmpty(page[Field_FileRefField].ToString()))
{
pageUrl = page[Field_FileRefField].ToString();
}
else
{
//skip page
continue;
}
start = DateTime.Now;
PageScanResult pageResult = new PageScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
PageUrl = pageUrl,
Library = sitePagesLibrary.RootFolder.ServerRelativeUrl,
};
// Is this page the web's home page?
if (pageUrl.EndsWith(homePageUrl, StringComparison.InvariantCultureIgnoreCase))
{
pageResult.HomePage = true;
}
// Get the type of the page
pageResult.PageType = page.PageType();
// Get page web parts
var pageAnalysis = page.WebParts(this.ScanJob.PageTransformation);
if (pageAnalysis != null)
{
pageResult.Layout = pageAnalysis.Item1.ToString().Replace("Wiki_", "").Replace("WebPart_", "");
pageResult.WebParts = pageAnalysis.Item2;
}
// Determine if this site contains a default "uncustomized" home page
bool isUncustomizedHomePage = false;
try
{
string pageName = "";
if (page.FieldValues.ContainsKey(Field_FileLeafRef) && !String.IsNullOrEmpty(page[Field_FileLeafRef].ToString()))
{
pageName = page[Field_FileLeafRef].ToString();
}
if (pageResult.HomePage && web.WebTemplate == "STS" && web.Configuration == 0 && pageName.Equals("home.aspx", StringComparison.InvariantCultureIgnoreCase))
{
bool homePageModernizationOptedOut = web.Features.Where(f => f.DefinitionId == FeatureId_Web_HomePage).Count() > 0;
if (!homePageModernizationOptedOut)
{
bool siteWasGroupified = web.Features.Where(f => f.DefinitionId == FeatureId_Web_GroupHomepage).Count() > 0;
if (!siteWasGroupified)
{
var wiki = page.FieldValues[Field_WikiField].ToString();
if (!string.IsNullOrEmpty(wiki))
{
var isHtmlUncustomized = IsHtmlUncustomized(wiki);
if (isHtmlUncustomized)
{
string pageType = GetPageWebPartInfo(pageResult.WebParts);
if (pageType == TeamSiteDefaultWebParts)
{
page.ContentType.EnsureProperty(p => p.DisplayFormTemplateName);
if (page.ContentType.DisplayFormTemplateName == "WikiEditForm")
{
isUncustomizedHomePage = true;
}
}
}
}
}
}
}
}
catch (Exception ex)
{
// no point in failing the scan if something goes wrong here
}
finally
{
pageResult.UncustomizedHomePage = isUncustomizedHomePage;
}
// Get page change information
pageResult.ModifiedAt = page.LastModifiedDateTime();
pageResult.ModifiedBy = page.LastModifiedBy();
// Grab this page from the search results to connect view information
string fullPageUrl = $"https://{new Uri(this.SiteCollectionUrl).DnsSafeHost}{pageUrl}";
if (pageResult.HomePage)
{
fullPageUrl = this.SiteUrl;
}
if (!this.ScanJob.SkipUsageInformation && this.pageSearchResults != null)
{
var searchPage = this.pageSearchResults.Where(x => x.Values.Contains(fullPageUrl)).FirstOrDefault();
if (searchPage != null)
{
// Recent = last 14 days
pageResult.ViewsRecent = searchPage["ViewsRecent"].ToInt32();
pageResult.ViewsRecentUniqueUsers = searchPage["ViewsRecentUniqueUsers"].ToInt32();
pageResult.ViewsLifeTime = searchPage["ViewsLifeTime"].ToInt32();
pageResult.ViewsLifeTimeUniqueUsers = searchPage["ViewsLifeTimeUniqueUsers"].ToInt32();
}
}
if (!this.ScanJob.PageScanResults.TryAdd(pageResult.PageUrl, pageResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add page scan result for {pageResult.PageUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "PageAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
var duration = new TimeSpan((DateTime.Now.Subtract(start).Ticks));
Console.WriteLine($"Scan of page {pageUrl} took {duration.Seconds} seconds");
}
catch (Exception ex)
{
ScanError error = new ScanError()
{
Error = ex.Message,
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "MainPageAnalyzerLoop",
Field2 = ex.StackTrace,
Field3 = pageUrl
};
// Send error to telemetry to make scanner better
if (this.ScanJob.ScannerTelemetry != null)
{
this.ScanJob.ScannerTelemetry.LogScanError(ex, error);
}
this.ScanJob.ScanErrors.Push(error);
Console.WriteLine("Error for page {1}: {0}", ex.Message, pageUrl);
}
}
}
}
}
}
finally
{
this.StopTime = DateTime.Now;
}
// return the duration of this scan
return(new TimeSpan((this.StopTime.Subtract(this.StartTime).Ticks)));
}
private void ModernizationScanJob_TimerJobRun(object sender, OfficeDevPnP.Core.Framework.TimerJobs.TimerJobRunEventArgs e)
{
// Validate ClientContext objects
if (e.WebClientContext == null || e.SiteClientContext == null)
{
ScanError error = new ScanError()
{
Error = "No valid ClientContext objects",
SiteURL = e.Url,
SiteColUrl = e.Url
};
this.ScanErrors.Push(error);
Console.WriteLine("Error for site {1}: {0}", "No valid ClientContext objects", e.Url);
// bail out
return;
}
// thread safe increase of the sites counter
IncreaseScannedSites();
try
{
// Set the first site collection done flag + perform telemetry
SetFirstSiteCollectionDone(e.WebClientContext, this.Name);
// Manually iterate over the content
IEnumerable <string> expandedSites = e.SiteClientContext.Site.GetAllSubSites();
bool isFirstSiteInList = true;
string siteCollectionUrl = "";
List <Dictionary <string, string> > pageSearchResults = null;
foreach (string site in expandedSites)
{
try
{
// thread safe increase of the webs counter
IncreaseScannedWebs();
// Clone the existing ClientContext for the sub web
using (ClientContext ccWeb = e.SiteClientContext.Clone(site))
{
Console.WriteLine("Processing site {0}...", site);
// Allow max server time out, might be needed for sites having a lot of users
ccWeb.RequestTimeout = Timeout.Infinite;
if (isFirstSiteInList)
{
// Perf optimization: do one call per site to load all the needed properties
var spSite = (ccWeb as ClientContext).Site;
ccWeb.Load(spSite, p => p.RootWeb, p => p.Url, p => p.GroupId);
ccWeb.Load(spSite.RootWeb, p => p.Id);
ccWeb.Load(spSite, p => p.UserCustomActions); // User custom action site level
ccWeb.Load(spSite, p => p.Features); // Features site level
ccWeb.ExecuteQueryRetry();
isFirstSiteInList = false;
}
// Perf optimization: do one call per web to load all the needed properties
ccWeb.Load(ccWeb.Web, p => p.Id, p => p.Title, p => p.Url);
ccWeb.Load(ccWeb.Web, p => p.WebTemplate, p => p.Configuration);
ccWeb.Load(ccWeb.Web, p => p.MasterUrl, p => p.CustomMasterUrl, // master page check
p => p.AlternateCssUrl, // Alternate CSS
p => p.UserCustomActions); // Web user custom actions
ccWeb.Load(ccWeb.Web, p => p.Features); // Features web level
ccWeb.Load(ccWeb.Web, p => p.RootFolder); // web home page
ccWeb.ExecuteQueryRetry();
// Split load in multiple batches to minimize timeout exceptions
if (!SkipUserInformation)
{
ccWeb.Load(ccWeb.Web, p => p.SiteUsers, p => p.AssociatedOwnerGroup, p => p.AssociatedMemberGroup, p => p.AssociatedVisitorGroup); // site user and groups
ccWeb.Load(ccWeb.Web, p => p.HasUniqueRoleAssignments, p => p.RoleAssignments, p => p.SiteGroups.Include(s => s.Users)); // permission inheritance at web level
ccWeb.ExecuteQueryRetry();
ccWeb.Load(ccWeb.Web.AssociatedOwnerGroup, p => p.Users); // users in the Owners group
ccWeb.Load(ccWeb.Web.AssociatedMemberGroup, p => p.Users); // users in the Members group
ccWeb.Load(ccWeb.Web.AssociatedVisitorGroup, p => p.Users); // users in the Visitors group
ccWeb.ExecuteQueryRetry();
}
// Do things only once per site collection
if (string.IsNullOrEmpty(siteCollectionUrl))
{
// Cross check Url property availability
ccWeb.Site.EnsureProperty(s => s.Url);
siteCollectionUrl = ccWeb.Site.Url;
// Site scan
SiteAnalyzer siteAnalyzer = new SiteAnalyzer(site, siteCollectionUrl, this);
var siteScanDuration = siteAnalyzer.Analyze(ccWeb);
pageSearchResults = siteAnalyzer.PageSearchResults;
}
// Web scan
WebAnalyzer webAnalyzer = new WebAnalyzer(site, siteCollectionUrl, this, pageSearchResults);
var webScanDuration = webAnalyzer.Analyze(ccWeb);
}
}
catch (Exception ex)
{
ScanError error = new ScanError()
{
Error = ex.Message,
SiteColUrl = e.Url,
SiteURL = site,
Field1 = "MainWebLoop",
Field2 = ex.StackTrace,
};
this.ScanErrors.Push(error);
Console.WriteLine("Error for site {1}: {0}", ex.Message, site);
}
}
}
catch (Exception ex)
{
ScanError error = new ScanError()
{
Error = ex.Message,
SiteColUrl = e.Url,
SiteURL = e.Url,
Field1 = "MainSiteLoop",
Field2 = ex.StackTrace,
};
this.ScanErrors.Push(error);
Console.WriteLine("Error for site {1}: {0}", ex.Message, e.Url);
}
// Output the scanning progress
try
{
TimeSpan ts = DateTime.Now.Subtract(this.StartTime);
Console.WriteLine($"Thread: {Thread.CurrentThread.ManagedThreadId}. Processed {this.ScannedSites} of {this.SitesToScan} site collections ({Math.Round(((float)this.ScannedSites / (float)this.SitesToScan) * 100)}%). Process running for {ts.Days} days, {ts.Hours} hours, {ts.Minutes} minutes and {ts.Seconds} seconds.");
}
catch (Exception ex)
{
Console.WriteLine($"Error showing progress: {ex.ToString()}");
}
}
public override TimeSpan Analyze(ClientContext cc)
{
// Get site usage information
List <string> propertiesToRetrieve = new List <string>
{
"LastModifiedTime",
"ModifiedBy",
"DetectedLanguage",
"SPWebUrl",
"Path",
"Title",
"ViewsRecent",
"ViewsRecentUniqueUsers",
"ViewsLifeTime",
"ViewsLifeTimeUniqueUsers"
};
Uri rootSite = new Uri(this.SiteCollectionUrl);
string pathFilter = $"{rootSite.Scheme}://{rootSite.DnsSafeHost}/portals/personal/*";
Dictionary <string, BlogWebScanResult> tempWebResults = new Dictionary <string, BlogWebScanResult>();
var results = this.ScanJob.Search(cc.Web, $"path:{pathFilter} AND ContentTypeId:\"0x010100DA3A7E6E3DB34DFF8FDEDE1F4EBAF95D*\"", propertiesToRetrieve);
if (results != null && results.Count > 0)
{
foreach (var result in results)
{
string url = result["SPWebUrl"]?.ToLower().Replace($"{rootSite.Scheme}://{rootSite.DnsSafeHost}".ToLower(), "");
DateTime lastModified = DateTime.MinValue;
if (result["LastModifiedTime"] != null)
{
DateTime.TryParse(result["LastModifiedTime"], out lastModified);
}
BlogWebScanResult scanResult = null;
BlogPageScanResult blogPageScanResult;
if (tempWebResults.ContainsKey(url))
{
// Increase page counter
tempWebResults[url].BlogPageCount += 1;
// Build page record
blogPageScanResult = AddBlogPageResult($"{rootSite.Scheme}://{rootSite.DnsSafeHost}".ToLower(), url, result, lastModified);
}
else
{
scanResult = new BlogWebScanResult
{
SiteColUrl = result["SPWebUrl"],
SiteURL = result["SPWebUrl"],
WebRelativeUrl = url,
WebTemplate = "POINTPUBLISHINGPERSONAL#0",
BlogType = BlogType.Delve,
BlogPageCount = 1,
LastRecentBlogPageChange = lastModified,
LastRecentBlogPagePublish = lastModified,
Language = 1033
};
tempWebResults.Add(url, scanResult);
// Build page record
blogPageScanResult = AddBlogPageResult($"{rootSite.Scheme}://{rootSite.DnsSafeHost}".ToLower(), url, result, lastModified);
}
if (blogPageScanResult != null)
{
if (!this.ScanJob.BlogPageScanResults.TryAdd($"blogScanResult.PageURL.{Guid.NewGuid()}", blogPageScanResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add delve blog page scan result for {blogPageScanResult.SiteColUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "DelveBlogPageAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
}
// Copy the temp scan results to the actual structure
foreach (var blogWebResult in tempWebResults)
{
if (!this.ScanJob.BlogWebScanResults.TryAdd($"blogScanResult.WebURL.{Guid.NewGuid()}", blogWebResult.Value))
{
ScanError error = new ScanError()
{
Error = $"Could not add delve blog web scan result for {blogWebResult.Value.SiteColUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "DelveBlogWebAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
}
return(base.Analyze(cc));
}
/// <summary>
/// Analyses a web for it's blog page usage
/// </summary>
/// <param name="cc">ClientContext instance used to retrieve blog data</param>
/// <returns>Duration of the blog analysis</returns>
public override TimeSpan Analyze(ClientContext cc)
{
try
{
// Is this a blog site
if (cc.Web.WebTemplate.Equals("BLOG", StringComparison.InvariantCultureIgnoreCase))
{
var web = cc.Web;
base.Analyze(cc);
BlogWebScanResult blogWebScanResult = new BlogWebScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
WebRelativeUrl = this.SiteUrl.Replace(this.SiteCollectionUrl, ""),
};
// Log used web template
if (web.WebTemplate != null)
{
blogWebScanResult.WebTemplate = $"{web.WebTemplate}#{web.Configuration}";
}
// Load additional web properties
web.EnsureProperty(p => p.Language);
blogWebScanResult.Language = web.Language;
// Get the blog page list
List blogList = null;
var lists = web.GetListsToScan();
if (lists != null)
{
blogList = lists.Where(p => p.BaseTemplate == (int)ListTemplateType.Posts).FirstOrDefault();
}
// Query the blog posts
if (blogList != null)
{
CamlQuery query = CamlQuery.CreateAllItemsQuery(10000, new string[] { "Title", "Body", "NumComments", "PostCategory", "PublishedDate", "Modified", "Created", "Editor", "Author" });
var pages = blogList.GetItems(query);
cc.Load(pages);
cc.ExecuteQueryRetry();
if (pages != null)
{
blogWebScanResult.BlogPageCount = pages.Count;
blogWebScanResult.LastRecentBlogPageChange = blogList.LastItemUserModifiedDate;
blogWebScanResult.LastRecentBlogPagePublish = blogList.LastItemUserModifiedDate;
foreach (var page in pages)
{
string pageUrl = null;
try
{
if (page.FieldValues.ContainsKey(FileRefField) && !String.IsNullOrEmpty(page[FileRefField].ToString()))
{
pageUrl = page[FileRefField].ToString().ToLower();
}
else
{
//skip page
continue;
}
BlogPageScanResult blogPageScanResult = new BlogPageScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
WebRelativeUrl = blogWebScanResult.WebRelativeUrl,
PageRelativeUrl = pageUrl,
};
if (page.FieldValues.ContainsKey(FileRefField) && !String.IsNullOrEmpty(page[FileRefField].ToString()))
{
blogPageScanResult.PageTitle = page["Title"].ToString();
}
// Add modified information
blogPageScanResult.ModifiedBy = page.LastModifiedBy();
blogPageScanResult.ModifiedAt = page.LastModifiedDateTime();
blogPageScanResult.PublishedDate = page.LastPublishedDateTime();
if (blogPageScanResult.ModifiedAt > blogWebScanResult.LastRecentBlogPageChange)
{
blogWebScanResult.LastRecentBlogPageChange = blogPageScanResult.ModifiedAt;
}
if (blogPageScanResult.PublishedDate > blogWebScanResult.LastRecentBlogPagePublish)
{
blogWebScanResult.LastRecentBlogPagePublish = blogPageScanResult.PublishedDate;
}
if (!this.ScanJob.BlogPageScanResults.TryAdd($"blogScanResult.PageURL.{Guid.NewGuid()}", blogPageScanResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add blog page scan result for {blogPageScanResult.SiteColUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "BlogPageAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
catch (Exception ex)
{
ScanError error = new ScanError()
{
Error = ex.Message,
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "BlogPageAnalyzer",
Field2 = ex.StackTrace,
Field3 = pageUrl
};
this.ScanJob.ScanErrors.Push(error);
}
}
}
}
if (!this.ScanJob.BlogWebScanResults.TryAdd($"blogScanResult.WebURL.{Guid.NewGuid()}", blogWebScanResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add blog web scan result for {blogWebScanResult.SiteColUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "BlogWebAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
}
catch (Exception ex)
{
ScanError error = new ScanError()
{
Error = ex.Message,
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "BlogPageAnalyzerMainLoop",
Field2 = ex.StackTrace
};
this.ScanJob.ScanErrors.Push(error);
}
finally
{
this.StopTime = DateTime.Now;
}
// return the duration of this scan
return(new TimeSpan((this.StopTime.Subtract(this.StartTime).Ticks)));
}
private void VisioScanJob_TimerJobRun(object sender, OfficeDevPnP.Core.Framework.TimerJobs.TimerJobRunEventArgs e)
{
// Validate ClientContext objects
if (e.WebClientContext == null || e.SiteClientContext == null)
{
ScanError error = new ScanError()
{
Error = "No valid ClientContext objects",
SiteURL = e.Url,
SiteColUrl = e.Url
};
this.ScanErrors.Push(error);
Console.WriteLine("Error for site {1}: {0}", "No valid ClientContext objects", e.Url);
// bail out
return;
}
// thread safe increase of the sites counter
IncreaseScannedSites();
try
{
// Set the first site collection done flag + perform telemetry
SetFirstSiteCollectionDone(e.WebClientContext, this.Name);
// Manually iterate over the content
IEnumerable <string> expandedSites = e.SiteClientContext.Site.GetAllSubSites();
bool isFirstSiteInList = true;
string siteCollectionUrl = "";
List <Dictionary <string, string> > pageSearchResults = null;
foreach (string site in expandedSites)
{
try
{
// thread safe increase of the webs counter
IncreaseScannedWebs();
// Clone the existing ClientContext for the sub web
using (ClientContext ccWeb = e.SiteClientContext.Clone(site))
{
Console.WriteLine("Processing site {0}...", site);
// Allow max server time out, might be needed for sites having a lot of users
ccWeb.RequestTimeout = Timeout.Infinite;
if (isFirstSiteInList)
{
// Perf optimization: do one call per site to load all the needed properties
var spSite = (ccWeb as ClientContext).Site;
ccWeb.Load(spSite, p => p.Url);
ccWeb.ExecuteQueryRetry();
isFirstSiteInList = false;
}
ListCollection listCollection = ccWeb.Web.Lists;
ccWeb.Load(listCollection, coll => coll.Include(li => li.Title, li => li.Hidden, li => li.DefaultViewUrl, li => li.BaseTemplate, li => li.RootFolder));
ccWeb.ExecuteQueryRetry();
// Do things only once per site collection
if (string.IsNullOrEmpty(siteCollectionUrl))
{
// Cross check Url property availability
ccWeb.Site.EnsureProperty(s => s.Url);
siteCollectionUrl = ccWeb.Site.Url;
// Site scan
SiteAnalyzer siteAnalyzer = new SiteAnalyzer(site, siteCollectionUrl, this);
var siteScanDuration = siteAnalyzer.Analyze(ccWeb);
pageSearchResults = siteAnalyzer.PageSearchResults;
}
VisioWebPartAnalyzer visioWebPartAnalyzer = new VisioWebPartAnalyzer(site, siteCollectionUrl, this, pageSearchResults);
visioWebPartAnalyzer.Analyze(ccWeb);
}
}
catch (Exception ex)
{
ScanError error = new ScanError()
{
Error = ex.Message,
SiteColUrl = e.Url,
SiteURL = site,
Field1 = "MainWebLoop",
Field2 = ex.StackTrace,
};
this.ScanErrors.Push(error);
Console.WriteLine("Error for site {1}: {0}", ex.Message, site);
}
}
}
catch (Exception ex)
{
ScanError error = new ScanError()
{
Error = ex.Message,
SiteColUrl = e.Url,
SiteURL = e.Url,
Field1 = "MainSiteLoop",
Field2 = ex.StackTrace,
};
this.ScanErrors.Push(error);
Console.WriteLine("Error for site {1}: {0}", ex.Message, e.Url);
}
// Output the scanning progress
try
{
TimeSpan ts = DateTime.Now.Subtract(this.StartTime);
Console.WriteLine($"Thread: {Thread.CurrentThread.ManagedThreadId}. Processed {this.ScannedSites} of {this.SitesToScan} site collections ({Math.Round(((float)this.ScannedSites / (float)this.SitesToScan) * 100)}%). Process running for {ts.Days} days, {ts.Hours} hours, {ts.Minutes} minutes and {ts.Seconds} seconds.");
}
catch (Exception ex)
{
Console.WriteLine($"Error showing progress: {ex.ToString()}");
}
}
private void ReferenceScanJob_TimerJobRun(object sender, TimerJobRunEventArgs e)
{
// thread safe increase of the sites counter
IncreaseScannedSites();
try
{
Console.WriteLine("Processing site {0}...", e.Url);
#region Basic sample
/*
* // Set the first site collection done flag + perform telemetry
* SetFirstSiteCollectionDone(e.WebClientContext);
*
* // add your custom scan logic here, ensure the catch errors as we don't want to terminate scanning
* e.WebClientContext.Load(e.WebClientContext.Web, p => p.Title);
* e.WebClientContext.ExecuteQueryRetry();
* ScanResult result = new ScanResult()
* {
* SiteColUrl = e.Url,
* SiteURL = e.Url,
* SiteTitle = e.WebClientContext.Web.Title
* };
*
* // Store the scan result
* if (!ScanResults.TryAdd(e.Url, result))
* {
* ScanError error = new ScanError()
* {
* SiteURL = e.Url,
* SiteColUrl = e.Url,
* Error = "Could not add scan result for this site"
* };
* this.ScanErrors.Push(error);
* }
*/
#endregion
#region Search based sample
// Set the first site collection done flag + perform telemetry
SetFirstSiteCollectionDone(e.WebClientContext);
// Need to use search inside this site collection?
List <string> propertiesToRetrieve = new List <string>
{
"Title",
"SPSiteUrl",
"FileExtension",
"OriginalPath"
};
var searchResults = this.Search(e.SiteClientContext.Web, $"((fileextension=htm OR fileextension=html) AND contentclass=STS_ListItem_DocumentLibrary AND Path:{e.Url.TrimEnd('/')}/*)", propertiesToRetrieve);
foreach (var searchResult in searchResults)
{
ScanResult result = new ScanResult()
{
SiteColUrl = e.Url,
FileName = searchResult["OriginalPath"]
};
// Get web url
var webUrlData = Web.GetWebUrlFromPageUrl(e.SiteClientContext, result.FileName);
e.SiteClientContext.ExecuteQueryRetry();
result.SiteURL = webUrlData.Value;
// Store the scan result, use FileName as unique key in this sample
if (!ScanResults.TryAdd(result.FileName, result))
{
ScanError error = new ScanError()
{
SiteURL = e.Url,
SiteColUrl = e.Url,
Error = "Could not add scan result for this web"
};
this.ScanErrors.Push(error);
}
}
#endregion
#region Sub site iteration sample
/***
* // Set the first site collection done flag + perform telemetry
* SetFirstSiteCollectionDone(e.WebClientContext);
*
* // Manually iterate over the content
* IEnumerable<string> expandedSites = GetAllSubSites(e.SiteClientContext.Site);
* bool isFirstSiteInList = true;
* string siteCollectionUrl = "";
*
* foreach (string site in expandedSites)
* {
* // thread safe increase of the webs counter
* IncreaseScannedWebs();
*
* // Clone the existing ClientContext for the sub web
* using (ClientContext ccWeb = e.SiteClientContext.Clone(site))
* {
* Console.WriteLine("Processing site {0}...", site);
*
* if (isFirstSiteInList)
* {
* // Perf optimization: do one call per site to load all the needed properties
* var spSite = (ccWeb as ClientContext).Site;
* ccWeb.Load(spSite, p => p.RootWeb, p => p.Url);
* ccWeb.Load(spSite.RootWeb, p => p.Id);
*
* isFirstSiteInList = false;
* }
*
* // Perf optimization: do one call per web to load all the needed properties
* ccWeb.Load(ccWeb.Web, p => p.Id, p => p.Title);
* ccWeb.Load(ccWeb.Web, p => p.WebTemplate, p => p.Configuration);
* ccWeb.Load(ccWeb.Web, p => p.Lists.Include(li => li.UserCustomActions, li => li.Title, li => li.Hidden, li => li.DefaultViewUrl, li => li.BaseTemplate, li => li.RootFolder, li => li.ListExperienceOptions));
* ccWeb.ExecuteQueryRetry();
*
* // Fill site collection url
* if (string.IsNullOrEmpty(siteCollectionUrl))
* {
* siteCollectionUrl = ccWeb.Site.Url;
* }
*
* // Need to know if this is a sub site?
* if (ccWeb.Web.IsSubSite())
* {
* // Sub site specific logic
* }
*
* ScanResult result = new ScanResult()
* {
* SiteColUrl = e.Url,
* SiteURL = site,
* SiteTitle = ccWeb.Web.Title,
* };
*
* // Store the scan result
* if (!ScanResults.TryAdd(site, result))
* {
* ScanError error = new ScanError()
* {
* SiteURL = site,
* SiteColUrl = e.Url,
* Error = "Could not add scan result for this web"
* };
* this.ScanErrors.Push(error);
* }
* }
* }
**/
#endregion
}
catch (Exception ex)
{
ScanError error = new ScanError()
{
Error = ex.Message,
SiteColUrl = e.Url,
SiteURL = e.Url,
Field1 = "put additional info here"
};
this.ScanErrors.Push(error);
Console.WriteLine("Error for site {1}: {0}", ex.Message, e.Url);
}
// Output the scanning progress
try
{
TimeSpan ts = DateTime.Now.Subtract(this.StartTime);
Console.WriteLine($"Thread: {Thread.CurrentThread.ManagedThreadId}. Processed {this.ScannedSites} of {this.SitesToScan} site collections ({Math.Round(((float)this.ScannedSites / (float)this.SitesToScan) * 100)}%). Process running for {ts.Days} days, {ts.Hours} hours, {ts.Minutes} minutes and {ts.Seconds} seconds.");
}
catch (Exception ex)
{
Console.WriteLine($"Error showing progress: {ex.ToString()}");
}
}
/// <summary>
/// Analyze the web
/// </summary>
/// <param name="cc">ClientContext of the web to be analyzed</param>
/// <returns>Duration of the analysis</returns>
public override TimeSpan Analyze(ClientContext cc)
{
try
{
base.Analyze(cc);
var baseUri = new Uri(this.SiteUrl);
var webAppUrl = baseUri.Scheme + "://" + baseUri.Host;
var lists = cc.Web.GetListsToScan();
foreach (var list in lists)
{
try
{
this.ScanJob.IncreaseScannedLists();
ListScanResult listScanData;
if (list.DefaultViewUrl.ToLower().Contains(".aspx"))
{
File file = cc.Web.GetFileByServerRelativeUrl(list.DefaultViewUrl);
listScanData = file.ModernCompatability(list, ref this.ScanJob.ScanErrors);
}
else
{
listScanData = new ListScanResult()
{
BlockedByNotBeingAbleToLoadPage = true
};
}
if (listScanData != null && !listScanData.WorksInModern)
{
if (this.ScanJob.ExcludeListsOnlyBlockedByOobReasons && listScanData.OnlyBlockedByOOBReasons)
{
continue;
}
listScanData.SiteURL = this.SiteUrl;
listScanData.ListUrl = $"{webAppUrl}{list.DefaultViewUrl}";
listScanData.SiteColUrl = this.SiteCollectionUrl;
listScanData.ListTitle = list.Title;
if (!this.ScanJob.ListScanResults.TryAdd($"{Guid.NewGuid().ToString()}{webAppUrl}{list.DefaultViewUrl}", listScanData))
{
ScanError error = new ScanError()
{
Error = $"Could not add list scan result for {webAppUrl}{list.DefaultViewUrl} from web scan of {this.SiteUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "ListAnalyzer",
Field2 = $"{webAppUrl}{list.DefaultViewUrl}"
};
this.ScanJob.ScanErrors.Push(error);
}
}
}
catch (Exception ex)
{
ScanError error = new ScanError()
{
Error = ex.Message,
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "MainListAnalyzerLoop",
Field2 = ex.StackTrace,
Field3 = $"{webAppUrl}{list.DefaultViewUrl}"
};
// Send error to telemetry to make scanner better
if (this.ScanJob.ScannerTelemetry != null)
{
this.ScanJob.ScannerTelemetry.LogScanError(ex, error);
}
this.ScanJob.ScanErrors.Push(error);
Console.WriteLine("Error for page {1}: {0}", ex.Message, $"{webAppUrl}{list.DefaultViewUrl}");
}
}
}
finally
{
this.StopTime = DateTime.Now;
}
// return the duration of this scan
return(new TimeSpan((this.StopTime.Subtract(this.StartTime).Ticks)));
}
/// <summary>
/// Analyses a web for it's workflow usage
/// </summary>
/// <param name="cc">ClientContext instance used to retrieve workflow data</param>
/// <returns>Duration of the workflow analysis</returns>
public override TimeSpan Analyze(ClientContext cc)
{
try
{
base.Analyze(cc);
var baseUri = new Uri(this.SiteUrl);
var webAppUrl = baseUri.Scheme + "://" + baseUri.Host;
var lists = cc.Web.GetListsToScan(showHidden: true);
foreach (var list in lists)
{
if (list.BaseTemplate == (int)ListTemplateType.XMLForm ||
(!string.IsNullOrEmpty(list.DocumentTemplateUrl) && list.DocumentTemplateUrl.EndsWith(".xsn", StringComparison.InvariantCultureIgnoreCase))
)
{
// Form libraries depend on InfoPath
InfoPathScanResult infoPathScanResult = new InfoPathScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
InfoPathUsage = "FormLibrary",
ListTitle = list.Title,
ListId = list.Id,
ListUrl = list.RootFolder.ServerRelativeUrl,
Enabled = true,
InfoPathTemplate = !string.IsNullOrEmpty(list.DocumentTemplateUrl) ? Path.GetFileName(list.DocumentTemplateUrl) : "",
ItemCount = list.ItemCount,
LastItemUserModifiedDate = list.LastItemUserModifiedDate,
};
if (!this.ScanJob.InfoPathScanResults.TryAdd($"{infoPathScanResult.SiteURL}.{Guid.NewGuid()}", infoPathScanResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add formlibrary InfoPath scan result for {infoPathScanResult.SiteColUrl} and list {infoPathScanResult.ListUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "InfoPathAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
else if (list.BaseTemplate == (int)ListTemplateType.DocumentLibrary || list.BaseTemplate == (int)ListTemplateType.WebPageLibrary)
{
// verify if a form content type was attached to this list
cc.Load(list, p => p.ContentTypes.Include(c => c.Id, c => c.DocumentTemplateUrl));
cc.ExecuteQueryRetry();
var formContentTypeFound = list.ContentTypes.Where(c => c.Id.StringValue.StartsWith(FormBaseContentType, StringComparison.InvariantCultureIgnoreCase)).OrderBy(c => c.Id.StringValue.Length).FirstOrDefault();
if (formContentTypeFound != null)
{
// Form libraries depend on InfoPath
InfoPathScanResult infoPathScanResult = new InfoPathScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
InfoPathUsage = "ContentType",
ListTitle = list.Title,
ListId = list.Id,
ListUrl = list.RootFolder.ServerRelativeUrl,
Enabled = true,
InfoPathTemplate = !string.IsNullOrEmpty(formContentTypeFound.DocumentTemplateUrl) ? Path.GetFileName(formContentTypeFound.DocumentTemplateUrl) : "",
ItemCount = list.ItemCount,
LastItemUserModifiedDate = list.LastItemUserModifiedDate,
};
if (!this.ScanJob.InfoPathScanResults.TryAdd($"{infoPathScanResult.SiteURL}.{Guid.NewGuid()}", infoPathScanResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add contenttype InfoPath scan result for {infoPathScanResult.SiteColUrl} and list {infoPathScanResult.ListUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "InfoPathAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
}
else if (list.BaseTemplate == (int)ListTemplateType.GenericList)
{
try
{
Folder folder = cc.Web.GetFolderByServerRelativeUrl($"{list.RootFolder.ServerRelativeUrl}/Item");
cc.Load(folder, p => p.Properties);
cc.ExecuteQueryRetry();
if (folder.Properties.FieldValues.ContainsKey("_ipfs_infopathenabled") && folder.Properties.FieldValues.ContainsKey("_ipfs_solutionName"))
{
bool infoPathEnabled = true;
if (bool.TryParse(folder.Properties.FieldValues["_ipfs_infopathenabled"].ToString(), out bool infoPathEnabledParsed))
{
infoPathEnabled = infoPathEnabledParsed;
}
// List with an InfoPath customization
InfoPathScanResult infoPathScanResult = new InfoPathScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
InfoPathUsage = "CustomForm",
ListTitle = list.Title,
ListId = list.Id,
ListUrl = list.RootFolder.ServerRelativeUrl,
Enabled = infoPathEnabled,
InfoPathTemplate = folder.Properties.FieldValues["_ipfs_solutionName"].ToString(),
ItemCount = list.ItemCount,
LastItemUserModifiedDate = list.LastItemUserModifiedDate,
};
if (!this.ScanJob.InfoPathScanResults.TryAdd($"{infoPathScanResult.SiteURL}.{Guid.NewGuid()}", infoPathScanResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add customform InfoPath scan result for {infoPathScanResult.SiteColUrl} and list {infoPathScanResult.ListUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "InfoPathAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
}
catch (ServerException ex)
{
if (((ServerException)ex).ServerErrorTypeName == "System.IO.FileNotFoundException")
{
// Ignore
}
else
{
ScanError error = new ScanError()
{
Error = ex.Message,
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "InfoPathAnalyzer",
Field2 = ex.StackTrace,
Field3 = $"{webAppUrl}{list.DefaultViewUrl}"
};
// Send error to telemetry to make scanner better
if (this.ScanJob.ScannerTelemetry != null)
{
this.ScanJob.ScannerTelemetry.LogScanError(ex, error);
}
this.ScanJob.ScanErrors.Push(error);
}
}
}
}
}
finally
{
this.StopTime = DateTime.Now;
}
// return the duration of this scan
return(new TimeSpan((this.StopTime.Subtract(this.StartTime).Ticks)));
}
private void PermissiveScanJob_TimerJobRun(object sender, OfficeDevPnP.Core.Framework.TimerJobs.TimerJobRunEventArgs e)
{
// thread safe increase of the sites counter
IncreaseScannedSites();
try
{
Console.WriteLine("Processing site {0}...", e.Url);
// Set the first site collection done flag + perform telemetry
SetFirstSiteCollectionDone(e.WebClientContext);
// Need to use search inside this site collection?
List <string> propertiesToRetrieve = new List <string>
{
"SPSiteUrl",
"FileExtension",
"OriginalPath"
};
var searchResults = this.Search(e.SiteClientContext.Web, $"({this.GetBaseSearchQuery()} AND Path:{e.Url.TrimEnd('/')}/*)", propertiesToRetrieve);
foreach (var searchResult in searchResults)
{
ScanResult result = new ScanResult()
{
SiteColUrl = e.Url,
FileName = searchResult["OriginalPath"],
FileExtension = searchResult["FileExtension"]
};
// Analyse the files
var webUrlData = Web.GetWebUrlFromPageUrl(e.SiteClientContext, result.FileName);
Uri fileUri;
if (Uri.TryCreate(result.FileName, UriKind.Absolute, out fileUri) &&
(result.FileExtension.ToLower().Equals("html") || result.FileExtension.ToLower().Equals("htm")))
{
var fileContents = e.SiteClientContext.Web.GetFileAsString(fileUri.LocalPath);
var htmlScan = new HtmlScanner().Scan(fileContents);
result.EmbeddedLinkCount = htmlScan.LinkReferences;
result.EmbeddedLocalHtmlLinkCount = htmlScan.LocalHtmlLinkReferences;
result.EmbeddedScriptTagCount = htmlScan.ScriptReferences;
}
e.SiteClientContext.Load(e.SiteClientContext.Web, p => p.SiteUsers, p => p.AssociatedOwnerGroup);
e.SiteClientContext.Load(e.SiteClientContext.Web.AssociatedOwnerGroup, p => p.Users);
e.SiteClientContext.ExecuteQueryRetry();
if (e.SiteClientContext.Web.SiteUsers != null)
{
try
{
var admins = e.SiteClientContext.Web.SiteUsers.Where(p => p.IsSiteAdmin);
if (admins != null && admins.Count() > 0)
{
foreach (var admin in admins)
{
if (!string.IsNullOrEmpty(admin.Email))
{
result.SiteAdmins = AddSiteOwner(result.SiteAdmins, admin.Email);
}
}
}
}
catch
{
//Eat exceptions...rather log all files in the main result list instead of dropping some due to error getting owners
}
try
{
// grab folks from the Access Web App site owners group
if (e.SiteClientContext.Web.AssociatedOwnerGroup != null && e.SiteClientContext.Web.AssociatedOwnerGroup.Users != null && e.SiteClientContext.Web.AssociatedOwnerGroup.Users.Count > 0)
{
foreach (var owner in e.SiteClientContext.Web.AssociatedOwnerGroup.Users)
{
if (!string.IsNullOrEmpty(owner.Email))
{
result.SiteAdmins = AddSiteOwner(result.SiteAdmins, owner.Email);
}
}
}
}
catch
{
//Eat exceptions...rather log all files in the main result list instead of dropping some due to error getting owners
}
}
result.SiteURL = webUrlData.Value;
// Store the scan result, use FileName as unique key in this sample
if (!ScanResults.TryAdd(result.FileName, result))
{
ScanError error = new ScanError()
{
SiteURL = result.SiteURL,
SiteColUrl = e.Url,
Error = "Could not add scan result for this web"
};
this.ScanErrors.Push(error);
}
}
}
catch (Exception ex)
{
ScanError error = new ScanError()
{
Error = ex.Message,
SiteColUrl = e.Url,
SiteURL = e.Url,
};
this.ScanErrors.Push(error);
Console.WriteLine("Error for site {1}: {0}", ex.Message, e.Url);
}
// Output the scanning progress
try
{
TimeSpan ts = DateTime.Now.Subtract(this.StartTime);
Console.WriteLine($"Thread: {Thread.CurrentThread.ManagedThreadId}. Processed {this.ScannedSites} of {this.SitesToScan} site collections ({Math.Round(((float)this.ScannedSites / (float)this.SitesToScan) * 100)}%). Process running for {ts.Days} days, {ts.Hours} hours, {ts.Minutes} minutes and {ts.Seconds} seconds.");
}
catch (Exception ex)
{
Console.WriteLine($"Error showing progress: {ex.ToString()}");
}
}
/// <summary>
/// Analyze the web
/// </summary>
/// <param name="cc">ClientContext of the web to be analyzed</param>
/// <returns>Duration of the analysis</returns>
public override TimeSpan Analyze(ClientContext cc)
{
try
{
base.Analyze(cc);
// Ensure needed data is loaded
Web web = cc.Web;
web.EnsureProperties(p => p.UserCustomActions, p => p.AlternateCssUrl, p => p.CustomMasterUrl, p => p.MasterUrl, p => p.Features, p => p.WebTemplate, p => p.Configuration, p => p.HasUniqueRoleAssignments, p => p.RootFolder);
// Log in Site scan data that the scanned web is a sub site
if (web.IsSubSite())
{
SiteScanResult siteScanData;
if (this.ScanJob.SiteScanResults.TryGetValue(this.SiteCollectionUrl, out siteScanData))
{
if (!siteScanData.SubSites)
{
var clonedSiteScandata = siteScanData.Clone();
clonedSiteScandata.SubSites = true;
if (!this.ScanJob.SiteScanResults.TryUpdate(this.SiteCollectionUrl, clonedSiteScandata, siteScanData))
{
ScanError error = new ScanError()
{
Error = $"Could not add update site scan result for {this.SiteCollectionUrl} from web scan of {this.SiteUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "WebAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
}
// Check if we've broken permission inheritance in this site collection
if (web.HasUniqueRoleAssignments)
{
SiteScanResult siteScanData2;
if (this.ScanJob.SiteScanResults.TryGetValue(this.SiteCollectionUrl, out siteScanData2))
{
if (!siteScanData2.SubSitesWithBrokenPermissionInheritance)
{
var clonedSiteScandata = siteScanData2.Clone();
clonedSiteScandata.SubSitesWithBrokenPermissionInheritance = true;
if (!this.ScanJob.SiteScanResults.TryUpdate(this.SiteCollectionUrl, clonedSiteScandata, siteScanData2))
{
ScanError error = new ScanError()
{
Error = $"Could not add update site scan result for {this.SiteCollectionUrl} from web scan of {this.SiteUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "WebAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
}
}
}
// Perform specific analysis work
WebScanResult scanResult = new WebScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
};
// Log used web template
if (web.WebTemplate != null)
{
scanResult.WebTemplate = $"{web.WebTemplate}#{web.Configuration}";
}
// Page feature check: users can disable this to prevent modern page creation
scanResult.ModernPageWebFeatureDisabled = web.Features.Where(f => f.DefinitionId == FeatureId_Web_ModernPage).Count() == 0;
// List feature check: users can enabled this to prevent modern lists from working
scanResult.ModernListWebBlockingFeatureEnabled = web.Features.Where(f => f.DefinitionId == FeatureId_Web_ModernList).Count() > 0;
// Publishing web feature enabled
scanResult.WebPublishingFeatureEnabled = web.Features.Where(f => f.DefinitionId == FeatureId_Web_Publishing).Count() > 0;
// Log permission inheritance details
if (web.IsSubSite() && web.HasUniqueRoleAssignments)
{
scanResult.BrokenPermissionInheritance = web.HasUniqueRoleAssignments;
scanResult.Owners = web.GetOwners();
scanResult.Members = web.GetMembers();
scanResult.Visitors = web.GetVisitors();
scanResult.EveryoneClaimsGranted = web.ClaimsHaveRoleAssignment(this.ScanJob.EveryoneClaim, this.ScanJob.EveryoneExceptExternalUsersClaim);
}
// If the web template is STS#0, GROUP#0 or SITEPAGEPUBLISHING#0 then the feature was activated by SPO, other templates never got it
scanResult.ModernPageFeatureWasEnabledBySPO = false;
if (scanResult.WebTemplate.Equals("STS#0", StringComparison.InvariantCultureIgnoreCase) ||
scanResult.WebTemplate.Equals("GROUP#0", StringComparison.InvariantCulture) ||
scanResult.WebTemplate.Equals("SITEPAGEPUBLISHING#0", StringComparison.InvariantCulture))
{
// Since we did not enable the feature for all STS#0 sites (sites with publishing active did not get it, nor sites having a large number of webpart/wiki pages) we
// check if it should have been turned on by checking for the "site page" content type being added to the site pages library
var listsToScan = web.GetListsToScan();
var sitePagesLibrary = listsToScan.Where(p => p.BaseTemplate == (int)ListTemplateType.WebPageLibrary).FirstOrDefault();
if (sitePagesLibrary != null)
{
cc.Load(sitePagesLibrary, p => p.ContentTypes.Include(c => c.Id));
cc.ExecuteQueryRetry();
if (sitePagesLibrary.ContentTypes.BestMatch(SitePageContentTypeId) != null)
{
scanResult.ModernPageFeatureWasEnabledBySPO = true;
}
}
}
// Get information about the master pages used
if (!string.IsNullOrEmpty(web.MasterUrl) && !excludeMasterPage.Contains(web.MasterUrl.Substring(web.MasterUrl.LastIndexOf("/") + 1).ToLower()))
{
scanResult.MasterPage = web.MasterUrl;
}
if (!string.IsNullOrEmpty(web.CustomMasterUrl) && !excludeMasterPage.Contains(web.CustomMasterUrl.Substring(web.CustomMasterUrl.LastIndexOf("/") + 1).ToLower()))
{
scanResult.CustomMasterPage = web.CustomMasterUrl;
}
if (!string.IsNullOrEmpty(web.AlternateCssUrl))
{
scanResult.AlternateCSS = web.AlternateCssUrl;
}
// Get the user custom actions
scanResult.WebUserCustomActions = web.UserCustomActions.Analyze(this.SiteCollectionUrl, this.SiteUrl);
// Get home page from the web and check whether it's a modern page or not
scanResult.ModernHomePage = false;
var homePageUrl = web.RootFolder.WelcomePage;
var homepageName = System.IO.Path.GetFileName(web.RootFolder.WelcomePage);
if (string.IsNullOrEmpty(homepageName))
{
homepageName = "Home.aspx";
}
var sitePagesLibraryForWeb = web.GetListsToScan().Where(p => p.BaseTemplate == (int)ListTemplateType.WebPageLibrary).FirstOrDefault();
if (sitePagesLibraryForWeb != null)
{
var homePageFile = web.GetFileByServerRelativeUrl($"{sitePagesLibraryForWeb.RootFolder.ServerRelativeUrl}/{homepageName}");
cc.Load(homePageFile, f => f.ListItemAllFields, f => f.Exists);
cc.ExecuteQueryRetry();
if (homePageFile.Exists)
{
var item = homePageFile.ListItemAllFields;
if (item.FieldValues.ContainsKey(ClientSideApplicationId) && item[ClientSideApplicationId] != null && item[ClientSideApplicationId].ToString().Equals(FeatureId_Web_ModernPage.ToString(), StringComparison.InvariantCultureIgnoreCase))
{
scanResult.ModernHomePage = true;
}
}
}
// Push information from root web to respective SiteScanResult object
if (!cc.Web.IsSubSite())
{
SiteScanResult siteScanData;
if (this.ScanJob.SiteScanResults.TryGetValue(this.SiteCollectionUrl, out siteScanData))
{
var clonedSiteScandata = siteScanData.Clone();
clonedSiteScandata.WebPublishingFeatureEnabled = scanResult.WebPublishingFeatureEnabled;
clonedSiteScandata.ModernPageWebFeatureDisabled = scanResult.ModernPageWebFeatureDisabled;
clonedSiteScandata.ModernPageFeatureWasEnabledBySPO = scanResult.ModernPageFeatureWasEnabledBySPO;
clonedSiteScandata.ModernListWebBlockingFeatureEnabled = scanResult.ModernListWebBlockingFeatureEnabled;
clonedSiteScandata.ModernHomePage = scanResult.ModernHomePage;
clonedSiteScandata.MasterPage = (!String.IsNullOrEmpty(scanResult.MasterPage) || !String.IsNullOrEmpty(scanResult.CustomMasterPage));
clonedSiteScandata.AlternateCSS = !String.IsNullOrEmpty(scanResult.AlternateCSS);
clonedSiteScandata.WebUserCustomActions = scanResult.WebUserCustomActions;
if (!this.ScanJob.SiteScanResults.TryUpdate(this.SiteCollectionUrl, clonedSiteScandata, siteScanData))
{
ScanError error = new ScanError()
{
Error = $"Could not add update site scan result for {this.SiteCollectionUrl} from web scan of {this.SiteUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "WebAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
}
// Persist web results
if (!this.ScanJob.WebScanResults.TryAdd(this.SiteUrl, scanResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add web scan result for {this.SiteUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "WebAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
if (this.ScanJob.Mode == Mode.Full)
{
// Kickoff the page analysing
var pageAnalyzer = new PageAnalyzer(this.SiteUrl, this.SiteCollectionUrl, this.ScanJob, this.pageSearchResults);
pageAnalyzer.Analyze(cc);
}
}
finally
{
this.StopTime = DateTime.Now;
}
// return the duration of this scan
return(new TimeSpan((this.StopTime.Subtract(this.StartTime).Ticks)));
}
/// <summary>
/// Analyses a page
/// </summary>
/// <param name="cc">ClientContext instance used to retrieve page data</param>
/// <returns>Duration of the page analysis</returns>
public override TimeSpan Analyze(ClientContext cc)
{
try
{
base.Analyze(cc);
Site site = cc.Site;
site.EnsureProperties(p => p.Features);
Web web = cc.Web;
cc.Web.EnsureProperties(p => p.WebTemplate, p => p.Configuration, p => p.Features, p => p.Language, p => p.CustomMasterUrl, p => p.MasterUrl);
var homePageUrl = web.WelcomePage;
if (string.IsNullOrEmpty(homePageUrl))
{
// Will be case when the site home page is a web part page
homePageUrl = "default.aspx";
}
var listsToScan = web.GetListsToScan();
var sitePagesLibraries = listsToScan.Where(p => p.BaseTemplate == (int)ListTemplateType.WebPageLibrary);
if (sitePagesLibraries.Count() > 0)
{
foreach (var sitePagesLibrary in sitePagesLibraries)
{
CamlQuery query = new CamlQuery
{
ViewXml = CAMLQueryByExtension
};
var pages = sitePagesLibrary.GetItems(query);
web.Context.Load(pages);
web.Context.ExecuteQueryRetry();
if (pages.FirstOrDefault() != null)
{
DateTime start;
bool forceCheckout = sitePagesLibrary.ForceCheckout;
foreach (var page in pages)
{
string pageUrl = null;
try
{
if (page.FieldValues.ContainsKey(Field_FileRefField) && !String.IsNullOrEmpty(page[Field_FileRefField].ToString()))
{
pageUrl = page[Field_FileRefField].ToString();
// In home page only mode we only continue if this page is the site's home page
if (Options.IsHomePageOnly(this.ScanJob.Mode) && !pageUrl.EndsWith(homePageUrl, StringComparison.InvariantCultureIgnoreCase))
{
continue;
}
}
else
{
//skip page
continue;
}
start = DateTime.Now;
PageScanResult pageResult = new PageScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
PageUrl = pageUrl,
Library = sitePagesLibrary.RootFolder.ServerRelativeUrl,
};
// Is this page the web's home page?
if (pageUrl.EndsWith(homePageUrl, StringComparison.InvariantCultureIgnoreCase))
{
pageResult.HomePage = true;
}
// Get the type of the page
pageResult.PageType = page.PageType();
// Get page web parts
var pageAnalysis = page.WebParts(this.ScanJob.PageTransformation);
if (pageAnalysis != null)
{
pageResult.Layout = pageAnalysis.Item1.ToString().Replace("Wiki_", "").Replace("WebPart_", "");
pageResult.WebParts = pageAnalysis.Item2;
}
// Determine if this site contains a default "uncustomized" home page
bool isUncustomizedHomePage = false;
bool canModernizeHomePageWorked = false;
try
{
var canModernizeHomepage = web.CanModernizeHomepage;
web.Context.Load(canModernizeHomepage);
web.Context.ExecuteQueryRetry();
isUncustomizedHomePage = canModernizeHomepage.CanModernizeHomepage;
canModernizeHomePageWorked = true;
pageResult.UncustomizedHomePage = isUncustomizedHomePage;
}
catch (Exception ex)
{
}
// If for some reason the API did not work, then let's fall back to the old approach
if (!canModernizeHomePageWorked)
{
try
{
string pageName = "";
if (page.FieldValues.ContainsKey(Field_FileLeafRef) && !String.IsNullOrEmpty(page[Field_FileLeafRef].ToString()))
{
pageName = page[Field_FileLeafRef].ToString();
}
if (pageResult.HomePage && web.WebTemplate == "STS" && web.Configuration == 0)
{
bool publishingWebFeatureEnabled = web.Features.Where(f => f.DefinitionId == WebAnalyzer.FeatureId_Web_Publishing).Count() > 0;
bool publishingSiteFeatureEnabled = site.Features.Where(f => f.DefinitionId == SiteAnalyzer.FeatureId_Site_Publishing).Count() > 0;
bool homePageModernizationOptedOut = web.Features.Where(f => f.DefinitionId == FeatureId_Web_HomePage).Count() > 0;
if (!homePageModernizationOptedOut && !publishingSiteFeatureEnabled && !publishingWebFeatureEnabled)
{
bool siteWasGroupified = web.Features.Where(f => f.DefinitionId == FeatureId_Web_GroupHomepage).Count() > 0;
if (!siteWasGroupified)
{
// Check for master page url
if (web.MasterUrl.EndsWith("_catalogs/masterpage/seattle.master", StringComparison.InvariantCultureIgnoreCase))
{
// Check for home page name, only "default sts#0 home pages" should be considered, so
// home.aspx or the translated versions
var homePageName = this.ScanJob.Store.Get <string>(this.ScanJob.StoreOptions.GetKey(keyWikiHomePageName));
if (homePageName == null)
{
ClientResult <string> result = Microsoft.SharePoint.Client.Utilities.Utility.GetLocalizedString(web.Context, "$Resources:WikiPageHomePageName", "core", (int)web.Language);
web.Context.ExecuteQueryRetry();
homePageName = $"{new Regex(@"['´`]").Replace(result.Value, "")}.aspx";
this.ScanJob.Store.Set <string>(this.ScanJob.StoreOptions.GetKey(keyWikiHomePageName), homePageName, this.ScanJob.StoreOptions.EntryOptions);
}
if (pageName.Equals(homePageName, StringComparison.InvariantCultureIgnoreCase))
{
var wiki = page.FieldValues[Field_WikiField].ToString();
if (!string.IsNullOrEmpty(wiki))
{
var isHtmlUncustomized = IsHtmlUncustomized(wiki);
if (isHtmlUncustomized)
{
string pageType = GetPageWebPartInfo(pageResult.WebParts);
if (pageType == TeamSiteDefaultWebParts)
{
page.ContentType.EnsureProperty(p => p.DisplayFormTemplateName);
if (page.ContentType.DisplayFormTemplateName == "WikiEditForm")
{
isUncustomizedHomePage = true;
}
}
}
}
}
}
}
}
}
}
catch (Exception ex)
{
// no point in failing the scan if something goes wrong here
}
finally
{
pageResult.UncustomizedHomePage = isUncustomizedHomePage;
}
}
// Get page change information
pageResult.ModifiedAt = page.LastModifiedDateTime();
pageResult.ModifiedBy = page.LastModifiedBy();
// Grab this page from the search results to connect view information
string fullPageUrl = $"https://{new Uri(this.SiteCollectionUrl).DnsSafeHost}{pageUrl}";
if (pageResult.HomePage)
{
fullPageUrl = this.SiteUrl;
}
if (!this.ScanJob.SkipUsageInformation && this.pageSearchResults != null)
{
var searchPage = this.pageSearchResults.Where(x => x.Values.Contains(fullPageUrl)).FirstOrDefault();
if (searchPage != null)
{
// Recent = last 14 days
pageResult.ViewsRecent = searchPage["ViewsRecent"].ToInt32();
pageResult.ViewsRecentUniqueUsers = searchPage["ViewsRecentUniqueUsers"].ToInt32();
pageResult.ViewsLifeTime = searchPage["ViewsLifeTime"].ToInt32();
pageResult.ViewsLifeTimeUniqueUsers = searchPage["ViewsLifeTimeUniqueUsers"].ToInt32();
}
}
if (!this.ScanJob.PageScanResults.TryAdd(pageResult.PageUrl, pageResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add page scan result for {pageResult.PageUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "PageAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
var duration = new TimeSpan((DateTime.Now.Subtract(start).Ticks));
}
catch (Exception ex)
{
ScanError error = new ScanError()
{
Error = ex.Message,
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "MainPageAnalyzerLoop",
Field2 = ex.StackTrace,
Field3 = pageUrl
};
// Send error to telemetry to make scanner better
if (this.ScanJob.ScannerTelemetry != null)
{
this.ScanJob.ScannerTelemetry.LogScanError(ex, error);
}
this.ScanJob.ScanErrors.Push(error);
}
}
}
}
}
}
finally
{
this.StopTime = DateTime.Now;
}
// return the duration of this scan
return(new TimeSpan((this.StopTime.Subtract(this.StartTime).Ticks)));
}
/// <summary>
/// Analyses a web for it's workflow usage
/// </summary>
/// <param name="cc">ClientContext instance used to retrieve workflow data</param>
/// <returns>Duration of the workflow analysis</returns>
public override TimeSpan Analyze(ClientContext cc)
{
try
{
Web web = cc.Web;
// Pre-load needed properties in a single call
cc.Load(web, w => w.Id, w => w.ServerRelativeUrl, w => w.Url, w => w.WorkflowTemplates, w => w.WorkflowAssociations);
cc.Load(web, p => p.ContentTypes.Include(ct => ct.WorkflowAssociations, ct => ct.Name, ct => ct.StringId));
cc.Load(web, p => p.Lists.Include(li => li.Id, li => li.Title, li => li.Hidden, li => li.DefaultViewUrl, li => li.BaseTemplate, li => li.RootFolder, li => li.ItemCount, li => li.WorkflowAssociations));
cc.ExecuteQueryRetry();
var lists = web.Lists;
// *******************************************
// Site, reusable and list level 2013 workflow
// *******************************************
// Retrieve the 2013 site level workflow definitions (including unpublished ones)
WorkflowDefinition[] siteDefinitions = null;
// Retrieve the 2013 site level workflow subscriptions
WorkflowSubscription[] siteSubscriptions = null;
try
{
var servicesManager = new WorkflowServicesManager(web.Context, web);
var deploymentService = servicesManager.GetWorkflowDeploymentService();
var subscriptionService = servicesManager.GetWorkflowSubscriptionService();
var definitions = deploymentService.EnumerateDefinitions(false);
web.Context.Load(definitions);
var subscriptions = subscriptionService.EnumerateSubscriptions();
web.Context.Load(subscriptions);
web.Context.ExecuteQueryRetry();
siteDefinitions = definitions.ToArray();
siteSubscriptions = subscriptions.ToArray();
}
catch (ServerException)
{
// If there is no workflow service present in the farm this method will throw an error.
// Swallow the exception
}
// We've found SP2013 site scoped workflows
if (siteDefinitions.Count() > 0)
{
foreach (var siteDefinition in siteDefinitions.Where(p => p.RestrictToType.Equals("site", StringComparison.InvariantCultureIgnoreCase) || p.RestrictToType.Equals("universal", StringComparison.InvariantCultureIgnoreCase)))
{
// Check if this workflow is also in use
var siteWorkflowSubscriptions = siteSubscriptions.Where(p => p.DefinitionId.Equals(siteDefinition.Id));
if (siteWorkflowSubscriptions.Count() > 0)
{
foreach (var siteWorkflowSubscription in siteWorkflowSubscriptions)
{
WorkflowScanResult workflowScanResult = new WorkflowScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
ListTitle = "",
ListUrl = "",
ContentTypeId = "",
ContentTypeName = "",
Version = "2013",
Scope = "Site",
RestrictToType = siteDefinition.RestrictToType,
DefinitionName = siteDefinition.DisplayName,
DefinitionDescription = siteDefinition.Description,
SubscriptionName = siteWorkflowSubscription.Name,
HasSubscriptions = true,
Enabled = siteWorkflowSubscription.Enabled,
DefinitionId = siteDefinition.Id,
IsOOBWorkflow = false,
SubscriptionId = siteWorkflowSubscription.Id,
};
if (!this.ScanJob.WorkflowScanResults.TryAdd($"workflowScanResult.SiteURL.{Guid.NewGuid()}", workflowScanResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add 2013 site workflow scan result for {workflowScanResult.SiteColUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "WorkflowAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
}
else
{
WorkflowScanResult workflowScanResult = new WorkflowScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
ListTitle = "",
ListUrl = "",
ContentTypeId = "",
ContentTypeName = "",
Version = "2013",
Scope = "Site",
RestrictToType = siteDefinition.RestrictToType,
DefinitionName = siteDefinition.DisplayName,
DefinitionDescription = siteDefinition.Description,
SubscriptionName = "",
HasSubscriptions = false,
Enabled = false,
DefinitionId = siteDefinition.Id,
IsOOBWorkflow = false,
SubscriptionId = Guid.Empty,
};
if (!this.ScanJob.WorkflowScanResults.TryAdd($"workflowScanResult.SiteURL.{Guid.NewGuid()}", workflowScanResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add 2013 site workflow scan result for {workflowScanResult.SiteColUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "WorkflowAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
}
}
// We've found SP2013 list scoped workflows
if (siteDefinitions.Count() > 0)
{
foreach (var listDefinition in siteDefinitions.Where(p => p.RestrictToType.Equals("list", StringComparison.InvariantCultureIgnoreCase) || p.RestrictToType.Equals("universal", StringComparison.InvariantCultureIgnoreCase)))
{
// Check if this workflow is also in use
var listWorkflowSubscriptions = siteSubscriptions.Where(p => p.DefinitionId.Equals(listDefinition.Id));
if (listWorkflowSubscriptions.Count() > 0)
{
foreach (var listWorkflowSubscription in listWorkflowSubscriptions)
{
Guid associatedListId = Guid.Empty;
string associatedListTitle = "";
string associatedListUrl = "";
if (Guid.TryParse(GetWorkflowProperty(listWorkflowSubscription, "Microsoft.SharePoint.ActivationProperties.ListId"), out Guid associatedListIdValue))
{
associatedListId = associatedListIdValue;
// Lookup this list and update title and url
var listLookup = lists.Where(p => p.Id.Equals(associatedListId)).FirstOrDefault();
if (listLookup != null)
{
associatedListTitle = listLookup.Title;
associatedListUrl = listLookup.RootFolder.ServerRelativeUrl;
}
}
WorkflowScanResult workflowScanResult = new WorkflowScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
ListTitle = associatedListTitle,
ListUrl = associatedListUrl,
ListId = associatedListId,
ContentTypeId = "",
ContentTypeName = "",
Version = "2013",
Scope = "List",
RestrictToType = listDefinition.RestrictToType,
DefinitionName = listDefinition.DisplayName,
DefinitionDescription = listDefinition.Description,
SubscriptionName = listWorkflowSubscription.Name,
HasSubscriptions = true,
Enabled = listWorkflowSubscription.Enabled,
DefinitionId = listDefinition.Id,
IsOOBWorkflow = false,
SubscriptionId = listWorkflowSubscription.Id,
};
if (!this.ScanJob.WorkflowScanResults.TryAdd($"workflowScanResult.SiteURL.{Guid.NewGuid()}", workflowScanResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add 2013 list workflow scan result for {workflowScanResult.SiteColUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "WorkflowAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
}
else
{
WorkflowScanResult workflowScanResult = new WorkflowScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
ListTitle = "",
ListUrl = "",
ListId = Guid.Empty,
ContentTypeId = "",
ContentTypeName = "",
Version = "2013",
Scope = "List",
RestrictToType = listDefinition.RestrictToType,
DefinitionName = listDefinition.DisplayName,
DefinitionDescription = listDefinition.Description,
SubscriptionName = "",
HasSubscriptions = false,
Enabled = false,
DefinitionId = listDefinition.Id,
IsOOBWorkflow = false,
SubscriptionId = Guid.Empty,
};
if (!this.ScanJob.WorkflowScanResults.TryAdd($"workflowScanResult.SiteURL.{Guid.NewGuid()}", workflowScanResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add 2013 list workflow scan result for {workflowScanResult.SiteColUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "WorkflowAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
}
}
// ***********************************************
// Site, list and content type level 2010 workflow
// ***********************************************
// Find all places where we have workflows associated (=subscribed) to SharePoint objects
if (web.WorkflowAssociations.Count > 0)
{
foreach (var workflowAssociation in web.WorkflowAssociations)
{
this.sp2010WorkflowAssociations.Add(new SP2010WorkFlowAssociation()
{
Scope = "Site", WorkflowAssociation = workflowAssociation
});
}
}
foreach (var list in lists.Where(p => p.WorkflowAssociations.Count > 0))
{
foreach (var workflowAssociation in list.WorkflowAssociations)
{
this.sp2010WorkflowAssociations.Add(new SP2010WorkFlowAssociation()
{
Scope = "List", WorkflowAssociation = workflowAssociation, AssociatedList = list
});
}
}
foreach (var ct in web.ContentTypes.Where(p => p.WorkflowAssociations.Count > 0))
{
foreach (var workflowAssociation in ct.WorkflowAssociations)
{
this.sp2010WorkflowAssociations.Add(new SP2010WorkFlowAssociation()
{
Scope = "ContentType", WorkflowAssociation = workflowAssociation, AssociatedContentType = ct
});
}
}
// Process 2010 worflows
System.Collections.Generic.List <Guid> processedWorkflowAssociations = new System.Collections.Generic.List <Guid>(this.sp2010WorkflowAssociations.Count);
if (web.WorkflowTemplates.Count > 0)
{
foreach (var workflowTemplate in web.WorkflowTemplates)
{
// do we have workflows associated for this template?
var associatedWorkflows = this.sp2010WorkflowAssociations.Where(p => p.WorkflowAssociation.BaseId.Equals(workflowTemplate.Id));
if (associatedWorkflows.Count() > 0)
{
foreach (var associatedWorkflow in associatedWorkflows)
{
processedWorkflowAssociations.Add(associatedWorkflow.WorkflowAssociation.Id);
// Skip previous versions of a workflow
// todo: non-english sites will use another string
if (associatedWorkflow.WorkflowAssociation.Name.Contains("(Previous Version:"))
{
continue;
}
WorkflowScanResult workflowScanResult = new WorkflowScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
ListTitle = associatedWorkflow.AssociatedList != null ? associatedWorkflow.AssociatedList.Title : "",
ListUrl = associatedWorkflow.AssociatedList != null ? associatedWorkflow.AssociatedList.RootFolder.ServerRelativeUrl : "",
ListId = associatedWorkflow.AssociatedList != null ? associatedWorkflow.AssociatedList.Id : Guid.Empty,
ContentTypeId = associatedWorkflow.AssociatedContentType != null ? associatedWorkflow.AssociatedContentType.StringId : "",
ContentTypeName = associatedWorkflow.AssociatedContentType != null ? associatedWorkflow.AssociatedContentType.Name : "",
Version = "2010",
Scope = associatedWorkflow.Scope,
RestrictToType = "N/A",
DefinitionName = workflowTemplate.Name,
DefinitionDescription = workflowTemplate.Description,
SubscriptionName = associatedWorkflow.WorkflowAssociation.Name,
HasSubscriptions = true,
Enabled = associatedWorkflow.WorkflowAssociation.Enabled,
DefinitionId = workflowTemplate.Id,
IsOOBWorkflow = IsOOBWorkflow(workflowTemplate.Id.ToString()),
SubscriptionId = associatedWorkflow.WorkflowAssociation.Id,
};
if (!this.ScanJob.WorkflowScanResults.TryAdd($"workflowScanResult.SiteURL.{Guid.NewGuid()}", workflowScanResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add 2010 {associatedWorkflow.Scope} type workflow scan result for {workflowScanResult.SiteColUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "WorkflowAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
}
else
{
// Only add non OOB workflow templates when there's no associated workflow - makes the dataset smaller
if (!IsOOBWorkflow(workflowTemplate.Id.ToString()))
{
WorkflowScanResult workflowScanResult = new WorkflowScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
ListTitle = "",
ListUrl = "",
ListId = Guid.Empty,
ContentTypeId = "",
ContentTypeName = "",
Version = "2010",
Scope = "",
RestrictToType = "N/A",
DefinitionName = workflowTemplate.Name,
DefinitionDescription = workflowTemplate.Description,
SubscriptionName = "",
HasSubscriptions = false,
Enabled = false,
DefinitionId = workflowTemplate.Id,
IsOOBWorkflow = IsOOBWorkflow(workflowTemplate.Id.ToString()),
SubscriptionId = Guid.Empty,
};
if (!this.ScanJob.WorkflowScanResults.TryAdd($"workflowScanResult.SiteURL.{Guid.NewGuid()}", workflowScanResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add 2010 type workflow scan result for {workflowScanResult.SiteColUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "WorkflowAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
}
}
}
// Are there associated workflows for which we did not find a template
foreach (var associatedWorkflow in this.sp2010WorkflowAssociations)
{
if (!processedWorkflowAssociations.Contains(associatedWorkflow.WorkflowAssociation.Id))
{
// Skip previous versions of a workflow
// todo: non-english sites will use another string
if (associatedWorkflow.WorkflowAssociation.Name.Contains("(Previous Version:"))
{
continue;
}
WorkflowScanResult workflowScanResult = new WorkflowScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
ListTitle = associatedWorkflow.AssociatedList != null ? associatedWorkflow.AssociatedList.Title : "",
ListUrl = associatedWorkflow.AssociatedList != null ? associatedWorkflow.AssociatedList.RootFolder.ServerRelativeUrl : "",
ListId = associatedWorkflow.AssociatedList != null ? associatedWorkflow.AssociatedList.Id : Guid.Empty,
ContentTypeId = associatedWorkflow.AssociatedContentType != null ? associatedWorkflow.AssociatedContentType.StringId : "",
ContentTypeName = associatedWorkflow.AssociatedContentType != null ? associatedWorkflow.AssociatedContentType.Name : "",
Version = "2010",
Scope = associatedWorkflow.Scope,
RestrictToType = "N/A",
DefinitionName = associatedWorkflow.WorkflowAssociation.Name,
DefinitionDescription = "",
SubscriptionName = associatedWorkflow.WorkflowAssociation.Name,
HasSubscriptions = true,
Enabled = associatedWorkflow.WorkflowAssociation.Enabled,
DefinitionId = Guid.Empty,
IsOOBWorkflow = false,
SubscriptionId = associatedWorkflow.WorkflowAssociation.Id,
};
if (!this.ScanJob.WorkflowScanResults.TryAdd($"workflowScanResult.SiteURL.{Guid.NewGuid()}", workflowScanResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add 2010 {associatedWorkflow.Scope} type workflow scan result for {workflowScanResult.SiteColUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "WorkflowAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
}
}
finally
{
this.StopTime = DateTime.Now;
}
// return the duration of this scan
return(new TimeSpan((this.StopTime.Subtract(this.StartTime).Ticks)));
}
/// <summary>
/// Analyses a web for it's workflow usage
/// </summary>
/// <param name="cc">ClientContext instance used to retrieve workflow data</param>
/// <returns>Duration of the workflow analysis</returns>
public override TimeSpan Analyze(ClientContext cc)
{
try
{
// Workflow analysis does not work as the xoml / xaml files can't be read with Sites.Read.All permission
if (!this.ScanJob.AppOnlyHasFullControl)
{
return(TimeSpan.Zero);
}
Web web = cc.Web;
// Pre-load needed properties in a single call
cc.Load(web, w => w.Id, w => w.ServerRelativeUrl, w => w.Url, w => w.WorkflowTemplates, w => w.WorkflowAssociations);
cc.Load(web, p => p.ContentTypes.Include(ct => ct.WorkflowAssociations, ct => ct.Name, ct => ct.StringId));
cc.Load(web, p => p.Lists.Include(li => li.Id, li => li.Title, li => li.Hidden, li => li.DefaultViewUrl, li => li.BaseTemplate, li => li.RootFolder.ServerRelativeUrl, li => li.ItemCount, li => li.WorkflowAssociations));
cc.Load(cc.Site, p => p.RootWeb);
cc.Load(cc.Site.RootWeb, p => p.Lists.Include(li => li.Id, li => li.Title, li => li.Hidden, li => li.DefaultViewUrl, li => li.BaseTemplate, li => li.RootFolder.ServerRelativeUrl, li => li.ItemCount, li => li.WorkflowAssociations));
cc.ExecuteQueryRetry();
var lists = web.Lists;
// *******************************************
// Site, reusable and list level 2013 workflow
// *******************************************
// Retrieve the 2013 site level workflow definitions (including unpublished ones)
WorkflowDefinition[] siteDefinitions = null;
// Retrieve the 2013 site level workflow subscriptions
WorkflowSubscription[] siteSubscriptions = null;
try
{
var servicesManager = new WorkflowServicesManager(web.Context, web);
var deploymentService = servicesManager.GetWorkflowDeploymentService();
var subscriptionService = servicesManager.GetWorkflowSubscriptionService();
var definitions = deploymentService.EnumerateDefinitions(false);
web.Context.Load(definitions);
var subscriptions = subscriptionService.EnumerateSubscriptions();
web.Context.Load(subscriptions);
web.Context.ExecuteQueryRetry();
siteDefinitions = definitions.ToArray();
siteSubscriptions = subscriptions.ToArray();
}
catch (ServerException ex)
{
// If there is no workflow service present in the farm this method will throw an error.
// Swallow the exception
}
// We've found SP2013 site scoped workflows
if (siteDefinitions != null && siteDefinitions.Count() > 0)
{
foreach (var siteDefinition in siteDefinitions.Where(p => p.RestrictToType != null && (p.RestrictToType.Equals("site", StringComparison.InvariantCultureIgnoreCase) || p.RestrictToType.Equals("universal", StringComparison.InvariantCultureIgnoreCase))))
{
// Check if this workflow is also in use
var siteWorkflowSubscriptions = siteSubscriptions.Where(p => p.DefinitionId.Equals(siteDefinition.Id));
// Perform workflow analysis
WorkflowActionAnalysis workFlowAnalysisResult = null;
WorkflowTriggerAnalysis workFlowTriggerAnalysisResult = null;
if (Options.IncludeWorkflowWithDetails(this.ScanJob.Mode))
{
workFlowAnalysisResult = WorkflowManager.Instance.ParseWorkflowDefinition(siteDefinition.Xaml, WorkflowTypes.SP2013);
workFlowTriggerAnalysisResult = WorkflowManager.Instance.ParseWorkflowTriggers(GetWorkflowPropertyBool(siteDefinition.Properties, "SPDConfig.StartOnCreate"), GetWorkflowPropertyBool(siteDefinition.Properties, "SPDConfig.StartOnChange"), GetWorkflowPropertyBool(siteDefinition.Properties, "SPDConfig.StartManually"));
}
if (siteWorkflowSubscriptions.Count() > 0)
{
foreach (var siteWorkflowSubscription in siteWorkflowSubscriptions)
{
WorkflowScanResult workflowScanResult = new WorkflowScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
ListTitle = "",
ListUrl = "",
ContentTypeId = "",
ContentTypeName = "",
Version = "2013",
Scope = "Site",
RestrictToType = siteDefinition.RestrictToType,
DefinitionName = siteDefinition.DisplayName,
DefinitionDescription = siteDefinition.Description,
SubscriptionName = siteWorkflowSubscription.Name,
HasSubscriptions = true,
Enabled = siteWorkflowSubscription.Enabled,
DefinitionId = siteDefinition.Id,
IsOOBWorkflow = false,
SubscriptionId = siteWorkflowSubscription.Id,
UsedActions = workFlowAnalysisResult?.WorkflowActions,
ActionCount = workFlowAnalysisResult != null ? workFlowAnalysisResult.ActionCount : 0,
UsedTriggers = workFlowTriggerAnalysisResult?.WorkflowTriggers,
UnsupportedActionsInFlow = workFlowAnalysisResult?.UnsupportedActions,
UnsupportedActionCount = workFlowAnalysisResult != null ? workFlowAnalysisResult.UnsupportedAccountCount : 0,
LastDefinitionEdit = GetWorkflowPropertyDateTime(siteDefinition.Properties, "Definition.ModifiedDateUTC"),
LastSubscriptionEdit = GetWorkflowPropertyDateTime(siteWorkflowSubscription.PropertyDefinitions, "SharePointWorkflowContext.Subscription.ModifiedDateUTC"),
};
if (!this.ScanJob.WorkflowScanResults.TryAdd($"workflowScanResult.SiteURL.{Guid.NewGuid()}", workflowScanResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add 2013 site workflow scan result for {workflowScanResult.SiteColUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "WorkflowAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
}
else
{
WorkflowScanResult workflowScanResult = new WorkflowScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
ListTitle = "",
ListUrl = "",
ContentTypeId = "",
ContentTypeName = "",
Version = "2013",
Scope = "Site",
RestrictToType = siteDefinition.RestrictToType,
DefinitionName = siteDefinition.DisplayName,
DefinitionDescription = siteDefinition.Description,
SubscriptionName = "",
HasSubscriptions = false,
Enabled = false,
DefinitionId = siteDefinition.Id,
IsOOBWorkflow = false,
SubscriptionId = Guid.Empty,
UsedActions = workFlowAnalysisResult?.WorkflowActions,
ActionCount = workFlowAnalysisResult != null ? workFlowAnalysisResult.ActionCount : 0,
UnsupportedActionsInFlow = workFlowAnalysisResult?.UnsupportedActions,
UnsupportedActionCount = workFlowAnalysisResult != null ? workFlowAnalysisResult.UnsupportedAccountCount : 0,
UsedTriggers = workFlowTriggerAnalysisResult?.WorkflowTriggers,
LastDefinitionEdit = GetWorkflowPropertyDateTime(siteDefinition.Properties, "Definition.ModifiedDateUTC"),
};
if (!this.ScanJob.WorkflowScanResults.TryAdd($"workflowScanResult.SiteURL.{Guid.NewGuid()}", workflowScanResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add 2013 site workflow scan result for {workflowScanResult.SiteColUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "WorkflowAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
}
}
// We've found SP2013 list scoped workflows
if (siteDefinitions != null && siteDefinitions.Count() > 0)
{
foreach (var listDefinition in siteDefinitions.Where(p => p.RestrictToType != null && (p.RestrictToType.Equals("list", StringComparison.InvariantCultureIgnoreCase) || p.RestrictToType.Equals("universal", StringComparison.InvariantCultureIgnoreCase))))
{
// Check if this workflow is also in use
var listWorkflowSubscriptions = siteSubscriptions.Where(p => p.DefinitionId.Equals(listDefinition.Id));
// Perform workflow analysis
WorkflowActionAnalysis workFlowAnalysisResult = null;
WorkflowTriggerAnalysis workFlowTriggerAnalysisResult = null;
if (Options.IncludeWorkflowWithDetails(this.ScanJob.Mode))
{
workFlowAnalysisResult = WorkflowManager.Instance.ParseWorkflowDefinition(listDefinition.Xaml, WorkflowTypes.SP2013);
workFlowTriggerAnalysisResult = WorkflowManager.Instance.ParseWorkflowTriggers(GetWorkflowPropertyBool(listDefinition.Properties, "SPDConfig.StartOnCreate"), GetWorkflowPropertyBool(listDefinition.Properties, "SPDConfig.StartOnChange"), GetWorkflowPropertyBool(listDefinition.Properties, "SPDConfig.StartManually"));
}
if (listWorkflowSubscriptions.Count() > 0)
{
foreach (var listWorkflowSubscription in listWorkflowSubscriptions)
{
Guid associatedListId = Guid.Empty;
string associatedListTitle = "";
string associatedListUrl = "";
if (Guid.TryParse(GetWorkflowProperty(listWorkflowSubscription, "Microsoft.SharePoint.ActivationProperties.ListId"), out Guid associatedListIdValue))
{
associatedListId = associatedListIdValue;
// Lookup this list and update title and url
var listLookup = lists.Where(p => p.Id.Equals(associatedListId)).FirstOrDefault();
if (listLookup != null)
{
associatedListTitle = listLookup.Title;
associatedListUrl = listLookup.RootFolder.ServerRelativeUrl;
}
}
WorkflowScanResult workflowScanResult = new WorkflowScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
ListTitle = associatedListTitle,
ListUrl = associatedListUrl,
ListId = associatedListId,
ContentTypeId = "",
ContentTypeName = "",
Version = "2013",
Scope = "List",
RestrictToType = listDefinition.RestrictToType,
DefinitionName = listDefinition.DisplayName,
DefinitionDescription = listDefinition.Description,
SubscriptionName = listWorkflowSubscription.Name,
HasSubscriptions = true,
Enabled = listWorkflowSubscription.Enabled,
DefinitionId = listDefinition.Id,
IsOOBWorkflow = false,
SubscriptionId = listWorkflowSubscription.Id,
UsedActions = workFlowAnalysisResult?.WorkflowActions,
ActionCount = workFlowAnalysisResult != null ? workFlowAnalysisResult.ActionCount : 0,
UsedTriggers = workFlowTriggerAnalysisResult?.WorkflowTriggers,
UnsupportedActionsInFlow = workFlowAnalysisResult?.UnsupportedActions,
UnsupportedActionCount = workFlowAnalysisResult != null ? workFlowAnalysisResult.UnsupportedAccountCount : 0,
LastDefinitionEdit = GetWorkflowPropertyDateTime(listDefinition.Properties, "Definition.ModifiedDateUTC"),
LastSubscriptionEdit = GetWorkflowPropertyDateTime(listWorkflowSubscription.PropertyDefinitions, "SharePointWorkflowContext.Subscription.ModifiedDateUTC"),
};
if (!this.ScanJob.WorkflowScanResults.TryAdd($"workflowScanResult.SiteURL.{Guid.NewGuid()}", workflowScanResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add 2013 list workflow scan result for {workflowScanResult.SiteColUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "WorkflowAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
}
else
{
WorkflowScanResult workflowScanResult = new WorkflowScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
ListTitle = "",
ListUrl = "",
ListId = Guid.Empty,
ContentTypeId = "",
ContentTypeName = "",
Version = "2013",
Scope = "List",
RestrictToType = listDefinition.RestrictToType,
DefinitionName = listDefinition.DisplayName,
DefinitionDescription = listDefinition.Description,
SubscriptionName = "",
HasSubscriptions = false,
Enabled = false,
DefinitionId = listDefinition.Id,
IsOOBWorkflow = false,
SubscriptionId = Guid.Empty,
UsedActions = workFlowAnalysisResult?.WorkflowActions,
ActionCount = workFlowAnalysisResult != null ? workFlowAnalysisResult.ActionCount : 0,
UsedTriggers = workFlowTriggerAnalysisResult?.WorkflowTriggers,
UnsupportedActionsInFlow = workFlowAnalysisResult?.UnsupportedActions,
UnsupportedActionCount = workFlowAnalysisResult != null ? workFlowAnalysisResult.UnsupportedAccountCount : 0,
LastDefinitionEdit = GetWorkflowPropertyDateTime(listDefinition.Properties, "Definition.ModifiedDateUTC"),
};
if (!this.ScanJob.WorkflowScanResults.TryAdd($"workflowScanResult.SiteURL.{Guid.NewGuid()}", workflowScanResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add 2013 list workflow scan result for {workflowScanResult.SiteColUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "WorkflowAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
}
}
// ***********************************************
// Site, list and content type level 2010 workflow
// ***********************************************
// Find all places where we have workflows associated (=subscribed) to SharePoint objects
if (web.WorkflowAssociations.Count > 0)
{
foreach (var workflowAssociation in web.WorkflowAssociations)
{
this.sp2010WorkflowAssociations.Add(new SP2010WorkFlowAssociation()
{
Scope = "Site", WorkflowAssociation = workflowAssociation
});
}
}
foreach (var list in lists.Where(p => p.WorkflowAssociations.Count > 0))
{
foreach (var workflowAssociation in list.WorkflowAssociations)
{
this.sp2010WorkflowAssociations.Add(new SP2010WorkFlowAssociation()
{
Scope = "List", WorkflowAssociation = workflowAssociation, AssociatedList = list
});
}
}
foreach (var ct in web.ContentTypes.Where(p => p.WorkflowAssociations.Count > 0))
{
foreach (var workflowAssociation in ct.WorkflowAssociations)
{
this.sp2010WorkflowAssociations.Add(new SP2010WorkFlowAssociation()
{
Scope = "ContentType", WorkflowAssociation = workflowAssociation, AssociatedContentType = ct
});
}
}
// Process 2010 worflows
List <Guid> processedWorkflowAssociations = new List <Guid>(this.sp2010WorkflowAssociations.Count);
if (web.WorkflowTemplates.Count > 0)
{
// Process the templates
foreach (var workflowTemplate in web.WorkflowTemplates)
{
// do we have workflows associated for this template?
var associatedWorkflows = this.sp2010WorkflowAssociations.Where(p => p.WorkflowAssociation.BaseId.Equals(workflowTemplate.Id));
if (associatedWorkflows.Count() > 0)
{
// Perform workflow analysis
// If returning null than this workflow template was an OOB workflow one
WorkflowActionAnalysis workFlowAnalysisResult = null;
Tuple <string, DateTime> loadedWorkflow = null;
if (Options.IncludeWorkflowWithDetails(this.ScanJob.Mode))
{
loadedWorkflow = LoadWorkflowDefinition(cc, workflowTemplate);
if (!string.IsNullOrEmpty(loadedWorkflow?.Item1))
{
workFlowAnalysisResult = WorkflowManager.Instance.ParseWorkflowDefinition(loadedWorkflow.Item1, WorkflowTypes.SP2010);
}
}
foreach (var associatedWorkflow in associatedWorkflows)
{
processedWorkflowAssociations.Add(associatedWorkflow.WorkflowAssociation.Id);
// Skip previous versions of a workflow
// TODO: non-english sites will use another string
if (associatedWorkflow.WorkflowAssociation.Name.Contains("(Previous Version:"))
{
continue;
}
WorkflowTriggerAnalysis workFlowTriggerAnalysisResult = null;
if (Options.IncludeWorkflowWithDetails(this.ScanJob.Mode))
{
workFlowTriggerAnalysisResult = WorkflowManager.Instance.ParseWorkflowTriggers(associatedWorkflow.WorkflowAssociation.AutoStartCreate, associatedWorkflow.WorkflowAssociation.AutoStartChange, associatedWorkflow.WorkflowAssociation.AllowManual);
}
WorkflowScanResult workflowScanResult = new WorkflowScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
ListTitle = associatedWorkflow.AssociatedList != null ? associatedWorkflow.AssociatedList.Title : "",
ListUrl = associatedWorkflow.AssociatedList != null ? associatedWorkflow.AssociatedList.RootFolder.ServerRelativeUrl : "",
ListId = associatedWorkflow.AssociatedList != null ? associatedWorkflow.AssociatedList.Id : Guid.Empty,
ContentTypeId = associatedWorkflow.AssociatedContentType != null ? associatedWorkflow.AssociatedContentType.StringId : "",
ContentTypeName = associatedWorkflow.AssociatedContentType != null ? associatedWorkflow.AssociatedContentType.Name : "",
Version = "2010",
Scope = associatedWorkflow.Scope,
RestrictToType = "N/A",
DefinitionName = workflowTemplate.Name,
DefinitionDescription = workflowTemplate.Description,
SubscriptionName = associatedWorkflow.WorkflowAssociation.Name,
HasSubscriptions = true,
Enabled = associatedWorkflow.WorkflowAssociation.Enabled,
DefinitionId = workflowTemplate.Id,
IsOOBWorkflow = IsOOBWorkflow(workflowTemplate.Id.ToString()),
SubscriptionId = associatedWorkflow.WorkflowAssociation.Id,
UsedActions = workFlowAnalysisResult?.WorkflowActions,
ActionCount = workFlowAnalysisResult != null ? workFlowAnalysisResult.ActionCount : 0,
UsedTriggers = workFlowTriggerAnalysisResult?.WorkflowTriggers,
UnsupportedActionsInFlow = workFlowAnalysisResult?.UnsupportedActions,
UnsupportedActionCount = workFlowAnalysisResult != null ? workFlowAnalysisResult.UnsupportedAccountCount : 0,
LastDefinitionEdit = loadedWorkflow != null ? loadedWorkflow.Item2 : associatedWorkflow.WorkflowAssociation.Modified,
LastSubscriptionEdit = associatedWorkflow.WorkflowAssociation.Modified,
};
if (!this.ScanJob.WorkflowScanResults.TryAdd($"workflowScanResult.SiteURL.{Guid.NewGuid()}", workflowScanResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add 2010 {associatedWorkflow.Scope} type workflow scan result for {workflowScanResult.SiteColUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "WorkflowAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
}
else
{
// Only add non OOB workflow templates when there's no associated workflow - makes the dataset smaller
if (!IsOOBWorkflow(workflowTemplate.Id.ToString()))
{
// Perform workflow analysis
WorkflowActionAnalysis workFlowAnalysisResult = null;
WorkflowTriggerAnalysis workFlowTriggerAnalysisResult = null;
Tuple <string, DateTime> loadedWorkflow = null;
if (Options.IncludeWorkflowWithDetails(this.ScanJob.Mode))
{
loadedWorkflow = LoadWorkflowDefinition(cc, workflowTemplate);
if (!string.IsNullOrEmpty(loadedWorkflow?.Item1))
{
workFlowAnalysisResult = WorkflowManager.Instance.ParseWorkflowDefinition(loadedWorkflow.Item1, WorkflowTypes.SP2010);
}
workFlowTriggerAnalysisResult = WorkflowManager.Instance.ParseWorkflowTriggers(workflowTemplate.AutoStartCreate, workflowTemplate.AutoStartChange, workflowTemplate.AllowManual);
}
WorkflowScanResult workflowScanResult = new WorkflowScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
ListTitle = "",
ListUrl = "",
ListId = Guid.Empty,
ContentTypeId = "",
ContentTypeName = "",
Version = "2010",
Scope = "",
RestrictToType = "N/A",
DefinitionName = workflowTemplate.Name,
DefinitionDescription = workflowTemplate.Description,
SubscriptionName = "",
HasSubscriptions = false,
Enabled = false,
DefinitionId = workflowTemplate.Id,
IsOOBWorkflow = IsOOBWorkflow(workflowTemplate.Id.ToString()),
SubscriptionId = Guid.Empty,
UsedActions = workFlowAnalysisResult?.WorkflowActions,
ActionCount = workFlowAnalysisResult != null ? workFlowAnalysisResult.ActionCount : 0,
UsedTriggers = workFlowTriggerAnalysisResult?.WorkflowTriggers,
LastDefinitionEdit = loadedWorkflow != null ? loadedWorkflow.Item2 : DateTime.MinValue,
};
if (!this.ScanJob.WorkflowScanResults.TryAdd($"workflowScanResult.SiteURL.{Guid.NewGuid()}", workflowScanResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add 2010 type workflow scan result for {workflowScanResult.SiteColUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "WorkflowAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
}
}
}
// Are there associated workflows for which we did not find a template (especially when the WF is created for a list)
foreach (var associatedWorkflow in this.sp2010WorkflowAssociations)
{
if (!processedWorkflowAssociations.Contains(associatedWorkflow.WorkflowAssociation.Id))
{
// Skip previous versions of a workflow
// TODO: non-english sites will use another string
if (associatedWorkflow.WorkflowAssociation.Name.Contains("(Previous Version:"))
{
continue;
}
// Perform workflow analysis
WorkflowActionAnalysis workFlowAnalysisResult = null;
WorkflowTriggerAnalysis workFlowTriggerAnalysisResult = null;
Tuple <string, DateTime> loadedWorkflow = null;
if (Options.IncludeWorkflowWithDetails(this.ScanJob.Mode))
{
loadedWorkflow = LoadWorkflowDefinition(cc, associatedWorkflow.WorkflowAssociation);
if (!string.IsNullOrEmpty(loadedWorkflow?.Item1))
{
workFlowAnalysisResult = WorkflowManager.Instance.ParseWorkflowDefinition(loadedWorkflow.Item1, WorkflowTypes.SP2010);
}
workFlowTriggerAnalysisResult = WorkflowManager.Instance.ParseWorkflowTriggers(associatedWorkflow.WorkflowAssociation.AutoStartCreate, associatedWorkflow.WorkflowAssociation.AutoStartChange, associatedWorkflow.WorkflowAssociation.AllowManual);
}
WorkflowScanResult workflowScanResult = new WorkflowScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
ListTitle = associatedWorkflow.AssociatedList != null ? associatedWorkflow.AssociatedList.Title : "",
ListUrl = associatedWorkflow.AssociatedList != null ? associatedWorkflow.AssociatedList.RootFolder.ServerRelativeUrl : "",
ListId = associatedWorkflow.AssociatedList != null ? associatedWorkflow.AssociatedList.Id : Guid.Empty,
ContentTypeId = associatedWorkflow.AssociatedContentType != null ? associatedWorkflow.AssociatedContentType.StringId : "",
ContentTypeName = associatedWorkflow.AssociatedContentType != null ? associatedWorkflow.AssociatedContentType.Name : "",
Version = "2010",
Scope = associatedWorkflow.Scope,
RestrictToType = "N/A",
DefinitionName = associatedWorkflow.WorkflowAssociation.Name,
DefinitionDescription = "",
SubscriptionName = associatedWorkflow.WorkflowAssociation.Name,
HasSubscriptions = true,
Enabled = associatedWorkflow.WorkflowAssociation.Enabled,
DefinitionId = Guid.Empty,
IsOOBWorkflow = false,
SubscriptionId = associatedWorkflow.WorkflowAssociation.Id,
UsedActions = workFlowAnalysisResult?.WorkflowActions,
ActionCount = workFlowAnalysisResult != null ? workFlowAnalysisResult.ActionCount : 0,
UsedTriggers = workFlowTriggerAnalysisResult?.WorkflowTriggers,
LastSubscriptionEdit = associatedWorkflow.WorkflowAssociation.Modified,
LastDefinitionEdit = loadedWorkflow != null ? loadedWorkflow.Item2 : associatedWorkflow.WorkflowAssociation.Modified,
};
if (!this.ScanJob.WorkflowScanResults.TryAdd($"workflowScanResult.SiteURL.{Guid.NewGuid()}", workflowScanResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add 2010 {associatedWorkflow.Scope} type workflow scan result for {workflowScanResult.SiteColUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "WorkflowAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
}
}
catch (Exception ex)
{
ScanError error = new ScanError()
{
Error = ex.Message,
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "WorkflowAnalyzer",
Field2 = ex.StackTrace,
};
// Send error to telemetry to make scanner better
if (this.ScanJob.ScannerTelemetry != null)
{
this.ScanJob.ScannerTelemetry.LogScanError(ex, error);
}
this.ScanJob.ScanErrors.Push(error);
}
finally
{
this.StopTime = DateTime.Now;
}
// return the duration of this scan
return(new TimeSpan((this.StopTime.Subtract(this.StartTime).Ticks)));
}
/// <summary>
/// Analyze the web
/// </summary>
/// <param name="cc">ClientContext of the web to be analyzed</param>
/// <returns>Duration of the analysis</returns>
public override TimeSpan Analyze(ClientContext cc)
{
try
{
base.Analyze(cc);
Web web = cc.Web;
cc.Web.EnsureProperties(p => p.WebTemplate, p => p.Configuration, p => p.RootFolder);
var homePageUrl = web.RootFolder.WelcomePage;
var listsToScan = web.GetListsToScan();
var sitePagesLibraries = listsToScan.Where(p => p.BaseTemplate == (int)ListTemplateType.WebPageLibrary);
var assetLibraries = listsToScan.Where(p => p.IsSiteAssetsLibrary == true);
var librariesToScan = sitePagesLibraries.Union(assetLibraries);
if (librariesToScan.Count() > 0)
{
foreach (var libraryToScan in librariesToScan)
{
CamlQuery query = new CamlQuery
{
ViewXml = CAMLQueryByExtension
};
var pages = libraryToScan.GetItems(query);
web.Context.Load(pages);
web.Context.ExecuteQueryRetry();
if (pages.FirstOrDefault() != null)
{
DateTime start;
bool forceCheckout = libraryToScan.ForceCheckout;
foreach (var page in pages)
{
string pageUrl = null;
try
{
if (page.FieldValues.ContainsKey(FileRefField) && !String.IsNullOrEmpty(page[FileRefField].ToString()))
{
pageUrl = page[FileRefField].ToString();
}
else
{
//skip page
continue;
}
start = DateTime.Now;
VisioWebPartScanResult visioWebPartResult = null;
// Get page web parts
var foundWebParts = page.WebParts();
if (foundWebParts != null)
{
// Do we have a visio web part?
var visioParts = foundWebParts.Where(p => p.Type == "Microsoft.Office.Visio.Server.WebControls.VisioWebAccess, Microsoft.Office.Visio.Server, Version=16.0.0.0, Culture=neutral, PublicKeyToken=71e9bce111e9429c").ToList();
if (visioParts.Count > 0)
{
Console.WriteLine($"Visio web part found on page {pageUrl}");
visioWebPartResult = new VisioWebPartScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
PageUrl = pageUrl,
Library = libraryToScan.RootFolder.ServerRelativeUrl,
WebParts = foundWebParts,
};
// Get page change information
visioWebPartResult.ModifiedAt = page.LastModifiedDateTime();
visioWebPartResult.ModifiedBy = page.LastModifiedBy();
// Grab this page from the search results to connect view information
// Is this page the web's home page?
bool isHomePage = false;
if (pageUrl.EndsWith(homePageUrl, StringComparison.InvariantCultureIgnoreCase))
{
isHomePage = true;
}
string fullPageUrl = $"https://{new Uri(this.SiteCollectionUrl).DnsSafeHost}{pageUrl}";
if (isHomePage)
{
fullPageUrl = this.SiteUrl;
}
var searchPage = this.pageSearchResults.Where(x => x.Values.Contains(fullPageUrl)).FirstOrDefault();
if (searchPage != null)
{
// Recent = last 14 days
visioWebPartResult.ViewsRecent = searchPage["ViewsRecent"].ToInt32();
visioWebPartResult.ViewsRecentUniqueUsers = searchPage["ViewsRecentUniqueUsers"].ToInt32();
visioWebPartResult.ViewsLifeTime = searchPage["ViewsLifeTime"].ToInt32();
visioWebPartResult.ViewsLifeTimeUniqueUsers = searchPage["ViewsLifeTimeUniqueUsers"].ToInt32();
}
if (!this.ScanJob.VisioWebPartScanResults.TryAdd(visioWebPartResult.PageUrl, visioWebPartResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add page scan result for {visioWebPartResult.PageUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "PageAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
}
var duration = new TimeSpan((DateTime.Now.Subtract(start).Ticks));
Console.WriteLine($"Scan of page {pageUrl} took {duration.Seconds} seconds");
}
catch (Exception ex)
{
ScanError error = new ScanError()
{
Error = ex.Message,
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "MainPageAnalyzerLoop",
Field2 = ex.StackTrace,
Field3 = pageUrl
};
this.ScanJob.ScanErrors.Push(error);
Console.WriteLine("Error for page {1}: {0}", ex.Message, pageUrl);
}
}
}
}
}
}
finally
{
this.StopTime = DateTime.Now;
}
// return the duration of this scan
return(new TimeSpan((this.StopTime.Subtract(this.StartTime).Ticks)));
}
private static void AssertFailScanErrorHandler(ScanError scanError)
{
throw scanError;
}
/// <summary>
/// Analyses a web for it's workflow usage
/// </summary>
/// <param name="cc">ClientContext instance used to retrieve workflow data</param>
/// <returns>Duration of the workflow analysis</returns>
public override TimeSpan Analyze(ClientContext cc)
{
try
{
base.Analyze(cc);
var baseUri = new Uri(this.SiteUrl);
var webAppUrl = baseUri.Scheme + "://" + baseUri.Host;
var lists = cc.Web.GetListsToScan(showHidden: true);
foreach (var list in lists)
{
// Skip system lists, except for Converted Forms and Form Templates libraries
if ((list.IsSystemList || list.IsCatalog || list.IsSiteAssetsLibrary || list.IsEnterpriseGalleryLibrary) &&
((int)list.BaseTemplate != 10102 || !string.Equals(list.RootFolder.Name, "FormServerTemplates", StringComparison.InvariantCultureIgnoreCase)))
{
continue;
}
if ((int)list.BaseTemplate == 10102)
{
// Converted Forms library
// Stores converted InfoPath forms for browser rendering. This can become bloated
// with many versions stored after every republish. Recommended to clean this out
// after migrating away from InfoPath
if (list.ItemCount > 0)
{
InfoPathScanResult infoPathScanResult = new InfoPathScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
InfoPathUsage = "ConvertedFormLibrary",
ListTitle = list.Title,
ListId = list.Id,
ListUrl = list.RootFolder.ServerRelativeUrl,
Enabled = true,
InfoPathTemplate = string.Empty,
InfoPathTemplateUrl = string.Empty,
ItemCount = list.ItemCount,
LastItemUserModifiedDate = list.LastItemUserModifiedDate,
};
if (!this.ScanJob.InfoPathScanResults.TryAdd($"{infoPathScanResult.SiteURL}.{Guid.NewGuid()}", infoPathScanResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add formlibrary InfoPath scan result for {infoPathScanResult.SiteColUrl} and list {infoPathScanResult.ListUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "InfoPathAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
continue;
}
if (string.Equals(list.RootFolder.Name, "FormServerTemplates", StringComparison.InvariantCultureIgnoreCase))
{
// Form Templates library
// Any templates in here indicate sandbox solution/admin approved forms
// Possibly from an on-prem migration?
if (list.ItemCount > 0)
{
InfoPathScanResult infoPathScanResult = new InfoPathScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
InfoPathUsage = "FormServerTemplates",
ListTitle = list.Title,
ListId = list.Id,
ListUrl = list.RootFolder.ServerRelativeUrl,
Enabled = true,
InfoPathTemplate = string.Empty,
InfoPathTemplateUrl = string.Empty,
ItemCount = list.ItemCount,
LastItemUserModifiedDate = list.LastItemUserModifiedDate,
};
if (!this.ScanJob.InfoPathScanResults.TryAdd($"{infoPathScanResult.SiteURL}.{Guid.NewGuid()}", infoPathScanResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add formlibrary InfoPath scan result for {infoPathScanResult.SiteColUrl} and list {infoPathScanResult.ListUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "InfoPathAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
continue;
}
if (list.BaseType == BaseType.DocumentLibrary)
{
if (!String.IsNullOrEmpty(list.DocumentTemplateUrl) && list.DocumentTemplateUrl.EndsWith(".xsn", StringComparison.InvariantCultureIgnoreCase))
{
// Library using an InfoPath file as New Item template
InfoPathScanResult infoPathScanResult = new InfoPathScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
InfoPathUsage = "FormLibrary",
ListTitle = list.Title,
ListId = list.Id,
ListUrl = list.RootFolder.ServerRelativeUrl,
Enabled = true,
InfoPathTemplate = list.DocumentTemplateUrl,
InfoPathTemplateUrl = this.SiteUrl.TrimEnd('/') + "/_layouts/16/download.aspx?SourceUrl=" + list.DocumentTemplateUrl,
ItemCount = list.ItemCount,
LastItemUserModifiedDate = list.LastItemUserModifiedDate,
};
// Root relative URL for the template
var templateFile = cc.Web.GetFileByServerRelativeUrl(list.DocumentTemplateUrl);
infoPathScanResult = ScrapeXsn(cc, infoPathScanResult, templateFile);
if (!this.ScanJob.InfoPathScanResults.TryAdd($"{infoPathScanResult.SiteURL}.{Guid.NewGuid()}", infoPathScanResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add formlibrary InfoPath scan result for {infoPathScanResult.SiteColUrl} and list {infoPathScanResult.ListUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "InfoPathAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
// Iterate through all content types - we may not have a template assigned to the library itself,
// only to content types
cc.Load(list, p => p.ContentTypes.Include(i => i.Name, i => i.DocumentTemplate, i => i.DocumentTemplateUrl, i => i.Id));
cc.ExecuteQueryRetry();
foreach (var contentType in list.ContentTypes)
{
if ((!string.IsNullOrEmpty(contentType.DocumentTemplateUrl) &&
contentType.DocumentTemplateUrl.EndsWith(".xsn", StringComparison.InvariantCultureIgnoreCase)) ||
contentType.Id.StringValue.StartsWith(FormBaseContentType))
{
InfoPathScanResult ctInfoPathScanResult = new InfoPathScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
InfoPathUsage = "FormLibraryContentType",
ListTitle = list.Title,
ListId = list.Id,
ListUrl = list.RootFolder.ServerRelativeUrl,
Enabled = true,
InfoPathTemplate = contentType.DocumentTemplateUrl,
InfoPathTemplateUrl = this.SiteUrl.TrimEnd('/') + "/_layouts/16/download.aspx?SourceUrl=" + contentType.DocumentTemplateUrl,
ItemCount = list.ItemCount,
ContentTypeName = contentType.Name,
LastItemUserModifiedDate = list.LastItemUserModifiedDate,
};
// Root relative URL for the template
var templateFile = cc.Web.GetFileByServerRelativeUrl(contentType.DocumentTemplateUrl);
ctInfoPathScanResult = ScrapeXsn(cc, ctInfoPathScanResult, templateFile);
if (!this.ScanJob.InfoPathScanResults.TryAdd($"{ctInfoPathScanResult.SiteURL}.{Guid.NewGuid()}", ctInfoPathScanResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add formlibrary InfoPath scan result for {ctInfoPathScanResult.SiteColUrl} and list {ctInfoPathScanResult.ListUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "InfoPathAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
}
}
else if (list.BaseType == BaseType.GenericList)
{
try
{
Folder folder = cc.Web.GetFolderByServerRelativeUrl($"{list.RootFolder.ServerRelativeUrl}/Item");
cc.Load(folder, p => p.Properties);
cc.ExecuteQueryRetry();
if (folder.Properties.FieldValues.ContainsKey("_ipfs_infopathenabled") && folder.Properties.FieldValues.ContainsKey("_ipfs_solutionName"))
{
bool infoPathEnabled = true;
if (bool.TryParse(folder.Properties.FieldValues["_ipfs_infopathenabled"].ToString(), out bool infoPathEnabledParsed))
{
infoPathEnabled = infoPathEnabledParsed;
}
// List with an InfoPath customization
InfoPathScanResult infoPathScanResult = new InfoPathScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
InfoPathUsage = "CustomForm",
ListTitle = list.Title,
ListId = list.Id,
ListUrl = list.RootFolder.ServerRelativeUrl,
Enabled = infoPathEnabled,
InfoPathTemplate = folder.Properties.FieldValues["_ipfs_solutionName"].ToString(),
InfoPathTemplateUrl = this.SiteUrl.TrimEnd('/') + "/_layouts/16/download.aspx?SourceUrl=" + cc.Web.Url.TrimEnd('/') + "/Lists/" + list.RootFolder.Name + "/" + folder.Properties.FieldValues["_ipfs_solutionName"].ToString(),
ItemCount = list.ItemCount,
LastItemUserModifiedDate = list.LastItemUserModifiedDate,
};
// Run the scraper
var templateFile = list.RootFolder.GetFile($"item/{infoPathScanResult.InfoPathTemplate}");
infoPathScanResult = ScrapeXsn(cc, infoPathScanResult, templateFile);
if (!this.ScanJob.InfoPathScanResults.TryAdd($"{infoPathScanResult.SiteURL}.{Guid.NewGuid()}", infoPathScanResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add customform InfoPath scan result for {infoPathScanResult.SiteColUrl} and list {infoPathScanResult.ListUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "InfoPathAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
}
catch (ServerException ex)
{
if (((ServerException)ex).ServerErrorTypeName == "System.IO.FileNotFoundException")
{
// Ignore
}
else
{
ScanError error = new ScanError()
{
Error = ex.Message,
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "InfoPathAnalyzer",
Field2 = ex.StackTrace,
Field3 = $"{webAppUrl}{list.DefaultViewUrl}"
};
// Send error to telemetry to make scanner better
if (this.ScanJob.ScannerTelemetry != null)
{
this.ScanJob.ScannerTelemetry.LogScanError(ex, error);
}
this.ScanJob.ScanErrors.Push(error);
Console.WriteLine("Error during InfoPath analysis for list {1}: {0}", ex.Message, $"{webAppUrl}{list.DefaultViewUrl}");
}
}
catch (Exception ex)
{
}
}
}
}
catch (Exception ex)
{
}
finally
{
this.StopTime = DateTime.Now;
}
// return the duration of this scan
return(new TimeSpan((this.StopTime.Subtract(this.StartTime).Ticks)));
}
/// <summary>
/// Scans a list for "modern" compatibility
/// </summary>
/// <param name="file">List form page to start the scan from</param>
/// <param name="list">List linked to the form page</param>
/// <returns>Object describing modern compatiblity</returns>
public static ListScanResult ModernCompatability(this File file, List list, ref ConcurrentStack <ScanError> scanErrors)
{
if (file == null)
{
throw new ArgumentNullException("file");
}
if (list == null)
{
throw new ArgumentNullException("list");
}
ClientContext cc = file.Context as ClientContext;
ListScanResult result = new ListScanResult();
// Load properties
file.EnsureProperties(p => p.PageRenderType);
// If it works in modern, we're good
if (file.PageRenderType == ListPageRenderType.Modern)
{
// let's return since we know it will work
return(result);
}
else
{
result.PageRenderType = file.PageRenderType;
}
// Hmmm...it's not working, so let's list *all* reasons why it's not working in modern
// Step 1: load the tenant / site / web / list level blocking options
// Tenant
// Currently we've no API to detect tenant setting...but since we anyhow should scan all lists this does not matter that much
// Site
Site site = cc.Site;
site.EnsureProperties(p => p.Features, p => p.Url);
result.BlockedAtSiteLevel = site.Features.Where(f => f.DefinitionId == FeatureId_Site_Modern).Count() > 0;
// Web
cc.Web.EnsureProperties(p => p.Features, p => p.Url);
result.BlockedAtWebLevel = cc.Web.Features.Where(f => f.DefinitionId == FeatureId_Web_Modern).Count() > 0;
// List
list.EnsureProperties(p => p.ListExperienceOptions, p => p.UserCustomActions, p => p.BaseTemplate);
result.ListExperience = list.ListExperienceOptions;
result.XsltViewWebPartCompatibility.ListBaseTemplate = list.BaseTemplate;
if (list.ListExperienceOptions == ListExperience.ClassicExperience)
{
result.BlockedAtListLevel = true;
}
// Step 2: verify we can load a web part manager and ensure there's only one web part of the page
LimitedWebPartManager wpm;
try
{
wpm = file.GetLimitedWebPartManager(PersonalizationScope.Shared);
file.Context.Load(wpm.WebParts, wps => wps.Include(wp => wp.WebPart.Title, wp => wp.WebPart.Properties));
file.Context.ExecuteQueryRetry();
}
catch (Exception ex)
{
result.BlockedByNotBeingAbleToLoadPage = true;
result.BlockedByNotBeingAbleToLoadPageException = ex.ToString();
return(result);
}
if (wpm.WebParts.Count != 1)
{
result.BlockedByZeroOrMultipleWebParts = true;
return(result);
}
var webPart = wpm.WebParts[0].WebPart;
// Step 3: Inspect the web part used to render the list
// Step 3.1: JSLink web part check
if (webPart.Properties.FieldValues.Keys.Contains("JSLink"))
{
if (webPart.Properties["JSLink"] != null && !String.IsNullOrEmpty(webPart.Properties["JSLink"].ToString()) && webPart.Properties["JSLink"].ToString().ToLower() != "clienttemplates.js")
{
result.XsltViewWebPartCompatibility.BlockedByJSLink = true;
result.XsltViewWebPartCompatibility.JSLink = webPart.Properties["JSLink"].ToString();
}
}
// Step 3.2: XslLink web part check
if (webPart.Properties.FieldValues.Keys.Contains("XslLink"))
{
if (webPart.Properties["XslLink"] != null && !String.IsNullOrEmpty(webPart.Properties["XslLink"].ToString()) && webPart.Properties["XslLink"].ToString().ToLower() != "main.xsl")
{
result.XsltViewWebPartCompatibility.BlockedByXslLink = true;
result.XsltViewWebPartCompatibility.XslLink = webPart.Properties["XslLink"].ToString();
}
}
// Step 3.3: Xsl web part check
if (webPart.Properties.FieldValues.Keys.Contains("Xsl"))
{
if (webPart.Properties["Xsl"] != null && !String.IsNullOrEmpty(webPart.Properties["Xsl"].ToString()))
{
result.XsltViewWebPartCompatibility.BlockedByXsl = true;
}
}
// Step 3.4: Process fields in view
if (webPart.Properties.FieldValues.Keys.Contains("XmlDefinition"))
{
if (webPart.Properties["XmlDefinition"] != null && !String.IsNullOrEmpty(webPart.Properties["XmlDefinition"].ToString()))
{
try
{
// Get the fields in this view
var viewFields = GetViewFields(webPart.Properties["XmlDefinition"].ToString());
// Load fields in one go
List <Field> fieldsToProcess = new List <Field>(viewFields.Count);
try
{
foreach (var viewField in viewFields)
{
Field field = list.Fields.GetByInternalNameOrTitle(viewField);
cc.Load(field, p => p.JSLink, p => p.TypeAsString, p => p.FieldTypeKind, p => p.InternalName);
fieldsToProcess.Add(field);
}
cc.ExecuteQueryRetry();
}
catch
{
// try to load the fields again, but now individually so we can collect the needed errors + evaulate the fields that do load
fieldsToProcess.Clear();
foreach (var viewField in viewFields)
{
try
{
Field field = list.Fields.GetByInternalNameOrTitle(viewField);
cc.Load(field, p => p.JSLink, p => p.TypeAsString, p => p.FieldTypeKind);
cc.ExecuteQueryRetry();
fieldsToProcess.Add(field);
}
catch (Exception ex)
{
ScanError error = new ScanError()
{
Error = ex.Message,
SiteURL = cc.Web.Url,
SiteColUrl = site.Url
};
scanErrors.Push(error);
Console.WriteLine("Error for site {1}: {0}", ex.Message, cc.Web.Url);
}
}
}
// Verify the fields
foreach (var field in fieldsToProcess)
{
try
{
// JSLink on field
if (!string.IsNullOrEmpty(field.JSLink) && field.JSLink != "clienttemplates.js" &&
field.JSLink != "sp.ui.reputation.js" && !field.IsTaxField())
{
result.XsltViewWebPartCompatibility.BlockedByJSLinkField = true;
result.XsltViewWebPartCompatibility.JSLinkFields = string.IsNullOrEmpty(result.XsltViewWebPartCompatibility.JSLinkFields) ? $"{field.InternalName}" : $"{result.XsltViewWebPartCompatibility.JSLinkFields},{field.InternalName}";
}
//Business data field
if (field.IsBusinessDataField())
{
result.XsltViewWebPartCompatibility.BlockedByBusinessDataField = true;
result.XsltViewWebPartCompatibility.BusinessDataFields = string.IsNullOrEmpty(result.XsltViewWebPartCompatibility.BusinessDataFields) ? $"{field.InternalName}" : $"{result.XsltViewWebPartCompatibility.BusinessDataFields},{field.InternalName}";
}
// Geolocation field
if (field.FieldTypeKind == FieldType.Geolocation)
{
result.XsltViewWebPartCompatibility.BlockedByGeoLocationField = true;
result.XsltViewWebPartCompatibility.GeoLocationFields = string.IsNullOrEmpty(result.XsltViewWebPartCompatibility.GeoLocationFields) ? $"{field.InternalName}" : $"{result.XsltViewWebPartCompatibility.GeoLocationFields},{field.InternalName}";
}
// TaskOutcome field
if (field.IsTaskOutcomeField())
{
result.XsltViewWebPartCompatibility.BlockedByTaskOutcomeField = true;
result.XsltViewWebPartCompatibility.TaskOutcomeFields = string.IsNullOrEmpty(result.XsltViewWebPartCompatibility.TaskOutcomeFields) ? $"{field.InternalName}" : $"{result.XsltViewWebPartCompatibility.TaskOutcomeFields},{field.InternalName}";
}
// Publishing field
if (field.IsPublishingField())
{
result.XsltViewWebPartCompatibility.BlockedByPublishingField = true;
result.XsltViewWebPartCompatibility.PublishingFields = string.IsNullOrEmpty(result.XsltViewWebPartCompatibility.PublishingFields) ? $"{field.InternalName}" : $"{result.XsltViewWebPartCompatibility.PublishingFields},{field.InternalName}";
}
}
catch (Exception ex)
{
ScanError error = new ScanError()
{
Error = ex.Message,
SiteURL = cc.Web.Url,
SiteColUrl = site.Url,
};
scanErrors.Push(error);
Console.WriteLine("Error for site {1}: {0}", ex.Message, cc.Web.Url);
}
}
}
catch (Exception ex)
{
ScanError error = new ScanError()
{
Error = ex.Message,
SiteURL = cc.Web.Url,
SiteColUrl = site.Url
};
scanErrors.Push(error);
Console.WriteLine("Error for site {1}: {0}", ex.Message, cc.Web.Url);
}
}
}
// Step 3.5: Process list custom actions
if (list.UserCustomActions.Count > 0)
{
foreach (var customAction in list.UserCustomActions)
{
if (!string.IsNullOrEmpty(customAction.Location) && customAction.Location.Equals("scriptlink", StringComparison.InvariantCultureIgnoreCase))
{
if (!string.IsNullOrEmpty(customAction.ScriptSrc))
{
result.XsltViewWebPartCompatibility.BlockedByListCustomAction = true;
result.XsltViewWebPartCompatibility.ListCustomActions = string.IsNullOrEmpty(result.XsltViewWebPartCompatibility.ListCustomActions) ? $"{customAction.Name}" : $"{result.XsltViewWebPartCompatibility.ListCustomActions},{customAction.Name}";
}
}
}
}
// Step 3.6: managed metadata navigation is not an issue anymore
result.XsltViewWebPartCompatibility.BlockedByManagedMetadataNavFeature = false;
// Step 4: check the view
if (webPart.Properties.FieldValues.Keys.Contains("ViewFlags") && webPart.Properties["ViewFlags"] != null && !String.IsNullOrEmpty(webPart.Properties["ViewFlags"].ToString()))
{
uint flags;
if (uint.TryParse(webPart.Properties["ViewFlags"].ToString(), out flags))
{
if ((flags & ViewFlag_Gantt) != 0 || (flags & ViewFlag_Calendar) != 0 || (flags & ViewFlag_Grid) != 0)
{
result.XsltViewWebPartCompatibility.BlockedByViewType = true;
if ((flags & ViewFlag_Gantt) != 0)
{
result.XsltViewWebPartCompatibility.ViewType = "Gantt";
}
else if ((flags & ViewFlag_Calendar) != 0)
{
result.XsltViewWebPartCompatibility.ViewType = "Calendar";
}
else if ((flags & ViewFlag_Grid) != 0)
{
if (list.BaseTemplate == (int)ListTemplateType.GenericList ||
list.BaseTemplate == (int)ListTemplateType.DocumentLibrary)
{
// unblock...we've added support for datasheet rendering for custom lists in July 2018 (see https://techcommunity.microsoft.com/t5/Microsoft-SharePoint-Blog/Updates-to-metadata-handling-and-list-templates/ba-p/202113)
result.XsltViewWebPartCompatibility.BlockedByViewType = false;
}
else
{
result.XsltViewWebPartCompatibility.ViewType = "Grid";
}
}
}
}
}
// Step 5: check the list
// Step 5.1: check the base template
if (!list.CanRenderNewExperience())
{
result.XsltViewWebPartCompatibility.BlockedByListBaseTemplate = true;
result.XsltViewWebPartCompatibility.ListBaseTemplate = list.BaseTemplate;
}
return(result);
}
private static void AssertFailScanErrorHandler(ScanError scanError)
{
// Note: does not contain stack trace, so no point in wrapping in Exception() at this point.
throw scanError;
}
/// <summary>
/// Analyses a page
/// </summary>
/// <param name="cc">ClientContext instance used to retrieve page data</param>
/// <returns>Duration of the page analysis</returns>
public override TimeSpan Analyze(ClientContext cc)
{
try
{
base.Analyze(cc);
Web web = cc.Web;
cc.Web.EnsureProperties(p => p.WebTemplate, p => p.Configuration, p => p.RootFolder);
var homePageUrl = web.RootFolder.WelcomePage;
var listsToScan = web.GetListsToScan();
var sitePagesLibraries = listsToScan.Where(p => p.BaseTemplate == (int)ListTemplateType.WebPageLibrary);
if (sitePagesLibraries.Count() > 0)
{
foreach (var sitePagesLibrary in sitePagesLibraries)
{
CamlQuery query = new CamlQuery
{
ViewXml = CAMLQueryByExtension
};
var pages = sitePagesLibrary.GetItems(query);
web.Context.Load(pages);
web.Context.ExecuteQueryRetry();
if (pages.FirstOrDefault() != null)
{
DateTime start;
bool forceCheckout = sitePagesLibrary.ForceCheckout;
foreach (var page in pages)
{
string pageUrl = null;
try
{
if (page.FieldValues.ContainsKey(FileRefField) && !String.IsNullOrEmpty(page[FileRefField].ToString()))
{
pageUrl = page[FileRefField].ToString();
}
else
{
//skip page
continue;
}
start = DateTime.Now;
PageScanResult pageResult = new PageScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
PageUrl = pageUrl,
Library = sitePagesLibrary.RootFolder.ServerRelativeUrl,
};
// Is this page the web's home page?
if (pageUrl.EndsWith(homePageUrl, StringComparison.InvariantCultureIgnoreCase))
{
pageResult.HomePage = true;
}
// Get the type of the page
pageResult.PageType = page.PageType();
// Get page web parts
var pageAnalysis = page.WebParts(this.ScanJob.PageTransformation);
if (pageAnalysis != null)
{
pageResult.Layout = pageAnalysis.Item1.ToString().Replace("Wiki_", "").Replace("WebPart_", "");
pageResult.WebParts = pageAnalysis.Item2;
}
// Get page change information
pageResult.ModifiedAt = page.LastModifiedDateTime();
pageResult.ModifiedBy = page.LastModifiedBy();
// Grab this page from the search results to connect view information
string fullPageUrl = $"https://{new Uri(this.SiteCollectionUrl).DnsSafeHost}{pageUrl}";
if (pageResult.HomePage)
{
fullPageUrl = this.SiteUrl;
}
var searchPage = this.pageSearchResults.Where(x => x.Values.Contains(fullPageUrl)).FirstOrDefault();
if (searchPage != null)
{
// Recent = last 14 days
pageResult.ViewsRecent = searchPage["ViewsRecent"].ToInt32();
pageResult.ViewsRecentUniqueUsers = searchPage["ViewsRecentUniqueUsers"].ToInt32();
pageResult.ViewsLifeTime = searchPage["ViewsLifeTime"].ToInt32();
pageResult.ViewsLifeTimeUniqueUsers = searchPage["ViewsLifeTimeUniqueUsers"].ToInt32();
}
if (!this.ScanJob.PageScanResults.TryAdd(pageResult.PageUrl, pageResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add page scan result for {pageResult.PageUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "PageAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
var duration = new TimeSpan((DateTime.Now.Subtract(start).Ticks));
Console.WriteLine($"Scan of page {pageUrl} took {duration.Seconds} seconds");
}
catch (Exception ex)
{
ScanError error = new ScanError()
{
Error = ex.Message,
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "MainPageAnalyzerLoop",
Field2 = ex.StackTrace,
Field3 = pageUrl
};
this.ScanJob.ScanErrors.Push(error);
Console.WriteLine("Error for page {1}: {0}", ex.Message, pageUrl);
}
}
}
}
}
}
finally
{
this.StopTime = DateTime.Now;
}
// return the duration of this scan
return(new TimeSpan((this.StopTime.Subtract(this.StartTime).Ticks)));
}
private void ScanError(ScanError scanError)
{
ReportError(scanError.Line, String.Empty, scanError.Message);
}
/// <summary>
/// Analyze the web
/// </summary>
/// <param name="cc">ClientContext of the web to be analyzed</param>
/// <returns>Duration of the analysis</returns>
public override TimeSpan Analyze(ClientContext cc)
{
try
{
base.Analyze(cc);
// Ensure needed data is loaded
Web web = cc.Web;
web.EnsureProperties(p => p.UserCustomActions, p => p.AlternateCssUrl, p => p.CustomMasterUrl, p => p.MasterUrl, p => p.Features, p => p.WebTemplate, p => p.Configuration, p => p.HasUniqueRoleAssignments);
// Log in Site scan data that the scanned web is a sub site
if (web.IsSubSite())
{
SiteScanResult siteScanData;
if (this.ScanJob.SiteScanResults.TryGetValue(this.SiteCollectionUrl, out siteScanData))
{
if (!siteScanData.SubSites)
{
var clonedSiteScandata = siteScanData.Clone();
clonedSiteScandata.SubSites = true;
if (!this.ScanJob.SiteScanResults.TryUpdate(this.SiteCollectionUrl, clonedSiteScandata, siteScanData))
{
ScanError error = new ScanError()
{
Error = $"Could not add update site scan result for {this.SiteCollectionUrl} from web scan of {this.SiteUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "WebAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
}
// Check if we've broken permission inheritance in this site collection
if (web.HasUniqueRoleAssignments)
{
SiteScanResult siteScanData2;
if (this.ScanJob.SiteScanResults.TryGetValue(this.SiteCollectionUrl, out siteScanData2))
{
if (!siteScanData2.SubSitesWithBrokenPermissionInheritance)
{
var clonedSiteScandata = siteScanData2.Clone();
clonedSiteScandata.SubSitesWithBrokenPermissionInheritance = true;
if (!this.ScanJob.SiteScanResults.TryUpdate(this.SiteCollectionUrl, clonedSiteScandata, siteScanData2))
{
ScanError error = new ScanError()
{
Error = $"Could not add update site scan result for {this.SiteCollectionUrl} from web scan of {this.SiteUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "WebAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
}
}
}
// Perform specific analysis work
WebScanResult scanResult = new WebScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
};
// Log used web template
if (web.WebTemplate != null)
{
scanResult.WebTemplate = $"{web.WebTemplate}#{web.Configuration}";
}
// Page feature check: users can disable this to prevent modern page creation
scanResult.ModernPageWebFeatureDisabled = web.Features.Where(f => f.DefinitionId == FeatureId_Web_ModernPage).Count() == 0;
// List feature check: users can enabled this to prevent modern lists from working
scanResult.ModernListWebBlockingFeatureEnabled = web.Features.Where(f => f.DefinitionId == FeatureId_Web_ModernList).Count() > 0;
// Publishing web feature enabled
scanResult.WebPublishingFeatureEnabled = web.Features.Where(f => f.DefinitionId == FeatureId_Web_Publishing).Count() > 0;
// Site is using the publishing "Pages" library?
if (scanResult.WebPublishingFeatureEnabled)
{
if (!(scanResult.WebTemplate.Equals("BICENTERSITE#0", StringComparison.InvariantCultureIgnoreCase) ||
scanResult.WebTemplate.Equals("BLANKINTERNET#0", StringComparison.InvariantCulture) ||
scanResult.WebTemplate.Equals("SRCHCEN#0", StringComparison.InvariantCulture) ||
scanResult.WebTemplate.Equals("CMSPUBLISHING#0", StringComparison.InvariantCulture) ||
scanResult.WebTemplate.Equals("SRCHCENTERLITE#0", StringComparison.InvariantCulture) ||
scanResult.WebTemplate.Equals("POINTPUBLISHINGHUB#0", StringComparison.InvariantCulture) ||
scanResult.WebTemplate.Equals("POINTPUBLISHINGTOPIC#0", StringComparison.InvariantCulture) ||
scanResult.WebTemplate.Equals("ENTERWIKI#0", StringComparison.InvariantCulture)))
{
var pagesLibrary = web.GetListsToScan().Where(p => p.BaseTemplate == 850).FirstOrDefault();
if (pagesLibrary != null)
{
if (pagesLibrary.ItemCount > 0)
{
scanResult.PublishingPagesLibraryContainsPages = true;
}
}
}
}
// Log permission inheritance details
if (web.IsSubSite() && web.HasUniqueRoleAssignments)
{
scanResult.BrokenPermissionInheritance = web.HasUniqueRoleAssignments;
if (!this.ScanJob.SkipUserInformation)
{
scanResult.Owners = web.GetOwners();
scanResult.Members = web.GetMembers();
scanResult.Visitors = web.GetVisitors();
scanResult.EveryoneClaimsGranted = web.ClaimsHaveRoleAssignment(this.ScanJob.EveryoneClaim, this.ScanJob.EveryoneExceptExternalUsersClaim);
}
}
// If the web template is STS#0, GROUP#0 or SITEPAGEPUBLISHING#0 then the feature was activated by SPO, other templates never got it
scanResult.ModernPageFeatureWasEnabledBySPO = false;
if (scanResult.WebTemplate.Equals("STS#0", StringComparison.InvariantCultureIgnoreCase) ||
scanResult.WebTemplate.Equals("STS#3", StringComparison.InvariantCultureIgnoreCase) ||
scanResult.WebTemplate.Equals("GROUP#0", StringComparison.InvariantCulture) ||
scanResult.WebTemplate.Equals("SITEPAGEPUBLISHING#0", StringComparison.InvariantCulture))
{
// Since we did not enable the feature for all STS#0 sites (sites with publishing active did not get it, nor sites having a large number of webpart/wiki pages) we
// check if it should have been turned on by checking for the "site page" content type being added to the site pages library
var listsToScan = web.GetListsToScan();
var sitePagesLibrary = listsToScan.Where(p => p.BaseTemplate == (int)ListTemplateType.WebPageLibrary).FirstOrDefault();
if (sitePagesLibrary != null)
{
cc.Load(sitePagesLibrary, p => p.ContentTypes.Include(c => c.Id));
cc.ExecuteQueryRetry();
var sitePageContentTypeFound = sitePagesLibrary.ContentTypes.Where(c => c.Id.StringValue.StartsWith(SitePageContentTypeId, StringComparison.InvariantCultureIgnoreCase)).OrderBy(c => c.Id.StringValue.Length).FirstOrDefault();
if (sitePageContentTypeFound != null)
{
scanResult.ModernPageFeatureWasEnabledBySPO = true;
}
}
}
// Get information about the master pages used
if (!string.IsNullOrEmpty(web.MasterUrl) && !excludeMasterPage.Contains(web.MasterUrl.Substring(web.MasterUrl.LastIndexOf("/") + 1).ToLower()))
{
scanResult.MasterPage = web.MasterUrl;
}
if (!string.IsNullOrEmpty(web.CustomMasterUrl) && !excludeMasterPage.Contains(web.CustomMasterUrl.Substring(web.CustomMasterUrl.LastIndexOf("/") + 1).ToLower()))
{
scanResult.CustomMasterPage = web.CustomMasterUrl;
}
if (!string.IsNullOrEmpty(web.AlternateCssUrl))
{
scanResult.AlternateCSS = web.AlternateCssUrl;
}
// Get the user custom actions
scanResult.WebUserCustomActions = web.UserCustomActions.Analyze(this.SiteCollectionUrl, this.SiteUrl);
// Get home page from the web and check whether it's a modern page or not
scanResult.ModernHomePage = false;
var homePageUrl = web.WelcomePage;
if (string.IsNullOrEmpty(homePageUrl))
{
// Will be case when the site home page is a web part page
homePageUrl = "default.aspx";
}
var homepageName = System.IO.Path.GetFileName(homePageUrl);
var sitePagesLibraryForWeb = web.GetListsToScan().Where(p => p.BaseTemplate == (int)ListTemplateType.WebPageLibrary).FirstOrDefault();
if (sitePagesLibraryForWeb != null && homePageUrl.StartsWith("SitePages", StringComparison.InvariantCultureIgnoreCase))
{
var homePageFile = web.GetFileByServerRelativeUrl($"{(web.ServerRelativeUrl.Length == 1 ? "" : web.ServerRelativeUrl)}/{homePageUrl}");
cc.Load(homePageFile, f => f.ListItemAllFields, f => f.Exists);
cc.ExecuteQueryRetry();
if (homePageFile.Exists)
{
var item = homePageFile.ListItemAllFields;
if (item.FieldValues.ContainsKey(ClientSideApplicationId) && item[ClientSideApplicationId] != null && item[ClientSideApplicationId].ToString().Equals(FeatureId_Web_ModernPage.ToString(), StringComparison.InvariantCultureIgnoreCase))
{
scanResult.ModernHomePage = true;
}
}
}
// Push information from root web to respective SiteScanResult object
if (!cc.Web.IsSubSite())
{
SiteScanResult siteScanData;
if (this.ScanJob.SiteScanResults.TryGetValue(this.SiteCollectionUrl, out siteScanData))
{
var clonedSiteScandata = siteScanData.Clone();
clonedSiteScandata.WebPublishingFeatureEnabled = scanResult.WebPublishingFeatureEnabled;
clonedSiteScandata.ModernPageWebFeatureDisabled = scanResult.ModernPageWebFeatureDisabled;
clonedSiteScandata.ModernPageFeatureWasEnabledBySPO = scanResult.ModernPageFeatureWasEnabledBySPO;
clonedSiteScandata.ModernListWebBlockingFeatureEnabled = scanResult.ModernListWebBlockingFeatureEnabled;
clonedSiteScandata.ModernHomePage = scanResult.ModernHomePage;
clonedSiteScandata.MasterPage = (!String.IsNullOrEmpty(scanResult.MasterPage) || !String.IsNullOrEmpty(scanResult.CustomMasterPage));
clonedSiteScandata.AlternateCSS = !String.IsNullOrEmpty(scanResult.AlternateCSS);
clonedSiteScandata.WebUserCustomActions = scanResult.WebUserCustomActions;
if (scanResult.PublishingPagesLibraryContainsPages && clonedSiteScandata.PublishingPagesUsed == false)
{
clonedSiteScandata.PublishingPagesUsed = true;
}
if (!this.ScanJob.SiteScanResults.TryUpdate(this.SiteCollectionUrl, clonedSiteScandata, siteScanData))
{
ScanError error = new ScanError()
{
Error = $"Could not add update site scan result for {this.SiteCollectionUrl} from web scan of {this.SiteUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "WebAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
}
else
{
if (scanResult.PublishingPagesLibraryContainsPages)
{
if (this.ScanJob.SiteScanResults.TryGetValue(this.SiteCollectionUrl, out SiteScanResult siteScanData))
{
var clonedSiteScandata = siteScanData.Clone();
if (clonedSiteScandata.PublishingPagesUsed == false)
{
clonedSiteScandata.PublishingPagesUsed = true;
}
if (!this.ScanJob.SiteScanResults.TryUpdate(this.SiteCollectionUrl, clonedSiteScandata, siteScanData))
{
ScanError error = new ScanError()
{
Error = $"Could not add update site scan result for {this.SiteCollectionUrl} from web scan of {this.SiteUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "WebAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
}
}
// Persist web results
if (!this.ScanJob.WebScanResults.TryAdd(this.SiteUrl, scanResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add web scan result for {this.SiteUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "WebAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
if (Options.IncludeLists(this.ScanJob.Mode))
{
// Kickoff the list analyzer
var listAnalyzer = new ListAnalyzer(this.SiteUrl, this.SiteCollectionUrl, this.ScanJob);
listAnalyzer.Analyze(cc);
}
if (Options.IncludePage(this.ScanJob.Mode))
{
// Kickoff the page analysing
var pageAnalyzer = new PageAnalyzer(this.SiteUrl, this.SiteCollectionUrl, this.ScanJob, this.pageSearchResults);
pageAnalyzer.Analyze(cc);
}
if (Options.IncludePublishing(this.ScanJob.Mode))
{
// Kickoff publishing analysis
PublishingAnalyzer publishingAnalyzer = null;
if (this.ScanJob.SiteScanResults.TryGetValue(this.SiteCollectionUrl, out SiteScanResult siteScanData))
{
publishingAnalyzer = new PublishingAnalyzer(this.SiteUrl, this.SiteCollectionUrl, this.ScanJob, scanResult, siteScanData);
}
else
{
publishingAnalyzer = new PublishingAnalyzer(this.SiteUrl, this.SiteCollectionUrl, this.ScanJob, scanResult, null);
}
// Assign the current masterpage gallery results
publishingAnalyzer.MasterPageGalleryCustomization = this.MasterPageGalleryCustomization;
// Run the publishing analyzer
publishingAnalyzer.Analyze(cc);
// Store the masterpage gallery results
this.MasterPageGalleryCustomization = publishingAnalyzer.MasterPageGalleryCustomization;
}
if (Options.IncludeInfoPath(this.ScanJob.Mode))
{
// Kick off InfoPath analysis
var infoPathAnalyzer = new InfoPathAnalyzer(this.SiteUrl, this.SiteCollectionUrl, this.ScanJob);
infoPathAnalyzer.Analyze(cc);
}
if (Options.IncludeBlog(this.ScanJob.Mode))
{
// Kick off Blog analysis
var blogAnalyzer = new BlogAnalyzer(this.SiteUrl, this.SiteCollectionUrl, this.ScanJob);
blogAnalyzer.Analyze(cc);
}
// Place workflow as last scan as it's reloading the web.Lists with different properties. The GetListsToScan method will not reload and hence cause missing properties otherwise
if (Options.IncludeWorkflow(this.ScanJob.Mode))
{
// Kick off workflow analysis
var workflowAnalyzer = new WorkflowAnalyzer(this.SiteUrl, this.SiteCollectionUrl, this.ScanJob);
workflowAnalyzer.Analyze(cc);
}
}
finally
{
this.StopTime = DateTime.Now;
}
// return the duration of this scan
return(new TimeSpan((this.StopTime.Subtract(this.StartTime).Ticks)));
}
/// <summary>
/// Analyze the site collection
/// </summary>
/// <param name="cc">ClientContext of the site to be analyzed</param>
/// <returns>Duration of the analysis</returns>
public override TimeSpan Analyze(ClientContext cc)
{
try
{
base.Analyze(cc);
Site site = cc.Site;
site.EnsureProperties(p => p.UserCustomActions, p => p.Features, p => p.Url, p => p.GroupId);
Web web = cc.Web;
cc.Web.EnsureProperties(p => p.WebTemplate, p => p.Configuration);
// Perform specific analysis work
SiteScanResult scanResult = new SiteScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
};
// Persist web template of the root site
scanResult.WebTemplate = $"{web.WebTemplate}#{web.Configuration}";
// Get security information for this site
scanResult.Admins = web.GetAdmins();
scanResult.Owners = web.GetOwners();
scanResult.Members = web.GetMembers();
scanResult.Visitors = web.GetVisitors();
scanResult.Office365GroupId = site.GroupId;
scanResult.EveryoneClaimsGranted = web.ClaimsHaveRoleAssignment(this.ScanJob.EveryoneClaim, this.ScanJob.EveryoneExceptExternalUsersClaim);
scanResult.ModernListSiteBlockingFeatureEnabled = site.Features.Where(f => f.DefinitionId == FeatureId_Site_ModernList).Count() > 0;
scanResult.SitePublishingFeatureEnabled = site.Features.Where(f => f.DefinitionId == FeatureId_Site_Publishing).Count() > 0;
// Get site user custom actions
scanResult.SiteUserCustomActions = site.UserCustomActions.Analyze(this.SiteCollectionUrl, this.SiteUrl);
// Get site usage information
List <string> propertiesToRetrieve = new List <string>
{
"ViewsRecent",
"ViewsRecentUniqueUsers",
"ViewsLifeTime",
"ViewsLifeTimeUniqueUsers"
};
var results = this.ScanJob.Search(cc.Web, $"path:{this.SiteCollectionUrl} AND contentclass=STS_Site", propertiesToRetrieve);
if (results != null && results.Count == 1)
{
scanResult.ViewsRecent = results[0]["ViewsRecent"].ToInt32();
scanResult.ViewsRecentUniqueUsers = results[0]["ViewsRecentUniqueUsers"].ToInt32();
scanResult.ViewsLifeTime = results[0]["ViewsLifeTime"].ToInt32();
scanResult.ViewsLifeTimeUniqueUsers = results[0]["ViewsLifeTimeUniqueUsers"].ToInt32();
}
try
{
// Get tenant information
var siteInformation = this.ScanJob.SPOTenant.GetSitePropertiesByUrl(this.SiteCollectionUrl, true);
this.ScanJob.SPOTenant.Context.Load(siteInformation);
this.ScanJob.SPOTenant.Context.ExecuteQueryRetry();
if (!siteInformation.ServerObjectIsNull())
{
scanResult.SharingCapabilities = siteInformation.SharingCapability.ToString();
}
}
// Eat all exceptions for now
// TODO move to single loop after scanning has been done - post processing
catch { }
if (this.ScanJob.Mode == Mode.Full)
{
// Use search to retrieve all view information for the indexed webpart/wiki/clientside pages in this site collection
// Need to use search inside this site collection?
List <string> propertiesToRetrieveForPage = new List <string>
{
"OriginalPath",
"ViewsRecent",
"ViewsRecentUniqueUsers",
"ViewsLifeTime",
"ViewsLifeTimeUniqueUsers"
};
this.PageSearchResults = this.ScanJob.Search(cc.Web, $"path:{this.SiteCollectionUrl} AND fileextension=aspx AND (contentclass=STS_ListItem_WebPageLibrary OR contentclass=STS_Site OR contentclass=STS_Web)", propertiesToRetrieveForPage);
}
if (!this.ScanJob.SiteScanResults.TryAdd(this.SiteCollectionUrl, scanResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add site scan result for {this.SiteUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "SiteAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
finally
{
this.StopTime = DateTime.Now;
}
// return the duration of this scan
return(new TimeSpan((this.StopTime.Subtract(this.StartTime).Ticks)));
}
public override TimeSpan Analyze(ClientContext cc)
{
try
{
base.Analyze(cc);
// Only scan when it's a valid publishing portal
var pageCount = ContinueScanning(cc);
if (pageCount > 0 || pageCount == -1)
{
try
{
PublishingWebScanResult scanResult = new PublishingWebScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
WebRelativeUrl = this.SiteUrl.Replace(this.SiteCollectionUrl, ""),
WebTemplate = this.webScanResult.WebTemplate,
BrokenPermissionInheritance = this.webScanResult.BrokenPermissionInheritance,
PageCount = pageCount == -1 ? 0 : pageCount,
SiteMasterPage = this.webScanResult.CustomMasterPage,
SystemMasterPage = this.webScanResult.MasterPage,
AlternateCSS = this.webScanResult.AlternateCSS,
Admins = this.siteScanResult.Admins,
Owners = this.webScanResult.Owners,
UserCustomActions = new List <UserCustomActionResult>()
};
// User custom actions will play a role in complexity calculation
if (this.siteScanResult.SiteUserCustomActions != null && this.siteScanResult.SiteUserCustomActions.Count > 0)
{
scanResult.UserCustomActions.AddRange(this.siteScanResult.SiteUserCustomActions);
}
if (this.webScanResult.WebUserCustomActions != null && this.webScanResult.WebUserCustomActions.Count > 0)
{
scanResult.UserCustomActions.AddRange(this.webScanResult.WebUserCustomActions);
}
Web web = cc.Web;
// Load additional web properties
web.EnsureProperties(p => p.Language);
scanResult.Language = web.Language;
// PageLayouts handling
var availablePageLayouts = web.GetPropertyBagValueString(AvailablePageLayouts, "");
var defaultPageLayout = web.GetPropertyBagValueString(DefaultPageLayout, "");
if (string.IsNullOrEmpty(availablePageLayouts))
{
scanResult.PageLayoutsConfiguration = "Any";
}
else if (availablePageLayouts.Equals("__inherit", StringComparison.InvariantCultureIgnoreCase))
{
scanResult.PageLayoutsConfiguration = "Inherit from parent";
}
else
{
scanResult.PageLayoutsConfiguration = "Defined list";
// Fill the defined list
var element = XElement.Parse(availablePageLayouts);
var nodes = element.Descendants("layout");
if (nodes != null && nodes.Count() > 0)
{
string allowedPageLayouts = "";
foreach (var node in nodes)
{
allowedPageLayouts = allowedPageLayouts + node.Attribute("url").Value.Replace("_catalogs/masterpage/", "") + ",";
}
allowedPageLayouts = allowedPageLayouts.TrimEnd(new char[] { ',' });
scanResult.AllowedPageLayouts = allowedPageLayouts;
}
}
if (!string.IsNullOrEmpty(defaultPageLayout))
{
var element = XElement.Parse(defaultPageLayout);
scanResult.DefaultPageLayout = element.Attribute("url").Value.Replace("_catalogs/masterpage/", "");
}
// Navigation
var navigationSettings = web.GetNavigationSettings();
if (navigationSettings != null)
{
if (navigationSettings.GlobalNavigation.ManagedNavigation)
{
scanResult.GlobalNavigationType = "Managed";
}
else
{
scanResult.GlobalNavigationType = "Structural";
scanResult.GlobalStructuralNavigationMaxCount = navigationSettings.GlobalNavigation.MaxDynamicItems;
scanResult.GlobalStructuralNavigationShowPages = navigationSettings.GlobalNavigation.ShowPages;
scanResult.GlobalStructuralNavigationShowSiblings = navigationSettings.GlobalNavigation.ShowSiblings;
scanResult.GlobalStructuralNavigationShowSubSites = navigationSettings.GlobalNavigation.ShowSubsites;
}
if (navigationSettings.CurrentNavigation.ManagedNavigation)
{
scanResult.CurrentNavigationType = "Managed";
}
else
{
scanResult.CurrentNavigationType = "Structural";
scanResult.CurrentStructuralNavigationMaxCount = navigationSettings.CurrentNavigation.MaxDynamicItems;
scanResult.CurrentStructuralNavigationShowPages = navigationSettings.CurrentNavigation.ShowPages;
scanResult.CurrentStructuralNavigationShowSiblings = navigationSettings.CurrentNavigation.ShowSiblings;
scanResult.CurrentStructuralNavigationShowSubSites = navigationSettings.CurrentNavigation.ShowSubsites;
}
if (navigationSettings.GlobalNavigation.ManagedNavigation || navigationSettings.CurrentNavigation.ManagedNavigation)
{
scanResult.ManagedNavigationAddNewPages = navigationSettings.AddNewPagesToNavigation;
scanResult.ManagedNavigationCreateFriendlyUrls = navigationSettings.CreateFriendlyUrlsForNewPages;
// get information about the managed nav term set configuration
var managedNavXml = web.GetPropertyBagValueString(WebNavigationSettings, "");
if (!string.IsNullOrEmpty(managedNavXml))
{
var managedNavSettings = XElement.Parse(managedNavXml);
IEnumerable <XElement> navNodes = managedNavSettings.XPathSelectElements("./SiteMapProviderSettings/TaxonomySiteMapProviderSettings");
foreach (var node in navNodes)
{
if (node.Attribute("Name").Value.Equals("CurrentNavigationTaxonomyProvider", StringComparison.InvariantCulture))
{
if (node.Attribute("TermSetId") != null)
{
scanResult.CurrentManagedNavigationTermSetId = node.Attribute("TermSetId").Value;
}
else if (node.Attribute("UseParentSiteMap") != null)
{
scanResult.CurrentManagedNavigationTermSetId = "Inherit from parent";
}
}
else if (node.Attribute("Name").Value.Equals("GlobalNavigationTaxonomyProvider", StringComparison.InvariantCulture))
{
if (node.Attribute("TermSetId") != null)
{
scanResult.GlobalManagedNavigationTermSetId = node.Attribute("TermSetId").Value;
}
else if (node.Attribute("UseParentSiteMap") != null)
{
scanResult.GlobalManagedNavigationTermSetId = "Inherit from parent";
}
}
}
}
}
}
// Pages library
var pagesLibrary = web.GetListsToScan().Where(p => p.BaseTemplate == 850).FirstOrDefault();
if (pagesLibrary != null)
{
pagesLibrary.EnsureProperties(p => p.EnableModeration, p => p.EnableVersioning, p => p.EnableMinorVersions, p => p.EventReceivers, p => p.Fields, p => p.DefaultContentApprovalWorkflowId);
scanResult.LibraryEnableModeration = pagesLibrary.EnableModeration;
scanResult.LibraryEnableVersioning = pagesLibrary.EnableVersioning;
scanResult.LibraryEnableMinorVersions = pagesLibrary.EnableMinorVersions;
scanResult.LibraryItemScheduling = pagesLibrary.ItemSchedulingEnabled();
scanResult.LibraryApprovalWorkflowDefined = pagesLibrary.DefaultContentApprovalWorkflowId != Guid.Empty;
}
// Variations
if (scanResult.Level == 0)
{
var variationLabels = cc.GetVariationLabels();
string labels = "";
string sourceLabel = "";
foreach (var label in variationLabels)
{
labels = labels + $"{label.Title} ({label.Language}),";
if (label.IsSource)
{
sourceLabel = label.Title;
}
}
scanResult.VariationLabels = labels.TrimEnd(new char[] { ',' });;
scanResult.VariationSourceLabel = sourceLabel;
}
// Scan pages inside the pages library
if (pagesLibrary != null && Options.IncludePublishingWithPages(this.ScanJob.Mode))
{
CamlQuery query = new CamlQuery
{
ViewXml = CAMLQueryByExtension,
};
var pages = pagesLibrary.GetItems(query);
// Load additional page related information
IEnumerable <ListItem> enumerable = web.Context.LoadQuery(pages.IncludeWithDefaultProperties((ListItem item) => item.ContentType));
web.Context.ExecuteQueryRetry();
if (enumerable.FirstOrDefault() != null)
{
foreach (var page in enumerable)
{
string pageUrl = null;
try
{
if (page.FieldValues.ContainsKey(FileRefField) && !String.IsNullOrEmpty(page[FileRefField].ToString()))
{
pageUrl = page[FileRefField].ToString();
}
else
{
//skip page
continue;
}
// Basic information about the page
PublishingPageScanResult pageScanResult = new PublishingPageScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
WebRelativeUrl = scanResult.WebRelativeUrl,
PageRelativeUrl = scanResult.WebRelativeUrl.Length > 0 ? pageUrl.Replace(scanResult.WebRelativeUrl, "") : pageUrl,
};
// Page name
if (page.FieldValues.ContainsKey(FileLeafRefField) && !String.IsNullOrEmpty(page[FileLeafRefField].ToString()))
{
pageScanResult.PageName = page[FileLeafRefField].ToString();
}
// Get page change information
pageScanResult.ModifiedAt = page.LastModifiedDateTime();
if (!this.ScanJob.SkipUserInformation)
{
pageScanResult.ModifiedBy = page.LastModifiedBy();
}
// Page layout
pageScanResult.PageLayout = page.PageLayout();
pageScanResult.PageLayoutFile = page.PageLayoutFile().Replace(pageScanResult.SiteColUrl, "").Replace("/_catalogs/masterpage/", "");
// Customization status
if (this.MasterPageGalleryCustomization == null)
{
this.MasterPageGalleryCustomization = new Dictionary <string, CustomizedPageStatus>();
}
// Load the file to check the customization status, only do this if the file was not loaded before for this site collection
Uri uri = new Uri(page.PageLayoutFile());
var url = page.PageLayoutFile().Replace($"{uri.Scheme}://{uri.DnsSafeHost}".ToLower(), "");
if (!this.MasterPageGalleryCustomization.ContainsKey(url))
{
try
{
var publishingPageLayout = cc.Site.RootWeb.GetFileByServerRelativeUrl(url);
cc.Load(publishingPageLayout);
cc.ExecuteQueryRetry();
this.MasterPageGalleryCustomization.Add(url, publishingPageLayout.CustomizedPageStatus);
}
catch (Exception ex)
{
// eat potential exceptions
}
}
// store the page layout customization status
if (this.MasterPageGalleryCustomization.TryGetValue(url, out CustomizedPageStatus pageStatus))
{
if (pageStatus == CustomizedPageStatus.Uncustomized)
{
pageScanResult.PageLayoutWasCustomized = false;
}
else
{
pageScanResult.PageLayoutWasCustomized = true;
}
}
else
{
// If the file was not loaded for some reason then assume it was customized
pageScanResult.PageLayoutWasCustomized = true;
}
// Page audiences
var audiences = page.Audiences();
if (audiences != null)
{
pageScanResult.GlobalAudiences = audiences.GlobalAudiences;
pageScanResult.SecurityGroupAudiences = audiences.SecurityGroups;
pageScanResult.SharePointGroupAudiences = audiences.SharePointGroups;
}
// Contenttype
pageScanResult.ContentType = page.ContentType.Name;
pageScanResult.ContentTypeId = page.ContentType.Id.StringValue;
// Get page web parts
var pageAnalysis = page.WebParts(this.ScanJob.PageTransformation);
if (pageAnalysis != null)
{
pageScanResult.WebParts = pageAnalysis.Item2;
}
// Persist publishing page scan results
if (!this.ScanJob.PublishingPageScanResults.TryAdd(pageUrl, pageScanResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add publishing page scan result for {pageScanResult.PageRelativeUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "PublishingAnalyzer",
Field2 = pageScanResult.PageRelativeUrl,
};
this.ScanJob.ScanErrors.Push(error);
}
}
catch (Exception ex)
{
ScanError error = new ScanError()
{
Error = ex.Message,
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "MainPublishingPageAnalyzerLoop",
Field2 = ex.StackTrace,
Field3 = pageUrl
};
this.ScanJob.ScanErrors.Push(error);
Console.WriteLine("Error for page {1}: {0}", ex.Message, pageUrl);
}
}
}
}
// Persist publishing scan results
if (!this.ScanJob.PublishingWebScanResults.TryAdd(this.SiteUrl, scanResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add publishing scan result for {this.SiteUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "PublishingAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
catch (Exception ex)
{
ScanError error = new ScanError()
{
Error = ex.Message,
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "MainPublishingAnalyzerLoop",
Field2 = ex.StackTrace,
};
this.ScanJob.ScanErrors.Push(error);
Console.WriteLine("Error for web {1}: {0}", ex.Message, this.SiteUrl);
}
}
}
finally
{
this.StopTime = DateTime.Now;
}
// return the duration of this scan
return(new TimeSpan((this.StopTime.Subtract(this.StartTime).Ticks)));
}
/// <summary>
/// Analyze the site collection
/// </summary>
/// <param name="cc">ClientContext of the site to be analyzed</param>
/// <returns>Duration of the analysis</returns>
public override TimeSpan Analyze(ClientContext cc)
{
try
{
base.Analyze(cc);
Site site = cc.Site;
site.EnsureProperties(p => p.UserCustomActions, p => p.Features, p => p.Url, p => p.GroupId, p => p.Id);
Web web = cc.Web;
cc.Web.EnsureProperties(p => p.WebTemplate, p => p.Configuration);
if (cc.Web.WebTemplate.Equals("TEAMCHANNEL", StringComparison.InvariantCultureIgnoreCase))
{
return(new TimeSpan((this.StopTime.Subtract(this.StartTime).Ticks)));
}
SiteScanResult scanResult = new SiteScanResult()
{
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
SiteId = site.Id.ToString(),
};
// Perform specific analysis work
// Persist web template of the root site
scanResult.WebTemplate = $"{web.WebTemplate}#{web.Configuration}";
// Is this the root site collection of the tenant
scanResult.IsRootSite = new Uri(this.SiteCollectionUrl).PathAndQuery.Equals("/");
// Is this site group connected
scanResult.Office365GroupId = site.GroupId;
if (site.GroupId != Guid.Empty)
{
if (this.ScanJob.TeamifiedSiteCollectionsLoaded)
{
if (this.ScanJob.TeamifiedSiteCollections.Contains(site.GroupId))
{
scanResult.HasTeamsTeam = true;
}
else
{
scanResult.HasTeamsTeam = false;
}
}
else
{
// we did not have the needed permissions to load the groups, hence leave the nullable bool null
}
}
else
{
// We're sure there's no team as there's no group
scanResult.HasTeamsTeam = false;
}
// Search for site scoped search center url
if (web.AllProperties.FieldValues.ContainsKey("SRCH_ENH_FTR_URL_SITE"))
{
scanResult.SearchCenterUrl = web.AllProperties.FieldValues["SRCH_ENH_FTR_URL_SITE"] as string;
}
else if (web.AllProperties.FieldValues.ContainsKey("SRCH_ENH_FTR_URL_WEB"))
{
scanResult.SearchCenterUrl = web.AllProperties.FieldValues["SRCH_ENH_FTR_URL_WEB"] as string;
}
// If no search site override check for a search results page override
if (string.IsNullOrEmpty(scanResult.SearchCenterUrl))
{
CheckForCustomSearchExperience(web, scanResult, "SRCH_SB_SET_SITE");
if (string.IsNullOrEmpty(scanResult.SearchCenterUrl))
{
CheckForCustomSearchExperience(web, scanResult, "SRCH_SB_SET_WEB");
}
}
// Get security information for this site
if (!this.ScanJob.SkipUserInformation)
{
scanResult.Admins = web.GetAdmins();
scanResult.Owners = web.GetOwners();
scanResult.Members = web.GetMembers();
scanResult.Visitors = web.GetVisitors();
scanResult.EveryoneClaimsGranted = web.ClaimsHaveRoleAssignment(this.ScanJob.EveryoneClaim, this.ScanJob.EveryoneExceptExternalUsersClaim);
}
scanResult.ModernListSiteBlockingFeatureEnabled = site.Features.Where(f => f.DefinitionId == FeatureId_Site_ModernList).Count() > 0;
scanResult.SitePublishingFeatureEnabled = site.Features.Where(f => f.DefinitionId == FeatureId_Site_Publishing).Count() > 0;
// Get site user custom actions
scanResult.SiteUserCustomActions = site.UserCustomActions.Analyze(this.SiteCollectionUrl, this.SiteUrl);
if (!this.ScanJob.SkipUsageInformation)
{
// Get site usage information
List <string> propertiesToRetrieve = new List <string>
{
"ViewsRecent",
"ViewsRecentUniqueUsers",
"ViewsLifeTime",
"ViewsLifeTimeUniqueUsers"
};
var results = this.ScanJob.Search(cc.Web, $"path:{this.SiteCollectionUrl} AND contentclass=STS_Site", propertiesToRetrieve);
if (results != null && results.Count == 1)
{
scanResult.ViewsRecent = results[0]["ViewsRecent"].ToInt32();
scanResult.ViewsRecentUniqueUsers = results[0]["ViewsRecentUniqueUsers"].ToInt32();
scanResult.ViewsLifeTime = results[0]["ViewsLifeTime"].ToInt32();
scanResult.ViewsLifeTimeUniqueUsers = results[0]["ViewsLifeTimeUniqueUsers"].ToInt32();
}
}
if (!this.ScanJob.AppOnlyHasFullControl)
{
var siteInfo = this.ScanJob.AppOnlyManager.SiteInformation.Where(p => p.SiteUrl.Equals(this.SiteCollectionUrl, StringComparison.InvariantCultureIgnoreCase)).FirstOrDefault();
if (siteInfo != null && siteInfo.ExternalSharing.HasValue)
{
if (!siteInfo.ExternalSharing.Value)
{
scanResult.SharingCapabilities = "Disabled";
}
else
{
if (siteInfo.AllowGuestUserSignIn.HasValue && !siteInfo.AllowGuestUserSignIn.Value)
{
scanResult.SharingCapabilities = "ExternalUserAndGuestSharing";
}
else
{
scanResult.SharingCapabilities = "ExternalUserSharingOnly";
}
}
}
else
{
scanResult.SharingCapabilities = "Unknown";
}
}
else
{
try
{
// Get tenant information
var siteInformation = this.ScanJob.SPOTenant.GetSitePropertiesByUrl(this.SiteCollectionUrl, true);
this.ScanJob.SPOTenant.Context.Load(siteInformation);
this.ScanJob.SPOTenant.Context.ExecuteQueryRetry();
if (!siteInformation.ServerObjectIsNull())
{
scanResult.SharingCapabilities = siteInformation.SharingCapability.ToString();
}
}
// Eat all exceptions for now
// TODO move to single loop after scanning has been done - post processing
catch { }
}
if (Options.IncludePage(this.ScanJob.Mode))
{
// Use search to retrieve all view information for the indexed webpart/wiki/clientside pages in this site collection
// Need to use search inside this site collection?
List <string> propertiesToRetrieveForPage = new List <string>
{
"OriginalPath",
"ViewsRecent",
"ViewsRecentUniqueUsers",
"ViewsLifeTime",
"ViewsLifeTimeUniqueUsers"
};
if (new Uri(this.SiteCollectionUrl).PathAndQuery == "/")
{
this.PageSearchResults = this.ScanJob.Search(cc.Web, $"path={this.SiteCollectionUrl} AND fileextension=aspx AND (contentclass=STS_ListItem_WebPageLibrary OR contentclass=STS_Site OR contentclass=STS_Web)", propertiesToRetrieveForPage);
}
else
{
this.PageSearchResults = this.ScanJob.Search(cc.Web, $"path:{this.SiteCollectionUrl} AND fileextension=aspx AND (contentclass=STS_ListItem_WebPageLibrary OR contentclass=STS_Site OR contentclass=STS_Web)", propertiesToRetrieveForPage);
}
}
if (!this.ScanJob.SiteScanResults.TryAdd(this.SiteCollectionUrl, scanResult))
{
ScanError error = new ScanError()
{
Error = $"Could not add site scan result for {this.SiteUrl}",
SiteColUrl = this.SiteCollectionUrl,
SiteURL = this.SiteUrl,
Field1 = "SiteAnalyzer",
};
this.ScanJob.ScanErrors.Push(error);
}
}
finally
{
this.StopTime = DateTime.Now;
}
// return the duration of this scan
return(new TimeSpan((this.StopTime.Subtract(this.StartTime).Ticks)));
}