private ListInstance ProcessListItems(Web web,
List siteList,
ListInstance listInstance,
ProvisioningTemplateCreationInformation creationInfo,
Model.Configuration.Lists.Lists.ExtractListsListsConfiguration extractionConfig,
Model.Configuration.Lists.Lists.ExtractListsQueryConfiguration queryConfig,
Uri baseUri,
ListItemCollection items,
PnPMonitoredScope scope)
{
if (!string.IsNullOrEmpty(extractionConfig.KeyColumn))
{
listInstance.DataRows.KeyColumn = extractionConfig.KeyColumn;
listInstance.DataRows.UpdateBehavior = extractionConfig.UpdateBehavior;
}
var itemCount = 1;
foreach (var item in items)
{
WriteSubProgress("List", listInstance.Title, itemCount, items.Count);
var dataRow = ProcessDataRow(web, siteList, item, listInstance, extractionConfig, queryConfig, baseUri, creationInfo, scope);
listInstance.DataRows.Add(dataRow);
itemCount++;
}
return(listInstance);
}
private ProvisioningTemplate ProcessLibraryItems(Web web,
List siteList,
ProvisioningTemplate template,
ListInstance listInstance,
Model.Configuration.Lists.Lists.ExtractListsListsConfiguration extractionConfig,
Model.Configuration.Lists.Lists.ExtractListsQueryConfiguration queryConfig,
ProvisioningTemplateCreationInformation creationInfo,
PnPMonitoredScope scope,
ListItemCollection items,
Uri baseUri,
string defaultContentTypeId)
{
var itemCount = 1;
foreach (var item in items)
{
switch (item.FileSystemObjectType)
{
case FileSystemObjectType.File:
{
//PnP:File
ProcessDocumentRow(web, siteList, baseUri, item, listInstance, template, creationInfo, scope, itemCount, items.Count, defaultContentTypeId);
break;
}
case FileSystemObjectType.Folder:
{
//PnP:Folder
ProcessFolderRow(web, item, siteList, listInstance, queryConfig, template, scope);
break;
}
default:
{
//PnP:DataRow
ProcessDataRow(web, siteList, item, listInstance, extractionConfig, queryConfig, baseUri, creationInfo, scope);
break;
}
}
itemCount++;
}
return(template);
}
private Model.DataRow ProcessDataRow(Web web, List siteList, ListItem item, ListInstance listInstance, Model.Configuration.Lists.Lists.ExtractListsListsConfiguration extractionConfig, Model.Configuration.Lists.Lists.ExtractListsQueryConfiguration queryConfig, Uri baseUri, ProvisioningTemplateCreationInformation creationInfo, PnPMonitoredScope scope)
{
var dataRow = new Model.DataRow();
var filteredFieldValues = item.FieldValues.ToList();
if (queryConfig != null && queryConfig.ViewFields != null && queryConfig.ViewFields.Count > 0)
{
filteredFieldValues = item.FieldValues.Where(f => queryConfig.ViewFields.Contains(f.Key)).ToList();
}
if (queryConfig != null && queryConfig.IncludeAttachments)
{
filteredFieldValues = filteredFieldValues.Where(f => !f.Key.Equals("Attachments", StringComparison.InvariantCultureIgnoreCase)).ToList();
}
foreach (var fieldValue in filteredFieldValues)
{
var value = item.FieldValuesAsText[fieldValue.Key];
var skip = extractionConfig.SkipEmptyFields && string.IsNullOrEmpty(value);
if (!skip)
{
dataRow.Values.Add(fieldValue.Key, TokenizeValue(web, siteList.Fields.FirstOrDefault(f => f.InternalName == fieldValue.Key).TypeAsString, fieldValue, value));
}
}
if (queryConfig != null && queryConfig.IncludeAttachments && siteList.EnableAttachments && (bool)item["Attachments"])
{
#if !SP2013 && !SP2016
item.Context.Load(item, i => i.AttachmentFiles.Include(a => a.FileName, a => a.FileNameAsPath, a => a.ServerRelativePath));
#else
item.Context.Load(item, i => i.AttachmentFiles.Include(a => a.FileName, a => a.FileName, a => a.ServerRelativeUrl));
#endif
item.Context.ExecuteQueryRetry();
foreach (var attachmentFile in item.AttachmentFiles)
{
#if !SP2013 && !SP2016
var fullUri = new Uri(baseUri, attachmentFile.ServerRelativePath.DecodedUrl);
#else
var fullUri = new Uri(baseUri, attachmentFile.ServerRelativeUrl);
#endif
var folderPath = HttpUtility.UrlDecode(fullUri.Segments.Take(fullUri.Segments.Count() - 1).ToArray().Aggregate((i, x) => i + x).TrimEnd('/'));
var targetFolder = $"ListData/SITE_{web.Id.ToString("N")}/LIST_{siteList.Id.ToString("N")}/Attachments/{item.Id}";
dataRow.Attachments.Add(new Model.SharePoint.InformationArchitecture.DataRowAttachment()
{
#if !SP2013 && !SP2016
Name = attachmentFile.FileNameAsPath.DecodedUrl,
Src = $"{targetFolder}/{attachmentFile.FileNameAsPath.DecodedUrl}"
#else
Name = attachmentFile.FileName,
Src = $"{targetFolder}/{attachmentFile.FileName}"
#endif
});
private void ProcessFolderRow(Web web, ListItem listItem, List siteList, ListInstance listInstance, Model.Configuration.Lists.Lists.ExtractListsQueryConfiguration queryConfig, ProvisioningTemplate template, PnPMonitoredScope scope)
{
listItem.EnsureProperties(it => it.ParentList.RootFolder.ServerRelativeUrl);
string serverRelativeListUrl = listItem.ParentList.RootFolder.ServerRelativeUrl;
string folderPath = listItem.FieldValuesAsText["FileRef"].Substring(serverRelativeListUrl.Length).TrimStart(new char[] { '/' });
if (!string.IsNullOrWhiteSpace(folderPath))
{
//listItem.EnsureProperties(it => it.Folder.UniqueId);
string[] folderSegments = folderPath.Split('/');
Model.Folder pnpFolder = null;
for (int i = 0; i < folderSegments.Length; i++)
{
if (i == 0)
{
pnpFolder = listInstance.Folders.FirstOrDefault(f => f.Name.Equals(folderSegments[i], StringComparison.CurrentCultureIgnoreCase));
if (pnpFolder == null)
{
string pathToCurrentFolder = string.Format("{0}/{1}", serverRelativeListUrl, string.Join("/", folderSegments.Take(i + 1)));
pnpFolder = ExtractFolderSettings(web, siteList, pathToCurrentFolder, scope, queryConfig);
listInstance.Folders.Add(pnpFolder);
}
}
else
{
var childFolder = pnpFolder.Folders.FirstOrDefault(f => f.Name.Equals(folderSegments[i], StringComparison.CurrentCultureIgnoreCase));
if (childFolder == null)
{
string pathToCurrentFolder = string.Format("{0}/{1}", serverRelativeListUrl, string.Join("/", folderSegments.Take(i + 1)));
childFolder = ExtractFolderSettings(web, siteList, pathToCurrentFolder, scope, queryConfig);
pnpFolder.Folders.Add(childFolder);
}
pnpFolder = childFolder;
}
}
}
}
public Model.Folder ExtractFolderSettings(Web web, List siteList, string serverRelativePathToFolder, PnPMonitoredScope scope, Model.Configuration.Lists.Lists.ExtractListsQueryConfiguration queryConfig)
{
Model.Folder pnpFolder = null;
try
{
Microsoft.SharePoint.Client.Folder spFolder = web.GetFolderByServerRelativeUrl(serverRelativePathToFolder);
web.Context.Load(spFolder,
f => f.Name,
f => f.ServerRelativeUrl,
f => f.Properties,
f => f.ListItemAllFields,
f => f.ListItemAllFields.RoleAssignments,
f => f.ListItemAllFields.RoleAssignments.Include(r => r.Member, r => r.RoleDefinitionBindings),
f => f.ListItemAllFields.HasUniqueRoleAssignments,
f => f.ListItemAllFields.ParentList,
f => f.ListItemAllFields.ContentType.StringId);
web.Context.Load(web,
w => w.AssociatedOwnerGroup,
w => w.AssociatedMemberGroup,
w => w.AssociatedVisitorGroup,
w => w.Title,
w => w.Url,
w => w.RoleDefinitions.Include(r => r.RoleTypeKind, r => r.Name),
w => w.ContentTypes.Include(c => c.Id, c => c.Name, c => c.StringId));
web.Context.ExecuteQueryRetry();
pnpFolder = new Model.Folder(spFolder.Name);
//export PnPFolder Properties
if (spFolder.Properties.FieldValues.Any())
{
foreach (var propKey in spFolder.Properties.FieldValues.Keys.Where(k => !k.StartsWith("vti_") && !k.StartsWith("docset_")))
{
pnpFolder.PropertyBagEntries.Add(new PropertyBagEntry()
{
Key = propKey, Value = spFolder.Properties.FieldValues[propKey].ToString()
});
}
}
//export PnPFolder FieldValues
if (spFolder.ListItemAllFields.FieldValues.Any())
{
var list = spFolder.ListItemAllFields.ParentList;
var fields = list.Fields;
web.Context.Load(fields, fs => fs.IncludeWithDefaultProperties(f => f.TypeAsString, f => f.InternalName, f => f.Title));
web.Context.ExecuteQueryRetry();
var fieldValues = spFolder.ListItemAllFields.FieldValues;
var fieldValuesAsText = spFolder.ListItemAllFields.EnsureProperty(li => li.FieldValuesAsText).FieldValues;
#region //**** get correct Content Type
string ctId = string.Empty;
foreach (var ct in web.ContentTypes.OrderByDescending(c => c.StringId.Length))
{
if (spFolder.ListItemAllFields.ContentType.StringId.StartsWith(ct.StringId))
{
pnpFolder.ContentTypeID = ct.StringId;
break;
}
}
#endregion //**** get correct Content Type
var filteredFieldValues = fieldValues.ToList();
if (queryConfig != null && queryConfig.ViewFields != null && queryConfig.ViewFields.Count > 0)
{
filteredFieldValues = fieldValues.Where(f => queryConfig.ViewFields.Contains(f.Key)).ToList();
}
foreach (var fieldValue in filteredFieldValues)
{
if (fieldValue.Value != null && !string.IsNullOrEmpty(fieldValue.Value.ToString()))
{
var field = siteList.Fields.FirstOrDefault(fs => fs.InternalName == fieldValue.Key);
string value = string.Empty;
//ignore read only fields
if (!field.ReadOnlyField || WriteableReadOnlyField.Contains(field.InternalName.ToLower()))
{
value = TokenizeValue(web, field.TypeAsString, fieldValue, fieldValuesAsText[field.InternalName]);
}
if (fieldValue.Key.Equals("ContentTypeId", StringComparison.InvariantCultureIgnoreCase) || fieldValue.Key.Equals("Attachments", StringComparison.InvariantCultureIgnoreCase))
{
value = null; //ignore here since already in dataRow
}
if (fieldValue.Key.Equals("HTML_x0020_File_x0020_Type", StringComparison.CurrentCultureIgnoreCase) &&
fieldValuesAsText["HTML_x0020_File_x0020_Type"] == "OneNote.Notebook")
{
pnpFolder.Properties.Add("File_x0020_Type", "OneNote.Notebook");
pnpFolder.Properties.Add(fieldValue.Key, "OneNote.Notebook");
value = null;
}
// We process real values only
if (!string.IsNullOrWhiteSpace(value) && value != "[]")
{
pnpFolder.Properties.Add(fieldValue.Key, value);
}
}
}
}
}
catch (Exception ex)
{
scope.LogError(ex, "Extract of Folder {0} failed", serverRelativePathToFolder);
}
return(pnpFolder);
}
private ListItemCollectionPosition RetrieveItems(Web web, ProvisioningTemplate template, ProvisioningTemplateCreationInformation creationInfo, PnPMonitoredScope scope, List siteList, Model.Configuration.Lists.Lists.ExtractListsListsConfiguration extractionConfiguration, CamlQuery camlQuery, Model.Configuration.Lists.Lists.ExtractListsQueryConfiguration queryConfig, ListInstance listInstance, string defaultContentTypeId)
{
var items = siteList.GetItems(camlQuery);
siteList.Context.Load(items, i => i.IncludeWithDefaultProperties(li => li.FieldValuesAsText), i => i.ListItemCollectionPosition);
if (queryConfig != null && queryConfig.ViewFields != null && queryConfig.ViewFields.Count > 0)
{
foreach (var viewField in queryConfig.ViewFields)
{
if (siteList.Fields.FirstOrDefault(f => f.InternalName == viewField) != null)
{
siteList.Context.Load(items, i => i.Include(li => li[viewField]));
}
}
}
siteList.Context.ExecuteQueryRetry();
var baseUri = new Uri(web.Url);
if (siteList.BaseType == BaseType.DocumentLibrary)
{
ProcessLibraryItems(web, siteList, template, listInstance, extractionConfiguration, queryConfig, creationInfo, scope, items, baseUri, defaultContentTypeId);
}
else
{
ProcessListItems(web, siteList, listInstance, creationInfo, extractionConfiguration, queryConfig, baseUri, items, scope);
}
return(items.ListItemCollectionPosition);
}
public override ProvisioningTemplate ExtractObjects(Web web, ProvisioningTemplate template, ProvisioningTemplateCreationInformation creationInfo)
{
using (var scope = new PnPMonitoredScope(this.Name))
{
var lists = web.Lists;
web.EnsureProperties(w => w.ServerRelativeUrl, w => w.Url, w => w.Id);
web.Context.Load(lists,
lc => lc.IncludeWithDefaultProperties(
l => l.RootFolder.ServerRelativeUrl,
l => l.EnableAttachments,
l => l.ContentTypes,
l => l.Fields.IncludeWithDefaultProperties(
f => f.Id,
f => f.Title,
f => f.TypeAsString,
f => f.ReadOnlyField,
f => f.Hidden,
f => f.InternalName,
f => f.DefaultValue,
f => f.Required))
);
web.Context.ExecuteQueryRetry();
var allLists = new List <List>();
var listsToProcess = lists.AsEnumerable().Where(l => l.Hidden == false || l.Hidden == creationInfo.IncludeHiddenLists).ToArray();
foreach (var siteList in listsToProcess)
{
if (ShouldNotExtractList(creationInfo, siteList))
{
continue;
}
var extractionConfig = creationInfo.ExtractConfiguration.Lists.Lists.FirstOrDefault(e => e.Title.Equals(siteList.Title) || siteList.RootFolder.ServerRelativeUrl.EndsWith(e.Title, StringComparison.InvariantCultureIgnoreCase));
CamlQuery camlQuery = CamlQuery.CreateAllItemsQuery();
Model.Configuration.Lists.Lists.ExtractListsQueryConfiguration queryConfig = null;
if (extractionConfig.Query != null)
{
queryConfig = extractionConfig.Query;
camlQuery = new CamlQuery();
if (string.IsNullOrEmpty(queryConfig.CamlQuery))
{
queryConfig.CamlQuery = "<Order><FieldRef Name='ID' /></Order>";
}
string viewXml = $"<View Scope=\"RecursiveAll\"><Query>{queryConfig.CamlQuery}</Query>";
if (queryConfig.IncludeAttachments && siteList.EnableAttachments)
{
if (queryConfig.ViewFields == null)
{
queryConfig.ViewFields = new List <string>();
}
else if (!queryConfig.ViewFields.Contains("Attachments"))
{
queryConfig.ViewFields.Add("Attachments");
}
}
if (queryConfig.ViewFields != null && queryConfig.ViewFields.Count > 0)
{
viewXml += "<ViewFields>";
foreach (var viewField in queryConfig.ViewFields)
{
viewXml += $"<FieldRef Name='{viewField}' />";
}
viewXml += "</ViewFields>";
}
if (queryConfig.RowLimit > 0 || queryConfig.PageSize > 0)
{
viewXml += $"<RowLimit{(queryConfig.PageSize > 0 ? " Paged=\"TRUE\"" : "")}>{(queryConfig.PageSize > 0 ? queryConfig.PageSize : queryConfig.RowLimit)}</RowLimit>";
}
viewXml += "</View>";
camlQuery.ViewXml = viewXml;
}
var listInstance = template.Lists.FirstOrDefault(l => siteList.RootFolder.ServerRelativeUrl.Equals(UrlUtility.Combine(web.ServerRelativeUrl, l.Url)));
if (listInstance != null)
{
do
{
camlQuery.ListItemCollectionPosition = RetrieveItems(web, template, creationInfo, scope, siteList, extractionConfig, camlQuery, queryConfig, listInstance, siteList.ContentTypes[0].Id.StringValue);
} while (camlQuery.ListItemCollectionPosition != null);
}
}
}
return(template);
}
