/// <summary> /// Execute the cmdlet /// </summary> public override void ExecuteCmdlet() { base.ExecuteCmdlet(); // Initialize logging instance with Powershell logger ITraceLogger logger = new DefaultUsageLogger(LogVerbose, LogWarning, LogError); // Construct the model var SiteComponents = new SiteProvisionerModel() { Lists = new List <SPListDefinition>() }; var ctx = this.ClientContext; var contextWeb = this.ClientContext.Web; ctx.Load(contextWeb, ctxw => ctxw.ServerRelativeUrl, wctx => wctx.Url, ctxw => ctxw.Id); ctx.ExecuteQueryRetry(); var _targetList = Identity.GetList(contextWeb, lctx => lctx.ItemCount, lctx => lctx.EnableFolderCreation, lctx => lctx.RootFolder, lctx => lctx.RootFolder.ServerRelativeUrl, lctx => lctx.RootFolder.Folders); var webUri = new Uri(contextWeb.Url); // Will query the list to determine the last item id in the list var lastItemId = _targetList.QueryLastItemId(); logger.LogInformation("List with item count {0} has a last ID of {1}", _targetList.ItemCount, lastItemId); logger.LogInformation("List has folder creation = {0}", _targetList.EnableFolderCreation); // store the OOTB standard fields var ootbCamlFields = new List <string>() { "Title", "ID", "Author", "Editor", "Created", "Modified" }; // Skip these specific fields var skiptypes = new FieldType[] { FieldType.Invalid, FieldType.Computed, FieldType.ContentTypeId, FieldType.Invalid, FieldType.WorkflowStatus, FieldType.WorkflowEventType, FieldType.Threading, FieldType.ThreadIndex, FieldType.Recurrence, FieldType.PageSeparator, FieldType.OutcomeChoice, FieldType.CrossProjectLink, FieldType.ModStat, FieldType.Error, FieldType.MaxItems, FieldType.Attachments }; // pull a small portion of the list // wire up temporary array var sitelists = new List <SPListDefinition>() { ctx.GetListDefinition(_targetList, ExpandObjects, logger, skiptypes) }; if (sitelists.Any()) { var idx = 0; // lets add any list with NO lookups first var nolookups = sitelists.Where(sl => !sl.ListDependency.Any()); nolookups.ToList().ForEach(nolookup => { logger.LogInformation("adding list {0}", nolookup.ListName); nolookup.ProvisionOrder = idx++; SiteComponents.Lists.Add(nolookup); sitelists.Remove(nolookup); }); // order with first in stack var haslookups = sitelists.Where(sl => sl.ListDependency.Any()).OrderBy(ob => ob.ListDependency.Count()).ToList(); while (haslookups.Count() > 0) { var listPopped = haslookups.FirstOrDefault(); haslookups.Remove(listPopped); logger.LogInformation("popping list {0}", listPopped.ListName); if (listPopped.ListDependency.Any(listField => listPopped.ListName.Equals(listField, StringComparison.InvariantCultureIgnoreCase) || listPopped.InternalName.Equals(listField, StringComparison.InvariantCultureIgnoreCase))) { // the list has a dependency on itself logger.LogInformation("Adding list {0} with dependency on itself", listPopped.ListName); listPopped.ProvisionOrder = idx++; SiteComponents.Lists.Add(listPopped); } else if (listPopped.ListDependency.Any(listField => !SiteComponents.Lists.Any(sc => sc.ListName.Equals(listField, StringComparison.InvariantCultureIgnoreCase) || sc.InternalName.Equals(listField, StringComparison.InvariantCultureIgnoreCase)))) { // no list definition exists in the collection with the dependent lookup lists logger.LogWarning("List {0} depends on {1} which do not exist current collection", listPopped.ListName, string.Join(",", listPopped.ListDependency)); foreach (var dependent in listPopped.ListDependency.Where(dlist => !haslookups.Any(hl => hl.ListName == dlist))) { var sitelist = contextWeb.GetListByTitle(dependent, lctx => lctx.Id, lctx => lctx.Title, lctx => lctx.RootFolder.ServerRelativeUrl); var sitelistDefinition = ctx.GetListDefinition(contextWeb, sitelist, true, logger, skiptypes); haslookups.Add(sitelistDefinition); } haslookups.Add(listPopped); // add back to collection listPopped = null; } else { logger.LogInformation("Adding list {0} to collection", listPopped.ListName); listPopped.ProvisionOrder = idx++; SiteComponents.Lists.Add(listPopped); } } } foreach (var listDefinition in SiteComponents.Lists.OrderBy(ob => ob.ProvisionOrder)) { listDefinition.ListItems = new List <SPListItemDefinition>(); var camlFields = new List <string>(); var listTitle = listDefinition.ListName; var etlList = this.ClientContext.Web.GetListByTitle(listTitle, lctx => lctx.Id, lctx => lctx.ItemCount, lctx => lctx.EnableFolderCreation, lctx => lctx.RootFolder.ServerRelativeUrl); if (ExpandObjects) { // list fields var listFields = listDefinition.FieldDefinitions; if (listFields != null && listFields.Any()) { var filteredListFields = listFields.Where(lf => !skiptypes.Any(st => lf.FieldTypeKind == st)).ToList(); var notInCamlFields = listFields.Where(listField => !ootbCamlFields.Any(cf => cf.Equals(listField.InternalName, StringComparison.CurrentCultureIgnoreCase)) && !skiptypes.Any(st => listField.FieldTypeKind == st)); foreach (var listField in notInCamlFields) { logger.LogInformation("Processing list {0} field {1}", etlList.Title, listField.InternalName); camlFields.Add(listField.InternalName); } } } camlFields.AddRange(ootbCamlFields); var camlViewFields = CAML.ViewFields(camlFields.Select(s => CAML.FieldRef(s)).ToArray()); ListItemCollectionPosition itemPosition = null; var camlQueries = etlList.SafeCamlClauseFromThreshold(2000, CamlStatement, 0, lastItemId); foreach (var camlAndValue in camlQueries) { itemPosition = null; var camlWhereClause = (string.IsNullOrEmpty(camlAndValue) ? string.Empty : CAML.Where(camlAndValue)); var camlQuery = new CamlQuery() { ViewXml = CAML.ViewQuery( ViewScope.RecursiveAll, camlWhereClause, CAML.OrderBy(new OrderByField("ID")), camlViewFields, 500) }; try { while (true) { logger.LogWarning("CAML Query {0} at position {1}", camlWhereClause, (itemPosition == null ? string.Empty : itemPosition.PagingInfo)); camlQuery.ListItemCollectionPosition = itemPosition; var listItems = etlList.GetItems(camlQuery); etlList.Context.Load(listItems); etlList.Context.ExecuteQueryRetry(); itemPosition = listItems.ListItemCollectionPosition; foreach (var rbiItem in listItems) { var itemId = rbiItem.Id; var itemTitle = rbiItem.RetrieveListItemValue("Title"); var author = rbiItem.RetrieveListItemUserValue("Author"); var editor = rbiItem.RetrieveListItemUserValue("Editor"); logger.LogInformation("Title: {0}; Item ID: {1}", itemTitle, itemId); var newitem = new SPListItemDefinition() { Id = itemId, Title = itemTitle, Created = rbiItem.RetrieveListItemValue("Created").ToNullableDatetime(), Modified = rbiItem.RetrieveListItemValue("Modified").ToNullableDatetime() }; if (author != null) { newitem.CreatedBy = new SPPrincipalUserDefinition() { Email = author.ToUserEmailValue(), LoginName = author.ToUserValue(), Id = author.LookupId }; } if (editor != null) { newitem.ModifiedBy = new SPPrincipalUserDefinition() { Email = editor.ToUserEmailValue(), LoginName = editor.ToUserValue(), Id = editor.LookupId }; } if (ExpandObjects) { var fieldValuesToWrite = rbiItem.FieldValues.Where(rfield => !ootbCamlFields.Any(oc => rfield.Key.Equals(oc, StringComparison.CurrentCultureIgnoreCase))); foreach (var rbiField in fieldValuesToWrite) { newitem.ColumnValues.Add(new SPListItemFieldDefinition() { FieldName = rbiField.Key, FieldValue = rbiField.Value }); } } listDefinition.ListItems.Add(newitem); } if (itemPosition == null) { break; } } } catch (Exception ex) { logger.LogError(ex, "Failed to retrieve list item collection"); } } } WriteObject(SiteComponents); }
/// <summary> /// Process the request /// </summary> public override void ExecuteCmdlet() { base.ExecuteCmdlet(); // Initialize logging instance with Powershell logger ITraceLogger logger = new DefaultUsageLogger(LogVerbose, LogWarning, LogError); // SharePoint URI for XML parsing XNamespace ns = "http://schemas.microsoft.com/sharepoint/"; // Definition file to operate on SiteProvisionerModel siteDefinition = null; try { // Retreive JSON Provisioner file and deserialize it var filePath = new System.IO.FileInfo(this.ProvisionerFilePath); var filePathJSON = System.IO.File.ReadAllText(filePath.FullName); siteDefinition = JsonConvert.DeserializeObject <SiteProvisionerModel>(filePathJSON); } catch (Exception ex) { logger.LogError(ex, "Failed to parse {0} Exception {1}", ProvisionerFilePath, ex.Message); return; } if (ValidateJson) { logger.LogInformation("The file {0} is valid.", ProvisionerFilePath); return; } var provisionerChoices = siteDefinition.FieldChoices; // Load the Context var contextWeb = this.ClientContext.Web; this.ClientContext.Load(contextWeb, ctxw => ctxw.ServerRelativeUrl, ctxw => ctxw.Id, ctxw => ctxw.Fields.Include(inc => inc.InternalName, inc => inc.JSLink, inc => inc.Title, inc => inc.Id)); // All Site Columns var siteFields = this.ClientContext.LoadQuery(ClientContext.Web.AvailableFields .Include(inc => inc.InternalName, inc => inc.JSLink, inc => inc.Title, inc => inc.Id)); // pull Site Groups var groupQuery = this.ClientContext.LoadQuery(contextWeb.SiteGroups .Include(group => group.Id, group => group.Title, group => group.Description, group => group.AllowRequestToJoinLeave, group => group.AllowMembersEditMembership, group => group.AutoAcceptRequestToJoinLeave, group => group.OnlyAllowMembersViewMembership, group => group.RequestToJoinLeaveEmailSetting)); var contentTypes = this.ClientContext.LoadQuery(contextWeb.ContentTypes .Include( ict => ict.Id, ict => ict.Group, ict => ict.Description, ict => ict.Name, ict => ict.Hidden, ict => ict.JSLink, ict => ict.FieldLinks, ict => ict.Fields)); var collists = contextWeb.Lists; var lists = this.ClientContext.LoadQuery(collists .Include( linc => linc.Title, linc => linc.Id, linc => linc.Description, linc => linc.Hidden, linc => linc.OnQuickLaunch, linc => linc.BaseTemplate, linc => linc.ContentTypesEnabled, linc => linc.AllowContentTypes, linc => linc.EnableFolderCreation, linc => linc.IsApplicationList, linc => linc.IsCatalog, linc => linc.IsSiteAssetsLibrary, linc => linc.IsPrivate, linc => linc.IsSystemList, linc => linc.Views, linc => linc.Fields .Include( lft => lft.Id, lft => lft.AutoIndexed, lft => lft.CanBeDeleted, lft => lft.DefaultFormula, lft => lft.DefaultValue, lft => lft.Group, lft => lft.Description, lft => lft.EnforceUniqueValues, lft => lft.FieldTypeKind, lft => lft.Filterable, lft => lft.Hidden, lft => lft.Indexed, lft => lft.InternalName, lft => lft.JSLink, lft => lft.NoCrawl, lft => lft.ReadOnlyField, lft => lft.Required, lft => lft.SchemaXml, lft => lft.Scope, lft => lft.Title ), linc => linc.ContentTypes .Include( ict => ict.Id, ict => ict.Group, ict => ict.Description, ict => ict.Name, ict => ict.Hidden, ict => ict.JSLink, ict => ict.FieldLinks, ict => ict.Fields)).Where(w => !w.IsSystemList && !w.IsSiteAssetsLibrary)); this.ClientContext.ExecuteQueryRetry(); // creates groups contextWeb.Context.Load(contextWeb, hw => hw.CurrentUser); contextWeb.Context.ExecuteQueryRetry(); if (ProvisionGroups) { if (siteDefinition.Groups != null && siteDefinition.Groups.Any()) { logger.LogInformation("Group collection will be provisioned for {0} groups", siteDefinition.Groups.Count()); foreach (var groupDef in siteDefinition.Groups) { if (groupQuery.Any(g => g.Title.Equals(groupDef.Title, StringComparison.CurrentCultureIgnoreCase))) { var group = groupQuery.FirstOrDefault(g => g.Title.Equals(groupDef.Title, StringComparison.CurrentCultureIgnoreCase)); SiteGroups.Add(new SPGroupDefinitionModel() { Id = group.Id, Title = group.Title }); } else { var newgroup = contextWeb.GetOrCreateSiteGroups(groupDef); SiteGroups.Add(new SPGroupDefinitionModel() { Id = newgroup.Id, Title = newgroup.Title }); } } } } else { // we aren't going to examine the JSON to provision groups, pull from current SharePoint instance SiteGroups.AddRange(groupQuery.Select(gq => new SPGroupDefinitionModel() { Id = gq.Id, Title = gq.Title })); } // provision columns // Site Columns if (siteDefinition.FieldDefinitions != null && siteDefinition.FieldDefinitions.Any()) { logger.LogInformation("Field definitions will be provisioned for {0} fields", siteDefinition.FieldDefinitions.Count()); foreach (var fieldDef in siteDefinition.FieldDefinitions) { var column = contextWeb.CreateColumn(fieldDef, logger, SiteGroups, siteDefinition.FieldChoices); if (column == null) { logger.LogWarning("Failed to create column {0}.", fieldDef.InternalName); } else { SiteColumns.Add(new SPFieldDefinitionModel() { InternalName = column.InternalName, Title = column.Title, FieldGuid = column.Id }); } } } // provision content types if (siteDefinition.ContentTypes != null && siteDefinition.ContentTypes.Any()) { logger.LogInformation("Content types will be provisioned for {0} ctypes", siteDefinition.ContentTypes.Count()); foreach (var contentDef in siteDefinition.ContentTypes) { var contentTypeName = contentDef.Name; var contentTypeId = contentDef.ContentTypeId; if (!contextWeb.ContentTypeExistsByName(contentTypeName) && !contextWeb.ContentTypeExistsById(contentTypeId)) { logger.LogInformation("Provisioning content type {0}", contentTypeName); contextWeb.CreateContentType(contentTypeName, contentTypeId, (string.IsNullOrEmpty(contentDef.ContentTypeGroup) ? "CustomColumn" : contentDef.ContentTypeGroup)); } var provisionedContentType = contextWeb.GetContentTypeByName(contentTypeName, true); if (provisionedContentType != null) { logger.LogInformation("Found content type {0} and is read only {1}", contentTypeName, provisionedContentType.ReadOnly); if (!provisionedContentType.ReadOnly) { foreach (var fieldDef in contentDef.FieldLinks) { // Check if FieldLink exists in the Content Type if (!contextWeb.FieldExistsByNameInContentType(contentTypeName, fieldDef.Name)) { // Check if FieldLInk column is in the collection of FieldDefinition var siteColumn = siteDefinition.FieldDefinitions.FirstOrDefault(f => f.InternalName == fieldDef.Name); if (siteColumn != null && !contextWeb.FieldExistsByNameInContentType(contentTypeName, siteColumn.DisplayNameMasked)) { var column = this.SiteColumns.FirstOrDefault(f => f.InternalName == fieldDef.Name); if (column == null) { logger.LogWarning("Column {0} was not added to the collection", fieldDef.Name); } else { contextWeb.AddFieldToContentTypeByName(contentTypeName, column.FieldGuid, siteColumn.Required); } } } } } } } } // provision lists var listtoprocess = siteDefinition.Lists.OrderBy(w => w.ProvisionOrder).ToList(); listtoprocess.ForEach(listDef => { // Content Type var listName = listDef.ListName; var listDescription = listDef.ListDescription; // provision the list definition var siteList = contextWeb.CreateListFromDefinition(listDef, provisionerChoices); if (listDef.ContentTypeEnabled && listDef.HasContentTypes) { if (listDef.ContentTypes != null && listDef.ContentTypes.Any()) { logger.LogInformation("List {0} => Content types will be provisioned for {1} ctypes", listDef.ListName, listDef.ContentTypes.Count()); foreach (var contentDef in listDef.ContentTypes) { var contentTypeName = contentDef.Name; ContentType accessContentType = null; if (!contextWeb.ContentTypeExistsByName(listName, contentTypeName)) { if (siteDefinition.ContentTypes != null && siteDefinition.ContentTypes.Any(ct => ct.Name == contentTypeName)) { contextWeb.AddContentTypeToListByName(listName, contentTypeName, true); accessContentType = siteList.GetContentTypeByName(contentTypeName); } else { var ctypeInfo = contentDef.ToCreationObject(); accessContentType = siteList.ContentTypes.Add(ctypeInfo); siteList.Update(); siteList.Context.Load(accessContentType, tycp => tycp.Id, tycp => tycp.Name); siteList.Context.ExecuteQueryRetry(); } if (contentDef.DefaultContentType) { siteList.SetDefaultContentType(accessContentType.Id); } } } } } // Existing columns var internalNamesForList = listDef.FieldDefinitions.Select(s => s.InternalName).ToArray(); var internalNamesFoundInList = new List <string>(); try { var existingListColumns = siteList.GetFields(internalNamesForList); foreach (var column in existingListColumns) { ListColumns.Add(new SPFieldDefinitionModel() { InternalName = column.InternalName, Title = column.Title, FieldGuid = column.Id }); internalNamesFoundInList.Add(column.InternalName); } } catch (Exception ex) { logger.LogError(ex, "List {0} => failed to query columns by internal names {1}", listDef.ListName, ex.Message); } // List Columns var nonExistingListColumns = listDef.FieldDefinitions.Where(fd => !internalNamesFoundInList.Any(inf => fd.InternalName.Equals(inf, StringComparison.InvariantCultureIgnoreCase))); foreach (var fieldDef in nonExistingListColumns) { if (fieldDef.FromBaseType == true && fieldDef.SourceID.IndexOf(ns.NamespaceName, StringComparison.CurrentCultureIgnoreCase) > -1) { // OOTB Column var hostsitecolumn = siteFields.FirstOrDefault(fd => fd.InternalName == fieldDef.InternalName); if (hostsitecolumn != null && !siteList.FieldExistsByName(hostsitecolumn.InternalName)) { var column = siteList.Fields.Add(hostsitecolumn); siteList.Update(); siteList.Context.Load(column, cctx => cctx.Id, cctx => cctx.InternalName); siteList.Context.ExecuteQueryRetry(); } var sourceListColumns = siteList.GetFields(fieldDef.InternalName); foreach (var column in sourceListColumns) { ListColumns.Add(new SPFieldDefinitionModel() { InternalName = column.InternalName, Title = column.Title, FieldGuid = column.Id }); } } else if (fieldDef.FieldTypeKind == FieldType.Invalid && fieldDef.FieldTypeKindText.IndexOf("TaxonomyFieldType", StringComparison.InvariantCultureIgnoreCase) > -1) { // Taxonomy Column var hostsitecolumn = siteFields.FirstOrDefault(fd => fd.InternalName == fieldDef.InternalName); if (hostsitecolumn != null && !siteList.FieldExistsByName(hostsitecolumn.InternalName)) { var column = siteList.Fields.Add(hostsitecolumn); siteList.Update(); siteList.Context.Load(column, cctx => cctx.Id, cctx => cctx.InternalName); siteList.Context.ExecuteQueryRetry(); } var sourceListColumns = siteList.GetFields(fieldDef.InternalName); foreach (var column in sourceListColumns) { ListColumns.Add(new SPFieldDefinitionModel() { InternalName = column.InternalName, Title = column.Title, FieldGuid = column.Id }); } } else { var column = siteList.CreateListColumn(fieldDef, logger, SiteGroups, provisionerChoices); if (column == null) { logger.LogWarning("Failed to create column {0}.", fieldDef.InternalName); } else { ListColumns.Add(new SPFieldDefinitionModel() { InternalName = column.InternalName, Title = column.Title, FieldGuid = column.Id }); } } } // Where content types are enabled // Add the provisioned site columns or list columns to the content type if (listDef.ContentTypeEnabled && listDef.HasContentTypes) { foreach (var contentDef in listDef.ContentTypes) { var contentTypeName = contentDef.Name; var accessContentTypes = siteList.ContentTypes; IEnumerable <ContentType> allContentTypes = contextWeb.Context.LoadQuery(accessContentTypes.Where(f => f.Name == contentTypeName).Include(tcyp => tcyp.Id, tcyp => tcyp.Name)); contextWeb.Context.ExecuteQueryRetry(); if (allContentTypes != null) { var accessContentType = allContentTypes.FirstOrDefault(); foreach (var fieldInternalName in contentDef.FieldLinks) { var column = ListColumns.FirstOrDefault(f => f.InternalName == fieldInternalName.Name); if (column == null) { logger.LogWarning("List {0} => Failed to associate field link {1}.", listDef.ListName, fieldInternalName.Name); continue; } var fieldLinks = accessContentType.FieldLinks; contextWeb.Context.Load(fieldLinks, cf => cf.Include(inc => inc.Id, inc => inc.Name)); contextWeb.Context.ExecuteQueryRetry(); var convertedInternalName = column.DisplayNameMasked; if (!fieldLinks.Any(a => a.Name == column.InternalName || a.Name == convertedInternalName)) { logger.LogInformation("List {0} => Content Type {1} Adding Field {2}", listDef.ListName, contentTypeName, column.InternalName); var siteColumn = siteList.GetFieldById <Field>(column.FieldGuid); contextWeb.Context.ExecuteQueryRetry(); var flink = new FieldLinkCreationInformation { Field = siteColumn }; var flinkstub = accessContentType.FieldLinks.Add(flink); //if(fieldDef.Required) flinkstub.Required = fieldDef.Required; accessContentType.Update(false); contextWeb.Context.ExecuteQueryRetry(); } } } } } // Views if (listDef.Views != null && listDef.Views.Any()) { ViewCollection views = siteList.Views; contextWeb.Context.Load(views, f => f.Include(inc => inc.Id, inc => inc.Hidden, inc => inc.Title, inc => inc.DefaultView)); contextWeb.Context.ExecuteQueryRetry(); foreach (var modelView in listDef.Views) { try { var updatecaml = false; View view = null; if (views.Any(v => v.Title.Equals(modelView.Title, StringComparison.CurrentCultureIgnoreCase))) { logger.LogInformation("List {0} => View {1} found in list", listName, modelView.Title); view = views.FirstOrDefault(v => v.Title.Equals(modelView.Title, StringComparison.CurrentCultureIgnoreCase)); updatecaml = true; } else { logger.LogInformation("List {0} => Creating View {0} in list", listName, modelView.Title); view = siteList.CreateView(modelView.CalculatedInternalName, modelView.ViewCamlType, modelView.FieldRefName.ToArray(), modelView.RowLimit, modelView.DefaultView, modelView.ViewQuery, modelView.PersonalView, modelView.Paged); } // grab the view properties from the object view.EnsureProperties( mview => mview.Title, mview => mview.Scope, mview => mview.AggregationsStatus, mview => mview.Aggregations, mview => mview.DefaultView, mview => mview.Hidden, mview => mview.Toolbar, mview => mview.JSLink, mview => mview.ViewFields, vctx => vctx.ViewQuery ); if (modelView.FieldRefName != null && modelView.FieldRefName.Any()) { var currentFields = view.ViewFields; currentFields.RemoveAll(); modelView.FieldRefName.ToList().ForEach(vField => { currentFields.Add(vField.Trim()); }); } if (!string.IsNullOrEmpty(modelView.Aggregations)) { view.Aggregations = modelView.Aggregations; view.AggregationsStatus = modelView.AggregationsStatus; } if (modelView.Hidden.HasValue && modelView.Hidden == true) { view.Hidden = modelView.Hidden.Value; } if (modelView.ToolBarType.HasValue) { view.Toolbar = string.Format("<Toolbar Type=\"{0}\"/>", modelView.ToolBarType.ToString()); } if (updatecaml) { view.DefaultView = modelView.DefaultView; view.RowLimit = modelView.RowLimit; view.ViewQuery = modelView.ViewQuery; } if (modelView.HasJsLink && modelView.JsLink.IndexOf("clienttemplates.js") == -1) { view.JSLink = modelView.JsLink; } view.Scope = modelView.Scope; view.Title = modelView.Title; view.Update(); contextWeb.Context.Load(view, v => v.Title, v => v.Id, v => v.ServerRelativeUrl); contextWeb.Context.ExecuteQueryRetry(); } catch (Exception ex) { logger.LogError(ex, "List {0} => Failed to create view {1} with XML:{2}", listDef.ListName, modelView.Title, modelView.ViewQuery); } } } }); }
/// <summary> /// Process the request /// </summary> public override void ExecuteCmdlet() { base.ExecuteCmdlet(); // Initialize logging instance with Powershell logger ITraceLogger logger = new DefaultUsageLogger(LogVerbose, LogWarning, LogError); // Skip these specific fields var skiptypes = new FieldType[] { FieldType.Invalid, FieldType.Computed, FieldType.ContentTypeId, FieldType.Invalid, FieldType.WorkflowStatus, FieldType.WorkflowEventType, FieldType.Threading, FieldType.ThreadIndex, FieldType.Recurrence, FieldType.PageSeparator, FieldType.OutcomeChoice, FieldType.CrossProjectLink, FieldType.ModStat, FieldType.Error, FieldType.MaxItems, FieldType.Attachments }; // Definition file to operate on SiteProvisionerModel siteDefinition = null; try { // Retreive JSON Provisioner file and deserialize it var filePath = new System.IO.FileInfo(this.ProvisionerFilePath); var filePathJSON = System.IO.File.ReadAllText(filePath.FullName); siteDefinition = JsonConvert.DeserializeObject <SiteProvisionerModel>(filePathJSON); } catch (Exception ex) { logger.LogError(ex, "Failed to parse {0} Exception {1}", ProvisionerFilePath, ex.Message); return; } // Expectation is that list already exists in target location foreach (var customListDefinition in siteDefinition.Lists.OrderBy(ob => ob.ProvisionOrder)) { var etlList = this.ClientContext.Web.GetListByTitle(customListDefinition.ListName, lctx => lctx.Id, lctx => lctx.RootFolder.ServerRelativeUrl, lctx => lctx.Title); logger.LogInformation("List {0}", etlList.Title); var dataMigratedFieldModel = new SPFieldDefinitionModel(FieldType.Boolean) { InternalName = "DataMigrated", Title = "DataMigrated", AddToDefaultView = false, AutoIndexed = true, DefaultValue = "0", FieldGuid = new Guid("9a353694-a5b4-4be4-a77a-c67bb071fbb5"), FieldIndexed = true, GroupName = "customcolumns" }; var dataMigratedField = etlList.CreateListColumn(dataMigratedFieldModel, logger, null, null); var sourceFieldModel = new SPFieldDefinitionModel(FieldType.Integer) { InternalName = "SourceItemID", Title = "SourceItemID", AddToDefaultView = false, AutoIndexed = true, FieldGuid = new Guid("e57a0936-5e8b-45d6-8c1c-d3e971c5b570"), FieldIndexed = true, GroupName = "customcolumns" }; var sourceField = etlList.CreateListColumn(sourceFieldModel, logger, null, null); // pull the internal names from list definition var customFields = customListDefinition.FieldDefinitions .Where(lf => !skiptypes.Any(st => lf.FieldTypeKind == st)) .Select(s => s.InternalName); logger.LogWarning("Processing list {0} found {1} fields to be processed", etlList.Title, customFields.Count()); // enumerate list items and add them to the list if (customListDefinition.ListItems != null && customListDefinition.ListItems.Any()) { foreach (var item in customListDefinition.ListItems) { var lici = new ListItemCreationInformation(); var newItem = etlList.AddItem(lici); newItem[ConstantsFields.Field_Title] = item.Title; newItem["SourceItemID"] = item.Id; newItem["DataMigrated"] = true; newItem.Update(); logger.LogInformation("Processing list {0} Setting up Item {1}", etlList.Title, item.Id); var customColumns = item.ColumnValues.Where(cv => customFields.Any(cf => cf.Equals(cv.FieldName))); foreach (var spRefCol in customColumns) { var internalFieldName = spRefCol.FieldName; var itemColumnValue = spRefCol.FieldValue; if (IsLookup(customListDefinition, internalFieldName, out SPFieldDefinitionModel strParent)) { newItem[internalFieldName] = GetParentItemID(this.ClientContext, itemColumnValue, strParent, logger); } else if (IsUser(customListDefinition, internalFieldName)) { newItem[internalFieldName] = GetUserID(this.ClientContext, itemColumnValue, logger); } else { newItem[internalFieldName] = itemColumnValue; } newItem.Update(); } etlList.Context.ExecuteQueryRetry(); } } } }
/// <summary> /// Process the request /// </summary> public override void ExecuteCmdlet() { base.ExecuteCmdlet(); // File Info var fileInfo = new System.IO.FileInfo(this.ProvisionerFilePath); // Initialize logging instance with Powershell logger ITraceLogger logger = new DefaultUsageLogger(LogVerbose, LogWarning, LogError); // Skip these specific fields var skiptypes = new FieldType[] { FieldType.Computed, FieldType.ContentTypeId, FieldType.Invalid, FieldType.WorkflowStatus, FieldType.WorkflowEventType, FieldType.Threading, FieldType.ThreadIndex, FieldType.Recurrence, FieldType.PageSeparator, FieldType.OutcomeChoice, FieldType.CrossProjectLink, FieldType.ModStat, FieldType.Error, FieldType.MaxItems, FieldType.Attachments }; var skipcolumns = new string[] { "_Hidden", "Base Columns", "Content Feedback", "Core Contact and Calendar Columns", "Core Document Columns", "Core Task and Issue Columns", "Display Template Columns", "Document and Record Management Columns", "Enterprise Keywords Group", "Extended Columns", "JavaScript Display Template Columns", "Page Layout Columns", "Publishing Columns", "Reports", "Status Indicators", "Translation Columns" }; var skipcontenttypes = new string[] { "_Hidden", "Business Intelligence", "Community Content Types", "Digital Asset Content Types", "Display Template Content Types", "Document Content Types", "Document Set Content Types", "Folder Content Types", "Content Feedback", "Publishing Content Types", "Page Layout Content Types", "Special Content Types", "Group Work Content Types", "List Content Types" }; // Construct the model var SiteComponents = new SiteProvisionerModel(); // Load the Context var contextWeb = this.ClientContext.Web; this.ClientContext.Load(contextWeb, ctxw => ctxw.ServerRelativeUrl, ctxw => ctxw.Id); // Get Site/Web fields var fields = this.ClientContext.LoadQuery(contextWeb.Fields .Include( fctx => fctx.Id, fctx => fctx.FieldTypeKind, fctx => fctx.Group, lft => lft.AutoIndexed, lft => lft.CanBeDeleted, lft => lft.DefaultFormula, lft => lft.DefaultValue, lft => lft.Description, lft => lft.EnforceUniqueValues, lft => lft.Filterable, lft => lft.FromBaseType, lft => lft.Hidden, lft => lft.Indexed, lft => lft.InternalName, lft => lft.JSLink, lft => lft.NoCrawl, lft => lft.ReadOnlyField, lft => lft.Required, lft => lft.SchemaXml, lft => lft.Scope, lft => lft.Title)); var groupQuery = this.ClientContext.LoadQuery(contextWeb.SiteGroups .Include(group => group.Id, group => group.Title, group => group.Description, group => group.AllowRequestToJoinLeave, group => group.AllowMembersEditMembership, group => group.AutoAcceptRequestToJoinLeave, group => group.OnlyAllowMembersViewMembership, group => group.RequestToJoinLeaveEmailSetting)); var contentTypes = this.ClientContext.LoadQuery(contextWeb.ContentTypes .Include( ict => ict.Id, ict => ict.Group, ict => ict.Description, ict => ict.Name, ict => ict.Hidden, ict => ict.JSLink, ict => ict.FieldLinks, ict => ict.Fields)); var lists = this.ClientContext.LoadQuery(contextWeb.Lists .Include( linc => linc.Title, linc => linc.Id, linc => linc.Description, linc => linc.RootFolder.ServerRelativeUrl, linc => linc.Hidden, linc => linc.OnQuickLaunch, linc => linc.BaseTemplate, linc => linc.ContentTypesEnabled, linc => linc.AllowContentTypes, linc => linc.EnableFolderCreation, linc => linc.IsApplicationList, linc => linc.IsCatalog, linc => linc.IsSiteAssetsLibrary, linc => linc.IsPrivate, linc => linc.IsSystemList, lctx => lctx.SchemaXml).Where(w => !w.IsSystemList && !w.IsSiteAssetsLibrary)); this.ClientContext.ExecuteQueryRetry(); var weburl = TokenHelper.EnsureTrailingSlash(contextWeb.ServerRelativeUrl); if (groupQuery.Any()) { SiteComponents.Groups = new List <SPGroupDefinitionModel>(); foreach (var group in groupQuery) { var model = new SPGroupDefinitionModel() { Id = group.Id, Title = group.Title, Description = group.Description, AllowRequestToJoinLeave = group.AllowRequestToJoinLeave, AllowMembersEditMembership = group.AllowMembersEditMembership, AutoAcceptRequestToJoinLeave = group.AutoAcceptRequestToJoinLeave, OnlyAllowMembersViewMembership = group.OnlyAllowMembersViewMembership, RequestToJoinLeaveEmailSetting = group.RequestToJoinLeaveEmailSetting }; SiteComponents.Groups.Add(model); } } if (fields.Any()) { var webfields = new List <SPFieldDefinitionModel>(); foreach (var field in fields) { if (skiptypes.Any(st => field.FieldTypeKind == st) || skipcolumns.Any(sg => field.Group.Equals(sg, StringComparison.CurrentCultureIgnoreCase))) { continue; } try { var fieldModel = ClientContext.RetrieveField(field, logger, groupQuery); webfields.Add(fieldModel); } catch (Exception ex) { System.Diagnostics.Trace.TraceError("Failed to parse field {0} MSG:{1}", field.InternalName, ex.Message); } } SiteComponents.FieldDefinitions = webfields; } var contentTypesFieldset = new List <dynamic>(); if (contentTypes.Any()) { SiteComponents.ContentTypes = new List <SPContentTypeDefinition>(); foreach (ContentType contenttype in contentTypes) { // skip core content types if (skipcontenttypes.Any(sg => contenttype.Group.Equals(sg, StringComparison.CurrentCultureIgnoreCase))) { continue; } contenttype.EnsureProperties(ctp => ctp.Id, ctp => ctp.Group, ctp => ctp.Hidden, ctp => ctp.Description, ctp => ctp.Name, ctp => ctp.FieldLinks); var ctypemodel = new SPContentTypeDefinition() { ContentTypeId = contenttype.Id.StringValue, ContentTypeGroup = contenttype.Group, Hidden = contenttype.Hidden, Description = contenttype.Description, Name = contenttype.Name }; if (contenttype.FieldLinks.Any()) { ctypemodel.FieldLinks = new List <SPFieldLinkDefinitionModel>(); foreach (FieldLink fieldlink in contenttype.FieldLinks) { ctypemodel.FieldLinks.Add(new SPFieldLinkDefinitionModel() { Id = fieldlink.Id, Name = fieldlink.Name, Required = fieldlink.Required, Hidden = fieldlink.Hidden }); contentTypesFieldset.Add(new { ctypeid = contenttype.Id.StringValue, name = fieldlink.Name }); } } SiteComponents.ContentTypes.Add(ctypemodel); } } if (lists.Any()) { var sitelists = new List <SPListDefinition>(); foreach (List list in lists.Where(lwt => (!_filterLists || (_filterLists && SpecificLists.Any(sl => lwt.Title.Equals(sl, StringComparison.InvariantCultureIgnoreCase)))))) { var listdefinition = ClientContext.GetListDefinition(contextWeb, list, true, logger, skiptypes, groupQuery); sitelists.Add(listdefinition); } if (sitelists.Any()) { var idx = 0; SiteComponents.Lists = new List <SPListDefinition>(); // lets add any list with NO lookups first var nolookups = sitelists.Where(sl => !sl.ListDependency.Any()); nolookups.ToList().ForEach(nolookup => { logger.LogInformation("adding list {0}", nolookup.ListName); nolookup.ProvisionOrder = idx++; SiteComponents.Lists.Add(nolookup); sitelists.Remove(nolookup); }); // order with first in stack var haslookups = sitelists.Where(sl => sl.ListDependency.Any()).OrderBy(ob => ob.ListDependency.Count()).ToList(); while (haslookups.Count() > 0) { var listPopped = haslookups.FirstOrDefault(); haslookups.Remove(listPopped); logger.LogInformation("adding list {0}", listPopped.ListName); if (listPopped.ListDependency.Any(listField => listPopped.ListName.Equals(listField, StringComparison.InvariantCultureIgnoreCase) || listPopped.InternalName.Equals(listField, StringComparison.InvariantCultureIgnoreCase))) { // the list has a dependency on itself logger.LogInformation("Adding list {0} with dependency on itself", listPopped.ListName); listPopped.ProvisionOrder = idx++; SiteComponents.Lists.Add(listPopped); } else if (listPopped.ListDependency.Any(listField => !SiteComponents.Lists.Any(sc => sc.ListName.Equals(listField, StringComparison.InvariantCultureIgnoreCase) || sc.InternalName.Equals(listField, StringComparison.InvariantCultureIgnoreCase)))) { // no list definition exists in the collection with the dependent lookup lists logger.LogWarning("List {0} depends on {1} which do not exist current collection", listPopped.ListName, string.Join(",", listPopped.ListDependency)); foreach (var dependent in listPopped.ListDependency.Where(dlist => !haslookups.Any(hl => hl.ListName == dlist))) { var sitelist = contextWeb.GetListByTitle(dependent, lctx => lctx.Id, lctx => lctx.Title, lctx => lctx.RootFolder.ServerRelativeUrl); var listDefinition = ClientContext.GetListDefinition(contextWeb, sitelist, true, logger, skiptypes, groupQuery); haslookups.Add(listDefinition); } haslookups.Add(listPopped); // add back to collection listPopped = null; } else { logger.LogInformation("Adding list {0} to collection", listPopped.ListName); listPopped.ProvisionOrder = idx++; SiteComponents.Lists.Add(listPopped); } } } } // Write the JSON to disc var jsonsettings = new JsonSerializerSettings() { Formatting = Formatting.Indented, Culture = System.Globalization.CultureInfo.CurrentUICulture, DateFormatHandling = DateFormatHandling.IsoDateFormat, NullValueHandling = NullValueHandling.Ignore }; var json = JsonConvert.SerializeObject(SiteComponents, jsonsettings); System.IO.File.WriteAllText(fileInfo.FullName, json); }