/// <summary> /// Execute the cmdlet /// </summary> public override void ExecuteCmdlet() { base.ExecuteCmdlet(); Collection <SPListItemDefinition> results = new Collection <SPListItemDefinition>(); var _listSites = this.ClientContext.Web.Lists.GetByTitle(ListTitle); this.ClientContext.Load(_listSites); this.ClientContext.ExecuteQuery(); ListItemCollectionPosition itemPosition = null; try { while (true) { CamlQuery camlQuery = new CamlQuery(); camlQuery.ListItemCollectionPosition = itemPosition; camlQuery.ViewXml = @"<View><Query> <ViewFields> <FieldRef Name='Title'/> </ViewFields> <RowLimit>50</RowLimit> </Query></View>"; ListItemCollection listItems = _listSites.GetItems(camlQuery); this.ClientContext.Load(listItems); this.ClientContext.ExecuteQuery(); itemPosition = listItems.ListItemCollectionPosition; foreach (var rbiItem in listItems) { LogVerbose("Title: {0}; Item ID: {1}", rbiItem["Title"], rbiItem.Id); var newitem = new SPListItemDefinition() { Title = rbiItem.RetrieveListItemValue("Title"), Id = rbiItem.Id }; results.Add(newitem); } if (itemPosition == null) { break; } } } catch (Exception ex) { LogError(ex, "Failed to retrieve recycle bin collection"); } WriteObject(results, true); }
/// <summary> /// Execute the cmdlet /// </summary> public override void ExecuteCmdlet() { base.ExecuteCmdlet(); // Initialize logging instance with Powershell logger ITraceLogger logger = new DefaultUsageLogger(LogVerbose, LogWarning, LogError); // Construct the model var SiteComponents = new SiteProvisionerModel() { Lists = new List <SPListDefinition>() }; var ctx = this.ClientContext; var contextWeb = this.ClientContext.Web; ctx.Load(contextWeb, ctxw => ctxw.ServerRelativeUrl, wctx => wctx.Url, ctxw => ctxw.Id); ctx.ExecuteQueryRetry(); var _targetList = Identity.GetList(contextWeb, lctx => lctx.ItemCount, lctx => lctx.EnableFolderCreation, lctx => lctx.RootFolder, lctx => lctx.RootFolder.ServerRelativeUrl, lctx => lctx.RootFolder.Folders); var webUri = new Uri(contextWeb.Url); // Will query the list to determine the last item id in the list var lastItemId = _targetList.QueryLastItemId(); logger.LogInformation("List with item count {0} has a last ID of {1}", _targetList.ItemCount, lastItemId); logger.LogInformation("List has folder creation = {0}", _targetList.EnableFolderCreation); // store the OOTB standard fields var ootbCamlFields = new List <string>() { "Title", "ID", "Author", "Editor", "Created", "Modified" }; // Skip these specific fields var skiptypes = new FieldType[] { FieldType.Invalid, FieldType.Computed, FieldType.ContentTypeId, FieldType.Invalid, FieldType.WorkflowStatus, FieldType.WorkflowEventType, FieldType.Threading, FieldType.ThreadIndex, FieldType.Recurrence, FieldType.PageSeparator, FieldType.OutcomeChoice, FieldType.CrossProjectLink, FieldType.ModStat, FieldType.Error, FieldType.MaxItems, FieldType.Attachments }; // pull a small portion of the list // wire up temporary array var sitelists = new List <SPListDefinition>() { ctx.GetListDefinition(_targetList, ExpandObjects, logger, skiptypes) }; if (sitelists.Any()) { var idx = 0; // lets add any list with NO lookups first var nolookups = sitelists.Where(sl => !sl.ListDependency.Any()); nolookups.ToList().ForEach(nolookup => { logger.LogInformation("adding list {0}", nolookup.ListName); nolookup.ProvisionOrder = idx++; SiteComponents.Lists.Add(nolookup); sitelists.Remove(nolookup); }); // order with first in stack var haslookups = sitelists.Where(sl => sl.ListDependency.Any()).OrderBy(ob => ob.ListDependency.Count()).ToList(); while (haslookups.Count() > 0) { var listPopped = haslookups.FirstOrDefault(); haslookups.Remove(listPopped); logger.LogInformation("popping list {0}", listPopped.ListName); if (listPopped.ListDependency.Any(listField => listPopped.ListName.Equals(listField, StringComparison.InvariantCultureIgnoreCase) || listPopped.InternalName.Equals(listField, StringComparison.InvariantCultureIgnoreCase))) { // the list has a dependency on itself logger.LogInformation("Adding list {0} with dependency on itself", listPopped.ListName); listPopped.ProvisionOrder = idx++; SiteComponents.Lists.Add(listPopped); } else if (listPopped.ListDependency.Any(listField => !SiteComponents.Lists.Any(sc => sc.ListName.Equals(listField, StringComparison.InvariantCultureIgnoreCase) || sc.InternalName.Equals(listField, StringComparison.InvariantCultureIgnoreCase)))) { // no list definition exists in the collection with the dependent lookup lists logger.LogWarning("List {0} depends on {1} which do not exist current collection", listPopped.ListName, string.Join(",", listPopped.ListDependency)); foreach (var dependent in listPopped.ListDependency.Where(dlist => !haslookups.Any(hl => hl.ListName == dlist))) { var sitelist = contextWeb.GetListByTitle(dependent, lctx => lctx.Id, lctx => lctx.Title, lctx => lctx.RootFolder.ServerRelativeUrl); var sitelistDefinition = ctx.GetListDefinition(contextWeb, sitelist, true, logger, skiptypes); haslookups.Add(sitelistDefinition); } haslookups.Add(listPopped); // add back to collection listPopped = null; } else { logger.LogInformation("Adding list {0} to collection", listPopped.ListName); listPopped.ProvisionOrder = idx++; SiteComponents.Lists.Add(listPopped); } } } foreach (var listDefinition in SiteComponents.Lists.OrderBy(ob => ob.ProvisionOrder)) { listDefinition.ListItems = new List <SPListItemDefinition>(); var camlFields = new List <string>(); var listTitle = listDefinition.ListName; var etlList = this.ClientContext.Web.GetListByTitle(listTitle, lctx => lctx.Id, lctx => lctx.ItemCount, lctx => lctx.EnableFolderCreation, lctx => lctx.RootFolder.ServerRelativeUrl); if (ExpandObjects) { // list fields var listFields = listDefinition.FieldDefinitions; if (listFields != null && listFields.Any()) { var filteredListFields = listFields.Where(lf => !skiptypes.Any(st => lf.FieldTypeKind == st)).ToList(); var notInCamlFields = listFields.Where(listField => !ootbCamlFields.Any(cf => cf.Equals(listField.InternalName, StringComparison.CurrentCultureIgnoreCase)) && !skiptypes.Any(st => listField.FieldTypeKind == st)); foreach (var listField in notInCamlFields) { logger.LogInformation("Processing list {0} field {1}", etlList.Title, listField.InternalName); camlFields.Add(listField.InternalName); } } } camlFields.AddRange(ootbCamlFields); var camlViewFields = CAML.ViewFields(camlFields.Select(s => CAML.FieldRef(s)).ToArray()); ListItemCollectionPosition itemPosition = null; var camlQueries = etlList.SafeCamlClauseFromThreshold(2000, CamlStatement, 0, lastItemId); foreach (var camlAndValue in camlQueries) { itemPosition = null; var camlWhereClause = (string.IsNullOrEmpty(camlAndValue) ? string.Empty : CAML.Where(camlAndValue)); var camlQuery = new CamlQuery() { ViewXml = CAML.ViewQuery( ViewScope.RecursiveAll, camlWhereClause, CAML.OrderBy(new OrderByField("ID")), camlViewFields, 500) }; try { while (true) { logger.LogWarning("CAML Query {0} at position {1}", camlWhereClause, (itemPosition == null ? string.Empty : itemPosition.PagingInfo)); camlQuery.ListItemCollectionPosition = itemPosition; var listItems = etlList.GetItems(camlQuery); etlList.Context.Load(listItems); etlList.Context.ExecuteQueryRetry(); itemPosition = listItems.ListItemCollectionPosition; foreach (var rbiItem in listItems) { var itemId = rbiItem.Id; var itemTitle = rbiItem.RetrieveListItemValue("Title"); var author = rbiItem.RetrieveListItemUserValue("Author"); var editor = rbiItem.RetrieveListItemUserValue("Editor"); logger.LogInformation("Title: {0}; Item ID: {1}", itemTitle, itemId); var newitem = new SPListItemDefinition() { Id = itemId, Title = itemTitle, Created = rbiItem.RetrieveListItemValue("Created").ToNullableDatetime(), Modified = rbiItem.RetrieveListItemValue("Modified").ToNullableDatetime() }; if (author != null) { newitem.CreatedBy = new SPPrincipalUserDefinition() { Email = author.ToUserEmailValue(), LoginName = author.ToUserValue(), Id = author.LookupId }; } if (editor != null) { newitem.ModifiedBy = new SPPrincipalUserDefinition() { Email = editor.ToUserEmailValue(), LoginName = editor.ToUserValue(), Id = editor.LookupId }; } if (ExpandObjects) { var fieldValuesToWrite = rbiItem.FieldValues.Where(rfield => !ootbCamlFields.Any(oc => rfield.Key.Equals(oc, StringComparison.CurrentCultureIgnoreCase))); foreach (var rbiField in fieldValuesToWrite) { newitem.ColumnValues.Add(new SPListItemFieldDefinition() { FieldName = rbiField.Key, FieldValue = rbiField.Value }); } } listDefinition.ListItems.Add(newitem); } if (itemPosition == null) { break; } } } catch (Exception ex) { logger.LogError(ex, "Failed to retrieve list item collection"); } } } WriteObject(SiteComponents); }
/// <summary> /// Execute the REST API querying the list with paging /// </summary> public override void ExecuteCmdlet() { base.ExecuteCmdlet(); Collection <SPListItemDefinition> results = new Collection <SPListItemDefinition>(); try { var creds = SPIaCConnection.CurrentConnection.GetActiveCredentials(); var spourl = new Uri(this.ClientContext.Url); var spocreds = new Microsoft.SharePoint.Client.SharePointOnlineCredentials(creds.UserName, creds.Password); var spocookies = spocreds.GetAuthenticationCookie(spourl); var spocontainer = new System.Net.CookieContainer(); spocontainer.SetCookies(spourl, spocookies); // region Consume the web service var ListService = string.Format("{0}/_api/web/lists/getByTitle('{1}')/ItemCount", this.ClientContext.Url, this.ListTitle); var webRequest = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(ListService); webRequest.Credentials = new System.Net.NetworkCredential(creds.UserName, creds.Password); webRequest.Method = "GET"; webRequest.Accept = "application/json;odata=verbose"; webRequest.CookieContainer = spocontainer; var webResponse = webRequest.GetResponse(); using (Stream webStream = webResponse.GetResponseStream()) { using (StreamReader responseReader = new StreamReader(webStream)) { string response = responseReader.ReadToEnd(); var jobj = JObject.Parse(response); var itemCount = jobj["d"]["ItemCount"]; LogVerbose("ItemCount:{0}", itemCount); } } var successFlag = true; ListService = string.Format("{0}/_api/web/lists/getByTitle('{1}')/items?$top={2}", this.ClientContext.Url, this.ListTitle, this.Throttle); while (successFlag) { LogVerbose("Paging:{0}", ListService); successFlag = false; webRequest = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(ListService); webRequest.Credentials = new System.Net.NetworkCredential(creds.UserName, creds.Password); webRequest.Method = "GET"; webRequest.Accept = "application/json;odata=verbose"; webRequest.CookieContainer = spocontainer; webResponse = webRequest.GetResponse(); using (Stream webStream = webResponse.GetResponseStream()) { using (StreamReader responseReader = new StreamReader(webStream)) { string response = responseReader.ReadToEnd(); var jobj = JObject.Parse(response); var jarr = (JArray)jobj["d"]["results"]; var jnextpage = jobj["d"]["__next"]; foreach (JObject j in jarr) { LogVerbose("ItemID:{0}", j["Id"]); var newitem = new SPListItemDefinition() { Title = j["Title"].ToObject <string>(), Id = j["Id"].ToObject <int>() }; results.Add(newitem); } if (jnextpage != null && !String.IsNullOrEmpty(jnextpage.ToString())) { successFlag = true; ListService = jnextpage.ToString(); } } } } WriteObject(results, true); } catch (Exception ex) { LogError(ex, "Failed in GetListItemCount for Library {0}", ListTitle); } }