/// <summary>
        /// Execute the cmdlet
        /// </summary>
        public override void ExecuteCmdlet()
        {
            base.ExecuteCmdlet();

            Collection <SPListItemDefinition> results = new Collection <SPListItemDefinition>();

            var _listSites = this.ClientContext.Web.Lists.GetByTitle(ListTitle);

            this.ClientContext.Load(_listSites);
            this.ClientContext.ExecuteQuery();

            ListItemCollectionPosition itemPosition = null;

            try
            {
                while (true)
                {
                    CamlQuery camlQuery = new CamlQuery();
                    camlQuery.ListItemCollectionPosition = itemPosition;
                    camlQuery.ViewXml = @"<View><Query>
                                            <ViewFields>
                                                <FieldRef Name='Title'/>
                                             </ViewFields>
                                            <RowLimit>50</RowLimit>
                                        </Query></View>";
                    ListItemCollection listItems = _listSites.GetItems(camlQuery);
                    this.ClientContext.Load(listItems);
                    this.ClientContext.ExecuteQuery();
                    itemPosition = listItems.ListItemCollectionPosition;

                    foreach (var rbiItem in listItems)
                    {
                        LogVerbose("Title: {0}; Item ID: {1}", rbiItem["Title"], rbiItem.Id);
                        var newitem = new SPListItemDefinition()
                        {
                            Title = rbiItem.RetrieveListItemValue("Title"),
                            Id    = rbiItem.Id
                        };
                        results.Add(newitem);
                    }

                    if (itemPosition == null)
                    {
                        break;
                    }
                }
            }
            catch (Exception ex)
            {
                LogError(ex, "Failed to retrieve recycle bin collection");
            }

            WriteObject(results, true);
        }
        /// <summary>
        /// Execute the cmdlet
        /// </summary>
        public override void ExecuteCmdlet()
        {
            base.ExecuteCmdlet();


            // Initialize logging instance with Powershell logger
            ITraceLogger logger = new DefaultUsageLogger(LogVerbose, LogWarning, LogError);


            // Construct the model
            var SiteComponents = new SiteProvisionerModel()
            {
                Lists = new List <SPListDefinition>()
            };

            var ctx        = this.ClientContext;
            var contextWeb = this.ClientContext.Web;

            ctx.Load(contextWeb, ctxw => ctxw.ServerRelativeUrl, wctx => wctx.Url, ctxw => ctxw.Id);
            ctx.ExecuteQueryRetry();

            var _targetList = Identity.GetList(contextWeb, lctx => lctx.ItemCount, lctx => lctx.EnableFolderCreation, lctx => lctx.RootFolder, lctx => lctx.RootFolder.ServerRelativeUrl, lctx => lctx.RootFolder.Folders);


            var webUri = new Uri(contextWeb.Url);

            // Will query the list to determine the last item id in the list
            var lastItemId = _targetList.QueryLastItemId();

            logger.LogInformation("List with item count {0} has a last ID of {1}", _targetList.ItemCount, lastItemId);
            logger.LogInformation("List has folder creation = {0}", _targetList.EnableFolderCreation);

            // store the OOTB standard fields
            var ootbCamlFields = new List <string>()
            {
                "Title", "ID", "Author", "Editor", "Created", "Modified"
            };

            // Skip these specific fields
            var skiptypes = new FieldType[]
            {
                FieldType.Invalid,
                FieldType.Computed,
                FieldType.ContentTypeId,
                FieldType.Invalid,
                FieldType.WorkflowStatus,
                FieldType.WorkflowEventType,
                FieldType.Threading,
                FieldType.ThreadIndex,
                FieldType.Recurrence,
                FieldType.PageSeparator,
                FieldType.OutcomeChoice,
                FieldType.CrossProjectLink,
                FieldType.ModStat,
                FieldType.Error,
                FieldType.MaxItems,
                FieldType.Attachments
            };

            // pull a small portion of the list
            // wire up temporary array
            var sitelists = new List <SPListDefinition>()
            {
                ctx.GetListDefinition(_targetList, ExpandObjects, logger, skiptypes)
            };

            if (sitelists.Any())
            {
                var idx = 0;

                // lets add any list with NO lookups first
                var nolookups = sitelists.Where(sl => !sl.ListDependency.Any());
                nolookups.ToList().ForEach(nolookup =>
                {
                    logger.LogInformation("adding list {0}", nolookup.ListName);
                    nolookup.ProvisionOrder = idx++;
                    SiteComponents.Lists.Add(nolookup);
                    sitelists.Remove(nolookup);
                });

                // order with first in stack
                var haslookups = sitelists.Where(sl => sl.ListDependency.Any()).OrderBy(ob => ob.ListDependency.Count()).ToList();
                while (haslookups.Count() > 0)
                {
                    var listPopped = haslookups.FirstOrDefault();
                    haslookups.Remove(listPopped);
                    logger.LogInformation("popping list {0}", listPopped.ListName);

                    if (listPopped.ListDependency.Any(listField => listPopped.ListName.Equals(listField, StringComparison.InvariantCultureIgnoreCase) ||
                                                      listPopped.InternalName.Equals(listField, StringComparison.InvariantCultureIgnoreCase)))
                    {
                        // the list has a dependency on itself
                        logger.LogInformation("Adding list {0} with dependency on itself", listPopped.ListName);
                        listPopped.ProvisionOrder = idx++;
                        SiteComponents.Lists.Add(listPopped);
                    }
                    else if (listPopped.ListDependency.Any(listField =>
                                                           !SiteComponents.Lists.Any(sc => sc.ListName.Equals(listField, StringComparison.InvariantCultureIgnoreCase) ||
                                                                                     sc.InternalName.Equals(listField, StringComparison.InvariantCultureIgnoreCase))))
                    {
                        // no list definition exists in the collection with the dependent lookup lists
                        logger.LogWarning("List {0} depends on {1} which do not exist current collection", listPopped.ListName, string.Join(",", listPopped.ListDependency));
                        foreach (var dependent in listPopped.ListDependency.Where(dlist => !haslookups.Any(hl => hl.ListName == dlist)))
                        {
                            var sitelist           = contextWeb.GetListByTitle(dependent, lctx => lctx.Id, lctx => lctx.Title, lctx => lctx.RootFolder.ServerRelativeUrl);
                            var sitelistDefinition = ctx.GetListDefinition(contextWeb, sitelist, true, logger, skiptypes);
                            haslookups.Add(sitelistDefinition);
                        }

                        haslookups.Add(listPopped); // add back to collection
                        listPopped = null;
                    }
                    else
                    {
                        logger.LogInformation("Adding list {0} to collection", listPopped.ListName);
                        listPopped.ProvisionOrder = idx++;
                        SiteComponents.Lists.Add(listPopped);
                    }
                }
            }

            foreach (var listDefinition in SiteComponents.Lists.OrderBy(ob => ob.ProvisionOrder))
            {
                listDefinition.ListItems = new List <SPListItemDefinition>();

                var camlFields = new List <string>();
                var listTitle  = listDefinition.ListName;
                var etlList    = this.ClientContext.Web.GetListByTitle(listTitle,
                                                                       lctx => lctx.Id, lctx => lctx.ItemCount, lctx => lctx.EnableFolderCreation, lctx => lctx.RootFolder.ServerRelativeUrl);

                if (ExpandObjects)
                {
                    // list fields
                    var listFields = listDefinition.FieldDefinitions;
                    if (listFields != null && listFields.Any())
                    {
                        var filteredListFields = listFields.Where(lf => !skiptypes.Any(st => lf.FieldTypeKind == st)).ToList();
                        var notInCamlFields    = listFields.Where(listField => !ootbCamlFields.Any(cf => cf.Equals(listField.InternalName, StringComparison.CurrentCultureIgnoreCase)) && !skiptypes.Any(st => listField.FieldTypeKind == st));
                        foreach (var listField in notInCamlFields)
                        {
                            logger.LogInformation("Processing list {0} field {1}", etlList.Title, listField.InternalName);
                            camlFields.Add(listField.InternalName);
                        }
                    }
                }

                camlFields.AddRange(ootbCamlFields);
                var camlViewFields = CAML.ViewFields(camlFields.Select(s => CAML.FieldRef(s)).ToArray());


                ListItemCollectionPosition itemPosition = null;
                var camlQueries = etlList.SafeCamlClauseFromThreshold(2000, CamlStatement, 0, lastItemId);
                foreach (var camlAndValue in camlQueries)
                {
                    itemPosition = null;
                    var camlWhereClause = (string.IsNullOrEmpty(camlAndValue) ? string.Empty : CAML.Where(camlAndValue));
                    var camlQuery       = new CamlQuery()
                    {
                        ViewXml = CAML.ViewQuery(
                            ViewScope.RecursiveAll,
                            camlWhereClause,
                            CAML.OrderBy(new OrderByField("ID")),
                            camlViewFields,
                            500)
                    };

                    try
                    {
                        while (true)
                        {
                            logger.LogWarning("CAML Query {0} at position {1}", camlWhereClause, (itemPosition == null ? string.Empty : itemPosition.PagingInfo));
                            camlQuery.ListItemCollectionPosition = itemPosition;
                            var listItems = etlList.GetItems(camlQuery);
                            etlList.Context.Load(listItems);
                            etlList.Context.ExecuteQueryRetry();
                            itemPosition = listItems.ListItemCollectionPosition;

                            foreach (var rbiItem in listItems)
                            {
                                var itemId    = rbiItem.Id;
                                var itemTitle = rbiItem.RetrieveListItemValue("Title");
                                var author    = rbiItem.RetrieveListItemUserValue("Author");
                                var editor    = rbiItem.RetrieveListItemUserValue("Editor");
                                logger.LogInformation("Title: {0}; Item ID: {1}", itemTitle, itemId);

                                var newitem = new SPListItemDefinition()
                                {
                                    Id       = itemId,
                                    Title    = itemTitle,
                                    Created  = rbiItem.RetrieveListItemValue("Created").ToNullableDatetime(),
                                    Modified = rbiItem.RetrieveListItemValue("Modified").ToNullableDatetime()
                                };

                                if (author != null)
                                {
                                    newitem.CreatedBy = new SPPrincipalUserDefinition()
                                    {
                                        Email     = author.ToUserEmailValue(),
                                        LoginName = author.ToUserValue(),
                                        Id        = author.LookupId
                                    };
                                }
                                if (editor != null)
                                {
                                    newitem.ModifiedBy = new SPPrincipalUserDefinition()
                                    {
                                        Email     = editor.ToUserEmailValue(),
                                        LoginName = editor.ToUserValue(),
                                        Id        = editor.LookupId
                                    };
                                }


                                if (ExpandObjects)
                                {
                                    var fieldValuesToWrite = rbiItem.FieldValues.Where(rfield => !ootbCamlFields.Any(oc => rfield.Key.Equals(oc, StringComparison.CurrentCultureIgnoreCase)));
                                    foreach (var rbiField in fieldValuesToWrite)
                                    {
                                        newitem.ColumnValues.Add(new SPListItemFieldDefinition()
                                        {
                                            FieldName  = rbiField.Key,
                                            FieldValue = rbiField.Value
                                        });
                                    }
                                }

                                listDefinition.ListItems.Add(newitem);
                            }

                            if (itemPosition == null)
                            {
                                break;
                            }
                        }
                    }
                    catch (Exception ex)
                    {
                        logger.LogError(ex, "Failed to retrieve list item collection");
                    }
                }
            }


            WriteObject(SiteComponents);
        }
Example #3
0
        /// <summary>
        /// Execute the REST API querying the list with paging
        /// </summary>
        public override void ExecuteCmdlet()
        {
            base.ExecuteCmdlet();

            Collection <SPListItemDefinition> results = new Collection <SPListItemDefinition>();

            try
            {
                var creds        = SPIaCConnection.CurrentConnection.GetActiveCredentials();
                var spourl       = new Uri(this.ClientContext.Url);
                var spocreds     = new Microsoft.SharePoint.Client.SharePointOnlineCredentials(creds.UserName, creds.Password);
                var spocookies   = spocreds.GetAuthenticationCookie(spourl);
                var spocontainer = new System.Net.CookieContainer();
                spocontainer.SetCookies(spourl, spocookies);

                // region Consume the web service
                var ListService = string.Format("{0}/_api/web/lists/getByTitle('{1}')/ItemCount", this.ClientContext.Url, this.ListTitle);
                var webRequest  = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(ListService);
                webRequest.Credentials     = new System.Net.NetworkCredential(creds.UserName, creds.Password);
                webRequest.Method          = "GET";
                webRequest.Accept          = "application/json;odata=verbose";
                webRequest.CookieContainer = spocontainer;

                var webResponse = webRequest.GetResponse();
                using (Stream webStream = webResponse.GetResponseStream())
                {
                    using (StreamReader responseReader = new StreamReader(webStream))
                    {
                        string response  = responseReader.ReadToEnd();
                        var    jobj      = JObject.Parse(response);
                        var    itemCount = jobj["d"]["ItemCount"];
                        LogVerbose("ItemCount:{0}", itemCount);
                    }
                }

                var successFlag = true;
                ListService = string.Format("{0}/_api/web/lists/getByTitle('{1}')/items?$top={2}", this.ClientContext.Url, this.ListTitle, this.Throttle);
                while (successFlag)
                {
                    LogVerbose("Paging:{0}", ListService);
                    successFlag                = false;
                    webRequest                 = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(ListService);
                    webRequest.Credentials     = new System.Net.NetworkCredential(creds.UserName, creds.Password);
                    webRequest.Method          = "GET";
                    webRequest.Accept          = "application/json;odata=verbose";
                    webRequest.CookieContainer = spocontainer;

                    webResponse = webRequest.GetResponse();
                    using (Stream webStream = webResponse.GetResponseStream())
                    {
                        using (StreamReader responseReader = new StreamReader(webStream))
                        {
                            string response  = responseReader.ReadToEnd();
                            var    jobj      = JObject.Parse(response);
                            var    jarr      = (JArray)jobj["d"]["results"];
                            var    jnextpage = jobj["d"]["__next"];

                            foreach (JObject j in jarr)
                            {
                                LogVerbose("ItemID:{0}", j["Id"]);
                                var newitem = new SPListItemDefinition()
                                {
                                    Title = j["Title"].ToObject <string>(),
                                    Id    = j["Id"].ToObject <int>()
                                };
                                results.Add(newitem);
                            }

                            if (jnextpage != null && !String.IsNullOrEmpty(jnextpage.ToString()))
                            {
                                successFlag = true;
                                ListService = jnextpage.ToString();
                            }
                        }
                    }
                }

                WriteObject(results, true);
            }
            catch (Exception ex)
            {
                LogError(ex, "Failed in GetListItemCount for Library {0}", ListTitle);
            }
        }