/// <summary>
/// Execute the cmdlet
/// </summary>
public override void ExecuteCmdlet()
{
base.ExecuteCmdlet();
Collection <SPListItemDefinition> results = new Collection <SPListItemDefinition>();
var _listSites = this.ClientContext.Web.Lists.GetByTitle(ListTitle);
this.ClientContext.Load(_listSites);
this.ClientContext.ExecuteQuery();
ListItemCollectionPosition itemPosition = null;
try
{
while (true)
{
CamlQuery camlQuery = new CamlQuery();
camlQuery.ListItemCollectionPosition = itemPosition;
camlQuery.ViewXml = @"<View><Query>
<ViewFields>
<FieldRef Name='Title'/>
</ViewFields>
<RowLimit>50</RowLimit>
</Query></View>";
ListItemCollection listItems = _listSites.GetItems(camlQuery);
this.ClientContext.Load(listItems);
this.ClientContext.ExecuteQuery();
itemPosition = listItems.ListItemCollectionPosition;
foreach (var rbiItem in listItems)
{
LogVerbose("Title: {0}; Item ID: {1}", rbiItem["Title"], rbiItem.Id);
var newitem = new SPListItemDefinition()
{
Title = rbiItem.RetrieveListItemValue("Title"),
Id = rbiItem.Id
};
results.Add(newitem);
}
if (itemPosition == null)
{
break;
}
}
}
catch (Exception ex)
{
LogError(ex, "Failed to retrieve recycle bin collection");
}
WriteObject(results, true);
}
/// <summary>
/// Execute the cmdlet
/// </summary>
public override void ExecuteCmdlet()
{
base.ExecuteCmdlet();
// Initialize logging instance with Powershell logger
ITraceLogger logger = new DefaultUsageLogger(LogVerbose, LogWarning, LogError);
// Construct the model
var SiteComponents = new SiteProvisionerModel()
{
Lists = new List <SPListDefinition>()
};
var ctx = this.ClientContext;
var contextWeb = this.ClientContext.Web;
ctx.Load(contextWeb, ctxw => ctxw.ServerRelativeUrl, wctx => wctx.Url, ctxw => ctxw.Id);
ctx.ExecuteQueryRetry();
var _targetList = Identity.GetList(contextWeb, lctx => lctx.ItemCount, lctx => lctx.EnableFolderCreation, lctx => lctx.RootFolder, lctx => lctx.RootFolder.ServerRelativeUrl, lctx => lctx.RootFolder.Folders);
var webUri = new Uri(contextWeb.Url);
// Will query the list to determine the last item id in the list
var lastItemId = _targetList.QueryLastItemId();
logger.LogInformation("List with item count {0} has a last ID of {1}", _targetList.ItemCount, lastItemId);
logger.LogInformation("List has folder creation = {0}", _targetList.EnableFolderCreation);
// store the OOTB standard fields
var ootbCamlFields = new List <string>()
{
"Title", "ID", "Author", "Editor", "Created", "Modified"
};
// Skip these specific fields
var skiptypes = new FieldType[]
{
FieldType.Invalid,
FieldType.Computed,
FieldType.ContentTypeId,
FieldType.Invalid,
FieldType.WorkflowStatus,
FieldType.WorkflowEventType,
FieldType.Threading,
FieldType.ThreadIndex,
FieldType.Recurrence,
FieldType.PageSeparator,
FieldType.OutcomeChoice,
FieldType.CrossProjectLink,
FieldType.ModStat,
FieldType.Error,
FieldType.MaxItems,
FieldType.Attachments
};
// pull a small portion of the list
// wire up temporary array
var sitelists = new List <SPListDefinition>()
{
ctx.GetListDefinition(_targetList, ExpandObjects, logger, skiptypes)
};
if (sitelists.Any())
{
var idx = 0;
// lets add any list with NO lookups first
var nolookups = sitelists.Where(sl => !sl.ListDependency.Any());
nolookups.ToList().ForEach(nolookup =>
{
logger.LogInformation("adding list {0}", nolookup.ListName);
nolookup.ProvisionOrder = idx++;
SiteComponents.Lists.Add(nolookup);
sitelists.Remove(nolookup);
});
// order with first in stack
var haslookups = sitelists.Where(sl => sl.ListDependency.Any()).OrderBy(ob => ob.ListDependency.Count()).ToList();
while (haslookups.Count() > 0)
{
var listPopped = haslookups.FirstOrDefault();
haslookups.Remove(listPopped);
logger.LogInformation("popping list {0}", listPopped.ListName);
if (listPopped.ListDependency.Any(listField => listPopped.ListName.Equals(listField, StringComparison.InvariantCultureIgnoreCase) ||
listPopped.InternalName.Equals(listField, StringComparison.InvariantCultureIgnoreCase)))
{
// the list has a dependency on itself
logger.LogInformation("Adding list {0} with dependency on itself", listPopped.ListName);
listPopped.ProvisionOrder = idx++;
SiteComponents.Lists.Add(listPopped);
}
else if (listPopped.ListDependency.Any(listField =>
!SiteComponents.Lists.Any(sc => sc.ListName.Equals(listField, StringComparison.InvariantCultureIgnoreCase) ||
sc.InternalName.Equals(listField, StringComparison.InvariantCultureIgnoreCase))))
{
// no list definition exists in the collection with the dependent lookup lists
logger.LogWarning("List {0} depends on {1} which do not exist current collection", listPopped.ListName, string.Join(",", listPopped.ListDependency));
foreach (var dependent in listPopped.ListDependency.Where(dlist => !haslookups.Any(hl => hl.ListName == dlist)))
{
var sitelist = contextWeb.GetListByTitle(dependent, lctx => lctx.Id, lctx => lctx.Title, lctx => lctx.RootFolder.ServerRelativeUrl);
var sitelistDefinition = ctx.GetListDefinition(contextWeb, sitelist, true, logger, skiptypes);
haslookups.Add(sitelistDefinition);
}
haslookups.Add(listPopped); // add back to collection
listPopped = null;
}
else
{
logger.LogInformation("Adding list {0} to collection", listPopped.ListName);
listPopped.ProvisionOrder = idx++;
SiteComponents.Lists.Add(listPopped);
}
}
}
foreach (var listDefinition in SiteComponents.Lists.OrderBy(ob => ob.ProvisionOrder))
{
listDefinition.ListItems = new List <SPListItemDefinition>();
var camlFields = new List <string>();
var listTitle = listDefinition.ListName;
var etlList = this.ClientContext.Web.GetListByTitle(listTitle,
lctx => lctx.Id, lctx => lctx.ItemCount, lctx => lctx.EnableFolderCreation, lctx => lctx.RootFolder.ServerRelativeUrl);
if (ExpandObjects)
{
// list fields
var listFields = listDefinition.FieldDefinitions;
if (listFields != null && listFields.Any())
{
var filteredListFields = listFields.Where(lf => !skiptypes.Any(st => lf.FieldTypeKind == st)).ToList();
var notInCamlFields = listFields.Where(listField => !ootbCamlFields.Any(cf => cf.Equals(listField.InternalName, StringComparison.CurrentCultureIgnoreCase)) && !skiptypes.Any(st => listField.FieldTypeKind == st));
foreach (var listField in notInCamlFields)
{
logger.LogInformation("Processing list {0} field {1}", etlList.Title, listField.InternalName);
camlFields.Add(listField.InternalName);
}
}
}
camlFields.AddRange(ootbCamlFields);
var camlViewFields = CAML.ViewFields(camlFields.Select(s => CAML.FieldRef(s)).ToArray());
ListItemCollectionPosition itemPosition = null;
var camlQueries = etlList.SafeCamlClauseFromThreshold(2000, CamlStatement, 0, lastItemId);
foreach (var camlAndValue in camlQueries)
{
itemPosition = null;
var camlWhereClause = (string.IsNullOrEmpty(camlAndValue) ? string.Empty : CAML.Where(camlAndValue));
var camlQuery = new CamlQuery()
{
ViewXml = CAML.ViewQuery(
ViewScope.RecursiveAll,
camlWhereClause,
CAML.OrderBy(new OrderByField("ID")),
camlViewFields,
500)
};
try
{
while (true)
{
logger.LogWarning("CAML Query {0} at position {1}", camlWhereClause, (itemPosition == null ? string.Empty : itemPosition.PagingInfo));
camlQuery.ListItemCollectionPosition = itemPosition;
var listItems = etlList.GetItems(camlQuery);
etlList.Context.Load(listItems);
etlList.Context.ExecuteQueryRetry();
itemPosition = listItems.ListItemCollectionPosition;
foreach (var rbiItem in listItems)
{
var itemId = rbiItem.Id;
var itemTitle = rbiItem.RetrieveListItemValue("Title");
var author = rbiItem.RetrieveListItemUserValue("Author");
var editor = rbiItem.RetrieveListItemUserValue("Editor");
logger.LogInformation("Title: {0}; Item ID: {1}", itemTitle, itemId);
var newitem = new SPListItemDefinition()
{
Id = itemId,
Title = itemTitle,
Created = rbiItem.RetrieveListItemValue("Created").ToNullableDatetime(),
Modified = rbiItem.RetrieveListItemValue("Modified").ToNullableDatetime()
};
if (author != null)
{
newitem.CreatedBy = new SPPrincipalUserDefinition()
{
Email = author.ToUserEmailValue(),
LoginName = author.ToUserValue(),
Id = author.LookupId
};
}
if (editor != null)
{
newitem.ModifiedBy = new SPPrincipalUserDefinition()
{
Email = editor.ToUserEmailValue(),
LoginName = editor.ToUserValue(),
Id = editor.LookupId
};
}
if (ExpandObjects)
{
var fieldValuesToWrite = rbiItem.FieldValues.Where(rfield => !ootbCamlFields.Any(oc => rfield.Key.Equals(oc, StringComparison.CurrentCultureIgnoreCase)));
foreach (var rbiField in fieldValuesToWrite)
{
newitem.ColumnValues.Add(new SPListItemFieldDefinition()
{
FieldName = rbiField.Key,
FieldValue = rbiField.Value
});
}
}
listDefinition.ListItems.Add(newitem);
}
if (itemPosition == null)
{
break;
}
}
}
catch (Exception ex)
{
logger.LogError(ex, "Failed to retrieve list item collection");
}
}
}
WriteObject(SiteComponents);
}
/// <summary>
/// Execute the REST API querying the list with paging
/// </summary>
public override void ExecuteCmdlet()
{
base.ExecuteCmdlet();
Collection <SPListItemDefinition> results = new Collection <SPListItemDefinition>();
try
{
var creds = SPIaCConnection.CurrentConnection.GetActiveCredentials();
var spourl = new Uri(this.ClientContext.Url);
var spocreds = new Microsoft.SharePoint.Client.SharePointOnlineCredentials(creds.UserName, creds.Password);
var spocookies = spocreds.GetAuthenticationCookie(spourl);
var spocontainer = new System.Net.CookieContainer();
spocontainer.SetCookies(spourl, spocookies);
// region Consume the web service
var ListService = string.Format("{0}/_api/web/lists/getByTitle('{1}')/ItemCount", this.ClientContext.Url, this.ListTitle);
var webRequest = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(ListService);
webRequest.Credentials = new System.Net.NetworkCredential(creds.UserName, creds.Password);
webRequest.Method = "GET";
webRequest.Accept = "application/json;odata=verbose";
webRequest.CookieContainer = spocontainer;
var webResponse = webRequest.GetResponse();
using (Stream webStream = webResponse.GetResponseStream())
{
using (StreamReader responseReader = new StreamReader(webStream))
{
string response = responseReader.ReadToEnd();
var jobj = JObject.Parse(response);
var itemCount = jobj["d"]["ItemCount"];
LogVerbose("ItemCount:{0}", itemCount);
}
}
var successFlag = true;
ListService = string.Format("{0}/_api/web/lists/getByTitle('{1}')/items?$top={2}", this.ClientContext.Url, this.ListTitle, this.Throttle);
while (successFlag)
{
LogVerbose("Paging:{0}", ListService);
successFlag = false;
webRequest = (System.Net.HttpWebRequest)System.Net.WebRequest.Create(ListService);
webRequest.Credentials = new System.Net.NetworkCredential(creds.UserName, creds.Password);
webRequest.Method = "GET";
webRequest.Accept = "application/json;odata=verbose";
webRequest.CookieContainer = spocontainer;
webResponse = webRequest.GetResponse();
using (Stream webStream = webResponse.GetResponseStream())
{
using (StreamReader responseReader = new StreamReader(webStream))
{
string response = responseReader.ReadToEnd();
var jobj = JObject.Parse(response);
var jarr = (JArray)jobj["d"]["results"];
var jnextpage = jobj["d"]["__next"];
foreach (JObject j in jarr)
{
LogVerbose("ItemID:{0}", j["Id"]);
var newitem = new SPListItemDefinition()
{
Title = j["Title"].ToObject <string>(),
Id = j["Id"].ToObject <int>()
};
results.Add(newitem);
}
if (jnextpage != null && !String.IsNullOrEmpty(jnextpage.ToString()))
{
successFlag = true;
ListService = jnextpage.ToString();
}
}
}
}
WriteObject(results, true);
}
catch (Exception ex)
{
LogError(ex, "Failed in GetListItemCount for Library {0}", ListTitle);
}
}
