public bool CreateSystemJobDeletionJob(IOrganizationService service, out string errorMsg)
{
try
{
ConditionExpression conEx = new ConditionExpression("statecode", ConditionOperator.Equal, 3);
FilterExpression fiEx = new FilterExpression();
fiEx.AddCondition(conEx);
BulkDeleteRequest request = new BulkDeleteRequest
{
JobName = "Bulk Delete Completed System Jobs",
CCRecipients = new Guid[] { },
ToRecipients = new Guid[] { },
RecurrencePattern = string.Empty,
QuerySet = new QueryExpression[]
{
new QueryExpression {
EntityName = "asyncoperation", Criteria = fiEx
}
}
};
BulkDeleteResponse response = (BulkDeleteResponse)service.Execute(request);
errorMsg = string.Empty;
return(true);
}
catch (FaultException <OrganizationServiceFault> e)
{
errorMsg = e.Message;
Trace.WriteLine(e.Message);
return(false);
}
}
public BulkDeleteResponse BulkDelete(QueryRequest request)
{
var url = string.Format("{0}/{1}/_delete_by_query", _baseUrl, _index);
var requestText = request.GenerateQuery();
if (logger.IsDebugEnabled)
{
logger.Debug("+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+");
logger.Debug(string.Format("Delete Request {0} Started:{1}Query:{2}", request.UniqueIdentifer, DateTime.Now.ToShortTimeString(), request.ToString()));
}
try
{
var responseJson = PostAsync(url, requestText).Result;
BulkDeleteResponse response = JsonConvert.DeserializeObject <BulkDeleteResponse>(responseJson);
response.RequestIdentifier = request.UniqueIdentifer;
if (logger.IsDebugEnabled)
{
logger.Debug(string.Format("Delete Request {0} completed: {1}; {2} Records deleted", request.UniqueIdentifer, DateTime.Now.ToShortTimeString(), response.Deleted));
logger.Debug("+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+");
}
return(response);
}
catch (Exception ex)
{
logger.Error(string.Format("{0} -{1}- Failed. {2}", "DoSearch", request.UniqueIdentifer, ex.Message));
throw;
}
}
public bool CreateBulkCaseDeletionJob(IOrganizationService service, out string errorMsg)
{
try
{
ConditionExpression conditionExp = new ConditionExpression(IncidentEntity.ProtectedCase, ConditionOperator.NotEqual, true);
FilterExpression filterExp = new FilterExpression();
filterExp.AddCondition(conditionExp);
BulkDeleteRequest request = new BulkDeleteRequest
{
JobName = "Delete Unprotected Records - Bulk Case Deletion",
ToRecipients = new Guid[] { },
CCRecipients = new Guid[] { },
RunNow = true,
RecurrencePattern = string.Empty,
QuerySet = new QueryExpression[]
{
new QueryExpression {
EntityName = IncidentEntity.LogicalName, Criteria = filterExp
}
}
};
BulkDeleteResponse response = (BulkDeleteResponse)service.Execute(request);
Trace.WriteLine($"Bulk Deletion Job response: {response.ToString()}");
errorMsg = null;
return(true);
}
catch (FaultException <OrganizationServiceFault> e)
{
errorMsg = e.Message;
Trace.WriteLine(e.Message);
return(false);
}
}
/// <summary>
/// Driver function for bulk import.
/// </summary>
/// <returns></returns>
private async Task RunBulkImportAndBulkDeleteAsync()
{
// Import documents into the collection
await this.RunBulkImport();
// Fetch <PartitionKey><DocumentId> tuples to delete in bulk
List <Tuple <string, string> > pkIdTuplesToDelete = GeneratePartitionKeyDocumentIdTuplesToBulkDelete();
long totalNumberOfDocumentsDeleted = 0;
double totalRequestUnitsConsumed = 0;
double totalTimeTakenSec = 0;
BulkDeleteResponse bulkDeleteResponse = null;
BulkExecutor bulkExecutor = new BulkExecutor(this.client, this.dataCollection);
await bulkExecutor.InitializeAsync();
try
{
bulkDeleteResponse = await bulkExecutor.BulkDeleteAsync(pkIdTuplesToDelete);
totalNumberOfDocumentsDeleted = bulkDeleteResponse.NumberOfDocumentsDeleted;
totalRequestUnitsConsumed = bulkDeleteResponse.TotalRequestUnitsConsumed;
totalTimeTakenSec = bulkDeleteResponse.TotalTimeTaken.TotalSeconds;
}
catch (DocumentClientException de)
{
Trace.TraceError("Document client exception: {0}", de);
}
catch (Exception e)
{
Trace.TraceError("Exception: {0}", e);
}
Trace.WriteLine("\n\n--------------------------------------------------------------------- ");
Trace.WriteLine("Executing bulk delete:");
Trace.WriteLine("--------------------------------------------------------------------- ");
Trace.WriteLine("\n\nOverall summary of bulk delete:");
Trace.WriteLine("--------------------------------------------------------------------- ");
Trace.WriteLine(String.Format("Deleted {0} docs @ {1} writes/s, {2} RU/s in {3} sec",
totalNumberOfDocumentsDeleted,
Math.Round(totalNumberOfDocumentsDeleted / totalTimeTakenSec),
Math.Round(totalRequestUnitsConsumed / totalTimeTakenSec),
totalTimeTakenSec));
Trace.WriteLine(String.Format("Average RU consumption per document delete: {0}",
(totalRequestUnitsConsumed / totalNumberOfDocumentsDeleted)));
Trace.WriteLine("--------------------------------------------------------------------- \n");
//-----------------------------------------------------------------------------------------------
}
public async Task DeleteAsync(IEnumerable <string> ids)
{
var documentCollection = await _documentClient.ReadDocumentCollectionAsync(GetCollectionUri()).ConfigureAwait(false);
var bulkExecutor = new BulkExecutor(_documentClient as Documents.Client.DocumentClient, documentCollection);
await bulkExecutor.InitializeAsync().ConfigureAwait(false);
var entries = ids.Select(x => new Tuple <string, string>(x, x)).ToList();
BulkDeleteResponse bulkDeleteResponse = null;
do
{
bulkDeleteResponse = await bulkExecutor
.BulkDeleteAsync(entries)
.ConfigureAwait(false);
} while (bulkDeleteResponse.NumberOfDocumentsDeleted < entries.Count && bulkDeleteResponse.NumberOfDocumentsDeleted > 0);
}
public async Task Delete <T>(string collectionName, List <T> toDelete) where T : CosmosObject
{
var bulkExecutor = await BuildClientAsync(collectionName);
var partitionKeyProperty = GetPartitionKeyProp <T>();
var deleteOperations = toDelete.Select(doc => {
var partitionKeyValue = (string)typeof(T).GetProperty(partitionKeyProperty.Name).GetValue(doc, null);
return(new Tuple <string, string>(partitionKeyValue, doc.id));
}).ToList();
BulkDeleteResponse bulkDeleteResponse = null;
try
{
bulkDeleteResponse = await bulkExecutor.BulkDeleteAsync(
deleteOperations,
deleteBatchSize : 1000,
cancellationToken : new CancellationTokenSource().Token
);
}
catch (DocumentClientException de)
{
Trace.TraceError("Document client exception: {0}", de);
}
catch (Exception e)
{
Trace.TraceError("Exception: {0}", e);
}
Trace.WriteLine("\n\n--------------------------------------------------------------------- ");
Trace.WriteLine("Executing bulk delete:");
Trace.WriteLine("--------------------------------------------------------------------- ");
Trace.WriteLine("\n\nOverall summary of bulk delete:");
Trace.WriteLine("--------------------------------------------------------------------- ");
Trace.WriteLine(String.Format("Deleted {0} docs", bulkDeleteResponse.NumberOfDocumentsDeleted));
Trace.WriteLine("--------------------------------------------------------------------- \n");
}
public void BulkCaseDeletion(IOrganizationService _orgServ)
{
OrganizationServiceContext _orgContext = new OrganizationServiceContext(_orgServ);
ConditionExpression conEx = new ConditionExpression("shg_casesavefield", ConditionOperator.NotEqual, true);
FilterExpression fiEx = new FilterExpression();
fiEx.AddCondition(conEx);
BulkDeleteRequest request = new BulkDeleteRequest
{
JobName = "Delete unsaved cases",
ToRecipients = new Guid[] { },
CCRecipients = new Guid[] { },
RecurrencePattern = string.Empty,
QuerySet = new QueryExpression[]
{
new QueryExpression {
EntityName = "incident", Criteria = fiEx
}
}
};
BulkDeleteResponse response = (BulkDeleteResponse)_orgServ.Execute(request);
}
public void BulkActivityDeletion(IOrganizationService _orgServ)
{
OrganizationServiceContext _orgContext = new OrganizationServiceContext(_orgServ);
ConditionExpression conEx = new ConditionExpression("regardingobjectid", ConditionOperator.Null);
FilterExpression fiEx = new FilterExpression();
fiEx.AddCondition(conEx);
BulkDeleteRequest request = new BulkDeleteRequest
{
JobName = "Delete unlinked activites",
ToRecipients = new Guid[] { },
CCRecipients = new Guid[] { },
RecurrencePattern = string.Empty,
QuerySet = new QueryExpression[]
{
new QueryExpression {
EntityName = "activitypointer", Criteria = fiEx
}
}
};
BulkDeleteResponse response = (BulkDeleteResponse)_orgServ.Execute(request);
}
public void WhenProcessOutputWithMoreItemsThanBatchSizeShouldRunInsertBatchDocuments()
{
var dateTime = new DateTime(2018, 08, 30);
var fileName = "some-file-name";
var codeChurn =
new DailyCodeChurn()
{
Added = 1,
ChangesBefore = 2,
ChangesAfter = 3,
Deleted = 4,
FileName = "abc",
Timestamp = "2018/08/30 00:00:00",
Authors = new List <DailyCodeChurnAuthor>()
{
new DailyCodeChurnAuthor()
{
Author = "author1",
NumberOfChanges = 1
},
new DailyCodeChurnAuthor()
{
Author = "author2",
NumberOfChanges = 2
}
}
};
var dict = new Dictionary <DateTime, Dictionary <string, DailyCodeChurn> >()
{
{ dateTime, new Dictionary <string, DailyCodeChurn> {
{ fileName, codeChurn }
} },
{ dateTime.AddDays(1), new Dictionary <string, DailyCodeChurn> {
{ fileName, codeChurn }
} },
{ dateTime.AddDays(2), new Dictionary <string, DailyCodeChurn> {
{ fileName, codeChurn }
} },
{ dateTime.AddDays(3), new Dictionary <string, DailyCodeChurn> {
{ fileName, codeChurn }
} },
{ dateTime.AddDays(4), new Dictionary <string, DailyCodeChurn> {
{ fileName, codeChurn }
} }
};
var bulkDeleteResponse = new BulkDeleteResponse();
var cosmosBulkImportSummary = new CosmosBulkImportSummary();
dataDocumentRepositoryMock
.Setup(x => x.BatchDeleteDocuments(dict.First().Key, dict.Last().Key, ProjectName, DocumentType.CodeChurn))
.Returns(bulkDeleteResponse);
dataDocumentRepositoryMock
.Setup(x => x.BatchInsertCosmosDocuments(It.IsAny <List <CosmosDataDocument <DailyCodeChurn> > >(), It.IsAny <Action <CosmosBulkImportSummary> >()))
.Returns(cosmosBulkImportSummary);
sut.ProcessOutput(OutputType.CosmosDb, string.Empty, dict);
dataDocumentRepositoryMock.Verify(x => x.DeleteMultipleDocuments(It.Is <List <CosmosDataDocument <DailyCodeChurn> > >(items =>
items.Any(y => CompareDailyCodeChurn(y.Data[0], codeChurn)))), Times.Never);
dataDocumentRepositoryMock.Verify(x => x.CreateDataDocument(It.Is <CosmosDataDocument <DailyCodeChurn> >(
y => CompareDailyCodeChurn(y.Data[0], codeChurn))), Times.Never);
dataDocumentRepositoryMock.Verify(x => x.BatchDeleteDocuments(dict.First().Key, dict.Last().Key, ProjectName, DocumentType.CodeChurn), Times.Exactly(1));
}
private static BulkDeleteResult DeleteInternal(IOrganizationService service, QueryExpression[] querySets)
{
BulkDeleteResult result = new BulkDeleteResult();
// Create the bulk delete request
BulkDeleteRequest bulkDeleteRequest = new BulkDeleteRequest()
{
// Set the request properties
JobName = "Temp Bulk Delete " + DateTime.Now,
QuerySet = querySets,
ToRecipients = new Guid[0],
CCRecipients = new Guid[0],
RecurrencePattern = string.Empty
};
// Submit the bulk delete job.
// NOTE: Because this is an asynchronous operation, the response will be immediate.
BulkDeleteResponse response = (BulkDeleteResponse)service.Execute(bulkDeleteRequest);
Guid asyncId = response.JobId;
var bulkOperation = GetBulkDeleteOperation(service, asyncId);
// Monitor the async operation through polling until it is complete or max polling time expires.
int secondsTicker = MaxAsynchronousRequestTimeout;
while (secondsTicker > 0)
{
// Make sure that the async operation was retrieved.
if (bulkOperation != null && bulkOperation.StateCode.Value == BulkDeleteOperationState.Completed)
{
result.TimedOut = false;
break;
}
// Wait a second for async operation to become active.
System.Threading.Thread.Sleep(1000);
secondsTicker--;
// Retrieve the entity again
bulkOperation = GetBulkDeleteOperation(service, asyncId);
}
if (result.TimedOut == null)
{
result.TimedOut = true;
}
// Validate that the operation was completed.
if (bulkOperation == null)
{
result.Result = "The Bulk Operation for Async " + asyncId + " was not found.";
}
else
{
result.DeletedCount = bulkOperation.SuccessCount ?? 0;
result.DeleteFailedCount = bulkOperation.FailureCount ?? 0;
if (bulkOperation.StateCode.Value != BulkDeleteOperationState.Completed ||
bulkOperation.StatusCode.Value != (int)bulkdeleteoperation_statuscode.Succeeded)
{
// This happens if it took longer than the polling time allowed
// for this operation to finish.
result.Result = "The operation took longer than the polling time allowed for this operation to finish.";
}
else if (result.DeleteFailedCount > 0)
{
result.Result = string.Format("The opertion had {0} failures and {1} successful deletions",
result.DeletedCount, result.DeleteFailedCount);
}
else
{
result.Result = string.Format("The operation had {0} successful deletions",
result.DeletedCount);
}
}
service.Delete(BulkDeleteOperation.EntityLogicalName, bulkOperation.Id);
// We have to update the AsyncOperation to be in a Completed state before we can delete it.
service.InitializeEntity <Entities.AsyncOperation>(bulkOperation.AsyncOperationId.Id,
a => { a.StateCode = AsyncOperationState.Completed; });
// Not sure if the status code needs to be set...
//a.StatusCode = new OptionSetValue((int)asyncoperation_statuscode.Succeeded) });
service.Delete(Entities.AsyncOperation.EntityLogicalName, bulkOperation.AsyncOperationId.Id);
return(result);
}
// <summary>
/// Performs the main operation of the sample - performs a bulk delete on inactive
/// opportunities and activities to remove them from the system.
/// </summary>
private static void PerformBulkDeleteBackup(CrmServiceClient service)
{
try
{
// Query for a system user to send an email to after the bulk delete
// operation completes.
var userRequest = new WhoAmIRequest();
var userResponse = (WhoAmIResponse)service.Execute(userRequest);
Guid currentUserId = userResponse.UserId;
// Create a condition for a bulk delete request.
// NOTE: This sample uses very specific queries for deleting records
// that have been manually exported in order to free space.
QueryExpression opportunitiesQuery = BuildOpportunityQuery();
// Create the bulk delete request.
var bulkDeleteRequest = new BulkDeleteRequest();
// Set the request properties.
bulkDeleteRequest.JobName = "Backup Bulk Delete";
// Querying activities
bulkDeleteRequest.QuerySet = new QueryExpression[]
{
opportunitiesQuery,
BuildActivityQuery(Task.EntityLogicalName),
BuildActivityQuery(Fax.EntityLogicalName),
BuildActivityQuery(PhoneCall.EntityLogicalName),
BuildActivityQuery(Email.EntityLogicalName),
BuildActivityQuery(Letter.EntityLogicalName),
BuildActivityQuery(Appointment.EntityLogicalName),
BuildActivityQuery(ServiceAppointment.EntityLogicalName),
BuildActivityQuery(CampaignResponse.EntityLogicalName),
BuildActivityQuery(RecurringAppointmentMaster.EntityLogicalName)
};
// Set the start time for the bulk delete.
bulkDeleteRequest.StartDateTime = DateTime.Now;
// Set the required recurrence pattern.
bulkDeleteRequest.RecurrencePattern = String.Empty;
// Set email activity properties.
bulkDeleteRequest.SendEmailNotification = false;
bulkDeleteRequest.ToRecipients = new Guid[] { currentUserId };
bulkDeleteRequest.CCRecipients = new Guid[] { };
// Submit the bulk delete job.
// NOTE: Because this is an asynchronous operation, the response will be immediate.
_bulkDeleteResponse =
(BulkDeleteResponse)service.Execute(bulkDeleteRequest);
Console.WriteLine("The bulk delete operation has been requested.");
CheckSuccess(service);
}
catch (System.Web.Services.Protocols.SoapException)
{
// Perform error handling here.
throw;
}
catch (Exception)
{
throw;
}
}
/// <summary>
/// Performs the main operation of the sample - performs a bulk delete on inactive
/// opportunities and activities to remove them from the system.
/// </summary>
private void PerformBulkDeleteBackup()
{
try
{
// Query for a system user to send an email to after the bulk delete
// operation completes.
var userRequest = new WhoAmIRequest();
var userResponse = (WhoAmIResponse)_serviceProxy.Execute(userRequest);
Guid currentUserId = userResponse.UserId;
//<snippetBulkDelete>
// Create a condition for a bulk delete request.
// NOTE: This sample uses very specific queries for deleting records
// that have been manually exported in order to free space.
QueryExpression opportunitiesQuery = BuildOpportunityQuery();
// Create the bulk delete request.
BulkDeleteRequest bulkDeleteRequest = new BulkDeleteRequest();
// Set the request properties.
bulkDeleteRequest.JobName = "Backup Bulk Delete";
// Querying activities
bulkDeleteRequest.QuerySet = new QueryExpression[]
{
opportunitiesQuery,
BuildActivityQuery(Task.EntityLogicalName),
BuildActivityQuery(Fax.EntityLogicalName),
BuildActivityQuery(PhoneCall.EntityLogicalName),
BuildActivityQuery(Email.EntityLogicalName),
BuildActivityQuery(Letter.EntityLogicalName),
BuildActivityQuery(Appointment.EntityLogicalName),
BuildActivityQuery(ServiceAppointment.EntityLogicalName),
BuildActivityQuery(CampaignResponse.EntityLogicalName),
BuildActivityQuery(RecurringAppointmentMaster.EntityLogicalName)
};
// Set the start time for the bulk delete.
bulkDeleteRequest.StartDateTime = DateTime.Now;
// Set the required recurrence pattern.
bulkDeleteRequest.RecurrencePattern = String.Empty;
// Set email activity properties.
bulkDeleteRequest.SendEmailNotification = false;
bulkDeleteRequest.ToRecipients = new Guid[] { currentUserId };
bulkDeleteRequest.CCRecipients = new Guid[] { };
// Submit the bulk delete job.
// NOTE: Because this is an asynchronous operation, the response will be immediate.
_bulkDeleteResponse =
(BulkDeleteResponse)_serviceProxy.Execute(bulkDeleteRequest);
Console.WriteLine("The bulk delete operation has been requested.");
//</snippetBulkDelete>
CheckSuccess();
}
catch (System.Web.Services.Protocols.SoapException)
{
// Perform error handling here.
throw;
}
catch (Exception)
{
throw;
}
}
