/// <summary>
/// Reprocesses lookup relationship fields that were missed during the initial import and
/// populates them, once all the records are loaded into the sandbox. Similar to
/// UpdateRecursiveField.
/// </summary>
private PopulateSandboxResult ReprocessObjects(List <ObjectTransformer> objectsToReprocess)
{
var results = new PopulateSandboxResult();
foreach (var obj in objectsToReprocess)
{
foreach (var field in obj.LookupsToReprocess)
{
ProgressUpdate(string.Format("Reprocessing referenced objects for {0} field {1}",
field.ObjectName, field.FieldName));
List <sObject> updateList = new List <sObject>();
foreach (var idPair in field.IdPairs)
{
var updateObj = CreateSobjectWithLookup(field.ObjectName,
field.RelatedObjectName, field.FieldName, idPair);
if (updateObj != null)
{
updateList.Add(updateObj);
}
}
// switch the IDs with the new ones in the sandbox
foreach (sObject rowLoop in updateList)
{
rowLoop.Id = _relationMapper.RecallNewId(field.ObjectName, rowLoop.Id);
}
ProgressUpdate(string.Format("Updating {0} {1} records",
updateList.Count, field.ObjectName));
var result = UpdateRecords(field.ObjectName, updateList);
results.ObjectResults.Add(result);
}
}
return(results);
}
public PopulateSandboxResult Start(IProgress <string> progress)
{
_progress = progress;
var res = new PopulateSandboxResult();
ProgressUpdate("Logging in to Salesforce API ...");
LoginToBoth();
ProgressUpdate("Fetching metadata...");
var apiNameArray = _instructions.SbbObjects.Select(o => o.ApiName).ToArray();
_sourceTasks.FetchObjectMetadata(apiNameArray);
ProgressUpdate("Getting inactive users...");
var inactiveUserIds = _targetTasks.GetInactiveUsers();
string currentUserId = _targetTasks.GetCurrentUserId();
ProgressUpdate("Checking for missing users...");
var sourceUserIds = _sourceTasks.GetAllUsers();
var targetUserIds = _targetTasks.GetAllUsers();
var missingUserIds = sourceUserIds.Except(targetUserIds).ToList();
logger.DebugFormat("Found {0} users in Source that are not in Target", missingUserIds.Count());
foreach (SbbObject objLoop in _instructions.SbbObjects)
{
ProgressUpdate(string.Format("Starting to process {0}", objLoop.ApiName));
var objres = new PopulateObjectResult();
var transformer = new ObjectTransformer();
transformer.RelationMapper = _relationMapper;
transformer.InactiveUserIds = inactiveUserIds;
transformer.MissingUserIds = missingUserIds;
transformer.CurrentUserId = currentUserId;
transformer.SbbObjectInstructions = objLoop;
objres.ApiName = objLoop.ApiName;
res.ObjectResults.Add(objres);
transformer.ObjectRelationships = _sourceTasks.GetObjectRelationships(objLoop.ApiName);
transformer.RecursiveRelationshipField = transformer.ObjectRelationships.FirstOrDefault(d => d.Value == objLoop.ApiName).Key;
if (transformer.RecursiveRelationshipField != null)
{
logger.DebugFormat("Object {0} has a recurive relation to iteself in field {1}",
objLoop.ApiName, transformer.RecursiveRelationshipField);
}
List <sObject> sourceData = null;
try
{
sourceData = _sourceTasks.GetDataFromSObject(objLoop.ApiName, objLoop.Filter);
}
catch (Exception e)
{
string errMess = string.Format("Error while fetching data for {0}: {1}", objLoop.ApiName, e.Message);
throw new ApplicationException(errMess, e);
}
objres.SourceRows = sourceData.Count();
ProgressUpdate(string.Format("Received {0} {1} records from source", sourceData.Count, objLoop.ApiName));
// get working info and transform objects
var workingList = new List <ObjectTransformer.sObjectWrapper>();
foreach (sObject rowLoop in sourceData)
{
var wrap = new ObjectTransformer.sObjectWrapper();
wrap.OriginalId = rowLoop.Id;
wrap.sObj = rowLoop;
transformer.ApplyTransformations(wrap);
workingList.Add(wrap);
}
// insert objects in batches
int batchSize = 100;
int done = 0;
bool allDone = false;
if (workingList.Count == 0)
{
allDone = true;
}
while (!allDone)
{
var workBatch = workingList.Skip(done).Take(batchSize).ToList();
done += workBatch.Count;
if (done >= workingList.Count)
{
allDone = true;
}
var insertRes = _targetTasks.InsertSObjects(objLoop.ApiName,
workBatch.Select(w => w.sObj).ToArray());
for (int i = 0; i < insertRes.Length; i++)
{
if (!string.IsNullOrEmpty(insertRes[i].NewId))
{
workBatch[i].NewId = insertRes[i].NewId;
objres.SuccessCount += 1;
_relationMapper.Remember(objLoop.ApiName, workBatch[i].OriginalId, workBatch[i].NewId);
}
else
{
workBatch[i].ErrorMessage = insertRes[i].ErrorMessage;
logger.WarnFormat("Error when inserting {0} {1} into target: {2}",
objLoop.ApiName, workBatch[i].OriginalId, workBatch[i].ErrorMessage);
objres.FailCount += 1;
}
}
}
// inserts done.
// if there's a recurive field, do update
if (transformer.RecursiveRelationshipField != null)
{
UpdateRecursiveField(objLoop.ApiName, workingList, transformer.RecursiveRelationshipField);
}
ProgressUpdate(string.Format("Summary for {0}: Success {1} Fail {2}",
objLoop.ApiName, objres.SuccessCount, objres.FailCount));
}
// log summary
ProgressUpdate("************************************************");
foreach (var resLoop in res.ObjectResults)
{
ProgressUpdate(string.Format("Summary for {0}: Success {1} Fail {2}",
resLoop.ApiName, resLoop.SuccessCount, resLoop.FailCount));
}
ProgressUpdate("************************************************");
return(res);
}
public PopulateSandboxResult Start(IProgress <string> progress)
{
_progress = progress;
var res = new PopulateSandboxResult();
ProgressUpdate("Logging in to Salesforce API ...");
LoginToBoth();
ProgressUpdate("Fetching metadata...");
var apiNameArray = _instructions.SbbObjects.Select(o => o.ApiName).ToArray();
_sourceTasks.FetchObjectMetadata(apiNameArray);
ProgressUpdate("Getting inactive users...");
var inactiveUserIds = _targetTasks.GetInactiveUsers();
string currentUserId = _targetTasks.GetCurrentUserId();
ProgressUpdate("Checking for missing users...");
var sourceUserIds = _sourceTasks.GetAllUsers();
var targetUserIds = _targetTasks.GetAllUsers();
var missingUserIds = sourceUserIds.Except(targetUserIds).ToList();
logger.DebugFormat("Found {0} users in Source that are not in Target", missingUserIds.Count());
var processedObjects = new List <String> {
"User"
}; // user is copied already
var objectsToReprocess = new List <ObjectTransformer>();
foreach (SbbObject objLoop in _instructions.SbbObjects)
{
ProgressUpdate(string.Format("Starting to process {0}", objLoop.ApiName));
var objres = new PopulateObjectResult();
var transformer = new ObjectTransformer();
transformer.RelationMapper = _relationMapper;
transformer.InactiveUserIds = inactiveUserIds;
transformer.MissingUserIds = missingUserIds;
transformer.CurrentUserId = currentUserId;
transformer.SbbObjectInstructions = objLoop;
objres.ApiName = objLoop.ApiName;
res.ObjectResults.Add(objres);
transformer.ObjectRelationships = _sourceTasks.GetObjectRelationships(objLoop.ApiName);
transformer.RecursiveRelationshipField = transformer.ObjectRelationships.FirstOrDefault(d => d.Value == objLoop.ApiName).Key;
if (transformer.RecursiveRelationshipField != null)
{
logger.DebugFormat("Object {0} has a recurive relation to iteself in field {1}",
objLoop.ApiName, transformer.RecursiveRelationshipField);
}
// find lookups on this object that can't be populated, to reprocess later
transformer.LookupsToReprocess = transformer.ObjectRelationships
.Where(d => d.Value != objLoop.ApiName) // where it's not a recursive relationship
.Where(d => !processedObjects.Contains(d.Value)) // and the referenced record doesn't exist yet
.Where(d => !objLoop.SbbFieldOptions.Any( // and it's not one of the skipped fields
e => e.ApiName.Equals(d.Key) &&
e.Skip))
// TODO: but is still one of the included object types (e.g. not Contact -> "rh2__PS_Describe__c")
.Select(d => new LookupInfo
{
FieldName = d.Key,
ObjectName = objLoop.ApiName,
RelatedObjectName = d.Value
})
.ToList();
if (transformer.LookupsToReprocess.Count > 0)
{
objectsToReprocess.Add(transformer);
var fields = transformer.LookupsToReprocess.Select(lookup => lookup.FieldName);
logger.DebugFormat("Object {0} has lookups that will need to be reprocessed: {1}",
objLoop.ApiName,
String.Join(", ", fields));
}
List <sObject> sourceData = null;
try
{
sourceData = _sourceTasks.GetDataFromSObject(objLoop.ApiName, objLoop.Filter);
}
catch (Exception e)
{
string errMess = string.Format("Error while fetching data for {0}: {1}", objLoop.ApiName, e.Message);
throw new ApplicationException(errMess, e);
}
objres.SourceRows = sourceData.Count();
ProgressUpdate(string.Format("Received {0} {1} records from source", sourceData.Count, objLoop.ApiName));
// get working info and transform objects
var workingList = new List <ObjectTransformer.sObjectWrapper>();
foreach (sObject rowLoop in sourceData)
{
var wrap = new ObjectTransformer.sObjectWrapper();
wrap.OriginalId = rowLoop.Id;
wrap.sObj = rowLoop;
transformer.ApplyTransformations(wrap);
workingList.Add(wrap);
}
// insert objects in batches
int batchSize = 100;
int done = 0;
bool allDone = false;
if (workingList.Count == 0)
{
allDone = true;
}
while (!allDone)
{
var workBatch = workingList.Skip(done).Take(batchSize).ToList();
done += workBatch.Count;
if (done >= workingList.Count)
{
allDone = true;
}
var insertRes = _targetTasks.InsertSObjects(objLoop.ApiName,
workBatch.Select(w => w.sObj).ToArray());
for (int i = 0; i < insertRes.Length; i++)
{
if (!string.IsNullOrEmpty(insertRes[i].NewId))
{
workBatch[i].NewId = insertRes[i].NewId;
objres.SuccessCount += 1;
_relationMapper.Remember(objLoop.ApiName, workBatch[i].OriginalId, workBatch[i].NewId);
}
else
{
workBatch[i].ErrorMessage = insertRes[i].ErrorMessage;
logger.WarnFormat("Error when inserting {0} {1} into target: {2}",
objLoop.ApiName, workBatch[i].OriginalId, workBatch[i].ErrorMessage);
objres.FailCount += 1;
}
}
}
// inserts done.
// if there's a recurive field, do update
if (transformer.RecursiveRelationshipField != null)
{
UpdateRecursiveField(objLoop.ApiName, workingList, transformer.RecursiveRelationshipField);
}
ProgressUpdate(string.Format("Summary for {0}: Success {1} Fail {2}",
objLoop.ApiName, objres.SuccessCount, objres.FailCount));
processedObjects.Add(objLoop.ApiName);
}
// reprocess lookup relationships that can be populated now that the inserts are done
var reprocessingResults = ReprocessObjects(objectsToReprocess);
// log summary
ProgressUpdate("************************************************");
foreach (var resLoop in res.ObjectResults)
{
ProgressUpdate(string.Format("Summary for {0}: Success {1} Fail {2}",
resLoop.ApiName, resLoop.SuccessCount, resLoop.FailCount));
}
// log reprocssing summary
foreach (var resLoop in reprocessingResults.ObjectResults)
{
ProgressUpdate(string.Format("Reprocessing summary for {0}: Success {1} Fail {2}",
resLoop.ApiName, resLoop.SuccessCount, resLoop.FailCount));
}
ProgressUpdate("************************************************");
return(res);
}