/// <summary>
/// Run the job
/// </summary>
/// <param name="job"></param>
/// <returns></returns>
public override bool Run()
{
var progressWriter = new MultiThreadFileWriter(ProgressFilePath);
Logger.LogInformation($"Checking {ProgressFilePath} existence...");
// Load already processed items from tracking file if exists
var processedItems = new List <string>();
if (File.Exists(ProgressFilePath))
{
Logger.LogInformation($"File {ProgressFilePath} detected! Continue process at it last state");
var lines = File.ReadAllLines(ProgressFilePath);
processedItems = lines.ToList();
}
else
{
Logger.LogInformation($"File {ProgressFilePath} not detected! Start process from 0");
}
var jobName = GetName();
var query = GetQuery(CallerId);
query.PageInfo.Count = JobSettings.QueryRecordLimit;
query.NoLock = true;
var results = ProxiesPool.MainProxy.RetrieveAll(query);
Logger.LogInformation($"Retrieved {results.Entities.Count} records from CRM");
var processedItemCount = 0;
var stopwatch = Stopwatch.StartNew();
var data = PrepareData(results.Entities);
var dataCount = data.Count();
Logger.LogInformation($"{dataCount} records to process");
var threads = (this.OverrideThreadNumber.HasValue) ? this.OverrideThreadNumber : JobSettings.ThreadNumber;
var progressDisplayStep = (this.OverrideProgressDisplayStep.HasValue) ? this.OverrideProgressDisplayStep.Value : DefaultProgressDisplayStep;
Parallel.ForEach(data,
new ParallelOptions()
{
MaxDegreeOfParallelism = threads.Value
},
() =>
{
var proxy = ProxiesPool.GetProxy();
return(new
{
Proxy = proxy
});
},
(item, loopState, context) =>
{
var jobExecutionContext = new JobExecutionContext(context.Proxy, item);
jobExecutionContext.PushMetrics(base.ContextProperties);
// Increment progress index
Interlocked.Increment(ref processedItemCount);
// Increment progress bar every x records
if (processedItemCount % progressDisplayStep == 0)
{
Logger.LogInformation($"Processing record {processedItemCount} / {dataCount}");
}
// Exit if record has already been processed
if (processedItems.Contains(item.Id.ToString()))
{
return(context);
}
try
{
ProcessRecord(jobExecutionContext);
Logger.LogSuccess("Record processed with success!", jobExecutionContext.DumpMetrics());
// Track job progress
progressWriter.Write(item.Id.ToString());
}
catch (FaultException <OrganizationServiceFault> faultException)
{
var properties = jobExecutionContext.DumpMetrics().MergeWith(faultException.ExportProperties());
Logger.LogFailure(faultException, properties);
}
catch (Exception ex)
{
Logger.LogFailure(ex, jobExecutionContext.DumpMetrics());
}
return(context);
},
(context) =>
{
context.Proxy.Dispose();
});
stopwatch.Stop();
var speed = Utilities.GetSpeed(stopwatch.Elapsed.TotalMilliseconds, results.Entities.Count);
Logger.LogInformation($"{dataCount} records processed in {stopwatch.Elapsed.TotalSeconds} => {stopwatch.Elapsed:g} [Speed = {speed}]!");
if (File.Exists(ProgressFilePath))
{
File.Delete(ProgressFilePath);
Logger.LogInformation($"Progress file {ProgressFilePath} removed!");
}
return(true);
}
/// <summary>
/// Run the job
/// </summary>
/// <param name="job"></param>
/// <returns></returns>
public override bool Run()
{
var jobName = GetName();
var defaultFileSeparator = GetInputFileSeparator().First();
List <string> lines = new List <string>();
// Check if pivot file exists
if (File.Exists(GetPivotFilePath()))
{
var fileLines = File.ReadAllLines(GetPivotFilePath());
lines = fileLines.ToList();
Logger.LogInformation($"Retrieved {lines.Count} from file {GetPivotFilePath()}");
}
else
{
// Load file content
var fileLines = File.ReadAllLines(GetInputFilePath());
lines = fileLines.ToList();
Logger.LogInformation($"Retrieved {lines.Count} from file {GetInputFilePath()}");
// Create pivot file that track progress and outcome
var header = lines.First();
var pivotLines = new List <string>()
{
string.Concat(header, defaultFileSeparator, PivotUniqueMarker, defaultFileSeparator, "RecordId", defaultFileSeparator, "Outcome", defaultFileSeparator, "Details")
};
File.WriteAllLines(GetPivotFilePath(), pivotLines, Encoding.UTF8);
}
var pivotFileWriter = new MultiThreadFileWriter(GetPivotFilePath());
var processedItemCount = 0;
var stopwatch = Stopwatch.StartNew();
var threads = (this.OverrideThreadNumber.HasValue) ? this.OverrideThreadNumber : JobSettings.ThreadNumber;
var linesToProcess = lines.Skip(1);
Parallel.ForEach(
linesToProcess,
new ParallelOptions()
{
MaxDegreeOfParallelism = threads.Value
},
() =>
{
var proxy = ProxiesPool.GetProxy();
return(new
{
Proxy = proxy
});
},
(line, loopState, context) =>
{
var jobExecutionContext = new JobExecutionContext(context.Proxy);
jobExecutionContext.PushMetrics(base.ContextProperties);
// Increment progress index
Interlocked.Increment(ref processedItemCount);
var isPivotLine = line.Contains(PivotUniqueMarker);
if (isPivotLine)
{
// TODO : Handle already processed pivot lines to replay errors
return(context);
}
Entity record = null;
try
{
var lineData = line.Split(GetInputFileSeparator(), StringSplitOptions.RemoveEmptyEntries);
// Retrieve CRM record based on current line
record = SearchRecord(context.Proxy, lineData);
jobExecutionContext.PushRecordToMetrics(record);
ProcessRecord(jobExecutionContext, lineData);
Logger.LogSuccess("Record processed with success!", jobExecutionContext.DumpMetrics());
// Track progress and outcome
var pivotLine = string.Concat(line,
defaultFileSeparator, PivotUniqueMarker,
defaultFileSeparator, record.Id.ToString() /* RecordId */,
defaultFileSeparator, "OK" /* Outcome */,
defaultFileSeparator, "Success" /*Details */);
pivotFileWriter.Write(pivotLine);
}
catch (FaultException <OrganizationServiceFault> faultException)
{
var properties = jobExecutionContext.DumpMetrics().MergeWith(faultException.ExportProperties());
Logger.LogFailure(faultException, properties);
// Track progress and outcome
var pivotLine = string.Concat(line,
defaultFileSeparator, PivotUniqueMarker,
defaultFileSeparator, record?.Id.ToString() /* RecordId */,
defaultFileSeparator, "KO" /* Outcome */,
defaultFileSeparator, faultException.Message /*Details */);
pivotFileWriter.Write(pivotLine);
}
catch (Exception ex)
{
Logger.LogFailure(ex, jobExecutionContext.DumpMetrics());
// Track progress and outcome
var pivotLine = string.Concat(line,
defaultFileSeparator, PivotUniqueMarker,
defaultFileSeparator, record?.Id.ToString() /* RecordId */,
defaultFileSeparator, "KO" /* Outcome */,
defaultFileSeparator, ex.Message /*Details */);
pivotFileWriter.Write(pivotLine);
}
return(context);
},
(context) =>
{
context.Proxy.Dispose();
});
stopwatch.Stop();
var speed = Utilities.GetSpeed(stopwatch.Elapsed.TotalMilliseconds, lines.Count);
Logger.LogInformation($"{lines.Count} records processed in {stopwatch.Elapsed.TotalSeconds} => {stopwatch.Elapsed.ToString("g")} [Speed = {speed}]!");
return(true);
}