/// <summary> /// Run the job /// </summary> /// <param name="job"></param> /// <returns></returns> public override bool Run() { var progressWriter = new MultiThreadFileWriter(ProgressFilePath); Logger.LogInformation($"Checking {ProgressFilePath} existence..."); // Load already processed items from tracking file if exists var processedItems = new List <string>(); if (File.Exists(ProgressFilePath)) { Logger.LogInformation($"File {ProgressFilePath} detected! Continue process at it last state"); var lines = File.ReadAllLines(ProgressFilePath); processedItems = lines.ToList(); } else { Logger.LogInformation($"File {ProgressFilePath} not detected! Start process from 0"); } var jobName = GetName(); var query = GetQuery(CallerId); query.PageInfo.Count = JobSettings.QueryRecordLimit; query.NoLock = true; var results = ProxiesPool.MainProxy.RetrieveAll(query); Logger.LogInformation($"Retrieved {results.Entities.Count} records from CRM"); var processedItemCount = 0; var stopwatch = Stopwatch.StartNew(); var data = PrepareData(results.Entities); var dataCount = data.Count(); Logger.LogInformation($"{dataCount} records to process"); var threads = (this.OverrideThreadNumber.HasValue) ? this.OverrideThreadNumber : JobSettings.ThreadNumber; var progressDisplayStep = (this.OverrideProgressDisplayStep.HasValue) ? this.OverrideProgressDisplayStep.Value : DefaultProgressDisplayStep; Parallel.ForEach(data, new ParallelOptions() { MaxDegreeOfParallelism = threads.Value }, () => { var proxy = ProxiesPool.GetProxy(); return(new { Proxy = proxy }); }, (item, loopState, context) => { var jobExecutionContext = new JobExecutionContext(context.Proxy, item); jobExecutionContext.PushMetrics(base.ContextProperties); // Increment progress index Interlocked.Increment(ref processedItemCount); // Increment progress bar every x records if (processedItemCount % progressDisplayStep == 0) { Logger.LogInformation($"Processing record {processedItemCount} / {dataCount}"); } // Exit if record has already been processed if (processedItems.Contains(item.Id.ToString())) { return(context); } try { ProcessRecord(jobExecutionContext); Logger.LogSuccess("Record processed with success!", jobExecutionContext.DumpMetrics()); // Track job progress progressWriter.Write(item.Id.ToString()); } catch (FaultException <OrganizationServiceFault> faultException) { var properties = jobExecutionContext.DumpMetrics().MergeWith(faultException.ExportProperties()); Logger.LogFailure(faultException, properties); } catch (Exception ex) { Logger.LogFailure(ex, jobExecutionContext.DumpMetrics()); } return(context); }, (context) => { context.Proxy.Dispose(); }); stopwatch.Stop(); var speed = Utilities.GetSpeed(stopwatch.Elapsed.TotalMilliseconds, results.Entities.Count); Logger.LogInformation($"{dataCount} records processed in {stopwatch.Elapsed.TotalSeconds} => {stopwatch.Elapsed:g} [Speed = {speed}]!"); if (File.Exists(ProgressFilePath)) { File.Delete(ProgressFilePath); Logger.LogInformation($"Progress file {ProgressFilePath} removed!"); } return(true); }
/// <summary> /// Run the job /// </summary> /// <param name="job"></param> /// <returns></returns> public override bool Run() { var jobName = GetName(); var defaultFileSeparator = GetInputFileSeparator().First(); List <string> lines = new List <string>(); // Check if pivot file exists if (File.Exists(GetPivotFilePath())) { var fileLines = File.ReadAllLines(GetPivotFilePath()); lines = fileLines.ToList(); Logger.LogInformation($"Retrieved {lines.Count} from file {GetPivotFilePath()}"); } else { // Load file content var fileLines = File.ReadAllLines(GetInputFilePath()); lines = fileLines.ToList(); Logger.LogInformation($"Retrieved {lines.Count} from file {GetInputFilePath()}"); // Create pivot file that track progress and outcome var header = lines.First(); var pivotLines = new List <string>() { string.Concat(header, defaultFileSeparator, PivotUniqueMarker, defaultFileSeparator, "RecordId", defaultFileSeparator, "Outcome", defaultFileSeparator, "Details") }; File.WriteAllLines(GetPivotFilePath(), pivotLines, Encoding.UTF8); } var pivotFileWriter = new MultiThreadFileWriter(GetPivotFilePath()); var processedItemCount = 0; var stopwatch = Stopwatch.StartNew(); var threads = (this.OverrideThreadNumber.HasValue) ? this.OverrideThreadNumber : JobSettings.ThreadNumber; var linesToProcess = lines.Skip(1); Parallel.ForEach( linesToProcess, new ParallelOptions() { MaxDegreeOfParallelism = threads.Value }, () => { var proxy = ProxiesPool.GetProxy(); return(new { Proxy = proxy }); }, (line, loopState, context) => { var jobExecutionContext = new JobExecutionContext(context.Proxy); jobExecutionContext.PushMetrics(base.ContextProperties); // Increment progress index Interlocked.Increment(ref processedItemCount); var isPivotLine = line.Contains(PivotUniqueMarker); if (isPivotLine) { // TODO : Handle already processed pivot lines to replay errors return(context); } Entity record = null; try { var lineData = line.Split(GetInputFileSeparator(), StringSplitOptions.RemoveEmptyEntries); // Retrieve CRM record based on current line record = SearchRecord(context.Proxy, lineData); jobExecutionContext.PushRecordToMetrics(record); ProcessRecord(jobExecutionContext, lineData); Logger.LogSuccess("Record processed with success!", jobExecutionContext.DumpMetrics()); // Track progress and outcome var pivotLine = string.Concat(line, defaultFileSeparator, PivotUniqueMarker, defaultFileSeparator, record.Id.ToString() /* RecordId */, defaultFileSeparator, "OK" /* Outcome */, defaultFileSeparator, "Success" /*Details */); pivotFileWriter.Write(pivotLine); } catch (FaultException <OrganizationServiceFault> faultException) { var properties = jobExecutionContext.DumpMetrics().MergeWith(faultException.ExportProperties()); Logger.LogFailure(faultException, properties); // Track progress and outcome var pivotLine = string.Concat(line, defaultFileSeparator, PivotUniqueMarker, defaultFileSeparator, record?.Id.ToString() /* RecordId */, defaultFileSeparator, "KO" /* Outcome */, defaultFileSeparator, faultException.Message /*Details */); pivotFileWriter.Write(pivotLine); } catch (Exception ex) { Logger.LogFailure(ex, jobExecutionContext.DumpMetrics()); // Track progress and outcome var pivotLine = string.Concat(line, defaultFileSeparator, PivotUniqueMarker, defaultFileSeparator, record?.Id.ToString() /* RecordId */, defaultFileSeparator, "KO" /* Outcome */, defaultFileSeparator, ex.Message /*Details */); pivotFileWriter.Write(pivotLine); } return(context); }, (context) => { context.Proxy.Dispose(); }); stopwatch.Stop(); var speed = Utilities.GetSpeed(stopwatch.Elapsed.TotalMilliseconds, lines.Count); Logger.LogInformation($"{lines.Count} records processed in {stopwatch.Elapsed.TotalSeconds} => {stopwatch.Elapsed.ToString("g")} [Speed = {speed}]!"); return(true); }