/// <summary>
        /// Run the job
        /// </summary>
        /// <param name="job"></param>
        /// <returns></returns>
        public override bool Run()
        {
            var progressWriter = new MultiThreadFileWriter(ProgressFilePath);

            Logger.LogInformation($"Checking {ProgressFilePath} existence...");
            // Load already processed items from tracking file if exists
            var processedItems = new List <string>();

            if (File.Exists(ProgressFilePath))
            {
                Logger.LogInformation($"File {ProgressFilePath} detected! Continue process at it last state");

                var lines = File.ReadAllLines(ProgressFilePath);
                processedItems = lines.ToList();
            }
            else
            {
                Logger.LogInformation($"File {ProgressFilePath} not detected! Start process from 0");
            }

            var jobName = GetName();
            var query   = GetQuery(CallerId);

            query.PageInfo.Count = JobSettings.QueryRecordLimit;
            query.NoLock         = true;

            var results = ProxiesPool.MainProxy.RetrieveAll(query);

            Logger.LogInformation($"Retrieved {results.Entities.Count} records from CRM");
            var processedItemCount = 0;
            var stopwatch          = Stopwatch.StartNew();
            var data      = PrepareData(results.Entities);
            var dataCount = data.Count();

            Logger.LogInformation($"{dataCount} records to process");

            var threads             = (this.OverrideThreadNumber.HasValue) ? this.OverrideThreadNumber : JobSettings.ThreadNumber;
            var progressDisplayStep = (this.OverrideProgressDisplayStep.HasValue) ? this.OverrideProgressDisplayStep.Value : DefaultProgressDisplayStep;

            Parallel.ForEach(data,
                             new ParallelOptions()
            {
                MaxDegreeOfParallelism = threads.Value
            },
                             () =>
            {
                var proxy = ProxiesPool.GetProxy();
                return(new
                {
                    Proxy = proxy
                });
            },
                             (item, loopState, context) =>
            {
                var jobExecutionContext = new JobExecutionContext(context.Proxy, item);
                jobExecutionContext.PushMetrics(base.ContextProperties);

                // Increment progress index
                Interlocked.Increment(ref processedItemCount);

                // Increment progress bar every x records
                if (processedItemCount % progressDisplayStep == 0)
                {
                    Logger.LogInformation($"Processing record {processedItemCount} / {dataCount}");
                }

                // Exit if record has already been processed
                if (processedItems.Contains(item.Id.ToString()))
                {
                    return(context);
                }

                try
                {
                    ProcessRecord(jobExecutionContext);
                    Logger.LogSuccess("Record processed with success!", jobExecutionContext.DumpMetrics());

                    // Track job progress
                    progressWriter.Write(item.Id.ToString());
                }
                catch (FaultException <OrganizationServiceFault> faultException)
                {
                    var properties = jobExecutionContext.DumpMetrics().MergeWith(faultException.ExportProperties());
                    Logger.LogFailure(faultException, properties);
                }
                catch (Exception ex)
                {
                    Logger.LogFailure(ex, jobExecutionContext.DumpMetrics());
                }

                return(context);
            },
                             (context) =>
            {
                context.Proxy.Dispose();
            });

            stopwatch.Stop();
            var speed = Utilities.GetSpeed(stopwatch.Elapsed.TotalMilliseconds, results.Entities.Count);

            Logger.LogInformation($"{dataCount} records processed in {stopwatch.Elapsed.TotalSeconds} => {stopwatch.Elapsed:g} [Speed = {speed}]!");

            if (File.Exists(ProgressFilePath))
            {
                File.Delete(ProgressFilePath);
                Logger.LogInformation($"Progress file {ProgressFilePath} removed!");
            }

            return(true);
        }
Example #2
0
        /// <summary>
        /// Run the job
        /// </summary>
        /// <param name="job"></param>
        /// <returns></returns>
        public override bool Run()
        {
            var jobName = GetName();
            var defaultFileSeparator = GetInputFileSeparator().First();

            List <string> lines = new List <string>();

            // Check if pivot file exists
            if (File.Exists(GetPivotFilePath()))
            {
                var fileLines = File.ReadAllLines(GetPivotFilePath());
                lines = fileLines.ToList();

                Logger.LogInformation($"Retrieved {lines.Count} from file {GetPivotFilePath()}");
            }
            else
            {
                // Load file content
                var fileLines = File.ReadAllLines(GetInputFilePath());
                lines = fileLines.ToList();

                Logger.LogInformation($"Retrieved {lines.Count} from file {GetInputFilePath()}");

                // Create pivot file that track progress and outcome
                var header     = lines.First();
                var pivotLines = new List <string>()
                {
                    string.Concat(header, defaultFileSeparator, PivotUniqueMarker, defaultFileSeparator, "RecordId", defaultFileSeparator, "Outcome", defaultFileSeparator, "Details")
                };
                File.WriteAllLines(GetPivotFilePath(), pivotLines, Encoding.UTF8);
            }
            var pivotFileWriter = new MultiThreadFileWriter(GetPivotFilePath());

            var processedItemCount = 0;
            var stopwatch          = Stopwatch.StartNew();

            var threads = (this.OverrideThreadNumber.HasValue) ? this.OverrideThreadNumber : JobSettings.ThreadNumber;

            var linesToProcess = lines.Skip(1);

            Parallel.ForEach(
                linesToProcess,
                new ParallelOptions()
            {
                MaxDegreeOfParallelism = threads.Value
            },
                () =>
            {
                var proxy = ProxiesPool.GetProxy();
                return(new
                {
                    Proxy = proxy
                });
            },
                (line, loopState, context) =>
            {
                var jobExecutionContext = new JobExecutionContext(context.Proxy);
                jobExecutionContext.PushMetrics(base.ContextProperties);

                // Increment progress index
                Interlocked.Increment(ref processedItemCount);

                var isPivotLine = line.Contains(PivotUniqueMarker);
                if (isPivotLine)
                {
                    // TODO : Handle already processed pivot lines to replay errors
                    return(context);
                }
                Entity record = null;
                try
                {
                    var lineData = line.Split(GetInputFileSeparator(), StringSplitOptions.RemoveEmptyEntries);

                    // Retrieve CRM record based on current line
                    record = SearchRecord(context.Proxy, lineData);
                    jobExecutionContext.PushRecordToMetrics(record);
                    ProcessRecord(jobExecutionContext, lineData);
                    Logger.LogSuccess("Record processed with success!", jobExecutionContext.DumpMetrics());

                    // Track progress and outcome
                    var pivotLine = string.Concat(line,
                                                  defaultFileSeparator, PivotUniqueMarker,
                                                  defaultFileSeparator, record.Id.ToString() /* RecordId */,
                                                  defaultFileSeparator, "OK" /* Outcome */,
                                                  defaultFileSeparator, "Success" /*Details */);
                    pivotFileWriter.Write(pivotLine);
                }
                catch (FaultException <OrganizationServiceFault> faultException)
                {
                    var properties = jobExecutionContext.DumpMetrics().MergeWith(faultException.ExportProperties());
                    Logger.LogFailure(faultException, properties);

                    // Track progress and outcome
                    var pivotLine = string.Concat(line,
                                                  defaultFileSeparator, PivotUniqueMarker,
                                                  defaultFileSeparator, record?.Id.ToString() /* RecordId */,
                                                  defaultFileSeparator, "KO" /* Outcome */,
                                                  defaultFileSeparator, faultException.Message /*Details */);
                    pivotFileWriter.Write(pivotLine);
                }
                catch (Exception ex)
                {
                    Logger.LogFailure(ex, jobExecutionContext.DumpMetrics());

                    // Track progress and outcome
                    var pivotLine = string.Concat(line,
                                                  defaultFileSeparator, PivotUniqueMarker,
                                                  defaultFileSeparator, record?.Id.ToString() /* RecordId */,
                                                  defaultFileSeparator, "KO" /* Outcome */,
                                                  defaultFileSeparator, ex.Message /*Details */);
                    pivotFileWriter.Write(pivotLine);
                }

                return(context);
            },
                (context) =>
            {
                context.Proxy.Dispose();
            });
            stopwatch.Stop();
            var speed = Utilities.GetSpeed(stopwatch.Elapsed.TotalMilliseconds, lines.Count);

            Logger.LogInformation($"{lines.Count} records processed in {stopwatch.Elapsed.TotalSeconds} => {stopwatch.Elapsed.ToString("g")} [Speed = {speed}]!");

            return(true);
        }