public IGeneratedRecord Next(Type spec, uint seqNo) { var aNewRec = Activator.CreateInstance(spec); spec.GetProperties(BuiltIns.SpecPropertyFlags).ToList().ForEach(p => { // build the options and handle any special attrs for this field var gOpts = GeneratorsStatic.ParseGeneratorOptionPairs(p); gOpts.Config = _config; var fieldGen = GeneratorsStatic.DefaultFieldGeneratorTypes[p.PropertyType]; var isSeqNo = false; FieldLinkerTypeAttribute linker = null; p.GetCustomAttributes(false).ToList().ForEach(cAttr => { if (cAttr is GeneratorTypeAttribute) { fieldGen = ((GeneratorTypeAttribute)cAttr).Type; } else if (cAttr is SequenceNumberAttribute) { isSeqNo = true; } else if (cAttr is FieldLinkerTypeAttribute) { linker = (FieldLinkerTypeAttribute)cAttr; } }); // verify the field generator options, specifically that: // - SequenceNumberAttribute can only be applied to 'uint' and 'ulong' fields // - SequenceNumberAttribute and FieldLinkerTypeAttribute cannot coexist on a field if (isSeqNo && p.PropertyType != typeof(uint) && p.PropertyType != typeof(ulong)) { throw new InvalidOperationException($"SequenceNumberAttribute applied to field {p}, " + $"which is of type {p.PropertyType}: it must be 'uint' or 'ulong'"); } if (isSeqNo && linker != null) { throw new InvalidProgramException("Sequence number field cannot be also be linked"); } // generate the new field by (in order): // - linking it to another field if specified, or // - using the current sequence number if specified, or // - make blank if specified and the random choice comes up positive, or // - generate using the specified field generator and options. object generated; if (linker != null) { var linkedField = spec.GetProperty(linker.LinkedFieldName, BuiltIns.SpecPropertyFlags); var linkerInst = (IFieldLinker)Activator.CreateInstance(linker.LinkerType); generated = linkerInst.LinkField(linkedField.GetValue(aNewRec), ref gOpts); } else if (isSeqNo) { generated = seqNo; } else if (gOpts.BlankFrequency != FieldGeneratorOptions.BlankFrequencyDisabled && new Random().Next(100) <= gOpts.BlankFrequency) { generated = null; } else { try { generated = GeneratorsStatic.GeneratorCache[fieldGen].GenerateField(ref gOpts); } catch (Exception ex) { Console.WriteLine($"GenerateField for '{p.Name}' failed at seqNo {seqNo} and spec {spec}: {ex.Message}"); throw; } } p.SetValue(aNewRec, generated); }); return((IGeneratedRecord)aNewRec); }
public override bool Execute() { if (Status != PipelineStatus.Pending) { return(false); } _start = DateTime.UtcNow; Console.WriteLine($"{this}: Executing pipeline {Id} at {_start}"); Status = PipelineStatus.Executing; Artifacts = new List <SinkStageArtifact>(); Results = new List <PersistenceStageResult>(); GeneratorsStatic.ThreadInit(); _source = new GeneratorSource(ref _config); // One day, we'll support multi-spec generation... Type iSpec = _specs.First(); var transformStageSw = new Stopwatch(); transformStageSw.Start(); // for each spec, run the pipeline once for each record // to be generated _sinks.ForEach(sink => sink.Prepare()); for (_seqNo = 0; !_canceled && _seqNo < _config.Count; _seqNo++) { // source stage IGeneratedRecord nextSrc; try { nextSrc = _source.Next(iSpec, _seqNo); } catch (Exception e) { Console.WriteLine($"{this}: Source-stage failure at seqNo={_seqNo} of {_config.Count}:\n{e}"); continue; } // intermediate stage _filters.ForEach(filter => { if (filter.Prepare() && filter.Sink(nextSrc)) { nextSrc = filter.Next(iSpec, _seqNo); } filter.Finish(); }); // sink stage _sinks.ForEach(sink => sink.Sink(nextSrc)); } transformStageSw.Stop(); Console.WriteLine($"{this}: Transform stage took {transformStageSw.Elapsed}"); if (!_canceled) { // persist stage var persistStageSw = new Stopwatch(); persistStageSw.Start(); Status = PipelineStatus.Persisting; // capture elapsed time to this point for persisters' use var extraMeta = new Dictionary <string, object>() { { "tranformStageElapsed", Elapsed } }; _sinks.ForEach(sink => { var artifact = sink.Finish(); if (artifact != null) { Artifacts.Add(artifact); _persisters.ForEach(p => { Results.Add(p.Persist(artifact, extraMeta)); if (artifact.ByteStream != null && artifact.ByteStream.CanSeek) { artifact.ByteStream.Seek(0, SeekOrigin.Begin); } }); } }); Artifacts.ForEach(a => a.Cleanup()); persistStageSw.Stop(); Console.WriteLine($"{this}: Persist stage took {persistStageSw.Elapsed}"); } else { _sinks.Select(sink => sink.Finish()).ToList().ForEach(art => art?.Cleanup()); } Status = _canceled ? PipelineStatus.Canceled : (_seqNo < _config.Count ? PipelineStatus.Failed : PipelineStatus.Success); _end = DateTime.UtcNow; if (CompletionHandler != null) { CompletionHandler(this); } return(Status == PipelineStatus.Success); }