internal FileProcessor(string inFile, string outLocation)
{
var config = new OrchestratorConfig()
{
ReportProgress = true,
ProgressInterval = 1000,
ProgressChangedHandler = (s, e) => { if (e.Phase == Phase.Intake)
{
Console.Write($"\rProcessed {e.RecCnt:N0} records so far...");
}
},
PhaseFinishedHandler = (s, e) => { if (e.Phase == Phase.Intake)
{
Console.WriteLine($"\rProcessed {e.RecCnt:N0} records. Done! ");
}
},
InputDataKind = KindOfTextData.Delimited,
HeadersInFirstInputRow = true,
InputFileName = inFile,
TransformerType = TransformerType.RecordFilter,
RecordFilterPredicate = r => (string)r["NPPES Provider State"] == "NJ" && ((string)r["Specialty Description"]).ToLower() == "dentist",
OutputDataKind = KindOfTextData.Delimited,
HeadersInFirstOutputRow = true,
OutputFileName = outLocation + Path.DirectorySeparatorChar + Path.GetFileNameWithoutExtension(inFile) + "_NJ_dentists.csv"
};
Orchestrator = OrchestratorCreator.GetEtlOrchestrator(config);
}
internal FileProcessor(string inFile, string outLocation)
{
var outFileBody = outLocation + Path.DirectorySeparatorChar + Path.GetFileNameWithoutExtension(inFile);
var outFileExt = Path.GetExtension(inFile);
var config = new OrchestratorConfig()
{
GlobalCacheElements = new string[] { "LowCnt|0", "HighCnt|0", "AllCnt|0", "IsaElems", "GsElems" },
DefaultX12SegmentDelimiter = "~\r\n",
InputDataKind = KindOfTextData.X12,
InputFileName = inFile,
ClusterMarker = SegmentStartsCluster,
MarkerStartsCluster = true, //predicate (marker) matches the first record in cluster
PrependHeadCluster = true, // to contain ISA/GS segments for _high file
AppendFootCluster = true, // to contain IEA/GE segments for _high file
RecordInitiator = StoreIsaAndGsSegments,
PropertyBinEntities = PropertyBinAttachedTo.Clusters,
DeferTransformation = DeferTransformation.UntilRecordInitiation,
ConcurrencyLevel = 4,
TransformerType = TransformerType.Clusterbound,
ClusterboundTransformer = ProcessX12Transaction,
RouterType = RouterType.PerCluster,
ClusterRouter = SendToLowOrHigh,
OutputDataKind = KindOfTextData.X12,
OutputFileNames = outFileBody + "_low" + outFileExt + "|" + outFileBody + "_high" + outFileExt //1st: less than $1,000; 2nd: $1,000 or more
};
Orchestrator = OrchestratorCreator.GetEtlOrchestrator(config);
}
public OrchestratorController(IViewStartup view, IOrchestrator orchestrator, IRouter router)
{
_router = router;
_orchestrator = orchestrator;
_view = view;
_view.SetController(this);
}
public DataSampling(IOrchestrator proxy)
{
_sample = new Queue <Temperature>();
proxy.Training.Subscribe(this, async signal =>
{
Reference <Sample> message = null;
lock (_sample)
{
_sample.Enqueue(signal);
if (_sample.Count >= _windowMaxSamples)
{
message = new Reference <Sample>()
{
Message = new Sample()
{
Data = _sample.ToArray()
}
};
for (int i = 0; i < _maxDelayPercentage * _windowMaxSamples / 100; i++)
{
_sample.Dequeue();
}
}
}
if (message != null)
{
await Samples.PublishAsync(message);
}
return(MessageResult.Ok);
});
}
static Program()
{
var container = new Container();
DependencyInjection.RegisterServices(container);
Orchestrator = container.GetInstance <IOrchestrator>();
}
internal DataProcessor(Func <Task <string> > intakeSupplierAsync, Action <string> outputConsumer, Action <int> progressHandler)
{
var config = new OrchestratorConfig()
{
ReportProgress = true,
ProgressInterval = 10,
ProgressChangedHandler = (s, e) => { if (e.Phase == Phase.Intake)
{
progressHandler(e.RecCnt);
}
},
PhaseFinishedHandler = (s, e) => { if (e.Phase == Phase.Intake)
{
progressHandler(e.RecCnt);
}
},
InputDataKind = KindOfTextData.Delimited,
InputFields = "PlaneDescription,IataCode,IcaoCode",
AsyncIntake = true,
TransformerType = TransformerType.Universal,
UniversalTransformer = FilterAndReorganizeFields,
AllowTransformToAlterFields = true,
OutputDataKind = KindOfTextData.Flat,
OutputFields = "IataCode|4,Hyphen|2,PlaneDescription|70",
ExcludeExtraneousFields = true
};
config.SetAsyncIntakeSupplier(intakeSupplierAsync);
config.SetOutputConsumer(outputConsumer);
Orchestrator = OrchestratorCreator.GetEtlOrchestrator(config);
}
/// <summary>
/// Add or Change Orchestrator
/// </summary>
/// <typeparam name="T">the orchestrator type</typeparam>
/// <param name="orchestrator">the Orchestrator to add/change</param>
/// <returns>this Orchestration (Fluent API)</returns>
public IOrchestration SetOrchestrator <T>(IOrchestrator <T> orchestrator)
{
if (string.IsNullOrEmpty(orchestrator.Name))
{
throw new ArgumentNullException("Orchestator.Name", "Orchestator name cannot be empty or null");
}
Type typKey = typeof(T);
if (!Orchestrators.ContainsKey(typKey))
{
Orchestrators.Add(typKey, orchestrator);
Variables.SetVariable(orchestrator.Name, true);
orchestrator.Start();
}
else
{
var orch = Orchestrators[typKey] as IOrchestrator <T>;
orch.Stop();
orch.Dispose();
Orchestrators[typKey] = orchestrator;
}
return(this);
}
internal ReportingOrchestratorDecorator(
IOrchestrator decorated,
Report report)
{
_decorated = decorated;
_report = report;
}
public Visualization(IOrchestrator proxy)
{
proxy.Visualization.Subscribe(this, async(e) =>
{
await AddSampleAsync(e);
return(MessageResult.Ok);
});
}
public MainWindow()
{
_playState = false;
_startExecution = false;
_videoLoaded = false;
InitializeComponent();
_Orchestrator = new BasicOrchestrator(Settings.Default.SamplingRate);
_Runtime = new RuntimeVisualization.RuntimeWindow();
}
public Ingestor(StatefulServiceContext context, IRouter router)
: base(context)
{
this._proxyFactory = new ServiceProxyFactory((c) =>
{
return(new FabricTransportServiceRemotingClientFactory());
});
orchestratorClient = _proxyFactory.CreateServiceProxy <IOrchestrator>(new Uri(Common.Names.OrchestratorServiceUri), new ServicePartitionKey(1));
this._router = router;
}
public ModelTraining(IOrchestrator proxy)
{
var sample = new Queue <Temperature>();
Twin.Subscribe(async twin =>
{
Logger.LogInformation("Twin update");
lock (_sync)
{
_aggregationSize = twin.AggregationSize;
_tumblingWindowPercentage = twin.TumblingWindowPercentage;
}
await Twin.ReportAsync(twin);
return(TwinResult.Ok);
});
proxy.Training.Subscribe(this, async signal =>
{
Model model = null;
lock (sample)
{
sample.Enqueue(signal);
if (sample.Count >= _aggregationSize)
{
var kMeansTraining = new KMeansTraining(_numClusters);
kMeansTraining.TrainModel(sample.Select(e => new[] { e.TimeStamp, e.Value }).ToArray());
model = new Model
{
Algorithm = Algorithm.kMeans,
DataJson = kMeansTraining.SerializeModel()
};
for (var i = 0; i < _tumblingWindowPercentage * _aggregationSize / 100; i++)
{
sample.Dequeue();
}
}
}
if (model != null)
{
await Model.PublishAsync(model).ConfigureAwait(false);
}
return(MessageResult.Ok);
});
}
public GetWorkOrderQueryHandler(ILogAs logAs,
IMapper mapper,
IOrchestrator orchestrator,
IWorkOrderReadRepository workOrderReadRepository,
IWorkOrderWriteRepository workOrderWriteRepository,
IPlatoOrderProvider platoOrderProvider,
IDomainConverter domainConverter)
{
_logAs = logAs ?? throw Error.ArgumentNull(nameof(logAs));
_mapper = mapper ?? throw Error.ArgumentNull(nameof(mapper));
_orchestrator = orchestrator ?? throw Error.ArgumentNull(nameof(orchestrator));
_workOrderReadRepository = workOrderReadRepository ?? throw Error.ArgumentNull(nameof(workOrderReadRepository));
_workOrderWriteRepository = workOrderWriteRepository ?? throw Error.ArgumentNull(nameof(workOrderWriteRepository));
_platoOrderProvider = platoOrderProvider ?? throw Error.ArgumentNull(nameof(platoOrderProvider));
_domainConverter = domainConverter ?? throw Error.ArgumentNull(nameof(domainConverter));
}
public DataSampling(IOrchestrator proxy)
{
Twin.Subscribe(async twin =>
{
lock (_sync)
{
_aggregationSize = twin.AggregationSize;
_maxDelayPercentage = twin.MaxDelayPercentage;
}
await Twin.ReportAsync(twin);
return(TwinResult.Ok);
});
_sample = new Queue <Temperature>();
proxy.Sampling.Subscribe(this, async signal =>
{
Reference <DataAggregate> message = null;
lock (_sample)
{
_sample.Enqueue(signal);
if (_sample.Count >= _aggregationSize)
{
message = new Reference <DataAggregate>()
{
Message = new DataAggregate()
{
Values = _sample.Select(e => new double[2] {
e.TimeStamp, e.Value
}).ToArray(),
CorrelationID = "IOrchestrator.Sampling"
}
};
for (int i = 0; i < _maxDelayPercentage * _aggregationSize / 100; i++)
{
_sample.Dequeue();
}
}
}
if (message != null)
{
await Aggregate.PublishAsync(message);
}
return(MessageResult.Ok);
});
}
public void ConstructorShouldFailWhenOrchestratorIsNull()
{
// Arrange
var logAs = new Mock <ILogAs>().Object;
IMapper mapper = new Mock <IMapper>().Object;
IOrchestrator orchestrator = null;
var workOrderReadrepository = new Mock <IWorkOrderReadRepository>().Object;
var workOrderWriteRepository = new Mock <IWorkOrderWriteRepository>().Object;
var platoOrderProvider = new Mock <IPlatoOrderProvider>().Object;
var domainConverter = new Mock <IDomainConverter>().Object;
// Act
Action ctor = () => { new GetWorkOrderQueryHandler(logAs, mapper, orchestrator, workOrderReadrepository, workOrderWriteRepository, platoOrderProvider, domainConverter); };
// Assert
ctor.Should().Throw <ArgumentNullException>();
}
private void GenerateWatch(IOrchestrator orchestrator, LeprechaunConfigurationBuilder configuration)
{
try
{
var metadata = GenerateMetadata(orchestrator, configuration);
GenerateCode(metadata);
}
catch (Exception ex)
{
// during watch we don't want errors to terminate the application
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine(ex.Message);
Console.ForegroundColor = ConsoleColor.Gray;
Console.WriteLine(ex.StackTrace);
Console.ResetColor();
}
}
private IReadOnlyList <ConfigurationCodeGenerationMetadata> GenerateMetadata(IOrchestrator orchestrator, LeprechaunConfigurationBuilder configuration)
{
var metadataTimer = new Stopwatch();
metadataTimer.Start();
// we generate template data that will feed code generation
var metadata = orchestrator.GenerateMetadata(configuration.Configurations);
metadataTimer.Stop();
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine(
$"Loaded metadata for {metadata.Count} configurations ({metadata.Sum(m => m.Metadata.Count)} total templates) in {metadataTimer.ElapsedMilliseconds}ms.");
Console.ResetColor();
return(metadata);
}
public ModelTraining(IOrchestrator proxy)
{
_sample = new Queue <Temperature>();
Twin.Subscribe(async twin =>
{
Console.WriteLine($"{typeof(ModelTraining).Name}::Twin update");
lock (_sync)
{
_aggregationSize = twin.AggregationSize;
_tumblingWindowPercentage = twin.TumblingWindowPercentage;
}
await Twin.ReportAsync(twin);
return(TwinResult.Ok);
});
proxy.Training.Subscribe(this, async signal =>
{
Model model = null;
lock (_sample)
{
_sample.Enqueue(signal);
if (_sample.Count >= _aggregationSize)
{
_kMeansTraining = new KMeansTraining(_numClusters);
_kMeansTraining.TrainModel(_sample.Select(e => new double[] { e.TimeStamp, e.Value }).ToArray());
model = new Model()
{
Algorithm = ThermostatApplication.Algorithm.kMeans,
DataJson = _kMeansTraining.SerializeModel()
};
for (int i = 0; i < _tumblingWindowPercentage * _aggregationSize / 100; i++)
{
_sample.Dequeue();
}
}
}
if (model != null)
{
await Model.PublishAsync(model).ConfigureAwait(false);
}
return(MessageResult.Ok);
});
}
public AnomalyDetection(IOrchestrator preprocessor, IDataSampling trainer)
{
preprocessor.Detection.Subscribe(this, async signal =>
{
int cluster = 0;
if (_kMeansClustering != null)
{
lock (_syncClustering)
if (_kMeansClustering != null)
{
cluster = _kMeansClustering.Classify(new double[] { signal.Value, signal.Minimum, signal.Maximum });
}
}
if (cluster < 0)
{
System.Console.WriteLine("_____________________________Anomaly detected__________________________________");
await Anomaly.PublishAsync(new Anomaly()
{
Temperature = signal
});
}
return(MessageResult.Ok);
});
trainer.Samples.Subscribe(this, async(sampleReference) =>
{
//if the messages has been stored and forwarded, but the file has been deleted (e.g. a restart)
//then the message can be empty (null)
if (sampleReference == null)
{
return(MessageResult.Ok);
}
System.Console.WriteLine(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
lock (_syncSample)
_sample = sampleReference.Message.Data.Select(e => new double[] { e.Value, e.Minimum, e.Maximum }).ToArray();
lock (_syncClustering)
_kMeansClustering = new KMeansClustering(_sample, _numClusters);
return(MessageResult.Ok);
});
}
public AnomalyDetection(IOrchestrator orcherstratorProxy, IDataAggregator aggregatorProxy)
{
orcherstratorProxy.Detection.Subscribe(this, async signal =>
{
int cluster = 0;
if (_kMeansClustering != null)
{
lock (_syncClustering)
if (_kMeansClustering != null)
{
cluster = _kMeansClustering.Classify(new double[] { signal.Value });
}
}
if (cluster < 0)
{
System.Console.WriteLine("_____________________________Anomaly detected__________________________________");
await Anomaly.PublishAsync(new Anomaly()
{
Temperature = signal
});
}
return(MessageResult.Ok);
});
aggregatorProxy.Aggregate.Subscribe(this, async(sampleReference) =>
{
//if the messages has been stored and forwarded, but the file has been deleted (e.g. a restart)
//then the message can be empty (null)
if (sampleReference == null)
{
return(MessageResult.Ok);
}
System.Console.WriteLine(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
lock (_syncSample)
_sample = sampleReference.Message.Values;
lock (_syncClustering)
_kMeansClustering = new KMeansClustering(_sample, _numClusters);
return(MessageResult.Ok);
});
}
public Visualization(IOrchestrator proxy)
{
_chartDataDictionary = new Dictionary <string, Chart>();
proxy.Visualization.Subscribe(this, async(e) =>
{
await RenderAsync(e);
return(MessageResult.Ok);
});
Twin.Subscribe(async twin =>
{
Console.WriteLine($"{typeof(Visualization).Name}::Twin update");
ConfigureCharts(twin);
return(TwinResult.Ok);
});
}
internal FileProcessor(string inFile, string outLocation)
{
var config = new OrchestratorConfig()
{
InputDataKind = KindOfTextData.Delimited,
InputFields = "PlaneDescription,IataCode,IcaoCode",
InputFileName = inFile,
TransformerType = TransformerType.Universal,
UniversalTransformer = FilterAndReorganizeFields,
AllowTransformToAlterFields = true,
OutputDataKind = KindOfTextData.Flat,
OutputFields = "IataCode|4,Hyphen|2,PlaneDescription|70",
ExcludeExtraneousFields = true,
OutputFileName = outLocation + Path.DirectorySeparatorChar + Path.GetFileNameWithoutExtension(inFile) + ".txt"
};
Orchestrator = OrchestratorCreator.GetEtlOrchestrator(config);
}
internal FileProcessor(string inFile, string outLocation)
{
var inputIsCsv = Path.GetExtension(inFile).ToLower() == ".csv";
var outFileWithoutExtension = outLocation + Path.DirectorySeparatorChar + Path.GetFileNameWithoutExtension(inFile);
var config = new OrchestratorConfig(LoggerCreator.CreateLogger(LoggerType.LogFile, "Unbound JSON translation to/from CSV.", LogEntrySeverity.Information))
{
ReportProgress = true,
ProgressInterval = 1000,
ProgressChangedHandler = (s, e) => { if (e.Phase == Phase.Intake)
{
Console.Write($"\rProcessed {e.RecCnt:N0} records so far...");
}
},
PhaseFinishedHandler = (s, e) => { if (e.Phase == Phase.Intake)
{
Console.WriteLine($"\rProcessed {e.RecCnt:N0} records. Done! ");
}
},
InputFileName = inFile
};
if (inputIsCsv) // CSV to UnboundJSON
{
config.InputDataKind = KindOfTextData.Delimited;
config.HeadersInFirstInputRow = true;
config.AllowTransformToAlterFields = true;
config.OutputDataKind = KindOfTextData.UnboundJSON;
config.XmlJsonOutputSettings = "IndentChars| "; // pretty print
config.OutputFileName = outFileWithoutExtension + ".json";
}
else // UnboundJSON to CSV
{
config.InputDataKind = KindOfTextData.UnboundJSON;
config.AllowOnTheFlyInputFields = true; // TODO: consider UnboundJSON ignoring this setting like X12
config.AllowTransformToAlterFields = true; //IMPORTANT! otherwise null items will be produced!
config.OutputDataKind = KindOfTextData.Delimited;
config.OutputFileName = outFileWithoutExtension + ".csv";
config.HeadersInFirstOutputRow = true;
}
Orchestrator = OrchestratorCreator.GetEtlOrchestrator(config);
}
public Visualization(IOrchestrator proxy, IConfigurationRoot configuration)
{
_configuration = configuration;
_chartDataDictionary = new Dictionary <string, Chart>();
proxy.Visualization.Subscribe(this, async e =>
{
await RenderAsync(e);
return(MessageResult.Ok);
});
Twin.Subscribe(twin =>
{
Logger.LogInformation("Twin update");
ConfigureCharts(twin);
return(Task.FromResult(TwinResult.Ok));
});
}
public T CreateOrchestrator <T>(ProviderType providerType, string dbName, bool useChangeTracking = false) where T : IOrchestrator
{
// Get connection string
var cs = HelperDatabase.GetConnectionString(providerType, dbName);
IOrchestrator orchestrator = null;
if (typeof(T) == typeof(RemoteOrchestrator))
{
orchestrator = new RemoteOrchestrator();
}
else if (typeof(T) == typeof(LocalOrchestrator))
{
orchestrator = new LocalOrchestrator();
}
else if (typeof(T) == typeof(WebServerOrchestrator))
{
orchestrator = new WebServerOrchestrator();
}
if (orchestrator == null)
{
throw new Exception("Orchestrator does not exists");
}
switch (providerType)
{
case ProviderType.Sql:
orchestrator.Provider = useChangeTracking ? new SqlSyncChangeTrackingProvider(cs) : new SqlSyncProvider(cs);
break;
case ProviderType.MySql:
orchestrator.Provider = new MySqlSyncProvider(cs);
break;
case ProviderType.Sqlite:
orchestrator.Provider = new SqliteSyncProvider(cs);
break;
}
return((T)orchestrator);
}
internal FileProcessor(string inFile, string outLocation)
{
var config = new OrchestratorConfig()
{
InputDataKind = KindOfTextData.Flat,
InputFileName = inFile,
HeadersInFirstInputRow = true,
TrimInputValues = true,
InputFields = "|8,|15,|34,|41,|12,|20,|10,|48,|64,|16,|29,|23,|5,|5,|14,|38,|36,|52,|11",
AllowTransformToAlterFields = true,
ConcurrencyLevel = 4,
TransformerType = TransformerType.Universal,
UniversalTransformer = FilterRecsAndExtractFields,
OutputDataKind = KindOfTextData.Arbitrary,
ArbitraryOutputDefs = new string[] { "INSERT INTO MyPeople VALUES('{Key}', ", "'{LName}', ", "'{FName}', ", "'{SSN}')" },
LeaderContents = "CREATE TABLE MyPeople (ID char(12), LastName char(20), FirstName char(12), SSN char(11))\r\nGO",
OutputFileName = outLocation + Path.DirectorySeparatorChar + Path.GetFileNameWithoutExtension(inFile) + ".sql"
};
Orchestrator = OrchestratorCreator.GetEtlOrchestrator(config);
}
public RunnableProjectOrchestrator(IOrchestrator basicOrchestrator)
{
_basicOrchestrator = basicOrchestrator;
}
public BookController(IOrchestrator orquestrator)
{
_orquestrator = orquestrator;
}
public void Initialize()
{
_target = new Orchestrator();
}
public ProjectionFunction(IOrchestrator orchestrator)
{
_orchestrator = orchestrator;
}
