public override void Open(IRFProcessingContext context)
{
switch (SiteType)
{
case RFFTPSiteType.SFTP:
if (!string.IsNullOrWhiteSpace(ConnectionKeyPath))
{
if (Password.IsBlank())
{
_connection = new SFTPConnection(Hostname, Port, Username, ConnectionKeyPath, ConnectionKeyPassword);
}
else
{
_connection = new SFTPConnection(Hostname, Port, Username, Password, ConnectionKeyPath, ConnectionKeyPassword);
}
}
else
{
_connection = new SFTPConnection(Hostname, Port, Username, Password);
}
break;
case RFFTPSiteType.FTP:
_connection = new FTPConnection(Hostname, Port, Username, Password);
break;
default:
throw new RFSystemException(this, "Unknown FTP site type {0}", SiteType);
}
}
public RFProcessingResult Process(RFInstruction i, IRFProcessingContext processingContext)
{
try
{
if (i is RFProcessInstruction)
{
var pi = i as RFProcessInstruction;
if (_processes.ContainsKey(pi.ProcessName))
{
var process = _processes[pi.ProcessName];
return(ProcessInstruction(process, i as RFProcessInstruction, processingContext));
}
else
{
var msg = String.Format("Process {0} referenced by instruction {1} not found in engine configuration.", pi.ProcessName, i);
Log.Error(this, msg);
return(RFProcessingResult.Error(new string[] { msg }, false));
}
}
}
catch (Exception ex)
{
Log.Exception(this, ex, "Exception processing instruction {0}", i);
return(RFProcessingResult.Error(new string[] { ex.Message }, false));
}
return(new RFProcessingResult());
}
public static long ImportCatalogUpdates(IRFProcessingContext context, string path)
{
long c = 0;
foreach (var f in Directory.GetFiles(path, "*.zip").OrderBy(f => f))
{
context.SystemLog.Info(typeof(RFCatalogMaintainer), "Importing updates from {0}", f);
using (var fs = new FileStream(f, FileMode.Open, FileAccess.Read))
{
try
{
foreach (var entry in ZIPUtils.UnzipArchive(fs))
{
try
{
var document = RFXMLSerializer.DeserializeContract(typeof(RFDocument).FullName, new string(Encoding.UTF8.GetChars(entry.Item2))) as RFDocument;
if (document != null && context.SaveEntry(document, false, false))
{
c++;
}
}
catch (Exception ex)
{
context.SystemLog.Error(typeof(RFCatalogMaintainer), "Error importing entry {0}: {1}", entry.Item1, ex.Message);
}
}
}
catch (Exception ex)
{
context.SystemLog.Error(typeof(RFCatalogMaintainer), "Error processing .zip file {0}: {1}", f, ex.Message);
}
}
}
return(c);
}
public static bool IsPresentationMode(IRFProcessingContext context, string username)
{
username = username.ToLower();
var preferences = GetPresentationPreferences(context);
return(preferences.Active.Contains(username));
}
public static long ExportCatalogUpdates(IRFProcessingContext context, string path, RFDate startDate, RFDate?endDate = null, string password = null)
{
long c = 0;
var keysInScope = context.SearchKeys(typeof(RFCatalogKey), startDate, endDate, 99999, null, false).Where(k => k.IsValid && k.Key.Plane == RFPlane.User).ToList();
foreach (var keyDate in keysInScope.GroupBy(k => k.UpdateTime.Date))
{
var fileName = Path.Combine(path, String.Format("RIFF_{0}_Updates_{1}.zip", context.Environment, keyDate.Key.ToString("yyyyMMdd")));
context.SystemLog.Info(typeof(RFCatalogMaintainer), "Exporting {0} documents into {1}", keyDate.Count(), fileName);
var exportableDocuments = new Dictionary <string, byte[]>();
long cnt = 1;
foreach (var key in keyDate)
{
var doc = context.LoadEntry(key.Key) as RFDocument;
if (doc != null)
{
var docName = RFFileHelpers.SanitizeFileName(string.Format("{0}_{1}_{2}_{3}_{4}.xml",
doc.Key.GraphInstance?.ValueDate?.ToString() ?? "none",
doc.Key.GraphInstance?.Name ?? "none",
doc.Key.GetType().Name,
doc.Key.FriendlyString(),
cnt++));
exportableDocuments.Add(docName, Encoding.UTF8.GetBytes(RFXMLSerializer.PrettySerializeContract(doc)));
c++;
}
}
ZIPUtils.ZipFiles(fileName, exportableDocuments, password);
}
return(c);
}
private static PresentationPreferences GetPresentationPreferences(IRFProcessingContext context)
{
var presentationKey = PresentationKey();
var preferences = context.LoadDocumentContent <PresentationPreferences>(presentationKey);
return(preferences ?? new PresentationPreferences());
}
public RFSchedulerService(IRFProcessingContext context, List <Func <IRFProcessingContext, RFSchedulerConfig> > configFuncs)
{
_context = context;
_configFuncs = configFuncs;
_lastTrigger = DateTime.Now;
Reload();
}
public static List <RFCatalogKeyData> RefreshCache(IRFProcessingContext context, string username)
{
using (var dataEditor = new RFDataEditorActivity(context, username))
{
var cache = dataEditor.GetDocuments(null, null, null, 0, null, true).Select(d => new RFCatalogKeyData
{
Key = JsonConvert.SerializeObject(d.Key),
FriendlyString = d.Key.FriendlyString(),
KeyReference = d.KeyReference,
KeyType = RFReflectionHelpers.TrimType(d.KeyType),
KeyTypeFull = d.KeyType,
Plane = d.Key.Plane.ToString(),
ContentType = RFReflectionHelpers.TrimType(d.ContentType),
ContentTypeFull = d.ContentType,
DataSize = d.DataSize,
Metadata = JsonConvert.SerializeObject(d.Metadata),
IsValid = d.IsValid,
UpdateTime = d.UpdateTime,
Instance = d.Key.GraphInstance != null ? d.Key.GraphInstance.Name : null,
ValueDate = (d.Key.GraphInstance != null && d.Key.GraphInstance.ValueDate.HasValue && d.Key.GraphInstance.ValueDate.Value != RFDate.NullDate) ?
d.Key.GraphInstance.ValueDate.Value.ToJavascript() : null
}).ToList();
lock (_sync)
{
_cache = cache;
}
return(cache);
}
}
public void Initialize(IRFProcessingContext context, RFEngineDefinition engineConfig, string connectionString)
{
_context = context;
_engineConfig = engineConfig;
_connectionString = connectionString;
Initialize();
}
public override RFSchedulerConfig SchedulerConfig(IRFProcessingContext context)
{
return(new RFSchedulerConfig
{
Range = RangeFunc(context),
Schedules = SchedulesFunc(context),
IsEnabled = context.UserConfig.GetBool(CONFIG_SECTION, TaskName, false, true, "Is Enabled")
});
}
public override RFSchedulerConfig SchedulerConfig(IRFProcessingContext context)
{
return(new RFSchedulerConfig
{
IsEnabled = true,
Range = RangeFunc(),
Schedules = SchedulesFunc()
});
}
/// <summary>
/// Do not override or call base when you do
/// </summary>
public virtual void Initialize(RFEngineProcessorParam p, IRFProcessingContext context, RFKeyDomain keyDomain, string processName)
{
InstanceParams = p?.ConvertTo <P>();
Context = context;
KeyDomain = keyDomain;
ProcessName = processName;
_isCancelling = false;
_processEntry = null;
Log = new RFProcessLog(Context.SystemLog, context.UserLog, this);
}
protected RFAttributionActivity(IRFProcessingContext context,
Func <RFDate, RFGraphInstance> getInstanceFunc,
Func <RFEnum, RFCatalogKey> keyCreatorFunc,
RFEnum latestEnum,
RFEnum templateEnum,
string userName) : base(context, userName)
{
mGetInstanceFunc = getInstanceFunc;
mKeyCreatorFunc = keyCreatorFunc;
mLatestEnum = latestEnum;
mTemplateEnum = templateEnum;
}
public RFConsoleExecutor(
RFEngineDefinition config,
IRFProcessingContext context,
EngineConfigElement engine,
IRFEngineConsole engineConsole)
{
_config = config;
_context = context;
_engine = engine;
_engineConsole = engineConsole;
_isExiting = false;
}
public override void Initialize(RFEngineProcessorParam p, IRFProcessingContext context, RFKeyDomain keyDomain, string processName)
{
base.Initialize(p, context, keyDomain, processName);
if (InstanceParams is RFEngineProcessorGraphInstanceParam)
{
GraphInstance = (InstanceParams as RFEngineProcessorGraphInstanceParam).Instance;
}
else
{
throw new RFSystemException(this, "Unable to extract GraphInstance from params");
}
}
public static bool SetPresentationMode(IRFProcessingContext context, string username, bool active)
{
username = username.ToLower();
var preferences = GetPresentationPreferences(context);
if (active && !preferences.Active.Contains(username))
{
preferences.Active.Add(username);
}
else if (!active && preferences.Active.Contains(username))
{
preferences.Active.Remove(username);
}
context.SaveDocument(PresentationKey(), preferences, false);
return(true);
}
protected AttributionController(
IRFProcessingContext context,
RFEngineDefinition engineConfig,
Func <IRFProcessingContext, string, A> activityFunc,
RedirectTarget applyRedirect,
RedirectTarget errorRedirect = null) : base(context, engineConfig)
{
_activityFunc = activityFunc;
_applyRedirect = applyRedirect;
_errorRedirect = errorRedirect;
if (_errorRedirect == null)
{
_errorRedirect = new RedirectTarget
{
Action = "Index",
Area = "",
Controller = "Home"
};
}
}
protected List <RFCatalogKey> SaveDomain(RFGraphProcessorDomain domain, IRFProcessingContext context)
{
var updates = new List <RFCatalogKey>();
var domainType = domain.GetType();
int numUpdates = 0;
foreach (var propertyInfo in domainType.GetProperties())
{
var ioBehaviour = propertyInfo.GetCustomAttributes(typeof(RFIOBehaviourAttribute), true).FirstOrDefault() as RFIOBehaviourAttribute;
if (ioBehaviour != null && (ioBehaviour.IOBehaviour == RFIOBehaviour.Output || ioBehaviour.IOBehaviour == RFIOBehaviour.State))
{
var value = propertyInfo.GetValue(domain);
if (value != null)
{
foreach (var outputMapping in Config.IOMappings.Where(m => m.Property.Name == propertyInfo.Name))
{
var outputKey = outputMapping.Key.CreateForInstance(this.GraphInstance);
var options = RFGraphInstance.ImplyOptions(outputMapping);
// by default date will be set from the graph instance
if (options.DateBehaviour == RFDateBehaviour.Dateless)
{
outputKey.GraphInstance.ValueDate = null;
}
bool isState = (ioBehaviour.IOBehaviour == RFIOBehaviour.State);
var hasUpdated = context.SaveEntry(new RFDocument
{
Content = value,
Key = outputKey,
Type = value.GetType().FullName
}, !isState);
if (hasUpdated)
{
updates.Add(outputKey);
numUpdates++;
}
}
}
}
}
return(updates);
}
public void Initialize(IRFProcessingContext serviceContext)
{
Log.Info(this, "Initializing RFSimpleEngine.");
foreach (var processConfig in _config.Processes)
{
var newProcess = new RFEngineProcess(processConfig.Value.Name, processConfig.Value as RFEngineProcessDefinition, _config.KeyDomain);
_processes.Add(processConfig.Value.Name, newProcess);
}
foreach (var trigger in _config.Triggers)
{
_reactors.Add(new RFTriggerReactor(trigger, _context.GetReadingContext()));
}
foreach (var graphConfig in _config.Graphs.Values)
{
AddGraph(graphConfig);
}
foreach (var service in _config.Services)
{
var backgroundService = new RFBackgroundServiceComponent(_context, service.Value(serviceContext));
_services.Add(backgroundService);
_reactors.Add(new RFServiceReactor {
ServiceName = service.Key, Service = backgroundService
});
}
if (_config.Schedules.Any())
{
var schedulerService = new RFBackgroundServiceComponent(_context, new RFSchedulerService(serviceContext, _config.Schedules));
_services.Add(schedulerService);
_reactors.Add(new RFServiceReactor {
ServiceName = RFSchedulerService.SERVICE_NAME, Service = schedulerService
});
}
}
public void React(RFEvent e, IRFProcessingContext processingContext)
{
var allInstructions = new BlockingCollection <RFInstruction>();
if (e is RFIntervalEvent ie) // silently store intervals in database?
{
processingContext.SaveEntry(new RFDocument
{
Content = ie.Interval,
Key = _config.IntervalDocumentKey(),
Type = typeof(RFInterval).FullName
}, false, true);
}
Parallel.ForEach(_reactors, reactor =>
{
try
{
var instructions = reactor.React(e);
if (instructions != null)
{
foreach (var instruction in instructions)
{
allInstructions.Add(instruction);
}
}
}
catch (Exception ex)
{
Log.Exception(this, ex, "Reactor {0} threw an exception processing event {1}", reactor, e);
}
});
foreach (var instruction in allInstructions)
{
processingContext.QueueInstruction(this, instruction);
}
}
protected RIFFApiController(IRFProcessingContext context, RFEngineDefinition engineConfig)
{
_context = context;
_engineConfig = engineConfig;
}
protected RFProcessingResult ProcessInstruction(RFEngineProcess process, RFProcessInstruction i, IRFProcessingContext processingContext)
{
var result = new RFProcessingResult();
try
{
var sw = Stopwatch.StartNew();
var startTime = DateTimeOffset.Now;
bool completed = false;
try
{
var processorInstance = process.CreateInstance();
if (processorInstance != null)
{
if (_config.MaxRuntime.Ticks > 0)
{
var maxRuntime = TimeSpan.FromTicks(Math.Max(processorInstance.MaxRuntime().Ticks, _config.MaxRuntime.Ticks));
var timerTask = Task.Delay(maxRuntime).ContinueWith(t =>
{
if (!completed)
{
try
{
processorInstance.Cancel();
}
catch (Exception ex)
{
Log.Warning(this, "Exception cancelling process {0}: {1}", process.Name, ex.Message);
}
throw new TimeoutException(String.Format("Cancelling process {0} as it's taken too long (max runtime = {1} seconds).", process.Name, maxRuntime.TotalSeconds));
}
});
}
result = process.RunInstance(processorInstance, i, processingContext);
}
}
catch (Exception ex) // hard exception, or softs should have bene handled by now
{
var message = ex.InnerException?.Message ?? ex.Message;
result.AddMessage(message);
result.IsError = true;
result.ShouldRetry |= (ex is DbException || ex is TimeoutException || ex is RFTransientSystemException || ex?.InnerException is DbException || ex?.InnerException is TimeoutException);
Log.Exception(this, ex, "Exception running process {0}", process.Name);
/*processingContext.UserLog.LogEntry(new RFUserLogEntry
* {
* Action = "Error",
* Area = null,
* Description = String.Format("Error running process {0}: {1}", process.Name, message),
* IsUserAction = false,
* IsWarning = true,
* Processor = process.Name
* });*/
}
completed = true;
LogProcessingStat(process, i, sw, startTime);
Log.Debug(this, String.Format("Engine: process {0} process took {1} ms.", process.Name, sw.ElapsedMilliseconds));
}
catch (Exception ex) // a really bad system exception
{
Log.Exception(this, ex, "Exception processing instruction {0} by process {1}", i, process);
result.AddMessage(ex.Message);
result.IsError = true;
result.ShouldRetry |= (ex is DbException || ex is TimeoutException || ex is RFTransientSystemException || ex?.InnerException is DbException || ex?.InnerException is TimeoutException);
}
return(result);
}
public RFConfigActivity(IRFProcessingContext context, string userName) : base(context, userName)
{
}
public RFSecureActivity(IRFProcessingContext context, string userName, RFKeyDomain keyDomain, RFEnum vaultEnum) : base(context, userName.ToLower().Trim())
{
_vaultKey = RFKeyVaultKey.Create(keyDomain, vaultEnum);
}
public abstract void Open(IRFProcessingContext context);
public override void Open(IRFProcessingContext context)
{
mContext = context;
}
public DataQueryController(IRFProcessingContext context, RFEngineDefinition engineConfig) : base(context, engineConfig)
{
}
public RFDataEditorActivity(IRFProcessingContext context, string userName) : base(context, userName)
{
}
protected RFGraphProcessorDomain LoadDomain(IRFGraphProcessorInstance processor, IRFProcessingContext context, ref SortedSet <string> missingInputs)
{
var domain = processor.CreateDomain();
domain.Instance = GraphInstance;
var domainType = domain.GetType();
var missingNames = new ConcurrentBag <string>();
try
{
foreach (var propertyInfo in domainType.GetProperties())
{
var ioBehaviour = propertyInfo.GetCustomAttributes(typeof(RFIOBehaviourAttribute), true).FirstOrDefault() as RFIOBehaviourAttribute;
if (ioBehaviour != null && (ioBehaviour.IOBehaviour == RFIOBehaviour.Input || ioBehaviour.IOBehaviour == RFIOBehaviour.State))
{
var inputMapping = Config.IOMappings.SingleOrDefault(m => m.Property.Name == propertyInfo.Name);
if (inputMapping != null)
{
var options = RFGraphInstance.ImplyOptions(inputMapping);
if (options.DateBehaviour == RFDateBehaviour.Range)
{
if (inputMapping.RangeRequestFunc == null)
{
if (ioBehaviour.IsMandatory)
{
Context.SystemLog.Warning(this, "No range specified for mandatory ranged input {0} on process {1}", propertyInfo.FullName(), this.ProcessName);
}
continue;
}
var dateRange = inputMapping.RangeRequestFunc(GraphInstance);
options.DateBehaviour = RFDateBehaviour.Exact; // override to load one-by-one
var rangeInput = Activator.CreateInstance(propertyInfo.PropertyType) as IRFRangeInput;
foreach (var vd in dateRange)
{
var inputKey = inputMapping.Key.CreateForInstance(GraphInstance.WithDate(vd));
var item = context.LoadEntry(inputKey, options);
if (item != null && item is RFDocument)
{
var content = (item as RFDocument).Content;
if (content != null)
{
rangeInput.Add(vd, content);
}
}
}
try
{
if (propertyInfo.PropertyType.IsAssignableFrom(rangeInput.GetType()))
{
propertyInfo.SetValue(domain, rangeInput);
}
else
{
Context.SystemLog.Warning(this, "Not assigning value of type {0} to property {1} of type {2} due to type mismatch [{3}]", rangeInput.GetType().FullName,
propertyInfo.FullName(), propertyInfo.PropertyType.FullName, domain.Instance?.ValueDate);
}
}
catch (Exception)
{
Context.SystemLog.Info(this, "Domain mismatch on property {0} for vd {1} - stale data?", propertyInfo.FullName(), domain.Instance.ValueDate);
}
}
else
{
var inputKey = inputMapping.Key.CreateForInstance(GraphInstance);
var item = context.LoadEntry(inputKey, options);
if (item != null && item is RFDocument)
{
try
{
var content = (item as RFDocument).Content;
if (content != null)
{
if (propertyInfo.PropertyType.IsAssignableFrom(content.GetType()))
{
propertyInfo.SetValue(domain, content);
}
else
{
Context.SystemLog.Warning(this, "Not assigning value of type {0} to property {1} of type {2} due to type mismatch [{3}]", content.GetType().FullName,
propertyInfo.FullName(), propertyInfo.PropertyType.FullName, domain.Instance?.ValueDate);
}
}
}
catch (Exception)
{
Context.SystemLog.Info(this, "Domain mismatch on property {0} for vd {1} - stale data?", propertyInfo.FullName(), domain.Instance.ValueDate);
}
}
else if (ioBehaviour.IsMandatory)
{
missingNames.Add(propertyInfo.Name);
// return null;// performance short circuit - only report one missing
}
}
}
}
}
missingInputs.UnionWith(missingNames);
if (missingInputs.Count > 0)
{
return(null);
}
return(domain);
}
catch (Exception ex)
{
Context.SystemLog.Exception(this, "Error loading domain", ex);
return(null);
}
}
public RFInputReportsActivity(IRFProcessingContext context) : base(context, null)
{
}
