public PipelineContext(PipelineContext ctx, DatasourceAdmin ds, DatasourceReport report)
{
var eng = ctx.ImportEngine;
Switches = ctx.Switches;
NewLastUpdated = eng.StartTimeUtc;
ImportEngine = eng;
RunAdministrations = eng.RunAdministrations;
DatasourceAdmin = ds;
DatasourceReport = report;
Pipeline = ds.Pipeline;
ImportLog = eng.ImportLog.Clone(ds.Name);
DebugLog = eng.DebugLog.Clone(ds.Name);
ErrorLog = eng.ErrorLog.Clone(ds.Name);
MissedLog = eng.MissedLog.Clone(ds.Name);
ImportFlags = eng.ImportFlags;
LogAdds = (ds.LogAdds > 0) ? ds.LogAdds : eng.LogAdds;
MaxAdds = (ds.MaxAdds >= 0) ? ds.MaxAdds : eng.MaxAdds;
MaxEmits = (ds.MaxEmits >= 0) ? ds.MaxEmits : eng.MaxEmits;
if (MaxEmits < 0 && (ImportFlags & _ImportFlags.MaxAddsToMaxEmits) != 0)
{
MaxEmits = MaxAdds;
}
ImportLog.Log("Current maxAdds={0}, maxEmits={1}", MaxAdds, MaxEmits);
}
private String getEndpointName(String name, DatasourceAdmin ds)
{
if (name != null)
{
if (name[0] != '.')
{
return(name);
}
//Name with a dot indicates an amendment on the default name
String defName = getEndpointName(null, ds);
int idx = defName.LastIndexOf('.');
return((idx < 0 ? defName : defName.Substring(0, idx)) + name);
}
name = ds.EndpointName;
if (name != null)
{
return(name);
}
if (DefaultEndpoint != null)
{
return(DefaultEndpoint.Replace("*", ds.Name));
}
return(ds.Name);
}
public DatasourceReport(DatasourceAdmin ds)
{
utcStart = DateTime.UtcNow;
DatasourceName = ds.Name;
ErrorState = _ErrorState.Running;
Stats = "Running...";
}
/// <summary>
/// Check if we don't have unresolved action endpoints
/// </summary>
public void CheckEndpoints(PipelineContext ctx, DatasourceAdmin ds)
{
if (ds.EndpointName != null)
{
return;
}
if (DefaultEndpoint == null || DefaultEndpoint.IndexOf('*') >= 0)
{
if (definedActions.Any(a => !a.Action.HasEndpointName))
{
ctx.ImportEngine.Endpoints.CheckDataEndpoint(ctx, getEndpointName(null, ds), true);
}
}
}
static bool isActive(String[] enabledDSses, DatasourceAdmin da)
{
if (enabledDSses == null)
{
return(da.Active);
}
for (int i = 0; i < enabledDSses.Length; i++)
{
if (da.Name.Equals(enabledDSses[i], StringComparison.InvariantCultureIgnoreCase))
{
return(true);
}
}
return(false);
}
public DateTime GetLastOKRunDateShifted(DatasourceAdmin ds)
{
var a = GetLastOKRun(ds.Name);
return(a == null ? DateTime.MinValue : a.RunDateUtc.AddSeconds(+ds.ShiftLastRuntime));
}
public DateTime GetLastOKRunDate(DatasourceAdmin ds)
{
var a = GetLastOKRun(ds.Name);
return(a == null ? DateTime.MinValue : a.RunDateUtc);
}
public ImportReport Import(String[] enabledDSses = null)
{
var ret = new ImportReport();
RunAdministrations = RunAdminSettings.Load();
StartTimeUtc = DateTime.UtcNow;
ImportLog.Log();
ImportLog.Log(new String('_', 80));
ImportLog.Log(_LogType.ltProgress, "Starting import. VirtMem={3:F1}GB, Flags={0}, MaxAdds={1}, ActiveDS's='{2}'.", ImportFlags, MaxAdds, enabledDSses == null ? null : String.Join(", ", enabledDSses), OS.GetTotalVirtualMemory() / (1024 * 1024 * 1024.0));
PipelineContext mainCtx = new PipelineContext(this);
try
{
_ErrorState stateFilter = _ErrorState.All;
if ((ImportFlags & _ImportFlags.IgnoreLimited) != 0)
{
stateFilter &= ~_ErrorState.Limited;
}
if ((ImportFlags & _ImportFlags.IgnoreErrors) != 0)
{
stateFilter &= ~_ErrorState.Error;
}
Endpoints.Open(mainCtx);
for (int i = 0; i < Datasources.Count; i++)
{
DatasourceAdmin admin = Datasources[i];
if (!String.IsNullOrEmpty(OverrideEndpoint))
{
ImportLog.Log(_LogType.ltWarning, "Datsource {0} will run with the override endpoint={1}", admin.Name, OverrideEndpoint);
admin.EndpointName = OverrideEndpoint;
}
if (!String.IsNullOrEmpty(OverrideEndpoint))
{
ImportLog.Log(_LogType.ltWarning, "Datsource {0} will run with the override pipeline={1}", admin.Name, OverridePipeline);
//admin.p = OverrideEndpoint;
}
if (!isActive(enabledDSses, admin))
{
ImportLog.Log(_LogType.ltProgress, "[{0}]: not active", admin.Name);
continue;
}
var report = new DatasourceReport(admin);
ret.Add(report);
PipelineContext ctx = new PipelineContext(mainCtx, admin, report);
var pipeline = admin.Pipeline;
try
{
admin.Import(ctx);
mainCtx.ErrorState |= (ctx.ErrorState & stateFilter);
if (ctx.LastError != null)
{
mainCtx.LastError = ctx.LastError;
}
report.MarkEnded(ctx);
}
catch (Exception err)
{
mainCtx.LastError = err;
mainCtx.ErrorState |= (ctx.ErrorState & stateFilter) | _ErrorState.Error;
report.MarkEnded(ctx);
throw;
}
Endpoints.OptClosePerDatasource(ctx);
foreach (var c in Converters)
{
c.DumpMissed(ctx);
}
}
ImportLog.Log(_LogType.ltProgress, "Import ended");
ProcessHostCollection.StopAll();
Endpoints.Close(mainCtx);
RunAdminSettings.Save(RunAdministrations);
}
catch (Exception err2)
{
mainCtx.LastError = err2;
throw;
}
finally
{
try
{
Endpoints.CloseFinally(mainCtx);
}
catch (Exception e2)
{
ErrorLog.Log(e2);
ImportLog.Log(e2);
}
try
{
ProcessHostCollection.StopAll();
}
catch (Exception e2)
{
ErrorLog.Log(e2);
ImportLog.Log(e2);
}
try
{
ret.SetGlobalStatus(mainCtx);
if (Reporter != null)
{
Reporter.SendReport(mainCtx, ret);
}
}
catch (Exception e2)
{
ErrorLog.Log(e2);
ImportLog.Log(e2);
}
}
//ret.SetGlobalStatus(mainCtx);
ImportLog.Log("Unknown switches: [{0}]", ret.UnknownSwitches);
ImportLog.Log("Mentioned switches: [{0}]", ret.MentionedSwitches);
return(ret);
}