public PipelineContext(PipelineContext ctx, DatasourceAdmin ds, DatasourceReport report) { var eng = ctx.ImportEngine; Switches = ctx.Switches; NewLastUpdated = eng.StartTimeUtc; ImportEngine = eng; RunAdministrations = eng.RunAdministrations; DatasourceAdmin = ds; DatasourceReport = report; Pipeline = ds.Pipeline; ImportLog = eng.ImportLog.Clone(ds.Name); DebugLog = eng.DebugLog.Clone(ds.Name); ErrorLog = eng.ErrorLog.Clone(ds.Name); MissedLog = eng.MissedLog.Clone(ds.Name); ImportFlags = eng.ImportFlags; LogAdds = (ds.LogAdds > 0) ? ds.LogAdds : eng.LogAdds; MaxAdds = (ds.MaxAdds >= 0) ? ds.MaxAdds : eng.MaxAdds; MaxEmits = (ds.MaxEmits >= 0) ? ds.MaxEmits : eng.MaxEmits; if (MaxEmits < 0 && (ImportFlags & _ImportFlags.MaxAddsToMaxEmits) != 0) { MaxEmits = MaxAdds; } ImportLog.Log("Current maxAdds={0}, maxEmits={1}", MaxAdds, MaxEmits); }
private String getEndpointName(String name, DatasourceAdmin ds) { if (name != null) { if (name[0] != '.') { return(name); } //Name with a dot indicates an amendment on the default name String defName = getEndpointName(null, ds); int idx = defName.LastIndexOf('.'); return((idx < 0 ? defName : defName.Substring(0, idx)) + name); } name = ds.EndpointName; if (name != null) { return(name); } if (DefaultEndpoint != null) { return(DefaultEndpoint.Replace("*", ds.Name)); } return(ds.Name); }
public DatasourceReport(DatasourceAdmin ds) { utcStart = DateTime.UtcNow; DatasourceName = ds.Name; ErrorState = _ErrorState.Running; Stats = "Running..."; }
/// <summary> /// Check if we don't have unresolved action endpoints /// </summary> public void CheckEndpoints(PipelineContext ctx, DatasourceAdmin ds) { if (ds.EndpointName != null) { return; } if (DefaultEndpoint == null || DefaultEndpoint.IndexOf('*') >= 0) { if (definedActions.Any(a => !a.Action.HasEndpointName)) { ctx.ImportEngine.Endpoints.CheckDataEndpoint(ctx, getEndpointName(null, ds), true); } } }
static bool isActive(String[] enabledDSses, DatasourceAdmin da) { if (enabledDSses == null) { return(da.Active); } for (int i = 0; i < enabledDSses.Length; i++) { if (da.Name.Equals(enabledDSses[i], StringComparison.InvariantCultureIgnoreCase)) { return(true); } } return(false); }
public DateTime GetLastOKRunDateShifted(DatasourceAdmin ds) { var a = GetLastOKRun(ds.Name); return(a == null ? DateTime.MinValue : a.RunDateUtc.AddSeconds(+ds.ShiftLastRuntime)); }
public DateTime GetLastOKRunDate(DatasourceAdmin ds) { var a = GetLastOKRun(ds.Name); return(a == null ? DateTime.MinValue : a.RunDateUtc); }
public ImportReport Import(String[] enabledDSses = null) { var ret = new ImportReport(); RunAdministrations = RunAdminSettings.Load(); StartTimeUtc = DateTime.UtcNow; ImportLog.Log(); ImportLog.Log(new String('_', 80)); ImportLog.Log(_LogType.ltProgress, "Starting import. VirtMem={3:F1}GB, Flags={0}, MaxAdds={1}, ActiveDS's='{2}'.", ImportFlags, MaxAdds, enabledDSses == null ? null : String.Join(", ", enabledDSses), OS.GetTotalVirtualMemory() / (1024 * 1024 * 1024.0)); PipelineContext mainCtx = new PipelineContext(this); try { _ErrorState stateFilter = _ErrorState.All; if ((ImportFlags & _ImportFlags.IgnoreLimited) != 0) { stateFilter &= ~_ErrorState.Limited; } if ((ImportFlags & _ImportFlags.IgnoreErrors) != 0) { stateFilter &= ~_ErrorState.Error; } Endpoints.Open(mainCtx); for (int i = 0; i < Datasources.Count; i++) { DatasourceAdmin admin = Datasources[i]; if (!String.IsNullOrEmpty(OverrideEndpoint)) { ImportLog.Log(_LogType.ltWarning, "Datsource {0} will run with the override endpoint={1}", admin.Name, OverrideEndpoint); admin.EndpointName = OverrideEndpoint; } if (!String.IsNullOrEmpty(OverrideEndpoint)) { ImportLog.Log(_LogType.ltWarning, "Datsource {0} will run with the override pipeline={1}", admin.Name, OverridePipeline); //admin.p = OverrideEndpoint; } if (!isActive(enabledDSses, admin)) { ImportLog.Log(_LogType.ltProgress, "[{0}]: not active", admin.Name); continue; } var report = new DatasourceReport(admin); ret.Add(report); PipelineContext ctx = new PipelineContext(mainCtx, admin, report); var pipeline = admin.Pipeline; try { admin.Import(ctx); mainCtx.ErrorState |= (ctx.ErrorState & stateFilter); if (ctx.LastError != null) { mainCtx.LastError = ctx.LastError; } report.MarkEnded(ctx); } catch (Exception err) { mainCtx.LastError = err; mainCtx.ErrorState |= (ctx.ErrorState & stateFilter) | _ErrorState.Error; report.MarkEnded(ctx); throw; } Endpoints.OptClosePerDatasource(ctx); foreach (var c in Converters) { c.DumpMissed(ctx); } } ImportLog.Log(_LogType.ltProgress, "Import ended"); ProcessHostCollection.StopAll(); Endpoints.Close(mainCtx); RunAdminSettings.Save(RunAdministrations); } catch (Exception err2) { mainCtx.LastError = err2; throw; } finally { try { Endpoints.CloseFinally(mainCtx); } catch (Exception e2) { ErrorLog.Log(e2); ImportLog.Log(e2); } try { ProcessHostCollection.StopAll(); } catch (Exception e2) { ErrorLog.Log(e2); ImportLog.Log(e2); } try { ret.SetGlobalStatus(mainCtx); if (Reporter != null) { Reporter.SendReport(mainCtx, ret); } } catch (Exception e2) { ErrorLog.Log(e2); ImportLog.Log(e2); } } //ret.SetGlobalStatus(mainCtx); ImportLog.Log("Unknown switches: [{0}]", ret.UnknownSwitches); ImportLog.Log("Mentioned switches: [{0}]", ret.MentionedSwitches); return(ret); }