public void SetGlobalStatus(PipelineContext ctx)
{
errorState = ctx.ErrorState;
ErrorMessage = ctx.LastError == null ? null : ctx.LastError.Message;
MentionedSwitches = ctx.Switches.GeAskedSwitches();
UnknownSwitches = ctx.Switches.GeUnknownSwitches();
}
public void Add(DatasourceReport rep)
{
errorState |= rep.ErrorState;
//if (rep.Errors > 0 || rep.ErrorMessage != null)
// hasErrors = true;
DatasourceReports.Add(rep);
}
public DatasourceReport(DatasourceAdmin ds)
{
utcStart = DateTime.UtcNow;
DatasourceName = ds.Name;
ErrorState = _ErrorState.Running;
Stats = "Running...";
}
public RunAdministration(JObject obj)
{
RunDateUtc = obj.ReadDate(ADM_DATE);
DataSource = obj.ReadStr(ADM_DS);
State = Invariant.ToEnum(obj.ReadStr(ADM_STATE), _ErrorState.Error);
ImportFlags = Invariant.ToEnum(obj.ReadStr(ADM_FLAGS), (_ImportFlags)0);
Added = obj.ReadInt(ADM_ADDED);
Deleted = obj.ReadInt(ADM_DELETED);
Skipped = obj.ReadInt(ADM_SKIPPED);
Emitted = obj.ReadInt(ADM_EMITTED);
Errors = obj.ReadInt(ADM_ERRORS);
}
public RunAdministration(PipelineContext ctx)
{
RunDateUtc = ctx.NewLastUpdated;
DataSource = ctx.DatasourceAdmin.Name;
State = ctx.ErrorState;
Added = ctx.Added;
Deleted = ctx.Deleted;
Skipped = ctx.Skipped;
Emitted = ctx.Emitted;
Errors = ctx.Errors;
ImportFlags = ctx.ImportFlags;
}
public void MarkEnded(PipelineContext ctx)
{
ElapsedSeconds = (int)(DateTime.UtcNow - utcStart).TotalSeconds;
Added = ctx.Added;
Deleted = ctx.Deleted;
Emitted = ctx.Emitted;
Errors = ctx.Errors;
Skipped = ctx.Skipped;
ErrorMessage = ctx.LastError == null ? null : ctx.LastError.Message;
StringBuilder sb = new StringBuilder();
sb.Append("Elapsed=");
sb.Append(Pretty.PrintElapsed(ElapsedSeconds));
sb.Append(", ");
sb.Append(ctx.GetStats());
Stats = sb.ToString();
ErrorState = ctx.ErrorState == _ErrorState.Running ? _ErrorState.OK : ctx.ErrorState;
}
public ImportReport Import(String[] enabledDSses = null)
{
var ret = new ImportReport();
RunAdministrations = RunAdminSettings.Load();
StartTimeUtc = DateTime.UtcNow;
ImportLog.Log();
ImportLog.Log(new String('_', 80));
ImportLog.Log(_LogType.ltProgress, "Starting import. VirtMem={3:F1}GB, Flags={0}, MaxAdds={1}, ActiveDS's='{2}'.", ImportFlags, MaxAdds, enabledDSses == null ? null : String.Join(", ", enabledDSses), OS.GetTotalVirtualMemory() / (1024 * 1024 * 1024.0));
PipelineContext mainCtx = new PipelineContext(this);
try
{
_ErrorState stateFilter = _ErrorState.All;
if ((ImportFlags & _ImportFlags.IgnoreLimited) != 0)
{
stateFilter &= ~_ErrorState.Limited;
}
if ((ImportFlags & _ImportFlags.IgnoreErrors) != 0)
{
stateFilter &= ~_ErrorState.Error;
}
Endpoints.Open(mainCtx);
for (int i = 0; i < Datasources.Count; i++)
{
DatasourceAdmin admin = Datasources[i];
if (!String.IsNullOrEmpty(OverrideEndpoint))
{
ImportLog.Log(_LogType.ltWarning, "Datsource {0} will run with the override endpoint={1}", admin.Name, OverrideEndpoint);
admin.EndpointName = OverrideEndpoint;
}
if (!String.IsNullOrEmpty(OverrideEndpoint))
{
ImportLog.Log(_LogType.ltWarning, "Datsource {0} will run with the override pipeline={1}", admin.Name, OverridePipeline);
//admin.p = OverrideEndpoint;
}
if (!isActive(enabledDSses, admin))
{
ImportLog.Log(_LogType.ltProgress, "[{0}]: not active", admin.Name);
continue;
}
var report = new DatasourceReport(admin);
ret.Add(report);
PipelineContext ctx = new PipelineContext(mainCtx, admin, report);
var pipeline = admin.Pipeline;
try
{
admin.Import(ctx);
mainCtx.ErrorState |= (ctx.ErrorState & stateFilter);
if (ctx.LastError != null)
{
mainCtx.LastError = ctx.LastError;
}
report.MarkEnded(ctx);
}
catch (Exception err)
{
mainCtx.LastError = err;
mainCtx.ErrorState |= (ctx.ErrorState & stateFilter) | _ErrorState.Error;
report.MarkEnded(ctx);
throw;
}
Endpoints.OptClosePerDatasource(ctx);
foreach (var c in Converters)
{
c.DumpMissed(ctx);
}
}
ImportLog.Log(_LogType.ltProgress, "Import ended");
ProcessHostCollection.StopAll();
Endpoints.Close(mainCtx);
RunAdminSettings.Save(RunAdministrations);
}
catch (Exception err2)
{
mainCtx.LastError = err2;
throw;
}
finally
{
try
{
Endpoints.CloseFinally(mainCtx);
}
catch (Exception e2)
{
ErrorLog.Log(e2);
ImportLog.Log(e2);
}
try
{
ProcessHostCollection.StopAll();
}
catch (Exception e2)
{
ErrorLog.Log(e2);
ImportLog.Log(e2);
}
try
{
ret.SetGlobalStatus(mainCtx);
if (Reporter != null)
{
Reporter.SendReport(mainCtx, ret);
}
}
catch (Exception e2)
{
ErrorLog.Log(e2);
ImportLog.Log(e2);
}
}
//ret.SetGlobalStatus(mainCtx);
ImportLog.Log("Unknown switches: [{0}]", ret.UnknownSwitches);
ImportLog.Log("Mentioned switches: [{0}]", ret.MentionedSwitches);
return(ret);
}
