/// <summary>Runs the export job.</summary>
/// <param name="exportJob">The export job.</param>
/// <param name="result">The result.</param>
/// <param name="scheduleHistoryItem">The schedule history item.</param>
public void Export(ExportImportJob exportJob, ExportImportResult result, ScheduleHistoryItem scheduleHistoryItem)
{
var exportDto = JsonConvert.DeserializeObject <ExportDto>(exportJob.JobObject);
if (exportDto == null)
{
exportJob.CompletedOnDate = DateUtils.GetDatabaseUtcTime();
exportJob.JobStatus = JobStatus.Failed;
return;
}
this.timeoutSeconds = GetTimeoutPerSlot();
var dbName = Path.Combine(ExportFolder, exportJob.Directory, Constants.ExportDbName);
var finfo = new FileInfo(dbName);
dbName = finfo.FullName;
var checkpoints = EntitiesController.Instance.GetJobChekpoints(exportJob.JobId);
// Delete so we start a fresh export database; only if there is no previous checkpoint exists
if (checkpoints.Count == 0)
{
if (finfo.Directory != null && finfo.Directory.Exists)
{
finfo.Directory.Delete(true);
}
// Clear all the files in finfo.Directory. Create if doesn't exists.
finfo.Directory?.Create();
result.AddSummary("Starting Exporting Repository", finfo.Name);
}
else
{
if (finfo.Directory != null && finfo.Directory.Exists)
{
result.AddSummary("Resuming Exporting Repository", finfo.Name);
}
else
{
scheduleHistoryItem.AddLogNote("Resuming data not found.");
result.AddSummary("Resuming data not found.", finfo.Name);
return;
}
}
exportJob.JobStatus = JobStatus.InProgress;
// there must be one parent implementor at least for this to work
var implementors = Util.GetPortableImplementors().ToList();
var parentServices = implementors.Where(imp => string.IsNullOrEmpty(imp.ParentCategory)).ToList();
implementors = implementors.Except(parentServices).ToList();
var nextLevelServices = new List <BasePortableService>();
var includedItems = GetAllCategoriesToInclude(exportDto, implementors);
if (includedItems.Count == 0)
{
scheduleHistoryItem.AddLogNote("Export NOT Possible");
scheduleHistoryItem.AddLogNote("<br/>No items selected for exporting");
result.AddSummary("Export NOT Possible", "No items selected for exporting");
exportJob.CompletedOnDate = DateUtils.GetDatabaseUtcTime();
exportJob.JobStatus = JobStatus.Failed;
return;
}
scheduleHistoryItem.AddLogNote($"<br/><b>SITE EXPORT Preparing Check Points. JOB #{exportJob.JobId}: {exportJob.Name}</b>");
this.PrepareCheckPoints(exportJob.JobId, parentServices, implementors, includedItems, checkpoints);
scheduleHistoryItem.AddLogNote($"<br/><b>SITE EXPORT Started. JOB #{exportJob.JobId}: {exportJob.Name}</b>");
scheduleHistoryItem.AddLogNote($"<br/>Between [{exportDto.FromDateUtc ?? Constants.MinDbTime}] and [{exportDto.ToDateUtc:g}]");
var firstIteration = true;
AddJobToCache(exportJob);
using (var ctx = new ExportImportRepository(dbName))
{
ctx.AddSingleItem(exportDto);
do
{
foreach (var service in parentServices.OrderBy(x => x.Priority))
{
if (exportJob.IsCancelled)
{
exportJob.JobStatus = JobStatus.Cancelled;
break;
}
if (implementors.Count > 0)
{
// collect children for next iteration
var children =
implementors.Where(imp => service.Category.Equals(imp.ParentCategory, IgnoreCaseComp));
nextLevelServices.AddRange(children);
implementors = implementors.Except(nextLevelServices).ToList();
}
if ((firstIteration && includedItems.Any(x => x.Equals(service.Category, IgnoreCaseComp))) ||
(!firstIteration && includedItems.Any(x => x.Equals(service.ParentCategory, IgnoreCaseComp))))
{
var serviceAssembly = service.GetType().Assembly.GetName().Name;
service.Result = result;
service.Repository = ctx;
service.CheckCancelled = CheckCancelledCallBack;
service.CheckPointStageCallback = this.CheckpointCallback;
service.CheckPoint = checkpoints.FirstOrDefault(cp => cp.Category == service.Category && cp.AssemblyName == serviceAssembly);
if (service.CheckPoint == null)
{
service.CheckPoint = new ExportImportChekpoint
{
JobId = exportJob.JobId,
Category = service.Category,
AssemblyName = serviceAssembly,
StartDate = DateUtils.GetDatabaseUtcTime(),
};
// persist the record in db
this.CheckpointCallback(service);
}
else if (service.CheckPoint.StartDate == Null.NullDate)
{
service.CheckPoint.StartDate = DateUtils.GetDatabaseUtcTime();
}
try
{
service.ExportData(exportJob, exportDto);
}
finally
{
this.AddLogsToDatabase(exportJob.JobId, result.CompleteLog);
}
scheduleHistoryItem.AddLogNote("<br/>Exported: " + service.Category);
}
}
firstIteration = false;
parentServices = new List <BasePortableService>(nextLevelServices);
nextLevelServices.Clear();
if (implementors.Count > 0 && parentServices.Count == 0)
{
// WARN: this is a case where there is a broken parent-children hierarchy
// and/or there are BasePortableService implementations without a known parent.
parentServices = implementors;
implementors.Clear();
scheduleHistoryItem.AddLogNote(
"<br/><b>Orphaned services:</b> " + string.Join(",", parentServices.Select(x => x.Category)));
}
}while (parentServices.Count > 0 && !this.TimeIsUp);
RemoveTokenFromCache(exportJob);
}
if (this.TimeIsUp)
{
result.AddSummary(
$"Job time slot ({this.timeoutSeconds} sec) expired",
"Job will resume in the next scheduler iteration");
}
else if (exportJob.JobStatus == JobStatus.InProgress)
{
// Create Export Summary for manifest file.
var summary = new ImportExportSummary();
using (var ctx = new ExportImportRepository(dbName))
{
BaseController.BuildJobSummary(exportJob.Directory, ctx, summary);
}
DoPacking(exportJob, dbName);
// Complete the job.
exportJob.JobStatus = JobStatus.Successful;
SetLastJobStartTime(scheduleHistoryItem.ScheduleID, exportJob.CreatedOnDate);
var exportController = new ExportController();
var exportFileInfo = new ExportFileInfo
{
ExportPath = exportJob.Directory,
ExportSize = Util.FormatSize(GetExportSize(Path.Combine(ExportFolder, exportJob.Directory))),
};
summary.ExportFileInfo = exportFileInfo;
exportController.CreatePackageManifest(exportJob, exportFileInfo, summary);
}
}
/// <summary>Runs the import job.</summary>
/// <param name="importJob">The import job.</param>
/// <param name="result">The result.</param>
/// <param name="scheduleHistoryItem">The schedule history item.</param>
public void Import(ExportImportJob importJob, ExportImportResult result, ScheduleHistoryItem scheduleHistoryItem)
{
scheduleHistoryItem.AddLogNote($"<br/><b>SITE IMPORT Started. JOB #{importJob.JobId}</b>");
this.timeoutSeconds = GetTimeoutPerSlot();
var importDto = JsonConvert.DeserializeObject <ImportDto>(importJob.JobObject);
if (importDto == null)
{
importJob.CompletedOnDate = DateUtils.GetDatabaseUtcTime();
importJob.JobStatus = JobStatus.Failed;
return;
}
var dbName = Path.Combine(ExportFolder, importJob.Directory, Constants.ExportDbName);
var finfo = new FileInfo(dbName);
if (!finfo.Exists)
{
DoUnPacking(importJob);
finfo = new FileInfo(dbName);
}
if (!finfo.Exists)
{
scheduleHistoryItem.AddLogNote("<br/>Import file not found. Name: " + dbName);
importJob.CompletedOnDate = DateUtils.GetDatabaseUtcTime();
importJob.JobStatus = JobStatus.Failed;
return;
}
using (var ctx = new ExportImportRepository(dbName))
{
var exportedDto = ctx.GetSingleItem <ExportDto>();
var exportVersion = new Version(exportedDto.SchemaVersion);
var importVersion = new Version(importDto.SchemaVersion);
if (importVersion < exportVersion)
{
importJob.CompletedOnDate = DateUtils.GetDatabaseUtcTime();
importJob.JobStatus = JobStatus.Failed;
scheduleHistoryItem.AddLogNote("Import NOT Possible");
var msg =
$"Exported version ({exportedDto.SchemaVersion}) is newer than import engine version ({importDto.SchemaVersion})";
result.AddSummary("Import NOT Possible", msg);
return;
}
var checkpoints = EntitiesController.Instance.GetJobChekpoints(importJob.JobId);
if (checkpoints.Count == 0)
{
result.AddSummary("Starting Importing Repository", finfo.Name);
result.AddSummary("Importing File Size", Util.FormatSize(finfo.Length));
CleanupDatabaseIfDirty(ctx);
}
else
{
result.AddSummary("Resuming Importing Repository", finfo.Name);
}
var implementors = Util.GetPortableImplementors().ToList();
var parentServices = implementors.Where(imp => string.IsNullOrEmpty(imp.ParentCategory)).ToList();
importJob.Name = exportedDto.ExportName;
importJob.Description = exportedDto.ExportDescription;
importJob.JobStatus = JobStatus.InProgress;
// there must be one parent implementor at least for this to work
implementors = implementors.Except(parentServices).ToList();
var nextLevelServices = new List <BasePortableService>();
var includedItems = GetAllCategoriesToInclude(exportedDto, implementors);
scheduleHistoryItem.AddLogNote($"<br/><b>SITE IMPORT Preparing Check Points. JOB #{importJob.JobId}: {importJob.Name}</b>");
this.PrepareCheckPoints(importJob.JobId, parentServices, implementors, includedItems, checkpoints);
var firstIteration = true;
AddJobToCache(importJob);
do
{
foreach (var service in parentServices.OrderBy(x => x.Priority))
{
if (importJob.IsCancelled)
{
importJob.JobStatus = JobStatus.Cancelled;
break;
}
if (implementors.Count > 0)
{
// collect children for next iteration
var children =
implementors.Where(imp => service.Category.Equals(imp.ParentCategory, IgnoreCaseComp));
nextLevelServices.AddRange(children);
implementors = implementors.Except(nextLevelServices).ToList();
}
if ((firstIteration && includedItems.Any(x => x.Equals(service.Category, IgnoreCaseComp))) ||
(!firstIteration && includedItems.Any(x => x.Equals(service.ParentCategory, IgnoreCaseComp))))
{
var serviceAssembly = service.GetType().Assembly.GetName().Name;
service.Result = result;
service.Repository = ctx;
service.CheckCancelled = CheckCancelledCallBack;
service.CheckPointStageCallback = this.CheckpointCallback;
service.CheckPoint = checkpoints.FirstOrDefault(cp => cp.Category == service.Category && cp.AssemblyName == serviceAssembly)
?? new ExportImportChekpoint
{
JobId = importJob.JobId,
AssemblyName = serviceAssembly,
Category = service.Category,
Progress = 0,
StartDate = DateUtils.GetDatabaseUtcTime(),
};
if (service.CheckPoint.StartDate == Null.NullDate)
{
service.CheckPoint.StartDate = DateUtils.GetDatabaseUtcTime();
}
this.CheckpointCallback(service);
try
{
service.ImportData(importJob, importDto);
}
finally
{
this.AddLogsToDatabase(importJob.JobId, result.CompleteLog);
}
scheduleHistoryItem.AddLogNote("<br/>Imported: " + service.Category);
}
}
firstIteration = false;
parentServices = new List <BasePortableService>(nextLevelServices);
nextLevelServices.Clear();
if (implementors.Count > 0 && parentServices.Count == 0)
{
// WARN: this is a case where there is a broken parent-children hierarchy
// and/or there are BasePortableService implementations without a known parent.
parentServices = implementors;
implementors.Clear();
scheduleHistoryItem.AddLogNote(
"<br/><b>Orphaned services:</b> " + string.Join(",", parentServices.Select(x => x.Category)));
}
}while (parentServices.Count > 0 && !this.TimeIsUp);
RemoveTokenFromCache(importJob);
if (this.TimeIsUp)
{
result.AddSummary(
$"Job time slot ({this.timeoutSeconds} sec) expired",
"Job will resume in the next scheduler iteration");
}
else if (importJob.JobStatus == JobStatus.InProgress)
{
importJob.JobStatus = JobStatus.Successful;
if (importDto.ExportDto.IncludeContent)
{
PagesExportService.ResetContentsFlag(ctx);
}
}
}
}
public override void DoWork()
{
try
{
//TODO: do some clean-up for very old import/export jobs/logs
var job = EntitiesController.Instance.GetFirstActiveJob();
if (job == null)
{
ScheduleHistoryItem.Succeeded = true;
ScheduleHistoryItem.AddLogNote("<br/>No Site Export/Import jobs queued for processing.");
}
else if (job.IsCancelled)
{
job.JobStatus = JobStatus.Cancelled;
EntitiesController.Instance.UpdateJobStatus(job);
ScheduleHistoryItem.Succeeded = true;
ScheduleHistoryItem.AddLogNote("<br/>Site Export/Import jobs was previously cancelled.");
}
else
{
job.JobStatus = JobStatus.InProgress;
EntitiesController.Instance.UpdateJobStatus(job);
var result = new ExportImportResult
{
JobId = job.JobId,
};
var engine = new ExportImportEngine();
var succeeded = true;
switch (job.JobType)
{
case JobType.Export:
try
{
engine.Export(job, result, ScheduleHistoryItem);
}
catch (Exception ex)
{
result.AddLogEntry("EXCEPTION exporting job #" + job.JobId, ex.Message, ReportLevel.Error);
engine.AddLogsToDatabase(job.JobId, result.CompleteLog);
throw;
}
EntitiesController.Instance.UpdateJobStatus(job);
break;
case JobType.Import:
try
{
engine.Import(job, result, ScheduleHistoryItem);
}
catch (ThreadAbortException)
{
ScheduleHistoryItem.TimeLapse = EmergencyScheduleFrequency;
ScheduleHistoryItem.TimeLapseMeasurement = EmergencyScheduleFrequencyUnit;
ScheduleHistoryItem.RetryTimeLapse = EmergencyScheduleRetry;
ScheduleHistoryItem.RetryTimeLapseMeasurement = EmergencyScheduleRetryUnit;
ScheduleHistoryItem.RetainHistoryNum = EmergencyHistoryNumber;
SchedulingController.UpdateSchedule(ScheduleHistoryItem);
SchedulingController.PurgeScheduleHistory();
Logger.Error("The Schduler item stopped because main thread stopped, set schedule into emergency mode so it will start after app restart.");
succeeded = false;
}
catch (Exception ex)
{
result.AddLogEntry("EXCEPTION importing job #" + job.JobId, ex.Message, ReportLevel.Error);
engine.AddLogsToDatabase(job.JobId, result.CompleteLog);
throw;
}
EntitiesController.Instance.UpdateJobStatus(job);
if (job.JobStatus == JobStatus.Successful || job.JobStatus == JobStatus.Cancelled)
{
// clear everything to be sure imported items take effect
DataCache.ClearCache();
}
break;
default:
throw new Exception("Unknown job type: " + job.JobType);
}
ScheduleHistoryItem.Succeeded = true;
//restore schedule item running timelapse to default.
if (succeeded &&
ScheduleHistoryItem.TimeLapse == EmergencyScheduleFrequency &&
ScheduleHistoryItem.TimeLapseMeasurement == EmergencyScheduleFrequencyUnit)
{
ScheduleHistoryItem.TimeLapse = DefaultScheduleFrequency;
ScheduleHistoryItem.TimeLapseMeasurement = DefaultScheduleFrequencyUnit;
ScheduleHistoryItem.RetryTimeLapse = DefaultScheduleRetry;
ScheduleHistoryItem.RetryTimeLapseMeasurement = DefaultScheduleRetryUnit;
ScheduleHistoryItem.RetainHistoryNum = DefaultHistoryNumber;
SchedulingController.UpdateSchedule(ScheduleHistoryItem);
}
var sb = new StringBuilder();
var jobType = Localization.GetString("JobType_" + job.JobType, Constants.SharedResources);
var jobStatus = Localization.GetString("JobStatus_" + job.JobStatus, Constants.SharedResources);
sb.AppendFormat("<br/><b>{0} {1}</b>", jobType, jobStatus);
var summary = result.Summary;
if (summary.Count > 0)
{
sb.Append("<br/><b>Summary:</b><ul>");
foreach (var entry in summary)
{
sb.Append($"<li>{entry.Name}: {entry.Value}</li>");
}
sb.Append("</ul>");
}
ScheduleHistoryItem.AddLogNote(sb.ToString());
engine.AddLogsToDatabase(job.JobId, result.CompleteLog);
Logger.Trace("Site Export/Import: Job Finished");
}
//SetLastSuccessfulIndexingDateTime(ScheduleHistoryItem.ScheduleID, ScheduleHistoryItem.StartDate);
}
catch (Exception ex)
{
ScheduleHistoryItem.Succeeded = false;
ScheduleHistoryItem.AddLogNote("<br/>Export/Import EXCEPTION: " + ex.Message);
Errored(ref ex);
// this duplicates the logging
//if (ScheduleHistoryItem.ScheduleSource != ScheduleSource.STARTED_FROM_BEGIN_REQUEST)
//{
// Exceptions.LogException(ex);
//}
}
}