public override void Execute()
{
var eventModel = CurrentActionProperties(TargetCollection, this.GetType().Name);
OnTrigger(new PreingestEventArgs {
Description = String.Format("Start polishing Opex for container '{0}'.", TargetCollection), Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Started, PreingestAction = eventModel
});
var anyMessages = new List <String>();
bool isSuccess = false;
try
{
string sessionFolder = Path.Combine(ApplicationSettings.DataFolderName, SessionGuid.ToString());
BodySettings settings = new SettingsReader(this.ApplicationSettings.DataFolderName, SessionGuid).GetSettings();
if (settings == null)
{
throw new ApplicationException("Settings are not saved!");
}
if (String.IsNullOrEmpty(settings.Polish))
{
throw new ApplicationException("Polish setting is empty!");
}
DirectoryInfo directoryInfoSessionFolder = new DirectoryInfo(sessionFolder);
List <FileInfo> opexFiles = directoryInfoSessionFolder.GetFiles("*.opex", SearchOption.AllDirectories).ToList();
if (opexFiles.Count == 0)
{
throw new ApplicationException("Zero OPEX files found!");
}
string xsltTranformationServiceUrl = String.Format("http://{0}:{1}/hooks/xslt-transformation", ApplicationSettings.TransformationServerName, ApplicationSettings.TransformationServerPort);
string stylesheetFileLocation = Path.Combine(ApplicationSettings.PreWashFolder, settings.Polish);
var jsonData = new List <string>();
int countTotal = opexFiles.Count;
OnTrigger(new PreingestEventArgs
{
Description = String.Format("Transforming: {0} files", countTotal),
Initiate = DateTimeOffset.Now,
ActionType = PreingestActionStates.Executing,
PreingestAction = eventModel
});
if (!String.IsNullOrEmpty(settings.UseSaxon))
{
bool useSaxon = settings.UseSaxon.Equals("Ja", StringComparison.InvariantCultureIgnoreCase);
if (TransformationSetting.UseSaxon != useSaxon)
{
TransformationSetting.UseSaxon = useSaxon;
}
}
opexFiles.ForEach(file =>
{
string result = string.Empty;
try
{
if (TransformationSetting.UseSaxon)
{
using (HttpClient client = new HttpClient())
{
client.Timeout = Timeout.InfiniteTimeSpan;
HttpResponseMessage response = client.GetAsync(String.Format("{0}?xsl={1}&xml={2}", xsltTranformationServiceUrl, stylesheetFileLocation, file.FullName)).Result;
response.EnsureSuccessStatusCode();
result = response.Content.ReadAsStringAsync().Result;
}
}
else
{
var opex = XDocument.Load(file.FullName).ToString();
var xsl = XDocument.Load(stylesheetFileLocation).ToString();
result = Transform(xsl, opex);
}
}
catch (Exception e)
{
anyMessages.Add(String.Format("Transformation with Opex file '{0}' failed! {1} {2}", file.FullName, e.Message, e.StackTrace));
}
finally
{
if (!String.IsNullOrEmpty(result))
{
try
{
XDocument doc = XDocument.Parse(result);
doc.Save(file.FullName);
}
catch (Exception ie)
{
anyMessages.Add(String.Format("Saving Opex file '{0}' failed! {1} {2}", file.FullName, ie.Message, ie.StackTrace));
}
finally
{
jsonData.Add(String.Format("Transformation processed: {0}", file.FullName));
}
}
}
});
//update opex
var profiles = opexFiles.Select(item => new { Xml = XDocument.Load(item.FullName).ToString(), FullName = item.FullName }).Where(item => item.Xml.Contains("<Files>")).ToList();
countTotal = profiles.Count;
OnTrigger(new PreingestEventArgs
{
Description = String.Format("Updating profile (Opex files): Total {0} files.", countTotal),
Initiate = DateTimeOffset.Now,
ActionType = PreingestActionStates.Executing,
PreingestAction = eventModel
});
profiles.ForEach(item =>
{
FileInfo opex = new FileInfo(item.FullName);
FileInfo[] currentContent = opex.Directory.GetFiles().Where(content => content.FullName != item.FullName).ToArray();
var currentOpexMetadataFile = Preingest.WebApi.Utilities.DeserializerHelper.DeSerializeObjectFromXmlFile <Noord.Hollands.Archief.Entities.Opex.opexMetadata>(item.Xml);
//overwrite
if (currentContent.Count() > 0)
{
currentOpexMetadataFile.Transfer.Manifest = new Noord.Hollands.Archief.Entities.Opex.manifest();
currentOpexMetadataFile.Transfer.Manifest.Files = currentContent.Select(item => new Noord.Hollands.Archief.Entities.Opex.file
{
size = item.Length,
typeSpecified = true,
type = item.Extension.Equals(".opex", StringComparison.InvariantCultureIgnoreCase) ? Noord.Hollands.Archief.Entities.Opex.fileType.metadata : Noord.Hollands.Archief.Entities.Opex.fileType.content,
sizeSpecified = true,
Value = item.Name
}).ToArray();
string output = Preingest.WebApi.Utilities.SerializerHelper.SerializeObjectToString(currentOpexMetadataFile);
var newDoc = XDocument.Parse(output);
newDoc.Save(item.FullName);
}
});
OnTrigger(new PreingestEventArgs {
Description = String.Format("Reading Opex files for XSD schema validation."), Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Executing, PreingestAction = eventModel
});
var newOpexFiles = directoryInfoSessionFolder.GetFiles("*.opex").ToList();
var schemaList = SchemaHandler.GetSchemaList();
var strXsd = schemaList["Noord.Hollands.Archief.Preingest.WebApi.Schema.OPEX-Metadata.xsd"];
var xsdTmpFilename = Path.GetTempFileName();
var xsd = XDocument.Parse(strXsd);
xsd.Save(xsdTmpFilename);
OnTrigger(new PreingestEventArgs {
Description = String.Format("Validate Opex files with XSD schema."), Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Executing, PreingestAction = eventModel
});
newOpexFiles.ForEach(opex =>
{
try
{
if (String.IsNullOrEmpty(opex.FullName) || !File.Exists(opex.FullName))
{
throw new FileNotFoundException(String.Format("Opex file with location '{0}' is empty or not found!", opex.FullName));
}
XDocument xml = XDocument.Load(opex.FullName);
SchemaValidationHandler.Validate(xml.ToString(), xsdTmpFilename);
}
catch (Exception validate)
{
anyMessages.Add(String.Format("Schema validation error for Opex file '{0}'! {1} {2}", opex.FullName, validate.Message, validate.StackTrace));
}
});
try { File.Delete(xsdTmpFilename); } catch { }
eventModel.Properties.Messages = anyMessages.ToArray();
eventModel.ActionData = jsonData.ToArray();
eventModel.ActionResult.ResultValue = PreingestActionResults.Success;
eventModel.Summary.Processed = opexFiles.Count;
eventModel.Summary.Accepted = (opexFiles.Count - anyMessages.Count);
eventModel.Summary.Rejected = anyMessages.Count;
if (eventModel.Summary.Rejected > 0)
{
eventModel.ActionResult.ResultValue = PreingestActionResults.Error;
}
else
{
eventModel.ActionResult.ResultValue = PreingestActionResults.Success;
}
isSuccess = true;
}
catch (Exception e)
{
isSuccess = false;
anyMessages.Clear();
anyMessages.Add(String.Format("Run polish with collection: '{0}' failed!", TargetCollection));
anyMessages.Add(e.Message);
anyMessages.Add(e.StackTrace);
Logger.LogError(e, "Run polish with collection: '{0}' failed!", TargetCollection);
eventModel.ActionResult.ResultValue = PreingestActionResults.Failed;
eventModel.Properties.Messages = anyMessages.ToArray();
eventModel.Summary.Processed = 1;
eventModel.Summary.Accepted = 0;
eventModel.Summary.Rejected = 1;
OnTrigger(new PreingestEventArgs {
Description = "An exception occured while running checksum!", Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Failed, PreingestAction = eventModel
});
}
finally
{
if (isSuccess)
{
OnTrigger(new PreingestEventArgs {
Description = "Polish run with a collection is done.", Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Completed, PreingestAction = eventModel
});
}
}
}
public override void Execute()
{
var eventModel = CurrentActionProperties(TargetCollection, this.GetType().Name);
OnTrigger(new PreingestEventArgs {
Description = String.Format("Start building Opex for container '{0}'.", TargetCollection), Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Started, PreingestAction = eventModel
});
var anyMessages = new List <String>();
bool isSuccess = false;
try
{
base.Execute();
string sessionFolder = Path.Combine(ApplicationSettings.DataFolderName, SessionGuid.ToString());
BodySettings settings = new SettingsReader(this.ApplicationSettings.DataFolderName, SessionGuid).GetSettings();
if (settings == null)
{
throw new ApplicationException("Settings are not saved!");
}
if (!String.IsNullOrEmpty(settings.MergeRecordAndFile))
{
bool doMerge = settings.MergeRecordAndFile.Equals("Ja", StringComparison.InvariantCultureIgnoreCase);
if (doMerge && InheritanceSetting.MethodResult == InheritanceMethod.None)
{
//true && true
this.InheritanceSetting.MethodResult = InheritanceMethod.Combine;
}
}
string addNameSpaces = Path.Combine(ApplicationSettings.PreWashFolder, OpexItem.ADD_NAMESPACE);
string stripNameSpaces = Path.Combine(ApplicationSettings.PreWashFolder, OpexItem.STRIP_NAMESPACE);
string opexFolders = Path.Combine(ApplicationSettings.PreWashFolder, OpexItem.OPEX_FOLDERS);
string opexFolderFiles = Path.Combine(ApplicationSettings.PreWashFolder, OpexItem.OPEX_FOLDER_FILES);
string opexFiles = Path.Combine(ApplicationSettings.PreWashFolder, OpexItem.OPEX_FILES);
string opexFinalize = Path.Combine(ApplicationSettings.PreWashFolder, OpexItem.OPEX_FINALIZE);
if (!File.Exists(addNameSpaces))
{
throw new FileNotFoundException(String.Format("Stylesheet file not found '{0}'!", addNameSpaces));
}
if (!File.Exists(stripNameSpaces))
{
throw new FileNotFoundException(String.Format("Stylesheet file not found '{0}'!", stripNameSpaces));
}
if (!File.Exists(opexFolders))
{
throw new FileNotFoundException(String.Format("Stylesheet file not found '{0}'!", opexFolders));
}
if (!File.Exists(opexFolderFiles))
{
throw new FileNotFoundException(String.Format("Stylesheet file not found '{0}'!", opexFolderFiles));
}
if (!File.Exists(opexFiles))
{
throw new FileNotFoundException(String.Format("Stylesheet file not found '{0}'!", opexFiles));
}
if (!File.Exists(opexFinalize))
{
throw new FileNotFoundException(String.Format("Stylesheet file not found '{0}'!", opexFinalize));
}
List <StylesheetItem> stylesheetList = new List <StylesheetItem>();
OnTrigger(new PreingestEventArgs {
Description = String.Format("Load settings for building Opex.", TargetCollection), Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Executing, PreingestAction = eventModel
});
var stylesheets = Directory.GetFiles(ApplicationSettings.PreWashFolder, "*-opex-*.xsl").Select(item => new StylesheetItem {
KeyLocation = item, XmlContent = XDocument.Load(item).ToString()
}).ToArray();
stylesheetList.AddRange(stylesheets);
List <OpexItem> potentionalOpexList = new List <OpexItem>();
OnTrigger(new PreingestEventArgs {
Description = String.Format("Reading collection to build Opex for container '{0}'.", TargetCollection), Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Executing, PreingestAction = eventModel
});
//trigger event read collections
if (this.IsToPX)
{
var listOfMetadata = new DirectoryInfo(Path.Combine(ApplicationSettings.DataFolderName, SessionGuid.ToString())).GetFiles("*.metadata", SearchOption.AllDirectories).Select(item
=> new OpexItem(this.TargetFolder, item.FullName,
XDocument.Load(item.FullName).ToString(),
item.Directory.GetFiles("*.metadata", SearchOption.TopDirectoryOnly).Count())).ToArray();
potentionalOpexList.AddRange(listOfMetadata);
OnTrigger(new PreingestEventArgs {
Description = String.Format("Found '{0}' metadata ({1}) files.", potentionalOpexList.Count, this.IsToPX ? "ToPX" : "MDTO"), Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Executing, PreingestAction = eventModel
});
}
if (this.IsMDTO)
{
var listOfMetadata = new DirectoryInfo(Path.Combine(ApplicationSettings.DataFolderName, SessionGuid.ToString())).GetFiles("*.xml", SearchOption.AllDirectories).Where(item => item.Name.EndsWith(".mdto.xml", StringComparison.InvariantCultureIgnoreCase)).Select(item
=> new OpexItem(this.TargetFolder, item.FullName,
XDocument.Load(item.FullName).ToString(),
item.Directory.GetFiles("*.xml", SearchOption.TopDirectoryOnly).Where(item
=> item.Name.EndsWith(".mdto.xml", StringComparison.InvariantCultureIgnoreCase)).Count())).ToArray();
potentionalOpexList.AddRange(listOfMetadata);
OnTrigger(new PreingestEventArgs {
Description = String.Format("Found '{0}' metadata ({1}) files.", potentionalOpexList.Count, this.IsToPX ? "ToPX" : "MDTO"), Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Executing, PreingestAction = eventModel
});
}
OnTrigger(new PreingestEventArgs {
Description = String.Format("Creating Opex files."), Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Executing, PreingestAction = eventModel
});
//trigger event process opex
potentionalOpexList.ForEach(item =>
{
item.InitializeOpex(stylesheetList);
});
var jsonData = new List <String>();
OnTrigger(new PreingestEventArgs {
Description = String.Format("Removing original metadata files."), Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Executing, PreingestAction = eventModel
});
//remove the old metadata files topx or mdto
potentionalOpexList.ForEach(item =>
{
try
{
File.Delete(item.KeyLocation);
}
catch (Exception delete)
{
anyMessages.Add(String.Format("Deleting file '{0}' failed! {1} {2}", item.KeyLocation, delete.Message, delete.StackTrace));
}
});
OnTrigger(new PreingestEventArgs {
Description = String.Format("Saving Opex files (only file level)."), Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Executing, PreingestAction = eventModel
});
var sortedPotentionalOpexList = new List <OpexItem>();
if (InheritanceSetting.MethodResult == InheritanceMethod.Combine)
{
var sorted = potentionalOpexList.Where(item => !item.IsFile).GroupBy(item => item.Level, (key, opex) => new { Level = key, Item = opex }).OrderByDescending(item
=> item.Level).FirstOrDefault();
sortedPotentionalOpexList.AddRange(sorted.Item.OfType <OpexItem>());
}
//save bestand level first
potentionalOpexList.Where(item => item.IsFile).ToList().ForEach(item =>
{
try
{
var metadataList = new List <System.Xml.XmlElement>();
if (InheritanceSetting.MethodResult == InheritanceMethod.Combine)
{
var parts = item.KeyLocation.Split('/', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToArray();
var startsWith = parts.Skip(0).Take(parts.Count() - 1).ToArray();
var currentItemPath = "/" + Path.Combine(startsWith);
var profile = sortedPotentionalOpexList.FirstOrDefault(sorted => sorted.KeyLocation.StartsWith(currentItemPath));
if (profile == null)
{
throw new ApplicationException(String.Format("Adding profile to DescriptiveMetadata failed! Cannot find profile record in '{0}'", currentItemPath));
}
var xmlProfileDoc = new System.Xml.XmlDocument();
xmlProfileDoc.LoadXml(profile.OriginalMetadata);
metadataList.Add(xmlProfileDoc.DocumentElement);
}
if (InheritanceSetting.MethodResult == InheritanceMethod.None)
{
metadataList.Clear();
}
item.UpdateOpexFileLevel(ApplicationSettings.ChecksumServerName, ApplicationSettings.ChecksumServerPort, metadataList, true);
var xml = XDocument.Parse(item.FinalUpdatedOpexMetadata);
xml.Save(item.KeyOpexLocation);
}
catch (Exception save)
{
anyMessages.Add(String.Format("Saving Opex file '{0}' failed! {1} {2}", item.KeyOpexLocation, save.Message, save.StackTrace));
}
});
OnTrigger(new PreingestEventArgs {
Description = String.Format("Saving Opex files (folder levels)."), Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Executing, PreingestAction = eventModel
});
//update mappen levels
//save the rest of the folder levels
potentionalOpexList.Where(item => !item.IsFile).ToList().ForEach(item =>
{
try
{
item.UpdateOpexFolderLevel();
var xml = XDocument.Parse(item.FinalUpdatedOpexMetadata);
xml.Save(item.KeyOpexLocation);
}
catch (Exception save)
{
anyMessages.Add(String.Format("Saving Opex file '{0}' failed! {1} {2}", item.KeyOpexLocation, save.Message, save.StackTrace));
}
});
OnTrigger(new PreingestEventArgs {
Description = String.Format("Rounding up Opex."), Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Executing, PreingestAction = eventModel
});
//create container opex file,
//move the result to a folder named 'opex';
//this folder will be used by the bucket container for upload to S3
var currentCollectionDI = new DirectoryInfo(TargetFolder);
var currentCollectionFoldername = currentCollectionDI.GetDirectories().OrderBy(item => item.CreationTime).FirstOrDefault();
if (currentCollectionFoldername == null)
{
throw new DirectoryNotFoundException("Expanded collection folder not found!");
}
var opexContainerFilename = CreateContainerOpexMetadata(currentCollectionDI, currentCollectionFoldername.Name);
var opexUploadFolder = Directory.CreateDirectory(Path.Combine(TargetFolder, "opex"));
opexContainerFilename.MoveTo(Path.Combine(opexUploadFolder.FullName, opexContainerFilename.Name), true);
currentCollectionFoldername.MoveTo(Path.Combine(opexUploadFolder.FullName, currentCollectionFoldername.Name));
OnTrigger(new PreingestEventArgs {
Description = String.Format("Reading Opex files for XSD schema validation."), Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Executing, PreingestAction = eventModel
});
var newOpexFiles = opexUploadFolder.GetFiles("*.opex").ToList();
var schemaList = SchemaHandler.GetSchemaList();
var strXsd = schemaList["Noord.Hollands.Archief.Preingest.WebApi.Schema.OPEX-Metadata.xsd"];
var xsdTmpFilename = Path.GetTempFileName();
var xsd = XDocument.Parse(strXsd);
xsd.Save(xsdTmpFilename);
OnTrigger(new PreingestEventArgs {
Description = String.Format("Validate Opex files with XSD schema."), Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Executing, PreingestAction = eventModel
});
newOpexFiles.ForEach(opex =>
{
try
{
if (String.IsNullOrEmpty(opex.FullName) || !File.Exists(opex.FullName))
{
throw new FileNotFoundException(String.Format("Opex file with location '{0}' is empty or not found!", opex.FullName));
}
XDocument xml = XDocument.Load(opex.FullName);
SchemaValidationHandler.Validate(xml.ToString(), xsdTmpFilename);
}
catch (Exception validate)
{
anyMessages.Add(String.Format("Schema validation error for Opex file '{0}'! {1} {2}", opex.FullName, validate.Message, validate.StackTrace));
}
});
try { File.Delete(xsdTmpFilename); } catch { }
var result = potentionalOpexList.Select(item => String.Format("{0} >> {1}", item.KeyLocation, item.KeyOpexLocation)).ToArray();
jsonData.AddRange(result);
opexContainerFilename.Refresh();
jsonData.Add("All moved to folder 'opex'.");
jsonData.Add(opexContainerFilename.FullName);
eventModel.ActionData = jsonData.ToArray();
eventModel.Summary.Processed = potentionalOpexList.Count;
eventModel.Summary.Accepted = potentionalOpexList.Count;
eventModel.Summary.Rejected = anyMessages.Count();
eventModel.Properties.Messages = anyMessages.ToArray();
if (eventModel.Summary.Rejected > 0)
{
eventModel.ActionResult.ResultValue = PreingestActionResults.Error;
}
else
{
eventModel.ActionResult.ResultValue = PreingestActionResults.Success;
}
isSuccess = true;
}
catch (Exception e)
{
isSuccess = false;
anyMessages.Clear();
anyMessages.Add(String.Format("Build Opex with collection: '{0}' failed!", TargetCollection));
anyMessages.Add(e.Message);
anyMessages.Add(e.StackTrace);
Logger.LogError(e, "Build Opex with collection: '{0}' failed!", TargetCollection);
eventModel.ActionResult.ResultValue = PreingestActionResults.Failed;
eventModel.Properties.Messages = anyMessages.ToArray();
eventModel.Summary.Processed = 1;
eventModel.Summary.Accepted = 0;
eventModel.Summary.Rejected = 1;
OnTrigger(new PreingestEventArgs {
Description = "An exception occured while building opex!", Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Failed, PreingestAction = eventModel
});
}
finally
{
if (isSuccess)
{
OnTrigger(new PreingestEventArgs {
Description = "Build Opex with a collection is done.", Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Completed, PreingestAction = eventModel
});
}
}
}
