private void SetItemsSummary(DirectoryInfo target, PreingestEventArgs eventArgs)
{
eventArgs.Description = String.Format("Start counting objects.", TargetFolder);
OnTrigger(eventArgs);
FileInfo[] files = target.GetFiles("*.*", SearchOption.AllDirectories);
String extension = this.IsMDTO ? EXTENSION_MDTO : EXTENSION_TOPX;
//With MDTO , extra condition check with xml files
//directoryInfoSessionFolder.GetFiles("*.xml", SearchOption.AllDirectories).Where(item => item.Name.EndsWith(".mdto.xml", StringComparison.InvariantCultureIgnoreCase)).Count();
if (this.IsToPX)
{
var listOfMetadata = files.Where(item => item.FullName.EndsWith(extension, StringComparison.InvariantCultureIgnoreCase)).ToList();
XNamespace ns = "http://www.nationaalarchief.nl/ToPX/v2.3";
var xmlList = listOfMetadata.Select(xml => XDocument.Load(xml.FullName)).ToList();
var aggregaties = xmlList.Where(xml => xml.Root.Elements(ns + "aggregatie").Count() > 0).ToList();
var bestanden = xmlList.Where(xml => xml.Root.Elements(ns + "bestand").Count() > 0).ToList();
String[] messages = new String[]
{
String.Format("Archief : {0} item(s)", aggregaties.Count(item => item.Root.Element(ns + "aggregatie").Element(ns + "aggregatieniveau").Value.Equals("archief", StringComparison.InvariantCultureIgnoreCase))),
String.Format("Series : {0} item(s)", aggregaties.Count(item => item.Root.Element(ns + "aggregatie").Element(ns + "aggregatieniveau").Value.Equals("serie", StringComparison.InvariantCultureIgnoreCase))),
String.Format("Record : {0} item(s)", aggregaties.Count(item => item.Root.Element(ns + "aggregatie").Element(ns + "aggregatieniveau").Value.Equals("record", StringComparison.InvariantCultureIgnoreCase))),
String.Format("Dossier : {0} item(s)", aggregaties.Count(item => item.Root.Element(ns + "aggregatie").Element(ns + "aggregatieniveau").Value.Equals("dossier", StringComparison.InvariantCultureIgnoreCase))),
String.Format("Bestand : {0} item(s)", bestanden.Count())
};
//save the summary
eventArgs.PreingestAction.Properties.Messages = messages;
eventArgs.PreingestAction.Summary.Processed = listOfMetadata.Count;
}
if (this.IsMDTO)
{
//TODO
throw new ApplicationException("Not supported in this current version yet!");
}
}
private void ScanPath(UnixDirectoryInfo dirinfo, PreingestEventArgs passEventArgs)
{
passEventArgs.Description = String.Format("Processing folder '{0}'.", dirinfo.FullName);
OnTrigger(passEventArgs);
dirinfo.FileAccessPermissions = FileAccessPermissions.AllPermissions;
foreach (var fileinfo in dirinfo.GetFileSystemEntries())
{
switch (fileinfo.FileType)
{
case FileTypes.RegularFile:
fileinfo.FileAccessPermissions = FileAccessPermissions.AllPermissions;
break;
case FileTypes.Directory:
ScanPath((UnixDirectoryInfo)fileinfo, passEventArgs);
break;
default:
/* Do nothing for symlinks or other weird things. */
break;
}
}
}
public override void Execute()
{
var eventModel = CurrentActionProperties(TargetCollection, this.GetType().Name);
OnTrigger(new PreingestEventArgs {
Description = String.Format("Start expanding container '{0}'.", TargetCollection), Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Started, PreingestAction = eventModel
});
string output = string.Empty;
string error = string.Empty;
var anyMessages = new List <String>();
bool isSuccess = false;
try
{
if (!File.Exists(TargetCollection))
{
throw new FileNotFoundException("Collection not found!", TargetCollection);
}
string sessionFolder = Path.Combine(ApplicationSettings.DataFolderName, SessionGuid.ToString());
using (var tarProcess = new Process()
{
StartInfo = new ProcessStartInfo
{
FileName = "tar",
Arguments = String.Format("-C \"{0}\" -oxvf \"{1}\"", sessionFolder, TargetCollection),
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true,
}
})
{
tarProcess.Start();
OnTrigger(new PreingestEventArgs {
Description = "Container is expanding content.", Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Executing, PreingestAction = eventModel
});
this.Logger.LogDebug("Unpacking container '{0}'", TargetCollection);
output = tarProcess.StandardOutput.ReadToEnd();
error = tarProcess.StandardError.ReadToEnd();
tarProcess.WaitForExit();
}
if (!String.IsNullOrEmpty(output))
{
this.Logger.LogDebug(output);
}
if (!String.IsNullOrEmpty(error))
{
this.Logger.LogDebug(error);
}
var fileInformation = new FileInfo(TargetCollection);
anyMessages.Add(String.Concat("Name : ", fileInformation.Name));
anyMessages.Add(String.Concat("Extension : ", fileInformation.Extension));
anyMessages.Add(String.Concat("Size : ", fileInformation.Length));
anyMessages.Add(String.Concat("CreationTime : ", fileInformation.CreationTimeUtc));
anyMessages.Add(String.Concat("LastAccessTime : ", fileInformation.LastAccessTimeUtc));
anyMessages.Add(String.Concat("LastWriteTime : ", fileInformation.LastWriteTimeUtc));
eventModel.Properties.Messages = anyMessages.ToArray();
bool isWindows = RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.Windows);
if (!isWindows)
{
var unixDirInfo = new UnixDirectoryInfo(sessionFolder);
//trigger event executing
var passEventArgs = new PreingestEventArgs {
Description = String.Format("Execute chmod 777 for container '{0}'", TargetCollection), Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Executing, PreingestAction = eventModel
};
OnTrigger(passEventArgs);
ScanPath(unixDirInfo, passEventArgs);
}
isSuccess = true;
}
catch (Exception e)
{
isSuccess = false;
anyMessages.Clear();
anyMessages.Add(String.Format("Unpack container file: '{0}' failed!", TargetCollection));
anyMessages.Add(e.Message);
anyMessages.Add(e.StackTrace);
Logger.LogError(e, "Unpack container file: '{0}' failed!", TargetCollection);
eventModel.ActionResult.ResultValue = PreingestActionResults.Failed;
eventModel.Properties.Messages = anyMessages.ToArray();
eventModel.Summary.Processed = 1;
eventModel.Summary.Accepted = 0;
eventModel.Summary.Rejected = 1;
OnTrigger(new PreingestEventArgs {
Description = "An exception occured while unpacking a container!", Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Failed, PreingestAction = eventModel
});
}
finally
{
if (isSuccess)
{
eventModel.ActionResult.ResultValue = PreingestActionResults.Success;
eventModel.Summary.Processed = 1;
eventModel.Summary.Accepted = 1;
eventModel.Summary.Rejected = 0;
eventModel.ActionData = output.Split(Environment.NewLine);
OnTrigger(new PreingestEventArgs {
Description = "Unpacking the container is done.", Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Completed, PreingestAction = eventModel
});
}
}
}
private void DirectoryRecursion(DirectoryInfo currentFolder, List <NamingItem> procesResult, PreingestEventArgs model)
{
model.Description = String.Format("Checking folder '{0}'.", currentFolder.FullName);
OnTrigger(model);
this.Logger.LogDebug("Checking folder '{0}'.", currentFolder.FullName);
bool checkResult = ContainsInvalidCharacters(currentFolder.Name);
bool checkResultNames = ContainsAnyDOSNames(currentFolder.Name);
var errorMessages = new List <String>();
if (checkResult)
{
errorMessages.Add(String.Format("Een of meerdere niet-toegestane bijzondere tekens komen voor in de map - of bestandsnaam '{0} ({1})'.", currentFolder.Name, currentFolder.FullName));
}
if (checkResultNames)
{
errorMessages.Add(String.Format("De map of het bestand '{0} ({1})' heeft een niet-toegestane naam.", currentFolder.Name, currentFolder.FullName));
}
procesResult.Add(new NamingItem {
ContainsInvalidCharacters = checkResult, ContainsDosNames = checkResultNames, Name = currentFolder.FullName, ErrorMessages = errorMessages.ToArray()
});
currentFolder.GetFiles().ToList().ForEach(item =>
{
model.Description = String.Format("Checking file '{0}'.", item.FullName);
OnTrigger(model);
this.Logger.LogDebug("Checking file '{0}'", currentFolder.FullName);
var errorMessages = new List <String>();
bool checkResult = ContainsInvalidCharacters(item.Name);
if (checkResult)
{
errorMessages.Add(String.Format("Een of meerdere niet-toegestane bijzondere tekens komen voor in de map - of bestandsnaam '{0} ({1})'", item.Name, item.FullName));
}
bool checkResultNames = ContainsAnyDOSNames(item.Name);
if (checkResultNames)
{
errorMessages.Add(String.Format("De map of het bestand '{0} ({1})' heeft een niet-toegestane naam.", item.Name, item.FullName));
}
procesResult.Add(new NamingItem {
ContainsInvalidCharacters = checkResult, ContainsDosNames = checkResultNames, Name = item.FullName, ErrorMessages = errorMessages.ToArray()
});
});
foreach (var directory in currentFolder.GetDirectories())
{
DirectoryRecursion(directory, procesResult, model);
}
}
public override void Execute()
{
var eventModel = CurrentActionProperties(TargetCollection, this.GetType().Name);
OnTrigger(new PreingestEventArgs {
Description = "Start sidecar structure validation.", Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Started, PreingestAction = eventModel
});
bool isSucces = false;
List <ISidecar> sidecarTreeNode = new List <ISidecar>();
try
{
base.Execute();
var collection = new DirectoryInfo(TargetFolder).GetDirectories().OrderBy(item => item.CreationTime).First();
if (collection == null)
{
throw new DirectoryNotFoundException(String.Format("Folder '{0}' not found!", TargetFolder));
}
PreingestEventArgs execEventArgs = new PreingestEventArgs {
Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Executing, PreingestAction = eventModel
};
var start = DateTime.Now;
Logger.LogInformation("Start validate sidecar structure in '{0}'.", TargetFolder);
Logger.LogInformation("Start time {0}", start);
SetItemsSummary(collection, execEventArgs);
List <String> total = new List <string>();
var structureMessages = ValidateStructure(collection, execEventArgs);
if (structureMessages.Count > 0)
{
total.AddRange(structureMessages);
}
else
{
var aggregationMessages = ValidateAggregation(collection, execEventArgs);
total.AddRange(aggregationMessages);
}
eventModel.Summary.Accepted = 0; // validationResult.Where(item => item.IsCorrect).Count();
eventModel.Summary.Rejected = total.Count; // validationResult.Where(item => !item.IsCorrect).Count();
eventModel.ActionData = total.ToArray();
if (eventModel.Summary.Rejected > 0)
{
eventModel.ActionResult.ResultValue = PreingestActionResults.Error;
}
else
{
eventModel.ActionResult.ResultValue = PreingestActionResults.Success;
}
var end = DateTime.Now;
Logger.LogInformation("End of the validation for the sidecar structure.");
Logger.LogInformation("End time {0}", end);
TimeSpan processTime = (TimeSpan)(end - start);
Logger.LogInformation(String.Format("Processed in {0} ms.", processTime));
isSucces = true;
}
catch (Exception e)
{
isSucces = false;
Logger.LogError(e, "Exception occured in sidecar structure validation!");
var anyMessages = new List <String>();
anyMessages.Clear();
anyMessages.Add("Exception occured in sidecar structure validation!");
anyMessages.Add(e.Message);
anyMessages.Add(e.StackTrace);
eventModel.Summary.Processed = -1;
eventModel.Summary.Accepted = -1;
eventModel.Summary.Rejected = -1;
eventModel.ActionResult.ResultValue = PreingestActionResults.Failed;
eventModel.Properties.Messages = anyMessages.ToArray();
OnTrigger(new PreingestEventArgs {
Description = "An exception occured in sidecar structure validation!", Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Failed, PreingestAction = eventModel
});
}
finally
{
if (isSucces)
{
OnTrigger(new PreingestEventArgs {
Description = "Sidecar structure validation is done.", Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Completed, PreingestAction = eventModel, SidecarStructure = sidecarTreeNode
});
}
}
}
private List <String> ValidateAggregation(DirectoryInfo target, PreingestEventArgs eventArgs)
{
eventArgs.Description = String.Format("Start validate aggregation combination.", TargetFolder);
OnTrigger(eventArgs);
List <String> messages = new List <string>();
FileInfo[] files = target.GetFiles("*.*", SearchOption.AllDirectories);
String extension = this.IsMDTO ? EXTENSION_MDTO : EXTENSION_TOPX;
var listOfBinary = files.Where(item => !item.FullName.EndsWith(extension)).ToList();
var listOfMetadata = files.Where(item => item.FullName.EndsWith(extension)).ToList();
//assumption, need extra double check!!!
//With MDTO , extra condition check with xml files
//directoryInfoSessionFolder.GetFiles("*.xml", SearchOption.AllDirectories).Where(item => item.Name.EndsWith(".mdto.xml", StringComparison.InvariantCultureIgnoreCase)).Count();
string collectionArchiveName = string.Empty;
//opex container
if (target.GetFiles((this.IsToPX) ? String.Concat("*", EXTENSION_TOPX) : String.Concat("*", EXTENSION_MDTO), SearchOption.TopDirectoryOnly).Count() == 1)
{
//current folder is archive collection name
collectionArchiveName = target.Name;
}
else
{
if (target.GetDirectories().Count() == 0)
{
throw new DirectoryNotFoundException(String.Format("No collection folder found in '{0}'!", target.FullName));
}
//first folder is archive collection name
collectionArchiveName = target.GetDirectories().FirstOrDefault().Name;
}
if (this.IsToPX)
{
/** topx
* Archief
* Serie 0..*
* Dossier 0..*
*
* Serie
* Serie 0..*
*
* Dossier
* Record 0..*
* Bestand 1..*,
* Dossier 0..*
**/
var scanResult = AggregationToPX(listOfBinary, listOfMetadata, collectionArchiveName);
foreach (var keyValuePair in scanResult)
{
string currentBestandMetadataFile = keyValuePair.Key;
var structureConstruction = keyValuePair.Value.Select(item => item.Root.Elements().First().Name.LocalName).Reverse();
XNamespace ns = keyValuePair.Value.Select(item => item.Root.GetDefaultNamespace()).Distinct().First();
var aggregationLevel = keyValuePair.Value.Select(item =>
(item.Root.Elements().First().Name.LocalName == "bestand") ? "Bestand" :
item.Root.Elements().First().Element(ns + "aggregatieniveau").Value).Reverse();
//aggregation construction test
var structureValue = String.Join('|', structureConstruction);
var levelValue = String.Join('|', aggregationLevel);
string pattern = @"^(aggregatie\|){1,}(bestand){1}$";
System.Text.RegularExpressions.Regex regex = new System.Text.RegularExpressions.Regex(pattern);
bool isMatch = regex.IsMatch(structureValue);
if (!isMatch)
{
messages.Add(String.Format("Invalid ToPX element(s) construction {0} (top/down) found.", structureValue));
}
//aggregation levels acceptable route
pattern = @"^(Archief\|){1}(Serie\|){0,}(Dossier\|){1,}(Record\|){0,}(Bestand){1}$";
regex = new System.Text.RegularExpressions.Regex(pattern);
isMatch = regex.IsMatch(levelValue);
if (!isMatch)
{
messages.Add(String.Format("Invalid aggregation levels {0} (top/down) found for: {1}", levelValue, currentBestandMetadataFile));
}
}
}
//TODO : MDTO PART
if (this.IsMDTO)
{
//mdto
/***
* InformatieObject (Archief)
* InformatieObject (Serie)
* InformatieObject (Dossier)
* InformatieObject (Archiefstuk)
* Bestand
*
* var scanResult = AggregationMDTO(listOfBinary, listOfMetadata, collectionArchiveName);
*
* foreach (var keyValuePair in scanResult)
* {
* string currentBestandMetadataFile = keyValuePair.Key;
*
* var construction = keyValuePair.Value.Select(item
* => item.Root.Element("informatieobject") == null ? item.Root.Element("bestand").Name.LocalName : item.Root.Element("informatieobject").Name.LocalName).Reverse();
*
* var structureValue = String.Join('|', construction);
*
* //aggregatie element test
* string pattern = @"^(informatieobject\|){1,}(bestand){1}$";
*
* System.Text.RegularExpressions.Regex regex = new System.Text.RegularExpressions.Regex(pattern);
* bool isMatch = regex.IsMatch(structureValue);
*
* if (!isMatch)
* {
* //add validation error
* //this.ValidationErrorList.Add(new ValidationError
* //{
* // CurrentMetadataFullname = currentBestandMetadataFile,
* // Message = String.Format("Invalid MDTO element(s) construction {0} (top/down) found.", structureValue),
* // FullException = new ApplicationException(String.Format("Invalid aggregation construction {0} (top/down) found.", structureValue))
* //});
* }
* //^(Archief\|){0,}(Serie\|){0,}(Dossier\|){1,}(Archiefstuk\|){1,}(Bestand){1}$
* }
**/
}
return(messages);
}
private List <String> ValidateStructure(DirectoryInfo target, PreingestEventArgs eventArgs)
{
eventArgs.Description = String.Format("Start validate collection structure.", TargetFolder);
OnTrigger(eventArgs);
List <String> messages = new List <string>();
if (target == null)
{
throw new ArgumentNullException("Input parameter is null! Expect DirectoryInfo as input.");
}
if (!target.Exists)
{
throw new DirectoryNotFoundException(String.Format("Not found: '{0}'.", target.FullName));
}
FileInfo[] files = target.GetFiles("*.*", SearchOption.AllDirectories);
DirectoryInfo[] directories = Directory.GetDirectories(target.FullName, "*", SearchOption.AllDirectories).Select(item => new DirectoryInfo(item)).ToArray();
int sum = (files.Count() + directories.Count() + 1); // plus current folder itself;
if ((sum % 2) != 0) //odd, not even. Expected even....
{
messages.Add(String.Format("The total sum of objects is not even! Expect an even count of objects. Files: {0}, Directories: {1}", files.Count(), directories.Count() + 1));
}
//total list of objects (stripped without the .metadata);
List <FileSystemInfo> strippedCombinedCollection = new List <FileSystemInfo>();
var filesStrippedMetadataExtension = files.Where(item
=> !item.Extension.Equals(this.IsMDTO
? EXTENSION_MDTO : EXTENSION_TOPX, StringComparison.InvariantCultureIgnoreCase)).ToArray();
strippedCombinedCollection.AddRange(filesStrippedMetadataExtension);
strippedCombinedCollection.AddRange(directories);
var filesOnlyMetadataExtension = files.Where(item
=> item.Extension.Equals(this.IsMDTO ?
EXTENSION_MDTO : EXTENSION_TOPX, StringComparison.InvariantCultureIgnoreCase)).ToList();
//With MDTO , extra condition check with xml files
//directoryInfoSessionFolder.GetFiles("*.xml", SearchOption.AllDirectories).Where(item => item.Name.EndsWith(".mdto.xml", StringComparison.InvariantCultureIgnoreCase)).Count();
//check for each object if it's paired with a metadata file ToPX (.metadata) or MDTO (.mdto.xml);
foreach (var collectionObject in strippedCombinedCollection)
{
string fullnameMetadata = String.Empty;
if (collectionObject is FileInfo)
{
fullnameMetadata = String.Concat(collectionObject.FullName, this.IsMDTO
? EXTENSION_MDTO : EXTENSION_TOPX);
}
else
{
fullnameMetadata = String.Concat(Path.Combine(collectionObject.FullName, collectionObject.Name), this.IsMDTO
? EXTENSION_MDTO : EXTENSION_TOPX);
}
bool exists = filesOnlyMetadataExtension.Exists((informationObject) =>
{
return(File.Exists(fullnameMetadata));
});
if (!exists)
{
string message = String.Format("Missing {0} metadata file for object '{1}'.", this.IsMDTO
? EXTENSION_MDTO : EXTENSION_TOPX, collectionObject.Name);
StringBuilder messageBuilder = new StringBuilder();
messageBuilder.AppendLine(message);
messageBuilder.AppendLine(String.Format("Object type: {0}.", (collectionObject is FileInfo) ? "File" : "Directory"));
messageBuilder.AppendLine(String.Format("Target: {0}.", collectionObject.FullName));
messageBuilder.AppendLine(String.Format("Expected: {0}.", fullnameMetadata));
messages.Add(messageBuilder.ToString());
}
}
return(messages);
}
