private void SetItemsSummary(DirectoryInfo target, PreingestEventArgs eventArgs)
        {
            eventArgs.Description = String.Format("Start counting objects.", TargetFolder);
            OnTrigger(eventArgs);

            FileInfo[] files     = target.GetFiles("*.*", SearchOption.AllDirectories);
            String     extension = this.IsMDTO ? EXTENSION_MDTO : EXTENSION_TOPX;

            //With MDTO , extra condition check with xml files
            //directoryInfoSessionFolder.GetFiles("*.xml", SearchOption.AllDirectories).Where(item => item.Name.EndsWith(".mdto.xml", StringComparison.InvariantCultureIgnoreCase)).Count();

            if (this.IsToPX)
            {
                var listOfMetadata = files.Where(item => item.FullName.EndsWith(extension, StringComparison.InvariantCultureIgnoreCase)).ToList();

                XNamespace ns = "http://www.nationaalarchief.nl/ToPX/v2.3";

                var xmlList = listOfMetadata.Select(xml => XDocument.Load(xml.FullName)).ToList();

                var aggregaties = xmlList.Where(xml => xml.Root.Elements(ns + "aggregatie").Count() > 0).ToList();
                var bestanden   = xmlList.Where(xml => xml.Root.Elements(ns + "bestand").Count() > 0).ToList();

                String[] messages = new String[]
                {
                    String.Format("Archief : {0} item(s)", aggregaties.Count(item => item.Root.Element(ns + "aggregatie").Element(ns + "aggregatieniveau").Value.Equals("archief", StringComparison.InvariantCultureIgnoreCase))),
                    String.Format("Series : {0} item(s)", aggregaties.Count(item => item.Root.Element(ns + "aggregatie").Element(ns + "aggregatieniveau").Value.Equals("serie", StringComparison.InvariantCultureIgnoreCase))),
                    String.Format("Record : {0} item(s)", aggregaties.Count(item => item.Root.Element(ns + "aggregatie").Element(ns + "aggregatieniveau").Value.Equals("record", StringComparison.InvariantCultureIgnoreCase))),
                    String.Format("Dossier : {0} item(s)", aggregaties.Count(item => item.Root.Element(ns + "aggregatie").Element(ns + "aggregatieniveau").Value.Equals("dossier", StringComparison.InvariantCultureIgnoreCase))),
                    String.Format("Bestand : {0} item(s)", bestanden.Count())
                };
                //save the summary
                eventArgs.PreingestAction.Properties.Messages = messages;
                eventArgs.PreingestAction.Summary.Processed   = listOfMetadata.Count;
            }

            if (this.IsMDTO)
            {
                //TODO
                throw new ApplicationException("Not supported in this current version yet!");
            }
        }
        private void ScanPath(UnixDirectoryInfo dirinfo, PreingestEventArgs passEventArgs)
        {
            passEventArgs.Description = String.Format("Processing folder '{0}'.", dirinfo.FullName);
            OnTrigger(passEventArgs);
            dirinfo.FileAccessPermissions = FileAccessPermissions.AllPermissions;
            foreach (var fileinfo in dirinfo.GetFileSystemEntries())
            {
                switch (fileinfo.FileType)
                {
                case FileTypes.RegularFile:
                    fileinfo.FileAccessPermissions = FileAccessPermissions.AllPermissions;
                    break;

                case FileTypes.Directory:
                    ScanPath((UnixDirectoryInfo)fileinfo, passEventArgs);
                    break;

                default:
                    /* Do nothing for symlinks or other weird things. */
                    break;
                }
            }
        }
        public override void Execute()
        {
            var eventModel = CurrentActionProperties(TargetCollection, this.GetType().Name);

            OnTrigger(new PreingestEventArgs {
                Description = String.Format("Start expanding container '{0}'.", TargetCollection), Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Started, PreingestAction = eventModel
            });

            string output = string.Empty;
            string error  = string.Empty;

            var  anyMessages = new List <String>();
            bool isSuccess   = false;

            try
            {
                if (!File.Exists(TargetCollection))
                {
                    throw new FileNotFoundException("Collection not found!", TargetCollection);
                }

                string sessionFolder = Path.Combine(ApplicationSettings.DataFolderName, SessionGuid.ToString());

                using (var tarProcess = new Process()
                {
                    StartInfo = new ProcessStartInfo
                    {
                        FileName = "tar",
                        Arguments = String.Format("-C \"{0}\" -oxvf \"{1}\"", sessionFolder, TargetCollection),
                        RedirectStandardOutput = true,
                        RedirectStandardError = true,
                        UseShellExecute = false,
                        CreateNoWindow = true,
                    }
                })
                {
                    tarProcess.Start();
                    OnTrigger(new PreingestEventArgs {
                        Description = "Container is expanding content.", Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Executing, PreingestAction = eventModel
                    });

                    this.Logger.LogDebug("Unpacking container '{0}'", TargetCollection);

                    output = tarProcess.StandardOutput.ReadToEnd();
                    error  = tarProcess.StandardError.ReadToEnd();

                    tarProcess.WaitForExit();
                }

                if (!String.IsNullOrEmpty(output))
                {
                    this.Logger.LogDebug(output);
                }

                if (!String.IsNullOrEmpty(error))
                {
                    this.Logger.LogDebug(error);
                }

                var fileInformation = new FileInfo(TargetCollection);
                anyMessages.Add(String.Concat("Name : ", fileInformation.Name));
                anyMessages.Add(String.Concat("Extension : ", fileInformation.Extension));
                anyMessages.Add(String.Concat("Size : ", fileInformation.Length));
                anyMessages.Add(String.Concat("CreationTime : ", fileInformation.CreationTimeUtc));
                anyMessages.Add(String.Concat("LastAccessTime : ", fileInformation.LastAccessTimeUtc));
                anyMessages.Add(String.Concat("LastWriteTime : ", fileInformation.LastWriteTimeUtc));

                eventModel.Properties.Messages = anyMessages.ToArray();

                bool isWindows = RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.Windows);
                if (!isWindows)
                {
                    var unixDirInfo = new UnixDirectoryInfo(sessionFolder);
                    //trigger event executing
                    var passEventArgs = new PreingestEventArgs {
                        Description = String.Format("Execute chmod 777 for container '{0}'", TargetCollection), Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Executing, PreingestAction = eventModel
                    };
                    OnTrigger(passEventArgs);
                    ScanPath(unixDirInfo, passEventArgs);
                }

                isSuccess = true;
            }
            catch (Exception e)
            {
                isSuccess = false;
                anyMessages.Clear();
                anyMessages.Add(String.Format("Unpack container file: '{0}' failed!", TargetCollection));
                anyMessages.Add(e.Message);
                anyMessages.Add(e.StackTrace);

                Logger.LogError(e, "Unpack container file: '{0}' failed!", TargetCollection);

                eventModel.ActionResult.ResultValue = PreingestActionResults.Failed;
                eventModel.Properties.Messages      = anyMessages.ToArray();
                eventModel.Summary.Processed        = 1;
                eventModel.Summary.Accepted         = 0;
                eventModel.Summary.Rejected         = 1;

                OnTrigger(new PreingestEventArgs {
                    Description = "An exception occured while unpacking a container!", Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Failed, PreingestAction = eventModel
                });
            }
            finally
            {
                if (isSuccess)
                {
                    eventModel.ActionResult.ResultValue = PreingestActionResults.Success;
                    eventModel.Summary.Processed        = 1;
                    eventModel.Summary.Accepted         = 1;
                    eventModel.Summary.Rejected         = 0;
                    eventModel.ActionData = output.Split(Environment.NewLine);

                    OnTrigger(new PreingestEventArgs {
                        Description = "Unpacking the container is done.", Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Completed, PreingestAction = eventModel
                    });
                }
            }
        }
示例#4
0
        private void DirectoryRecursion(DirectoryInfo currentFolder, List <NamingItem> procesResult, PreingestEventArgs model)
        {
            model.Description = String.Format("Checking folder '{0}'.", currentFolder.FullName);
            OnTrigger(model);

            this.Logger.LogDebug("Checking folder '{0}'.", currentFolder.FullName);

            bool checkResult      = ContainsInvalidCharacters(currentFolder.Name);
            bool checkResultNames = ContainsAnyDOSNames(currentFolder.Name);
            var  errorMessages    = new List <String>();

            if (checkResult)
            {
                errorMessages.Add(String.Format("Een of meerdere niet-toegestane bijzondere tekens komen voor in de map - of bestandsnaam '{0} ({1})'.", currentFolder.Name, currentFolder.FullName));
            }
            if (checkResultNames)
            {
                errorMessages.Add(String.Format("De map of het bestand '{0} ({1})' heeft een niet-toegestane naam.", currentFolder.Name, currentFolder.FullName));
            }

            procesResult.Add(new NamingItem {
                ContainsInvalidCharacters = checkResult, ContainsDosNames = checkResultNames, Name = currentFolder.FullName, ErrorMessages = errorMessages.ToArray()
            });

            currentFolder.GetFiles().ToList().ForEach(item =>
            {
                model.Description = String.Format("Checking file '{0}'.", item.FullName);
                OnTrigger(model);

                this.Logger.LogDebug("Checking file '{0}'", currentFolder.FullName);
                var errorMessages = new List <String>();
                bool checkResult  = ContainsInvalidCharacters(item.Name);
                if (checkResult)
                {
                    errorMessages.Add(String.Format("Een of meerdere niet-toegestane bijzondere tekens komen voor in de map - of bestandsnaam '{0} ({1})'", item.Name, item.FullName));
                }
                bool checkResultNames = ContainsAnyDOSNames(item.Name);
                if (checkResultNames)
                {
                    errorMessages.Add(String.Format("De map of het bestand '{0} ({1})' heeft een niet-toegestane naam.", item.Name, item.FullName));
                }

                procesResult.Add(new NamingItem {
                    ContainsInvalidCharacters = checkResult, ContainsDosNames = checkResultNames, Name = item.FullName, ErrorMessages = errorMessages.ToArray()
                });
            });

            foreach (var directory in currentFolder.GetDirectories())
            {
                DirectoryRecursion(directory, procesResult, model);
            }
        }
        public override void Execute()
        {
            var eventModel = CurrentActionProperties(TargetCollection, this.GetType().Name);

            OnTrigger(new PreingestEventArgs {
                Description = "Start sidecar structure validation.", Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Started, PreingestAction = eventModel
            });
            bool isSucces = false;

            List <ISidecar> sidecarTreeNode = new List <ISidecar>();

            try
            {
                base.Execute();

                var collection = new DirectoryInfo(TargetFolder).GetDirectories().OrderBy(item => item.CreationTime).First();
                if (collection == null)
                {
                    throw new DirectoryNotFoundException(String.Format("Folder '{0}' not found!", TargetFolder));
                }

                PreingestEventArgs execEventArgs = new PreingestEventArgs {
                    Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Executing, PreingestAction = eventModel
                };

                var start = DateTime.Now;
                Logger.LogInformation("Start validate sidecar structure in '{0}'.", TargetFolder);
                Logger.LogInformation("Start time {0}", start);

                SetItemsSummary(collection, execEventArgs);

                List <String> total             = new List <string>();
                var           structureMessages = ValidateStructure(collection, execEventArgs);
                if (structureMessages.Count > 0)
                {
                    total.AddRange(structureMessages);
                }
                else
                {
                    var aggregationMessages = ValidateAggregation(collection, execEventArgs);
                    total.AddRange(aggregationMessages);
                }

                eventModel.Summary.Accepted = 0;           // validationResult.Where(item => item.IsCorrect).Count();
                eventModel.Summary.Rejected = total.Count; // validationResult.Where(item => !item.IsCorrect).Count();

                eventModel.ActionData = total.ToArray();

                if (eventModel.Summary.Rejected > 0)
                {
                    eventModel.ActionResult.ResultValue = PreingestActionResults.Error;
                }
                else
                {
                    eventModel.ActionResult.ResultValue = PreingestActionResults.Success;
                }

                var end = DateTime.Now;
                Logger.LogInformation("End of the validation for the sidecar structure.");
                Logger.LogInformation("End time {0}", end);
                TimeSpan processTime = (TimeSpan)(end - start);
                Logger.LogInformation(String.Format("Processed in {0} ms.", processTime));

                isSucces = true;
            }
            catch (Exception e)
            {
                isSucces = false;
                Logger.LogError(e, "Exception occured in sidecar structure validation!");

                var anyMessages = new List <String>();
                anyMessages.Clear();
                anyMessages.Add("Exception occured in sidecar structure validation!");
                anyMessages.Add(e.Message);
                anyMessages.Add(e.StackTrace);

                eventModel.Summary.Processed = -1;
                eventModel.Summary.Accepted  = -1;
                eventModel.Summary.Rejected  = -1;

                eventModel.ActionResult.ResultValue = PreingestActionResults.Failed;
                eventModel.Properties.Messages      = anyMessages.ToArray();

                OnTrigger(new PreingestEventArgs {
                    Description = "An exception occured in sidecar structure validation!", Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Failed, PreingestAction = eventModel
                });
            }
            finally
            {
                if (isSucces)
                {
                    OnTrigger(new PreingestEventArgs {
                        Description = "Sidecar structure validation is done.", Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Completed, PreingestAction = eventModel, SidecarStructure = sidecarTreeNode
                    });
                }
            }
        }
        private List <String> ValidateAggregation(DirectoryInfo target, PreingestEventArgs eventArgs)
        {
            eventArgs.Description = String.Format("Start validate aggregation combination.", TargetFolder);
            OnTrigger(eventArgs);

            List <String> messages = new List <string>();

            FileInfo[] files = target.GetFiles("*.*", SearchOption.AllDirectories);

            String extension = this.IsMDTO ? EXTENSION_MDTO : EXTENSION_TOPX;

            var listOfBinary   = files.Where(item => !item.FullName.EndsWith(extension)).ToList();
            var listOfMetadata = files.Where(item => item.FullName.EndsWith(extension)).ToList();

            //assumption, need extra double check!!!
            //With MDTO , extra condition check with xml files
            //directoryInfoSessionFolder.GetFiles("*.xml", SearchOption.AllDirectories).Where(item => item.Name.EndsWith(".mdto.xml", StringComparison.InvariantCultureIgnoreCase)).Count();

            string collectionArchiveName = string.Empty;

            //opex container
            if (target.GetFiles((this.IsToPX) ? String.Concat("*", EXTENSION_TOPX) : String.Concat("*", EXTENSION_MDTO), SearchOption.TopDirectoryOnly).Count() == 1)
            {
                //current folder is archive collection name
                collectionArchiveName = target.Name;
            }
            else
            {
                if (target.GetDirectories().Count() == 0)
                {
                    throw new DirectoryNotFoundException(String.Format("No collection folder found in '{0}'!", target.FullName));
                }
                //first folder is archive collection name
                collectionArchiveName = target.GetDirectories().FirstOrDefault().Name;
            }

            if (this.IsToPX)
            {
                /** topx
                 * Archief
                 *  Serie 0..*
                 *  Dossier 0..*
                 *
                 *  Serie
                 *      Serie 0..*
                 *
                 * Dossier
                 *  Record 0..*
                 *  Bestand 1..*,
                 *  Dossier 0..*
                 **/

                var scanResult = AggregationToPX(listOfBinary, listOfMetadata, collectionArchiveName);

                foreach (var keyValuePair in scanResult)
                {
                    string currentBestandMetadataFile = keyValuePair.Key;

                    var        structureConstruction = keyValuePair.Value.Select(item => item.Root.Elements().First().Name.LocalName).Reverse();
                    XNamespace ns = keyValuePair.Value.Select(item => item.Root.GetDefaultNamespace()).Distinct().First();

                    var aggregationLevel = keyValuePair.Value.Select(item =>
                                                                     (item.Root.Elements().First().Name.LocalName == "bestand") ? "Bestand" :
                                                                     item.Root.Elements().First().Element(ns + "aggregatieniveau").Value).Reverse();

                    //aggregation construction test
                    var structureValue = String.Join('|', structureConstruction);
                    var levelValue     = String.Join('|', aggregationLevel);

                    string pattern = @"^(aggregatie\|){1,}(bestand){1}$";
                    System.Text.RegularExpressions.Regex regex = new System.Text.RegularExpressions.Regex(pattern);
                    bool isMatch = regex.IsMatch(structureValue);
                    if (!isMatch)
                    {
                        messages.Add(String.Format("Invalid ToPX element(s) construction {0} (top/down) found.", structureValue));
                    }

                    //aggregation levels acceptable route
                    pattern = @"^(Archief\|){1}(Serie\|){0,}(Dossier\|){1,}(Record\|){0,}(Bestand){1}$";
                    regex   = new System.Text.RegularExpressions.Regex(pattern);
                    isMatch = regex.IsMatch(levelValue);
                    if (!isMatch)
                    {
                        messages.Add(String.Format("Invalid aggregation levels {0} (top/down) found for: {1}", levelValue, currentBestandMetadataFile));
                    }
                }
            }

            //TODO : MDTO PART

            if (this.IsMDTO)
            {
                //mdto

                /***
                 * InformatieObject (Archief)
                 *      InformatieObject (Serie)
                 *          InformatieObject (Dossier)
                 *              InformatieObject (Archiefstuk)
                 *                  Bestand
                 *
                 * var scanResult = AggregationMDTO(listOfBinary, listOfMetadata, collectionArchiveName);
                 *
                 * foreach (var keyValuePair in scanResult)
                 * {
                 *  string currentBestandMetadataFile = keyValuePair.Key;
                 *
                 *  var construction = keyValuePair.Value.Select(item
                 *      => item.Root.Element("informatieobject") == null ? item.Root.Element("bestand").Name.LocalName : item.Root.Element("informatieobject").Name.LocalName).Reverse();
                 *
                 *  var structureValue = String.Join('|', construction);
                 *
                 *  //aggregatie element test
                 *  string pattern = @"^(informatieobject\|){1,}(bestand){1}$";
                 *
                 *  System.Text.RegularExpressions.Regex regex = new System.Text.RegularExpressions.Regex(pattern);
                 *  bool isMatch = regex.IsMatch(structureValue);
                 *
                 *  if (!isMatch)
                 *  {
                 *      //add validation error
                 *      //this.ValidationErrorList.Add(new ValidationError
                 *      //{
                 *      //    CurrentMetadataFullname = currentBestandMetadataFile,
                 *      //    Message = String.Format("Invalid MDTO element(s) construction {0} (top/down) found.", structureValue),
                 *      //    FullException = new ApplicationException(String.Format("Invalid aggregation construction {0} (top/down) found.", structureValue))
                 *      //});
                 *  }
                 *  //^(Archief\|){0,}(Serie\|){0,}(Dossier\|){1,}(Archiefstuk\|){1,}(Bestand){1}$
                 * }
                 **/
            }

            return(messages);
        }
        private List <String> ValidateStructure(DirectoryInfo target, PreingestEventArgs eventArgs)
        {
            eventArgs.Description = String.Format("Start validate collection structure.", TargetFolder);
            OnTrigger(eventArgs);

            List <String> messages = new List <string>();

            if (target == null)
            {
                throw new ArgumentNullException("Input parameter is null! Expect DirectoryInfo as input.");
            }

            if (!target.Exists)
            {
                throw new DirectoryNotFoundException(String.Format("Not found: '{0}'.", target.FullName));
            }

            FileInfo[]      files       = target.GetFiles("*.*", SearchOption.AllDirectories);
            DirectoryInfo[] directories = Directory.GetDirectories(target.FullName, "*", SearchOption.AllDirectories).Select(item => new DirectoryInfo(item)).ToArray();

            int sum = (files.Count() + directories.Count() + 1); // plus current folder itself;

            if ((sum % 2) != 0)                                  //odd, not even. Expected even....
            {
                messages.Add(String.Format("The total sum of objects is not even! Expect an even count of objects. Files: {0}, Directories: {1}", files.Count(), directories.Count() + 1));
            }

            //total list of objects (stripped without the .metadata);
            List <FileSystemInfo> strippedCombinedCollection = new List <FileSystemInfo>();
            var filesStrippedMetadataExtension = files.Where(item
                                                             => !item.Extension.Equals(this.IsMDTO
                ? EXTENSION_MDTO : EXTENSION_TOPX, StringComparison.InvariantCultureIgnoreCase)).ToArray();

            strippedCombinedCollection.AddRange(filesStrippedMetadataExtension);
            strippedCombinedCollection.AddRange(directories);

            var filesOnlyMetadataExtension = files.Where(item
                                                         => item.Extension.Equals(this.IsMDTO ?
                                                                                  EXTENSION_MDTO : EXTENSION_TOPX, StringComparison.InvariantCultureIgnoreCase)).ToList();

            //With MDTO , extra condition check with xml files
            //directoryInfoSessionFolder.GetFiles("*.xml", SearchOption.AllDirectories).Where(item => item.Name.EndsWith(".mdto.xml", StringComparison.InvariantCultureIgnoreCase)).Count();

            //check for each object if it's paired with a metadata file ToPX (.metadata) or MDTO (.mdto.xml);
            foreach (var collectionObject in strippedCombinedCollection)
            {
                string fullnameMetadata = String.Empty;
                if (collectionObject is FileInfo)
                {
                    fullnameMetadata = String.Concat(collectionObject.FullName, this.IsMDTO
                        ? EXTENSION_MDTO : EXTENSION_TOPX);
                }
                else
                {
                    fullnameMetadata = String.Concat(Path.Combine(collectionObject.FullName, collectionObject.Name), this.IsMDTO
                        ? EXTENSION_MDTO : EXTENSION_TOPX);
                }

                bool exists = filesOnlyMetadataExtension.Exists((informationObject) =>
                {
                    return(File.Exists(fullnameMetadata));
                });

                if (!exists)
                {
                    string message = String.Format("Missing {0} metadata file for object '{1}'.", this.IsMDTO
                        ? EXTENSION_MDTO : EXTENSION_TOPX, collectionObject.Name);

                    StringBuilder messageBuilder = new StringBuilder();
                    messageBuilder.AppendLine(message);
                    messageBuilder.AppendLine(String.Format("Object type: {0}.", (collectionObject is FileInfo) ? "File" : "Directory"));
                    messageBuilder.AppendLine(String.Format("Target: {0}.", collectionObject.FullName));
                    messageBuilder.AppendLine(String.Format("Expected: {0}.", fullnameMetadata));

                    messages.Add(messageBuilder.ToString());
                }
            }

            return(messages);
        }