コード例 #1
0
        public override void Execute()
        {
            var eventModel = CurrentActionProperties(TargetCollection, this.GetType().Name);

            OnTrigger(new PreingestEventArgs {
                Description = String.Format("Start polishing Opex for container '{0}'.", TargetCollection), Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Started, PreingestAction = eventModel
            });

            var  anyMessages = new List <String>();
            bool isSuccess   = false;

            try
            {
                string       sessionFolder = Path.Combine(ApplicationSettings.DataFolderName, SessionGuid.ToString());
                BodySettings settings      = new SettingsReader(this.ApplicationSettings.DataFolderName, SessionGuid).GetSettings();

                if (settings == null)
                {
                    throw new ApplicationException("Settings are not saved!");
                }

                if (String.IsNullOrEmpty(settings.Polish))
                {
                    throw new ApplicationException("Polish setting is empty!");
                }

                DirectoryInfo   directoryInfoSessionFolder = new DirectoryInfo(sessionFolder);
                List <FileInfo> opexFiles = directoryInfoSessionFolder.GetFiles("*.opex", SearchOption.AllDirectories).ToList();

                if (opexFiles.Count == 0)
                {
                    throw new ApplicationException("Zero OPEX files found!");
                }

                string xsltTranformationServiceUrl = String.Format("http://{0}:{1}/hooks/xslt-transformation", ApplicationSettings.TransformationServerName, ApplicationSettings.TransformationServerPort);
                string stylesheetFileLocation      = Path.Combine(ApplicationSettings.PreWashFolder, settings.Polish);

                var jsonData = new List <string>();

                int countTotal = opexFiles.Count;

                OnTrigger(new PreingestEventArgs
                {
                    Description     = String.Format("Transforming: {0} files", countTotal),
                    Initiate        = DateTimeOffset.Now,
                    ActionType      = PreingestActionStates.Executing,
                    PreingestAction = eventModel
                });

                if (!String.IsNullOrEmpty(settings.UseSaxon))
                {
                    bool useSaxon = settings.UseSaxon.Equals("Ja", StringComparison.InvariantCultureIgnoreCase);
                    if (TransformationSetting.UseSaxon != useSaxon)
                    {
                        TransformationSetting.UseSaxon = useSaxon;
                    }
                }

                opexFiles.ForEach(file =>
                {
                    string result = string.Empty;

                    try
                    {
                        if (TransformationSetting.UseSaxon)
                        {
                            using (HttpClient client = new HttpClient())
                            {
                                client.Timeout = Timeout.InfiniteTimeSpan;
                                HttpResponseMessage response = client.GetAsync(String.Format("{0}?xsl={1}&xml={2}", xsltTranformationServiceUrl, stylesheetFileLocation, file.FullName)).Result;
                                response.EnsureSuccessStatusCode();

                                result = response.Content.ReadAsStringAsync().Result;
                            }
                        }
                        else
                        {
                            var opex = XDocument.Load(file.FullName).ToString();
                            var xsl  = XDocument.Load(stylesheetFileLocation).ToString();

                            result = Transform(xsl, opex);
                        }
                    }
                    catch (Exception e)
                    {
                        anyMessages.Add(String.Format("Transformation with Opex file '{0}' failed! {1} {2}", file.FullName, e.Message, e.StackTrace));
                    }
                    finally
                    {
                        if (!String.IsNullOrEmpty(result))
                        {
                            try
                            {
                                XDocument doc = XDocument.Parse(result);
                                doc.Save(file.FullName);
                            }
                            catch (Exception ie)
                            {
                                anyMessages.Add(String.Format("Saving Opex file '{0}' failed! {1} {2}", file.FullName, ie.Message, ie.StackTrace));
                            }
                            finally
                            {
                                jsonData.Add(String.Format("Transformation processed: {0}", file.FullName));
                            }
                        }
                    }
                });

                //update opex
                var profiles = opexFiles.Select(item => new { Xml = XDocument.Load(item.FullName).ToString(), FullName = item.FullName }).Where(item => item.Xml.Contains("<Files>")).ToList();
                countTotal = profiles.Count;

                OnTrigger(new PreingestEventArgs
                {
                    Description     = String.Format("Updating profile (Opex files): Total {0} files.", countTotal),
                    Initiate        = DateTimeOffset.Now,
                    ActionType      = PreingestActionStates.Executing,
                    PreingestAction = eventModel
                });

                profiles.ForEach(item =>
                {
                    FileInfo opex             = new FileInfo(item.FullName);
                    FileInfo[] currentContent = opex.Directory.GetFiles().Where(content => content.FullName != item.FullName).ToArray();

                    var currentOpexMetadataFile = Preingest.WebApi.Utilities.DeserializerHelper.DeSerializeObjectFromXmlFile <Noord.Hollands.Archief.Entities.Opex.opexMetadata>(item.Xml);
                    //overwrite
                    if (currentContent.Count() > 0)
                    {
                        currentOpexMetadataFile.Transfer.Manifest       = new Noord.Hollands.Archief.Entities.Opex.manifest();
                        currentOpexMetadataFile.Transfer.Manifest.Files = currentContent.Select(item => new Noord.Hollands.Archief.Entities.Opex.file
                        {
                            size          = item.Length,
                            typeSpecified = true,
                            type          = item.Extension.Equals(".opex", StringComparison.InvariantCultureIgnoreCase) ? Noord.Hollands.Archief.Entities.Opex.fileType.metadata : Noord.Hollands.Archief.Entities.Opex.fileType.content,
                            sizeSpecified = true,
                            Value         = item.Name
                        }).ToArray();

                        string output = Preingest.WebApi.Utilities.SerializerHelper.SerializeObjectToString(currentOpexMetadataFile);
                        var newDoc    = XDocument.Parse(output);
                        newDoc.Save(item.FullName);
                    }
                });

                OnTrigger(new PreingestEventArgs {
                    Description = String.Format("Reading Opex files for XSD schema validation."), Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Executing, PreingestAction = eventModel
                });

                var newOpexFiles = directoryInfoSessionFolder.GetFiles("*.opex").ToList();
                var schemaList   = SchemaHandler.GetSchemaList();

                var strXsd         = schemaList["Noord.Hollands.Archief.Preingest.WebApi.Schema.OPEX-Metadata.xsd"];
                var xsdTmpFilename = Path.GetTempFileName();
                var xsd            = XDocument.Parse(strXsd);
                xsd.Save(xsdTmpFilename);

                OnTrigger(new PreingestEventArgs {
                    Description = String.Format("Validate Opex files with XSD schema."), Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Executing, PreingestAction = eventModel
                });

                newOpexFiles.ForEach(opex =>
                {
                    try
                    {
                        if (String.IsNullOrEmpty(opex.FullName) || !File.Exists(opex.FullName))
                        {
                            throw new FileNotFoundException(String.Format("Opex file with location '{0}' is empty or not found!", opex.FullName));
                        }

                        XDocument xml = XDocument.Load(opex.FullName);

                        SchemaValidationHandler.Validate(xml.ToString(), xsdTmpFilename);
                    }
                    catch (Exception validate)
                    {
                        anyMessages.Add(String.Format("Schema validation error for Opex file '{0}'! {1} {2}", opex.FullName, validate.Message, validate.StackTrace));
                    }
                });

                try { File.Delete(xsdTmpFilename); } catch { }

                eventModel.Properties.Messages = anyMessages.ToArray();
                eventModel.ActionData          = jsonData.ToArray();

                eventModel.ActionResult.ResultValue = PreingestActionResults.Success;
                eventModel.Summary.Processed        = opexFiles.Count;
                eventModel.Summary.Accepted         = (opexFiles.Count - anyMessages.Count);
                eventModel.Summary.Rejected         = anyMessages.Count;

                if (eventModel.Summary.Rejected > 0)
                {
                    eventModel.ActionResult.ResultValue = PreingestActionResults.Error;
                }
                else
                {
                    eventModel.ActionResult.ResultValue = PreingestActionResults.Success;
                }

                isSuccess = true;
            }
            catch (Exception e)
            {
                isSuccess = false;
                anyMessages.Clear();
                anyMessages.Add(String.Format("Run polish with collection: '{0}' failed!", TargetCollection));
                anyMessages.Add(e.Message);
                anyMessages.Add(e.StackTrace);

                Logger.LogError(e, "Run polish with collection: '{0}' failed!", TargetCollection);

                eventModel.ActionResult.ResultValue = PreingestActionResults.Failed;
                eventModel.Properties.Messages      = anyMessages.ToArray();
                eventModel.Summary.Processed        = 1;
                eventModel.Summary.Accepted         = 0;
                eventModel.Summary.Rejected         = 1;

                OnTrigger(new PreingestEventArgs {
                    Description = "An exception occured while running checksum!", Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Failed, PreingestAction = eventModel
                });
            }
            finally
            {
                if (isSuccess)
                {
                    OnTrigger(new PreingestEventArgs {
                        Description = "Polish run with a collection is done.", Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Completed, PreingestAction = eventModel
                    });
                }
            }
        }
コード例 #2
0
        public override void Execute()
        {
            var eventModel = CurrentActionProperties(TargetCollection, this.GetType().Name);

            OnTrigger(new PreingestEventArgs {
                Description = String.Format("Start building Opex for container '{0}'.", TargetCollection), Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Started, PreingestAction = eventModel
            });

            var  anyMessages = new List <String>();
            bool isSuccess   = false;

            try
            {
                base.Execute();

                string       sessionFolder = Path.Combine(ApplicationSettings.DataFolderName, SessionGuid.ToString());
                BodySettings settings      = new SettingsReader(this.ApplicationSettings.DataFolderName, SessionGuid).GetSettings();

                if (settings == null)
                {
                    throw new ApplicationException("Settings are not saved!");
                }

                if (!String.IsNullOrEmpty(settings.MergeRecordAndFile))
                {
                    bool doMerge = settings.MergeRecordAndFile.Equals("Ja", StringComparison.InvariantCultureIgnoreCase);

                    if (doMerge && InheritanceSetting.MethodResult == InheritanceMethod.None)
                    {
                        //true && true
                        this.InheritanceSetting.MethodResult = InheritanceMethod.Combine;
                    }
                }

                string addNameSpaces   = Path.Combine(ApplicationSettings.PreWashFolder, OpexItem.ADD_NAMESPACE);
                string stripNameSpaces = Path.Combine(ApplicationSettings.PreWashFolder, OpexItem.STRIP_NAMESPACE);
                string opexFolders     = Path.Combine(ApplicationSettings.PreWashFolder, OpexItem.OPEX_FOLDERS);
                string opexFolderFiles = Path.Combine(ApplicationSettings.PreWashFolder, OpexItem.OPEX_FOLDER_FILES);
                string opexFiles       = Path.Combine(ApplicationSettings.PreWashFolder, OpexItem.OPEX_FILES);
                string opexFinalize    = Path.Combine(ApplicationSettings.PreWashFolder, OpexItem.OPEX_FINALIZE);

                if (!File.Exists(addNameSpaces))
                {
                    throw new FileNotFoundException(String.Format("Stylesheet file not found '{0}'!", addNameSpaces));
                }
                if (!File.Exists(stripNameSpaces))
                {
                    throw new FileNotFoundException(String.Format("Stylesheet file not found '{0}'!", stripNameSpaces));
                }
                if (!File.Exists(opexFolders))
                {
                    throw new FileNotFoundException(String.Format("Stylesheet file not found '{0}'!", opexFolders));
                }
                if (!File.Exists(opexFolderFiles))
                {
                    throw new FileNotFoundException(String.Format("Stylesheet file not found '{0}'!", opexFolderFiles));
                }
                if (!File.Exists(opexFiles))
                {
                    throw new FileNotFoundException(String.Format("Stylesheet file not found '{0}'!", opexFiles));
                }
                if (!File.Exists(opexFinalize))
                {
                    throw new FileNotFoundException(String.Format("Stylesheet file not found '{0}'!", opexFinalize));
                }

                List <StylesheetItem> stylesheetList = new List <StylesheetItem>();

                OnTrigger(new PreingestEventArgs {
                    Description = String.Format("Load settings for building Opex.", TargetCollection), Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Executing, PreingestAction = eventModel
                });

                var stylesheets = Directory.GetFiles(ApplicationSettings.PreWashFolder, "*-opex-*.xsl").Select(item => new StylesheetItem {
                    KeyLocation = item, XmlContent = XDocument.Load(item).ToString()
                }).ToArray();
                stylesheetList.AddRange(stylesheets);

                List <OpexItem> potentionalOpexList = new List <OpexItem>();

                OnTrigger(new PreingestEventArgs {
                    Description = String.Format("Reading collection to build Opex for container '{0}'.", TargetCollection), Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Executing, PreingestAction = eventModel
                });

                //trigger event read collections
                if (this.IsToPX)
                {
                    var listOfMetadata = new DirectoryInfo(Path.Combine(ApplicationSettings.DataFolderName, SessionGuid.ToString())).GetFiles("*.metadata", SearchOption.AllDirectories).Select(item
                                                                                                                                                                                                => new OpexItem(this.TargetFolder, item.FullName,
                                                                                                                                                                                                                XDocument.Load(item.FullName).ToString(),
                                                                                                                                                                                                                item.Directory.GetFiles("*.metadata", SearchOption.TopDirectoryOnly).Count())).ToArray();
                    potentionalOpexList.AddRange(listOfMetadata);
                    OnTrigger(new PreingestEventArgs {
                        Description = String.Format("Found '{0}' metadata ({1}) files.", potentionalOpexList.Count, this.IsToPX ? "ToPX" : "MDTO"), Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Executing, PreingestAction = eventModel
                    });
                }

                if (this.IsMDTO)
                {
                    var listOfMetadata = new DirectoryInfo(Path.Combine(ApplicationSettings.DataFolderName, SessionGuid.ToString())).GetFiles("*.xml", SearchOption.AllDirectories).Where(item => item.Name.EndsWith(".mdto.xml", StringComparison.InvariantCultureIgnoreCase)).Select(item
                                                                                                                                                                                                                                                                                       => new OpexItem(this.TargetFolder, item.FullName,
                                                                                                                                                                                                                                                                                                       XDocument.Load(item.FullName).ToString(),
                                                                                                                                                                                                                                                                                                       item.Directory.GetFiles("*.xml", SearchOption.TopDirectoryOnly).Where(item
                                                                                                                                                                                                                                                                                                                                                                             => item.Name.EndsWith(".mdto.xml", StringComparison.InvariantCultureIgnoreCase)).Count())).ToArray();
                    potentionalOpexList.AddRange(listOfMetadata);
                    OnTrigger(new PreingestEventArgs {
                        Description = String.Format("Found '{0}' metadata ({1}) files.", potentionalOpexList.Count, this.IsToPX ? "ToPX" : "MDTO"), Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Executing, PreingestAction = eventModel
                    });
                }

                OnTrigger(new PreingestEventArgs {
                    Description = String.Format("Creating Opex files."), Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Executing, PreingestAction = eventModel
                });
                //trigger event process opex
                potentionalOpexList.ForEach(item =>
                {
                    item.InitializeOpex(stylesheetList);
                });

                var jsonData = new List <String>();

                OnTrigger(new PreingestEventArgs {
                    Description = String.Format("Removing original metadata files."), Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Executing, PreingestAction = eventModel
                });

                //remove the old metadata files topx or mdto
                potentionalOpexList.ForEach(item =>
                {
                    try
                    {
                        File.Delete(item.KeyLocation);
                    }
                    catch (Exception delete)
                    {
                        anyMessages.Add(String.Format("Deleting file '{0}' failed! {1} {2}", item.KeyLocation, delete.Message, delete.StackTrace));
                    }
                });

                OnTrigger(new PreingestEventArgs {
                    Description = String.Format("Saving Opex files (only file level)."), Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Executing, PreingestAction = eventModel
                });

                var sortedPotentionalOpexList = new List <OpexItem>();

                if (InheritanceSetting.MethodResult == InheritanceMethod.Combine)
                {
                    var sorted = potentionalOpexList.Where(item => !item.IsFile).GroupBy(item => item.Level, (key, opex) => new { Level = key, Item = opex }).OrderByDescending(item
                                                                                                                                                                                => item.Level).FirstOrDefault();
                    sortedPotentionalOpexList.AddRange(sorted.Item.OfType <OpexItem>());
                }

                //save bestand level first
                potentionalOpexList.Where(item => item.IsFile).ToList().ForEach(item =>
                {
                    try
                    {
                        var metadataList = new List <System.Xml.XmlElement>();

                        if (InheritanceSetting.MethodResult == InheritanceMethod.Combine)
                        {
                            var parts           = item.KeyLocation.Split('/', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToArray();
                            var startsWith      = parts.Skip(0).Take(parts.Count() - 1).ToArray();
                            var currentItemPath = "/" + Path.Combine(startsWith);

                            var profile = sortedPotentionalOpexList.FirstOrDefault(sorted => sorted.KeyLocation.StartsWith(currentItemPath));

                            if (profile == null)
                            {
                                throw new ApplicationException(String.Format("Adding profile to DescriptiveMetadata failed! Cannot find profile record in '{0}'", currentItemPath));
                            }

                            var xmlProfileDoc = new System.Xml.XmlDocument();
                            xmlProfileDoc.LoadXml(profile.OriginalMetadata);
                            metadataList.Add(xmlProfileDoc.DocumentElement);
                        }

                        if (InheritanceSetting.MethodResult == InheritanceMethod.None)
                        {
                            metadataList.Clear();
                        }

                        item.UpdateOpexFileLevel(ApplicationSettings.ChecksumServerName, ApplicationSettings.ChecksumServerPort, metadataList, true);
                        var xml = XDocument.Parse(item.FinalUpdatedOpexMetadata);
                        xml.Save(item.KeyOpexLocation);
                    }
                    catch (Exception save)
                    {
                        anyMessages.Add(String.Format("Saving Opex file '{0}' failed! {1} {2}", item.KeyOpexLocation, save.Message, save.StackTrace));
                    }
                });

                OnTrigger(new PreingestEventArgs {
                    Description = String.Format("Saving Opex files (folder levels)."), Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Executing, PreingestAction = eventModel
                });

                //update mappen levels
                //save the rest of the folder levels
                potentionalOpexList.Where(item => !item.IsFile).ToList().ForEach(item =>
                {
                    try
                    {
                        item.UpdateOpexFolderLevel();
                        var xml = XDocument.Parse(item.FinalUpdatedOpexMetadata);
                        xml.Save(item.KeyOpexLocation);
                    }
                    catch (Exception save)
                    {
                        anyMessages.Add(String.Format("Saving Opex file '{0}' failed! {1} {2}", item.KeyOpexLocation, save.Message, save.StackTrace));
                    }
                });

                OnTrigger(new PreingestEventArgs {
                    Description = String.Format("Rounding up Opex."), Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Executing, PreingestAction = eventModel
                });

                //create container opex file,
                //move the result to a folder named 'opex';
                //this folder will be used by the bucket container for upload to S3
                var currentCollectionDI         = new DirectoryInfo(TargetFolder);
                var currentCollectionFoldername = currentCollectionDI.GetDirectories().OrderBy(item => item.CreationTime).FirstOrDefault();
                if (currentCollectionFoldername == null)
                {
                    throw new DirectoryNotFoundException("Expanded collection folder not found!");
                }

                var opexContainerFilename = CreateContainerOpexMetadata(currentCollectionDI, currentCollectionFoldername.Name);
                var opexUploadFolder      = Directory.CreateDirectory(Path.Combine(TargetFolder, "opex"));

                opexContainerFilename.MoveTo(Path.Combine(opexUploadFolder.FullName, opexContainerFilename.Name), true);
                currentCollectionFoldername.MoveTo(Path.Combine(opexUploadFolder.FullName, currentCollectionFoldername.Name));

                OnTrigger(new PreingestEventArgs {
                    Description = String.Format("Reading Opex files for XSD schema validation."), Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Executing, PreingestAction = eventModel
                });

                var newOpexFiles = opexUploadFolder.GetFiles("*.opex").ToList();
                var schemaList   = SchemaHandler.GetSchemaList();

                var strXsd         = schemaList["Noord.Hollands.Archief.Preingest.WebApi.Schema.OPEX-Metadata.xsd"];
                var xsdTmpFilename = Path.GetTempFileName();
                var xsd            = XDocument.Parse(strXsd);
                xsd.Save(xsdTmpFilename);

                OnTrigger(new PreingestEventArgs {
                    Description = String.Format("Validate Opex files with XSD schema."), Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Executing, PreingestAction = eventModel
                });

                newOpexFiles.ForEach(opex =>
                {
                    try
                    {
                        if (String.IsNullOrEmpty(opex.FullName) || !File.Exists(opex.FullName))
                        {
                            throw new FileNotFoundException(String.Format("Opex file with location '{0}' is empty or not found!", opex.FullName));
                        }

                        XDocument xml = XDocument.Load(opex.FullName);

                        SchemaValidationHandler.Validate(xml.ToString(), xsdTmpFilename);
                    }
                    catch (Exception validate)
                    {
                        anyMessages.Add(String.Format("Schema validation error for Opex file '{0}'! {1} {2}", opex.FullName, validate.Message, validate.StackTrace));
                    }
                });

                try { File.Delete(xsdTmpFilename); } catch { }

                var result = potentionalOpexList.Select(item => String.Format("{0} >> {1}", item.KeyLocation, item.KeyOpexLocation)).ToArray();
                jsonData.AddRange(result);
                opexContainerFilename.Refresh();
                jsonData.Add("All moved to folder 'opex'.");
                jsonData.Add(opexContainerFilename.FullName);

                eventModel.ActionData          = jsonData.ToArray();
                eventModel.Summary.Processed   = potentionalOpexList.Count;
                eventModel.Summary.Accepted    = potentionalOpexList.Count;
                eventModel.Summary.Rejected    = anyMessages.Count();
                eventModel.Properties.Messages = anyMessages.ToArray();

                if (eventModel.Summary.Rejected > 0)
                {
                    eventModel.ActionResult.ResultValue = PreingestActionResults.Error;
                }
                else
                {
                    eventModel.ActionResult.ResultValue = PreingestActionResults.Success;
                }

                isSuccess = true;
            }
            catch (Exception e)
            {
                isSuccess = false;
                anyMessages.Clear();
                anyMessages.Add(String.Format("Build Opex with collection: '{0}' failed!", TargetCollection));
                anyMessages.Add(e.Message);
                anyMessages.Add(e.StackTrace);

                Logger.LogError(e, "Build Opex with collection: '{0}' failed!", TargetCollection);

                eventModel.ActionResult.ResultValue = PreingestActionResults.Failed;
                eventModel.Properties.Messages      = anyMessages.ToArray();
                eventModel.Summary.Processed        = 1;
                eventModel.Summary.Accepted         = 0;
                eventModel.Summary.Rejected         = 1;

                OnTrigger(new PreingestEventArgs {
                    Description = "An exception occured while building opex!", Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Failed, PreingestAction = eventModel
                });
            }
            finally
            {
                if (isSuccess)
                {
                    OnTrigger(new PreingestEventArgs {
                        Description = "Build Opex with a collection is done.", Initiate = DateTimeOffset.Now, ActionType = PreingestActionStates.Completed, PreingestAction = eventModel
                    });
                }
            }
        }