/// <summary> Check if a folder has a BibID restriction ( i.e., incoming items must start with 'XYZ..' ) and then checks each package /// found in that folder has a valid BibID, or moves it to the failures folder </summary> /// <param name="BuilderFolder"> Builder folder upon which to perform all work </param> /// <param name="IncomingPackages"> List of valid incoming packages, which may be modified by this process </param> /// <param name="Deletes"> List of valid deletes, which may be modifyed by this process </param> public override void DoWork(Actionable_Builder_Source_Folder BuilderFolder, List <Incoming_Digital_Resource> IncomingPackages, List <Incoming_Digital_Resource> Deletes) { // If this folder is limited on what BibID roots it accepts, check that now if (BuilderFolder.BibID_Roots_Restrictions.Length > 0) { if ((Directory.Exists(BuilderFolder.Processing_Folder)) && (Directory.GetDirectories(BuilderFolder.Processing_Folder).Length > 0)) { // Get the list of all packages in the processing folder List <Incoming_Digital_Resource> packages = BuilderFolder.Packages_For_Processing; // Step through each package foreach (Incoming_Digital_Resource resource in packages) { string[] starts = BuilderFolder.BibID_Roots_Restrictions.Split("|,;".ToCharArray()); bool okay = starts.Any(ThisStart => resource.Folder_Name.IndexOf(ThisStart, StringComparison.OrdinalIgnoreCase) == 0); // If not okay.. it is a failure if (!okay) { OnError("Package " + resource.Folder_Name + " has invalid BibID for " + BuilderFolder.Folder_Name + " incoming folder ( " + BuilderFolder.Folder_Name + " )", resource.BibID + ":" + resource.VID, "INCOMING", -1); // Move this resource if (!resource.Move(BuilderFolder.Failures_Folder)) { OnError("Unable to move folder " + resource.Folder_Name + " in " + BuilderFolder.Folder_Name + " to the failures folder", resource.BibID + ":" + resource.VID, resource.METS_Type_String, -1); } } } } } }
/// <summary> Check if a folder has a BibID restriction ( i.e., incoming items must start with 'XYZ..' ) and then checks each package /// found in that folder has a valid BibID, or moves it to the failures folder </summary> /// <param name="BuilderFolder"> Builder folder upon which to perform all work </param> /// <param name="IncomingPackages"> List of valid incoming packages, which may be modified by this process </param> /// <param name="Deletes"> List of valid deletes, which may be modifyed by this process </param> public override void DoWork(Actionable_Builder_Source_Folder BuilderFolder, List<Incoming_Digital_Resource> IncomingPackages, List<Incoming_Digital_Resource> Deletes) { // If this folder is limited on what BibID roots it accepts, check that now if (BuilderFolder.BibID_Roots_Restrictions.Length > 0) { if ((Directory.Exists(BuilderFolder.Processing_Folder)) && (Directory.GetDirectories(BuilderFolder.Processing_Folder).Length > 0)) { // Get the list of all packages in the processing folder List<Incoming_Digital_Resource> packages = BuilderFolder.Packages_For_Processing; // Step through each package foreach (Incoming_Digital_Resource resource in packages) { string[] starts = BuilderFolder.BibID_Roots_Restrictions.Split("|,;".ToCharArray()); bool okay = starts.Any(ThisStart => resource.Folder_Name.IndexOf(ThisStart, StringComparison.OrdinalIgnoreCase) == 0); // If not okay.. it is a failure if (!okay) { OnError("Package " + resource.Folder_Name + " has invalid BibID for " + BuilderFolder.Folder_Name + " incoming folder ( " + BuilderFolder.Folder_Name + " )", resource.BibID + ":" + resource.VID, "INCOMING", -1); // Move this resource if (!resource.Move(BuilderFolder.Failures_Folder)) { OnError("Unable to move folder " + resource.Folder_Name + " in " + BuilderFolder.Folder_Name + " to the failures folder", resource.BibID + ":" + resource.VID, resource.METS_Type_String, -1); } } } } } }
/// <summary> Check for appropriately aged folders and moves them into the related processing folder </summary> /// <param name="BuilderFolder"> Builder folder upon which to perform all work </param> /// <param name="IncomingPackages"> List of valid incoming packages, which may be modified by this process </param> /// <param name="Deletes"> List of valid deletes, which may be modifyed by this process </param> public override void DoWork(Actionable_Builder_Source_Folder BuilderFolder, List<Incoming_Digital_Resource> IncomingPackages, List<Incoming_Digital_Resource> Deletes) { try { // Move all eligible packages from the FTP folders to the processing folders if (Settings.Builder.Verbose_Flag) OnProcess("Worker_BulkLoader.Move_Appropriate_Inbound_Packages_To_Processing: Checking incoming folder " + BuilderFolder.Inbound_Folder, String.Empty, String.Empty, String.Empty, -1); if (BuilderFolder.Items_Exist_In_Inbound) { if (Settings.Builder.Verbose_Flag) OnProcess("Worker_BulkLoader.Move_Appropriate_Inbound_Packages_To_Processing: Found either files or subdirectories in " + BuilderFolder.Inbound_Folder, String.Empty, String.Empty, String.Empty, -1); if (Settings.Builder.Verbose_Flag) OnProcess("Checking inbound packages for aging and possibly moving to processing", String.Empty, String.Empty, String.Empty, -1); String outMessage; if (!BuilderFolder.Move_From_Inbound_To_Processing(out outMessage)) { if (outMessage.Length > 0) OnError(outMessage, String.Empty, String.Empty, -1); OnError("Unspecified error moving files from inbound to processing", String.Empty, String.Empty, -1); } else { if ((Settings.Builder.Verbose_Flag) && (outMessage.Length > 0)) OnProcess(outMessage, String.Empty, String.Empty, String.Empty, -1); } // Try to get rid of any empty folders try { string[] subdirs = Directory.GetDirectories(BuilderFolder.Inbound_Folder); foreach (string thisSubdir in subdirs) { if ((Directory.GetFiles(thisSubdir).Length == 0) && (Directory.GetDirectories(thisSubdir).Length == 0)) { Directory.Delete(thisSubdir); } } } catch { } } else if (Settings.Builder.Verbose_Flag) OnProcess("Worker_BulkLoader.Move_Appropriate_Inbound_Packages_To_Processing: No subdirectories or files found in incoming folder " + BuilderFolder.Inbound_Folder, String.Empty, String.Empty, String.Empty, -1); } catch (Exception ee) { OnError("Error in harvesting packages from inbound folders to processing\n" + ee.Message, String.Empty, String.Empty, -1); } }
/// <summary> Check for folders that are not named in BibID or BibiD/VID format and do not have metadata. For /// these items, creates a BibID/VID folder with minimal metadata from the folder name. </summary> /// <param name="BuilderFolder"> Builder folder upon which to perform all work </param> /// <param name="IncomingPackages"> List of valid incoming packages, which may be modified by this process </param> /// <param name="Deletes"> List of valid deletes, which may be modifyed by this process </param> public override void DoWork(Actionable_Builder_Source_Folder BuilderFolder, List<Incoming_Digital_Resource> IncomingPackages, List<Incoming_Digital_Resource> Deletes) { string[] subdirs = Directory.GetDirectories(BuilderFolder.Inbound_Folder); foreach ( string thisSubDir in subdirs) { try { string thisSubDirName = (new DirectoryInfo(thisSubDir)).Name; // Must have some files to continue if (Directory.GetFiles(thisSubDir).Length == 0) continue; // Need to check if this MAY be a valid BibID. // Need to make this a bit more specific in the future, as it will skip ANY folders // that are ten digits long right now. if ((thisSubDir.Length == 10) || ((thisSubDir.Length == 16) && (thisSubDirName[0] == '_'))) continue; // Look for a METS file or any source of metadata in the folder if ((Directory.GetFiles(thisSubDir, "*.mets").Length > 0) || (Directory.GetFiles(thisSubDir, "*.xml").Length > 0)) continue; // Clean any additional periods in the filenames first string[] allFiles = Directory.GetFiles(thisSubDir); foreach (string thisFile in allFiles) { string fileName = Path.GetFileName(thisFile); if (Regex.Matches(fileName, "\\.").Count > 1) { string newFileName = fileName; while (Regex.Matches(newFileName, "\\.").Count > 1) { char[] charArr = newFileName.ToCharArray(); charArr[newFileName.IndexOf(".")] = '_'; // freely modify the array newFileName = new string(charArr); } File.Move(thisFile, Path.Combine(thisSubDir, newFileName)); } } // Create the new object SobekCM_Item newItem = new SobekCM_Item(); newItem.Bib_Info.SobekCM_Type = TypeOfResource_SobekCM_Enum.Archival; newItem.Bib_Info.Main_Title.Title = thisSubDirName; newItem.Bib_Info.Add_Identifier(thisSubDirName); newItem.Bib_Info.Source.Code = Arguments[1]; newItem.Bib_Info.Source.Statement = Arguments[2]; newItem.BibID = Arguments[0]; newItem.VID = "00001"; // Save this item, for the necessary bibid SobekCM_Item_Database.Save_New_Digital_Resource(newItem, false, false, "Builder", "Created BibID folder from '" + thisSubDirName + "'", -1); string newFolderName = newItem.BibID + "_" + newItem.VID; string newFolder = Path.Combine(BuilderFolder.Inbound_Folder, newFolderName); Directory.Move(thisSubDir, newFolder); newItem.Source_Directory = newFolder; newItem.Save_METS(); } catch (Exception ee) { Console.WriteLine("Error moving directory " + ee.Message); } } }
/// <summary> Check for folders that are not named in BibID or BibiD/VID format and do not have metadata. For /// these items, creates a BibID/VID folder with minimal metadata from the folder name. </summary> /// <param name="BuilderFolder"> Builder folder upon which to perform all work </param> /// <param name="IncomingPackages"> List of valid incoming packages, which may be modified by this process </param> /// <param name="Deletes"> List of valid deletes, which may be modifyed by this process </param> public override void DoWork(Actionable_Builder_Source_Folder BuilderFolder, List <Incoming_Digital_Resource> IncomingPackages, List <Incoming_Digital_Resource> Deletes) { string[] subdirs = Directory.GetDirectories(BuilderFolder.Inbound_Folder); foreach (string thisSubDir in subdirs) { try { string thisSubDirName = (new DirectoryInfo(thisSubDir)).Name; // Must have some files to continue if (Directory.GetFiles(thisSubDir).Length == 0) { continue; } // Need to check if this MAY be a valid BibID. // Need to make this a bit more specific in the future, as it will skip ANY folders // that are ten digits long right now. if ((thisSubDir.Length == 10) || ((thisSubDir.Length == 16) && (thisSubDirName[0] == '_'))) { continue; } // Look for a METS file or any source of metadata in the folder if ((Directory.GetFiles(thisSubDir, "*.mets").Length > 0) || (Directory.GetFiles(thisSubDir, "*.xml").Length > 0)) { continue; } // Clean any additional periods in the filenames first string[] allFiles = Directory.GetFiles(thisSubDir); foreach (string thisFile in allFiles) { string fileName = Path.GetFileName(thisFile); if (Regex.Matches(fileName, "\\.").Count > 1) { string newFileName = fileName; while (Regex.Matches(newFileName, "\\.").Count > 1) { char[] charArr = newFileName.ToCharArray(); charArr[newFileName.IndexOf(".")] = '_'; // freely modify the array newFileName = new string(charArr); } File.Move(thisFile, Path.Combine(thisSubDir, newFileName)); } } // Create the new object SobekCM_Item newItem = new SobekCM_Item(); newItem.Bib_Info.SobekCM_Type = TypeOfResource_SobekCM_Enum.Archival; newItem.Bib_Info.Main_Title.Title = thisSubDirName; newItem.Bib_Info.Add_Identifier(thisSubDirName); newItem.Bib_Info.Source.Code = Arguments[1]; newItem.Bib_Info.Source.Statement = Arguments[2]; newItem.BibID = Arguments[0]; newItem.VID = "00001"; // Save this item, for the necessary bibid SobekCM_Item_Database.Save_New_Digital_Resource(newItem, false, false, "Builder", "Created BibID folder from '" + thisSubDirName + "'", -1); string newFolderName = newItem.BibID + "_" + newItem.VID; string newFolder = Path.Combine(BuilderFolder.Inbound_Folder, newFolderName); Directory.Move(thisSubDir, newFolder); newItem.Source_Directory = newFolder; newItem.Save_METS(); } catch (Exception ee) { Console.WriteLine("Error moving directory " + ee.Message); } } }
/// <summary> Validates the metadata for each folder and classified as package either adding /// a new item or updating an existing item versus a package requesting a delete </summary> /// <param name="BuilderFolder"> Builder folder upon which to perform all work </param> /// <param name="IncomingPackages"> List of valid incoming packages, which may be modified by this process </param> /// <param name="Deletes"> List of valid deletes, which may be modifyed by this process </param> public override void DoWork(Actionable_Builder_Source_Folder BuilderFolder, List<Incoming_Digital_Resource> IncomingPackages, List<Incoming_Digital_Resource> Deletes) { if (Settings.Builder.Verbose_Flag) OnProcess("ValidateAndClassifyModule.Perform_BulkLoader: Begin validating and classifying packages in incoming/process folders", "Verbose", String.Empty, String.Empty, -1); // If the maximum number of (incoming, non-delete) packages have already been set aside to process, no need to continue on this folder if ((MultiInstance_Builder_Settings.Instance_Package_Limit > 0) && (IncomingPackages.Count >= MultiInstance_Builder_Settings.Instance_Package_Limit)) { OnProcess("...Package validation aborted - maximum number of packages ( " + MultiInstance_Builder_Settings.Instance_Package_Limit + " ) reached", "Verbose", String.Empty, String.Empty, -1); return; } try { // Only continue if the processing folder exists and has subdirectories if ((Directory.Exists(BuilderFolder.Processing_Folder)) && (Directory.GetDirectories(BuilderFolder.Processing_Folder).Length > 0)) { // Get the list of all packages in the processing folder if (Settings.Builder.Verbose_Flag) OnProcess("....Validate packages for " + BuilderFolder.Folder_Name, "Verbose", String.Empty, String.Empty, -1); List<Incoming_Digital_Resource> packages = BuilderFolder.Packages_For_Processing; if (packages.Count > 0) { // Create the METS validation objects if (thisMetsValidator == null) { if (Settings.Builder.Verbose_Flag) OnProcess("ValidateAndClassifyModule.Constructor: Created Validators", "Verbose", String.Empty, String.Empty, -1); thisMetsValidator = new SobekCM_METS_Validator(String.Empty); metsSchemeValidator = new METS_Validator_Object(false); } // Step through each package foreach (Incoming_Digital_Resource resource in packages) { // Validate the categorize the package if (Settings.Builder.Verbose_Flag) OnProcess("........Checking '" + resource.Folder_Name + "'", "Verbose", resource.Folder_Name.Replace("_", ":"), String.Empty, -1); // If there is no METS file, use special code to check this if (Directory.GetFiles(resource.Resource_Folder, "*.mets*").Length == 0) { DirectoryInfo noMetsDirInfo = new DirectoryInfo(resource.Resource_Folder); string vid = noMetsDirInfo.Name; if (noMetsDirInfo.Parent != null) // Should never be null { string bibid = noMetsDirInfo.Parent.Name; if ((vid.Length == 16) && (vid[10] == '_')) { bibid = vid.Substring(0, 10); vid = vid.Substring(11, 5); } // Is this allowed in this incomnig folder? if (BuilderFolder.Allow_Folders_No_Metadata) { if (itemTable == null) { DataSet itemListFromDb = SobekCM_Database.Get_Item_List(true, null); // Reload the settings if (itemListFromDb == null) { OnError("ValidateAndClassifyModule : Unable to pull the item table from the database", String.Empty, String.Empty, -1); return; } // Save the item table itemTable = itemListFromDb.Tables[0]; } DataRow[] selected = itemTable.Select("BibID='" + bibid + "' and VID='" + vid + "'"); if (selected.Length > 0) { resource.BibID = bibid; resource.VID = vid; resource.Resource_Type = Incoming_Digital_Resource.Incoming_Digital_Resource_Type.FOLDER_OF_FILES; IncomingPackages.Add(resource); } else { OnError("METS-less folder is not a valid BibID/VID combination", resource.Folder_Name.Replace("_", ":"), "NONE", -1); // Move this resource if (!resource.Move(BuilderFolder.Failures_Folder)) { OnError("Unable to move folder " + resource.Folder_Name + " in " + BuilderFolder.Folder_Name + " to the failures folder", resource.Folder_Name.Replace("_", ":"), "NONE", -1); } } } else { OnError("METS-less folders are not allowed in " + BuilderFolder.Folder_Name, bibid + ":" + vid, "NONE", -1); // Move this resource if (!resource.Move(BuilderFolder.Failures_Folder)) { OnError("Unable to move folder " + resource.Folder_Name + " in " + BuilderFolder.Folder_Name + " to the failures folder", resource.Folder_Name.Replace("_", ":"), "NONE", -1); } } } } else { long validateId = OnProcess("....Validating METS file for " + resource.Folder_Name, "Folder Process", resource.Folder_Name.Replace("_", ":"), "UNKNOWN", -1); string validation_errors = Validate_and_Read_METS(resource, thisMetsValidator, metsSchemeValidator); // Save any errors to the main log if (validation_errors.Length > 0) { // Save the validation errors to the main log Save_Validation_Errors_To_Log(validation_errors, resource.Source_Folder.Folder_Name.Replace("_", ":"), validateId); // Move this resource if (!resource.Move(BuilderFolder.Failures_Folder)) { OnError("Unable to move folder " + resource.Folder_Name + " in " + BuilderFolder.Folder_Name + " to the failures folder", resource.Folder_Name.Replace("_", ":"), "NONE", -1); } } else { // Categorize remaining packages by type switch (resource.Resource_Type) { case Incoming_Digital_Resource.Incoming_Digital_Resource_Type.PARTIAL_PACKAGE: case Incoming_Digital_Resource.Incoming_Digital_Resource_Type.COMPLETE_PACKAGE: IncomingPackages.Add(resource); break; case Incoming_Digital_Resource.Incoming_Digital_Resource_Type.METADATA_UPDATE: if (BuilderFolder.Allow_Metadata_Updates) { IncomingPackages.Add(resource); } else { OnError("Metadata update is not allowed in " + BuilderFolder.Folder_Name, resource.Folder_Name.Replace("_", ":"), "METADATA UPDATE", -1); // Move this resource if (!resource.Move(BuilderFolder.Failures_Folder)) { OnError("Unable to move folder " + resource.Folder_Name + " to failures", resource.Folder_Name.Replace("_", ":"), String.Empty, -1); } } break; case Incoming_Digital_Resource.Incoming_Digital_Resource_Type.DELETE: if (BuilderFolder.Allow_Deletes) { Deletes.Add(resource); } else { OnError("Delete is not allowed in " + BuilderFolder.Folder_Name, resource.Folder_Name.Replace("_", ":"), "DELETE", -1); // Move this resource if (!resource.Move(BuilderFolder.Failures_Folder)) { OnError("Unable to move folder " + resource.Folder_Name + " to failures", resource.Folder_Name.Replace("_", ":"), String.Empty, -1); } } break; } } } // If the maximum number of (incoming, non-delete) packages has now been met, no need to classify anymore if ((MultiInstance_Builder_Settings.Instance_Package_Limit > 0) && (IncomingPackages.Count >= MultiInstance_Builder_Settings.Instance_Package_Limit)) { OnProcess("...Package validation aborted - maximum number of packages ( " + MultiInstance_Builder_Settings.Instance_Package_Limit + " ) reached", "Verbose", String.Empty, String.Empty, -1); return; } } } } } catch (Exception ee) { OnError("Error in harvesting packages from processing : " + ee.Message + "\n" + ee.StackTrace , String.Empty, String.Empty, -1); } }
/// <summary> Method performs the work of the folder-level builder module </summary> /// <param name="BuilderFolder"> Builder folder upon which to perform all work </param> /// <param name="IncomingPackages"> List of valid incoming packages, which may be modified by this process </param> /// <param name="Deletes"> List of valid deletes, which may be modifyed by this process </param> public abstract void DoWork(Actionable_Builder_Source_Folder BuilderFolder, List<Incoming_Digital_Resource> IncomingPackages, List<Incoming_Digital_Resource> Deletes);
/// <summary> Check for appropriately aged folders and moves them into the related processing folder </summary> /// <param name="BuilderFolder"> Builder folder upon which to perform all work </param> /// <param name="IncomingPackages"> List of valid incoming packages, which may be modified by this process </param> /// <param name="Deletes"> List of valid deletes, which may be modifyed by this process </param> public override void DoWork(Actionable_Builder_Source_Folder BuilderFolder, List <Incoming_Digital_Resource> IncomingPackages, List <Incoming_Digital_Resource> Deletes) { try { // Move all eligible packages from the FTP folders to the processing folders if (Settings.Builder.Verbose_Flag) { OnProcess("Worker_BulkLoader.Move_Appropriate_Inbound_Packages_To_Processing: Checking incoming folder " + BuilderFolder.Inbound_Folder, String.Empty, String.Empty, String.Empty, -1); } if (BuilderFolder.Items_Exist_In_Inbound) { if (Settings.Builder.Verbose_Flag) { OnProcess("Worker_BulkLoader.Move_Appropriate_Inbound_Packages_To_Processing: Found either files or subdirectories in " + BuilderFolder.Inbound_Folder, String.Empty, String.Empty, String.Empty, -1); } if (Settings.Builder.Verbose_Flag) { OnProcess("Checking inbound packages for aging and possibly moving to processing", String.Empty, String.Empty, String.Empty, -1); } String outMessage; if (!BuilderFolder.Move_From_Inbound_To_Processing(Settings, out outMessage)) { if (outMessage.Length > 0) { OnError(outMessage, String.Empty, String.Empty, -1); } OnError("Unspecified error moving files from inbound to processing", String.Empty, String.Empty, -1); } else { if ((Settings.Builder.Verbose_Flag) && (outMessage.Length > 0)) { OnProcess(outMessage, String.Empty, String.Empty, String.Empty, -1); } } // Try to get rid of any empty folders try { string[] subdirs = Directory.GetDirectories(BuilderFolder.Inbound_Folder); foreach (string thisSubdir in subdirs) { if ((Directory.GetFiles(thisSubdir).Length == 0) && (Directory.GetDirectories(thisSubdir).Length == 0)) { Directory.Delete(thisSubdir); } } } catch { } } else if (Settings.Builder.Verbose_Flag) { OnProcess("Worker_BulkLoader.Move_Appropriate_Inbound_Packages_To_Processing: No subdirectories or files found in incoming folder " + BuilderFolder.Inbound_Folder, String.Empty, String.Empty, String.Empty, -1); } } catch (Exception ee) { OnError("Error in harvesting packages from inbound folders to processing\n" + ee.Message, String.Empty, String.Empty, -1); } }
/// <summary> Validates the metadata for each folder and classified as package either adding /// a new item or updating an existing item versus a package requesting a delete </summary> /// <param name="BuilderFolder"> Builder folder upon which to perform all work </param> /// <param name="IncomingPackages"> List of valid incoming packages, which may be modified by this process </param> /// <param name="Deletes"> List of valid deletes, which may be modifyed by this process </param> public override void DoWork(Actionable_Builder_Source_Folder BuilderFolder, List <Incoming_Digital_Resource> IncomingPackages, List <Incoming_Digital_Resource> Deletes) { if (Settings.Builder.Verbose_Flag) { OnProcess("ValidateAndClassifyModule.Perform_BulkLoader: Begin validating and classifying packages in incoming/process folders", "Verbose", String.Empty, String.Empty, -1); } // If the maximum number of (incoming, non-delete) packages have already been set aside to process, no need to continue on this folder if ((MultiInstance_Builder_Settings.Instance_Package_Limit > 0) && (IncomingPackages.Count >= MultiInstance_Builder_Settings.Instance_Package_Limit)) { OnProcess("...Package validation aborted - maximum number of packages ( " + MultiInstance_Builder_Settings.Instance_Package_Limit + " ) reached", "Verbose", String.Empty, String.Empty, -1); return; } try { // Only continue if the processing folder exists and has subdirectories if ((Directory.Exists(BuilderFolder.Processing_Folder)) && (Directory.GetDirectories(BuilderFolder.Processing_Folder).Length > 0)) { // Get the list of all packages in the processing folder if (Settings.Builder.Verbose_Flag) { OnProcess("....Validate packages for " + BuilderFolder.Folder_Name, "Verbose", String.Empty, String.Empty, -1); } List <Incoming_Digital_Resource> packages = BuilderFolder.Packages_For_Processing; if (packages.Count > 0) { // Create the METS validation objects if (thisMetsValidator == null) { if (Settings.Builder.Verbose_Flag) { OnProcess("ValidateAndClassifyModule.Constructor: Created Validators", "Verbose", String.Empty, String.Empty, -1); } thisMetsValidator = new SobekCM_METS_Validator(String.Empty); metsSchemeValidator = new METS_Validator_Object(false); } // Step through each package foreach (Incoming_Digital_Resource resource in packages) { // Validate the categorize the package if (Settings.Builder.Verbose_Flag) { OnProcess("........Checking '" + resource.Folder_Name + "'", "Verbose", resource.Folder_Name.Replace("_", ":"), String.Empty, -1); } // If there is no METS file, use special code to check this if (Directory.GetFiles(resource.Resource_Folder, "*.mets*").Length == 0) { DirectoryInfo noMetsDirInfo = new DirectoryInfo(resource.Resource_Folder); string vid = noMetsDirInfo.Name; if (noMetsDirInfo.Parent != null) // Should never be null { string bibid = noMetsDirInfo.Parent.Name; if ((vid.Length == 16) && (vid[10] == '_')) { bibid = vid.Substring(0, 10); vid = vid.Substring(11, 5); } // Is this allowed in this incomnig folder? if (BuilderFolder.Allow_Folders_No_Metadata) { if (itemTable == null) { DataSet itemListFromDb = Engine_Library.Database.Engine_Database.Item_List(true, null); // Reload the settings if (itemListFromDb == null) { OnError("ValidateAndClassifyModule : Unable to pull the item table from the database", String.Empty, String.Empty, -1); return; } // Save the item table itemTable = itemListFromDb.Tables[0]; } DataRow[] selected = itemTable.Select("BibID='" + bibid + "' and VID='" + vid + "'"); if (selected.Length > 0) { resource.BibID = bibid; resource.VID = vid; resource.Resource_Type = Incoming_Digital_Resource.Incoming_Digital_Resource_Type.FOLDER_OF_FILES; IncomingPackages.Add(resource); } else { OnError("METS-less folder is not a valid BibID/VID combination", resource.Folder_Name.Replace("_", ":"), "NONE", -1); // Move this resource if (!resource.Move(BuilderFolder.Failures_Folder)) { OnError("Unable to move folder " + resource.Folder_Name + " in " + BuilderFolder.Folder_Name + " to the failures folder", resource.Folder_Name.Replace("_", ":"), "NONE", -1); } } } else { OnError("METS-less folders are not allowed in " + BuilderFolder.Folder_Name, bibid + ":" + vid, "NONE", -1); // Move this resource if (!resource.Move(BuilderFolder.Failures_Folder)) { OnError("Unable to move folder " + resource.Folder_Name + " in " + BuilderFolder.Folder_Name + " to the failures folder", resource.Folder_Name.Replace("_", ":"), "NONE", -1); } } } } else { long validateId = OnProcess("....Validating METS file for " + resource.Folder_Name, "Folder Process", resource.Folder_Name.Replace("_", ":"), "UNKNOWN", -1); string validation_errors = Validate_and_Read_METS(resource, thisMetsValidator, metsSchemeValidator); // Save any errors to the main log if (validation_errors.Length > 0) { // Save the validation errors to the main log Save_Validation_Errors_To_Log(validation_errors, resource.Source_Folder.Folder_Name.Replace("_", ":"), validateId); // Move this resource if (!resource.Move(BuilderFolder.Failures_Folder)) { OnError("Unable to move folder " + resource.Folder_Name + " in " + BuilderFolder.Folder_Name + " to the failures folder", resource.Folder_Name.Replace("_", ":"), "NONE", -1); } } else { // Categorize remaining packages by type switch (resource.Resource_Type) { case Incoming_Digital_Resource.Incoming_Digital_Resource_Type.PARTIAL_PACKAGE: case Incoming_Digital_Resource.Incoming_Digital_Resource_Type.COMPLETE_PACKAGE: IncomingPackages.Add(resource); break; case Incoming_Digital_Resource.Incoming_Digital_Resource_Type.METADATA_UPDATE: if (BuilderFolder.Allow_Metadata_Updates) { IncomingPackages.Add(resource); } else { OnError("Metadata update is not allowed in " + BuilderFolder.Folder_Name, resource.Folder_Name.Replace("_", ":"), "METADATA UPDATE", -1); // Move this resource if (!resource.Move(BuilderFolder.Failures_Folder)) { OnError("Unable to move folder " + resource.Folder_Name + " to failures", resource.Folder_Name.Replace("_", ":"), String.Empty, -1); } } break; case Incoming_Digital_Resource.Incoming_Digital_Resource_Type.DELETE: if (BuilderFolder.Allow_Deletes) { Deletes.Add(resource); } else { OnError("Delete is not allowed in " + BuilderFolder.Folder_Name, resource.Folder_Name.Replace("_", ":"), "DELETE", -1); // Move this resource if (!resource.Move(BuilderFolder.Failures_Folder)) { OnError("Unable to move folder " + resource.Folder_Name + " to failures", resource.Folder_Name.Replace("_", ":"), String.Empty, -1); } } break; } } } // If the maximum number of (incoming, non-delete) packages has now been met, no need to classify anymore if ((MultiInstance_Builder_Settings.Instance_Package_Limit > 0) && (IncomingPackages.Count >= MultiInstance_Builder_Settings.Instance_Package_Limit)) { OnProcess("...Package validation aborted - maximum number of packages ( " + MultiInstance_Builder_Settings.Instance_Package_Limit + " ) reached", "Verbose", String.Empty, String.Empty, -1); return; } } } } } catch (Exception ee) { OnError("Error in harvesting packages from processing : " + ee.Message + "\n" + ee.StackTrace, String.Empty, String.Empty, -1); } }
/// <summary> Method performs the work of the folder-level builder module </summary> /// <param name="BuilderFolder"> Builder folder upon which to perform all work </param> /// <param name="IncomingPackages"> List of valid incoming packages, which may be modified by this process </param> /// <param name="Deletes"> List of valid deletes, which may be modifyed by this process </param> public abstract void DoWork(Actionable_Builder_Source_Folder BuilderFolder, List <Incoming_Digital_Resource> IncomingPackages, List <Incoming_Digital_Resource> Deletes);
/// <summary> Check if there are very large JPEGs which should be converted to TIFF for JPEG2000 and smaller JPEG creation </summary> /// <param name="BuilderFolder"> Builder folder upon which to perform all work </param> /// <param name="IncomingPackages"> List of valid incoming packages, which may be modified by this process </param> /// <param name="Deletes"> List of valid deletes, which may be modifyed by this process </param> public override void DoWork(Actionable_Builder_Source_Folder BuilderFolder, List <Incoming_Digital_Resource> IncomingPackages, List <Incoming_Digital_Resource> Deletes) { throw new NotImplementedException(); }
/// <summary> Check if there are very large JPEGs which should be converted to TIFF for JPEG2000 and smaller JPEG creation </summary> /// <param name="BuilderFolder"> Builder folder upon which to perform all work </param> /// <param name="IncomingPackages"> List of valid incoming packages, which may be modified by this process </param> /// <param name="Deletes"> List of valid deletes, which may be modifyed by this process </param> public override void DoWork(Actionable_Builder_Source_Folder BuilderFolder, List<Incoming_Digital_Resource> IncomingPackages, List<Incoming_Digital_Resource> Deletes) { throw new NotImplementedException(); }