/// <summary> Check if a folder has a BibID restriction ( i.e., incoming items must start with 'XYZ..' ) and then checks each package
/// found in that folder has a valid BibID, or moves it to the failures folder </summary>
/// <param name="BuilderFolder"> Builder folder upon which to perform all work </param>
/// <param name="IncomingPackages"> List of valid incoming packages, which may be modified by this process </param>
/// <param name="Deletes"> List of valid deletes, which may be modifyed by this process </param>
public override void DoWork(Actionable_Builder_Source_Folder BuilderFolder, List <Incoming_Digital_Resource> IncomingPackages, List <Incoming_Digital_Resource> Deletes)
{
// If this folder is limited on what BibID roots it accepts, check that now
if (BuilderFolder.BibID_Roots_Restrictions.Length > 0)
{
if ((Directory.Exists(BuilderFolder.Processing_Folder)) && (Directory.GetDirectories(BuilderFolder.Processing_Folder).Length > 0))
{
// Get the list of all packages in the processing folder
List <Incoming_Digital_Resource> packages = BuilderFolder.Packages_For_Processing;
// Step through each package
foreach (Incoming_Digital_Resource resource in packages)
{
string[] starts = BuilderFolder.BibID_Roots_Restrictions.Split("|,;".ToCharArray());
bool okay = starts.Any(ThisStart => resource.Folder_Name.IndexOf(ThisStart, StringComparison.OrdinalIgnoreCase) == 0);
// If not okay.. it is a failure
if (!okay)
{
OnError("Package " + resource.Folder_Name + " has invalid BibID for " + BuilderFolder.Folder_Name + " incoming folder ( " + BuilderFolder.Folder_Name + " )", resource.BibID + ":" + resource.VID, "INCOMING", -1);
// Move this resource
if (!resource.Move(BuilderFolder.Failures_Folder))
{
OnError("Unable to move folder " + resource.Folder_Name + " in " + BuilderFolder.Folder_Name + " to the failures folder", resource.BibID + ":" + resource.VID, resource.METS_Type_String, -1);
}
}
}
}
}
}
/// <summary> Check if a folder has a BibID restriction ( i.e., incoming items must start with 'XYZ..' ) and then checks each package
/// found in that folder has a valid BibID, or moves it to the failures folder </summary>
/// <param name="BuilderFolder"> Builder folder upon which to perform all work </param>
/// <param name="IncomingPackages"> List of valid incoming packages, which may be modified by this process </param>
/// <param name="Deletes"> List of valid deletes, which may be modifyed by this process </param>
public override void DoWork(Actionable_Builder_Source_Folder BuilderFolder, List<Incoming_Digital_Resource> IncomingPackages, List<Incoming_Digital_Resource> Deletes)
{
// If this folder is limited on what BibID roots it accepts, check that now
if (BuilderFolder.BibID_Roots_Restrictions.Length > 0)
{
if ((Directory.Exists(BuilderFolder.Processing_Folder)) && (Directory.GetDirectories(BuilderFolder.Processing_Folder).Length > 0))
{
// Get the list of all packages in the processing folder
List<Incoming_Digital_Resource> packages = BuilderFolder.Packages_For_Processing;
// Step through each package
foreach (Incoming_Digital_Resource resource in packages)
{
string[] starts = BuilderFolder.BibID_Roots_Restrictions.Split("|,;".ToCharArray());
bool okay = starts.Any(ThisStart => resource.Folder_Name.IndexOf(ThisStart, StringComparison.OrdinalIgnoreCase) == 0);
// If not okay.. it is a failure
if (!okay)
{
OnError("Package " + resource.Folder_Name + " has invalid BibID for " + BuilderFolder.Folder_Name + " incoming folder ( " + BuilderFolder.Folder_Name + " )", resource.BibID + ":" + resource.VID, "INCOMING", -1);
// Move this resource
if (!resource.Move(BuilderFolder.Failures_Folder))
{
OnError("Unable to move folder " + resource.Folder_Name + " in " + BuilderFolder.Folder_Name + " to the failures folder", resource.BibID + ":" + resource.VID, resource.METS_Type_String, -1);
}
}
}
}
}
}
/// <summary> Check for appropriately aged folders and moves them into the related processing folder </summary>
/// <param name="BuilderFolder"> Builder folder upon which to perform all work </param>
/// <param name="IncomingPackages"> List of valid incoming packages, which may be modified by this process </param>
/// <param name="Deletes"> List of valid deletes, which may be modifyed by this process </param>
public override void DoWork(Actionable_Builder_Source_Folder BuilderFolder, List<Incoming_Digital_Resource> IncomingPackages, List<Incoming_Digital_Resource> Deletes)
{
try
{
// Move all eligible packages from the FTP folders to the processing folders
if (Settings.Builder.Verbose_Flag)
OnProcess("Worker_BulkLoader.Move_Appropriate_Inbound_Packages_To_Processing: Checking incoming folder " + BuilderFolder.Inbound_Folder, String.Empty, String.Empty, String.Empty, -1);
if (BuilderFolder.Items_Exist_In_Inbound)
{
if (Settings.Builder.Verbose_Flag)
OnProcess("Worker_BulkLoader.Move_Appropriate_Inbound_Packages_To_Processing: Found either files or subdirectories in " + BuilderFolder.Inbound_Folder, String.Empty, String.Empty, String.Empty, -1);
if (Settings.Builder.Verbose_Flag)
OnProcess("Checking inbound packages for aging and possibly moving to processing", String.Empty, String.Empty, String.Empty, -1);
String outMessage;
if (!BuilderFolder.Move_From_Inbound_To_Processing(out outMessage))
{
if (outMessage.Length > 0) OnError(outMessage, String.Empty, String.Empty, -1);
OnError("Unspecified error moving files from inbound to processing", String.Empty, String.Empty, -1);
}
else
{
if ((Settings.Builder.Verbose_Flag) && (outMessage.Length > 0))
OnProcess(outMessage, String.Empty, String.Empty, String.Empty, -1);
}
// Try to get rid of any empty folders
try
{
string[] subdirs = Directory.GetDirectories(BuilderFolder.Inbound_Folder);
foreach (string thisSubdir in subdirs)
{
if ((Directory.GetFiles(thisSubdir).Length == 0) && (Directory.GetDirectories(thisSubdir).Length == 0))
{
Directory.Delete(thisSubdir);
}
}
}
catch { }
}
else if (Settings.Builder.Verbose_Flag)
OnProcess("Worker_BulkLoader.Move_Appropriate_Inbound_Packages_To_Processing: No subdirectories or files found in incoming folder " + BuilderFolder.Inbound_Folder, String.Empty, String.Empty, String.Empty, -1);
}
catch (Exception ee)
{
OnError("Error in harvesting packages from inbound folders to processing\n" + ee.Message, String.Empty, String.Empty, -1);
}
}
/// <summary> Check for folders that are not named in BibID or BibiD/VID format and do not have metadata. For
/// these items, creates a BibID/VID folder with minimal metadata from the folder name. </summary>
/// <param name="BuilderFolder"> Builder folder upon which to perform all work </param>
/// <param name="IncomingPackages"> List of valid incoming packages, which may be modified by this process </param>
/// <param name="Deletes"> List of valid deletes, which may be modifyed by this process </param>
public override void DoWork(Actionable_Builder_Source_Folder BuilderFolder, List<Incoming_Digital_Resource> IncomingPackages, List<Incoming_Digital_Resource> Deletes)
{
string[] subdirs = Directory.GetDirectories(BuilderFolder.Inbound_Folder);
foreach ( string thisSubDir in subdirs)
{
try
{
string thisSubDirName = (new DirectoryInfo(thisSubDir)).Name;
// Must have some files to continue
if (Directory.GetFiles(thisSubDir).Length == 0)
continue;
// Need to check if this MAY be a valid BibID.
// Need to make this a bit more specific in the future, as it will skip ANY folders
// that are ten digits long right now.
if ((thisSubDir.Length == 10) || ((thisSubDir.Length == 16) && (thisSubDirName[0] == '_')))
continue;
// Look for a METS file or any source of metadata in the folder
if ((Directory.GetFiles(thisSubDir, "*.mets").Length > 0) || (Directory.GetFiles(thisSubDir, "*.xml").Length > 0))
continue;
// Clean any additional periods in the filenames first
string[] allFiles = Directory.GetFiles(thisSubDir);
foreach (string thisFile in allFiles)
{
string fileName = Path.GetFileName(thisFile);
if (Regex.Matches(fileName, "\\.").Count > 1)
{
string newFileName = fileName;
while (Regex.Matches(newFileName, "\\.").Count > 1)
{
char[] charArr = newFileName.ToCharArray();
charArr[newFileName.IndexOf(".")] = '_'; // freely modify the array
newFileName = new string(charArr);
}
File.Move(thisFile, Path.Combine(thisSubDir, newFileName));
}
}
// Create the new object
SobekCM_Item newItem = new SobekCM_Item();
newItem.Bib_Info.SobekCM_Type = TypeOfResource_SobekCM_Enum.Archival;
newItem.Bib_Info.Main_Title.Title = thisSubDirName;
newItem.Bib_Info.Add_Identifier(thisSubDirName);
newItem.Bib_Info.Source.Code = Arguments[1];
newItem.Bib_Info.Source.Statement = Arguments[2];
newItem.BibID = Arguments[0];
newItem.VID = "00001";
// Save this item, for the necessary bibid
SobekCM_Item_Database.Save_New_Digital_Resource(newItem, false, false, "Builder", "Created BibID folder from '" + thisSubDirName + "'", -1);
string newFolderName = newItem.BibID + "_" + newItem.VID;
string newFolder = Path.Combine(BuilderFolder.Inbound_Folder, newFolderName);
Directory.Move(thisSubDir, newFolder);
newItem.Source_Directory = newFolder;
newItem.Save_METS();
}
catch (Exception ee)
{
Console.WriteLine("Error moving directory " + ee.Message);
}
}
}
/// <summary> Check for folders that are not named in BibID or BibiD/VID format and do not have metadata. For
/// these items, creates a BibID/VID folder with minimal metadata from the folder name. </summary>
/// <param name="BuilderFolder"> Builder folder upon which to perform all work </param>
/// <param name="IncomingPackages"> List of valid incoming packages, which may be modified by this process </param>
/// <param name="Deletes"> List of valid deletes, which may be modifyed by this process </param>
public override void DoWork(Actionable_Builder_Source_Folder BuilderFolder, List <Incoming_Digital_Resource> IncomingPackages, List <Incoming_Digital_Resource> Deletes)
{
string[] subdirs = Directory.GetDirectories(BuilderFolder.Inbound_Folder);
foreach (string thisSubDir in subdirs)
{
try
{
string thisSubDirName = (new DirectoryInfo(thisSubDir)).Name;
// Must have some files to continue
if (Directory.GetFiles(thisSubDir).Length == 0)
{
continue;
}
// Need to check if this MAY be a valid BibID.
// Need to make this a bit more specific in the future, as it will skip ANY folders
// that are ten digits long right now.
if ((thisSubDir.Length == 10) || ((thisSubDir.Length == 16) && (thisSubDirName[0] == '_')))
{
continue;
}
// Look for a METS file or any source of metadata in the folder
if ((Directory.GetFiles(thisSubDir, "*.mets").Length > 0) || (Directory.GetFiles(thisSubDir, "*.xml").Length > 0))
{
continue;
}
// Clean any additional periods in the filenames first
string[] allFiles = Directory.GetFiles(thisSubDir);
foreach (string thisFile in allFiles)
{
string fileName = Path.GetFileName(thisFile);
if (Regex.Matches(fileName, "\\.").Count > 1)
{
string newFileName = fileName;
while (Regex.Matches(newFileName, "\\.").Count > 1)
{
char[] charArr = newFileName.ToCharArray();
charArr[newFileName.IndexOf(".")] = '_'; // freely modify the array
newFileName = new string(charArr);
}
File.Move(thisFile, Path.Combine(thisSubDir, newFileName));
}
}
// Create the new object
SobekCM_Item newItem = new SobekCM_Item();
newItem.Bib_Info.SobekCM_Type = TypeOfResource_SobekCM_Enum.Archival;
newItem.Bib_Info.Main_Title.Title = thisSubDirName;
newItem.Bib_Info.Add_Identifier(thisSubDirName);
newItem.Bib_Info.Source.Code = Arguments[1];
newItem.Bib_Info.Source.Statement = Arguments[2];
newItem.BibID = Arguments[0];
newItem.VID = "00001";
// Save this item, for the necessary bibid
SobekCM_Item_Database.Save_New_Digital_Resource(newItem, false, false, "Builder", "Created BibID folder from '" + thisSubDirName + "'", -1);
string newFolderName = newItem.BibID + "_" + newItem.VID;
string newFolder = Path.Combine(BuilderFolder.Inbound_Folder, newFolderName);
Directory.Move(thisSubDir, newFolder);
newItem.Source_Directory = newFolder;
newItem.Save_METS();
}
catch (Exception ee)
{
Console.WriteLine("Error moving directory " + ee.Message);
}
}
}
/// <summary> Validates the metadata for each folder and classified as package either adding
/// a new item or updating an existing item versus a package requesting a delete </summary>
/// <param name="BuilderFolder"> Builder folder upon which to perform all work </param>
/// <param name="IncomingPackages"> List of valid incoming packages, which may be modified by this process </param>
/// <param name="Deletes"> List of valid deletes, which may be modifyed by this process </param>
public override void DoWork(Actionable_Builder_Source_Folder BuilderFolder, List<Incoming_Digital_Resource> IncomingPackages, List<Incoming_Digital_Resource> Deletes)
{
if (Settings.Builder.Verbose_Flag)
OnProcess("ValidateAndClassifyModule.Perform_BulkLoader: Begin validating and classifying packages in incoming/process folders", "Verbose", String.Empty, String.Empty, -1);
// If the maximum number of (incoming, non-delete) packages have already been set aside to process, no need to continue on this folder
if ((MultiInstance_Builder_Settings.Instance_Package_Limit > 0) && (IncomingPackages.Count >= MultiInstance_Builder_Settings.Instance_Package_Limit))
{
OnProcess("...Package validation aborted - maximum number of packages ( " + MultiInstance_Builder_Settings.Instance_Package_Limit + " ) reached", "Verbose", String.Empty, String.Empty, -1);
return;
}
try
{
// Only continue if the processing folder exists and has subdirectories
if ((Directory.Exists(BuilderFolder.Processing_Folder)) && (Directory.GetDirectories(BuilderFolder.Processing_Folder).Length > 0))
{
// Get the list of all packages in the processing folder
if (Settings.Builder.Verbose_Flag)
OnProcess("....Validate packages for " + BuilderFolder.Folder_Name, "Verbose", String.Empty, String.Empty, -1);
List<Incoming_Digital_Resource> packages = BuilderFolder.Packages_For_Processing;
if (packages.Count > 0)
{
// Create the METS validation objects
if (thisMetsValidator == null)
{
if (Settings.Builder.Verbose_Flag)
OnProcess("ValidateAndClassifyModule.Constructor: Created Validators", "Verbose", String.Empty, String.Empty, -1);
thisMetsValidator = new SobekCM_METS_Validator(String.Empty);
metsSchemeValidator = new METS_Validator_Object(false);
}
// Step through each package
foreach (Incoming_Digital_Resource resource in packages)
{
// Validate the categorize the package
if (Settings.Builder.Verbose_Flag)
OnProcess("........Checking '" + resource.Folder_Name + "'", "Verbose", resource.Folder_Name.Replace("_", ":"), String.Empty, -1);
// If there is no METS file, use special code to check this
if (Directory.GetFiles(resource.Resource_Folder, "*.mets*").Length == 0)
{
DirectoryInfo noMetsDirInfo = new DirectoryInfo(resource.Resource_Folder);
string vid = noMetsDirInfo.Name;
if (noMetsDirInfo.Parent != null) // Should never be null
{
string bibid = noMetsDirInfo.Parent.Name;
if ((vid.Length == 16) && (vid[10] == '_'))
{
bibid = vid.Substring(0, 10);
vid = vid.Substring(11, 5);
}
// Is this allowed in this incomnig folder?
if (BuilderFolder.Allow_Folders_No_Metadata)
{
if (itemTable == null)
{
DataSet itemListFromDb = SobekCM_Database.Get_Item_List(true, null);
// Reload the settings
if (itemListFromDb == null)
{
OnError("ValidateAndClassifyModule : Unable to pull the item table from the database", String.Empty, String.Empty, -1);
return;
}
// Save the item table
itemTable = itemListFromDb.Tables[0];
}
DataRow[] selected = itemTable.Select("BibID='" + bibid + "' and VID='" + vid + "'");
if (selected.Length > 0)
{
resource.BibID = bibid;
resource.VID = vid;
resource.Resource_Type = Incoming_Digital_Resource.Incoming_Digital_Resource_Type.FOLDER_OF_FILES;
IncomingPackages.Add(resource);
}
else
{
OnError("METS-less folder is not a valid BibID/VID combination", resource.Folder_Name.Replace("_", ":"), "NONE", -1);
// Move this resource
if (!resource.Move(BuilderFolder.Failures_Folder))
{
OnError("Unable to move folder " + resource.Folder_Name + " in " + BuilderFolder.Folder_Name + " to the failures folder", resource.Folder_Name.Replace("_", ":"), "NONE", -1);
}
}
}
else
{
OnError("METS-less folders are not allowed in " + BuilderFolder.Folder_Name, bibid + ":" + vid, "NONE", -1);
// Move this resource
if (!resource.Move(BuilderFolder.Failures_Folder))
{
OnError("Unable to move folder " + resource.Folder_Name + " in " + BuilderFolder.Folder_Name + " to the failures folder", resource.Folder_Name.Replace("_", ":"), "NONE", -1);
}
}
}
}
else
{
long validateId = OnProcess("....Validating METS file for " + resource.Folder_Name, "Folder Process", resource.Folder_Name.Replace("_", ":"), "UNKNOWN", -1);
string validation_errors = Validate_and_Read_METS(resource, thisMetsValidator, metsSchemeValidator);
// Save any errors to the main log
if (validation_errors.Length > 0)
{
// Save the validation errors to the main log
Save_Validation_Errors_To_Log(validation_errors, resource.Source_Folder.Folder_Name.Replace("_", ":"), validateId);
// Move this resource
if (!resource.Move(BuilderFolder.Failures_Folder))
{
OnError("Unable to move folder " + resource.Folder_Name + " in " + BuilderFolder.Folder_Name + " to the failures folder", resource.Folder_Name.Replace("_", ":"), "NONE", -1);
}
}
else
{
// Categorize remaining packages by type
switch (resource.Resource_Type)
{
case Incoming_Digital_Resource.Incoming_Digital_Resource_Type.PARTIAL_PACKAGE:
case Incoming_Digital_Resource.Incoming_Digital_Resource_Type.COMPLETE_PACKAGE:
IncomingPackages.Add(resource);
break;
case Incoming_Digital_Resource.Incoming_Digital_Resource_Type.METADATA_UPDATE:
if (BuilderFolder.Allow_Metadata_Updates)
{
IncomingPackages.Add(resource);
}
else
{
OnError("Metadata update is not allowed in " + BuilderFolder.Folder_Name, resource.Folder_Name.Replace("_", ":"), "METADATA UPDATE", -1);
// Move this resource
if (!resource.Move(BuilderFolder.Failures_Folder))
{
OnError("Unable to move folder " + resource.Folder_Name + " to failures", resource.Folder_Name.Replace("_", ":"), String.Empty, -1);
}
}
break;
case Incoming_Digital_Resource.Incoming_Digital_Resource_Type.DELETE:
if (BuilderFolder.Allow_Deletes)
{
Deletes.Add(resource);
}
else
{
OnError("Delete is not allowed in " + BuilderFolder.Folder_Name, resource.Folder_Name.Replace("_", ":"), "DELETE", -1);
// Move this resource
if (!resource.Move(BuilderFolder.Failures_Folder))
{
OnError("Unable to move folder " + resource.Folder_Name + " to failures", resource.Folder_Name.Replace("_", ":"), String.Empty, -1);
}
}
break;
}
}
}
// If the maximum number of (incoming, non-delete) packages has now been met, no need to classify anymore
if ((MultiInstance_Builder_Settings.Instance_Package_Limit > 0) && (IncomingPackages.Count >= MultiInstance_Builder_Settings.Instance_Package_Limit))
{
OnProcess("...Package validation aborted - maximum number of packages ( " + MultiInstance_Builder_Settings.Instance_Package_Limit + " ) reached", "Verbose", String.Empty, String.Empty, -1);
return;
}
}
}
}
}
catch (Exception ee)
{
OnError("Error in harvesting packages from processing : " + ee.Message + "\n" + ee.StackTrace , String.Empty, String.Empty, -1);
}
}
/// <summary> Method performs the work of the folder-level builder module </summary> /// <param name="BuilderFolder"> Builder folder upon which to perform all work </param> /// <param name="IncomingPackages"> List of valid incoming packages, which may be modified by this process </param> /// <param name="Deletes"> List of valid deletes, which may be modifyed by this process </param> public abstract void DoWork(Actionable_Builder_Source_Folder BuilderFolder, List<Incoming_Digital_Resource> IncomingPackages, List<Incoming_Digital_Resource> Deletes);
/// <summary> Check for appropriately aged folders and moves them into the related processing folder </summary>
/// <param name="BuilderFolder"> Builder folder upon which to perform all work </param>
/// <param name="IncomingPackages"> List of valid incoming packages, which may be modified by this process </param>
/// <param name="Deletes"> List of valid deletes, which may be modifyed by this process </param>
public override void DoWork(Actionable_Builder_Source_Folder BuilderFolder, List <Incoming_Digital_Resource> IncomingPackages, List <Incoming_Digital_Resource> Deletes)
{
try
{
// Move all eligible packages from the FTP folders to the processing folders
if (Settings.Builder.Verbose_Flag)
{
OnProcess("Worker_BulkLoader.Move_Appropriate_Inbound_Packages_To_Processing: Checking incoming folder " + BuilderFolder.Inbound_Folder, String.Empty, String.Empty, String.Empty, -1);
}
if (BuilderFolder.Items_Exist_In_Inbound)
{
if (Settings.Builder.Verbose_Flag)
{
OnProcess("Worker_BulkLoader.Move_Appropriate_Inbound_Packages_To_Processing: Found either files or subdirectories in " + BuilderFolder.Inbound_Folder, String.Empty, String.Empty, String.Empty, -1);
}
if (Settings.Builder.Verbose_Flag)
{
OnProcess("Checking inbound packages for aging and possibly moving to processing", String.Empty, String.Empty, String.Empty, -1);
}
String outMessage;
if (!BuilderFolder.Move_From_Inbound_To_Processing(Settings, out outMessage))
{
if (outMessage.Length > 0)
{
OnError(outMessage, String.Empty, String.Empty, -1);
}
OnError("Unspecified error moving files from inbound to processing", String.Empty, String.Empty, -1);
}
else
{
if ((Settings.Builder.Verbose_Flag) && (outMessage.Length > 0))
{
OnProcess(outMessage, String.Empty, String.Empty, String.Empty, -1);
}
}
// Try to get rid of any empty folders
try
{
string[] subdirs = Directory.GetDirectories(BuilderFolder.Inbound_Folder);
foreach (string thisSubdir in subdirs)
{
if ((Directory.GetFiles(thisSubdir).Length == 0) && (Directory.GetDirectories(thisSubdir).Length == 0))
{
Directory.Delete(thisSubdir);
}
}
}
catch { }
}
else if (Settings.Builder.Verbose_Flag)
{
OnProcess("Worker_BulkLoader.Move_Appropriate_Inbound_Packages_To_Processing: No subdirectories or files found in incoming folder " + BuilderFolder.Inbound_Folder, String.Empty, String.Empty, String.Empty, -1);
}
}
catch (Exception ee)
{
OnError("Error in harvesting packages from inbound folders to processing\n" + ee.Message, String.Empty, String.Empty, -1);
}
}
/// <summary> Validates the metadata for each folder and classified as package either adding
/// a new item or updating an existing item versus a package requesting a delete </summary>
/// <param name="BuilderFolder"> Builder folder upon which to perform all work </param>
/// <param name="IncomingPackages"> List of valid incoming packages, which may be modified by this process </param>
/// <param name="Deletes"> List of valid deletes, which may be modifyed by this process </param>
public override void DoWork(Actionable_Builder_Source_Folder BuilderFolder, List <Incoming_Digital_Resource> IncomingPackages, List <Incoming_Digital_Resource> Deletes)
{
if (Settings.Builder.Verbose_Flag)
{
OnProcess("ValidateAndClassifyModule.Perform_BulkLoader: Begin validating and classifying packages in incoming/process folders", "Verbose", String.Empty, String.Empty, -1);
}
// If the maximum number of (incoming, non-delete) packages have already been set aside to process, no need to continue on this folder
if ((MultiInstance_Builder_Settings.Instance_Package_Limit > 0) && (IncomingPackages.Count >= MultiInstance_Builder_Settings.Instance_Package_Limit))
{
OnProcess("...Package validation aborted - maximum number of packages ( " + MultiInstance_Builder_Settings.Instance_Package_Limit + " ) reached", "Verbose", String.Empty, String.Empty, -1);
return;
}
try
{
// Only continue if the processing folder exists and has subdirectories
if ((Directory.Exists(BuilderFolder.Processing_Folder)) && (Directory.GetDirectories(BuilderFolder.Processing_Folder).Length > 0))
{
// Get the list of all packages in the processing folder
if (Settings.Builder.Verbose_Flag)
{
OnProcess("....Validate packages for " + BuilderFolder.Folder_Name, "Verbose", String.Empty, String.Empty, -1);
}
List <Incoming_Digital_Resource> packages = BuilderFolder.Packages_For_Processing;
if (packages.Count > 0)
{
// Create the METS validation objects
if (thisMetsValidator == null)
{
if (Settings.Builder.Verbose_Flag)
{
OnProcess("ValidateAndClassifyModule.Constructor: Created Validators", "Verbose", String.Empty, String.Empty, -1);
}
thisMetsValidator = new SobekCM_METS_Validator(String.Empty);
metsSchemeValidator = new METS_Validator_Object(false);
}
// Step through each package
foreach (Incoming_Digital_Resource resource in packages)
{
// Validate the categorize the package
if (Settings.Builder.Verbose_Flag)
{
OnProcess("........Checking '" + resource.Folder_Name + "'", "Verbose", resource.Folder_Name.Replace("_", ":"), String.Empty, -1);
}
// If there is no METS file, use special code to check this
if (Directory.GetFiles(resource.Resource_Folder, "*.mets*").Length == 0)
{
DirectoryInfo noMetsDirInfo = new DirectoryInfo(resource.Resource_Folder);
string vid = noMetsDirInfo.Name;
if (noMetsDirInfo.Parent != null) // Should never be null
{
string bibid = noMetsDirInfo.Parent.Name;
if ((vid.Length == 16) && (vid[10] == '_'))
{
bibid = vid.Substring(0, 10);
vid = vid.Substring(11, 5);
}
// Is this allowed in this incomnig folder?
if (BuilderFolder.Allow_Folders_No_Metadata)
{
if (itemTable == null)
{
DataSet itemListFromDb = Engine_Library.Database.Engine_Database.Item_List(true, null);
// Reload the settings
if (itemListFromDb == null)
{
OnError("ValidateAndClassifyModule : Unable to pull the item table from the database", String.Empty, String.Empty, -1);
return;
}
// Save the item table
itemTable = itemListFromDb.Tables[0];
}
DataRow[] selected = itemTable.Select("BibID='" + bibid + "' and VID='" + vid + "'");
if (selected.Length > 0)
{
resource.BibID = bibid;
resource.VID = vid;
resource.Resource_Type = Incoming_Digital_Resource.Incoming_Digital_Resource_Type.FOLDER_OF_FILES;
IncomingPackages.Add(resource);
}
else
{
OnError("METS-less folder is not a valid BibID/VID combination", resource.Folder_Name.Replace("_", ":"), "NONE", -1);
// Move this resource
if (!resource.Move(BuilderFolder.Failures_Folder))
{
OnError("Unable to move folder " + resource.Folder_Name + " in " + BuilderFolder.Folder_Name + " to the failures folder", resource.Folder_Name.Replace("_", ":"), "NONE", -1);
}
}
}
else
{
OnError("METS-less folders are not allowed in " + BuilderFolder.Folder_Name, bibid + ":" + vid, "NONE", -1);
// Move this resource
if (!resource.Move(BuilderFolder.Failures_Folder))
{
OnError("Unable to move folder " + resource.Folder_Name + " in " + BuilderFolder.Folder_Name + " to the failures folder", resource.Folder_Name.Replace("_", ":"), "NONE", -1);
}
}
}
}
else
{
long validateId = OnProcess("....Validating METS file for " + resource.Folder_Name, "Folder Process", resource.Folder_Name.Replace("_", ":"), "UNKNOWN", -1);
string validation_errors = Validate_and_Read_METS(resource, thisMetsValidator, metsSchemeValidator);
// Save any errors to the main log
if (validation_errors.Length > 0)
{
// Save the validation errors to the main log
Save_Validation_Errors_To_Log(validation_errors, resource.Source_Folder.Folder_Name.Replace("_", ":"), validateId);
// Move this resource
if (!resource.Move(BuilderFolder.Failures_Folder))
{
OnError("Unable to move folder " + resource.Folder_Name + " in " + BuilderFolder.Folder_Name + " to the failures folder", resource.Folder_Name.Replace("_", ":"), "NONE", -1);
}
}
else
{
// Categorize remaining packages by type
switch (resource.Resource_Type)
{
case Incoming_Digital_Resource.Incoming_Digital_Resource_Type.PARTIAL_PACKAGE:
case Incoming_Digital_Resource.Incoming_Digital_Resource_Type.COMPLETE_PACKAGE:
IncomingPackages.Add(resource);
break;
case Incoming_Digital_Resource.Incoming_Digital_Resource_Type.METADATA_UPDATE:
if (BuilderFolder.Allow_Metadata_Updates)
{
IncomingPackages.Add(resource);
}
else
{
OnError("Metadata update is not allowed in " + BuilderFolder.Folder_Name, resource.Folder_Name.Replace("_", ":"), "METADATA UPDATE", -1);
// Move this resource
if (!resource.Move(BuilderFolder.Failures_Folder))
{
OnError("Unable to move folder " + resource.Folder_Name + " to failures", resource.Folder_Name.Replace("_", ":"), String.Empty, -1);
}
}
break;
case Incoming_Digital_Resource.Incoming_Digital_Resource_Type.DELETE:
if (BuilderFolder.Allow_Deletes)
{
Deletes.Add(resource);
}
else
{
OnError("Delete is not allowed in " + BuilderFolder.Folder_Name, resource.Folder_Name.Replace("_", ":"), "DELETE", -1);
// Move this resource
if (!resource.Move(BuilderFolder.Failures_Folder))
{
OnError("Unable to move folder " + resource.Folder_Name + " to failures", resource.Folder_Name.Replace("_", ":"), String.Empty, -1);
}
}
break;
}
}
}
// If the maximum number of (incoming, non-delete) packages has now been met, no need to classify anymore
if ((MultiInstance_Builder_Settings.Instance_Package_Limit > 0) && (IncomingPackages.Count >= MultiInstance_Builder_Settings.Instance_Package_Limit))
{
OnProcess("...Package validation aborted - maximum number of packages ( " + MultiInstance_Builder_Settings.Instance_Package_Limit + " ) reached", "Verbose", String.Empty, String.Empty, -1);
return;
}
}
}
}
}
catch (Exception ee)
{
OnError("Error in harvesting packages from processing : " + ee.Message + "\n" + ee.StackTrace, String.Empty, String.Empty, -1);
}
}
/// <summary> Method performs the work of the folder-level builder module </summary> /// <param name="BuilderFolder"> Builder folder upon which to perform all work </param> /// <param name="IncomingPackages"> List of valid incoming packages, which may be modified by this process </param> /// <param name="Deletes"> List of valid deletes, which may be modifyed by this process </param> public abstract void DoWork(Actionable_Builder_Source_Folder BuilderFolder, List <Incoming_Digital_Resource> IncomingPackages, List <Incoming_Digital_Resource> Deletes);
/// <summary> Check if there are very large JPEGs which should be converted to TIFF for JPEG2000 and smaller JPEG creation </summary>
/// <param name="BuilderFolder"> Builder folder upon which to perform all work </param>
/// <param name="IncomingPackages"> List of valid incoming packages, which may be modified by this process </param>
/// <param name="Deletes"> List of valid deletes, which may be modifyed by this process </param>
public override void DoWork(Actionable_Builder_Source_Folder BuilderFolder, List <Incoming_Digital_Resource> IncomingPackages, List <Incoming_Digital_Resource> Deletes)
{
throw new NotImplementedException();
}
/// <summary> Check if there are very large JPEGs which should be converted to TIFF for JPEG2000 and smaller JPEG creation </summary>
/// <param name="BuilderFolder"> Builder folder upon which to perform all work </param>
/// <param name="IncomingPackages"> List of valid incoming packages, which may be modified by this process </param>
/// <param name="Deletes"> List of valid deletes, which may be modifyed by this process </param>
public override void DoWork(Actionable_Builder_Source_Folder BuilderFolder, List<Incoming_Digital_Resource> IncomingPackages, List<Incoming_Digital_Resource> Deletes)
{
throw new NotImplementedException();
}