public static List <IamsItem> queryMetadataAPIs(String spURL, String spList, List <HMDObject> itemList, bool writeToSharepoint)
        {
            // This function makes sure that you aren't querying Aleph for manuscripts, or similar
            List <IamsItem> IAMSRecords = new List <IamsItem>();

            foreach (HMDObject item in itemList)
            {
                if (item.MetadataSource.ToUpper().ToLower().Contains("aleph"))
                {
                    List <AlephItem> returnedAlephRecords = GetAlephRecords(item);
                    string           outFolder            = Environment.GetFolderPath(Environment.SpecialFolder.Desktop);
                    outFolder += @"\HMDSharepoint_AlephRecords\";
                    outFolder += item.Shelfmark + "_" + item.SystemNumber;
                    if (!Directory.Exists(outFolder))
                    {
                        Directory.CreateDirectory(outFolder);
                    }
                    writeAlephCSV(item.Shelfmark, returnedAlephRecords, outFolder);
                }
                else if (item.MetadataSource.ToLower().ToUpper().Contains("IAMS"))
                {
                    var IAMSitem = GetIAMSRecords(item);
                    IAMSRecords.Add(IAMSitem);
                }
            }
            foreach (var iamsItem in IAMSRecords)
            {
                if (iamsItem.SharepointID != null)
                {
                    try
                    {
                        String columnName   = "IAMSCatalogueStatusIsPublished";
                        var    theShelfmark = iamsItem.ItemShelfmark;
                        var    itemID       = iamsItem.SharepointID;
                        var    message      = "";
                        if (iamsItem.CatalogueStatus.ToUpper().ToLower().Contains("published"))
                        {
                            message = "Yes";
                        }
                        else
                        {
                            message = "No";
                        }
                        if (writeToSharepoint)
                        {
                            SharepointTools.CreateSharepointColumn(spURL, "Digitisation Workflow", columnName);
                            SharepointTools.WriteToSharepointColumnByID(spURL, spList, columnName, theShelfmark, itemID, message);
                        }

                        if (iamsItem.ChildRecordTitles.Count > 0)
                        {
                            // write IAMS child record titles to a CSV
                            string outFolder = Environment.GetFolderPath(Environment.SpecialFolder.Desktop);
                            outFolder += @"\HMDSharepoint_IamsChildRecords\";
                            outFolder += iamsItem.ItemShelfmark;
                            if (!Directory.Exists(outFolder))
                            {
                                Directory.CreateDirectory(outFolder);
                            }
                            writeIAMSCSV(iamsItem, outFolder);
                        }
                    }

                    catch (Exception ex)
                    {
                        Console.WriteLine("Problem writing IAMS catalogue status. \nException {0}", ex);
                    }
                    try
                    {
                        String columnName   = "Shelfmark_HasIAMSDeleteFlag";
                        var    theShelfmark = iamsItem.ItemShelfmark;
                        var    itemID       = iamsItem.SharepointID;
                        var    message      = "";
                        if (iamsItem.DeleteFlagPresent)
                        {
                            message = "Yes";
                        }
                        else
                        {
                            message = "No";
                        }
                        if (writeToSharepoint)
                        {
                            SharepointTools.CreateSharepointColumn(spURL, "Digitisation Workflow", columnName);
                            SharepointTools.WriteToSharepointColumnByID(spURL, spList, columnName, theShelfmark, itemID, message);
                        }
                    }
                    catch (Exception ex)
                    {
                        Console.WriteLine("Problem writing IAMS delete flag status. \nException {0}", ex);
                    }
                }
                else
                {
                    continue;
                }
            }

            return(IAMSRecords);
        }
예제 #2
0
        static void Main(string[] args)
        {
            bool runProgram = true;

            while (runProgram)
            {
                var  startTime = DateTime.Now;
                bool debug     = false;


                bool   useIngestSpreadsheet = false;
                String restartProgram       = String.Empty;

                // Start of sharepoint-specific items
                Console.WriteLine("This program can use the sharepoint site or a user-provided ingest spreadsheet.\nUse ingest spreadsheet?");
                String useInputIngestSpreadsheet = Console.ReadLine();
                if (useInputIngestSpreadsheet.ToLower().Equals("yes") || useInputIngestSpreadsheet.ToLower().Equals("y"))
                {
                    useIngestSpreadsheet = true;
                    Console.WriteLine("Using ingest spreadsheet...");
                }
                else
                {
                    Console.WriteLine("Using sharepoint...");
                }
                if (!useIngestSpreadsheet)
                {
                    // Set the environment (use 'test' or 'prod')
                    var env = "test";



                    Console.WriteLine("You are currently running {0}. Switch to prod? yes/no", env);
                    String inputEnv = Console.ReadLine();
                    if (inputEnv.ToLower().Equals("yes") || inputEnv.ToLower().Equals("y"))
                    {
                        env = "prod";
                    }


                    // do initial setup for sharepoint jobs
                    Dictionary <String, String> variablesDictionary = InitialSetup(env);
                    Assert.IsNotNull(variablesDictionary); // Check we set the variables properly
                    var           spURL         = variablesDictionary["spURL"];
                    var           project       = variablesDictionary["project"];
                    List <String> inputVariable = new List <string>();
                    Assert.IsTrue(RunInitialTests(spURL));

                    Console.WriteLine("This program searches sharepoint using either the 'Project' field or the 'Shelfmark' field.\nShelfmarks can be used individually or provided in a shelfmarks.txt file in this folder, with one shelfmark per line.\nUse shelfmarks? (yes/no)");
                    String useShelfmarksYN = Console.ReadLine();
                    if (useShelfmarksYN.ToLower().Equals("yes") || useShelfmarksYN.ToLower().Equals("y"))
                    {
                        Console.WriteLine("Use list of shelfmarks? (yes/no)");
                        String inputShelfmarkList = Console.ReadLine();
                        if (inputShelfmarkList.ToLower().Equals("yes") || inputShelfmarkList.ToLower().Equals("y"))
                        {
                            Console.WriteLine("Searching for 'shelfmarks.txt' file in current folder...");
                            var currentDir          = Directory.GetCurrentDirectory();
                            var shelfmark_list_path = currentDir + "\\shelfmarks.txt";

                            if (System.IO.File.Exists(shelfmark_list_path))
                            {
                                Console.WriteLine("Found shelfmarks.txt file in current folder!");
                                foreach (var line in System.IO.File.ReadLines(shelfmark_list_path))
                                {
                                    inputVariable.Add(@"<FieldRef Name ='Title'/><Value Type = 'Text'>" + line + @"</Value>");
                                }
                            }
                            else
                            {
                                Console.WriteLine("Could not find file 'shelfmarks.txt' in current folder. Using single shelfmark...");
                                Console.WriteLine("Enter a shelfmark name (must match Sharepoint site)");
                                String inputSingleShelfmark = Console.ReadLine();
                                if (inputSingleShelfmark.Length > 0)
                                {
                                    inputVariable.Add(@"<FieldRef Name ='Title'/><Value Type = 'Text'>" + inputSingleShelfmark + @"</Value>");
                                }
                                else // kick it back and use defaults if nothing provided
                                {
                                    Console.WriteLine("No shelfmark provided! Using default project");
                                    inputVariable.Add(@"<FieldRef Name ='Project_x0020_Name'/><Value Type = 'Text'>" + project + @"</Value>");
                                }
                            }
                        }
                        else
                        {
                            Console.WriteLine("Enter a shelfmark name (must match Sharepoint site)");
                            String inputSingleShelfmark = Console.ReadLine();
                            if (inputSingleShelfmark.Length > 0)
                            {
                                inputVariable.Add(@"<FieldRef Name ='Title'/><Value Type = 'Text'>" + inputSingleShelfmark + @"</Value>");
                            }
                            else // kick it back and use defaults if nothing provided
                            {
                                Console.WriteLine("No shelfmark provided! Using default project");
                                inputVariable.Add(@"<FieldRef Name ='Project_x0020_Name'/><Value Type = 'Text'>" + project + @"</Value>");
                            }
                        }
                    }
                    else
                    {
                        Console.WriteLine("Project is currently set to {0}, change this? (yes/no)", project);
                        String inputProjectYN = Console.ReadLine();
                        if (inputProjectYN.ToLower().Equals("yes") || inputProjectYN.ToLower().Equals("y"))
                        {
                            Console.WriteLine("Type a project name (must match sharepoint record)", project);
                            String inputProject = Console.ReadLine();
                            if (inputProject.Length > 0)
                            {
                                inputVariable.Add(@"<FieldRef Name ='Project_x0020_Name'/><Value Type = 'Text'>" + inputProject + @"</Value>");
                            }
                            else
                            {
                                Console.WriteLine("No project provided! Using default project.");
                                inputVariable.Add(@"<FieldRef Name ='Project_x0020_Name'/><Value Type = 'Text'>" + project + @"</Value>");
                            }
                        }
                        else
                        {
                            Console.WriteLine("Using default project {0}", project);
                            inputVariable.Add(@"<FieldRef Name ='Project_x0020_Name'/><Value Type = 'Text'>" + project + @"</Value>");
                        }
                    }

                    // Check we assigned something to the input variable
                    Assert.AreNotEqual(0, inputVariable.Count);

                    //===================== Checks to run ====================
                    bool reportShelfmarkCheckStatus       = false;
                    bool runShelfmarkCharacterChecks      = false;
                    bool runImageOrderGenerationXMLChecks = false;
                    bool queryMetadata = false;


                    Console.WriteLine("Run shelfmark source folder checks? yes/no");
                    String inputSourceFolderCheck = Console.ReadLine();
                    if (inputSourceFolderCheck.ToLower().Equals("yes") || inputSourceFolderCheck.ToLower().Equals("y"))
                    {
                        reportShelfmarkCheckStatus = true;
                    }
                    Console.WriteLine("Run shelfmark protected character check? yes/no");
                    String inputSMCharCheck = Console.ReadLine();
                    if (inputSMCharCheck.ToLower().Equals("yes") || inputSMCharCheck.ToLower().Equals("y"))
                    {
                        runShelfmarkCharacterChecks = true;
                    }
                    Console.WriteLine("Generate image order CSV and perform ALTO XML checks? yes/no");
                    String inputImageOrderGen = Console.ReadLine();
                    if (inputImageOrderGen.ToLower().Equals("yes") || inputImageOrderGen.ToLower().Equals("y"))
                    {
                        runImageOrderGenerationXMLChecks = true;
                    }
                    Console.WriteLine("Run query against descriptive metadata APIs? yes/no");
                    String inputMDAPIQuery = Console.ReadLine();
                    if (inputMDAPIQuery.ToLower().Equals("yes") || inputMDAPIQuery.ToLower().Equals("y"))
                    {
                        queryMetadata = true;
                    }
                    // =======================================================



                    // Get the 'Digitisation Workflow' list fields and print them out:
                    if (debug)
                    {
                        var DigitisationWorkflowTitles = SharepointTools.GetSharePointListFields(spURL, "Digitisation Workflow");
                        Assert.IsNotNull(DigitisationWorkflowTitles.Count);
                    }

                    // ============== Preliminary stuff - has to be run every time ================
                    // This is the "grab all the info from sharepoint" part of things

                    // Get the contents of the "ID", "Shelfmark" and "Source Folder" columns in the 'Digitisation Workflow' list

                    try
                    {
                        for (int counter = 0; counter < inputVariable.Count; ++counter)
                        {
                            var inputVar = inputVariable[counter];
                            Console.WriteLine("Processing input {0} of {1}", counter + 1, inputVariable.Count);


                            var DigitisationWorkflow_ID_Title_SourceFolders = SharepointTools.GetSharePointListFieldContents(spURL, "Digitisation Workflow", env, inputVar);
                            if (DigitisationWorkflow_ID_Title_SourceFolders.Count < 1)
                            {
                                continue; // goes on to next input...
                            }

                            //  Check source folders - requires the above two lines to work
                            var SourceFolderStatus = SharepointTools.CheckSourceFolderExists(DigitisationWorkflow_ID_Title_SourceFolders);
                            Assert.IsNotNull(SourceFolderStatus.Count);
                            if (debug)
                            {
                                Assert.IsTrue(TextOutputFunctions.OutputListOfLists(SourceFolderStatus, "sourceFolderStatus"));
                            }

                            // =============================================================================

                            // ================= Optional checks - some take ~10 mins to read all of sharepoint, so don't run every time =============

                            // Validates that the value of the 'Source Folder' field exists as a valid path on the network
                            // Attempts to write the results to a sharepoint column for each row (== shelfmark)
                            if (reportShelfmarkCheckStatus)
                            {
                                Console.WriteLine("Writing source folder validation status to Sharepoint...");
                                String SharePointSourceFolderCheck = "SourceFolderValid";
                                Assert.IsTrue(SharepointTools.CreateSharepointColumn(spURL, "Digitisation Workflow", SharePointSourceFolderCheck));
                                Assert.IsTrue(SharepointTools.ReportSourceFolderStatus(spURL, "Digitisation Workflow", SharePointSourceFolderCheck, SourceFolderStatus));
                            }

                            // Checks for protected characters in Shelfmark names
                            // Attempts to write the results to sharepoint column
                            if (runShelfmarkCharacterChecks)
                            {
                                Console.WriteLine("=======================================\nChecking shelfmarks for protected characters\n=======================================");
                                // Add in Shelfmark protected chars check here
                                String SharePointColumnShelfmarkCheck = "ShelfmarkCheck";
                                Assert.IsTrue(SharepointTools.CreateSharepointColumn(spURL, "Digitisation Workflow", SharePointColumnShelfmarkCheck));
                                List <HMDObject> badShelfmarks            = SharepointTools.BadShelfmarkNames(SourceFolderStatus);
                                String           shelfmarkCharacterStatus = "";

                                var thisItem = 1;

                                foreach (var item in badShelfmarks)
                                {
                                    if (badShelfmarks.Count > 20)
                                    {
                                        if (thisItem % 10 == 0)
                                        {
                                            Console.WriteLine("{0}/{1}", thisItem, badShelfmarks.Count);
                                        }
                                    }
                                    else
                                    {
                                        Console.WriteLine("{0}/{1}", thisItem, badShelfmarks.Count);
                                    }
                                    thisItem += 1;
                                    var    ID = item.ID;
                                    String SM = item.Shelfmark;
                                    if (item.BadShelfmark)
                                    {
                                        shelfmarkCharacterStatus = "Protected character(s) found";
                                    }
                                    else
                                    {
                                        shelfmarkCharacterStatus = "Valid";
                                    }
                                    Assert.IsTrue(SharepointTools.WriteToSharepointColumnByID(spURL, "Digitisation Workflow", SharePointColumnShelfmarkCheck, SM, ID, shelfmarkCharacterStatus));
                                }
                            }

                            // Get the labels (image order, image type etc) for all shelfmarks passed into this function
                            // Write these out in an image order csv


                            // Also runs ALTO XML checks - do they exist with the same names as TIFs and in the same number as TIFs?
                            // Are they version 2.0 or older?
                            if (runImageOrderGenerationXMLChecks)
                            {
                                bool writeToSharepoint = true;
                                bool addColumns        = false;
                                if (addColumns)
                                {
                                    // Add columns for XML checking
                                    String SharePointColumnXMLCheck = "ALTOXMLCheck";
                                    Assert.IsTrue(SharepointTools.CreateSharepointColumn(spURL, "Digitisation Workflow", SharePointColumnXMLCheck));
                                }
                                Console.WriteLine("=======================================\nQuerying metadata APIs to retrieve child shelfmark information...\n=======================================");
                                var iamsRecords = LibraryAPIs.queryMetadataAPIs(spURL, "Digitisation Workflow", SourceFolderStatus, writeToSharepoint);

                                var allShelfmarkFiles = InputOrderSpreadsheetTools.listAllShelfmarkFilesTIFXML(SourceFolderStatus, env, spURL, "Digitisation Workflow", iamsRecords, writeToSharepoint);
                                Assert.IsNotNull(allShelfmarkFiles);
                            }

                            if (queryMetadata)
                            {
                                bool writeToSharepoint = true;
                                LibraryAPIs.queryMetadataAPIs(spURL, "Digitisation Workflow", SourceFolderStatus, writeToSharepoint);
                            }
                            // ======================================================================

                            // Deprecated?
                            // var sourceFolderXMLFiles = SharepointTools.GetSourceFolderXMLs(SourceFolderStatus, true);
                            //Assert.IsNotNull(sourceFolderXMLFiles);
                        }
                    }
                    catch (Exception ex)
                    {
                        Console.WriteLine("Error while processing query.\n Exception: {0}", ex);
                    }

                    TimeSpan ts      = DateTime.Now - startTime;
                    var      runTime = ts.TotalSeconds.ToString();
                    Console.WriteLine("=======================================\nFinished in {0} seconds\n=======================================", runTime);
                    Console.WriteLine("Perform another query? (yes/no)");
                    restartProgram = Console.ReadLine();
                }
                else // else you will use an ingest spreadsheet rather than sharepoint
                {
                    // This will get the source folder status automatically (or should do...)

                    try
                    {
                        Console.WriteLine("Searching for an ingest spreadsheet file with name [IngestSpreadsheet.xlsx] and sheet name [PSIP Generator Fields].");
                        var ingestSpreadsheetPath = Directory.GetCurrentDirectory() + "\\IngestSpreadsheet.xlsx";
                        if (!File.Exists(ingestSpreadsheetPath))
                        {
                            Console.WriteLine("Error: Expected to find an ingest spreadsheet in the current directory with name 'IngestSpreadsheet.xlsx'");
                            continue; // this should kick you back to the start of the while loop
                        }

                        //===================== Checks to run ====================

                        bool runImageOrderGenerationXMLChecks = false;
                        bool queryMetadata = false;

                        Console.WriteLine("Generate image order CSV and perform ALTO XML checks? yes/no");
                        String inputImageOrderGen = Console.ReadLine();
                        if (inputImageOrderGen.ToLower().Equals("yes") || inputImageOrderGen.ToLower().Equals("y"))
                        {
                            runImageOrderGenerationXMLChecks = true;
                        }
                        Console.WriteLine("Run query against descriptive metadata APIs? yes/no");
                        String inputMDAPIQuery = Console.ReadLine();
                        if (inputMDAPIQuery.ToLower().Equals("yes") || inputMDAPIQuery.ToLower().Equals("y"))
                        {
                            queryMetadata = true;
                        }
                        // =======================================================


                        // find an alternative to linq-to-excel here...

                        // Need to create a "HMDSPObject" to pass to the relevant bits of the tool and bypass sharepoint...
                        // Turn off writing to sharepoint with a bool or something...
                        List <HMDSPObject> ingestSpreadsheetItems = new List <HMDSPObject>();


                        using (var stream = File.Open(ingestSpreadsheetPath, FileMode.Open, FileAccess.Read))
                        {
                            // Auto-detect format, supports:
                            //  - Binary Excel files (2.0-2003 format; *.xls)
                            //  - OpenXml Excel files (2007 format; *.xlsx, *.xlsb)
                            using (var reader = ExcelReaderFactory.CreateReader(stream))
                            {
                                var result = reader.AsDataSet(new ExcelDataSetConfiguration()
                                {
                                    ConfigureDataTable = (_) => new ExcelDataTableConfiguration()
                                    {
                                        UseHeaderRow = true
                                    }
                                });

                                var table       = result.Tables;
                                var resultTable = table["PSIP Generator Fields"];


                                int dummySharepointID = -9999;
                                foreach (System.Data.DataRow dr in resultTable.Rows)
                                {
                                    HMDSPObject item = new HMDSPObject();
                                    item.Title          = dr["Shelfmark"].ToString();
                                    item.ID             = dummySharepointID.ToString(); // won't have a sharepoint ID
                                    item.Location       = dr["Source Folder"].ToString();
                                    item.MetadataSource = dr["Descriptive Metadata Source"].ToString();
                                    item.SystemNumber   = dr["System number"].ToString();
                                    dummySharepointID  += 1;

                                    ingestSpreadsheetItems.Add(item);
                                }
                            }
                        }


                        var SourceFolderStatus = SharepointTools.CheckSourceFolderExists(ingestSpreadsheetItems);
                        // generates a list of HMDObjects... then the software treats this exactly like it had come from sharepoint
                        if (SourceFolderStatus.Count < 1)
                        {
                            Console.WriteLine("Error: No items found in ingest spreadsheet.\nMake sure the spreadsheet contains the following columns:");
                            Console.WriteLine("[Shelfmark] [Source Folder] [Descriptive Metadata Source] [System number]");
                            Console.WriteLine("Perform another query? (yes/no)");
                            restartProgram = Console.ReadLine();
                        }
                        else
                        {
                            // proceed
                            var  env = "prod";           // dictates behaviour in the imageorder csv - find to leave as prod here
                            bool writeToSharepoint = false;
                            var  spURL             = ""; // make sure we're not using sharepoint at all here
                            if (runImageOrderGenerationXMLChecks)
                            {
                                Console.WriteLine("=======================================\nQuerying metadata APIs to retrieve child shelfmark information...\n=======================================");
                                var iamsRecords = LibraryAPIs.queryMetadataAPIs(spURL, "Digitisation Workflow", SourceFolderStatus, writeToSharepoint);

                                var allShelfmarkFiles = InputOrderSpreadsheetTools.listAllShelfmarkFilesTIFXML(SourceFolderStatus, env, spURL, "Digitisation Workflow", iamsRecords, writeToSharepoint);
                                Assert.IsNotNull(allShelfmarkFiles);
                            }
                            if (queryMetadata)
                            {
                                LibraryAPIs.queryMetadataAPIs(spURL, "Digitisation Workflow", SourceFolderStatus, writeToSharepoint);
                            }
                        }
                    }
                    catch (Exception ex)
                    {
                        Console.WriteLine("Error while processing ingest spreadsheet.\nException: {0}", ex);
                    }
                    TimeSpan ts      = DateTime.Now - startTime;
                    var      runTime = ts.TotalSeconds.ToString();
                    Console.WriteLine("=======================================\nFinished in {0} seconds\n=======================================", runTime);
                    Console.WriteLine("Perform another query? (yes/no)");
                    restartProgram = Console.ReadLine();
                }


                if (restartProgram.ToLower().Equals("no") || restartProgram.ToLower().Equals("n"))
                {
                    Console.WriteLine("Press any key to exit.");
                    Console.ReadKey();
                    runProgram = false;
                }
                Console.Clear();
            }


            return;
        }