public void start()
        {
            oai.OAI objOAI = new oai.OAI("http://doaj.org/oai.article");

            ListRecord objRecord = new ListRecord();
            ListIdentifier objIdentifier = new ListIdentifier();

            DateTime fromdate = new DateTime(2004, 1, 1);
            for (int day = 1 ; day < 4; ++day)
            {
                DateTime thisdate = new DateTime(2014, 4, day);

                objRecord = objOAI.ListRecords("", fromdate.ToString("yyyy-MM-dd"), thisdate.ToString("yyyy-MM-dd"));
                objIdentifier = objOAI.ListIdentifiers("");
                ////Console.WriteLine("Record:" + objRecord.record);

                //To hold duplicate sql tuples not inserted
                System.Collections.ArrayList duprecords = new System.Collections.ArrayList();

                //process through all records (100 per unique search
                foreach (var item in objRecord.record)
                {
                    //per record, make a db context object to do an insert.
                    using (var db = new ArticleVeryFullMetadata_02Context())
                    {
                        var articlemeda = new T_ArticleVeryFullMeDa_02();

                        Record rec = (Record)item;
                        OAI_DC metadatalist = (OAI_DC)rec.metadata;
                        StringBuilder tmpbuilder = new StringBuilder();
                        foreach (var ttl in metadatalist.title)
                        {
                            tmpbuilder.Append(ttl.ToString());
                            tmpbuilder.Append(" ");

                        }
                        try
                        {
                            tmpbuilder.Remove(tmpbuilder.Length - 1, 1);
                        }
                        catch (System.ArgumentOutOfRangeException e)
                        {
                            tmpbuilder.Append("N/A");
                        }
                        //tmpbuilder.Remove(tmpbuilder.Length - 2, 2);
                        articlemeda.Title = tmpbuilder.ToString();
                        //Console.WriteLine("Title: " + tmpbuilder.ToString());
                        tmpbuilder.Clear();
                        //Console.WriteLine("Creator (Authors): ");
                        foreach (var creator in metadatalist.creator)
                        {
                            if (creator.ToString().Length < 40)
                            {
                                tmpbuilder.Append(creator.ToString());
                                tmpbuilder.Append("$");
                            }
                            else
                            {
                                tmpbuilder.Append(creator.ToString().Substring(0, 40));
                                tmpbuilder.Append("$");
                            }
                        }
                        try
                        {
                            tmpbuilder.Remove(tmpbuilder.Length - 1, 1);
                        }
                        catch (System.ArgumentOutOfRangeException e)
                        {
                            tmpbuilder.Append("N/A");
                        }
                        //tmpbuilder.Remove(tmpbuilder.Length - 2, 2);
                        articlemeda.Creator = tmpbuilder.ToString();
                        //Console.WriteLine("Creator: " + tmpbuilder.ToString());
                        tmpbuilder.Clear();

                        foreach (var desc in metadatalist.description)
                        {
                            tmpbuilder.Append(desc.ToString());
                            tmpbuilder.Append(" ");
                        }
                        try
                        {
                            tmpbuilder.Remove(tmpbuilder.Length - 1, 1);
                        }
                        catch (System.ArgumentOutOfRangeException e)
                        {
                            tmpbuilder.Append("N/A");
                        }
                        //tmpbuilder.Remove(tmpbuilder.Length - 2, 2);
                        articlemeda.Description = tmpbuilder.ToString();
                        //Console.WriteLine("Description (abstract): " + tmpbuilder.ToString());
                        tmpbuilder.Clear();

                        foreach (var pub in metadatalist.publisher)
                        {
                            tmpbuilder.Append(pub.ToString());
                            tmpbuilder.Append("$");
                        }
                        try
                        {
                            tmpbuilder.Remove(tmpbuilder.Length - 1, 1);
                        }
                        catch (System.ArgumentOutOfRangeException e)
                        {
                            tmpbuilder.Append("N/A");
                        }
                        articlemeda.Publisher = tmpbuilder.ToString();
                        //Console.WriteLine("Publisher: " + tmpbuilder.ToString());
                        tmpbuilder.Clear();

                        //Console.WriteLine("Identifiers (Id numbers):");
                        //If inserting to T_ArticleVeryFullMeDa
                        int count = 0;
                        foreach (var ids in metadatalist.identifier)
                        {
                            tmpbuilder.Append(ids.ToString());
                            if (count == 0)
                                articlemeda.aSSN = tmpbuilder.ToString();
                            else if (count == 1)
                                articlemeda.bSSN = tmpbuilder.ToString();
                            else if (count == 2)
                                articlemeda.ExtraId1 = tmpbuilder.ToString();
                            else if (count == 3)
                                articlemeda.ExtraId2 = tmpbuilder.ToString();
                            //Console.WriteLine("Identifiers: " + tmpbuilder.ToString());
                            tmpbuilder.Clear();
                            ++count;
                        }
                        ////Console.WriteLine("Identifiers: " + tmpbuilder.ToString());
                        tmpbuilder.Clear();
                        //If inserting to T_ArticleFullMeDa
                        /*
                        foreach (var ids in metadatalist.identifier)
                        {
                            tmpbuilder.Append(ids.ToString());
                            tmpbuilder.Append(", ");
                        }
                        tmpbuilder.Remove(tmpbuilder.Length - 2, 2);
                        articlemeda.Identifier = tmpbuilder.ToString();
                        //Console.WriteLine("Identifiers: " + tmpbuilder.ToString());
                        tmpbuilder.Clear();
                        */
                        foreach (var lang in metadatalist.language)
                        {
                            tmpbuilder.Append(lang.ToString());
                            tmpbuilder.Append("$");
                        }
                        try
                        {
                            tmpbuilder.Remove(tmpbuilder.Length - 1, 1);
                        }
                        catch (System.ArgumentOutOfRangeException e)
                        {
                            tmpbuilder.Append("N/A");
                        }
                        //tmpbuilder.Remove(tmpbuilder.Length - 2, 2);
                        articlemeda.PubLanguage = tmpbuilder.ToString();
                        //Console.WriteLine("Language: " + tmpbuilder.ToString());
                        tmpbuilder.Clear();

                        foreach (var rel in metadatalist.relation)
                        {
                            tmpbuilder.Append(rel.ToString());
                            tmpbuilder.Append(", ");
                        }
                        try
                        {
                            tmpbuilder.Remove(tmpbuilder.Length - 2, 2);
                        }
                        catch (System.ArgumentOutOfRangeException e)
                        {
                            tmpbuilder.Append("N/A");
                        }
                        //tmpbuilder.Remove(tmpbuilder.Length - 2, 2);
                        articlemeda.Relation = tmpbuilder.ToString();
                        //Console.WriteLine("Relation (Source): " + tmpbuilder.ToString());
                        tmpbuilder.Clear();

                        foreach (var sub in metadatalist.subject)
                        {
                            if (sub.ToString().Length < 40)
                            {
                                tmpbuilder.Append(sub.ToString());
                                tmpbuilder.Append("$");
                            }
                            else
                            {
                                tmpbuilder.Append(sub.ToString().Substring(0, 40));
                                tmpbuilder.Append("$");
                            }
                        }
                        try
                        {
                            tmpbuilder.Remove(tmpbuilder.Length - 1, 1);
                        }
                        catch (System.ArgumentOutOfRangeException e)
                        {
                            tmpbuilder.Append("N/A");
                        }
                        //tmpbuilder.Remove(tmpbuilder.Length - 2, 2);
                        articlemeda.Subject = tmpbuilder.ToString();
                        //Console.WriteLine("Subjects:" + tmpbuilder.ToString());
                        tmpbuilder.Clear();

                        foreach (var typ in metadatalist.type)
                        {
                            tmpbuilder.Append(typ.ToString());
                            tmpbuilder.Append("$");
                        }
                        try
                        {
                            tmpbuilder.Remove(tmpbuilder.Length - 1, 1);
                        }
                        catch (System.ArgumentOutOfRangeException e)
                        {
                            tmpbuilder.Append("N/A");
                        }
                        //tmpbuilder.Remove(tmpbuilder.Length - 2, 2);
                        articlemeda.Type = tmpbuilder.ToString();
                        //Console.WriteLine("Type: " + tmpbuilder.ToString());
                        tmpbuilder.Clear();

                        foreach (var dt in metadatalist.date)
                        {
                            tmpbuilder.Append(dt.ToString());
                            tmpbuilder.Append(", ");
                            //Console.WriteLine("PubDate: " + dt.ToString());
                        }
                        try
                        {
                            tmpbuilder.Remove(tmpbuilder.Length - 2, 2);
                        }
                        catch (System.ArgumentOutOfRangeException e)
                        {
                            tmpbuilder.Append("N/A");
                        }
                        //tmpbuilder.Remove(tmpbuilder.Length - 2, 2);'
                        System.DateTime tmpdate = new System.DateTime();

                        try
                        {

                            tmpdate = System.DateTime.ParseExact(tmpbuilder.ToString(), "yyyy-MM-ddTHH:mm:ssZ", null);
                            articlemeda.PubDate = tmpdate;
                        }
                        catch (System.FormatException e)
                        {
                            tmpbuilder.Append("N/A");
                        }

                        //Console.WriteLine("PubDate: " + tmpbuilder.ToString());
                        tmpbuilder.Clear();
                        //2013-12-01T00:00:00Z

                        articlemeda.EntryDate = DateTime.Today;

                        db.T_ArticleVeryFullMeDa_02.Add(articlemeda);

                        try
                        {
                            if (db.SaveChanges() > 0)
                            {
                                Console.WriteLine("Saved to DB");
                            }
                        }
                        catch (DbEntityValidationException e)
                        {
                            foreach (var eve in e.EntityValidationErrors)
                            {
                                Console.WriteLine("Entity of type \"{0}\" in state \"{1}\" has the following validation errors:",
                                    eve.Entry.Entity.GetType().Name, eve.Entry.State);
                                foreach (var ve in eve.ValidationErrors)
                                {
                                    Console.WriteLine("- Property: \"{0}\", Error: \"{1}\"",
                                        ve.PropertyName, ve.ErrorMessage);

                                }
                            }
                            continue;
                        }
                        catch (DbUpdateException dbupe)
                        {
                            //UpdateException upe = new UpdateException();
                            var innexc = dbupe.InnerException;
                            if ((innexc.GetType()) == typeof(UpdateException))
                            {
                                var sqlinnexc = innexc.InnerException;
                                if ((sqlinnexc.GetType()) == typeof(SqlException))
                                {
                                    SqlException sex = (SqlException)sqlinnexc;

                                    switch (sex.Number)
                                    {
                                        case 242:
                                            duprecords.Add(articlemeda);
                                            break;
                                        case 2601:
                                            duprecords.Add(articlemeda);
                                            break;
                                        case 2627:
                                            duprecords.Add(articlemeda);
                                            break;
                                        default:
                                            throw;

                                    }
                                }

                            }
                        }
                        //db.SaveChanges();
                    }

                }

            }
            //Console.ReadKey();
        }
Exemple #2
0
        public ListIdentifier ListIdentifiers(string sPrefix,
            string sset,
            string sfrom,
            string suntil,
            ResumptionToken objToken)
        {
            System.IO.Stream objStream;
            ListIdentifier objList = new ListIdentifier();
            Identifiers objRecord;
            ResumptionToken myToken;
            string tmp = "";
            System.Net.HttpWebRequest wr;
            System.Xml.XmlTextReader rd;

            if (sPrefix.Length == 0)
            {
                sPrefix = "oai_dc";
            }

            if (objToken == null)
            {
                if (sset.Length != 0)
                {
                    sset = "&set=" + sset;
                }
                if (sfrom.Length != 0)
                {
                    sfrom = "&from=" + sfrom;
                }
                if (suntil.Length != 0)
                {
                    suntil = "&until=" + suntil;
                }

                prequestURL = baseURL + "?verb=ListIdentifiers&metadataPrefix=" + sPrefix + sset + sfrom + suntil;

            }
            else
            {
                prequestURL = baseURL + "?verb=ListIdentifiers&resumptionToken=" + objToken.resumptionToken;
                //This is where we handle the resumptionToken
            }
            //======================================================
            // If you wanted to support additional metadata formats,
            // you would just need to have additional handlers.
            //======================================================
            //Console.Write(sURL);
            try
            {
                wr = (System.Net.HttpWebRequest)System.Net.HttpWebRequest.Create(prequestURL);
                wr.UserAgent = cUserAgent;
                System.Net.WebResponse response = wr.GetResponse();
                objStream = response.GetResponseStream();
                System.IO.StreamReader reader = new System.IO.StreamReader(objStream);
                prawXML = reader.ReadToEnd();
                reader.Close();
                rd = new System.Xml.XmlTextReader(prawXML, System.Xml.XmlNodeType.Document, null);
            }
            catch (Exception e)
            {
                error.errorName = "badConnection";
                error.errorDescription = e.Message + "<br>Unable to connect to " + baseURL;
                return null;
            }

            while (rd.Read())
            {
                if (rd.NodeType == System.Xml.XmlNodeType.Element)
                {
                    if (rd.Name == "responseDate")
                    {
                        presponseDate = rd.ReadString();
                    }
                    else if (rd.Name == "ListIdentifiers")
                    {
                        do
                        {
                            if (rd.Name == "header")
                            {
                                tmp = rd.ReadOuterXml();
                                //tmp += ParseOAIContainer(ref rd, "header", true);
                                //Console.WriteLine("In the Function: " + tmp);
                                objRecord = new Identifiers(tmp);
                                objList.record.Add(objRecord);
                                //return objRecord;
                            }
                            else if (rd.Name == "resumptionToken")
                            {
                                tmp = rd.ReadOuterXml();
                                myToken = new ResumptionToken(tmp);
                                objList.token = myToken;
                            }
                            else rd.Read(); // Added the Read() and will never occur with the ReadInnerXml()

                        } while (rd.Name != "ListIdentifiers"); // loop
                    }
                    else if (rd.Name == "error")
                    {
                        error.errorName = rd.GetAttribute("code");
                        error.errorDescription = rd.ReadString();
                        rd.Close();
                        return null;
                    }
                }
            }

            rd.Close();
            return objList;
        }