public void splitToAuthorSmall()
 {
     //using (var dbArtVeryFull = new ArticleVeryFullMetadataContext())
     using (var dbArtVeryFull = new ArticleVeryFullMetadata_02Context())
     {
         //using (var tblAuthSmall = new AuthorSmallMetadataContext())
         using (var tblAuthSmall = new AuthorSmallMetadata_02Context())
         {
             char[] delimeters = { '$' };
             var query = from vFullArticle in dbArtVeryFull.T_ArticleVeryFullMeDa_02
                         select vFullArticle;
             foreach (var names in query)
             {
                 foreach (var name in names.Creator.Split(delimeters))
                 {
                     //var authTupl = new T_AuthorSmallMeDa();
                     var authTupl = new T_AuthorSmallMeDa_02();
                     authTupl.Title = names.Title;
                     authTupl.PubDate = names.PubDate;
                     authTupl.Author = name;
                     authTupl.Link = names.Relation;
                     //tblAuthSmall.T_AuthorSmallMeDa.Add(authTupl);
                     tblAuthSmall.T_AuthorSmallMeDa_02.Add(authTupl);
                     try
                     {
                         tblAuthSmall.SaveChanges();
                     }
                     catch (DbUpdateException dupe)
                     {
                         tblAuthSmall.T_AuthorSmallMeDa_02.Remove(authTupl);
                         Console.WriteLine(dupe.ToString());
                     }
                     catch (DbEntityValidationException e)
                     {
                         tblAuthSmall.T_AuthorSmallMeDa_02.Remove(authTupl);
                         foreach (var eve in e.EntityValidationErrors)
                         {
                             Console.WriteLine("Entity of type \"{0}\" in state \"{1}\" has the following validation errors:",
                                 eve.Entry.Entity.GetType().Name, eve.Entry.State);
                             foreach (var ve in eve.ValidationErrors)
                             {
                                 Console.WriteLine("- Property: \"{0}\", Error: \"{1}\"",
                                     ve.PropertyName, ve.ErrorMessage);
                             }
                         }
                         Console.WriteLine(e.ToString());
                     }
                 }
             }
         }
         Console.WriteLine("Press any key...");
         Console.ReadKey();
     }
 }
 public void splitToSubjectSmall()
 {
     //using (var dbArtVeryFull = new ArticleVeryFullMetadataContext())
     using (var dbArtVeryFull = new ArticleVeryFullMetadata_02Context())
     {
         //using (var tblSubjSmall = new SubjectSmallMetadataContext())
         using (var tblSubjSmall = new SubjectSmallMetadata_02Context())
         {
             char[] delimeters = { '$' };
             var query = from vFullArticle in dbArtVeryFull.T_ArticleVeryFullMeDa_02
                         select vFullArticle;
             foreach (var subjs in query)
             {
                 foreach (var sub in subjs.Subject.Split(delimeters))
                 {
                     //var subjTupl = new T_SubjectSmallMeDa();
                     var subjTupl = new T_SubjectSmallMeDa_02();
                     subjTupl.Title = subjs.Title;
                     subjTupl.PubDate = subjs.PubDate;
                     subjTupl.Subject = sub;
                     subjTupl.Link = subjs.Relation;
                     tblSubjSmall.T_SubjectSmallMeDa_02.Add(subjTupl);
                     try
                     {
                         tblSubjSmall.SaveChanges();
                     }
                     catch (DbUpdateException dupe)
                     {
                         tblSubjSmall.T_SubjectSmallMeDa_02.Remove(subjTupl);
                         continue;
                     }
                     catch (DbEntityValidationException e)
                     {
                         tblSubjSmall.T_SubjectSmallMeDa_02.Remove(subjTupl);
                         foreach (var eve in e.EntityValidationErrors)
                         {
                             Console.WriteLine("Entity of type \"{0}\" in state \"{1}\" has the following validation errors:",
                                 eve.Entry.Entity.GetType().Name, eve.Entry.State);
                             foreach (var ve in eve.ValidationErrors)
                             {
                                 Console.WriteLine("- Property: \"{0}\", Error: \"{1}\"",
                                     ve.PropertyName, ve.ErrorMessage);
                             }
                         }
                         continue;
                     }
                 }
             }
         }
         Console.WriteLine("Press any key...");
         Console.ReadKey();
     }
 }
Пример #3
0
        public void start()
        {
            oai.OAI objOAI = new oai.OAI("http://doaj.org/oai.article");

            ListRecord objRecord = new ListRecord();
            ListIdentifier objIdentifier = new ListIdentifier();

            DateTime fromdate = new DateTime(2004, 1, 1);
            for (int day = 1 ; day < 4; ++day)
            {
                DateTime thisdate = new DateTime(2014, 4, day);

                objRecord = objOAI.ListRecords("", fromdate.ToString("yyyy-MM-dd"), thisdate.ToString("yyyy-MM-dd"));
                objIdentifier = objOAI.ListIdentifiers("");
                ////Console.WriteLine("Record:" + objRecord.record);

                //To hold duplicate sql tuples not inserted
                System.Collections.ArrayList duprecords = new System.Collections.ArrayList();

                //process through all records (100 per unique search
                foreach (var item in objRecord.record)
                {
                    //per record, make a db context object to do an insert.
                    using (var db = new ArticleVeryFullMetadata_02Context())
                    {
                        var articlemeda = new T_ArticleVeryFullMeDa_02();

                        Record rec = (Record)item;
                        OAI_DC metadatalist = (OAI_DC)rec.metadata;
                        StringBuilder tmpbuilder = new StringBuilder();
                        foreach (var ttl in metadatalist.title)
                        {
                            tmpbuilder.Append(ttl.ToString());
                            tmpbuilder.Append(" ");

                        }
                        try
                        {
                            tmpbuilder.Remove(tmpbuilder.Length - 1, 1);
                        }
                        catch (System.ArgumentOutOfRangeException e)
                        {
                            tmpbuilder.Append("N/A");
                        }
                        //tmpbuilder.Remove(tmpbuilder.Length - 2, 2);
                        articlemeda.Title = tmpbuilder.ToString();
                        //Console.WriteLine("Title: " + tmpbuilder.ToString());
                        tmpbuilder.Clear();
                        //Console.WriteLine("Creator (Authors): ");
                        foreach (var creator in metadatalist.creator)
                        {
                            if (creator.ToString().Length < 40)
                            {
                                tmpbuilder.Append(creator.ToString());
                                tmpbuilder.Append("$");
                            }
                            else
                            {
                                tmpbuilder.Append(creator.ToString().Substring(0, 40));
                                tmpbuilder.Append("$");
                            }
                        }
                        try
                        {
                            tmpbuilder.Remove(tmpbuilder.Length - 1, 1);
                        }
                        catch (System.ArgumentOutOfRangeException e)
                        {
                            tmpbuilder.Append("N/A");
                        }
                        //tmpbuilder.Remove(tmpbuilder.Length - 2, 2);
                        articlemeda.Creator = tmpbuilder.ToString();
                        //Console.WriteLine("Creator: " + tmpbuilder.ToString());
                        tmpbuilder.Clear();

                        foreach (var desc in metadatalist.description)
                        {
                            tmpbuilder.Append(desc.ToString());
                            tmpbuilder.Append(" ");
                        }
                        try
                        {
                            tmpbuilder.Remove(tmpbuilder.Length - 1, 1);
                        }
                        catch (System.ArgumentOutOfRangeException e)
                        {
                            tmpbuilder.Append("N/A");
                        }
                        //tmpbuilder.Remove(tmpbuilder.Length - 2, 2);
                        articlemeda.Description = tmpbuilder.ToString();
                        //Console.WriteLine("Description (abstract): " + tmpbuilder.ToString());
                        tmpbuilder.Clear();

                        foreach (var pub in metadatalist.publisher)
                        {
                            tmpbuilder.Append(pub.ToString());
                            tmpbuilder.Append("$");
                        }
                        try
                        {
                            tmpbuilder.Remove(tmpbuilder.Length - 1, 1);
                        }
                        catch (System.ArgumentOutOfRangeException e)
                        {
                            tmpbuilder.Append("N/A");
                        }
                        articlemeda.Publisher = tmpbuilder.ToString();
                        //Console.WriteLine("Publisher: " + tmpbuilder.ToString());
                        tmpbuilder.Clear();

                        //Console.WriteLine("Identifiers (Id numbers):");
                        //If inserting to T_ArticleVeryFullMeDa
                        int count = 0;
                        foreach (var ids in metadatalist.identifier)
                        {
                            tmpbuilder.Append(ids.ToString());
                            if (count == 0)
                                articlemeda.aSSN = tmpbuilder.ToString();
                            else if (count == 1)
                                articlemeda.bSSN = tmpbuilder.ToString();
                            else if (count == 2)
                                articlemeda.ExtraId1 = tmpbuilder.ToString();
                            else if (count == 3)
                                articlemeda.ExtraId2 = tmpbuilder.ToString();
                            //Console.WriteLine("Identifiers: " + tmpbuilder.ToString());
                            tmpbuilder.Clear();
                            ++count;
                        }
                        ////Console.WriteLine("Identifiers: " + tmpbuilder.ToString());
                        tmpbuilder.Clear();
                        //If inserting to T_ArticleFullMeDa
                        /*
                        foreach (var ids in metadatalist.identifier)
                        {
                            tmpbuilder.Append(ids.ToString());
                            tmpbuilder.Append(", ");
                        }
                        tmpbuilder.Remove(tmpbuilder.Length - 2, 2);
                        articlemeda.Identifier = tmpbuilder.ToString();
                        //Console.WriteLine("Identifiers: " + tmpbuilder.ToString());
                        tmpbuilder.Clear();
                        */
                        foreach (var lang in metadatalist.language)
                        {
                            tmpbuilder.Append(lang.ToString());
                            tmpbuilder.Append("$");
                        }
                        try
                        {
                            tmpbuilder.Remove(tmpbuilder.Length - 1, 1);
                        }
                        catch (System.ArgumentOutOfRangeException e)
                        {
                            tmpbuilder.Append("N/A");
                        }
                        //tmpbuilder.Remove(tmpbuilder.Length - 2, 2);
                        articlemeda.PubLanguage = tmpbuilder.ToString();
                        //Console.WriteLine("Language: " + tmpbuilder.ToString());
                        tmpbuilder.Clear();

                        foreach (var rel in metadatalist.relation)
                        {
                            tmpbuilder.Append(rel.ToString());
                            tmpbuilder.Append(", ");
                        }
                        try
                        {
                            tmpbuilder.Remove(tmpbuilder.Length - 2, 2);
                        }
                        catch (System.ArgumentOutOfRangeException e)
                        {
                            tmpbuilder.Append("N/A");
                        }
                        //tmpbuilder.Remove(tmpbuilder.Length - 2, 2);
                        articlemeda.Relation = tmpbuilder.ToString();
                        //Console.WriteLine("Relation (Source): " + tmpbuilder.ToString());
                        tmpbuilder.Clear();

                        foreach (var sub in metadatalist.subject)
                        {
                            if (sub.ToString().Length < 40)
                            {
                                tmpbuilder.Append(sub.ToString());
                                tmpbuilder.Append("$");
                            }
                            else
                            {
                                tmpbuilder.Append(sub.ToString().Substring(0, 40));
                                tmpbuilder.Append("$");
                            }
                        }
                        try
                        {
                            tmpbuilder.Remove(tmpbuilder.Length - 1, 1);
                        }
                        catch (System.ArgumentOutOfRangeException e)
                        {
                            tmpbuilder.Append("N/A");
                        }
                        //tmpbuilder.Remove(tmpbuilder.Length - 2, 2);
                        articlemeda.Subject = tmpbuilder.ToString();
                        //Console.WriteLine("Subjects:" + tmpbuilder.ToString());
                        tmpbuilder.Clear();

                        foreach (var typ in metadatalist.type)
                        {
                            tmpbuilder.Append(typ.ToString());
                            tmpbuilder.Append("$");
                        }
                        try
                        {
                            tmpbuilder.Remove(tmpbuilder.Length - 1, 1);
                        }
                        catch (System.ArgumentOutOfRangeException e)
                        {
                            tmpbuilder.Append("N/A");
                        }
                        //tmpbuilder.Remove(tmpbuilder.Length - 2, 2);
                        articlemeda.Type = tmpbuilder.ToString();
                        //Console.WriteLine("Type: " + tmpbuilder.ToString());
                        tmpbuilder.Clear();

                        foreach (var dt in metadatalist.date)
                        {
                            tmpbuilder.Append(dt.ToString());
                            tmpbuilder.Append(", ");
                            //Console.WriteLine("PubDate: " + dt.ToString());
                        }
                        try
                        {
                            tmpbuilder.Remove(tmpbuilder.Length - 2, 2);
                        }
                        catch (System.ArgumentOutOfRangeException e)
                        {
                            tmpbuilder.Append("N/A");
                        }
                        //tmpbuilder.Remove(tmpbuilder.Length - 2, 2);'
                        System.DateTime tmpdate = new System.DateTime();

                        try
                        {

                            tmpdate = System.DateTime.ParseExact(tmpbuilder.ToString(), "yyyy-MM-ddTHH:mm:ssZ", null);
                            articlemeda.PubDate = tmpdate;
                        }
                        catch (System.FormatException e)
                        {
                            tmpbuilder.Append("N/A");
                        }

                        //Console.WriteLine("PubDate: " + tmpbuilder.ToString());
                        tmpbuilder.Clear();
                        //2013-12-01T00:00:00Z

                        articlemeda.EntryDate = DateTime.Today;

                        db.T_ArticleVeryFullMeDa_02.Add(articlemeda);

                        try
                        {
                            if (db.SaveChanges() > 0)
                            {
                                Console.WriteLine("Saved to DB");
                            }
                        }
                        catch (DbEntityValidationException e)
                        {
                            foreach (var eve in e.EntityValidationErrors)
                            {
                                Console.WriteLine("Entity of type \"{0}\" in state \"{1}\" has the following validation errors:",
                                    eve.Entry.Entity.GetType().Name, eve.Entry.State);
                                foreach (var ve in eve.ValidationErrors)
                                {
                                    Console.WriteLine("- Property: \"{0}\", Error: \"{1}\"",
                                        ve.PropertyName, ve.ErrorMessage);

                                }
                            }
                            continue;
                        }
                        catch (DbUpdateException dbupe)
                        {
                            //UpdateException upe = new UpdateException();
                            var innexc = dbupe.InnerException;
                            if ((innexc.GetType()) == typeof(UpdateException))
                            {
                                var sqlinnexc = innexc.InnerException;
                                if ((sqlinnexc.GetType()) == typeof(SqlException))
                                {
                                    SqlException sex = (SqlException)sqlinnexc;

                                    switch (sex.Number)
                                    {
                                        case 242:
                                            duprecords.Add(articlemeda);
                                            break;
                                        case 2601:
                                            duprecords.Add(articlemeda);
                                            break;
                                        case 2627:
                                            duprecords.Add(articlemeda);
                                            break;
                                        default:
                                            throw;

                                    }
                                }

                            }
                        }
                        //db.SaveChanges();
                    }

                }

            }
            //Console.ReadKey();
        }