public void splitToAuthorSmall() { //using (var dbArtVeryFull = new ArticleVeryFullMetadataContext()) using (var dbArtVeryFull = new ArticleVeryFullMetadata_02Context()) { //using (var tblAuthSmall = new AuthorSmallMetadataContext()) using (var tblAuthSmall = new AuthorSmallMetadata_02Context()) { char[] delimeters = { '$' }; var query = from vFullArticle in dbArtVeryFull.T_ArticleVeryFullMeDa_02 select vFullArticle; foreach (var names in query) { foreach (var name in names.Creator.Split(delimeters)) { //var authTupl = new T_AuthorSmallMeDa(); var authTupl = new T_AuthorSmallMeDa_02(); authTupl.Title = names.Title; authTupl.PubDate = names.PubDate; authTupl.Author = name; authTupl.Link = names.Relation; //tblAuthSmall.T_AuthorSmallMeDa.Add(authTupl); tblAuthSmall.T_AuthorSmallMeDa_02.Add(authTupl); try { tblAuthSmall.SaveChanges(); } catch (DbUpdateException dupe) { tblAuthSmall.T_AuthorSmallMeDa_02.Remove(authTupl); Console.WriteLine(dupe.ToString()); } catch (DbEntityValidationException e) { tblAuthSmall.T_AuthorSmallMeDa_02.Remove(authTupl); foreach (var eve in e.EntityValidationErrors) { Console.WriteLine("Entity of type \"{0}\" in state \"{1}\" has the following validation errors:", eve.Entry.Entity.GetType().Name, eve.Entry.State); foreach (var ve in eve.ValidationErrors) { Console.WriteLine("- Property: \"{0}\", Error: \"{1}\"", ve.PropertyName, ve.ErrorMessage); } } Console.WriteLine(e.ToString()); } } } } Console.WriteLine("Press any key..."); Console.ReadKey(); } }
public void splitToSubjectSmall() { //using (var dbArtVeryFull = new ArticleVeryFullMetadataContext()) using (var dbArtVeryFull = new ArticleVeryFullMetadata_02Context()) { //using (var tblSubjSmall = new SubjectSmallMetadataContext()) using (var tblSubjSmall = new SubjectSmallMetadata_02Context()) { char[] delimeters = { '$' }; var query = from vFullArticle in dbArtVeryFull.T_ArticleVeryFullMeDa_02 select vFullArticle; foreach (var subjs in query) { foreach (var sub in subjs.Subject.Split(delimeters)) { //var subjTupl = new T_SubjectSmallMeDa(); var subjTupl = new T_SubjectSmallMeDa_02(); subjTupl.Title = subjs.Title; subjTupl.PubDate = subjs.PubDate; subjTupl.Subject = sub; subjTupl.Link = subjs.Relation; tblSubjSmall.T_SubjectSmallMeDa_02.Add(subjTupl); try { tblSubjSmall.SaveChanges(); } catch (DbUpdateException dupe) { tblSubjSmall.T_SubjectSmallMeDa_02.Remove(subjTupl); continue; } catch (DbEntityValidationException e) { tblSubjSmall.T_SubjectSmallMeDa_02.Remove(subjTupl); foreach (var eve in e.EntityValidationErrors) { Console.WriteLine("Entity of type \"{0}\" in state \"{1}\" has the following validation errors:", eve.Entry.Entity.GetType().Name, eve.Entry.State); foreach (var ve in eve.ValidationErrors) { Console.WriteLine("- Property: \"{0}\", Error: \"{1}\"", ve.PropertyName, ve.ErrorMessage); } } continue; } } } } Console.WriteLine("Press any key..."); Console.ReadKey(); } }
public void start() { oai.OAI objOAI = new oai.OAI("http://doaj.org/oai.article"); ListRecord objRecord = new ListRecord(); ListIdentifier objIdentifier = new ListIdentifier(); DateTime fromdate = new DateTime(2004, 1, 1); for (int day = 1 ; day < 4; ++day) { DateTime thisdate = new DateTime(2014, 4, day); objRecord = objOAI.ListRecords("", fromdate.ToString("yyyy-MM-dd"), thisdate.ToString("yyyy-MM-dd")); objIdentifier = objOAI.ListIdentifiers(""); ////Console.WriteLine("Record:" + objRecord.record); //To hold duplicate sql tuples not inserted System.Collections.ArrayList duprecords = new System.Collections.ArrayList(); //process through all records (100 per unique search foreach (var item in objRecord.record) { //per record, make a db context object to do an insert. using (var db = new ArticleVeryFullMetadata_02Context()) { var articlemeda = new T_ArticleVeryFullMeDa_02(); Record rec = (Record)item; OAI_DC metadatalist = (OAI_DC)rec.metadata; StringBuilder tmpbuilder = new StringBuilder(); foreach (var ttl in metadatalist.title) { tmpbuilder.Append(ttl.ToString()); tmpbuilder.Append(" "); } try { tmpbuilder.Remove(tmpbuilder.Length - 1, 1); } catch (System.ArgumentOutOfRangeException e) { tmpbuilder.Append("N/A"); } //tmpbuilder.Remove(tmpbuilder.Length - 2, 2); articlemeda.Title = tmpbuilder.ToString(); //Console.WriteLine("Title: " + tmpbuilder.ToString()); tmpbuilder.Clear(); //Console.WriteLine("Creator (Authors): "); foreach (var creator in metadatalist.creator) { if (creator.ToString().Length < 40) { tmpbuilder.Append(creator.ToString()); tmpbuilder.Append("$"); } else { tmpbuilder.Append(creator.ToString().Substring(0, 40)); tmpbuilder.Append("$"); } } try { tmpbuilder.Remove(tmpbuilder.Length - 1, 1); } catch (System.ArgumentOutOfRangeException e) { tmpbuilder.Append("N/A"); } //tmpbuilder.Remove(tmpbuilder.Length - 2, 2); articlemeda.Creator = tmpbuilder.ToString(); //Console.WriteLine("Creator: " + tmpbuilder.ToString()); tmpbuilder.Clear(); foreach (var desc in metadatalist.description) { tmpbuilder.Append(desc.ToString()); tmpbuilder.Append(" "); } try { tmpbuilder.Remove(tmpbuilder.Length - 1, 1); } catch (System.ArgumentOutOfRangeException e) { tmpbuilder.Append("N/A"); } //tmpbuilder.Remove(tmpbuilder.Length - 2, 2); articlemeda.Description = tmpbuilder.ToString(); //Console.WriteLine("Description (abstract): " + tmpbuilder.ToString()); tmpbuilder.Clear(); foreach (var pub in metadatalist.publisher) { tmpbuilder.Append(pub.ToString()); tmpbuilder.Append("$"); } try { tmpbuilder.Remove(tmpbuilder.Length - 1, 1); } catch (System.ArgumentOutOfRangeException e) { tmpbuilder.Append("N/A"); } articlemeda.Publisher = tmpbuilder.ToString(); //Console.WriteLine("Publisher: " + tmpbuilder.ToString()); tmpbuilder.Clear(); //Console.WriteLine("Identifiers (Id numbers):"); //If inserting to T_ArticleVeryFullMeDa int count = 0; foreach (var ids in metadatalist.identifier) { tmpbuilder.Append(ids.ToString()); if (count == 0) articlemeda.aSSN = tmpbuilder.ToString(); else if (count == 1) articlemeda.bSSN = tmpbuilder.ToString(); else if (count == 2) articlemeda.ExtraId1 = tmpbuilder.ToString(); else if (count == 3) articlemeda.ExtraId2 = tmpbuilder.ToString(); //Console.WriteLine("Identifiers: " + tmpbuilder.ToString()); tmpbuilder.Clear(); ++count; } ////Console.WriteLine("Identifiers: " + tmpbuilder.ToString()); tmpbuilder.Clear(); //If inserting to T_ArticleFullMeDa /* foreach (var ids in metadatalist.identifier) { tmpbuilder.Append(ids.ToString()); tmpbuilder.Append(", "); } tmpbuilder.Remove(tmpbuilder.Length - 2, 2); articlemeda.Identifier = tmpbuilder.ToString(); //Console.WriteLine("Identifiers: " + tmpbuilder.ToString()); tmpbuilder.Clear(); */ foreach (var lang in metadatalist.language) { tmpbuilder.Append(lang.ToString()); tmpbuilder.Append("$"); } try { tmpbuilder.Remove(tmpbuilder.Length - 1, 1); } catch (System.ArgumentOutOfRangeException e) { tmpbuilder.Append("N/A"); } //tmpbuilder.Remove(tmpbuilder.Length - 2, 2); articlemeda.PubLanguage = tmpbuilder.ToString(); //Console.WriteLine("Language: " + tmpbuilder.ToString()); tmpbuilder.Clear(); foreach (var rel in metadatalist.relation) { tmpbuilder.Append(rel.ToString()); tmpbuilder.Append(", "); } try { tmpbuilder.Remove(tmpbuilder.Length - 2, 2); } catch (System.ArgumentOutOfRangeException e) { tmpbuilder.Append("N/A"); } //tmpbuilder.Remove(tmpbuilder.Length - 2, 2); articlemeda.Relation = tmpbuilder.ToString(); //Console.WriteLine("Relation (Source): " + tmpbuilder.ToString()); tmpbuilder.Clear(); foreach (var sub in metadatalist.subject) { if (sub.ToString().Length < 40) { tmpbuilder.Append(sub.ToString()); tmpbuilder.Append("$"); } else { tmpbuilder.Append(sub.ToString().Substring(0, 40)); tmpbuilder.Append("$"); } } try { tmpbuilder.Remove(tmpbuilder.Length - 1, 1); } catch (System.ArgumentOutOfRangeException e) { tmpbuilder.Append("N/A"); } //tmpbuilder.Remove(tmpbuilder.Length - 2, 2); articlemeda.Subject = tmpbuilder.ToString(); //Console.WriteLine("Subjects:" + tmpbuilder.ToString()); tmpbuilder.Clear(); foreach (var typ in metadatalist.type) { tmpbuilder.Append(typ.ToString()); tmpbuilder.Append("$"); } try { tmpbuilder.Remove(tmpbuilder.Length - 1, 1); } catch (System.ArgumentOutOfRangeException e) { tmpbuilder.Append("N/A"); } //tmpbuilder.Remove(tmpbuilder.Length - 2, 2); articlemeda.Type = tmpbuilder.ToString(); //Console.WriteLine("Type: " + tmpbuilder.ToString()); tmpbuilder.Clear(); foreach (var dt in metadatalist.date) { tmpbuilder.Append(dt.ToString()); tmpbuilder.Append(", "); //Console.WriteLine("PubDate: " + dt.ToString()); } try { tmpbuilder.Remove(tmpbuilder.Length - 2, 2); } catch (System.ArgumentOutOfRangeException e) { tmpbuilder.Append("N/A"); } //tmpbuilder.Remove(tmpbuilder.Length - 2, 2);' System.DateTime tmpdate = new System.DateTime(); try { tmpdate = System.DateTime.ParseExact(tmpbuilder.ToString(), "yyyy-MM-ddTHH:mm:ssZ", null); articlemeda.PubDate = tmpdate; } catch (System.FormatException e) { tmpbuilder.Append("N/A"); } //Console.WriteLine("PubDate: " + tmpbuilder.ToString()); tmpbuilder.Clear(); //2013-12-01T00:00:00Z articlemeda.EntryDate = DateTime.Today; db.T_ArticleVeryFullMeDa_02.Add(articlemeda); try { if (db.SaveChanges() > 0) { Console.WriteLine("Saved to DB"); } } catch (DbEntityValidationException e) { foreach (var eve in e.EntityValidationErrors) { Console.WriteLine("Entity of type \"{0}\" in state \"{1}\" has the following validation errors:", eve.Entry.Entity.GetType().Name, eve.Entry.State); foreach (var ve in eve.ValidationErrors) { Console.WriteLine("- Property: \"{0}\", Error: \"{1}\"", ve.PropertyName, ve.ErrorMessage); } } continue; } catch (DbUpdateException dbupe) { //UpdateException upe = new UpdateException(); var innexc = dbupe.InnerException; if ((innexc.GetType()) == typeof(UpdateException)) { var sqlinnexc = innexc.InnerException; if ((sqlinnexc.GetType()) == typeof(SqlException)) { SqlException sex = (SqlException)sqlinnexc; switch (sex.Number) { case 242: duprecords.Add(articlemeda); break; case 2601: duprecords.Add(articlemeda); break; case 2627: duprecords.Add(articlemeda); break; default: throw; } } } } //db.SaveChanges(); } } } //Console.ReadKey(); }