public void start()
{
oai.OAI objOAI = new oai.OAI("http://doaj.org/oai.article");
ListRecord objRecord = new ListRecord();
ListIdentifier objIdentifier = new ListIdentifier();
DateTime fromdate = new DateTime(2004, 1, 1);
for (int day = 1 ; day < 4; ++day)
{
DateTime thisdate = new DateTime(2014, 4, day);
objRecord = objOAI.ListRecords("", fromdate.ToString("yyyy-MM-dd"), thisdate.ToString("yyyy-MM-dd"));
objIdentifier = objOAI.ListIdentifiers("");
////Console.WriteLine("Record:" + objRecord.record);
//To hold duplicate sql tuples not inserted
System.Collections.ArrayList duprecords = new System.Collections.ArrayList();
//process through all records (100 per unique search
foreach (var item in objRecord.record)
{
//per record, make a db context object to do an insert.
using (var db = new ArticleVeryFullMetadata_02Context())
{
var articlemeda = new T_ArticleVeryFullMeDa_02();
Record rec = (Record)item;
OAI_DC metadatalist = (OAI_DC)rec.metadata;
StringBuilder tmpbuilder = new StringBuilder();
foreach (var ttl in metadatalist.title)
{
tmpbuilder.Append(ttl.ToString());
tmpbuilder.Append(" ");
}
try
{
tmpbuilder.Remove(tmpbuilder.Length - 1, 1);
}
catch (System.ArgumentOutOfRangeException e)
{
tmpbuilder.Append("N/A");
}
//tmpbuilder.Remove(tmpbuilder.Length - 2, 2);
articlemeda.Title = tmpbuilder.ToString();
//Console.WriteLine("Title: " + tmpbuilder.ToString());
tmpbuilder.Clear();
//Console.WriteLine("Creator (Authors): ");
foreach (var creator in metadatalist.creator)
{
if (creator.ToString().Length < 40)
{
tmpbuilder.Append(creator.ToString());
tmpbuilder.Append("$");
}
else
{
tmpbuilder.Append(creator.ToString().Substring(0, 40));
tmpbuilder.Append("$");
}
}
try
{
tmpbuilder.Remove(tmpbuilder.Length - 1, 1);
}
catch (System.ArgumentOutOfRangeException e)
{
tmpbuilder.Append("N/A");
}
//tmpbuilder.Remove(tmpbuilder.Length - 2, 2);
articlemeda.Creator = tmpbuilder.ToString();
//Console.WriteLine("Creator: " + tmpbuilder.ToString());
tmpbuilder.Clear();
foreach (var desc in metadatalist.description)
{
tmpbuilder.Append(desc.ToString());
tmpbuilder.Append(" ");
}
try
{
tmpbuilder.Remove(tmpbuilder.Length - 1, 1);
}
catch (System.ArgumentOutOfRangeException e)
{
tmpbuilder.Append("N/A");
}
//tmpbuilder.Remove(tmpbuilder.Length - 2, 2);
articlemeda.Description = tmpbuilder.ToString();
//Console.WriteLine("Description (abstract): " + tmpbuilder.ToString());
tmpbuilder.Clear();
foreach (var pub in metadatalist.publisher)
{
tmpbuilder.Append(pub.ToString());
tmpbuilder.Append("$");
}
try
{
tmpbuilder.Remove(tmpbuilder.Length - 1, 1);
}
catch (System.ArgumentOutOfRangeException e)
{
tmpbuilder.Append("N/A");
}
articlemeda.Publisher = tmpbuilder.ToString();
//Console.WriteLine("Publisher: " + tmpbuilder.ToString());
tmpbuilder.Clear();
//Console.WriteLine("Identifiers (Id numbers):");
//If inserting to T_ArticleVeryFullMeDa
int count = 0;
foreach (var ids in metadatalist.identifier)
{
tmpbuilder.Append(ids.ToString());
if (count == 0)
articlemeda.aSSN = tmpbuilder.ToString();
else if (count == 1)
articlemeda.bSSN = tmpbuilder.ToString();
else if (count == 2)
articlemeda.ExtraId1 = tmpbuilder.ToString();
else if (count == 3)
articlemeda.ExtraId2 = tmpbuilder.ToString();
//Console.WriteLine("Identifiers: " + tmpbuilder.ToString());
tmpbuilder.Clear();
++count;
}
////Console.WriteLine("Identifiers: " + tmpbuilder.ToString());
tmpbuilder.Clear();
//If inserting to T_ArticleFullMeDa
/*
foreach (var ids in metadatalist.identifier)
{
tmpbuilder.Append(ids.ToString());
tmpbuilder.Append(", ");
}
tmpbuilder.Remove(tmpbuilder.Length - 2, 2);
articlemeda.Identifier = tmpbuilder.ToString();
//Console.WriteLine("Identifiers: " + tmpbuilder.ToString());
tmpbuilder.Clear();
*/
foreach (var lang in metadatalist.language)
{
tmpbuilder.Append(lang.ToString());
tmpbuilder.Append("$");
}
try
{
tmpbuilder.Remove(tmpbuilder.Length - 1, 1);
}
catch (System.ArgumentOutOfRangeException e)
{
tmpbuilder.Append("N/A");
}
//tmpbuilder.Remove(tmpbuilder.Length - 2, 2);
articlemeda.PubLanguage = tmpbuilder.ToString();
//Console.WriteLine("Language: " + tmpbuilder.ToString());
tmpbuilder.Clear();
foreach (var rel in metadatalist.relation)
{
tmpbuilder.Append(rel.ToString());
tmpbuilder.Append(", ");
}
try
{
tmpbuilder.Remove(tmpbuilder.Length - 2, 2);
}
catch (System.ArgumentOutOfRangeException e)
{
tmpbuilder.Append("N/A");
}
//tmpbuilder.Remove(tmpbuilder.Length - 2, 2);
articlemeda.Relation = tmpbuilder.ToString();
//Console.WriteLine("Relation (Source): " + tmpbuilder.ToString());
tmpbuilder.Clear();
foreach (var sub in metadatalist.subject)
{
if (sub.ToString().Length < 40)
{
tmpbuilder.Append(sub.ToString());
tmpbuilder.Append("$");
}
else
{
tmpbuilder.Append(sub.ToString().Substring(0, 40));
tmpbuilder.Append("$");
}
}
try
{
tmpbuilder.Remove(tmpbuilder.Length - 1, 1);
}
catch (System.ArgumentOutOfRangeException e)
{
tmpbuilder.Append("N/A");
}
//tmpbuilder.Remove(tmpbuilder.Length - 2, 2);
articlemeda.Subject = tmpbuilder.ToString();
//Console.WriteLine("Subjects:" + tmpbuilder.ToString());
tmpbuilder.Clear();
foreach (var typ in metadatalist.type)
{
tmpbuilder.Append(typ.ToString());
tmpbuilder.Append("$");
}
try
{
tmpbuilder.Remove(tmpbuilder.Length - 1, 1);
}
catch (System.ArgumentOutOfRangeException e)
{
tmpbuilder.Append("N/A");
}
//tmpbuilder.Remove(tmpbuilder.Length - 2, 2);
articlemeda.Type = tmpbuilder.ToString();
//Console.WriteLine("Type: " + tmpbuilder.ToString());
tmpbuilder.Clear();
foreach (var dt in metadatalist.date)
{
tmpbuilder.Append(dt.ToString());
tmpbuilder.Append(", ");
//Console.WriteLine("PubDate: " + dt.ToString());
}
try
{
tmpbuilder.Remove(tmpbuilder.Length - 2, 2);
}
catch (System.ArgumentOutOfRangeException e)
{
tmpbuilder.Append("N/A");
}
//tmpbuilder.Remove(tmpbuilder.Length - 2, 2);'
System.DateTime tmpdate = new System.DateTime();
try
{
tmpdate = System.DateTime.ParseExact(tmpbuilder.ToString(), "yyyy-MM-ddTHH:mm:ssZ", null);
articlemeda.PubDate = tmpdate;
}
catch (System.FormatException e)
{
tmpbuilder.Append("N/A");
}
//Console.WriteLine("PubDate: " + tmpbuilder.ToString());
tmpbuilder.Clear();
//2013-12-01T00:00:00Z
articlemeda.EntryDate = DateTime.Today;
db.T_ArticleVeryFullMeDa_02.Add(articlemeda);
try
{
if (db.SaveChanges() > 0)
{
Console.WriteLine("Saved to DB");
}
}
catch (DbEntityValidationException e)
{
foreach (var eve in e.EntityValidationErrors)
{
Console.WriteLine("Entity of type \"{0}\" in state \"{1}\" has the following validation errors:",
eve.Entry.Entity.GetType().Name, eve.Entry.State);
foreach (var ve in eve.ValidationErrors)
{
Console.WriteLine("- Property: \"{0}\", Error: \"{1}\"",
ve.PropertyName, ve.ErrorMessage);
}
}
continue;
}
catch (DbUpdateException dbupe)
{
//UpdateException upe = new UpdateException();
var innexc = dbupe.InnerException;
if ((innexc.GetType()) == typeof(UpdateException))
{
var sqlinnexc = innexc.InnerException;
if ((sqlinnexc.GetType()) == typeof(SqlException))
{
SqlException sex = (SqlException)sqlinnexc;
switch (sex.Number)
{
case 242:
duprecords.Add(articlemeda);
break;
case 2601:
duprecords.Add(articlemeda);
break;
case 2627:
duprecords.Add(articlemeda);
break;
default:
throw;
}
}
}
}
//db.SaveChanges();
}
}
}
//Console.ReadKey();
}
public ListIdentifier ListIdentifiers(string sPrefix,
string sset,
string sfrom,
string suntil,
ResumptionToken objToken)
{
System.IO.Stream objStream;
ListIdentifier objList = new ListIdentifier();
Identifiers objRecord;
ResumptionToken myToken;
string tmp = "";
System.Net.HttpWebRequest wr;
System.Xml.XmlTextReader rd;
if (sPrefix.Length == 0)
{
sPrefix = "oai_dc";
}
if (objToken == null)
{
if (sset.Length != 0)
{
sset = "&set=" + sset;
}
if (sfrom.Length != 0)
{
sfrom = "&from=" + sfrom;
}
if (suntil.Length != 0)
{
suntil = "&until=" + suntil;
}
prequestURL = baseURL + "?verb=ListIdentifiers&metadataPrefix=" + sPrefix + sset + sfrom + suntil;
}
else
{
prequestURL = baseURL + "?verb=ListIdentifiers&resumptionToken=" + objToken.resumptionToken;
//This is where we handle the resumptionToken
}
//======================================================
// If you wanted to support additional metadata formats,
// you would just need to have additional handlers.
//======================================================
//Console.Write(sURL);
try
{
wr = (System.Net.HttpWebRequest)System.Net.HttpWebRequest.Create(prequestURL);
wr.UserAgent = cUserAgent;
System.Net.WebResponse response = wr.GetResponse();
objStream = response.GetResponseStream();
System.IO.StreamReader reader = new System.IO.StreamReader(objStream);
prawXML = reader.ReadToEnd();
reader.Close();
rd = new System.Xml.XmlTextReader(prawXML, System.Xml.XmlNodeType.Document, null);
}
catch (Exception e)
{
error.errorName = "badConnection";
error.errorDescription = e.Message + "<br>Unable to connect to " + baseURL;
return null;
}
while (rd.Read())
{
if (rd.NodeType == System.Xml.XmlNodeType.Element)
{
if (rd.Name == "responseDate")
{
presponseDate = rd.ReadString();
}
else if (rd.Name == "ListIdentifiers")
{
do
{
if (rd.Name == "header")
{
tmp = rd.ReadOuterXml();
//tmp += ParseOAIContainer(ref rd, "header", true);
//Console.WriteLine("In the Function: " + tmp);
objRecord = new Identifiers(tmp);
objList.record.Add(objRecord);
//return objRecord;
}
else if (rd.Name == "resumptionToken")
{
tmp = rd.ReadOuterXml();
myToken = new ResumptionToken(tmp);
objList.token = myToken;
}
else rd.Read(); // Added the Read() and will never occur with the ReadInnerXml()
} while (rd.Name != "ListIdentifiers"); // loop
}
else if (rd.Name == "error")
{
error.errorName = rd.GetAttribute("code");
error.errorDescription = rd.ReadString();
rd.Close();
return null;
}
}
}
rd.Close();
return objList;
}