public static void AddMetadataToDatabase(OaiPmhContext context, int objId, byte objType, byte metaType, Metadata metadata)
{
context.Metadata.Add(metadata);
context.SaveChanges();
context.ObjectMetadata.Add(new ObjectMetadata()
{
ObjectId = objId,
ObjectType = objType,
MetadataType = metaType,
MetadataId = metadata.MetadataId
});
}
public static void AddRecHeaderToDatabase(
OaiPmhContext context,
Header header,
OAIDataProvider dataProvider,
bool createNewIdentifier,
string identifierBase)
{
header.OAIDataProviderId = dataProvider.OAIDataProviderId;
context.Header.Add(header);
context.SaveChanges();
if (createNewIdentifier)
{
if(identifierBase.ElementAt(identifierBase.Length -1) != ':')
{
identifierBase += ':';
}
header.OAI_Identifier = identifierBase + header.HeaderId;
context.Entry(header).State = System.Data.EntityState.Modified;
}
}
private static async Task<string> HarvestRecordsAsync(
OAIDataProvider dataProvider,
string url,
string metadataPrefix,
Enums.DeDuplication deDup,
bool updateStats,
bool harvestFiles,
bool isList,
int retryCount = 3)
{
if (dataProvider == null)
{
if (updateStats)
{
HarvestStats[dataProvider.OAIDataProviderId].HarvestOptions.Stats.Status = "Exception";
HarvestStats[dataProvider.OAIDataProviderId].HarvestOptions.Stats.Message = "Data provider is not initialized";
}
return null;
}
if (string.IsNullOrEmpty(metadataPrefix))
{
if (updateStats)
{
HarvestStats[dataProvider.OAIDataProviderId].HarvestOptions.Stats.Status = "Exception";
HarvestStats[dataProvider.OAIDataProviderId].HarvestOptions.Stats.Message = "Metadata format is not provided";
}
return null;
}
try
{
XDocument xd;
try
{
xd = await GetAndParseXMLAsync(url).ConfigureAwait(false);
}
catch (Exception e)
{
HarvestStats[dataProvider.OAIDataProviderId].HarvestOptions.Stats.Status = "Retrying";
HarvestStats[dataProvider.OAIDataProviderId].HarvestOptions.Stats.Message = e.Message;
return "retry";
}
/* we validate if it's enabled */
bool errors = false;
if (Properties.validateXml && Properties.schemas.Count > 0)
{
List<string> eMsgs = new List<string>();
xd.Validate(Properties.schemas, (o, e) => { errors = true; eMsgs.Add(e.Message); });
}
if (Properties.validateXml ? !errors : true)
{
/* get harvest date */
DateTime harvestDate;
bool isHarvestDateTime;
MlDecode.ResponseDate(ref xd, out harvestDate, out isHarvestDateTime);
XElement listRecords = isList ? xd.Root.Element(MlNamespaces.oaiNs + "ListRecords") :
xd.Root.Element(MlNamespaces.oaiNs + "GetRecord");
if (listRecords != null)
{
/* parse records */
List<RecordQueryResult> records = new List<RecordQueryResult>();
foreach (var record in listRecords.Elements(MlNamespaces.oaiNs + "record"))
{
var rec = ParseRecordAsync(record, metadataPrefix).Result;
if (!rec.Header.Deleted)
{
records.Add(rec);
}
}
int itemsPerPage = records.Count;
if (records.Count > 0)
{
using (var context = new OaiPmhContext())
{
/* try to deduplicate (if selected) and add records to database */
try
{
/* update timestamp of last harvesting */
context.OAIDataProvider.Attach(dataProvider);
if (!isList)
{
dataProvider.LastHarvesting = harvestDate;
context.Entry(dataProvider).State = EntityState.Modified;
}
DeDuplicate.Records(
records,
context,
deDup);
/* add records to database */
foreach (var record in records)
{
if (harvestFiles)
{
FileHarvester.GetFile(dataProvider, record);
}
RecordQueryResult.AddRecordToDatabase(
record,
context,
dataProvider,
metadataPrefix,
harvestDate,
Properties.addProvenanceToHarvestedRecords,
Properties.createNewIdentifierForHarvestedRecords,
Properties.identifierBase,
isHarvestDateTime);
}
context.SaveChanges();
}
catch (DbEntityValidationException dbEx)
{
if (updateStats)
{
HarvestStats[dataProvider.OAIDataProviderId].HarvestOptions.Stats.Status = "Exception";
HarvestStats[dataProvider.OAIDataProviderId].HarvestOptions.Stats.Message = "Database exception occured. Please contact administrator";
}
SaveXMLWithErrors(xd, dbEx, dataProvider.RepositoryName);
}
catch (Exception e)
{
if (updateStats)
{
HarvestStats[dataProvider.OAIDataProviderId].HarvestOptions.Stats.Status = "Exception";
HarvestStats[dataProvider.OAIDataProviderId].HarvestOptions.Stats.Message = e.Message;
}
/* for debugging purpose only */
string msg = e.Message;
}
}
if (!isList)
{
return records.Count > 0 ? records[0].Header.HeaderId.ToString() : null;
}
}
var resumption = listRecords.Element(MlNamespaces.oaiNs + "resumptionToken");
if (resumption != null)
{
/* set complete list size and current progress */
if (updateStats)
{
int completeListSize = 0;
int cursor = 0;
var listSizeAttribute = resumption.Attribute("completeListSize");
var cursorAttribute = resumption.Attribute("cursor");
if ((listSizeAttribute != null &&
int.TryParse(listSizeAttribute.Value, out completeListSize)) &&
(cursorAttribute != null &&
int.TryParse(cursorAttribute.Value, out cursor)))
{
HarvestStats[dataProvider.OAIDataProviderId].HarvestOptions.Stats.RatioAll = completeListSize;
HarvestStats[dataProvider.OAIDataProviderId].HarvestOptions.Stats.RatioDone = cursor + itemsPerPage;
}
}
if (!String.IsNullOrEmpty(resumption.Value))
{
return resumption.Value;
}
}
}
}
else if (updateStats)
{
HarvestStats[dataProvider.OAIDataProviderId].HarvestOptions.Stats.Status = "Exception";
HarvestStats[dataProvider.OAIDataProviderId].HarvestOptions.Stats.Message = "Validation has failed";
}
}
catch (Exception e)
{
if (updateStats)
{
HarvestStats[dataProvider.OAIDataProviderId].HarvestOptions.Stats.Status = "Exception";
HarvestStats[dataProvider.OAIDataProviderId].HarvestOptions.Stats.Message = e.Message;
}
/* for debugging purpose only */
string msg = e.Message;
}
return null;
}
public static void AddToDatabase(OaiPmhContext context, int objId, byte objType, byte metaType, Metadata provenance)
{
context.Metadata.Add(provenance);
context.SaveChanges();
context.ObjectMetadata.Add(new ObjectMetadata()
{
ObjectId = objId,
ObjectType = objType,
MetadataType = metaType,
MetadataId = provenance.MetadataId
});
if (provenance.NestedElements != null)
{
int prevId = provenance.MetadataId;
foreach (var item in provenance.NestedElements)
{
item.AdditionalInt1 = prevId;
context.Metadata.Add(item);
context.SaveChanges();
context.ObjectMetadata.Add(new ObjectMetadata()
{
ObjectId = objId,
ObjectType = objType,
MetadataType = metaType,
MetadataId = item.MetadataId
});
prevId = item.MetadataId;
}
}
}
public static OAIDataProvider AddOrUpdateDataProvider(string baseURL, OAIDataProvider dataProvider)
{
using (var context = new OaiPmhContext())
{
OAIDataProvider dp = null;
bool isUpdateMode = dataProvider != null && dataProvider.OAIDataProviderId != 0;
if (isUpdateMode)
{
/* get data provider to update */
context.Configuration.ProxyCreationEnabled = false;
dp = context.OAIDataProvider.Where(d => d.OAIDataProviderId == dataProvider.OAIDataProviderId).FirstOrDefault();
}
else if (!string.IsNullOrEmpty(baseURL))
{
/* get and parse XML document */
dp = IdentifyDataProvider(baseURL);
}
if (dp != null && isUpdateMode ? true : !context.OAIDataProvider.Where(d => d.BaseURL == dp.BaseURL).Any())
{
if (dataProvider != null)
{
dp.Function = dataProvider.Function;
dp.FirstSource = dataProvider.FirstSource;
dp.SecondSource = dataProvider.SecondSource;
}
if (!isUpdateMode)
{
context.OAIDataProvider.Add(dp);
}
context.SaveChanges();
return dp;
}
}
return null;
}
public static bool DeleteSetting(string name)
{
if (!string.IsNullOrEmpty(name))
{
using (var context = new OaiPmhContext())
{
Property setting = context.Property.Where(s => s.Key == name).FirstOrDefault();
if (setting != null)
{
context.Property.Remove(setting);
context.SaveChanges();
return true;
}
}
}
return false;
}
public static bool AddOrUpdateSetting(Property newSetting)
{
if (!string.IsNullOrEmpty(newSetting.Key))
{
using (var context = new OaiPmhContext())
{
if (newSetting.Value == null)
{
newSetting.Value = "";
}
Property setting = context.Property.Where(s => s.Key == newSetting.Key).FirstOrDefault();
if (setting == null)
{
/* add */
context.Property.Add(newSetting);
context.SaveChanges();
return true;
}
/* update */
setting.Value = newSetting.Value;
setting.Section = newSetting.Section;
context.SaveChanges();
return true;
}
}
return false;
}
internal static OAIDataProvider ReIdentifyDataProvider(int id)
{
using (var context = new OaiPmhContext())
{
context.Configuration.ProxyCreationEnabled = false;
var dataProvider = context.OAIDataProvider.Where(d => d.OAIDataProviderId == id).FirstOrDefault();
if (dataProvider != null)
{
var dp = IdentifyDataProvider(dataProvider.BaseURL);
if (dp != null)
{
dataProvider.AdminEmail = dp.AdminEmail;
dataProvider.BaseURL = dp.BaseURL;
dataProvider.Compression = dp.Compression;
dataProvider.DeletedRecord = dp.DeletedRecord;
dataProvider.EarliestDatestamp = dp.EarliestDatestamp;
dataProvider.Granularity = dp.Granularity;
dataProvider.ProtocolVersion = dp.ProtocolVersion;
dataProvider.RepositoryName = dp.RepositoryName;
context.SaveChanges();
return dataProvider;
}
}
}
return null;
}
public static bool DeleteDataProvider(int id)
{
using (var context = new OaiPmhContext())
{
var dataProvider = context.OAIDataProvider.Where(d => d.OAIDataProviderId == id).FirstOrDefault();
if (dataProvider != null)
{
context.OAIDataProvider.Remove(dataProvider);
context.SaveChanges();
return true;
}
}
return false;
}