public static void AddRecordToDatabase(
RecordQueryResult record,
OaiPmhContext context,
OAIDataProvider dp,
string metadataPrefix,
DateTime harvestDate,
bool addProvenance,
bool createNewIdentifier,
string identifierBase,
bool isHarvestDateTime)
{
if (addProvenance)
{
record.About.Add(Provenance.NewMeta(harvestDate,
isHarvestDateTime,
createNewIdentifier,
dp.BaseURL,
record.Header.OAI_Identifier,
record.Header.Datestamp.HasValue ? record.Header.Datestamp.Value : DateTime.MinValue,
record.Header.IsDatestampDateTime,
FormatList.GetNamespace(metadataPrefix)));
}
/* add header */
Header.AddRecHeaderToDatabase(
context,
record.Header,
dp,
createNewIdentifier,
identifierBase);
/* add metadata */
DbQueries.AddRecMetadataToDatabase(
context,
record.Header.HeaderId,
record.Metadata);
/* add about */
DbQueries.AddRecAboutToDatabase(
context,
record.Header.HeaderId,
record.About);
}
public static XDocument GetRecord(string identifier, string metadataPrefix, List<XElement> errorList, bool? loadAbout)
{
List<XElement> errors = errorList;
bool isIdentifier = !String.IsNullOrEmpty(identifier);
if (!isIdentifier)
{
errors.Add(MlErrors.badIdentifierArgument);
}
bool isPrefixOk = !String.IsNullOrEmpty(metadataPrefix);
if (!isPrefixOk)
{
errors.Add(MlErrors.badMetadataArgument);
}
else if (FormatList.Prefix2Int(metadataPrefix) == 0)
{
errors.Add(MlErrors.cannotDisseminateFormat);
isPrefixOk = false;
}
bool isAbout = loadAbout.HasValue ? loadAbout.Value : Properties.loadAbout;
RecordQueryResult record = null;
if (isIdentifier && isPrefixOk)
{
using (var context = new OaiPmhContext())
{
Header header = DbQueries.GetHeader(context, identifier).FirstOrDefault();
if (header == null)
{
errors.Add(MlErrors.idDoesNotExist);
}
else
{
var formatNum = FormatList.Prefix2Int(metadataPrefix);
/* execute query */
var recQuery = (from om in context.ObjectMetadata
join md in context.Metadata on om.MetadataId equals md.MetadataId
where om.ObjectId == header.HeaderId
where om.ObjectType == Enums.ObjectType.OAIRecord
where ((om.MetadataType == Enums.MetadataType.Metadata && (md.MdFormat & formatNum) != 0) ||
(om.MetadataType == Enums.MetadataType.About))
group md by om.MetadataType into grp
select grp).ToList();
record = new RecordQueryResult()
{
Header = header,
Metadata = (Metadata)recQuery.Where(g => g.Key == Enums.MetadataType.Metadata).SelectMany(g => g).FirstOrDefault(),
About = isAbout ? recQuery.Where(g => g.Key == Enums.MetadataType.About).SelectMany(g => g).Cast<Metadata>().ToList()
: null
};
if (record == null || record.Metadata == null)
{
errors.Add(MlErrors.cannotDisseminateRecordFormat);
}
}
}
}
XElement request = new XElement("request",
new XAttribute("verb", "GetRecord"),
isIdentifier ? new XAttribute("identifier", identifier) : null,
isPrefixOk ? new XAttribute("metadataPrefix", metadataPrefix) : null,
Properties.baseURL);
if (errors.Count > 0)
{
errors.Insert(0, request); /* add request on the first position, that it will be diplayed before errors */
return CreateXml(errors.ToArray());
}
XElement theRecord = new XElement("GetRecord",
new XElement("record",
MlEncode.HeaderItem(record.Header, Properties.granularity),
MlEncode.Metadata(record.Metadata, Properties.granularity)),
isAbout ? MlEncode.About(record.About, Properties.granularity) : null);
return CreateXml(new XElement[] { request, theRecord });
}
public static void GetFile(OAIDataProvider dataProvider, RecordQueryResult record)
{
if (dataProvider == null || record == null)
{
return;
}
string basePath = Properties.directoryForHarvestedFiles + "\\";
basePath += new Uri(dataProvider.BaseURL).Host + "\\";
if (Properties.overwriteHarvestedFiles ? true : string.IsNullOrEmpty(record.Header.FilePath))
{
if (!Directory.Exists(basePath))
{
Directory.CreateDirectory(basePath);
}
string filePath = null;
switch (dataProvider.Function)
{
case "FromPageOnly":
filePath = FromPageOnly(record, basePath, dataProvider.FirstSource);
break;
case "FromSourceTag":
filePath = FromSourceTag(record, dataProvider, basePath);
break;
default:
break;
}
if (!string.IsNullOrEmpty(filePath))
{
record.Header.FilePath = filePath;
}
}
}
public static string FromSourceTag(
RecordQueryResult record,
OAIDataProvider dataProvider,
string basePath)
{
try
{
if (record == null || dataProvider == null || string.IsNullOrEmpty(basePath))
{
return null;
}
string files = string.Empty;
foreach (var sourceItem in GetAllSources(record, dataProvider.FirstSource))
{
var fileName = string.IsNullOrEmpty(sourceItem.Name) ? sourceItem.AlternateName : sourceItem.Name;
string filePath = CreateUniqueFileName(basePath, fileName);
if (DownloadFile(filePath, sourceItem.AbsoluteUri))
{
files += filePath + "][";
}
}
if (!string.IsNullOrEmpty(files))
{
return files.Substring(0, files.Length - 2);
}
return FromPageOnly(record, basePath,
string.IsNullOrEmpty(dataProvider.SecondSource) ? dataProvider.FirstSource : dataProvider.SecondSource);
}
catch (Exception e)
{
/* for debugging purpose only */
string msg = e.Message;
}
return null;
}
public static string FromPageOnly(
RecordQueryResult record,
string basePath,
string source)
{
try
{
if (record == null || string.IsNullOrEmpty(basePath))
{
return null;
}
string files = string.Empty;
foreach (var sourceItem in GetAllSources(record, source))
{
string filePath = PageOnly(sourceItem, basePath);
if (!string.IsNullOrEmpty(filePath))
{
files += filePath;
}
}
if (!string.IsNullOrEmpty(files))
{
return files.Substring(0, files.Length - 2);
}
}
catch (Exception e)
{
/* for debugging purpose only */
string msg = e.Message;
}
return null;
}
public static IEnumerable<FileProperties> GetAllSources(RecordQueryResult record, string sourceProperty)
{
if (record == null || string.IsNullOrEmpty(sourceProperty))
{
yield break;
}
FileProperties tempProps;
var sourcePropertyValue = record.Metadata.GetType().GetProperty(sourceProperty).GetValue(record.Metadata, null).ToString();
var sourceList = MlEncode.ListElementValues(sourcePropertyValue).ToList();
List<string> formatList = new List<string>();
if (sourceProperty != "Format")
{
formatList = MlEncode.ListElementValues(record.Metadata.Format).ToList();
}
for (int i = 0; i < sourceList.Count; i++)
{
var value = sourceList[i];
if (!string.IsNullOrEmpty(value.Trim()))
{
tempProps = new FileProperties();
/* extract absolute URI and extension */
if (sourceProperty != "Format")
{
if (formatList.Count > 1 && i < formatList.Count)
{
tempProps.Extension = GetExtension(formatList[i]);
}
else if (formatList.Count == 1)
{
tempProps.Extension = GetExtension(formatList[0]);
}
else
{
tempProps.Extension = string.Empty;
}
tempProps.AbsoluteUri = value.Trim();
}
else if (value.StartsWith("application"))
{
var tmpSplit = value.Split(new char[] { '\n' }, StringSplitOptions.RemoveEmptyEntries);
if (tmpSplit.Length == 2)
{
tempProps.Extension = GetExtension(tmpSplit[0].Trim());
tempProps.AbsoluteUri = tmpSplit[1].Trim();
}
}
/* extract file name */
string fileName = tempProps.AbsoluteUri.Substring(tempProps.AbsoluteUri.LastIndexOf('/') + 1);
int questionmarkIndex = fileName.IndexOf('?');
if (questionmarkIndex > -1)
{
fileName = string.Empty;
}
if (MimeMapping.GetMimeMapping(fileName) == "application/octet-stream")
{
fileName += tempProps.Extension;
}
tempProps.Name = fileName;
tempProps.AlternateName = record.Header.OAI_Identifier;
/* check if this is an URI */
Uri sourceUri;
if (Uri.TryCreate(tempProps.AbsoluteUri, UriKind.Absolute, out sourceUri) &&
(sourceUri.Scheme == Uri.UriSchemeHttp ||
sourceUri.Scheme == Uri.UriSchemeHttps ||
sourceUri.Scheme == Uri.UriSchemeFtp ||
sourceUri.Scheme == Uri.UriSchemeFile))
{
tempProps.Uri = sourceUri;
yield return tempProps;
}
}
}
}