public static string FromPageOnly(
RecordQueryResult record,
string basePath,
string source)
{
try
{
if (record == null || string.IsNullOrEmpty(basePath))
{
return(null);
}
string files = string.Empty;
foreach (var sourceItem in GetAllSources(record, source))
{
string filePath = PageOnly(sourceItem, basePath);
if (!string.IsNullOrEmpty(filePath))
{
files += filePath;
}
}
if (!string.IsNullOrEmpty(files))
{
return(files.Substring(0, files.Length - 2));
}
}
catch (Exception e)
{
/* for debugging purpose only */
string msg = e.Message;
}
return(null);
}
public static void GetFile(OAIDataProvider dataProvider, RecordQueryResult record)
{
if (dataProvider == null || record == null)
{
return;
}
string basePath = Properties.directoryForHarvestedFiles + "\\";
basePath += new Uri(dataProvider.BaseURL).Host + "\\";
if (Properties.overwriteHarvestedFiles ? true : string.IsNullOrEmpty(record.Header.FilePath))
{
if (!Directory.Exists(basePath))
{
Directory.CreateDirectory(basePath);
}
string filePath = null;
switch (dataProvider.Function)
{
case "FromPageOnly":
filePath = FromPageOnly(record, basePath, dataProvider.FirstSource);
break;
case "FromSourceTag":
filePath = FromSourceTag(record, dataProvider, basePath);
break;
default:
break;
}
if (!string.IsNullOrEmpty(filePath))
{
record.Header.FilePath = filePath;
}
}
}
public static string FromSourceTag(
RecordQueryResult record,
OAIDataProvider dataProvider,
string basePath)
{
try
{
if (record == null || dataProvider == null || string.IsNullOrEmpty(basePath))
{
return(null);
}
string files = string.Empty;
foreach (var sourceItem in GetAllSources(record, dataProvider.FirstSource))
{
var fileName = string.IsNullOrEmpty(sourceItem.Name) ? sourceItem.AlternateName : sourceItem.Name;
string filePath = CreateUniqueFileName(basePath, fileName);
if (DownloadFile(filePath, sourceItem.AbsoluteUri))
{
files += filePath + "][";
}
}
if (!string.IsNullOrEmpty(files))
{
return(files.Substring(0, files.Length - 2));
}
return(FromPageOnly(record, basePath,
string.IsNullOrEmpty(dataProvider.SecondSource) ? dataProvider.FirstSource : dataProvider.SecondSource));
}
catch (Exception e)
{
/* for debugging purpose only */
string msg = e.Message;
}
return(null);
}
public static IEnumerable <FileProperties> GetAllSources(RecordQueryResult record, string sourceProperty)
{
if (record == null || string.IsNullOrEmpty(sourceProperty))
{
yield break;
}
FileProperties tempProps;
var sourcePropertyValue = record.Metadata.GetType().GetProperty(sourceProperty).GetValue(record.Metadata, null).ToString();
var sourceList = MlEncode.ListElementValues(sourcePropertyValue).ToList();
List <string> formatList = new List <string>();
if (sourceProperty != "Format")
{
formatList = MlEncode.ListElementValues(record.Metadata.Format).ToList();
}
for (int i = 0; i < sourceList.Count; i++)
{
var value = sourceList[i];
if (!string.IsNullOrEmpty(value.Trim()))
{
tempProps = new FileProperties();
/* extract absolute URI and extension */
if (sourceProperty != "Format")
{
if (formatList.Count > 1 && i < formatList.Count)
{
tempProps.Extension = GetExtension(formatList[i]);
}
else if (formatList.Count == 1)
{
tempProps.Extension = GetExtension(formatList[0]);
}
else
{
tempProps.Extension = string.Empty;
}
tempProps.AbsoluteUri = value.Trim();
}
else if (value.StartsWith("application"))
{
var tmpSplit = value.Split(new char[] { '\n' }, StringSplitOptions.RemoveEmptyEntries);
if (tmpSplit.Length == 2)
{
tempProps.Extension = GetExtension(tmpSplit[0].Trim());
tempProps.AbsoluteUri = tmpSplit[1].Trim();
}
}
/* extract file name */
string fileName = tempProps.AbsoluteUri.Substring(tempProps.AbsoluteUri.LastIndexOf('/') + 1);
int questionmarkIndex = fileName.IndexOf('?');
if (questionmarkIndex > -1)
{
fileName = string.Empty;
}
if (MimeMapping.GetMimeMapping(fileName) == "application/octet-stream")
{
fileName += tempProps.Extension;
}
tempProps.Name = fileName;
tempProps.AlternateName = record.Header.OAI_Identifier;
/* check if this is an URI */
Uri sourceUri;
if (Uri.TryCreate(tempProps.AbsoluteUri, UriKind.Absolute, out sourceUri) &&
(sourceUri.Scheme == Uri.UriSchemeHttp ||
sourceUri.Scheme == Uri.UriSchemeHttps ||
sourceUri.Scheme == Uri.UriSchemeFtp ||
sourceUri.Scheme == Uri.UriSchemeFile))
{
tempProps.Uri = sourceUri;
yield return(tempProps);
}
}
}
}
public static XDocument GetRecord(string identifier, string metadataPrefix, List <XElement> errorList, bool?loadAbout, Tenant tenant = null)
{
List <XElement> errors = errorList;
bool isIdentifier = !String.IsNullOrEmpty(identifier);
if (!isIdentifier)
{
errors.Add(MlErrors.badIdentifierArgument);
}
bool isPrefixOk = !String.IsNullOrEmpty(metadataPrefix);
if (!isPrefixOk)
{
errors.Add(MlErrors.badMetadataArgument);
}
else if (FormatList.Prefix2Int(metadataPrefix) == 0)
{
errors.Add(MlErrors.cannotDisseminateFormat);
isPrefixOk = false;
}
bool isAbout = loadAbout.HasValue ? loadAbout.Value : Properties.loadAbout;
RecordQueryResult record = null;
if (isIdentifier && isPrefixOk)
{
OAIHelper helper = new OAIHelper();
long datasetId = helper.ConvertToId(identifier);
Header header = helper.GetHeader(datasetId);
if (header == null)
{
errors.Add(MlErrors.idDoesNotExist);
}
else
{
record = new RecordQueryResult(header, helper.GetMetadata(datasetId, metadataPrefix));
// ToDo Check About in the documentaion of oai-pmh
record.About = null;
if (record == null || record.Metadata == null)
{
errors.Add(MlErrors.cannotDisseminateRecordFormat);
}
}
}
XElement request = new XElement("request",
new XAttribute("verb", "GetRecord"),
isIdentifier ? new XAttribute("identifier", identifier) : null,
isPrefixOk ? new XAttribute("metadataPrefix", metadataPrefix) : null,
Properties.baseURL);
if (errors.Count > 0)
{
errors.Insert(0, request); /* add request on the first position, that it will be diplayed before errors */
return(CreateXml(errors.ToArray()));
}
XElement theRecord = new XElement("GetRecord",
new XElement("record",
MlEncode.HeaderItem(record.Header, Properties.granularity),
MlEncode.Metadata(record.Metadata, Properties.granularity)),
isAbout ? MlEncode.About(record.About, Properties.granularity) : null);
return(CreateXml(new XElement[] { request, theRecord }));
}