public static string FromPageOnly( RecordQueryResult record, string basePath, string source) { try { if (record == null || string.IsNullOrEmpty(basePath)) { return(null); } string files = string.Empty; foreach (var sourceItem in GetAllSources(record, source)) { string filePath = PageOnly(sourceItem, basePath); if (!string.IsNullOrEmpty(filePath)) { files += filePath; } } if (!string.IsNullOrEmpty(files)) { return(files.Substring(0, files.Length - 2)); } } catch (Exception e) { /* for debugging purpose only */ string msg = e.Message; } return(null); }
public static void GetFile(OAIDataProvider dataProvider, RecordQueryResult record) { if (dataProvider == null || record == null) { return; } string basePath = Properties.directoryForHarvestedFiles + "\\"; basePath += new Uri(dataProvider.BaseURL).Host + "\\"; if (Properties.overwriteHarvestedFiles ? true : string.IsNullOrEmpty(record.Header.FilePath)) { if (!Directory.Exists(basePath)) { Directory.CreateDirectory(basePath); } string filePath = null; switch (dataProvider.Function) { case "FromPageOnly": filePath = FromPageOnly(record, basePath, dataProvider.FirstSource); break; case "FromSourceTag": filePath = FromSourceTag(record, dataProvider, basePath); break; default: break; } if (!string.IsNullOrEmpty(filePath)) { record.Header.FilePath = filePath; } } }
public static string FromSourceTag( RecordQueryResult record, OAIDataProvider dataProvider, string basePath) { try { if (record == null || dataProvider == null || string.IsNullOrEmpty(basePath)) { return(null); } string files = string.Empty; foreach (var sourceItem in GetAllSources(record, dataProvider.FirstSource)) { var fileName = string.IsNullOrEmpty(sourceItem.Name) ? sourceItem.AlternateName : sourceItem.Name; string filePath = CreateUniqueFileName(basePath, fileName); if (DownloadFile(filePath, sourceItem.AbsoluteUri)) { files += filePath + "]["; } } if (!string.IsNullOrEmpty(files)) { return(files.Substring(0, files.Length - 2)); } return(FromPageOnly(record, basePath, string.IsNullOrEmpty(dataProvider.SecondSource) ? dataProvider.FirstSource : dataProvider.SecondSource)); } catch (Exception e) { /* for debugging purpose only */ string msg = e.Message; } return(null); }
public static IEnumerable <FileProperties> GetAllSources(RecordQueryResult record, string sourceProperty) { if (record == null || string.IsNullOrEmpty(sourceProperty)) { yield break; } FileProperties tempProps; var sourcePropertyValue = record.Metadata.GetType().GetProperty(sourceProperty).GetValue(record.Metadata, null).ToString(); var sourceList = MlEncode.ListElementValues(sourcePropertyValue).ToList(); List <string> formatList = new List <string>(); if (sourceProperty != "Format") { formatList = MlEncode.ListElementValues(record.Metadata.Format).ToList(); } for (int i = 0; i < sourceList.Count; i++) { var value = sourceList[i]; if (!string.IsNullOrEmpty(value.Trim())) { tempProps = new FileProperties(); /* extract absolute URI and extension */ if (sourceProperty != "Format") { if (formatList.Count > 1 && i < formatList.Count) { tempProps.Extension = GetExtension(formatList[i]); } else if (formatList.Count == 1) { tempProps.Extension = GetExtension(formatList[0]); } else { tempProps.Extension = string.Empty; } tempProps.AbsoluteUri = value.Trim(); } else if (value.StartsWith("application")) { var tmpSplit = value.Split(new char[] { '\n' }, StringSplitOptions.RemoveEmptyEntries); if (tmpSplit.Length == 2) { tempProps.Extension = GetExtension(tmpSplit[0].Trim()); tempProps.AbsoluteUri = tmpSplit[1].Trim(); } } /* extract file name */ string fileName = tempProps.AbsoluteUri.Substring(tempProps.AbsoluteUri.LastIndexOf('/') + 1); int questionmarkIndex = fileName.IndexOf('?'); if (questionmarkIndex > -1) { fileName = string.Empty; } if (MimeMapping.GetMimeMapping(fileName) == "application/octet-stream") { fileName += tempProps.Extension; } tempProps.Name = fileName; tempProps.AlternateName = record.Header.OAI_Identifier; /* check if this is an URI */ Uri sourceUri; if (Uri.TryCreate(tempProps.AbsoluteUri, UriKind.Absolute, out sourceUri) && (sourceUri.Scheme == Uri.UriSchemeHttp || sourceUri.Scheme == Uri.UriSchemeHttps || sourceUri.Scheme == Uri.UriSchemeFtp || sourceUri.Scheme == Uri.UriSchemeFile)) { tempProps.Uri = sourceUri; yield return(tempProps); } } } }
public static XDocument GetRecord(string identifier, string metadataPrefix, List <XElement> errorList, bool?loadAbout, Tenant tenant = null) { List <XElement> errors = errorList; bool isIdentifier = !String.IsNullOrEmpty(identifier); if (!isIdentifier) { errors.Add(MlErrors.badIdentifierArgument); } bool isPrefixOk = !String.IsNullOrEmpty(metadataPrefix); if (!isPrefixOk) { errors.Add(MlErrors.badMetadataArgument); } else if (FormatList.Prefix2Int(metadataPrefix) == 0) { errors.Add(MlErrors.cannotDisseminateFormat); isPrefixOk = false; } bool isAbout = loadAbout.HasValue ? loadAbout.Value : Properties.loadAbout; RecordQueryResult record = null; if (isIdentifier && isPrefixOk) { OAIHelper helper = new OAIHelper(); long datasetId = helper.ConvertToId(identifier); Header header = helper.GetHeader(datasetId); if (header == null) { errors.Add(MlErrors.idDoesNotExist); } else { record = new RecordQueryResult(header, helper.GetMetadata(datasetId, metadataPrefix)); // ToDo Check About in the documentaion of oai-pmh record.About = null; if (record == null || record.Metadata == null) { errors.Add(MlErrors.cannotDisseminateRecordFormat); } } } XElement request = new XElement("request", new XAttribute("verb", "GetRecord"), isIdentifier ? new XAttribute("identifier", identifier) : null, isPrefixOk ? new XAttribute("metadataPrefix", metadataPrefix) : null, Properties.baseURL); if (errors.Count > 0) { errors.Insert(0, request); /* add request on the first position, that it will be diplayed before errors */ return(CreateXml(errors.ToArray())); } XElement theRecord = new XElement("GetRecord", new XElement("record", MlEncode.HeaderItem(record.Header, Properties.granularity), MlEncode.Metadata(record.Metadata, Properties.granularity)), isAbout ? MlEncode.About(record.About, Properties.granularity) : null); return(CreateXml(new XElement[] { request, theRecord })); }