public override async Task<Document> GetDocument(ParsedUri puri)
{
if (puri == null)
{
Debug.WriteLine("Error: empty URL provided.");
return null;
}
var doc = await base.GetDocument(puri);
if (doc == null)
{
try
{
var response = await HttpClient.GetAsync(new Uri(MetadataServiceUri, "metadata.json?url=" + puri.AbsoluteUri));
if (response.IsSuccessStatusCode)
{
doc = this.MetadataTranslationScope.Deserialize(await response.Content.ReadAsStreamAsync(), Format.Json) as Document;
}
}
catch (Exception e)
{
Debug.WriteLine("failed to get document: {0}", e.Message);
doc = null;
}
}
return doc;
}
public Channel(String pTitle, String pDescription, ParsedUri pLink, List<Item> pItems)
{
title = pTitle;
description = pDescription;
link = pLink;
items = pItems;
}
public PURLConnection(ParsedUri purl, HttpWebRequest request, Stream inputStream)
{
PURL = purl;
Stream = inputStream;
Request = request;
Good = true;
}
public PURLConnection(ParsedUri purl, HttpWebRequest request, Stream inputStream)
{
PURL = purl;
Stream = inputStream;
Request = request;
Good = true;
}
/// <summary>
///
/// Encapsulate the oodss client requst in a simpler GetMetadata call.
/// </summary>
/// <param name="url"></param>
/// <returns></returns>
public async Task<Document> RequestMetadata(ParsedUri uri)
{
Document result = null;
if (_metadataClient != null)
{
Debug.WriteLine("Performing asynchronous call");
ResponseMessage metadataResponse =
_metadataClient.RequestAsync(new MetadataRequest(uri.ToString())).Result;
Debug.WriteLine("Received asynchronous request ");
if (metadataResponse != null && metadataResponse is MetadataResponse)
{
result = (metadataResponse as MetadataResponse).Metadata;
}
else if (metadataResponse != null && metadataResponse is SemanticServiceError)
{
(metadataResponse as SemanticServiceError).Perform();
}
else
throw new Exception();
}
else
{
var requestUri = new ParsedUri(_serviceBaseUri, "?url=" + uri.AbsoluteUri);
result = await _metadataTypeScope.DeserializeUri(requestUri, Format.Xml) as Document;
}
return result;
}
public SemanticsSessionScope(SimplTypesScope metadataTranslationScope, string repoLocation, ParsedUri serviceUri,
EventHandler<EventArgs> onCompleted)
: base(metadataTranslationScope, repoLocation, onCompleted)
{
SemanticsSessionScope.Get = this;
MetadataServiceUri = serviceUri;
HttpClient = new HttpClient();
}
public WebViewParser(DocumentClosure closure)
{
_closure = closure;
SemanticsSessionScope = closure.SemanticsSessionScope;
_webView = SemanticsSessionScope.WebBrowserPool.Acquire();
_puri = closure.PURLConnection.ResponsePURL;
_requestTimedOut = new DispatcherTimer() { Interval = EXTRACTION_TIMEOUT };
_requestTimedOut.Tick += ExtractionRequestTimedOut;
}
public Item(String pTitle, String pDescription, ParsedUri pLink, String pGuid, String pAuthor, List<String> pCategorySet )
{
title = pTitle;
description = pDescription;
link = pLink;
guid = pGuid;
author = pAuthor;
categorySet = pCategorySet;
}
public virtual async Task<Document> GetDocument(ParsedUri location)
{
if (location == null)
return null;
Document doc;
GlobalDocumentCollection.TryGetDocument(location, out doc);
return doc;
}
private async void BtnGetMetadata_Click(object sender, RoutedEventArgs e)
{
// string serializedMetadata = UrlBox.Text;
// Object o = RepositoryMetadataTranslationScope.Get().Deserialize(serializedMetadata, StringFormat.Xml);
// StringBuilder sb = new StringBuilder();
// SimplTypesScope.Serialize(o, sb, StringFormat.Xml);
// String reserializedMetadatata = sb.ToString();
// Console.WriteLine(reserializedMetadatata);
//Expander expander = new Expander();
//TextBox metadataXML = new TextBox { TextWrapping = TextWrapping.Wrap, MinHeight = 100 };
//metadataXML.Text = reserializedMetadatata;
string urls = UrlBox.Text;
List<Task<Document>> extractionRequests = new List<Task<Document>>();
Dictionary<ParsedUri, DateTime> timeStamps = new Dictionary<ParsedUri, DateTime>();
foreach (var s in urls.Split(new string[] { Environment.NewLine }, StringSplitOptions.None))
{
Console.WriteLine("Requesting async extraction of: " + s);
ParsedUri puri = new ParsedUri(s);
{
Task<Document> t = _semanticsSessionScope.GetDocument(puri);
timeStamps.Add(puri, DateTime.Now);
//Alternate, if you want the document here:
//Document doc = await _semanticsSessionScope.GetDocument(puri);
extractionRequests.Add(t);
}
}
while (extractionRequests.Count > 0)
{
Task<Document> completedTask = await Task.WhenAny(extractionRequests);
extractionRequests.Remove(completedTask);
Document parsedDoc = await completedTask;
if (parsedDoc == null)
continue;
Expander expander = new Expander {Header = parsedDoc.Title};
TextBox metadataXML = new TextBox {TextWrapping = TextWrapping.Wrap, MinHeight = 100};
var s = timeStamps[parsedDoc.Location.Value];
Console.WriteLine(" ---------------------------------- Time to complete: " +
DateTime.Now.Subtract(s).TotalMilliseconds);
metadataXML.Text = await Task.Run(() => SimplTypesScope.Serialize(parsedDoc, StringFormat.Xml));
expander.Content = metadataXML;
MetadataTitleXMLContainer.Children.Add(expander);
}
}
public virtual async Task<Document> GetOrConstructDocument(ParsedUri location)
{
var doc = await GetDocument(location);
if(doc == null)
{
doc = MetaMetadataRepository.ConstructDocument(location, false);
if (doc != null)
GlobalDocumentCollection.AddDocument(doc, location);
}
return doc;
}
public MetadataServicesClient(SimplTypesScope metadatascope, SemanticsSessionScope semanticSessionScope, ParsedUri serviceUri, bool useWebSockets = false)
{
SimplTypesScope[] oodssAndMetadataScope = {metadatascope, DefaultServicesTranslations.Get()};
_metadataTypeScope = SimplTypesScope.Get("MetadataServicesTranslationScope",
oodssAndMetadataScope,
typeof (MetadataRequest),
typeof (MetadataResponse),
typeof (SemanticServiceError)
);
_serviceBaseUri = serviceUri;
if (useWebSockets)
{
_metadataClient = new WebSocketOODSSClient("127.0.0.1", 2018, _metadataTypeScope, semanticSessionScope);
_metadataClient.StartAsync();
}
}
public static ImageClipping ConstructClipping(ParsedUri location, Document sourceDoc, IdeaMacheUser creator)
{
var imageClipping = new ImageClipping
{
Media = new Image
{
Location = new MetadataParsedURL(location)
},
SourceDoc = sourceDoc,
CreativeActs = new List<CreativeAct>(),
};
imageClipping.CreativeActs.Add(new CreativeAct
{
Action = CreativeAct.CreativeAction.CurateClipping,
Time = new MetadataDate(DateTime.UtcNow),
Creator = creator
});
return imageClipping;
}
public ParsedUri Perform(ParsedUri input)
{
ParsedUri result = null;
if (input != null && match != null)
{
String str = input.ToString();
Match matcher = match.Match(str);
if (matcher.Success)
{
if (replace == null)
replace = "";
Debug.WriteLine(matcher.Groups[0].Value + " " + matcher.Groups[1].Value);
String rez = input.ToString().Replace(matcher.Groups[0].Value, replace);
for (int i = 1; i < matcher.Groups.Count; i++)
rez = rez.Replace("$" + i, matcher.Groups[i].Value);
result = new ParsedUri(rez);
}
}
return result;
}
private async void ShowFeeds()
{
Feeds.Clear();
if (FeedURLs != null)
{
foreach (string url in FeedURLs.Split(new char[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries))
{
var puri = new ParsedUri(url);
var parsedDoc = await _sessionScope.GetDocument(puri);
var feedViewModels = FeedViewModelFactory.GetFeedViewModels(parsedDoc);
foreach (var viewModel in feedViewModels)
{
Feeds.Add(viewModel);
}
this.NotifyPropertyChanged("Feeds");
}
}
}
/// <summary>
///
///
/// </summary>
/// <param name="baseUri"></param>
/// <param name="uri"></param>
public ParsedUri(ParsedUri baseUri, String uri)
: base(baseUri, uri)
{
}
/// <summary>
///
///
/// </summary>
/// <param name="baseUri"></param>
/// <param name="uri"></param>
public ParsedUri(ParsedUri baseUri, String uri)
: base(baseUri, uri)
{
}
public async static Task<SemanticsSessionScope> InitAsync(SimplTypesScope metadataTranslationScope, string repoLocation, ParsedUri serviceUri)
{
var scope = await Task.Run(() => new SemanticsSessionScope(metadataTranslationScope, repoLocation, serviceUri, null));
return scope;
}
public PURLConnection(ParsedUri purl)
{
PURL = purl;
}
public override async Task<Document> GetOrConstructDocument(ParsedUri location)
{
Document doc = await base.GetOrConstructDocument(location);
doc.SemanticsSessionScope = this;
return doc;
}
///<summary>
/// Get the old location from this.
/// Set the location of this to the newLocation.
/// Add a mapping in the GlobalCollection from newLocation to this.
/// Add the old location for this as an additionalLocation for this.
///</summary>
public void ChangeLocation(ParsedUri newLocation)
{
if (newLocation != null)
{
MetadataParsedURL origLocation = Location;
if (!origLocation.Value.Equals(newLocation))
{
Location = new MetadataParsedURL(newLocation);
if (SemanticsSessionScope != null)
SemanticsSessionScope.GlobalDocumentCollection.AddDocument(this, newLocation);
AddAdditionalLocation(origLocation);
}
}
}
public string GetUriLeftPart(ParsedUri parsedUri)
{
var urlWithQuery = parsedUri.GetComponents(UriComponents.HttpRequestUrl, UriFormat.UriEscaped);
var query = parsedUri.GetComponents(UriComponents.Query, UriFormat.UriEscaped);
var index = urlWithQuery.IndexOf(query);
string urlWithoutQuery = urlWithQuery;
if (index != -1)
{
urlWithoutQuery = urlWithQuery.Remove(index);
}
return urlWithoutQuery;
}
// throws IOException
///<summary>
/// Extract location ParsedURL parameters into a HashMap.
/// Let ParamOps operate on this Map.
/// Derive a new ParsedURL using the base of the original location's ParsedURL and the transformed parameter map.
/// If the new ParsedURL is different than the old one, make the old one an additional location for this,
/// and add the transformed ParsedURL to the DocumentLocationMap.
/// <p/>
/// If the location changes, then reconnect to the new one.
///</summary>
public override Object Perform(Object obj)
{
bool usingThisDoc = true; // if we are changing this document's location, or its child field's location
Document document = (Document) SemanticOperationHandler.SemanticOperationVariableMap.Get(SemanticOperationKeyWords.Metadata);
if (ObjectStr != null)
{
Object o = SemanticOperationHandler.SemanticOperationVariableMap.Get(ObjectStr);
if (o != null && o is Document)
{
document = (Document) o;
usingThisDoc = false;
}
}
ParsedUri origLocation = document.Location.value;
if (origLocation.IsFile)
{
Debug.WriteLine("Not doing <filter_location> because this is a file: " + origLocation);
return null;
}
SemanticsGlobalCollection<Document> globalCollection = SemanticOperationHandler.SemanticsScope.GlobalDocumentCollection;
bool locationChanged = false;
if (paramOps != null && paramOps.Count > 0)
{
Dictionary<String, String> parametersMap = new Dictionary<string, string>();
foreach (String param in origLocation.Query.Split('&'))
{
parametersMap.Add(param.Split('=')[0], param.Split('=')[1]);
}
if (parametersMap.Count == 0)
parametersMap = new Dictionary<string, string>(paramOps.Count);
foreach (ParamOp paramOp in paramOps)
{
paramOp.SemanticHandler = SemanticOperationHandler;
paramOp.TransformParams(parametersMap);
}
String str = origLocation.AbsolutePath + "?";
foreach (KeyValuePair<string, string> pair in parametersMap)
{
str += pair.Key + "=" + pair.Value + "&";
}
if (str[str.Length - 1] == '&')
str = str.Substring(0, str.Length - 1);
ParsedUri transformedLocation = new ParsedUri(str);
if (origLocation != transformedLocation)
{
document.Location.Value = transformedLocation;
locationChanged = true;
}
}
if (alternativeHosts != null)
{
String origHost = origLocation.Host;
foreach (String alternativeHost in alternativeHosts)
{
if (!origHost.Equals(alternativeHost))
{
ParsedUri newLocation = new ParsedUri(origLocation.AbsoluteUri.Replace(origLocation.Host, alternativeHost));
document.AddAdditionalLocation(new MetadataParsedURL(newLocation));
globalCollection.AddDocument(document, newLocation);
}
}
}
if (regex != null)
{
ParsedUri location = document.Location.Value;
ParsedUri regexURL = regex.Perform(location);
document.ChangeLocation(regexURL);
locationChanged = true;
}
if (stripPrefix != null)
{
String origlLocationString = origLocation.ToString();
int index = origlLocationString.IndexOf(stripPrefix);
if (index > 6)
{
String newLocationString = origlLocationString.Substring(0, index);
ParsedUri newLocation = new ParsedUri(newLocationString);
if (newLocation != null)
{
document.ChangeLocation(newLocation);
}
}
}
if (locationChanged && usingThisDoc) // if we are just changing the location of a field, we don't have to reconnect.
{
if (documentParser != null)
{
//documentParser.reConnect(); // changed the location, so we better connect again!
}
else
{
//imageClosure.RequestMetadata();
}
}
return null;
}
public PURLConnection(ParsedUri purl)
{
PURL = purl;
}
public async Task<object> DeserializeUri(ParsedUri uri, Format format = Format.Xml, TranslationContext context = null, IDeserializationHookStrategy deserializationHookStrategy = null)
{
object result = null;
var request = WebRequest.Create(uri);
if (request != null)
{
WebResponse response = await request.GetResponseAsync();
result = Deserialize(response.GetResponseStream(), context, deserializationHookStrategy, format);
}
return result;
}
public bool ProcessRedirect(Uri redirectedUri)
{
MetadataParsedURL originalMetadataPURL = originalDocument.Location;
ParsedUri originalPURL = originalMetadataPURL == null ? null : originalMetadataPURL.Value;
ParsedUri redirectedPURL = new ParsedUri(redirectedUri.AbsoluteUri);
Debug.WriteLine("try redirecting: " + originalPURL + " > " + redirectedPURL);
/*Document redirectedDocument = semanticsSessionScope.GetOrConstructDocument(redirectedPURL);
// note FIXME currently, GetOrConstructDocument does not return null!
if (redirectedDocument != null)
{
// existing document: the redirected url has been visited already.
if (originalPURL != redirectedPURL)
redirectedDocument.AddAdditionalLocation(originalMetadataPURL);
// TODO -- copy metadata from originalDocument?!!
documentClosure.ChangeDocument(redirectedDocument);
// TODO -- reconnect
return true;
}
else
{
// TODO -- redirect to a new location.
}*/
// TODO -- what if redirectedDocument is already in the queue or being downloaded?
return false;
}
public TranslationContext(ParsedUri baseUri)
{
_baseDirPurl = baseUri;
}
public string GetUriLeftPart(ParsedUri parsedUri)
{
return parsedUri.GetLeftPart(UriPartial.Path);
}