Extending Uri to support some additional features currently include .Domain and .Suffix
Наследование: System.Uri
        public override async Task<Document> GetDocument(ParsedUri puri)
        {
            if (puri == null)
            {
                Debug.WriteLine("Error: empty URL provided.");
                return null;
            }

            var doc = await base.GetDocument(puri);
            if (doc == null)
            {
                try
                {
                    var response = await HttpClient.GetAsync(new Uri(MetadataServiceUri, "metadata.json?url=" + puri.AbsoluteUri));
                    if (response.IsSuccessStatusCode)
                    {
                        doc = this.MetadataTranslationScope.Deserialize(await response.Content.ReadAsStreamAsync(), Format.Json) as Document;
                    }
                }
                catch (Exception e)
                {
                    Debug.WriteLine("failed to get document: {0}", e.Message);
                    doc = null;
                }
            }

            return doc;
        }
Пример #2
0
 public Channel(String pTitle, String pDescription, ParsedUri pLink, List<Item> pItems)
 {
     title = pTitle;
     description = pDescription;
     link = pLink;
     items = pItems;
 }
Пример #3
0
 public PURLConnection(ParsedUri purl, HttpWebRequest request, Stream inputStream)
 {
     PURL = purl;
     Stream = inputStream;
     Request = request;
     Good = true;
 }
Пример #4
0
 public PURLConnection(ParsedUri purl, HttpWebRequest request, Stream inputStream)
 {
     PURL    = purl;
     Stream  = inputStream;
     Request = request;
     Good    = true;
 }
        /// <summary>
        /// 
        /// Encapsulate the oodss client requst in a simpler GetMetadata call.
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public async Task<Document> RequestMetadata(ParsedUri uri)
        {
            Document result = null;

            if (_metadataClient != null)
            {
                Debug.WriteLine("Performing asynchronous call");
                ResponseMessage metadataResponse =
                    _metadataClient.RequestAsync(new MetadataRequest(uri.ToString())).Result;
                Debug.WriteLine("Received asynchronous request ");

                if (metadataResponse != null && metadataResponse is MetadataResponse)
                {
                    result = (metadataResponse as MetadataResponse).Metadata;
                }
                else if (metadataResponse != null && metadataResponse is SemanticServiceError)
                {
                    (metadataResponse as SemanticServiceError).Perform();
                }
                else
                    throw new Exception();
            }
            else
            {
                var requestUri = new ParsedUri(_serviceBaseUri, "?url=" + uri.AbsoluteUri);
                result = await _metadataTypeScope.DeserializeUri(requestUri, Format.Xml) as Document;
            }

            return result;
        }
        public SemanticsSessionScope(SimplTypesScope metadataTranslationScope, string repoLocation, ParsedUri serviceUri,
            EventHandler<EventArgs> onCompleted)
            : base(metadataTranslationScope, repoLocation, onCompleted)
        {
            SemanticsSessionScope.Get = this;

            MetadataServiceUri = serviceUri;
            HttpClient = new HttpClient();
        }
 public WebViewParser(DocumentClosure closure)
 {
     _closure = closure;
     SemanticsSessionScope = closure.SemanticsSessionScope;
     _webView = SemanticsSessionScope.WebBrowserPool.Acquire();
     _puri = closure.PURLConnection.ResponsePURL;
     _requestTimedOut = new DispatcherTimer() { Interval = EXTRACTION_TIMEOUT };
     _requestTimedOut.Tick += ExtractionRequestTimedOut;
 }
Пример #8
0
 public Item(String pTitle, String pDescription, ParsedUri pLink, String pGuid, String pAuthor, List<String> pCategorySet )
 {
     title = pTitle;
     description = pDescription;
     link = pLink;
     guid = pGuid;
     author = pAuthor;
     categorySet = pCategorySet;
 }
        public virtual async Task<Document> GetDocument(ParsedUri location)
        {
            if (location == null)
                return null;

            Document doc;
            GlobalDocumentCollection.TryGetDocument(location, out doc);

            return doc;
        }
        private async void BtnGetMetadata_Click(object sender, RoutedEventArgs e)
        {
            //            string serializedMetadata = UrlBox.Text;
            //            Object o = RepositoryMetadataTranslationScope.Get().Deserialize(serializedMetadata, StringFormat.Xml);
            //            StringBuilder sb = new StringBuilder();
            //            SimplTypesScope.Serialize(o, sb, StringFormat.Xml);
            //            String reserializedMetadatata = sb.ToString();
            //            Console.WriteLine(reserializedMetadatata);
            //Expander expander = new Expander();
            //TextBox metadataXML = new TextBox { TextWrapping = TextWrapping.Wrap, MinHeight = 100 };
            //metadataXML.Text = reserializedMetadatata;

            string urls = UrlBox.Text;

            List<Task<Document>> extractionRequests = new List<Task<Document>>();
            Dictionary<ParsedUri, DateTime> timeStamps = new Dictionary<ParsedUri, DateTime>();
            foreach (var s in urls.Split(new string[] { Environment.NewLine }, StringSplitOptions.None))
            {
                Console.WriteLine("Requesting async extraction of: " + s);

                ParsedUri puri = new ParsedUri(s);
                {
                    Task<Document> t = _semanticsSessionScope.GetDocument(puri);
                    timeStamps.Add(puri, DateTime.Now);
                    //Alternate, if you want the document here:
                    //Document doc = await _semanticsSessionScope.GetDocument(puri);

                    extractionRequests.Add(t);

                }
            }

            while (extractionRequests.Count > 0)
            {
                Task<Document> completedTask = await Task.WhenAny(extractionRequests);
                extractionRequests.Remove(completedTask);

                Document parsedDoc = await completedTask;
                if (parsedDoc == null)
                    continue;

                Expander expander = new Expander {Header = parsedDoc.Title};
                TextBox metadataXML = new TextBox {TextWrapping = TextWrapping.Wrap, MinHeight = 100};

                var s = timeStamps[parsedDoc.Location.Value];
                Console.WriteLine(" ---------------------------------- Time to complete: " +
                                  DateTime.Now.Subtract(s).TotalMilliseconds);
                metadataXML.Text = await Task.Run(() => SimplTypesScope.Serialize(parsedDoc, StringFormat.Xml));

                expander.Content = metadataXML;
                MetadataTitleXMLContainer.Children.Add(expander);
            }

        }
      public virtual async Task<Document> GetOrConstructDocument(ParsedUri location)
      {
          var doc = await GetDocument(location);
 
          if(doc == null)
          {
              doc = MetaMetadataRepository.ConstructDocument(location, false);
              if (doc != null)
                  GlobalDocumentCollection.AddDocument(doc, location);
          }
          
          return doc;
      }
        public MetadataServicesClient(SimplTypesScope metadatascope, SemanticsSessionScope semanticSessionScope, ParsedUri serviceUri, bool useWebSockets = false)
        {
            SimplTypesScope[] oodssAndMetadataScope = {metadatascope, DefaultServicesTranslations.Get()};

            _metadataTypeScope = SimplTypesScope.Get("MetadataServicesTranslationScope",
                                                        oodssAndMetadataScope,
                                                        typeof (MetadataRequest),
                                                        typeof (MetadataResponse),
                                                        typeof (SemanticServiceError)
                                                    );

            _serviceBaseUri = serviceUri;

            if (useWebSockets)
            {
                _metadataClient = new WebSocketOODSSClient("127.0.0.1", 2018, _metadataTypeScope, semanticSessionScope);
                _metadataClient.StartAsync();
            }
            
        }
Пример #13
0
        public static ImageClipping ConstructClipping(ParsedUri location, Document sourceDoc, IdeaMacheUser creator)
        {
            var imageClipping = new ImageClipping
            {
                Media = new Image
                {
                    Location = new MetadataParsedURL(location)
                },
                SourceDoc = sourceDoc,
                CreativeActs = new List<CreativeAct>(),

            };
            imageClipping.CreativeActs.Add(new CreativeAct
            {
                Action = CreativeAct.CreativeAction.CurateClipping,
                Time = new MetadataDate(DateTime.UtcNow),
                Creator = creator
            });

            return imageClipping;
        }
Пример #14
0
        public ParsedUri Perform(ParsedUri input)
        {
            ParsedUri result = null;
            if (input != null && match != null)
            {
                String str	= input.ToString();
                Match matcher	= match.Match(str);
                if (matcher.Success)
                {
                    if (replace == null)
                        replace	= "";
                    Debug.WriteLine(matcher.Groups[0].Value + " " + matcher.Groups[1].Value);
                    String rez = input.ToString().Replace(matcher.Groups[0].Value, replace);
                    for (int i = 1; i < matcher.Groups.Count; i++)
                        rez = rez.Replace("$" + i, matcher.Groups[i].Value);

                    result = new ParsedUri(rez);
                }
            }
            return result;
        }
        private async void ShowFeeds()
        {
            Feeds.Clear();
            if (FeedURLs != null)
            {
                foreach (string url in FeedURLs.Split(new char[] { '\n', '\r' }, StringSplitOptions.RemoveEmptyEntries))
                {
                    var puri = new ParsedUri(url);
                    var parsedDoc = await _sessionScope.GetDocument(puri);

                    var feedViewModels = FeedViewModelFactory.GetFeedViewModels(parsedDoc);
                    foreach (var viewModel in feedViewModels)
                    {
                        Feeds.Add(viewModel);
                    }
                    this.NotifyPropertyChanged("Feeds");

                }
            }
        }
Пример #16
0
 /// <summary>
 /// 
 /// 
 /// </summary>
 /// <param name="baseUri"></param>
 /// <param name="uri"></param>
 public ParsedUri(ParsedUri baseUri, String uri)
     : base(baseUri, uri)
 {
 }
Пример #17
0
 /// <summary>
 ///
 ///
 /// </summary>
 /// <param name="baseUri"></param>
 /// <param name="uri"></param>
 public ParsedUri(ParsedUri baseUri, String uri)
     : base(baseUri, uri)
 {
 }
 public async static Task<SemanticsSessionScope> InitAsync(SimplTypesScope metadataTranslationScope, string repoLocation, ParsedUri serviceUri)
 {
     var scope = await Task.Run(() => new SemanticsSessionScope(metadataTranslationScope, repoLocation, serviceUri, null));
     return scope;
 }
Пример #19
0
 public PURLConnection(ParsedUri purl)
 {
     PURL = purl;
 }
 public override async Task<Document> GetOrConstructDocument(ParsedUri location)
 {
     Document doc = await base.GetOrConstructDocument(location);
     doc.SemanticsSessionScope = this;
     return doc;
 }
Пример #21
0
 ///<summary>
 /// Get the old location from this.
 /// Set the location of this to the newLocation.
 /// Add a mapping in the GlobalCollection from newLocation to this.
 /// Add the old location for this as an additionalLocation for this.
 ///</summary>
 public void ChangeLocation(ParsedUri newLocation)
 {
     if (newLocation != null)
     {
         MetadataParsedURL origLocation = Location;
         if (!origLocation.Value.Equals(newLocation))
         {
             Location = new MetadataParsedURL(newLocation);
             if (SemanticsSessionScope != null)
                 SemanticsSessionScope.GlobalDocumentCollection.AddDocument(this, newLocation);
             AddAdditionalLocation(origLocation);
         }
     }
 }
        public string GetUriLeftPart(ParsedUri parsedUri)
        {

            var urlWithQuery = parsedUri.GetComponents(UriComponents.HttpRequestUrl, UriFormat.UriEscaped);
            var query = parsedUri.GetComponents(UriComponents.Query, UriFormat.UriEscaped);
            var index = urlWithQuery.IndexOf(query);
            string urlWithoutQuery = urlWithQuery;
            if (index != -1)
            {
                urlWithoutQuery = urlWithQuery.Remove(index);
            }
            return urlWithoutQuery;
        }
        // throws IOException
        ///<summary>
        /// Extract location ParsedURL parameters into a HashMap.
        /// Let ParamOps operate on this Map.
        /// Derive a new ParsedURL using the base of the original location's ParsedURL and the transformed parameter map.
        /// If the new ParsedURL is different than the old one, make the old one an additional location for this,
        /// and add the transformed ParsedURL to the DocumentLocationMap.
        /// <p/>
        /// If the location changes, then reconnect to the new one.
        ///</summary>
        public override Object Perform(Object obj)
        {
            bool usingThisDoc = true; // if we are changing this document's location, or its child field's location
            Document document = (Document) SemanticOperationHandler.SemanticOperationVariableMap.Get(SemanticOperationKeyWords.Metadata);
            if (ObjectStr != null)
            {
                Object o = SemanticOperationHandler.SemanticOperationVariableMap.Get(ObjectStr);
                if (o != null && o is Document)
                {
                    document = (Document) o;
                    usingThisDoc = false;
                }
            }

            ParsedUri origLocation = document.Location.value;
            if (origLocation.IsFile)
            {
                Debug.WriteLine("Not doing <filter_location> because this is a file: " + origLocation);
                return null;
            }

            SemanticsGlobalCollection<Document> globalCollection = SemanticOperationHandler.SemanticsScope.GlobalDocumentCollection;
            bool locationChanged = false;
            if (paramOps != null && paramOps.Count > 0)
            {
                Dictionary<String, String> parametersMap = new Dictionary<string, string>();

                foreach (String param in origLocation.Query.Split('&'))
                {
                    parametersMap.Add(param.Split('=')[0], param.Split('=')[1]);
                }

                if (parametersMap.Count == 0)
                    parametersMap = new Dictionary<string, string>(paramOps.Count);
                foreach (ParamOp paramOp in paramOps)
                {
                    paramOp.SemanticHandler = SemanticOperationHandler;
                    paramOp.TransformParams(parametersMap);
                }

                String str = origLocation.AbsolutePath + "?";
                foreach (KeyValuePair<string, string> pair in parametersMap)
                {
                    str += pair.Key + "=" + pair.Value + "&";
                }
                if (str[str.Length - 1] == '&')
                    str = str.Substring(0, str.Length - 1);

                ParsedUri transformedLocation = new ParsedUri(str);

                if (origLocation != transformedLocation)
                {
                    document.Location.Value = transformedLocation;

                    locationChanged			= true;
                }
            }
            if (alternativeHosts != null)
            {
                String origHost = origLocation.Host;
                foreach (String alternativeHost in alternativeHosts)
                {
                    if (!origHost.Equals(alternativeHost))
                    {
                        ParsedUri newLocation = new ParsedUri(origLocation.AbsoluteUri.Replace(origLocation.Host, alternativeHost));
                        document.AddAdditionalLocation(new MetadataParsedURL(newLocation));
                        globalCollection.AddDocument(document, newLocation);
                    }
                }
            }
            if (regex != null)
            {
                ParsedUri location = document.Location.Value;
                ParsedUri regexURL = regex.Perform(location);

                document.ChangeLocation(regexURL);

                locationChanged	= true;
            }
            if (stripPrefix != null)
            {
                String origlLocationString	= origLocation.ToString();
                int index	= origlLocationString.IndexOf(stripPrefix);
                if (index > 6)
                {
                    String newLocationString = origlLocationString.Substring(0, index);
                    ParsedUri newLocation = new ParsedUri(newLocationString);
                    if (newLocation != null)
                    {
                        document.ChangeLocation(newLocation);
                    }
                }
            }
            if (locationChanged && usingThisDoc) // if we are just changing the location of a field, we don't have to reconnect.
            {
                if (documentParser != null)
                {
                    //documentParser.reConnect(); // changed the location, so we better connect again!
                }
                else
                {
                    //imageClosure.RequestMetadata();
                }
            }
            return null;
        }
Пример #24
0
 public PURLConnection(ParsedUri purl)
 {
     PURL = purl;
 }
Пример #25
0
        public async Task<object> DeserializeUri(ParsedUri uri, Format format = Format.Xml, TranslationContext context = null, IDeserializationHookStrategy deserializationHookStrategy = null)
        {
            object result = null;
            
            var request = WebRequest.Create(uri);
            if (request != null)
            {
                WebResponse response = await request.GetResponseAsync();
                result = Deserialize(response.GetResponseStream(), context, deserializationHookStrategy, format);
            }

            return result;
        }
        public bool ProcessRedirect(Uri redirectedUri)
        {
            MetadataParsedURL originalMetadataPURL = originalDocument.Location;
            ParsedUri originalPURL = originalMetadataPURL == null ? null : originalMetadataPURL.Value;
            ParsedUri redirectedPURL = new ParsedUri(redirectedUri.AbsoluteUri);
            Debug.WriteLine("try redirecting: " + originalPURL + " > " + redirectedPURL);

            /*Document redirectedDocument = semanticsSessionScope.GetOrConstructDocument(redirectedPURL);
            // note FIXME currently, GetOrConstructDocument does not return null!
            if (redirectedDocument != null)
            {
                // existing document: the redirected url has been visited already.

                if (originalPURL != redirectedPURL)
                    redirectedDocument.AddAdditionalLocation(originalMetadataPURL);

                // TODO -- copy metadata from originalDocument?!!

                documentClosure.ChangeDocument(redirectedDocument);

                // TODO -- reconnect

                return true;
            }
            else
            {
                // TODO -- redirect to a new location.
            }*/

            // TODO -- what if redirectedDocument is already in the queue or being downloaded?

            return false;
        }
Пример #27
0
 public TranslationContext(ParsedUri baseUri)
 {
     _baseDirPurl = baseUri;
 }
 public string GetUriLeftPart(ParsedUri parsedUri)
 {
     return parsedUri.GetLeftPart(UriPartial.Path);
 }