Example #1
0
        public void ImportTime_query_arg_forces_import_property_creation()
        {
            // Log in as ADMIN
            Plug p = Utils.BuildPlugForAdmin();

            // Create a page
            string id   = null;
            string path = null;

            PageUtils.CreateRandomPage(p, out id, out path);

            // Set importTime property to page
            var importTime = DateTime.UtcNow.ToString("yyyyMMddHHmmss");

            p.At("pages", id, "contents")
            .With("edittime", DateTime.MaxValue.ToString("yyyyMMddHHmmss"))
            .With("redirects", "0")
            .With("importtime", importTime)
            .Post(DreamMessage.Ok(MimeType.TEXT_UTF8, "foo"), new Result <DreamMessage>()).Wait();

            // Retrieve property and assert the dates match
            var msg = p.At("pages", id, "properties", XUri.EncodeSegment("mindtouch.import#info")).Get(new Result <DreamMessage>()).Wait();

            Assert.IsTrue(msg.IsSuccessful);
            Assert.AreEqual(importTime, (msg.ToDocument()["date.modified"].AsDate ?? DateTime.MinValue).ToString("yyyyMMddHHmmss"), "Unexpected date");
        }
Example #2
0
        private XUri BuildLogHref(string sortedTerms, DateTime since, DateTime before)
        {
            var href = _apiUri.At("site", "query", "log", "=" + XUri.EncodeSegment(sortedTerms))
                       .With("since", DbUtils.ToString(since))
                       .With("before", DbUtils.ToString(before));

            return(href);
        }
        public static string UriBuild(
            [DekiScriptParam("base uri")] XUri uri,
            [DekiScriptParam("path segments to append (must a string or list of strings)", true)] object path,
            [DekiScriptParam("query parameters to append", true)] Hashtable args
            )
        {
            if (path is string)
            {
                uri = uri.AtPath((string)path);
            }
            else if (path is ArrayList)
            {
                foreach (string segment in (ArrayList)path)
                {
                    uri = uri.At(XUri.EncodeSegment(segment));
                }
            }
            if (args != null)
            {
                foreach (DictionaryEntry entry in args)
                {
                    string key = (string)entry.Key;

                    // remove existing parameter
                    uri = uri.WithoutParams(key);

                    // check if entry is a list of values
                    if (entry.Value is ArrayList)
                    {
                        foreach (var value in (ArrayList)entry.Value)
                        {
                            uri = uri.With(key, SysUtil.ChangeType <string>(value));
                        }
                    }
                    else if (entry.Value != null)
                    {
                        uri = uri.With(key, SysUtil.ChangeType <string>(entry.Value));
                    }
                }
            }
            return(uri.ToString());
        }
Example #4
0
        /// <summary>
        /// Retrieves the title as a ui uri segment (ui urls can be used to access pages by pre-pending the ui host).  The output is db encoded.
        /// Ex. "index.php?title=User:Admin/MyPage", "User:Admin/MyPage"
        /// </summary>
        public string AsUiUriPath(bool forceUseIndexPhp)
        {
            string path;
            string dbPath = XUri.Decode(AsUnprefixedDbPath());

            if (forceUseIndexPhp)
            {
                path = INDEX_PHP_TITLE + AppendFilenameQueryAnchor(NSAndPathToString(Namespace, XUri.EncodeQuery(dbPath).Replace("+", "%2b")), true);
            }
            else
            {
                // returns the title in a wiki ui consumable form
                StringBuilder pathBuilder = new StringBuilder();
                string[]      segments    = AsDbSegments(dbPath);
                bool          useIndexPhp = false;
                foreach (string segment in segments)
                {
                    if (-1 < segment.IndexOfAny(INDEX_PHP_CHARS))
                    {
                        useIndexPhp = true;
                        break;
                    }
                    if (0 < pathBuilder.Length)
                    {
                        pathBuilder.Append("/");
                    }
                    pathBuilder.Append(XUri.EncodeSegment(segment));
                }
                if (useIndexPhp)
                {
                    path = INDEX_PHP_TITLE + AppendFilenameQueryAnchor(NSAndPathToString(Namespace, XUri.EncodeQuery(dbPath).Replace("+", "%2b")), true);
                }
                else
                {
                    path = AppendFilenameQueryAnchor(NSAndPathToString(Namespace, pathBuilder.ToString().Replace("?", "%3f")), false);
                }
            }

            // URL encode ?'s so they don't get confused for the query and add the filename, anchor and query
            return(path);
        }
Example #5
0
        //--- Constructors ---
        public S3Storage(XDoc configuration, ILog log)
        {
            _timerFactory = TaskTimerFactory.Create(this);
            _log          = log;
            _publicKey    = configuration["publickey"].AsText;
            _privateKey   = configuration["privatekey"].AsText;
            _bucket       = configuration["bucket"].AsText;
            _prefix       = configuration["prefix"].AsText;
            if (string.IsNullOrEmpty(_publicKey))
            {
                throw new ArgumentException("Invalid Amazon S3 publickey");
            }
            if (string.IsNullOrEmpty(_privateKey))
            {
                throw new ArgumentException("Invalid Amazon S3 privatekey");
            }
            if (string.IsNullOrEmpty(_bucket))
            {
                throw new ArgumentException("Invalid Amazon S3 bucket");
            }
            if (string.IsNullOrEmpty(_prefix))
            {
                throw new ArgumentException("Invalid Amazon S3 prefix");
            }
            _tempDirectory = Path.Combine(Path.GetTempPath(), "s3_cache_" + XUri.EncodeSegment(_prefix));
            if (Directory.Exists(_tempDirectory))
            {
                Directory.Delete(_tempDirectory, true);
            }
            Directory.CreateDirectory(_tempDirectory);
            _allowRedirects  = configuration["allowredirects"].AsBool ?? false;
            _redirectTimeout = TimeSpan.FromSeconds(configuration["redirecttimeout"].AsInt ?? 60);
            _cacheTtl        = (configuration["cachetimeout"].AsInt ?? 60 * 60).Seconds();

            // initialize S3 plug
            _s3 = Plug.New("http://s3.amazonaws.com", TimeSpan.FromSeconds(configuration["timeout"].AsDouble ?? DEFAUTL_S3_TIMEOUT)).WithPreHandler(S3AuthenticationHeader).At(_bucket);
        }
Example #6
0
 public void EncodeSegment()
 {
     Assert.AreEqual("a^b", XUri.EncodeSegment("a^b"));
 }
Example #7
0
        private Yield GetCache(uint pageId, string wikiId, DateTime time, CultureInfo culture, Result <PageChangeCacheData> result)
        {
            // Note (arnec): going back 10 seconds before event, because timestamps in a request are not currently synced
            string keytime = time.ToString("yyyyMMddHHmm");
            string since   = time.Subtract(TimeSpan.FromSeconds(10)).ToString("yyyyMMddHHmmss");
            PageChangeCacheData cacheData;
            string key = string.Format("{0}:{1}:{2}:{3}", pageId, wikiId, keytime, culture);

            _log.DebugFormat("getting data for key: {0}", key);
            lock (_cache) {
                if (_cache.TryGetValue(key, out cacheData))
                {
                    result.Return(cacheData);
                    yield break;
                }
            }

            // fetch the page data
            Result <DreamMessage> pageResponse;

            yield return(pageResponse = _deki
                                        .At("pages", pageId.ToString())
                                        .WithHeader("X-Deki-Site", "id=" + wikiId)
                                        .With("redirects", "0").GetAsync());

            if (!pageResponse.Value.IsSuccessful)
            {
                _log.WarnFormat("Unable to fetch page '{0}' info: {1}", pageId, pageResponse.Value.Status);
                result.Return((PageChangeCacheData)null);
                yield break;
            }
            XDoc   page          = pageResponse.Value.ToDocument();
            string title         = page["title"].AsText;
            XUri   pageUri       = page["uri.ui"].AsUri;
            string pageUriString = CleanUriForEmail(pageUri);
            string unsubUri      = CleanUriForEmail(pageUri
                                                    .WithoutPathQueryFragment()
                                                    .At("index.php")
                                                    .With("title", "Special:PageAlerts")
                                                    .With("id", pageId.ToString()));

            // fetch the revision history
            Result <DreamMessage> feedResponse;

            yield return(feedResponse = _deki
                                        .At("pages", pageId.ToString(), "feed")
                                        .WithHeader("X-Deki-Site", "id=" + wikiId)
                                        .With("redirects", "0")
                                        .With("format", "raw")
                                        .With("since", since)
                                        .GetAsync());

            if (!feedResponse.Value.IsSuccessful)
            {
                _log.WarnFormat("Unable to fetch page '{0}' changes: {1}", pageId, feedResponse.Value.Status);
                result.Return((PageChangeCacheData)null);
                yield break;
            }

            // build the docs
            XDoc feed = feedResponse.Value.ToDocument()["change"];

            if (feed.ListLength == 0)
            {
                _log.WarnFormat("Change feed is empty for page: {0}", pageId);
                result.Return((PageChangeCacheData)null);
                yield break;
            }
            string who    = feed["rc_user_name"].AsText;
            string whoUri = CleanUriForEmail(pageUri.WithoutPathQueryFragment().At(XUri.EncodeSegment("User:"******"rc_comment"].AsText;
                string revisionUri  = CleanUriForEmail(pageUri.With("revision", change["rc_revision"].AsText));
                who    = change["rc_user_name"].AsText;
                whoUri = CleanUriForEmail(pageUri.WithoutPathQueryFragment().At(XUri.EncodeSegment("User:"******"rc_timestamp"].AsText);
                cacheData.Items.Add(item);
            }
            lock (_cache) {
                // even though we override the entry if one was created in the meantime
                // we do the existence check so that we don't set up two expiration timers;
                if (!_cache.ContainsKey(key))
                {
                    _cacheItemCallback(key, () => {
                        lock (_cache) {
                            _cache.Remove(key);
                        }
                    });
                }
                _cache[key] = cacheData;
            }
            result.Return(cacheData);
            yield break;
        }
Example #8
0
        public XDoc StockChart(
            [DekiExtParam("stock ticker symbol")] string symbol
            )
        {
            // check keys
            string app = Config["finance-app-id"].AsText;
            string sig = Config["finance-sig"].AsText;

            if (string.IsNullOrEmpty(app) || string.IsNullOrEmpty(sig))
            {
                return(new XDoc("html").Start("body").Start("span").Attr("style", "color:red;font-weight:bold;").Value("The Yahoo! Finance Application ID or Signature are missing").End().End());
            }

            // create control
            symbol = symbol.ToUpperInvariant();
            XDoc result = new XDoc("html").Start("body").Start("iframe")
                          .Attr("allowtransparency", "true")
                          .Attr("marginwidth", "0")
                          .Attr("marginheight", "0")
                          .Attr("hspace", "0")
                          .Attr("vspace", "0")
                          .Attr("frameborder", "0")
                          .Attr("scrolling", "no")
                          .Attr("src", string.Format("http://api.finance.yahoo.com/instrument/1.0/{0}/badge;chart=1y;quote/HTML?AppID={1}&sig={2}", XUri.EncodeSegment(symbol), XUri.EncodeQuery(app), XUri.EncodeQuery(sig)))
                          .Attr("width", "200px")
                          .Attr("height", "390px")
                          .Start("a").Attr("href", "http://finance.yahoo.com").Value("Yahoo! Finance").End()
                          .Elem("br")
                          .Start("a").Attr("href", "http://finance.yahoo.com/q?s=^GSPC/").Value("Quote for ^GSPC").End()
                          .End().End();

            return(result);
        }
 //--- Class Methods ---
 private static string Encode(string segment)
 {
     return(XUri.EncodeSegment(segment));
 }
Example #10
0
 public void Setup()
 {
     _tempDirectory = Path.Combine(Path.GetTempPath(), "s3_cache_" + XUri.EncodeSegment(PREFIX));
 }
Example #11
0
        private IEnumerator <IYield> UpdatePackages_Helper(Plug api, string wikiId, string apikey, bool force, bool init, Result <XDoc> result)
        {
            try {
                _status = PackageUpdaterStatus.Importing;
                yield return(Coroutine.Invoke(AuthenticateImportUser, api, wikiId, apikey, new Result <Plug>()).Set(x => api = x));

                XDoc license = null;
                yield return(api.At("license").With("apikey", apikey).Get(new Result <XDoc>()).Set(x => license = x));

                var importReport = new XDoc("packages");
                var first        = true;
                foreach (var directory in Directory.GetDirectories(_templatePackagePath))
                {
                    var    directoryName = Path.GetFileName(directory);
                    string restriction   = null;
                    if (directoryName.EqualsInvariantIgnoreCase("public"))
                    {
                        restriction = "Public";
                    }
                    else if (directoryName.EqualsInvariantIgnoreCase("semi-public"))
                    {
                        restriction = "Semi-Public";
                    }
                    else if (directoryName.EqualsInvariantIgnoreCase("private"))
                    {
                        restriction = "Private";
                    }
                    foreach (var package in Directory.GetFiles(directory, "*.mt*").OrderBy(x => x))
                    {
                        var ext = Path.GetExtension(package);
                        if (!(ext.EqualsInvariantIgnoreCase(".mtarc") || ext.EqualsInvariantIgnoreCase(".mtapp")))
                        {
                            continue;
                        }
                        if (!first)
                        {
                            importReport.End();
                        }
                        first = false;
                        importReport.Start("package").Elem("path", package);
                        ArchivePackageReader packageReader;
                        _log.DebugFormat("contemplating import of '{0}' for '{1}'", package, wikiId);
                        try {
                            packageReader = new ArchivePackageReader(package);
                        } catch (Exception e) {
                            SetError(importReport, wikiId, "error", e, "Unable to open package.");
                            continue;
                        }
                        Result <XDoc> manifestResult;
                        yield return(manifestResult = packageReader.ReadManifest(new Result <XDoc>()).Catch());

                        if (manifestResult.HasException)
                        {
                            SetError(importReport, wikiId, "error", manifestResult.Exception, "Unable to read package manifest.");
                            continue;
                        }
                        var manifest = manifestResult.Value;

                        // check for required capabilities
                        var capabilitiesSatisfied = true;
                        foreach (var capability in manifest["capability"])
                        {
                            var capabilityName  = capability["@name"].AsText;
                            var capabilityValue = capability["@value"].AsText.IfNullOrEmpty("enabled");
                            if (string.IsNullOrEmpty(capabilityName))
                            {
                                continue;
                            }
                            var licenseCapabilityValue = DekiLicense.GetCapability(license, capabilityName) ?? "";
                            if (!string.IsNullOrEmpty(capabilityName) && licenseCapabilityValue.EqualsInvariant(capabilityValue))
                            {
                                continue;
                            }
                            capabilitiesSatisfied = false;
                            SetError(importReport, wikiId, "error", null, "Missing capability '{0}' or incorrect capability value, expected '{1}' and got '{2}'.", capabilityName, capabilityValue, licenseCapabilityValue);
                        }
                        if (!capabilitiesSatisfied)
                        {
                            continue;
                        }

                        // add security xml if we are in a restriction enforcing path
                        if (!string.IsNullOrEmpty(restriction))
                        {
                            manifest.Start("security")
                            .Start("permissions.page")
                            .Elem("restriction", restriction)
                            .End()
                            .End();
                        }

                        // if package predates @date.created, take file modified
                        var dateCreated = manifest["@date.created"].AsDate ?? new FileInfo(package).LastWriteTime;

                        // check whether we should import this package
                        var filename    = Path.GetFileName(package);
                        var importOnce  = manifest["@import-once"].AsBool ?? false;
                        var initPackage = manifest["@init-only"].AsBool ?? false;
                        importReport.Elem("name", filename).Attr("date.created", dateCreated).Attr("preserve-local", manifest["@preserve-local"].AsBool ?? false);
                        var importPropertyName = XUri.EncodeSegment(PACKAGE_PROPERTY_NS + filename);
                        if (initPackage && !init && !force)
                        {
                            SetError(importReport, wikiId, "skipped", null, "package '{0}' is an init package and neither init or force flags were set, skipping", package);
                            continue;
                        }
                        if (force && !importOnce)
                        {
                            _log.DebugFormat("force is set and package '{0}' is not marked as oneTimeOnly, proceeding with import", package);
                        }
                        else
                        {
                            DreamMessage propertyResponse = null;
                            yield return(api.At("site", "properties", importPropertyName).Get(new Result <DreamMessage>()).Set(x => propertyResponse = x));

                            if (propertyResponse.IsSuccessful)
                            {
                                var importedPackage     = propertyResponse.ToDocument();
                                var importedPackageDate = importedPackage["date.created"].AsDate;
                                if (!importedPackageDate.HasValue)
                                {
                                    _log.WarnFormat("unable to retrieve imported package date.created for '{0}', treating package as new", package);
                                }
                                else if (dateCreated > importedPackageDate.Value)
                                {
                                    _log.DebugFormat("package '{0}' is newer, proceed with import ({1} > {2}", package, dateCreated, importedPackageDate.Value);
                                }
                                else
                                {
                                    SetError(importReport, wikiId, "skipped", null,
                                             "package '{0}' is not newer, skip import ({1} <= {2}.)",
                                             package,
                                             dateCreated,
                                             importedPackageDate.Value);
                                    continue;
                                }
                            }
                            else if (propertyResponse.Status == DreamStatus.Unauthorized)
                            {
                                _log.WarnFormat("apiuri has lost its authentication, dropping out");
                                throw new UnauthorizedAccessException("Authentication for was lost");
                            }
                            else
                            {
                                _log.DebugFormat("package '{0}' has not previously been imported, proceeding with import", package);
                            }
                        }

                        // import package
                        Result <Importer> importerResult;
                        yield return(importerResult = Importer.CreateAsync(api, manifest, "/", new Result <Importer>()).Catch());

                        if (importerResult.HasException)
                        {
                            SetError(importReport, wikiId, "error", importerResult.Exception, "Unable to create importer for package.");
                            continue;
                        }
                        var    importer      = importerResult.Value;
                        var    importManager = new ImportManager(importer, packageReader);
                        Result importResult;
                        yield return(importResult = importManager.ImportAsync(new Result()).Catch());

                        if (importResult.HasException)
                        {
                            SetError(importReport, wikiId, "error", importResult.Exception, "Import did not complete successfully.");
                            continue;
                        }

                        // write import data as site property
                        yield return(api.At("site", "properties", importPropertyName)
                                     .With("abort", "never")
                                     .With("description", string.Format("Import of package '{0}'", filename))
                                     .Put(new XDoc("package").Elem("date.created", dateCreated), new Result <DreamMessage>()));

                        importReport.Start("status").Attr("code", "ok").End();
                        _log.DebugFormat("sucessfully imported package '{0}'", package);
                    }
                }
                if (!first)
                {
                    importReport.End();
                }
                result.Return(importReport);
                yield break;
            } finally {
                _status = PackageUpdaterStatus.Idle;
            }
        }
Example #12
0
 private DreamMessage DirectoryGetWikiIdByHostname(string hostname)
 {
     return(CachedRequest(_directory.At("=" + hostname), XUri.EncodeSegment(hostname) + "-wikiid.xml"));
 }
Example #13
0
 private string GetUriTitle(string title)
 {
     return(XUri.EncodeSegment(title.ReplaceAll("-", "_", " ", "_")));
 }