public void ImportTime_query_arg_forces_import_property_creation()
{
// Log in as ADMIN
Plug p = Utils.BuildPlugForAdmin();
// Create a page
string id = null;
string path = null;
PageUtils.CreateRandomPage(p, out id, out path);
// Set importTime property to page
var importTime = DateTime.UtcNow.ToString("yyyyMMddHHmmss");
p.At("pages", id, "contents")
.With("edittime", DateTime.MaxValue.ToString("yyyyMMddHHmmss"))
.With("redirects", "0")
.With("importtime", importTime)
.Post(DreamMessage.Ok(MimeType.TEXT_UTF8, "foo"), new Result <DreamMessage>()).Wait();
// Retrieve property and assert the dates match
var msg = p.At("pages", id, "properties", XUri.EncodeSegment("mindtouch.import#info")).Get(new Result <DreamMessage>()).Wait();
Assert.IsTrue(msg.IsSuccessful);
Assert.AreEqual(importTime, (msg.ToDocument()["date.modified"].AsDate ?? DateTime.MinValue).ToString("yyyyMMddHHmmss"), "Unexpected date");
}
private XUri BuildLogHref(string sortedTerms, DateTime since, DateTime before)
{
var href = _apiUri.At("site", "query", "log", "=" + XUri.EncodeSegment(sortedTerms))
.With("since", DbUtils.ToString(since))
.With("before", DbUtils.ToString(before));
return(href);
}
public static string UriBuild(
[DekiScriptParam("base uri")] XUri uri,
[DekiScriptParam("path segments to append (must a string or list of strings)", true)] object path,
[DekiScriptParam("query parameters to append", true)] Hashtable args
)
{
if (path is string)
{
uri = uri.AtPath((string)path);
}
else if (path is ArrayList)
{
foreach (string segment in (ArrayList)path)
{
uri = uri.At(XUri.EncodeSegment(segment));
}
}
if (args != null)
{
foreach (DictionaryEntry entry in args)
{
string key = (string)entry.Key;
// remove existing parameter
uri = uri.WithoutParams(key);
// check if entry is a list of values
if (entry.Value is ArrayList)
{
foreach (var value in (ArrayList)entry.Value)
{
uri = uri.With(key, SysUtil.ChangeType <string>(value));
}
}
else if (entry.Value != null)
{
uri = uri.With(key, SysUtil.ChangeType <string>(entry.Value));
}
}
}
return(uri.ToString());
}
/// <summary>
/// Retrieves the title as a ui uri segment (ui urls can be used to access pages by pre-pending the ui host). The output is db encoded.
/// Ex. "index.php?title=User:Admin/MyPage", "User:Admin/MyPage"
/// </summary>
public string AsUiUriPath(bool forceUseIndexPhp)
{
string path;
string dbPath = XUri.Decode(AsUnprefixedDbPath());
if (forceUseIndexPhp)
{
path = INDEX_PHP_TITLE + AppendFilenameQueryAnchor(NSAndPathToString(Namespace, XUri.EncodeQuery(dbPath).Replace("+", "%2b")), true);
}
else
{
// returns the title in a wiki ui consumable form
StringBuilder pathBuilder = new StringBuilder();
string[] segments = AsDbSegments(dbPath);
bool useIndexPhp = false;
foreach (string segment in segments)
{
if (-1 < segment.IndexOfAny(INDEX_PHP_CHARS))
{
useIndexPhp = true;
break;
}
if (0 < pathBuilder.Length)
{
pathBuilder.Append("/");
}
pathBuilder.Append(XUri.EncodeSegment(segment));
}
if (useIndexPhp)
{
path = INDEX_PHP_TITLE + AppendFilenameQueryAnchor(NSAndPathToString(Namespace, XUri.EncodeQuery(dbPath).Replace("+", "%2b")), true);
}
else
{
path = AppendFilenameQueryAnchor(NSAndPathToString(Namespace, pathBuilder.ToString().Replace("?", "%3f")), false);
}
}
// URL encode ?'s so they don't get confused for the query and add the filename, anchor and query
return(path);
}
//--- Constructors ---
public S3Storage(XDoc configuration, ILog log)
{
_timerFactory = TaskTimerFactory.Create(this);
_log = log;
_publicKey = configuration["publickey"].AsText;
_privateKey = configuration["privatekey"].AsText;
_bucket = configuration["bucket"].AsText;
_prefix = configuration["prefix"].AsText;
if (string.IsNullOrEmpty(_publicKey))
{
throw new ArgumentException("Invalid Amazon S3 publickey");
}
if (string.IsNullOrEmpty(_privateKey))
{
throw new ArgumentException("Invalid Amazon S3 privatekey");
}
if (string.IsNullOrEmpty(_bucket))
{
throw new ArgumentException("Invalid Amazon S3 bucket");
}
if (string.IsNullOrEmpty(_prefix))
{
throw new ArgumentException("Invalid Amazon S3 prefix");
}
_tempDirectory = Path.Combine(Path.GetTempPath(), "s3_cache_" + XUri.EncodeSegment(_prefix));
if (Directory.Exists(_tempDirectory))
{
Directory.Delete(_tempDirectory, true);
}
Directory.CreateDirectory(_tempDirectory);
_allowRedirects = configuration["allowredirects"].AsBool ?? false;
_redirectTimeout = TimeSpan.FromSeconds(configuration["redirecttimeout"].AsInt ?? 60);
_cacheTtl = (configuration["cachetimeout"].AsInt ?? 60 * 60).Seconds();
// initialize S3 plug
_s3 = Plug.New("http://s3.amazonaws.com", TimeSpan.FromSeconds(configuration["timeout"].AsDouble ?? DEFAUTL_S3_TIMEOUT)).WithPreHandler(S3AuthenticationHeader).At(_bucket);
}
public void EncodeSegment()
{
Assert.AreEqual("a^b", XUri.EncodeSegment("a^b"));
}
private Yield GetCache(uint pageId, string wikiId, DateTime time, CultureInfo culture, Result <PageChangeCacheData> result)
{
// Note (arnec): going back 10 seconds before event, because timestamps in a request are not currently synced
string keytime = time.ToString("yyyyMMddHHmm");
string since = time.Subtract(TimeSpan.FromSeconds(10)).ToString("yyyyMMddHHmmss");
PageChangeCacheData cacheData;
string key = string.Format("{0}:{1}:{2}:{3}", pageId, wikiId, keytime, culture);
_log.DebugFormat("getting data for key: {0}", key);
lock (_cache) {
if (_cache.TryGetValue(key, out cacheData))
{
result.Return(cacheData);
yield break;
}
}
// fetch the page data
Result <DreamMessage> pageResponse;
yield return(pageResponse = _deki
.At("pages", pageId.ToString())
.WithHeader("X-Deki-Site", "id=" + wikiId)
.With("redirects", "0").GetAsync());
if (!pageResponse.Value.IsSuccessful)
{
_log.WarnFormat("Unable to fetch page '{0}' info: {1}", pageId, pageResponse.Value.Status);
result.Return((PageChangeCacheData)null);
yield break;
}
XDoc page = pageResponse.Value.ToDocument();
string title = page["title"].AsText;
XUri pageUri = page["uri.ui"].AsUri;
string pageUriString = CleanUriForEmail(pageUri);
string unsubUri = CleanUriForEmail(pageUri
.WithoutPathQueryFragment()
.At("index.php")
.With("title", "Special:PageAlerts")
.With("id", pageId.ToString()));
// fetch the revision history
Result <DreamMessage> feedResponse;
yield return(feedResponse = _deki
.At("pages", pageId.ToString(), "feed")
.WithHeader("X-Deki-Site", "id=" + wikiId)
.With("redirects", "0")
.With("format", "raw")
.With("since", since)
.GetAsync());
if (!feedResponse.Value.IsSuccessful)
{
_log.WarnFormat("Unable to fetch page '{0}' changes: {1}", pageId, feedResponse.Value.Status);
result.Return((PageChangeCacheData)null);
yield break;
}
// build the docs
XDoc feed = feedResponse.Value.ToDocument()["change"];
if (feed.ListLength == 0)
{
_log.WarnFormat("Change feed is empty for page: {0}", pageId);
result.Return((PageChangeCacheData)null);
yield break;
}
string who = feed["rc_user_name"].AsText;
string whoUri = CleanUriForEmail(pageUri.WithoutPathQueryFragment().At(XUri.EncodeSegment("User:"******"rc_comment"].AsText;
string revisionUri = CleanUriForEmail(pageUri.With("revision", change["rc_revision"].AsText));
who = change["rc_user_name"].AsText;
whoUri = CleanUriForEmail(pageUri.WithoutPathQueryFragment().At(XUri.EncodeSegment("User:"******"rc_timestamp"].AsText);
cacheData.Items.Add(item);
}
lock (_cache) {
// even though we override the entry if one was created in the meantime
// we do the existence check so that we don't set up two expiration timers;
if (!_cache.ContainsKey(key))
{
_cacheItemCallback(key, () => {
lock (_cache) {
_cache.Remove(key);
}
});
}
_cache[key] = cacheData;
}
result.Return(cacheData);
yield break;
}
public XDoc StockChart(
[DekiExtParam("stock ticker symbol")] string symbol
)
{
// check keys
string app = Config["finance-app-id"].AsText;
string sig = Config["finance-sig"].AsText;
if (string.IsNullOrEmpty(app) || string.IsNullOrEmpty(sig))
{
return(new XDoc("html").Start("body").Start("span").Attr("style", "color:red;font-weight:bold;").Value("The Yahoo! Finance Application ID or Signature are missing").End().End());
}
// create control
symbol = symbol.ToUpperInvariant();
XDoc result = new XDoc("html").Start("body").Start("iframe")
.Attr("allowtransparency", "true")
.Attr("marginwidth", "0")
.Attr("marginheight", "0")
.Attr("hspace", "0")
.Attr("vspace", "0")
.Attr("frameborder", "0")
.Attr("scrolling", "no")
.Attr("src", string.Format("http://api.finance.yahoo.com/instrument/1.0/{0}/badge;chart=1y;quote/HTML?AppID={1}&sig={2}", XUri.EncodeSegment(symbol), XUri.EncodeQuery(app), XUri.EncodeQuery(sig)))
.Attr("width", "200px")
.Attr("height", "390px")
.Start("a").Attr("href", "http://finance.yahoo.com").Value("Yahoo! Finance").End()
.Elem("br")
.Start("a").Attr("href", "http://finance.yahoo.com/q?s=^GSPC/").Value("Quote for ^GSPC").End()
.End().End();
return(result);
}
//--- Class Methods ---
private static string Encode(string segment)
{
return(XUri.EncodeSegment(segment));
}
public void Setup()
{
_tempDirectory = Path.Combine(Path.GetTempPath(), "s3_cache_" + XUri.EncodeSegment(PREFIX));
}
private IEnumerator <IYield> UpdatePackages_Helper(Plug api, string wikiId, string apikey, bool force, bool init, Result <XDoc> result)
{
try {
_status = PackageUpdaterStatus.Importing;
yield return(Coroutine.Invoke(AuthenticateImportUser, api, wikiId, apikey, new Result <Plug>()).Set(x => api = x));
XDoc license = null;
yield return(api.At("license").With("apikey", apikey).Get(new Result <XDoc>()).Set(x => license = x));
var importReport = new XDoc("packages");
var first = true;
foreach (var directory in Directory.GetDirectories(_templatePackagePath))
{
var directoryName = Path.GetFileName(directory);
string restriction = null;
if (directoryName.EqualsInvariantIgnoreCase("public"))
{
restriction = "Public";
}
else if (directoryName.EqualsInvariantIgnoreCase("semi-public"))
{
restriction = "Semi-Public";
}
else if (directoryName.EqualsInvariantIgnoreCase("private"))
{
restriction = "Private";
}
foreach (var package in Directory.GetFiles(directory, "*.mt*").OrderBy(x => x))
{
var ext = Path.GetExtension(package);
if (!(ext.EqualsInvariantIgnoreCase(".mtarc") || ext.EqualsInvariantIgnoreCase(".mtapp")))
{
continue;
}
if (!first)
{
importReport.End();
}
first = false;
importReport.Start("package").Elem("path", package);
ArchivePackageReader packageReader;
_log.DebugFormat("contemplating import of '{0}' for '{1}'", package, wikiId);
try {
packageReader = new ArchivePackageReader(package);
} catch (Exception e) {
SetError(importReport, wikiId, "error", e, "Unable to open package.");
continue;
}
Result <XDoc> manifestResult;
yield return(manifestResult = packageReader.ReadManifest(new Result <XDoc>()).Catch());
if (manifestResult.HasException)
{
SetError(importReport, wikiId, "error", manifestResult.Exception, "Unable to read package manifest.");
continue;
}
var manifest = manifestResult.Value;
// check for required capabilities
var capabilitiesSatisfied = true;
foreach (var capability in manifest["capability"])
{
var capabilityName = capability["@name"].AsText;
var capabilityValue = capability["@value"].AsText.IfNullOrEmpty("enabled");
if (string.IsNullOrEmpty(capabilityName))
{
continue;
}
var licenseCapabilityValue = DekiLicense.GetCapability(license, capabilityName) ?? "";
if (!string.IsNullOrEmpty(capabilityName) && licenseCapabilityValue.EqualsInvariant(capabilityValue))
{
continue;
}
capabilitiesSatisfied = false;
SetError(importReport, wikiId, "error", null, "Missing capability '{0}' or incorrect capability value, expected '{1}' and got '{2}'.", capabilityName, capabilityValue, licenseCapabilityValue);
}
if (!capabilitiesSatisfied)
{
continue;
}
// add security xml if we are in a restriction enforcing path
if (!string.IsNullOrEmpty(restriction))
{
manifest.Start("security")
.Start("permissions.page")
.Elem("restriction", restriction)
.End()
.End();
}
// if package predates @date.created, take file modified
var dateCreated = manifest["@date.created"].AsDate ?? new FileInfo(package).LastWriteTime;
// check whether we should import this package
var filename = Path.GetFileName(package);
var importOnce = manifest["@import-once"].AsBool ?? false;
var initPackage = manifest["@init-only"].AsBool ?? false;
importReport.Elem("name", filename).Attr("date.created", dateCreated).Attr("preserve-local", manifest["@preserve-local"].AsBool ?? false);
var importPropertyName = XUri.EncodeSegment(PACKAGE_PROPERTY_NS + filename);
if (initPackage && !init && !force)
{
SetError(importReport, wikiId, "skipped", null, "package '{0}' is an init package and neither init or force flags were set, skipping", package);
continue;
}
if (force && !importOnce)
{
_log.DebugFormat("force is set and package '{0}' is not marked as oneTimeOnly, proceeding with import", package);
}
else
{
DreamMessage propertyResponse = null;
yield return(api.At("site", "properties", importPropertyName).Get(new Result <DreamMessage>()).Set(x => propertyResponse = x));
if (propertyResponse.IsSuccessful)
{
var importedPackage = propertyResponse.ToDocument();
var importedPackageDate = importedPackage["date.created"].AsDate;
if (!importedPackageDate.HasValue)
{
_log.WarnFormat("unable to retrieve imported package date.created for '{0}', treating package as new", package);
}
else if (dateCreated > importedPackageDate.Value)
{
_log.DebugFormat("package '{0}' is newer, proceed with import ({1} > {2}", package, dateCreated, importedPackageDate.Value);
}
else
{
SetError(importReport, wikiId, "skipped", null,
"package '{0}' is not newer, skip import ({1} <= {2}.)",
package,
dateCreated,
importedPackageDate.Value);
continue;
}
}
else if (propertyResponse.Status == DreamStatus.Unauthorized)
{
_log.WarnFormat("apiuri has lost its authentication, dropping out");
throw new UnauthorizedAccessException("Authentication for was lost");
}
else
{
_log.DebugFormat("package '{0}' has not previously been imported, proceeding with import", package);
}
}
// import package
Result <Importer> importerResult;
yield return(importerResult = Importer.CreateAsync(api, manifest, "/", new Result <Importer>()).Catch());
if (importerResult.HasException)
{
SetError(importReport, wikiId, "error", importerResult.Exception, "Unable to create importer for package.");
continue;
}
var importer = importerResult.Value;
var importManager = new ImportManager(importer, packageReader);
Result importResult;
yield return(importResult = importManager.ImportAsync(new Result()).Catch());
if (importResult.HasException)
{
SetError(importReport, wikiId, "error", importResult.Exception, "Import did not complete successfully.");
continue;
}
// write import data as site property
yield return(api.At("site", "properties", importPropertyName)
.With("abort", "never")
.With("description", string.Format("Import of package '{0}'", filename))
.Put(new XDoc("package").Elem("date.created", dateCreated), new Result <DreamMessage>()));
importReport.Start("status").Attr("code", "ok").End();
_log.DebugFormat("sucessfully imported package '{0}'", package);
}
}
if (!first)
{
importReport.End();
}
result.Return(importReport);
yield break;
} finally {
_status = PackageUpdaterStatus.Idle;
}
}
private DreamMessage DirectoryGetWikiIdByHostname(string hostname)
{
return(CachedRequest(_directory.At("=" + hostname), XUri.EncodeSegment(hostname) + "-wikiid.xml"));
}
private string GetUriTitle(string title)
{
return(XUri.EncodeSegment(title.ReplaceAll("-", "_", " ", "_")));
}
