private void MakeDocument()
{
if (IsClosed)
{
throw new InvalidOperationException("message has already been closed");
}
if (_doc == null)
{
try {
MakeStream();
_doc = XDocFactory.From(_stream, ContentType);
if ((_doc == null) || _doc.IsEmpty)
{
throw new InvalidDataException(string.Format("message body with content type '{0}' is not well-formed xml", ContentType));
}
} finally {
if (_stream != null)
{
_stream.Close();
_stream = null;
_streamOpen = false;
}
}
}
}
public void DiffUnambiguosAddRemoveReplace()
{
// NOTE (steveb): test unamibiguous addition, deletion, and replacement
XDoc a = XDocFactory.From("<p>the quick brown fox jumped over the lazy dog</p>", MimeType.XML);
XDoc b = XDocFactory.From("<p>the slow black fox jumped far over the dog</p>", MimeType.XML);
Tuplet <ArrayDiffKind, XDocDiff.Token>[] diff = XDocDiff.Diff(a, b, 10000);
Match(diff[0], ArrayDiffKind.Same, XmlNodeType.Element, "p");
Match(diff[1], ArrayDiffKind.Same, XmlNodeType.EndElement, string.Empty);
Match(diff[2], ArrayDiffKind.Same, XmlNodeType.Text, "the");
Match(diff[3], ArrayDiffKind.Same, XmlNodeType.Text, " ");
Match(diff[4], ArrayDiffKind.Removed, XmlNodeType.Text, "quick");
Match(diff[5], ArrayDiffKind.Added, XmlNodeType.Text, "slow");
Match(diff[6], ArrayDiffKind.Same, XmlNodeType.Text, " ");
Match(diff[7], ArrayDiffKind.Removed, XmlNodeType.Text, "brown");
Match(diff[8], ArrayDiffKind.Added, XmlNodeType.Text, "black");
Match(diff[9], ArrayDiffKind.Same, XmlNodeType.Text, " ");
Match(diff[10], ArrayDiffKind.Same, XmlNodeType.Text, "fox");
Match(diff[11], ArrayDiffKind.Same, XmlNodeType.Text, " ");
Match(diff[12], ArrayDiffKind.Same, XmlNodeType.Text, "jumped");
Match(diff[13], ArrayDiffKind.Same, XmlNodeType.Text, " ");
Match(diff[14], ArrayDiffKind.Added, XmlNodeType.Text, "far");
Match(diff[15], ArrayDiffKind.Added, XmlNodeType.Text, " ");
Match(diff[16], ArrayDiffKind.Same, XmlNodeType.Text, "over");
Match(diff[17], ArrayDiffKind.Same, XmlNodeType.Text, " ");
Match(diff[18], ArrayDiffKind.Same, XmlNodeType.Text, "the");
Match(diff[19], ArrayDiffKind.Removed, XmlNodeType.Text, " ");
Match(diff[20], ArrayDiffKind.Removed, XmlNodeType.Text, "lazy");
Match(diff[21], ArrayDiffKind.Same, XmlNodeType.Text, " ");
Match(diff[22], ArrayDiffKind.Same, XmlNodeType.Text, "dog");
Match(diff[23], ArrayDiffKind.Same, XmlNodeType.None, "p");
}
//--- Methods ---
protected override void OnStart(string[] args)
{
TimeSpan time;
_sysEventLog.WriteEntry("host service starting", EventLogEntryType.Information);
// TODO (steveb): make settings file name & location configurable (use app settings)
string baseFolder = Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().GetModules()[0].FullyQualifiedName);
string startupFile = Path.Combine(baseFolder, "mindtouch.dream.startup.xml");
XDoc settings = XDoc.Empty;
settings = XDocFactory.LoadFrom(startupFile, MimeType.XML);
if (settings == null)
{
throw new Exception("invalid settings file");
}
// create environment
time = DebugUtil.Stopwatch(delegate() {
_host = new DreamHost(settings);
});
_sysEventLog.WriteEntry(string.Format("ApiKey: {0}", _host.Self.Uri.GetParam("apikey")), EventLogEntryType.Information);
_sysEventLog.WriteEntry(string.Format("initialized {0} secs", time.TotalSeconds), EventLogEntryType.Information);
// execute all scripts
time = DebugUtil.Stopwatch(delegate() {
_host.RunScripts(settings, null);
});
_sysEventLog.WriteEntry(string.Format("ready {0} secs", time.TotalSeconds), EventLogEntryType.Information);
}
public void UnSafeXhtml_cleanses_Video_with_sources()
{
XDoc doc = XDocFactory.From("<doc><video poster=\"\" onloadstart=\"\" onprogress=\"\"><source src=\"\" unsafe=\"\"/></video></doc>", MimeType.TEXT_XML);
DekiScriptLibrary.ValidateXHtml(doc, false, true);
Assert.AreEqual("<doc><video poster=\"\" onloadstart=\"\" onprogress=\"\"><source src=\"\" /></video></doc>", doc.ToString());
}
public void SafeXhtml_cleanses_Audio_element()
{
XDoc doc = XDocFactory.From("<doc><audio src=\"\" poster=\"\" autoplay=\"\" loop=\"\" controls=\"\" onloadstart=\"\" onprogress=\"\"></audio></doc>", MimeType.TEXT_XML);
DekiScriptLibrary.ValidateXHtml(doc, true, true);
Assert.AreEqual("<doc><audio src=\"\" autoplay=\"\" loop=\"\" controls=\"\"></audio></doc>", doc.ToString());
}
public XDoc GadgetScript(
[DekiExtParam("Google gadget script")] string script
)
{
// validate xml
XDoc xml = XDocFactory.From("<html><body>" + script + "</body></html>", MimeType.HTML)["//script"];
if ((xml == null) || !xml.HasName("script"))
{
throw new ArgumentException("Google gadget must contained in a <script> tag");
}
// validate uri
XUri uri = xml["@src"].AsUri ?? XUri.Localhost;
if (!uri.Host.EqualsInvariantIgnoreCase("gmodules.com") && !uri.Host.EqualsInvariantIgnoreCase("www.gmodules.com"))
{
throw new ArgumentException("Google gadget must be originating from gmodules.com");
}
// remove .js output option
uri = uri.WithoutParams("output");
// create <iframe> in which we'll load the gadget
return(NewIFrame(uri, SysUtil.ChangeType <float?>(uri.GetParam("w")), SysUtil.ChangeType <float>(uri.GetParam("h"))));
}
public override DekiAuthenticationService.User GetUser(string user)
{
User result = null;
XDoc doc = XDocFactory.LoadFrom(_path, MimeType.XML);
XDoc userNode = doc[string.Format("users/user[@name=\"{0}\"]", user)];
if (!userNode.IsEmpty)
{
List <Group> groups = GetGroups(doc, string.Format("groups/group[user/@name=\"{0}\"]", user));
if (groups.Count == 0)
{
groups.Add(new Group(string.Empty));
}
result = new User(
userNode["@name"].IsEmpty ? string.Empty : userNode["@name"].AsText,
userNode["email"].IsEmpty ? string.Empty : userNode["email"].AsText,
groups.ToArray()
);
result.Custom["fullname"] = userNode["fullname"].IsEmpty ? string.Empty : userNode["fullname"].AsText;
result.Custom["status"] = userNode["status"].IsEmpty ? "enabled" : userNode["status"].AsText;
}
return(result);
}
private ExpiringHashSet <string> .Entry SyncMeta(string filePath)
{
lock (_expirationEntries) {
DateTime?expire = null;
var ttl = TimeSpan.Zero;
var metaPath = GetMetaPath(filePath);
if (File.Exists(metaPath))
{
var meta = XDocFactory.LoadFrom(metaPath, MimeType.TEXT_XML);
expire = meta["expire.date"].AsDate;
ttl = TimeSpan.FromSeconds(meta["expire.ttl"].AsDouble ?? 0);
}
if (expire.HasValue)
{
// set up expiration
_expirationEntries.SetExpiration(filePath, expire.Value, ttl);
}
else
{
// no expiration anymore, so expiration needs to be removed
_expirationEntries.Delete(filePath);
}
return(_expirationEntries[filePath]);
}
}
private void MigrateLegacyStateFile()
{
// check if state file exists, and convert state to meta files
var statefile = Path.Combine(_path, STATE_FILENAME);
if (!File.Exists(statefile))
{
return;
}
var state = XDocFactory.LoadFrom(statefile, MimeType.XML);
// restore file expiration list
foreach (var entry in state["file"])
{
var filepath = Path.Combine(_path, entry["path"].Contents);
var when = entry["date.expire"].AsDate;
var ttl = TimeSpan.FromSeconds(entry["date.ttl"].AsDouble ?? 0);
if (!File.Exists(filepath) || !when.HasValue)
{
continue;
}
_log.DebugFormat("migrating file meta data. Scheduled for deletion at {0}", when);
WriteMeta(filepath, ttl, when);
}
File.Delete(statefile);
}
public void SafeXhtml_cleanses_Audio_src_attribute()
{
XDoc doc = XDocFactory.From("<doc><audio src=\"javascript://\" poster=\"javascript://\" autoplay=\"\" loop=\"\" controls=\"\"></audio></doc>", MimeType.TEXT_XML);
DekiScriptLibrary.ValidateXHtml(doc, true, true);
Assert.AreEqual("<doc><audio autoplay=\"\" loop=\"\" controls=\"\"></audio></doc>", doc.ToString());
}
private void LoadRecordsFromFileSystem()
{
string storagePath = Config["filestorage-path"].AsText;
if (string.IsNullOrEmpty(storagePath))
{
return;
}
if (!System.IO.Directory.Exists(storagePath))
{
return;
}
string[] files =
System.IO.Directory.GetFiles(storagePath, "*.xml", System.IO.SearchOption.TopDirectoryOnly);
if (files != null)
{
foreach (string file in files)
{
try {
DirectoryRecord record = new DirectoryRecord();
record.Name = System.IO.Path.GetFileNameWithoutExtension(file);
record.Value = XDocFactory.LoadFrom(file, MimeType.XML);
_directory[record.Name] = record;
}
catch (Exception) {
System.IO.File.Delete(file);
}
}
}
}
private object ProcessGraph(string graph, string app, string format)
{
if (string.IsNullOrEmpty(graph))
{
throw new DreamBadRequestException("invalid graph");
}
if (string.IsNullOrEmpty(app) || !File.Exists(app))
{
throw new DreamBadRequestException("application misconfigured or missing");
}
if (format == null)
{
throw new ArgumentNullException("format");
}
// determined the output format
switch (format.ToLowerInvariant())
{
case "png":
return(GetResoucePlug(graph, app, format, MimeType.PNG).Uri);
case "png+map": {
string id = StringUtil.CreateAlphaNumericKey(8);
// TODO (steveb): dot -Tcmapx -oTEMPFILE.map -Tgif -oTEMPFILE.png < source
// generate map file and replace attributes
XDoc map = XDocFactory.From(GetResoucePlug(graph, app, "cmapx", MimeType.XML).Get().AsText(), MimeType.XML);
map["@id"].ReplaceValue(id);
map["@name"].ReplaceValue(id);
// create composite result
return(new XDoc("html")
.Start("body")
.Start("img")
.Attr("src", GetResoucePlug(graph, app, "png", MimeType.PNG).Uri)
.Attr("usemap", "#" + id)
.End()
.Add(map)
.End());
}
case "svg":
return(new XDoc("html")
.Start("body")
.Start("embed")
.Attr("src", GetResoucePlug(graph, app, format, MimeType.SVG))
.Attr("type", MimeType.SVG.FullType)
.Attr("pluginspage", "http://www.adobe.com/svg/viewer/install/")
.End()
.End());
case "xdot":
return(GetResoucePlug(graph, app, format, MimeType.TEXT).Get().AsText());
default:
throw new DreamBadRequestException(string.Format("unknown format: {0}", format));
}
}
public void XDocFactory_parse_valid_XML()
{
string xml = "<root>mixed<inner>text</inner></root>";
XDoc doc = XDocFactory.From(xml, MimeType.XML);
Assert.IsFalse(doc.IsEmpty);
Assert.AreEqual(xml, doc.ToString());
}
public void Bug_MT8962_PostPageContentsWithFile()
{
Plug adminPlug = Utils.BuildPlugForAdmin();
// Create random contributor
string username = null;
string userid = null;
DreamMessage msg = UserUtils.CreateRandomUser(adminPlug, "Contributor", "password", out userid, out username);
// Login as user
Plug userPlug = Utils.BuildPlugForUser(username);
// Create a page
string page_id;
string page_path;
msg = PageUtils.CreateRandomPage(userPlug, out page_id, out page_path);
// Create a file
string fileName = FileUtils.CreateRamdomFile(null);
msg = DreamMessage.FromFile(fileName);
fileName = "foo.jpg";
// Upload file to page
msg = userPlug.At("pages", page_id, "files", "=" + fileName).Put(msg);
Assert.AreEqual(DreamStatus.Ok, msg.Status, "PUT request failed");
// update the page
XDoc pageContents = new XDoc("content");
pageContents.Attr("type", "application/x.deki-text");
pageContents.Attr("unsafe", "false");
pageContents.Start("body");
pageContents.Start("img");
pageContents.Attr("class", "internal default");
pageContents.Attr("src.path", "//");
pageContents.Attr("src.filename", fileName);
pageContents.End(); //img
pageContents.End(); //body
msg = userPlug.At("pages", page_id, "contents")
.With("edittime", "now")
.With("redirects", 0)
.With("reltopath", page_path)
.Post(pageContents);
Assert.AreEqual(DreamStatus.Ok, msg.Status, "page creation failed!");
msg = userPlug.At("pages", page_id, "contents")
.With("mode", "view")
.Get();
string contents = msg.AsDocument()["/content/body"].Contents;
XDoc imgDoc = XDocFactory.From(contents, MimeType.XML);
XUri imgSrcUri = XUri.TryParse(imgDoc["@src"].AsText);
Assert.IsNotNull(imgSrcUri, "img src uri is invalid!");
}
public void Can_create_BE_from_XDoc()
{
var doc = new XDoc("test").Elem("foo", StringUtil.CreateAlphaNumericKey(6));
var be = new ResourceContentBE(doc);
var doc2 = XDocFactory.From(be.ToStream(), be.MimeType);
Assert.AreEqual(doc, doc2);
}
public void Pack_and_unpack_manifest()
{
// Arrange
string directory = Path.Combine(Path.GetTempPath(), StringUtil.CreateAlphaNumericKey(6));
Directory.CreateDirectory(directory);
List <XDoc> docs = new List <XDoc>();
docs.Add(new XDoc("doc1").Attr("dataid", "a"));
docs.Add(new XDoc("doc2").Attr("dataid", "b"));
docs.Add(new XDoc("doc3").Attr("dataid", "c"));
List <Tuplet <string, MemoryStream> > data = new List <Tuplet <string, MemoryStream> >();
foreach (XDoc doc in docs)
{
string id = doc["@dataid"].AsText;
data.Add(new Tuplet <string, MemoryStream>(id, new MemoryStream(doc.ToBytes())));
}
XDoc manifest = new XDoc("manifest")
.Start("page").Attr("dataid", "a").End()
.Start("page").Attr("dataid", "b").End()
.Start("page").Attr("dataid", "c").End();
// Act
using (FilePackageWriter packageWriter = new FilePackageWriter(directory)) {
foreach (Tuplet <string, MemoryStream> tuple in data)
{
var item = new ExportItem(tuple.Item1, tuple.Item2, tuple.Item2.Length, new XDoc("item").Elem("path", "abc"));
packageWriter.WriteDataAsync(item, new Result()).Wait();
}
packageWriter.WriteManifest(manifest, new Result()).Wait();
}
XDoc manifest2;
List <XDoc> docs2 = new List <XDoc>();
using (FilePackageReader packageReader = new FilePackageReader(directory)) {
manifest2 = packageReader.ReadManifest(new Result <XDoc>()).Wait();
foreach (XDoc id in manifest2["*/@dataid"])
{
using (ImportItem item = packageReader.ReadData(new ImportItem(id.AsText, null, null), new Result <ImportItem>()).Wait()) {
using (StreamReader reader = new StreamReader(item.Data)) {
docs2.Add(XDocFactory.From(reader, MimeType.TEXT_XML));
}
}
}
}
// Assert
Assert.IsTrue(File.Exists(Path.Combine(directory, "package.xml")));
Assert.AreEqual(ToCanonical(manifest), ToCanonical(manifest2));
Assert.AreEqual(docs.Count, docs2.Count);
foreach (var doc in docs)
{
Assert.IsTrue(docs2.Select(x => x == doc).Any());
}
}
public void GlobalSetup()
{
_log.Debug("setup");
var assemblyPath = @"MindTouch.Deki.Util.Tests.dll";
_inspector = new TypeInspector();
_inspector.InspectAssembly(assemblyPath);
_xmlDocumentation = XDocFactory.LoadFrom(@"MindTouch.Deki.Util.Tests.xml", MimeType.TEXT_XML);
}
public static XDoc GenerateLicense(string[] licenseArgs)
{
Init();
Tuplet <int, Stream, Stream> exitValues = CallLicenseGenerator(licenseArgs);
Assert.AreEqual(0, exitValues.Item1, "Unexpected return code\n" + GetErrorMsg(exitValues.Item2) + GetErrorMsg(exitValues.Item3));
// Retrieve generated license
return(!File.Exists(_genLicensePath) ? null : XDocFactory.LoadFrom(_genLicensePath, MimeType.XML));
}
public void XDocFactory_parse_HTML_with_entity_codes_and_render_as_XHTML_without_entities()
{
string xhtml = "<html><body>Ω ℵ Ÿ € © ø </body></html>";
XDoc doc = XDocFactory.From(xhtml, MimeType.HTML);
// check if entities were converted
string text = doc["body"].ToXHtml(false);
Assert.AreEqual("<body>Ω ℵ Ÿ € © ø  </body>", text);
}
public void Dom_partial_evaluation_for_foreach2()
{
const string source = "<html xmlns:eval='http://mindtouch.com/2007/dekiscript'><body init='var x = 2'><div foreach='var y in num.series(1, 5)' where='y != x'><eval:expr>y</eval:expr></div></body></html>";
XDoc doc = XDocFactory.From(source, MimeType.XML);
DekiScriptExpression node = DekiScriptParser.Parse(doc);
node = _dekiScriptTester.Runtime.Optimize(node, DekiScriptEvalMode.Evaluate, _dekiScriptTester.Runtime.CreateEnv());
Assert.AreEqual("<html><body><div>1; </div><div>3; </div><div>4; </div><div>5; </div></body></html>", node.ToString());
}
public void MergeConflicting()
{
XDoc org = XDocFactory.From("<p>the <span>brown</span> box</p>", MimeType.XML);
XDoc left = XDocFactory.From("<p>the <em>brown</em> box</p>", MimeType.XML);
XDoc right = XDocFactory.From("<p>the <b>brown</b> fox</p>", MimeType.XML);
var conflict = false;
var result = XDocDiff.Merge(org, left, right, 10000, ArrayMergeDiffPriority.Left, out conflict);
Assert.AreEqual("<p>the <em>brown</em> fox</p>", result.ToString());
}
//--- Class Methods ---
public static XDekiScript LoadFrom(string filename)
{
XDoc doc = XDocFactory.LoadFrom(filename, MimeType.XML);
if (doc == null)
{
return(null);
}
return(new XDekiScript(doc));
}
public void Write_XDoc()
{
var doc = new XDoc("test").Start("content").Value("this is content").End();
var stream = new ChunkedMemoryStream();
doc.WriteTo(stream);
stream.Position = 0;
var newdoc = XDocFactory.From(stream, MimeType.XML);
Assert.AreEqual(doc, newdoc, "xdoc changed during serialization");
}
public FilePackageReader(string packageDirectory)
{
_packageDirectory = packageDirectory;
string path = Path.Combine(_packageDirectory, "package.xml");
if (!File.Exists(path))
{
throw new FileNotFoundException("Unable to locate manifest", path);
}
_package = XDocFactory.LoadFrom(path, MimeType.TEXT_XML);
}
// --- Methods ---
private void LoadDefaults()
{
// load default parameter values from the xml data file
_defaultValues = new NameValueCollection();
try {
XDoc defaultDoc = XDocFactory.LoadFrom("DefaultData.xml", MimeType.XML);
foreach (XDoc defaultValue in defaultDoc["/params/param"])
{
_defaultValues.Add(defaultValue["name"].Contents.Trim(), defaultValue["default"].Contents.Trim());
}
} catch { }
}
private Yield ReadPackage_Helper(Result result)
{
ZipEntry manifestEntry = _zipFile.GetEntry("package.xml");
Result <MemoryStream> readResult;
yield return(readResult = ReadZipStream(manifestEntry, new Result <MemoryStream>()));
using (TextReader reader = new StreamReader(readResult.Value)) {
_package = XDocFactory.From(reader, MimeType.TEXT_XML);
}
result.Return();
}
public object Fetch(
[DekiExtParam("document id")] string id
)
{
// fetch response from cache
CacheEntry result;
lock (_cacheLookup) {
_cacheLookup.TryGetValue(id, out result);
}
// check if we have a cached entry
XDoc document = null;
if (result != null)
{
_log.DebugFormat("cache hit for '{0}'", result.Id);
result.ResetMemoryExpiration();
if (result.Cache != null)
{
_log.DebugFormat("cache data in memory '{0}'", result.Id);
document = XDocFactory.From(result.Cache, MimeType.XML);
}
else
{
// we have the result on disk, so let's fetch it
DreamMessage msg = Storage.At(CACHE_DATA, result.Guid + ".bin").GetAsync().Wait();
if (msg.IsSuccessful)
{
_log.DebugFormat("cache data pulled from disk");
result.Cache = Encoding.UTF8.GetString(msg.AsBytes());
document = XDocFactory.From(result.Cache, MimeType.XML);
}
else
{
_log.DebugFormat("unable to fetch cache data from disk: {0}", msg.Status);
}
}
}
// check if we have a document to convert
if (document != null)
{
try {
DekiScriptList list = (DekiScriptList)DekiScriptLiteral.FromXml(document);
return(list[0]);
} catch {
// the cached entry is corrupted, remove it
Clear(id);
}
}
return(null);
}
private XDoc LoadExportDocumentFromFile(string exportDocumentPath)
{
if (!File.Exists(exportDocumentPath))
{
throw new ConfigurationException("No such export document: {0}", exportDocumentPath);
}
try {
return(XDocFactory.LoadFrom(exportDocumentPath, MimeType.TEXT_XML));
} catch (Exception e) {
throw new ConfigurationException("Unable to load '{0}': {1}", exportDocumentPath, e.Message);
}
}
public static ArrayList WebList(
[DekiScriptParam("XML source text or source uri")] string source,
[DekiScriptParam("xpath to list of values")] string xpath,
[DekiScriptParam("namespaces (default: none)", true)] Hashtable namespaces,
[DekiScriptParam("capture enclosing XML element (default: false)", true)] bool?xml,
[DekiScriptParam("caching duration in seconds (range: 60 - 86400; default: 300)", true)] double?ttl,
[DekiScriptParam("return nil if source could not be loaded (default: text with error message)", true)] bool?nilIfMissing
)
{
XUri uri = XUri.TryParse(source);
ArrayList result;
if (uri == null)
{
// source is a string -> convert sourcwe to an XML document and apply xpath
XDoc doc = XDocFactory.From(source, MimeType.XML);
if ((doc == null) || doc.IsEmpty)
{
result = nilIfMissing.GetValueOrDefault() ? null : new ArrayList();
}
else
{
result = AtXPathList(doc, xpath, namespaces, xml ?? false);
}
}
else
{
// we need to fetch an online document
string response = CachedWebGet(uri, ttl, nilIfMissing);
if (response == null)
{
return(null);
}
XDoc doc = XDocFactory.From(response, MimeType.XML);
if (doc.IsEmpty)
{
doc = XDocFactory.From(response, MimeType.HTML);
}
if (doc.IsEmpty)
{
// * source is a uri pointing to text document -> fetch source and convert to string (ignore xpath)
result = nilIfMissing.GetValueOrDefault() ? null : new ArrayList {
response
};
}
else
{
// * source is a uri pointing to xml document -> fetch source, convert to XML document, and apply xpath
result = AtXPathList(doc, xpath, namespaces, xml ?? false);
}
}
return(result);
}
private void BuildIndex(IndexInfo info)
{
_catalog.NewQuery(string.Format("SELECT id, revision, doc FROM {0}", _name)).Execute(delegate(IDataReader reader) {
while (reader.Read())
{
int id = reader.GetInt32(0);
int revision = reader.GetInt32(1);
XDoc doc = XDocFactory.From(reader.GetString(2), MimeType.TEXT_XML);
QueueSingleIndexUpdate(info, id, revision, doc);
}
});
}
