static byte[] ReadDfsFileContent(FastXml fx, int filestart, int fileend, string dfsfile)
{
long size;
{
int sizestart, sizeend;
if (!fx.FindTag("Size", filestart, fileend, out sizestart, out sizeend))
{
throw new Exception("LoadDfsFileContents: Expected <Size> for DFS file '" + dfsfile + "'");
}
string ssize = fx.GetInnerText(sizestart, sizeend);
if (!long.TryParse(ssize, out size) || size < 0)
{
throw new FormatException("LoadDfsFileContents: <Size> for DFS file '" + dfsfile + "' has invalid value: " + ssize);
}
if (size > 1342177280)
{
throw new Exception("LoadDfsFileContents: file size (" + size + ") for DFS file '" + dfsfile + "' too large to load into memory");
}
}
byte[] content = new byte[size];
int contentpos = 0;
{
int cstart = filestart;
int cend = fileend;
for (; ; )
{
int nodestart, nodeend;
if (!fx.FindTag("FileNode", cstart, cend, out nodestart, out nodeend))
{
break;
}
string chunkname = fx.GetTagInnerText("Name", nodestart, nodeend);
string[] chunkhosts = fx.GetTagInnerText("Host", nodestart, nodeend).Split(';');
int chunkpos = 0; // Excluding header.
for (int ichunkhost = 0; ; )
{
try
{
string chunkpath = ToNetworkPath(DfsDir, chunkhosts[ichunkhost]) + @"\" + chunkname;
using (System.IO.FileStream fs = new System.IO.FileStream(chunkpath, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read))
{
{
if (null == headerbuf)
{
headerbuf = new byte[32];
}
// Skip the dfs-chunk file-header...
int headerlength = 0;
{
if (4 != fs.Read(headerbuf, 0, 4))
{
throw new System.IO.IOException("Unable to read DFS chunk header: " + chunkpath);
}
{
headerlength = MyBytesToInt(headerbuf, 0);
if (headerlength > 4)
{
int remain = headerlength - 4;
if (remain > headerbuf.Length)
{
headerbuf = new byte[remain + 100];
}
MyStreamReadExact(fs, headerbuf, remain);
}
}
}
}
if (0 != chunkpos)
{
fs.Seek(chunkpos, System.IO.SeekOrigin.Current);
}
for (; ; )
{
int read = fs.Read(content, contentpos, content.Length - contentpos);
if (read < 1)
{
break;
}
chunkpos += read;
contentpos += read;
if (contentpos > content.Length)
{
throw new Exception("LoadDfsFileContent: read too much data from DFS file '" + dfsfile + "'; file size reported is inaccurate");
}
}
}
break;
}
catch (System.IO.IOException)
{
ichunkhost++;
if (ichunkhost >= chunkhosts.Length)
{
throw;
}
continue;
}
}
cstart = nodeend;
}
}
if (contentpos < content.Length)
{
#if DEBUG
throw new Exception("DEBUG: LoadDfsFileContent: (contentpos{" + contentpos
+ "} < content.Length{" + content.Length + "}) for DFS file '" + dfsfile + "'");
#endif
byte[] newcontent = new byte[contentpos];
Buffer.BlockCopy(content, 0, newcontent, 0, contentpos);
content = newcontent;
}
return content;
}
static Random rnd = null; // Single threaded.
static List<string> GetFileNodePaths(FastXml fx, int filestart, int fileend)
{
List<string> result = new List<string>();
int start = filestart, end = fileend;
for (; ; )
{
int xs, xe;
if (!fx.FindTag("FileNode", start, end, out xs, out xe))
{
break;
}
{
string nname = fx.GetTagInnerText("Name", xs, xe);
string[] nhosts = fx.GetTagInnerText("Host", xs, xe).Split(';');
for (int i = 0; i < nhosts.Length; i++)
{
result.Add(ToNetworkPath(DfsDir, nhosts[i]) + @"\" + nname);
}
}
start = xe;
}
return result;
}
static List<string> GetFileNodePaths(FastXml fx, string dfsfile)
{
int filestart, fileend;
if (!DfsFindFile(fx, dfsfile, out filestart, out fileend))
{
throw new Exception("File not found in DFS: " + dfsfile);
}
return GetFileNodePaths(fx, filestart, fileend);
}
public static void SetFileContent(string dfsfile, string dfsfiletype, byte[] content, int contentlength)
{
if (dfsfile.StartsWith("dfs://", StringComparison.OrdinalIgnoreCase))
{
dfsfile = dfsfile.Substring(6);
}
ValidateDfsProtocolDfsFileName(dfsfile);
FastXml fx;
using (LockDfsMutex())
{
fx = new FastXml(ReadDfs_unlocked(DfsDir + @"\dfs.xml"));
}
string[] slaves = fx.GetTagInnerText("SlaveList").Split(';');
int replfactor;
if (!int.TryParse(fx.GetTagInnerText("Replication"), out replfactor))
{
replfactor = 1;
}
string dfsfilexml = WriteDfsFileContent(dfsfile, dfsfiletype, content, contentlength, slaves, replfactor);
List<string> oldfilenodepaths = null;
using (LockDfsMutex())
{
fx = new FastXml(ReadDfs_unlocked(DfsDir + @"\dfs.xml"));
int fstart, fend;
if (DfsFindFile(fx, dfsfile, out fstart, out fend))
{
oldfilenodepaths = GetFileNodePaths(fx, fstart, fend);
fx.Replace(fstart, fend, dfsfilexml);
}
else
{
if (fx.FindTag("Files", out fstart, out fend))
{
fx.AppendChild(fstart, fend, dfsfilexml);
}
else
{
if (fx.FindTag("dfs", out fstart, out fend))
{
fx.AppendChild(fstart, fend, "<Files>" + dfsfilexml + "</Files>");
}
else
{
throw new Exception("Corrupt DFS.XML: dfs root tag not found");
}
}
}
{
string metabackupdir = null;
metabackupdir = fx.GetTagInnerText("MetaBackup");
if (null != metabackupdir && 0 == metabackupdir.Length)
{
metabackupdir = null;
}
WriteDfsXml_unlocked(fx.ToString(), DfsDir + @"\dfs.xml", metabackupdir);
}
}
if (null != oldfilenodepaths)
{
MySpace.DataMining.Threading.ThreadTools<string>.Parallel(
new Action<string>(
delegate(string chunkpath)
{
try
{
System.IO.File.Delete(chunkpath);
System.IO.File.Delete(chunkpath + ".zsa");
}
catch
{
}
}), oldfilenodepaths);
}
}
public static void DeleteFile(string dfsfile)
{
if (dfsfile.StartsWith("dfs://", StringComparison.OrdinalIgnoreCase))
{
dfsfile = dfsfile.Substring(6);
}
ValidateDfsProtocolDfsFileName(dfsfile);
FastXml fx;
List<string> oldfilenodepaths;
using (LockDfsMutex())
{
fx = new FastXml(ReadDfs_unlocked(DfsDir + @"\dfs.xml"));
{
int filestart, fileend;
if (!DfsFindFile(fx, dfsfile, out filestart, out fileend))
{
throw new Exception("File not found in DFS: " + dfsfile);
}
oldfilenodepaths = GetFileNodePaths(fx, filestart, fileend);
fx.Replace(filestart, fileend, "");
{
string metabackupdir = null;
metabackupdir = fx.GetTagInnerText("MetaBackup");
if (null != metabackupdir && 0 == metabackupdir.Length)
{
metabackupdir = null;
}
WriteDfsXml_unlocked(fx.ToString(), DfsDir + @"\dfs.xml", metabackupdir);
}
}
}
MySpace.DataMining.Threading.ThreadTools<string>.Parallel(
new Action<string>(
delegate(string chunkpath)
{
try
{
System.IO.File.Delete(chunkpath);
System.IO.File.Delete(chunkpath + ".zsa");
}
catch
{
}
}), oldfilenodepaths);
}
public static byte[] GetFileContent(string dfsfile)
{
if (dfsfile.StartsWith("dfs://", StringComparison.OrdinalIgnoreCase))
{
dfsfile = dfsfile.Substring(6);
}
ValidateDfsProtocolDfsFileName(dfsfile);
FastXml fx;
using (LockDfsMutex())
{
fx = new FastXml(ReadDfs_unlocked(DfsDir + @"\dfs.xml"));
}
int filestart, fileend;
if (!DfsFindFile(fx, dfsfile, out filestart, out fileend))
{
throw new Exception("The specified file '" + dfsfile + "' does not exist in DFS");
}
byte[] content = ReadDfsFileContent(fx, filestart, fileend, dfsfile);
return content;
}
public static int GetFilePartCount(string dfsfile)
{
if (dfsfile.StartsWith("dfs://", StringComparison.OrdinalIgnoreCase))
{
dfsfile = dfsfile.Substring(6);
}
ValidateDfsProtocolDfsFileName(dfsfile);
FastXml fx;
using (LockDfsMutex())
{
fx = new FastXml(ReadDfs_unlocked(DfsDir + @"\dfs.xml"));
}
int filestart, fileend;
if (!DfsFindFile(fx, dfsfile, out filestart, out fileend))
{
throw new Exception("File not found in DFS: " + dfsfile);
}
int partcount = 0;
int pcstart, pcend = filestart;
while (fx.FindTag("FileNode", pcend, fileend, out pcstart, out pcend))
{
partcount++;
}
return partcount;
}
public static string GetFileSizeString(string dfsfile)
{
if (dfsfile.StartsWith("dfs://", StringComparison.OrdinalIgnoreCase))
{
dfsfile = dfsfile.Substring(6);
}
ValidateDfsProtocolDfsFileName(dfsfile);
FastXml fx;
using (LockDfsMutex())
{
fx = new FastXml(ReadDfs_unlocked(DfsDir + @"\dfs.xml"));
}
int filestart, fileend;
if (!DfsFindFile(fx, dfsfile, out filestart, out fileend))
{
throw new Exception("File not found in DFS: " + dfsfile);
}
int sizestart, sizeend;
if (!fx.FindTag("Size", filestart, fileend, out sizestart, out sizeend))
{
throw new Exception("Expected <Size> for DFS file '" + dfsfile + "'");
}
string ssize = fx.GetInnerText(sizestart, sizeend);
return ssize;
}
internal static void _Test_()
{
#if DEBUG
{
FastXml fx = new FastXml("foo <foo> foo </foo> foo");
int start, end;
if (!fx.FindTag("foo", 0, out start, out end) || 4 != start || 20 != end)
{
System.Diagnostics.Debugger.Launch();
throw new Exception("TEST FAILED");
}
}
{
FastXml fx = new FastXml("foo <foo > foo </foo> foo");
int start, end;
if (!fx.FindTag("foo", 0, out start, out end) || 4 != start || 21 != end)
{
System.Diagnostics.Debugger.Launch();
throw new Exception("TEST FAILED");
}
}
{
FastXml fx = new FastXml("foo <foo foo=`hi`> foo </foo> foo".Replace('`', '"'));
int start, end;
if (!fx.FindTag("foo", 0, out start, out end) || 4 != start || 29 != end)
{
System.Diagnostics.Debugger.Launch();
throw new Exception("TEST FAILED");
}
}
{
FastXml fx = new FastXml("foo <foo/> foo </foo> foo");
int start, end;
if (!fx.FindTag("foo", 0, out start, out end) || 4 != start || 10 != end)
{
System.Diagnostics.Debugger.Launch();
throw new Exception("TEST FAILED");
}
}
{
FastXml fx = new FastXml("<foofoo> <foo> foo </foo> foo");
int start, end;
if (!fx.FindTag("foo", 0, out start, out end) || 9 != start || 25 != end)
{
System.Diagnostics.Debugger.Launch();
throw new Exception("TEST FAILED");
}
}
{
FastXml fx = new FastXml("<foofoo> <foo> foo </foofoo> </foo> foo");
int start, end;
if (!fx.FindTag("foo", 0, out start, out end) || 9 != start || 35 != end)
{
System.Diagnostics.Debugger.Launch();
throw new Exception("TEST FAILED");
}
}
{
FastXml fx = new FastXml("<foofoo foo=`foo`> <foo/> foo </foofoo> </foo> foo".Replace('`', '"'));
int start, end;
if (!fx.FindTag("foo", 0, out start, out end) || 19 != start || 25 != end)
{
System.Diagnostics.Debugger.Launch();
throw new Exception("TEST FAILED");
}
}
{
FastXml fx = new FastXml("<foofoo/> <foo/> foo </foofoo> </foo> foo");
int start, end;
if (!fx.FindTag("foo", 0, out start, out end) || 10 != start || 16 != end)
{
System.Diagnostics.Debugger.Launch();
throw new Exception("TEST FAILED");
}
}
{
FastXml fx = new FastXml("<foo>bar</foo>");
int start, end;
fx.FindTag("foo", 0, out start, out end);
string s = fx.GetInnerText(start, end);
if (s != "bar")
{
System.Diagnostics.Debugger.Launch();
throw new Exception("TEST FAILED");
}
}
{
FastXml fx = new FastXml(" <foo> bar </foo> ");
int start, end;
fx.FindTag("foo", 0, out start, out end);
string s = fx.GetInnerText(start, end);
if (s != "bar")
{
System.Diagnostics.Debugger.Launch();
throw new Exception("TEST FAILED");
}
}
{
FastXml fx = new FastXml(" <foo> <hello> bar </hello> baz </foo> ");
int start, end;
fx.FindTag("foo", 0, out start, out end);
string s = fx.GetInnerText(start, end);
if (s != "bar baz")
{
System.Diagnostics.Debugger.Launch();
throw new Exception("TEST FAILED");
}
}
{
FastXml fx = new FastXml("<foo> bar<x/><hello>baz</hello>");
int start, end;
fx.FindTag("foo", 0, out start, out end);
string s = fx.GetInnerText(start, end);
if (s != "bar baz")
{
System.Diagnostics.Debugger.Launch();
throw new Exception("TEST FAILED");
}
}
{
FastXml fx = new FastXml("<foo> bar<x/><hello/>baz</foo>");
int start, end;
fx.FindTag("foo", 0, out start, out end);
string s = fx.GetInnerText(start, end);
if (s != "bar baz")
{
System.Diagnostics.Debugger.Launch();
throw new Exception("TEST FAILED");
}
}
{
FastXml fx = new FastXml("<foo> bar<x>qux</x><hello/>baz</foo>");
int start, end;
fx.FindTag("foo", 0, out start, out end);
string s = fx.GetInnerText(start, end);
if (s != "bar qux baz")
{
System.Diagnostics.Debugger.Launch();
throw new Exception("TEST FAILED");
}
}
{
FastXml fx = new FastXml(" a <foo/> b ");
int start, end;
fx.FindTag("foo", 0, out start, out end);
string s = fx.GetInnerText(start, end);
if (s != "")
{
System.Diagnostics.Debugger.Launch();
throw new Exception("TEST FAILED");
}
}
{
FastXml fx = new FastXml(" <foo> a <x/> < b </foo> ");
int start, end;
fx.FindTag("foo", 0, out start, out end);
string s = fx.GetInnerText(start, end);
if (s != "a < b")
{
System.Diagnostics.Debugger.Launch();
throw new Exception("TEST FAILED");
}
}
#endif
}
static bool DfsFindFile(FastXml fx, string dfsfile, out int tagstart, out int tagend)
{
int filestart, fileend;
for (int pos = 0; fx.FindTag("DfsFile", pos, out filestart, out fileend); pos = fileend)
{
int namestart, nameend;
if (fx.FindTag("Name", filestart, fileend, out namestart, out nameend))
{
string onname = fx.GetInnerText(namestart, nameend);
if (0 == string.Compare(onname, dfsfile, StringComparison.OrdinalIgnoreCase))
{
tagstart = filestart;
tagend = fileend;
return true;
}
}
}
tagstart = 0;
tagend = 0;
return false;
}
