static byte[] ReadDfsFileContent(FastXml fx, int filestart, int fileend, string dfsfile)
{
long size;
{
int sizestart, sizeend;
if (!fx.FindTag("Size", filestart, fileend, out sizestart, out sizeend))
{
throw new Exception("LoadDfsFileContents: Expected <Size> for DFS file '" + dfsfile + "'");
}
string ssize = fx.GetInnerText(sizestart, sizeend);
if (!long.TryParse(ssize, out size) || size < 0)
{
throw new FormatException("LoadDfsFileContents: <Size> for DFS file '" + dfsfile + "' has invalid value: " + ssize);
}
if (size > 1342177280)
{
throw new Exception("LoadDfsFileContents: file size (" + size + ") for DFS file '" + dfsfile + "' too large to load into memory");
}
}
byte[] content = new byte[size];
int contentpos = 0;
{
int cstart = filestart;
int cend = fileend;
for (; ; )
{
int nodestart, nodeend;
if (!fx.FindTag("FileNode", cstart, cend, out nodestart, out nodeend))
{
break;
}
string chunkname = fx.GetTagInnerText("Name", nodestart, nodeend);
string[] chunkhosts = fx.GetTagInnerText("Host", nodestart, nodeend).Split(';');
int chunkpos = 0; // Excluding header.
for (int ichunkhost = 0; ; )
{
try
{
string chunkpath = ToNetworkPath(DfsDir, chunkhosts[ichunkhost]) + @"\" + chunkname;
using (System.IO.FileStream fs = new System.IO.FileStream(chunkpath, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read))
{
{
if (null == headerbuf)
{
headerbuf = new byte[32];
}
// Skip the dfs-chunk file-header...
int headerlength = 0;
{
if (4 != fs.Read(headerbuf, 0, 4))
{
throw new System.IO.IOException("Unable to read DFS chunk header: " + chunkpath);
}
{
headerlength = MyBytesToInt(headerbuf, 0);
if (headerlength > 4)
{
int remain = headerlength - 4;
if (remain > headerbuf.Length)
{
headerbuf = new byte[remain + 100];
}
MyStreamReadExact(fs, headerbuf, remain);
}
}
}
}
if (0 != chunkpos)
{
fs.Seek(chunkpos, System.IO.SeekOrigin.Current);
}
for (; ; )
{
int read = fs.Read(content, contentpos, content.Length - contentpos);
if (read < 1)
{
break;
}
chunkpos += read;
contentpos += read;
if (contentpos > content.Length)
{
throw new Exception("LoadDfsFileContent: read too much data from DFS file '" + dfsfile + "'; file size reported is inaccurate");
}
}
}
break;
}
catch (System.IO.IOException)
{
ichunkhost++;
if (ichunkhost >= chunkhosts.Length)
{
throw;
}
continue;
}
}
cstart = nodeend;
}
}
if (contentpos < content.Length)
{
#if DEBUG
throw new Exception("DEBUG: LoadDfsFileContent: (contentpos{" + contentpos
+ "} < content.Length{" + content.Length + "}) for DFS file '" + dfsfile + "'");
#endif
byte[] newcontent = new byte[contentpos];
Buffer.BlockCopy(content, 0, newcontent, 0, contentpos);
content = newcontent;
}
return content;
}
public static void DeleteFile(string dfsfile)
{
if (dfsfile.StartsWith("dfs://", StringComparison.OrdinalIgnoreCase))
{
dfsfile = dfsfile.Substring(6);
}
ValidateDfsProtocolDfsFileName(dfsfile);
FastXml fx;
List<string> oldfilenodepaths;
using (LockDfsMutex())
{
fx = new FastXml(ReadDfs_unlocked(DfsDir + @"\dfs.xml"));
{
int filestart, fileend;
if (!DfsFindFile(fx, dfsfile, out filestart, out fileend))
{
throw new Exception("File not found in DFS: " + dfsfile);
}
oldfilenodepaths = GetFileNodePaths(fx, filestart, fileend);
fx.Replace(filestart, fileend, "");
{
string metabackupdir = null;
metabackupdir = fx.GetTagInnerText("MetaBackup");
if (null != metabackupdir && 0 == metabackupdir.Length)
{
metabackupdir = null;
}
WriteDfsXml_unlocked(fx.ToString(), DfsDir + @"\dfs.xml", metabackupdir);
}
}
}
MySpace.DataMining.Threading.ThreadTools<string>.Parallel(
new Action<string>(
delegate(string chunkpath)
{
try
{
System.IO.File.Delete(chunkpath);
System.IO.File.Delete(chunkpath + ".zsa");
}
catch
{
}
}), oldfilenodepaths);
}
static Random rnd = null; // Single threaded.
static List<string> GetFileNodePaths(FastXml fx, int filestart, int fileend)
{
List<string> result = new List<string>();
int start = filestart, end = fileend;
for (; ; )
{
int xs, xe;
if (!fx.FindTag("FileNode", start, end, out xs, out xe))
{
break;
}
{
string nname = fx.GetTagInnerText("Name", xs, xe);
string[] nhosts = fx.GetTagInnerText("Host", xs, xe).Split(';');
for (int i = 0; i < nhosts.Length; i++)
{
result.Add(ToNetworkPath(DfsDir, nhosts[i]) + @"\" + nname);
}
}
start = xe;
}
return result;
}
public static void SetFileContent(string dfsfile, string dfsfiletype, byte[] content, int contentlength)
{
if (dfsfile.StartsWith("dfs://", StringComparison.OrdinalIgnoreCase))
{
dfsfile = dfsfile.Substring(6);
}
ValidateDfsProtocolDfsFileName(dfsfile);
FastXml fx;
using (LockDfsMutex())
{
fx = new FastXml(ReadDfs_unlocked(DfsDir + @"\dfs.xml"));
}
string[] slaves = fx.GetTagInnerText("SlaveList").Split(';');
int replfactor;
if (!int.TryParse(fx.GetTagInnerText("Replication"), out replfactor))
{
replfactor = 1;
}
string dfsfilexml = WriteDfsFileContent(dfsfile, dfsfiletype, content, contentlength, slaves, replfactor);
List<string> oldfilenodepaths = null;
using (LockDfsMutex())
{
fx = new FastXml(ReadDfs_unlocked(DfsDir + @"\dfs.xml"));
int fstart, fend;
if (DfsFindFile(fx, dfsfile, out fstart, out fend))
{
oldfilenodepaths = GetFileNodePaths(fx, fstart, fend);
fx.Replace(fstart, fend, dfsfilexml);
}
else
{
if (fx.FindTag("Files", out fstart, out fend))
{
fx.AppendChild(fstart, fend, dfsfilexml);
}
else
{
if (fx.FindTag("dfs", out fstart, out fend))
{
fx.AppendChild(fstart, fend, "<Files>" + dfsfilexml + "</Files>");
}
else
{
throw new Exception("Corrupt DFS.XML: dfs root tag not found");
}
}
}
{
string metabackupdir = null;
metabackupdir = fx.GetTagInnerText("MetaBackup");
if (null != metabackupdir && 0 == metabackupdir.Length)
{
metabackupdir = null;
}
WriteDfsXml_unlocked(fx.ToString(), DfsDir + @"\dfs.xml", metabackupdir);
}
}
if (null != oldfilenodepaths)
{
MySpace.DataMining.Threading.ThreadTools<string>.Parallel(
new Action<string>(
delegate(string chunkpath)
{
try
{
System.IO.File.Delete(chunkpath);
System.IO.File.Delete(chunkpath + ".zsa");
}
catch
{
}
}), oldfilenodepaths);
}
}