static void _MemCacheRelease(dfs.DfsFile df, bool force)
{
dfs dc = LoadDfsConfig();
string exception = "";
string[] slaves = dc.Slaves.SlaveList.Split(';');
{
//foreach (string slave in slaves)
MySpace.DataMining.Threading.ThreadTools <string> .Parallel(
new Action <string>(
delegate(string slave)
{
System.Net.Sockets.NetworkStream nstm = Surrogate.ConnectService(slave);
nstm.WriteByte((byte)'C');
nstm.WriteByte((byte)'r');
XContent.SendXContent(nstm, df.Name);
XContent.SendXContent(nstm, new byte[1] {
(byte)(force ? 1 : 0)
});
int ich = nstm.ReadByte();
if ('+' != ich)
{
string errmsg = null;
if ('-' == ich)
{
try
{
errmsg = XContent.ReceiveXString(nstm, null);
}
catch
{
}
}
lock (slaves)
{
string newexception;
if (null != errmsg)
{
newexception = ("Error received from DO service during MemCache rollback from " + slave + ": " + errmsg);
}
else
{
newexception = ("Did not receive a success signal from DO service during MemCache rollback from " + slave);
}
if (string.IsNullOrEmpty(exception) ||
-1 != exception.IndexOf("MemCacheWarning"))
{
exception = newexception;
}
}
}
}), slaves, slaves.Length);
}
if (!string.IsNullOrEmpty(exception))
{
throw new Exception(exception);
}
}
internal static void _KillMemCache_mt(List <dfs.DfsFile> delfiles, bool verbose)
{
dfs dc = LoadDfsConfig();
string[] slaves = dc.Slaves.SlaveList.Split(';');
foreach (dfs.DfsFile df in delfiles)
{
try
{
// Unpin shared memory segments on all machines.
_MemCacheRelease(df, true); // force=true
}
catch (System.Threading.ThreadAbortException)
{
}
catch (Exception e)
{
if (verbose)
{
string msg = e.Message;
if (-1 == msg.IndexOf("MemCacheWarning"))
{
Console.WriteLine("Warning: {0}", msg);
}
}
}
foreach (string slave in slaves)
{
try
{
string mdfp = Surrogate.NetworkPathForHost(slave)
+ @"\" + df.MemCache.MetaFileName;
System.IO.File.Delete(mdfp);
}
catch
{
}
}
}
_KillDataFileChunks_unlocked_mt(delfiles, false);
if (verbose)
{
foreach (dfs.DfsFile df in delfiles)
{
Console.WriteLine("Successfully deleted MemCache '{0}' ({1} parts)", df.Name, df.Nodes.Count);
}
}
}
internal FailoverInfo(dfs dc)
{
try
{
healthpluginpaths = GetHealthPluginPaths(dc);
}
catch
{
}
if (healthpluginpaths != null && healthpluginpaths.Length > 0)
{
diskcheck = new MySpace.DataMining.DistributedObjects.DiskCheck(healthpluginpaths);
}
rnd = new Random(unchecked(DateTime.Now.Millisecond + System.Diagnostics.Process.GetCurrentProcess().Id));
}
static void EnsureReplication(string dfsxmlpath, int replicationfactor)
{
bool toomany = false;
Console.WriteLine("Ensure data is replicated...");
{
dfs dc = dfs.ReadDfsConfig_unlocked(dfsxmlpath);
foreach (dfs.DfsFile df in dc.Files)
{
if (0 == string.Compare(DfsFileTypes.NORMAL, df.Type, StringComparison.OrdinalIgnoreCase) ||
0 == string.Compare(DfsFileTypes.BINARY_RECT, df.Type, StringComparison.OrdinalIgnoreCase))
{
foreach (dfs.DfsFile.FileNode fn in df.Nodes)
{
string[] nhosts = fn.Host.Split(';', ',');
if (nhosts.Length < replicationfactor)
{
throw new Exception("dfs://" + df.Name + " node " + fn.Name + " does not live on " + replicationfactor.ToString() + " machines");
}
else if (nhosts.Length > replicationfactor)
{
if (!toomany)
{
toomany = true;
Console.Error.WriteLine("Warning: too many replicates for one or more DFS file chunks");
}
}
for (int ni = 0; ni < nhosts.Length; ni++)
{
string np = Surrogate.NetworkPathForHost(nhosts[ni]) + @"\" + fn.Name;
if (!System.IO.File.Exists(np))
{
throw new Exception("dfs://" + df.Name + " node " + fn.Name + " does not actually live on host " + nhosts[ni] + " as indicated by meta-data [" + np + "]");
}
if (df.HasZsa)
{
if (!System.IO.File.Exists(np + ".zsa"))
{
throw new Exception("Sample data for dfs://" + df.Name + " node " + fn.Name + " (" + fn.Name + ".zsa) does not actually live on host " + nhosts[ni] + " as indicated by meta-data [" + np + ".zsa]");
}
}
}
}
}
}
}
}
public DfsStream(string dfsfile, bool PreserveOrder, bool MachineLock)
{
if (MachineLock)
{
this.Mutex = new System.Threading.Mutex(false, "DfsStream{24A86864-EED6-4680-AB0E-3BDE97262339}");
this.Mutex.WaitOne();
}
ReplicateStartIndex = StaticGlobals.Qizmt_BlockID;
surrogatedir = Surrogate.NetworkPathForHost(Surrogate.MasterHost);
dfs dc = dfs.ReadDfsConfig_unlocked(surrogatedir + @"\" + dfs.DFSXMLNAME);
this.RetryTimeout = dc.slave.CookTimeout;
this.RetryCount = dc.slave.CookRetries;
dfs.DfsFile df = dc.FindAny(dfsfile);
if (null == df)
{
throw new System.IO.FileNotFoundException("DFS file '" + dfsfile + "' not found", dfsfile);
}
if (0 != string.Compare(DfsFileTypes.NORMAL, df.Type, StringComparison.OrdinalIgnoreCase) &&
0 != string.Compare(DfsFileTypes.BINARY_RECT, df.Type, StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException("DFS file '" + df.Name + "' cannot be opened because file is of type " + df.Type);
}
this.reclen = df.RecordLength;
nodes = df.Nodes.ToArray();
if (!PreserveOrder)
{
Random rnd = new Random(unchecked (
System.Threading.Thread.CurrentThread.ManagedThreadId
+ DateTime.Now.Millisecond * 351
+ ReplicateStartIndex + nodes.Length * 6131));
for (int i = 0; i < nodes.Length; i++)
{
int ridx = rnd.Next(0, nodes.Length);
dfs.DfsFile.FileNode tmpnode = nodes[i];
nodes[i] = nodes[ridx];
nodes[ridx] = tmpnode;
}
}
}
public static void MemCacheLoad(string mcname)
{
if (mcname.StartsWith("dfs://", StringComparison.OrdinalIgnoreCase))
{
mcname = mcname.Substring(6);
}
dfs dc = LoadDfsConfig();
dfs.DfsFile df = dc.FindAny(mcname);
if (null == df || df.MemCache == null)
{
Console.Error.WriteLine("Error: '{0}' is not a MemCache", (null == df ? mcname : df.Name));
SetFailure();
return;
}
_MemCacheLoad(df);
}
static void BlankTest(string[] args)
{
if (args.Length <= 1 || !System.IO.File.Exists(args[1]))
{
throw new Exception("Expected path to DFS.xml");
}
string dfsxmlpath = args[1];
string dfsxmlpathbackup = dfsxmlpath + "$" + Guid.NewGuid().ToString();
string masterdir;
{
System.IO.FileInfo fi = new System.IO.FileInfo(dfsxmlpath);
masterdir = fi.DirectoryName; // Directory's full path.
}
Surrogate.SetNewMetaLocation(masterdir);
dfs dc = dfs.ReadDfsConfig_unlocked(dfsxmlpath);
string masterhost = System.Net.Dns.GetHostName();
string[] allmachines;
{
string[] sl = dc.Slaves.SlaveList.Split(';');
List <string> aml = new List <string>(sl.Length + 1);
aml.Add(masterhost);
foreach (string slave in sl)
{
if (0 != string.Compare(IPAddressUtil.GetName(slave), IPAddressUtil.GetName(masterhost), StringComparison.OrdinalIgnoreCase))
{
aml.Add(slave);
}
}
allmachines = aml.ToArray();
}
{
// Test logic:
}
Console.WriteLine("[PASSED] - " + string.Join(" ", args));
}
public static void X()
{
string[] inputstr = Console.ReadLine().Split(' ');
int CountHuman = int.Parse(inputstr[0]);
int MainHuman = int.Parse(inputstr[1]);
int[,] Matrix = new int[CountHuman, CountHuman];
for (int i = 0; i < CountHuman; i++)
{
string[] CurrentHuman = Console.ReadLine().Split(' ');
for (int j = 0; j < CountHuman; j++)
{
Matrix[i, j] = int.Parse(CurrentHuman[j]);
}
}
dfs d = new dfs(Matrix, MainHuman - 1, CountHuman, MainHuman - 1);
d.dfsmet();
Console.WriteLine(d.Res());
}
public static void MemCacheDelete(string mcname, bool verbose)
{
if (mcname.StartsWith("dfs://", StringComparison.OrdinalIgnoreCase))
{
mcname = mcname.Substring(6);
}
dfs.DfsFile df = null;
using (LockDfsMutex())
{
dfs dc = LoadDfsConfig();
int dfindex;
for (dfindex = 0; dfindex < dc.Files.Count; dfindex++)
{
if (0 == string.Compare(dc.Files[dfindex].Name, mcname, true))
{
df = dc.Files[dfindex];
break;
}
}
if (null == df)
{
//Console.Error.WriteLine("DFS file '{0}' does not exist", mcname);
//SetFailure();
return;
}
if (df.MemCache == null)
{
Console.Error.WriteLine("DFS file '{0}' is not a MemCache file", df.Name);
SetFailure();
return;
}
dc.Files.RemoveAt(dfindex);
UpdateDfsXml(dc);
}
{
List <dfs.DfsFile> delfiles = new List <dfs.DfsFile>(1);
delfiles.Add(df);
_KillMemCache_mt(delfiles, verbose);
}
}
static void _MemCacheLoad(dfs.DfsFile df)
{
dfs dc = LoadDfsConfig();
string[] slaves = dc.Slaves.SlaveList.Split(';');
{
//foreach (string slave in slaves)
MySpace.DataMining.Threading.ThreadTools <string> .Parallel(
new Action <string>(
delegate(string slave)
{
System.Net.Sockets.NetworkStream nstm = Surrogate.ConnectService(slave);
nstm.WriteByte((byte)'C');
nstm.WriteByte((byte)'l');
XContent.SendXContent(nstm, df.Name);
int ich = nstm.ReadByte();
if ('+' != ich)
{
string errmsg = null;
if ('-' == ich)
{
try
{
errmsg = XContent.ReceiveXString(nstm, null);
}
catch
{
}
}
if (null != errmsg)
{
throw new Exception("Error received from DO service during MemCache load from " + slave + ": " + errmsg);
}
throw new Exception("Did not receive a success signal from DO service during MemCache load from " + slave);
}
}), slaves, slaves.Length);
}
}
public static string GetMemCacheMetaFileHeader(dfs.DfsFile df)
{
StringBuilder sbstartmeta = new StringBuilder();
sbstartmeta.AppendFormat("name={1}{0}", Environment.NewLine, df.Name);
sbstartmeta.AppendFormat("blocksize={1}{0}", Environment.NewLine, df.MemCache.SegmentSize);
sbstartmeta.AppendFormat("rowlen={1}{0}", Environment.NewLine, df.MemCache.RowLength);
sbstartmeta.AppendFormat("keyoffset={1}{0}", Environment.NewLine, df.MemCache.KeyOffset);
sbstartmeta.AppendFormat("keylen={1}{0}", Environment.NewLine, df.MemCache.KeyLength);
sbstartmeta.AppendFormat("coloffsets={1}{0}", Environment.NewLine, df.MemCache.FieldOffsets);
sbstartmeta.AppendLine("-----------------------------------------");
return sbstartmeta.ToString();
}
public static dfs.DfsFile DfsFindAny(dfs dc, string dfspath)
{
return dc.FindAny(dfspath);
}
public static dfs.DfsFile DfsFind(dfs dc, string dfspath, string type)
{
return dc.Find(dfspath, type);
}
public static void _KillDllFileChunks_unlocked(dfs.DfsFile dfsdll, bool verbose)
{
dfs dc = LoadDfsConfig(); // Just to get the slave list.
string[] slaves = dc.Slaves.SlaveList.Split(',', ';');
for (int si = 0; si < slaves.Length; si++)
{
try
{
string slave = slaves[si];
string netpath = Surrogate.NetworkPathForHost(slave);
string cacpath = netpath + @"\" + dfs.DLL_DIR_NAME;
System.IO.File.Delete(cacpath + dfsdll.Name);
}
catch
{
}
}
}
static void Dfs(string cmd, string[] args)
{
//using (LockDfsMutex())
{
switch (cmd)
{
case "delete":
case "del":
case "rm":
case "delmt":
case "delst":
if (!dfs.DfsConfigExists(DFSXMLPATH))
{
Console.Error.WriteLine("DFS not setup; use: {0} format", appname);
SetFailure();
return;
}
if (args.Length < 1)
{
Console.Error.WriteLine("Error: {0} command needs argument: <path|wildcard>", cmd);
SetFailure();
return;
}
if (string.Compare(cmd, "delst", true) == 0)
{
DfsDelete(args[0]); //single threaded.
}
else
{
DfsDeleteMt(args[0], true);
}
break;
case "head":
{
if (!dfs.DfsConfigExists(DFSXMLPATH))
{
Console.Error.WriteLine("DFS not setup; use: {0} format", appname);
SetFailure();
return;
}
if (args.Length < 1 || '-' == args[0][0])
{
Console.Error.WriteLine("Error: dfs head command needs argument: <dfsfile>");
SetFailure();
return;
}
string[] specs;
if (args[0].StartsWith("dfs://", StringComparison.OrdinalIgnoreCase))
{
specs = args[0].Substring(6).Split(':');
}
else
{
specs = args[0].Split(':');
}
string RecordInfo = null;
int RecordLength = -1;
{
int iat = specs[0].IndexOf('@');
if (-1 != iat)
{
RecordInfo = specs[0].Substring(iat + 1);
specs[0] = specs[0].Substring(0, iat);
}
}
dfs dc = LoadDfsConfig();
dfs.DfsFile df = DfsFindAny(dc, specs[0]);
if (null == df)
{
Console.Error.WriteLine("File not found in DFS: {0}", specs[0]);
SetFailure();
return;
}
if (0 == string.Compare(DfsFileTypes.NORMAL, df.Type, StringComparison.OrdinalIgnoreCase))
{
if (null != RecordInfo)
{
Console.Error.WriteLine("Cannot specify record information '{0}'"
+ " for non- rectangular binary DFS file '{1}'",
RecordInfo, df.Name);
SetFailure();
return;
}
}
else if (0 == string.Compare(DfsFileTypes.BINARY_RECT, df.Type, StringComparison.OrdinalIgnoreCase))
{
if (string.IsNullOrEmpty(RecordInfo))
{
Console.Error.WriteLine("DFS file '{0}' is not of expected type", specs[0]);
Console.WriteLine("Must specify <dfsfile>@<record-types> for a rectangular binary DFS file");
SetFailure();
return;
}
RecordLength = df.RecordLength;
}
else
{
Console.Error.WriteLine("DFS file '{0}' is not of expected type", specs[0]);
SetFailure();
return;
}
if (df.Nodes.Count > 0)
{
string shost = "";
if (specs.Length >= 2 && specs[1].Length > 0)
{
shost = IPAddressUtil.GetName(specs[1]);
}
string partspec = "";
bool foundpart = true;
if (specs.Length >= 3)
{
partspec = specs[2];
foundpart = false;
}
int lc = 10;
if (args.Length >= 2)
{
lc = int.Parse(args[1]);
if (lc <= 0)
{
throw new FormatException("Line count makes no sense");
}
}
const int MAX_SIZE_PER_RECEIVE = 0x400 * 64;
byte[] fbuf = new byte[MAX_SIZE_PER_RECEIVE];
foreach (dfs.DfsFile.FileNode node in df.Nodes)
{
if (partspec.Length > 0)
{
if (0 == string.Compare(node.Name, partspec, StringComparison.OrdinalIgnoreCase))
{
if (ListContainsHost(node.Host, shost))
{
// Good!..
foundpart = true;
}
else
{
ConsoleFlush();
Console.Error.WriteLine(" Specified data-node chunk does not reside on specified host");
SetFailure();
return;
}
}
else
{
continue;
}
}
//string netpath = NetworkPathForHost(node.Host.Split(';')[0]) + @"\" + node.Name;
using (System.IO.Stream _fc = new DfsFileNodeStream(node, true, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read, FILE_BUFFER_SIZE))
{
System.IO.Stream fc = _fc;
if (1 == dc.slave.CompressDfsChunks)
{
fc = new System.IO.Compression.GZipStream(_fc, System.IO.Compression.CompressionMode.Decompress);
}
{
int xread = StreamReadLoop(fc, fbuf, 4);
if (4 == xread)
{
int hlen = MySpace.DataMining.DistributedObjects.Entry.BytesToInt(fbuf);
StreamReadExact(fc, fbuf, hlen - 4);
}
}
if (null == RecordInfo)
{
using (System.IO.StreamReader sr = new System.IO.StreamReader(fc))
{
for (; lc > 0; lc--)
{
string s = sr.ReadLine();
if (null == s)
{
break;
}
Console.WriteLine(s);
}
}
}
else
{
byte[] recbuf = new byte[RecordLength];
string[] types = RecordInfo.Split(',');
List<int> typesizes = new List<int>(types.Length);
int totalsizerequested = 0;
{
foreach (Surrogate.RecordInfo ri in Surrogate.EachRecord(RecordInfo))
{
if (ri.InKey)
{
throw new Exception("Unexpected [ found in record information");
}
totalsizerequested += ri.Size;
if (totalsizerequested <= RecordLength)
{
typesizes.Add(ri.Size);
}
}
}
{
for (; lc > 0; lc--)
{
if (RecordLength != StreamReadLoop(fc, recbuf, RecordLength))
{
break;
}
int offset = 0;
for (int it = 0; it < typesizes.Count;
offset += typesizes[it++])
{
if (0 != it)
{
Console.Write(' ');
}
if (char.IsDigit(types[it][0]))
{
Console.Write("<{0}>", typesizes[it]);
}
else if (0 == string.Compare("int", types[it], StringComparison.OrdinalIgnoreCase)
|| 0 == string.Compare("int32", types[it], StringComparison.OrdinalIgnoreCase)
|| 0 == string.Compare("nInt", types[it], StringComparison.OrdinalIgnoreCase)
)
{
bool isnullable = (char.ToLowerInvariant(types[it][0]) == 'n');
MySpace.DataMining.DistributedObjects.recordset rs;
if (isnullable)
{
if (0 != recbuf[0])
{
Console.Write("NULL");
continue;
}
else
{
rs = MySpace.DataMining.DistributedObjects.recordset.Prepare(
MySpace.DataMining.DistributedObjects.ByteSlice.Prepare(
recbuf, offset + 1, typesizes[it] - 1));
}
}
else
{
rs = MySpace.DataMining.DistributedObjects.recordset.Prepare(
MySpace.DataMining.DistributedObjects.ByteSlice.Prepare(
recbuf, offset, typesizes[it]));
}
int val = rs.GetInt();
if (isnullable)
{
val = (int)MySpace.DataMining.DistributedObjects.Entry.ToUInt32(val);
}
Console.Write(val);
}
else if (0 == string.Compare("long", types[it], StringComparison.OrdinalIgnoreCase)
|| 0 == string.Compare("int64", types[it], StringComparison.OrdinalIgnoreCase)
|| 0 == string.Compare("nLong", types[it], StringComparison.OrdinalIgnoreCase)
)
{
bool isnullable = (char.ToLowerInvariant(types[it][0]) == 'n');
MySpace.DataMining.DistributedObjects.recordset rs;
if (isnullable)
{
if (0 != recbuf[0])
{
Console.Write("NULL");
continue;
}
else
{
rs = MySpace.DataMining.DistributedObjects.recordset.Prepare(
MySpace.DataMining.DistributedObjects.ByteSlice.Prepare(
recbuf, offset + 1, typesizes[it] - 1));
}
}
else
{
rs = MySpace.DataMining.DistributedObjects.recordset.Prepare(
MySpace.DataMining.DistributedObjects.ByteSlice.Prepare(
recbuf, offset, typesizes[it]));
}
long val = rs.GetLong();
if (isnullable)
{
val = (long)MySpace.DataMining.DistributedObjects.Entry.ToUInt64(val);
}
Console.Write(val);
}
else if (0 == string.Compare("nDateTime", types[it], StringComparison.OrdinalIgnoreCase)
)
{
bool isnullable = (char.ToLowerInvariant(types[it][0]) == 'n');
MySpace.DataMining.DistributedObjects.recordset rs;
if (isnullable)
{
if (0 != recbuf[0])
{
Console.Write("NULL");
continue;
}
else
{
rs = MySpace.DataMining.DistributedObjects.recordset.Prepare(
MySpace.DataMining.DistributedObjects.ByteSlice.Prepare(
recbuf, offset + 1, typesizes[it] - 1));
}
}
else
{
rs = MySpace.DataMining.DistributedObjects.recordset.Prepare(
MySpace.DataMining.DistributedObjects.ByteSlice.Prepare(
recbuf, offset, typesizes[it]));
}
try
{
DateTime dtval = rs.GetDateTime();
string val = "\"" + dtval + "\"";
Console.Write(val);
}
catch
{
Console.Write("<error>");
}
}
else if (0 == string.Compare("double", types[it], StringComparison.OrdinalIgnoreCase)
|| 0 == string.Compare("nDouble", types[it], StringComparison.OrdinalIgnoreCase)
)
{
bool isnullable = (char.ToLowerInvariant(types[it][0]) == 'n');
MySpace.DataMining.DistributedObjects.recordset rs;
if (isnullable)
{
if (0 != recbuf[0])
{
Console.Write("NULL");
continue;
}
else
{
rs = MySpace.DataMining.DistributedObjects.recordset.Prepare(
MySpace.DataMining.DistributedObjects.ByteSlice.Prepare(
recbuf, offset + 1, typesizes[it] - 1));
}
}
else
{
rs = MySpace.DataMining.DistributedObjects.recordset.Prepare(
MySpace.DataMining.DistributedObjects.ByteSlice.Prepare(
recbuf, offset, typesizes[it]));
}
double val = Math.Round(rs.GetDouble(), 4);
Console.Write(val);
}
else if (types[it].StartsWith("nChar", StringComparison.OrdinalIgnoreCase)
)
{
bool isnullable = (char.ToLowerInvariant(types[it][0]) == 'n');
MySpace.DataMining.DistributedObjects.recordset rs;
if (isnullable)
{
if (0 != recbuf[0])
{
Console.Write("NULL");
continue;
}
else
{
rs = MySpace.DataMining.DistributedObjects.recordset.Prepare(
MySpace.DataMining.DistributedObjects.ByteSlice.Prepare(
recbuf, offset + 1, typesizes[it] - 1));
}
}
else
{
throw new NotSupportedException();
}
try
{
byte[] sbbuf = new byte[rs.Length];
rs.GetBytes(sbbuf, 0, sbbuf.Length);
string s = Encoding.Unicode.GetString(sbbuf).TrimEnd('\0');
string val = "\"" + s.Replace("\\", "\\\\").Replace("\"", "\\\"") + "\"";
Console.Write(val);
}
catch
{
Console.Write("<error>");
}
}
else
{
// Unhandled type:
Console.Write("<{0}>", types[it].ToUpper());
}
}
if (typesizes.Count > types.Length)
{
if (typesizes.Count > 0)
{
Console.Write(' ');
}
Console.Write("<clipped>");
}
/*else if(totalsizerequested < RecordLength)
{
if (typesizes.Count > 0)
{
Console.Write(' ');
}
Console.Write("<{0}>", RecordLength - totalsizerequested);
}*/
Console.WriteLine();
}
}
}
}
if (partspec.Length > 0)
{
break;
}
if (lc <= 0)
{
break;
}
}
if (lc > 0)
{
ConsoleFlush();
if (!foundpart)
{
Console.Error.WriteLine(" Specified data-node chunk not found");
}
else
{
if (partspec.Length > 0)
{
Console.Error.WriteLine(" Hit end of specified data-node chunk");
}
else
{
//Console.Error.WriteLine(" Hit end of file");
}
}
}
}
else
{
Console.Error.WriteLine(" No data-node chunks");
}
}
break;
case "rename":
case "ren":
case "move":
case "mv":
if (!dfs.DfsConfigExists(DFSXMLPATH))
{
Console.Error.WriteLine("DFS not setup; use: {0} format", appname);
SetFailure();
return;
}
if (args.Length < 2)
{
Console.Error.WriteLine("Error: dfs rename command needs arguments: <dfspath> <dfspath>");
SetFailure();
return;
}
DfsRename(args);
break;
case "swap":
if (!dfs.DfsConfigExists(DFSXMLPATH))
{
Console.Error.WriteLine("DFS not setup; use: {0} format", appname);
SetFailure();
return;
}
if (args.Length < 2)
{
Console.Error.WriteLine("Error: dfs swap command needs arguments: <dfspath> <dfspath>");
SetFailure();
return;
}
DfsSwap(args);
break;
case "countparts":
{
if (args.Length == 0)
{
Console.Error.WriteLine("Error: countparts command needs argument: <dfspath>");
SetFailure();
return;
}
if (!dfs.DfsConfigExists(DFSXMLPATH))
{
Console.Error.WriteLine("DFS not setup; use: {0} format", appname);
SetFailure();
return;
}
string dfsfilename = args[0];
dfs dc = LoadDfsConfig();
dfs.DfsFile df = DfsFindAny(dc, dfsfilename);
if (null == df)
{
Console.Error.WriteLine("No such file: {0}", dfsfilename);
return;
}
if (0 == string.Compare(df.Type, DfsFileTypes.NORMAL, StringComparison.OrdinalIgnoreCase)
|| 0 == string.Compare(df.Type, DfsFileTypes.BINARY_RECT, StringComparison.OrdinalIgnoreCase))
{
Console.WriteLine(df.Nodes.Count);
}
else
{
Console.Error.WriteLine("countparts not supported for file of type '{0}'", df.Type);
}
}
break;
case "filesize":
{
if (args.Length == 0)
{
Console.Error.WriteLine("Error: filesize command needs argument: <dfspath>");
SetFailure();
return;
}
if (!dfs.DfsConfigExists(DFSXMLPATH))
{
Console.Error.WriteLine("DFS not setup; use: {0} format", appname);
SetFailure();
return;
}
string dfsfilename = args[0];
dfs dc = LoadDfsConfig();
dfs.DfsFile df = DfsFindAny(dc, dfsfilename);
if (null == df)
{
Console.Error.WriteLine("No such file: {0}", dfsfilename);
SetFailure();
return;
}
if (0 == string.Compare(df.Type, DfsFileTypes.NORMAL, StringComparison.OrdinalIgnoreCase)
|| 0 == string.Compare(df.Type, DfsFileTypes.BINARY_RECT, StringComparison.OrdinalIgnoreCase))
{
Console.WriteLine(df.Size); // Byte count.
Console.WriteLine(Surrogate.GetFriendlyByteSize(df.Size)); // Friendly size.
}
else
{
Console.Error.WriteLine("filesize not supported for file of type '{0}'", df.Type);
SetFailure();
return;
}
}
break;
case "ls":
case "dir":
if (!dfs.DfsConfigExists(DFSXMLPATH))
{
Console.Error.WriteLine("DFS not setup; use: {0} format", appname);
SetFailure();
return;
}
{
int iarg = 0;
bool showhidden = false;
if (args.Length > iarg)
{
if ("-h" == args[iarg])
{
iarg++;
showhidden = true;
}
}
bool filterspecified = args.Length > iarg;
string filter = filterspecified ? args[iarg++] : "*";
if (filter.StartsWith("dfs://", StringComparison.OrdinalIgnoreCase))
{
filter = filter.Substring(6);
}
string srex = Surrogate.WildcardRegexString(filter);
System.Text.RegularExpressions.Regex rex = new System.Text.RegularExpressions.Regex(srex, System.Text.RegularExpressions.RegexOptions.IgnoreCase);
if (!filterspecified) // Only show [metadata] info if no specific filter.
{
long dcsize = dfs.GetDfsConfigSize(DFSXMLPATH);
string colorcode = "", endcolorcode = "";
ConsoleColor oldcolor = Console.ForegroundColor;
Console.ForegroundColor = ConsoleColor.Cyan;
if (isdspace)
{
colorcode = "\u00012";
endcolorcode = "\u00010";
}
Console.Write(" {0}{1,-40}{2} ", colorcode, "[metadata]", endcolorcode);
Console.WriteLine("{0,10}", GetFriendlyByteSize(dcsize));
Console.ForegroundColor = oldcolor;
}
long totalbytes = 0;
dfs dc = LoadDfsConfig();
string about = "";
int matchedCount = 0;
for (int i = 0; i < dc.Files.Count; i++)
{
if (rex.IsMatch(dc.Files[i].Name))
{
bool isnormalfile = 0 == string.Compare(dc.Files[i].Type, DfsFileTypes.NORMAL);
bool iszballfile = 0 == string.Compare(dc.Files[i].Type, DfsFileTypes.DELTA);
bool isjobsfile = 0 == string.Compare(dc.Files[i].Type, DfsFileTypes.JOB);
bool isdllfile = 0 == string.Compare(dc.Files[i].Type, DfsFileTypes.DLL);
bool istbl = false;
int RecordLength = dc.Files[i].RecordLength;
if (RecordLength > 0 || RecordLength == -2)
{
isnormalfile = true; // For most purposes here it's the same.
}
/*if (isnormalfile && dc.Files[i].Name.EndsWith(".tbl", StringComparison.OrdinalIgnoreCase))
{
istbl = true;
}*/
string ssize = " ";
if (isnormalfile || isdllfile) // jobs file doesn't update the file size yet!
{
ssize = GetFriendlyByteSize(dc.Files[i].Size);
if (dc.Files[i].Size < 0)
{
ssize = "?";
}
}
if (dc.Files[i].Size >= 0)
{
totalbytes += dc.Files[i].Size;
}
else
{
about = "~";
}
ConsoleColor oldcolor = ConsoleColor.Gray; // ...
string colorcode = "", endcolorcode = "";
int morespace = 0;
if (iszballfile)
{
oldcolor = Console.ForegroundColor;
Console.ForegroundColor = ConsoleColor.Cyan;
if (isdspace)
{
colorcode = "\u00012";
endcolorcode = "\u00010";
}
}
else if (isjobsfile)
{
if (isdspace)
{
colorcode = "\u00013";
endcolorcode = "\u00010";
}
}
else if (isdllfile || istbl)
{
oldcolor = Console.ForegroundColor;
Console.ForegroundColor = ConsoleColor.Cyan;
if (isdspace)
{
colorcode = "\u00012[\u00010";
endcolorcode = "\u00012]\u00010";
}
Console.ForegroundColor = oldcolor;
morespace += 2;
}
if (RecordLength > 0 || RecordLength == -2)
{
string srl = RecordLength > 0 ? RecordLength.ToString() : "?";
endcolorcode += "\u00012@" + srl + "\u00010";
morespace += 1 + srl.Length;
}
if (dc.Files[i].MemCache != null)
{
endcolorcode += " \u00012mc\u00010";
morespace += 3;
}
{
int iddx = dc.Files[i].Name.IndexOf(".$");
if (-1 != iddx)
{
iddx = dc.Files[i].Name.IndexOf(".$", iddx + 2);
if (-1 != iddx)
{
if (showhidden)
{
Console.ForegroundColor = ConsoleColor.Red;
colorcode = "\u00014";
endcolorcode = "\u00010";
}
else
{
continue;
}
}
}
}
if (isdllfile || istbl || RecordLength > 0 || RecordLength == -2)
{
Console.Write(" {0}{1}{2} ", colorcode, dc.Files[i].Name, endcolorcode);
int spacelen = 40 - dc.Files[i].Name.Length - morespace;
if (spacelen > 0)
{
Console.Write(new string(' ', spacelen));
}
}
else
{
Console.Write(" {0}{1,-40}{2} ", colorcode, dc.Files[i].Name, endcolorcode);
}
if (iszballfile || isjobsfile)
{
Console.ForegroundColor = oldcolor;
}
Console.Write("{0,10}", ssize);
if (isnormalfile)
{
Console.Write(" ({0} parts)", dc.Files[i].Nodes.Count);
}
Console.WriteLine();
matchedCount++;
}
}
Console.WriteLine(" {0} Distributed Files", matchedCount);
Console.WriteLine(" {0}{1} Used (data files)", about, GetFriendlyByteSize(totalbytes));
{
long freespace = 0;
long freemin = long.MaxValue;
int replicationFactor = dc.Replication;
string[] fslaves = dc.Slaves.SlaveList.Split(';');
//for (int fsi = 0; fsi < fslaves.Length; fsi++)
MySpace.DataMining.Threading.ThreadTools<string>.Parallel(
new Action<string>(
delegate(string fslave)
{
//string fslave = fslaves[fsi];
try
{
long x = (long)GetDiskFreeBytes(Surrogate.NetworkPathForHost(fslave));
if (replicationFactor > 1)
{
x = x / replicationFactor;
}
lock (fslaves)
{
if (x < freemin)
{
freemin = x;
}
freespace += x;
}
}
catch (Exception e)
{
LogOutputToFile("Error while calculating DFS disk spage usage: " + e.ToString());
/*if (!dc.IsFailoverEnabled)
{
throw;
}*/
}
}
), fslaves, fslaves.Length);
Console.WriteLine(" {0} Free ({1} node avg; {2} node min)",
GetFriendlyByteSize(freespace),
GetFriendlyByteSize(freespace / fslaves.Length),
GetFriendlyByteSize((freemin == long.MaxValue) ? 0 : freemin)
);
}
}
break;
case "copy":
case "cp":
if (args.Length < 2)
{
Console.Error.WriteLine("Error: dfs copy command needs arguments: <from-path> <to-path>");
SetFailure();
return;
}
{
bool isdfs0 = args[0].StartsWith("dfs://", StringComparison.OrdinalIgnoreCase);
bool isdfs1 = args[1].StartsWith("dfs://", StringComparison.OrdinalIgnoreCase);
if (isdfs0 && isdfs1)
{
Console.Error.WriteLine("Error: dfs copy DFS-to-DFS not supported yet");
SetFailure();
return;
}
if (!isdfs0 && !isdfs1)
{
//Console.Error.WriteLine("Error: dfs copy local-to-local not supported");
Console.Error.WriteLine("Error: dfs copy must contain at least one dfs://");
SetFailure();
return;
}
if (isdfs0)
{
DfsGet(args);
}
else //if (isdfs1)
{
DfsPut(args);
}
}
break;
case "get":
DfsGet(args);
break;
case "getbinary":
DfsGetBinary(args);
break;
case "put":
DfsPut(args);
break;
case "fput":
DfsFPut(args);
break;
case "fget":
DfsFGet(args);
break;
case "copyto":
DfsCopyTo(args);
break;
case "putbinary":
DfsPutBinary(args);
break;
case "bulkget":
DfsBulkGet(args);
break;
case "bulkput":
DfsBulkPut(args);
break;
case "shuffle":
DfsShuffle(args);
break;
case "getjobs":
if (args.Length < 1)
{
Console.Error.WriteLine("Argument expected: <localpath.dj>");
SetFailure();
return;
}
if (new System.IO.DirectoryInfo(args[0]).Exists)
{
Console.Error.WriteLine("Argument cannot be a directory. Argument expected: <localpath.dj>");
SetFailure();
return;
}
EnsureNetworkPath(args[0]);
//using (LockDfsMutex())
{
dfs dc = LoadDfsConfig();
int count = 0;
using (System.IO.StreamWriter sw = new System.IO.StreamWriter(args[0]))
{
for (int i = 0; i < dc.Files.Count; i++)
{
dfs.DfsFile f = dc.Files[i];
if (0 == string.Compare(f.Type, DfsFileTypes.JOB, StringComparison.OrdinalIgnoreCase))
{
try
{
if (f.Nodes.Count < 1)
{
throw new Exception("Error: -exec jobs file not in correct jobs DFS format");
}
dfs.DfsFile.FileNode fn = dc.Files[i].Nodes[0];
string content = System.IO.File.ReadAllText(Surrogate.NetworkPathForHost(fn.Host.Split(';')[0]) + @"\" + fn.Name);
sw.Write(f.Name);
sw.Write('\0');
sw.Write(f.Type);
sw.Write('\0');
sw.Write(content);
sw.Write('\0');
count++;
}
catch (Exception e)
{
Console.Error.WriteLine("Unable to get job '{0}': {1}", f.Name, e.Message);
}
}
}
}
Console.WriteLine("Saved {0} jobs files to jobs archive '{1}'", count, args[0]);
}
break;
case "putjobs":
if (args.Length < 1)
{
Console.Error.WriteLine("Argument expected: <localpath.dj>");
SetFailure();
return;
}
EnsureNetworkPath(args[0]);
{
string[] segs = System.IO.File.ReadAllText(args[0]).Split('\0'); // 0: name, 1: type, 2: content, etc.
int count = 0;
for (int si = 0; si + 2 < segs.Length; si += 3)
{
try
{
string fname = segs[si + 0];
string ftype = segs[si + 1];
string fcontent = segs[si + 2];
if (0 != string.Compare(ftype, DfsFileTypes.JOB, StringComparison.OrdinalIgnoreCase))
{
throw new Exception("File '" + fname + "' is of type '" + ftype + "', not of expected type '" + DfsFileTypes.JOB + "'");
}
else
{
if (!DfsPutJobsFileContent(fname, fcontent))
{
throw new Exception("Unable to write job '" + fname + "' to DFS; ensure that the file does not already exist in DFS");
}
}
Console.WriteLine(" {0}", fname);
count++;
}
catch (Exception e)
{
Console.Error.WriteLine("Problem importing job: {0}", e.Message);
}
}
Console.WriteLine("Done importing {0} jobs files into DFS", count);
}
break;
case "combine":
// Note: datanode chunk file header keeps the old file offset.
{
//System.Threading.Thread.Sleep(8000);
using (LockDfsMutex())
{
dfs dc = LoadDfsConfig();
List<string> inputs = new List<string>();
string outfn = null;
bool nextoutfn = false;
foreach (string arg in args)
{
if (nextoutfn)
{
if (null != outfn)
{
throw new Exception("Too many output files");
}
outfn = arg;
}
else
{
if ("+" == arg)
{
nextoutfn = true;
}
else if (arg.Length > 0 && '+' == arg[0])
{
if (null != outfn)
{
throw new Exception("Too many output files");
}
outfn = arg.Substring(1);
}
else
{
inputs.AddRange(SplitInputPaths(dc, arg));
}
}
}
if (0 == inputs.Count)
{
Console.Error.WriteLine("No input files to combine");
SetFailure();
return;
}
bool outisin = false;
if (null == outfn)
{
outfn = inputs[inputs.Count - 1];
outisin = true;
}
if (outfn.StartsWith("dfs://", StringComparison.OrdinalIgnoreCase))
{
outfn = outfn.Substring(6);
}
string reason = "";
if (dfs.IsBadFilename(outfn, out reason))
{
Console.Error.WriteLine("Invalid output file: {0}", reason);
SetFailure();
return;
}
if (null != DfsFindAny(dc, outfn))
{
if (outisin)
{
if (!QuietMode && InteractiveMode)
{
Console.Write("The specified file already exists in DFS; overwrite? ");
ConsoleFlush();
for (; ; )
{
string s = Console.ReadLine();
char ch = '\0';
if (0 != s.Length)
{
ch = char.ToUpper(s[0]);
}
if ('N' == ch)
{
Console.WriteLine(" Aborted by user");
return;
}
else if ('Y' == ch)
{
break; // !
}
else
{
Console.Write("Overwrite, yes or no? ");
ConsoleFlush();
}
}
}
}
else
{
Console.Error.WriteLine("Output file for combine already exists: {0}", outfn);
SetFailure();
return;
}
}
//if (verbose)
{
//Console.WriteLine("Combining {0} input files into file '{1}'", inputs.Count, outfn);
}
{
dfs.DfsFile dfout = new dfs.DfsFile();
dfout.Nodes = new List<dfs.DfsFile.FileNode>(inputs.Count * 32);
dfout.Name = outfn;
dfout.Size = 0;
int RecordLength = int.MinValue;
for (int i = 0; i < inputs.Count; i++)
{
dfs.DfsFile df = DfsFindAny(dc, inputs[i]);
if (null == df)
{
Console.Error.WriteLine("Combine error: input file '{0}' does not exist in DFS or was included more than once", inputs[i]);
SetFailure();
return;
}
if (0 != string.Compare(df.Type, DfsFileTypes.NORMAL, StringComparison.OrdinalIgnoreCase)
&& 0 != string.Compare(df.Type, DfsFileTypes.BINARY_RECT, StringComparison.OrdinalIgnoreCase))
{
Console.Error.WriteLine("DFS file '{0}' is not of expected type", df.Name);
SetFailure();
return;
}
{
int reclen = df.RecordLength;
if (int.MinValue != RecordLength
&& reclen != RecordLength)
{
Console.Error.WriteLine("Error: Record lengths of all input files must match; DFS file '{0}' has record length of {1}, expected record length of {2}", df.Name, (-1 == reclen) ? "<none>" : reclen.ToString(), (-1 == RecordLength) ? "<none>" : RecordLength.ToString());
SetFailure();
return;
}
RecordLength = reclen;
#if DEBUG
if (int.MinValue == RecordLength)
{
throw new Exception("DEBUG: (int.MinValue == RecordLength) after first file");
}
#endif
}
int j = dfout.Nodes.Count;
dfout.Nodes.AddRange(df.Nodes);
for (; j < dfout.Nodes.Count; j++)
{
dfout.Nodes[j].Position = dfout.Size; // !
dfout.Size += dfout.Nodes[j].Length; // !
}
dc.Files.Remove(df); // Ok since a failure will bail this out entirely, since the next DFS read re-loads.
}
if (RecordLength > 0)
{
dfout.XFileType = DfsFileTypes.BINARY_RECT + "@" + RecordLength.ToString();
}
dc.Files.Add(dfout);
UpdateDfsXml(dc); // !
//if (verbose)
{
Console.WriteLine("Combined {0} input files into file '{1}' of resulting size {2}", inputs.Count, outfn, GetFriendlyByteSize(dfout.Size));
}
}
}
}
break;
case "info":
case "information":
{
dfs dc = LoadDfsConfig();
if (null == dc)
{
Console.Error.WriteLine(" No " + dfs.DFSXMLNAME);
SetFailure();
}
else
{
string[] slaves = dc.Slaves.SlaveList.Split(',', ';');
bool mt = false;
bool shortname = true;
List<string> largs = new List<string>();
for (int i = 0; i < args.Length; i++)
{
string arg = args[i].ToLower();
switch(arg)
{
case "-mt":
mt = true;
break;
case "-s":
shortname = true;
break;
default:
largs.Add(arg);
break;
}
}
if (largs.Count == 0)
{
if (mt)
{
Dictionary<string, string> netpaths = new Dictionary<string, string>(slaves.Length);
Dictionary<string, string> hostnames = new Dictionary<string, string>(slaves.Length);
Dictionary<string, long> freesp = new Dictionary<string, long>(slaves.Length);
MySpace.DataMining.Threading.ThreadTools<string>.Parallel(
new Action<string>(delegate(string host)
{
host = host.ToUpper();
lock (hostnames)
{
if (shortname)
{
hostnames[host] = host;
}
else
{
hostnames[host] = IPAddressUtil.GetName(host);
}
string np = Surrogate.NetworkPathForHost(host);
netpaths[host] = np;
freesp[host] = (long)GetDiskFreeBytes(np);
}
}), slaves, slaves.Length);
{
string dfsfmt = "machines=";
for (int i = 0; i < slaves.Length; i++)
{
if (i != 0)
{
dfsfmt += ",";
}
dfsfmt += hostnames[slaves[i].ToUpper()];
}
dfsfmt += " processes=" + dc.Blocks.TotalCount.ToString();
if (dc.DataNodeBaseSize != dfs.DataNodeBaseSize_default)
{
dfsfmt += " datanodebasesize=" + dc.DataNodeBaseSize.ToString();
}
Console.WriteLine("[DFS information]");
Console.WriteLine(" Format: {0}", dfsfmt);
Console.WriteLine(" Files: {0}", dc.Files.Count);
}
MySpace.DataMining.Threading.ThreadTools<string>.Parallel(
new Action<string>(delegate(string host)
{
host = host.ToUpper();
try
{
string netpath = netpaths[host];
System.IO.DirectoryInfo netdi = new System.IO.DirectoryInfo(netpath);
long zdcount = 0;
long zdsizes = 0;
foreach (System.IO.FileInfo fi in (netdi).GetFiles("zd*.zd"))
{
zdcount++;
zdsizes += fi.Length;
}
long sbcount = 0;
long sbsizes = 0;
foreach (System.IO.FileInfo fi in (netdi).GetFiles(GetSnowballFilesWildcard("*")))
{
sbcount++;
sbsizes += fi.Length;
}
long diskfree = freesp[host];
lock (hostnames)
{
Console.WriteLine(" {0}:", hostnames[host]);
Console.WriteLine(" {0} data file parts", zdcount);
Console.WriteLine(" {0} cache parts", sbcount);
Console.WriteLine(" {0} total used", GetFriendlyByteSize(zdsizes + sbsizes));
Console.WriteLine(" {0} free", GetFriendlyByteSize(diskfree));
}
}
catch
{
string reason;
if (Surrogate.IsHealthySlaveMachine(host, out reason))
{
reason = "cannot query";
}
lock (hostnames)
{
Console.WriteLine(" Error: {0}", reason);
}
}
}), slaves, slaves.Length);
}
else
{
string[] netpaths = new string[slaves.Length];
Dictionary<int, long> freesp = new Dictionary<int, long>();
Console.WriteLine("[DFS information]");
{
string dfsfmt = "machines=";
for (int i = 0; i < slaves.Length; i++)
{
if (i != 0)
{
dfsfmt += ",";
}
dfsfmt += shortname ? slaves[i].ToUpper() : IPAddressUtil.GetName(slaves[i]);
string np = Surrogate.NetworkPathForHost(slaves[i]);
netpaths[i] = np;
freesp.Add(i, (long)GetDiskFreeBytes(np));
}
dfsfmt += " processes=" + dc.Blocks.TotalCount.ToString();
if (dc.DataNodeBaseSize != dfs.DataNodeBaseSize_default)
{
dfsfmt += " datanodebasesize=" + dc.DataNodeBaseSize.ToString();
}
Console.WriteLine(" Format: {0}", dfsfmt);
}
Console.WriteLine(" Files: {0}", dc.Files.Count);
List<KeyValuePair<int, long>> sfreesp = new List<KeyValuePair<int, long>>(freesp);
sfreesp.Sort(
delegate(KeyValuePair<int, long> firstPair, KeyValuePair<int, long> nextPair)
{
return -firstPair.Value.CompareTo(nextPair.Value);
}
);
foreach (KeyValuePair<int, long> item in sfreesp)
{
int si = item.Key;
Console.WriteLine(" {0}:", shortname ? slaves[si].ToUpper() : IPAddressUtil.GetName(slaves[si]));
try
{
string netpath = netpaths[si];
System.IO.DirectoryInfo netdi = new System.IO.DirectoryInfo(netpath);
long zdcount = 0;
long zdsizes = 0;
foreach (System.IO.FileInfo fi in (netdi).GetFiles("zd*.zd"))
{
zdcount++;
zdsizes += fi.Length;
}
long sbcount = 0;
long sbsizes = 0;
foreach (System.IO.FileInfo fi in (netdi).GetFiles(GetSnowballFilesWildcard("*")))
{
sbcount++;
sbsizes += fi.Length;
}
long diskfree = item.Value;
Console.WriteLine(" {0} data file parts", zdcount);
Console.WriteLine(" {0} cache parts", sbcount);
Console.WriteLine(" {0} total used", GetFriendlyByteSize(zdsizes + sbsizes));
Console.WriteLine(" {0} free", GetFriendlyByteSize(diskfree));
}
catch
{
string reason;
if (Surrogate.IsHealthySlaveMachine(slaves[si], out reason))
{
reason = "cannot query";
}
Console.WriteLine(" Error: {0}", reason);
}
}
}
}
else
{
if (-1 != largs[0].IndexOf(':'))
{
string[] specs = largs[0].Split(':'); // <file>:<host>
dfs.DfsFile df = DfsFindAny(dc, specs[0]);
if (null == df)
{
Console.Error.WriteLine(" No such file: {0}", specs[0]);
}
else if (0 == string.Compare(df.Type, DfsFileTypes.NORMAL, true)
|| 0 == string.Compare(df.Type, DfsFileTypes.BINARY_RECT, true))
{
bool HasSamples = df.RecordLength < 1;
Console.WriteLine("[DFS file information]");
Console.WriteLine(" DFS File: {0}", df.Name);
string shost = "";
{
Console.WriteLine(" Host: {0}", specs[1].ToUpper());
shost = IPAddressUtil.GetName(specs[1]);
}
/*string partspec = "";
if (specs.Length >= 3)
{
partspec = specs[2];
}*/
{
foreach (dfs.DfsFile.FileNode fn in df.Nodes)
{
int replindex = 0;
if (shost.Length > 0)
{
{
string[] fnxshosts = fn.Host.Split(';');
for (int i = 0; i < fnxshosts.Length; i++)
{
string fnshost = IPAddressUtil.GetName(fnxshosts[i]);
if (0 == string.Compare(shost, fnshost, StringComparison.OrdinalIgnoreCase))
{
replindex = i + 1;
break;
}
}
if (replindex < 1)
{
continue;
}
}
}
/*if (partspec.Length > 0)
{
if (0 != string.Compare(partspec, fn.Name, StringComparison.OrdinalIgnoreCase))
{
continue;
}
}*/
try
{
if (HasSamples)
{
Console.WriteLine(" {0} [{3}] ({1} data; {2} samples)",
fn.Name,
GetFriendlyByteSize((new System.IO.FileInfo(dfs.MapNodeToNetworkPath(fn))).Length),
GetFriendlyByteSize((new System.IO.FileInfo(dfs.MapNodeToNetworkPath(fn, true))).Length),
replindex
);
}
else
{
Console.WriteLine(" {0} [{2}] ({1} data)",
fn.Name,
GetFriendlyByteSize((new System.IO.FileInfo(dfs.MapNodeToNetworkPath(fn))).Length),
replindex
);
}
}
catch (Exception e)
{
LogOutputToFile(e.ToString());
Console.WriteLine(" {0}", fn.Name);
}
}
}
}
else
{
Console.Error.WriteLine("DFS file '{0}' is not of expected type", df.Type);
SetFailure();
return;
}
}
else
{
dfs.DfsFile df = DfsFindAny(dc, largs[0]);
if (null == df)
{
Console.Error.WriteLine(" No such file: {0}", largs[0]);
}
else
{
Console.WriteLine("[DFS file information]");
if (0 == string.Compare(df.Type, DfsFileTypes.NORMAL, true)
|| 0 == string.Compare(df.Type, DfsFileTypes.BINARY_RECT, true))
{
bool HasSamples = df.RecordLength < 1 && df.RecordLength != -2;
Console.WriteLine(" DFS File: {0}", df.Name);
int RecordLength = df.RecordLength;
if (RecordLength > 0 || RecordLength == -2)
{
Console.WriteLine(" Record Length: {0}", (RecordLength > 0 ? RecordLength.ToString() : "?"));
}
Console.WriteLine(" Size: {0} ({1})", GetFriendlyByteSize(df.Size), df.Size);
if (HasSamples)
{
long samplesize = 0;
MySpace.DataMining.Threading.ThreadTools<dfs.DfsFile.FileNode>.Parallel(
new Action<dfs.DfsFile.FileNode>(delegate(dfs.DfsFile.FileNode fn)
{
try
{
System.IO.FileInfo fi = new System.IO.FileInfo(dfs.MapNodeToNetworkPath(fn, true));
int ss = (int)fi.Length;
System.Threading.Interlocked.Add(ref samplesize, ss);
}
catch
{
}
}), df.Nodes, slaves.Length);
string avg = "0";
if (df.Nodes.Count > 0)
{
avg = GetFriendlyByteSize(samplesize / df.Nodes.Count);
}
Console.WriteLine(" Sample Size: {0} ({1} avg)", GetFriendlyByteSize(samplesize), avg);
}
Console.WriteLine(" Parts: {0}", df.Nodes.Count);
{
Dictionary<string, int> partsonhosts = new Dictionary<string, int>();
Dictionary<string, long> zdsizeonhosts = new Dictionary<string, long>();
Dictionary<string, long> zsasizeonhosts = new Dictionary<string, long>();
for (int i = 0; i < df.Nodes.Count; i++)
{
int value;
long zdsize;
long zsasize;
string[] xkeys = df.Nodes[i].Host.Split(';');
for (int ik = 0; ik < xkeys.Length; ik++)
{
string key = xkeys[ik].ToUpper();
if (partsonhosts.ContainsKey(key))
{
value = partsonhosts[key];
zdsize = zdsizeonhosts[key];
zsasize = zsasizeonhosts[key];
}
else
{
value = 0;
zdsize = 0;
zsasize = 0;
}
value++;
try
{
zdsize += (new System.IO.FileInfo(dfs.MapNodeToNetworkPath(df.Nodes[i]))).Length;
}
catch
{
}
if (HasSamples)
{
try
{
zsasize += (new System.IO.FileInfo(dfs.MapNodeToNetworkPath(df.Nodes[i], true))).Length;
}
catch
{
}
}
partsonhosts[key] = value;
zdsizeonhosts[key] = zdsize;
zsasizeonhosts[key] = zsasize;
}
}
foreach (KeyValuePair<string, int> kvp in partsonhosts)
{
if (HasSamples)
{
Console.WriteLine(" {0} chunks on {1} ({2} data; {3} samples)",
kvp.Value, kvp.Key,
GetFriendlyByteSize(zdsizeonhosts[kvp.Key]), GetFriendlyByteSize(zsasizeonhosts[kvp.Key])
);
}
else
{
Console.WriteLine(" {0} chunks on {1} ({2} data)",
kvp.Value, kvp.Key,
GetFriendlyByteSize(zdsizeonhosts[kvp.Key])
);
}
}
}
}
else if (0 == string.Compare(df.Type, "zsb", true))
{
Console.WriteLine(" DFS Delta: {0}", df.Name);
long sbsz = 0;
int sbparts = 0;
{
string fnwc = GetSnowballFilesWildcard(df.Name);
//string[] slaves = dc.Slaves.SlaveList.Split(',', ';');
try
{
for (int si = 0; si < slaves.Length; si++)
{
string netpath = Surrogate.NetworkPathForHost(slaves[si]);
foreach (System.IO.FileInfo fi in (new System.IO.DirectoryInfo(netpath)).GetFiles(fnwc))
{
sbparts++;
sbsz += fi.Length;
}
}
}
catch
{
sbparts = -1;
sbsz = -1;
}
}
Console.WriteLine(" Size: {0} ({1})", (sbsz >= 0) ? GetFriendlyByteSize(sbsz) : "?", (sbsz >= 0) ? sbsz.ToString() : "?");
Console.WriteLine(" Parts: {0}", (sbparts >= 0) ? sbparts.ToString() : "?");
Console.WriteLine(" Cached Inputs: {0}", df.Nodes.Count);
ConsoleColor oldcolor = ConsoleColor.Gray;
string colorcode = "";
string nodeName = "";
foreach (dfs.DfsFile.FileNode fn in df.Nodes)
{
Console.ForegroundColor = oldcolor;
Console.Write(" Input:");
if (fn.Name.StartsWith(invalidatedCacheToken))
{
nodeName = fn.Name.Substring(invalidatedCacheToken.Length + 1);
Console.ForegroundColor = ConsoleColor.DarkGray;
if (isdspace)
{
colorcode = "\u00015";
}
}
else
{
nodeName = fn.Name;
Console.ForegroundColor = oldcolor;
colorcode = "";
}
Console.WriteLine("{0}{1}{2}", colorcode, nodeName, colorcode.Length != 0 ? "\u00010" : "");
}
Console.ForegroundColor = oldcolor;
}
else
{
Console.Error.WriteLine(" No info for file of type '{0}'", df.Type);
}
}
}
}
}
}
break;
case "partinfo":
{
if (args.Length < 1)
{
Console.Error.WriteLine("qizmt partinfo <partname>");
SetFailure();
}
else
{
string nodename = args[0];
dfs dc = LoadDfsConfig();
string ownerfilename = null;
dfs.DfsFile.FileNode fn = DfsFindFileNode(dc, nodename, out ownerfilename);
if (fn == null)
{
Console.WriteLine("Part not found in dfs.xml");
}
else
{
Console.WriteLine();
Console.WriteLine("Owner file name: {0}", ownerfilename);
Console.WriteLine();
Console.WriteLine("Part paths in metadata:");
Console.WriteLine();
string[] nhosts = fn.Host.Split(';');
for (int hi = 0; hi < nhosts.Length; hi++)
{
Console.WriteLine(NetworkPathForHost(nhosts[hi]) + @"\" + fn.Name);
Console.WriteLine();
}
Console.WriteLine();
Console.WriteLine("Part paths in physical files:");
Console.WriteLine();
ConsoleColor oldcolor = Console.ForegroundColor;
string colorcode = "\u00014";
string endcolorcode = "\u00010";
for (int hi = 0; hi < nhosts.Length; hi++)
{
string ppath = NetworkPathForHost(nhosts[hi]) + @"\" + fn.Name;
if (!System.IO.File.Exists(ppath))
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("{0}{1} does not exist{2}", colorcode, ppath, endcolorcode);
Console.ForegroundColor = oldcolor;
}
else
{
Console.WriteLine(ppath);
}
Console.WriteLine();
}
}
}
}
break;
case "delchunk":
{
if (args.Length < 2)
{
Console.Error.WriteLine("qizmt delchunk <chunkname> <host>");
SetFailure();
return;
}
else
{
string nodename = args[0];
string delhost = args[1];
dfs.DfsFile.FileNode fn = null;
bool metaremoved = false;
using (LockDfsMutex())
{
dfs dc = LoadDfsConfig();
string ownerfilename = null;
fn = DfsFindFileNode(dc, nodename, out ownerfilename);
if (fn == null)
{
Console.WriteLine("Part not found in dfs.xml");
return;
}
else
{
string[] nhosts = fn.Host.Split(';');
string goodhosts = "";
for (int hi = 0; hi < nhosts.Length; hi++)
{
if (string.Compare(nhosts[hi], delhost, true) != 0)
{
if (goodhosts.Length > 0)
{
goodhosts += ';';
}
goodhosts += nhosts[hi];
}
else
{
metaremoved = true;
}
}
if (goodhosts.Length > 0)
{
fn.Host = goodhosts;
}
else
{
//remove this node all together
dfs.DfsFile df = DfsFindAny(dc, ownerfilename);
if (df == null)
{
Console.Error.WriteLine("Cannot locate owner file.");
return;
}
else
{
long filesize = 0;
List<dfs.DfsFile.FileNode> goodnodes = new List<dfs.DfsFile.FileNode>(df.Nodes.Count - 1);
for (int ni = 0; ni < df.Nodes.Count; ni++)
{
dfs.DfsFile.FileNode thisnode = df.Nodes[ni];
if (string.Compare(thisnode.Name, nodename, true) != 0)
{
goodnodes.Add(thisnode);
thisnode.Position = filesize;
filesize += thisnode.Length;
}
}
df.Size = filesize;
df.Nodes = goodnodes;
}
}
UpdateDfsXml(dc);
}
}
bool physicalfileremoved = false;
try
{
string ppath = NetworkPathForHost(delhost) + @"\" + fn.Name;
if (System.IO.File.Exists(ppath) || System.IO.File.Exists(ppath + ".zsa"))
{
System.IO.File.Delete(ppath);
System.IO.File.Delete(ppath + ".zsa");
physicalfileremoved = true;
}
}
catch
{
}
Console.WriteLine("Chunk deleted successfully from host:");
if (metaremoved)
{
Console.WriteLine("Metadata removed");
}
if (physicalfileremoved)
{
Console.WriteLine("Physical file deleted");
}
}
}
break;
case "\u0040format":
case "format":
{
EnterAdminCmd();
bool verify = false;
if (args.Length == 1 &&
(0 == string.Compare(args[0], "vacuum", true) || 0 == string.Compare(args[0], "vacuum=true", true)))
{
Console.Error.WriteLine("Use: {0} killall", appname);
}
else // Normal format...
{
int blockcount = -1;
int sortedblockcount = -1;
string[] slavelist = null;
int datanodebasesize = 0;
int zmapblockcount = 0;
int zblockcount = 0;
//int zblockaddbuffersize = 0;
//int zblockreadbuffersize = 0;
int filebuffersizeoverride = 0;
byte compresszmapblocks = 127;
byte compressdfschunks = 127;
int numargs = 0;
int replication = 0;
ulong btreeCapSize = 0;
int logexechistory = 0;
int cooktimeout = -1;
int cookretries = -1;
bool mt = false;
bool metaonly = false;
string metabackuplocation = null;
int failovertimeout = -1;
int failoverdocheck = -1;
foreach (string arg in args)
{
string optname = "", optvalue = "";
{
int oi = arg.IndexOf('=');
if (-1 == oi)
{
optname = arg;
optvalue = "";
}
else
{
optname = arg.Substring(0, oi);
optvalue = arg.Substring(oi + 1);
}
}
numargs++;
switch (optname.ToLower())
{
case "blocks":
case "processes":
case "groupedprocesses":
blockcount = int.Parse(optvalue);
break;
case "sortedprocesses":
sortedblockcount = int.Parse(optvalue);
break;
case "slaves":
case "machines":
if (optvalue[0] == '@')
{
slavelist = Surrogate.GetHostsFromFile(optvalue.Substring(1));
}
else
{
slavelist = optvalue.Split(';', ',');
}
break;
case "replication":
case "replicationfactor":
replication = int.Parse(optvalue);
break;
case "datanodebasesize":
datanodebasesize = ParseCapacity(optvalue);
break;
case "zmapblockcount":
zmapblockcount = int.Parse(optvalue);
break;
case "zblockcount":
zblockcount = int.Parse(optvalue);
break;
case "zblockaddbuffersize":
//zblockaddbuffersize = ParseCapacity(optvalue);
Console.Error.WriteLine("zblockaddbuffersize no longer supported, use FileBufferSizeOverride");
SetFailure();
return;
case "zblockreadbuffersize":
//zblockreadbuffersize = ParseCapacity(optvalue);
Console.Error.WriteLine("zblockreadbuffersize no longer supported, use FileBufferSizeOverride");
SetFailure();
return;
case "filebuffersizeoverride":
filebuffersizeoverride = ParseCapacity(optvalue);
break;
case "compresszmapblocks":
switch (optvalue.ToLower())
{
case "true":
case "1":
compresszmapblocks = 1;
break;
case "false":
case "0":
compresszmapblocks = 0;
break;
default:
throw new Exception("Unknown value for 'compresszmapblocks'");
}
break;
case "compressdfschunks":
switch (optvalue.ToLower())
{
case "true":
case "1":
compressdfschunks = 1;
break;
case "false":
case "0":
compressdfschunks = 0;
break;
default:
throw new Exception("Unknown value for 'compressdfschunks'");
}
break;
case "btreecapsize":
btreeCapSize = (ulong)AELight.ParseLongCapacity((optvalue));
break;
case "logexechistory":
logexechistory = int.Parse(optvalue);
break;
case "vacuum":
Console.Error.WriteLine("Error: 'vacuum' cannot be used with other options", arg);
return;
case "?":
numargs--;
break;
case "verify":
switch (optvalue.ToLower())
{
case "true":
case "1":
verify = true;
break;
case "false":
case "0":
verify = false;
break;
default:
throw new Exception("Unknown value for 'Verify'");
}
break;
case "cooktimeout":
cooktimeout = int.Parse(optvalue);
break;
case "cookretries":
cookretries = int.Parse(optvalue);
break;
case "multithreaded":
switch (optvalue.ToLower())
{
case "true":
case "1":
mt = true;
break;
case "false":
case "0":
mt = false;
break;
default:
throw new Exception("Unknown value for 'Multithreaded'");
}
break;
case "metaonly":
switch (optvalue.ToLower())
{
case "true":
case "1":
metaonly = true;
break;
case "false":
case "0":
metaonly = false;
break;
default:
throw new Exception("Unknown value for 'MetaOnly'");
}
break;
case "metabackuplocation":
case "metabackup":
metabackuplocation = optvalue;
break;
case "failovertimeout":
failovertimeout = int.Parse(optvalue);
break;
case "failoverdocheck":
failoverdocheck = int.Parse(optvalue);
break;
default:
Console.Error.WriteLine("Error: unknown option for dfs format: {0}", arg);
return;
}
}
if (0 == numargs)
{
Console.Error.WriteLine("Format arguments:");
Console.Error.WriteLine(" Machines=<host1>[,<host2>...]");
Console.Error.WriteLine(" [Processes=<num>]");
//Console.Error.WriteLine(" [SortedProcesses=<num>]");
Console.Error.WriteLine(" [Replication=<num>]");
Console.Error.WriteLine(" [DataNodeBaseSize=<size>]");
//Console.Error.WriteLine(" [ZMapBlockCount=<count>]");
Console.Error.WriteLine(" [ZBlockCount=<size>]");
//Console.Error.WriteLine(" [ZBlockAddBufferSize=<size>]");
//Console.Error.WriteLine(" [ZBlockReadBufferSize=<size>]");
Console.Error.WriteLine(" [FileBufferSizeOverride=<size>]");
Console.Error.WriteLine(" [CompressZMapBlocks=<bool>]");
Console.Error.WriteLine(" [CompressDfsChunks=<bool>]");
Console.Error.WriteLine(" [LogExecHistory=<num>]");
Console.Error.WriteLine(" [BTreeCapSize=<size>]");
Console.Error.WriteLine(" [CookTimeout=<ms>]");
Console.Error.WriteLine(" [CookRetries=<num>]");
Console.Error.WriteLine(" [MetaBackupLocation=<dir>]");
Console.Error.WriteLine(" [Verify=<bool>]");
Console.Error.WriteLine(" [Multithreaded=<bool>]");
Console.Error.WriteLine(" [MetaOnly=<bool>]");
return;
}
if (null == slavelist)
{
Console.Error.WriteLine("Error: \"Machines=<host1>[,<host2>...]\" expected");
SetFailure();
return;
}
{
Dictionary<string, bool> alls = new Dictionary<string, bool>(slavelist.Length);
foreach (string ss in slavelist)
{
string coolss = IPAddressUtil.GetName(ss);
if (alls.ContainsKey(coolss))
{
Console.Error.WriteLine("host in there twice {0} lol", ss);
SetFailure();
return;
}
alls.Add(coolss, true);
}
}
if (verify)
{
string[] sl = new string[1];
bool vOK = true;
foreach (string s in slavelist)
{
sl[0] = s;
if (!VerifyHostPermissions(sl))
{
Console.Error.WriteLine("Ensure the Windows service is installed and running on '{0}'", s);
vOK = false;
}
}
if (vOK)
{
Console.WriteLine("All machines are verified.");
}
else
{
Console.WriteLine();
Console.Error.WriteLine("Unable to format.");
SetFailure();
return;
}
}
if (dfs.DfsConfigExists(DFSXMLPATH, 1))
{
Console.WriteLine("DFS exists; reformatting...");
Console.WriteLine("Consider running killall after format");
if (!metaonly)
{
}
}
else
{
}
try
{
System.IO.File.Delete(DFSXMLPATH);
}
catch
{
}
{
dfs dc = new dfs();
dc.InitNew();
string sslavelist = "";
{
StringBuilder sb = new StringBuilder();
for (int i = 0; i < slavelist.Length; i++)
{
if (sb.Length != 0)
{
sb.Append(';');
}
sb.Append(slavelist[i].Trim());
}
sslavelist = sb.ToString();
}
dc.Slaves.SlaveList = sslavelist;
dc.Blocks = new dfs.ConfigBlocks();
if (blockcount <= 0)
{
blockcount = NearestPrimeGE(slavelist.Length * Surrogate.NumberOfProcessors);
}
dc.Blocks.TotalCount = blockcount;
if (sortedblockcount <= 0)
{
sortedblockcount = slavelist.Length * Surrogate.NumberOfProcessors;
}
dc.Blocks.SortedTotalCount = sortedblockcount;
if (datanodebasesize > 0)
{
dc.DataNodeBaseSize = datanodebasesize;
}
if (zmapblockcount > 0)
{
dc.slave.zblocks.count = zmapblockcount;
}
if (replication > 0)
{
if (replication > slavelist.Length)
{
Console.Error.WriteLine("Cannot format with replication factor higher than the number of machines in the cluster (replication {0} > {1} machines)", replication, slavelist.Length);
SetFailure();
return;
}
dc.Replication = replication;
}
if (btreeCapSize > 0)
{
dc.BTreeCapSize = btreeCapSize;
}
if (logexechistory > 0)
{
dc.LogExecHistory = logexechistory;
}
if (zblockcount > 0)
{
dc.slave.zblocks.count = zblockcount;
}
/*if (zblockaddbuffersize > 0)
{
dc.slave.zblocks.addbuffersize = zblockaddbuffersize;
}*/
/*if (zblockreadbuffersize > 0)
{
dc.slave.zblocks.readbuffersize = zblockreadbuffersize;
}*/
if (filebuffersizeoverride > 0)
{
dc.slave.FileBufferSizeOverride = filebuffersizeoverride;
}
if (127 != compressdfschunks)
{
dc.slave.CompressDfsChunks = compressdfschunks;
}
if (127 != compressdfschunks)
{
dc.slave.CompressZMapBlocks = compresszmapblocks;
}
if (cooktimeout >= 0)
{
dc.slave.CookTimeout = cooktimeout;
}
if (cookretries >= 0)
{
dc.slave.CookRetries = cookretries;
}
if (failovertimeout >= 0)
{
dc.FailoverTimeout = failovertimeout;
}
if (failoverdocheck >= 0)
{
dc.FailoverDoCheck = failoverdocheck;
}
try
{
if (null != metabackuplocation)
{
if (string.Empty == metabackuplocation)
{
dc.MetaBackup = "";
//Console.WriteLine("MetaBackupLocation explicitly disabled");
}
else
{
if (metabackuplocation.StartsWith(@"\\"))
{
dc.MetaBackup = metabackuplocation;
}
else // If not a network path, make it one (relative to current-machine/surrogate).
{
// Using GetHostName here becuase during format, the current machine is the surrogate.
dc.MetaBackup = Surrogate.LocalPathToNetworkPath(metabackuplocation, System.Net.Dns.GetHostName());
}
try
{
EnsureMetaBackupLocation(dc);
}
catch
{
dc.MetaBackup = "";
throw;
}
foreach (string fn in System.IO.Directory.GetFiles(dc.GetMetaBackupLocation()))
{
System.IO.File.Delete(fn);
}
}
}
}
catch (Exception e)
{
LogOutputToFile(e.ToString());
Console.Error.WriteLine(e.Message);
}
//Delete dfsxml from slaves.
{
string[] slaves = dc.Slaves.SlaveList.Split(';');
string self = GetSelfHost(slaves);
foreach (string slave in slaves)
{
if (self != slave)
{
string dfsxmlpath = Surrogate.NetworkPathForHost(slave) + "\\" + dfs.DFSXMLNAME;
// Not using dfs.DfsConfigExists() here because we're testing all slaves.
if (System.IO.File.Exists(dfsxmlpath))
{
System.IO.File.Delete(dfsxmlpath);
}
}
}
}
UpdateDfsXml(dc);
try
{
// Ensure master isn't an old slave.
System.IO.File.Delete(AELight_Dir + @"\slave.dat");
}
catch
{
}
foreach (string slave in dc.Slaves.SlaveList.Split(';'))
{
WriteSlaveDat(slave);
}
Console.WriteLine("DFS setup: {0} processes on {1} machines", dc.Blocks.TotalCount, slavelist.Length);
}
}
}
break;
case "invalidate":
if (!dfs.DfsConfigExists(DFSXMLPATH))
{
Console.Error.WriteLine("DFS not setup; use: {0} format", appname);
SetFailure();
return;
}
if (args.Length < 2)
{
Console.Error.WriteLine("Error: dfs invalidate command needs arguments: <cacheName> <fileNodeName>");
SetFailure();
return;
}
DfsInvalidateCachedFileNode(args);
break;
default:
Console.Error.WriteLine("Unrecognized DFS command: " + cmd);
SetFailure();
return;
}
}
}
static void _MemCacheLoad(dfs.DfsFile df)
{
dfs dc = LoadDfsConfig();
string[] slaves = dc.Slaves.SlaveList.Split(';');
{
//foreach (string slave in slaves)
MySpace.DataMining.Threading.ThreadTools<string>.Parallel(
new Action<string>(
delegate(string slave)
{
System.Net.Sockets.NetworkStream nstm = Surrogate.ConnectService(slave);
nstm.WriteByte((byte)'C');
nstm.WriteByte((byte)'l');
XContent.SendXContent(nstm, df.Name);
int ich = nstm.ReadByte();
if ('+' != ich)
{
string errmsg = null;
if ('-' == ich)
{
try
{
errmsg = XContent.ReceiveXString(nstm, null);
}
catch
{
}
}
if (null != errmsg)
{
throw new Exception("Error received from DO service during MemCache load from " + slave + ": " + errmsg);
}
throw new Exception("Did not receive a success signal from DO service during MemCache load from " + slave);
}
}), slaves, slaves.Length);
}
}
public static void CheckSummt(string hashname, string dfsreader)
{
dfs dc = LoadDfsConfig();
if (null == DfsFindAny(dc, dfsreader))
{
Console.Error.WriteLine("Input file does not exist in DFS: {0}", dfsreader);
SetFailure();
return;
}
string tempfnpost = ".$" + Guid.NewGuid().ToString() + ".$" + System.Diagnostics.Process.GetCurrentProcess().Id.ToString();
string jobsfn = "gen" + hashname + "-jobs.xml" + tempfnpost;
try
{
using (System.IO.StreamWriter sw = System.IO.File.CreateText(jobsfn))
{
sw.Write((@"<SourceCode>
<Jobs>
<Job Name=`Generate " + hashname + @"` Custodian=`` Email=``>
<IOSettings>
<JobType>mapreduce</JobType>
<KeyLength>4</KeyLength>
<DFSInput>" + dfsreader + @"</DFSInput>
<DFSOutput></DFSOutput>
<OutputMethod>grouped</OutputMethod>
</IOSettings>
<MapReduce>
<Map>
<![CDATA[
ulong lsum = 0;
public virtual void Map(ByteSlice line, MapOutput output)
{
byte[] buf = line.ToBytes();
for(int i=0; i < buf.Length ; i++)
{
lsum += buf[i];
}
if(StaticGlobals.Qizmt_Last)
{
recordset rValue = recordset.Prepare();
recordset rKey = recordset.Prepare();
rKey.PutInt(0);
rValue.PutULong(lsum);
output.Add(rKey,rValue);
}
}
]]>
</Map>
<Reduce>
<![CDATA[
public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output)
{
ulong lsum = 0;
for(int i = 0; i < values.Length; i++)
{
recordset res = recordset.Prepare(values.Items[i]);
ulong temp = res.GetULong();
lsum += temp;
}
StringBuilder sbresult = new StringBuilder(32);
sbresult.Append(`\n" + hashname + @"` + ` of `);
sbresult.Append(@`dfs://" + dfsreader + @"`);
sbresult.Append(`: `);
sbresult.Append(lsum);
DSpace_Log(sbresult.ToString());
}
]]>
</Reduce>
</MapReduce>
</Job>
</Jobs>
</SourceCode>
").Replace("`", "\""));
}
Console.WriteLine("Generating {0}...", hashname);
Exec("", LoadConfig(jobsfn), new string[] { }, false, false);
}
finally
{
try
{
System.IO.File.Delete(jobsfn);
}
catch
{
}
}
}
static void ServiceCommands(string[] args)
{
if (args.Length <= 1 || !System.IO.File.Exists(args[1]))
{
throw new Exception("Expected path to DFS.xml");
}
string dfsxmlpath = args[1];
dfs dc = dfs.ReadDfsConfig_unlocked(dfsxmlpath);
string masterhost = IPAddressUtil.GetName(Surrogate.MasterHost);
string[] allmachines;
{
string[] sl = dc.Slaves.SlaveList.Split(';');
List <string> aml = new List <string>(sl.Length + 1);
aml.Add(masterhost);
foreach (string slave in sl)
{
if (0 != string.Compare(IPAddressUtil.GetName(slave), IPAddressUtil.GetName(masterhost), StringComparison.OrdinalIgnoreCase))
{
aml.Add(IPAddressUtil.GetName(slave));
}
}
allmachines = aml.ToArray();
}
string tempdir = @"\\" + System.Net.Dns.GetHostName() + @"\c$\temp";
if (!System.IO.Directory.Exists(tempdir))
{
System.IO.Directory.CreateDirectory(tempdir);
}
string hoststxtpath = tempdir + @"\dspace-hosts-" + Guid.NewGuid().ToString() + ".txt";
System.IO.File.WriteAllText(hoststxtpath, "# Test" + Environment.NewLine + string.Join(Environment.NewLine, allmachines));
try
{
Console.WriteLine("Confirm initial cluster status...");
{
Dictionary <string, string> status = _GetStatusAll("");
if (status.Count != allmachines.Length)
{
throw new Exception("Qizmt servicestatusall: (status.Count != allmachines.Length)");
}
try
{
_EnsureAllStatus(status, allmachines, "RUNNING");
}
catch (Exception e)
{
throw new Exception("Qizmt servicestatusall: " + e.Message, e);
}
}
{
string arg = string.Join(",", allmachines);
Dictionary <string, string> status = _GetStatusAll(arg);
if (status.Count != allmachines.Length)
{
throw new Exception("Qizmt servicestatusall " + arg + ": (status.Count != allmachines.Length)");
}
try
{
_EnsureAllStatus(status, allmachines, "RUNNING");
}
catch (Exception e)
{
throw new Exception("Qizmt servicestatusall " + arg + ": " + e.Message, e);
}
}
{
string arg = "\"@" + hoststxtpath + "\"";
Dictionary <string, string> status = _GetStatusAll(arg);
if (status.Count != allmachines.Length)
{
throw new Exception("Qizmt servicestatusall " + arg + ": (status.Count != allmachines.Length)");
}
try
{
_EnsureAllStatus(status, allmachines, "RUNNING");
}
catch (Exception e)
{
throw new Exception("Qizmt servicestatusall " + arg + ": " + e.Message, e);
}
}
Console.WriteLine("Stopping surrogate...");
Console.WriteLine(Exec.Shell("Qizmt stopsurrogate"));
Console.WriteLine("Confirm stopped status of surrogate...");
{
string host, statusmsg;
if (!_GetStatusFromLine(Exec.Shell("Qizmt servicestatussurrogate"), out host, out statusmsg) ||
statusmsg.StartsWith("RUNNING"))
{
throw new Exception("Qizmt servicestatussurrogate");
}
}
{
Dictionary <string, string> status = _GetStatusAll("");
if (status.Count != allmachines.Length)
{
throw new Exception("(surrogate down check) Qizmt servicestatusall: (status.Count != allmachines.Length)");
}
try
{
_EnsureAllStatus(status, allmachines, "RUNNING");
throw new Exception("(surrogate down check) Qizmt servicestatusall: cluster reported healthy, but surrogate should be down");
}
catch
{
// Expected!
}
}
Console.WriteLine("Starting surrogate...");
Console.WriteLine(Exec.Shell("Qizmt startsurrogate"));
Console.WriteLine("Confirm started status of surrogate...");
{
string host, statusmsg;
if (!_GetStatusFromLine(Exec.Shell("Qizmt servicestatussurrogate"), out host, out statusmsg) ||
!statusmsg.StartsWith("RUNNING"))
{
throw new Exception("(restarted surrogate check) Qizmt servicestatussurrogate");
}
}
{
Dictionary <string, string> status = _GetStatusAll("");
if (status.Count != allmachines.Length)
{
throw new Exception("(restarted surrogate check) Qizmt servicestatusall: (status.Count != allmachines.Length)");
}
try
{
_EnsureAllStatus(status, allmachines, "RUNNING");
}
catch (Exception e)
{
throw new Exception("(restarted surrogate check) Qizmt servicestatusall: " + e.Message, e);
}
}
{
string arg = string.Join(",", allmachines);
Dictionary <string, string> status = _GetStatusAll(arg);
if (status.Count != allmachines.Length)
{
throw new Exception("(restarted surrogate check) Qizmt servicestatusall " + arg + ": (status.Count != allmachines.Length)");
}
try
{
_EnsureAllStatus(status, allmachines, "RUNNING");
}
catch (Exception e)
{
throw new Exception("(restarted surrogate check) Qizmt servicestatusall " + arg + ": " + e.Message, e);
}
}
{
string arg = "\"@" + hoststxtpath + "\"";
Dictionary <string, string> status = _GetStatusAll(arg);
if (status.Count != allmachines.Length)
{
throw new Exception("(restarted surrogate check) Qizmt servicestatusall " + arg + ": (status.Count != allmachines.Length)");
}
try
{
_EnsureAllStatus(status, allmachines, "RUNNING");
}
catch (Exception e)
{
throw new Exception("(restarted surrogate check) Qizmt servicestatusall " + arg + ": " + e.Message, e);
}
}
foreach (string xallargs in new string[] { "", string.Join(",", allmachines), "\"@" + hoststxtpath + "\"" })
{
Console.WriteLine("Stopping all services in cluster, including surrogate...");
Console.WriteLine(Exec.Shell("Qizmt stopall " + xallargs));
Console.WriteLine("Confirming stopped status of all services...");
{
Dictionary <string, string> status = _GetStatusAll("");
if (status.Count != allmachines.Length)
{
throw new Exception("stopall " + xallargs + ": (stopped all services check) Qizmt servicestatusall: (status.Count != allmachines.Length)");
}
try
{
_EnsureAllStatus(status, allmachines, "STOPPED");
}
catch (Exception e)
{
throw new Exception("stopall " + xallargs + ": (stopped all services check) Qizmt servicestatusall: " + e.Message, e);
}
}
{
string arg = string.Join(",", allmachines);
Dictionary <string, string> status = _GetStatusAll(arg);
if (status.Count != allmachines.Length)
{
throw new Exception("stopall " + xallargs + ": (stopped all services check) Qizmt servicestatusall " + arg + ": (status.Count != allmachines.Length)");
}
try
{
_EnsureAllStatus(status, allmachines, "STOPPED");
}
catch (Exception e)
{
throw new Exception("stopall " + xallargs + ": (stopped all services check) Qizmt servicestatusall " + arg + ": " + e.Message, e);
}
}
{
string arg = "\"@" + hoststxtpath + "\"";
Dictionary <string, string> status = _GetStatusAll(arg);
if (status.Count != allmachines.Length)
{
throw new Exception("stopall " + xallargs + ": (stopped all services check) Qizmt servicestatusall " + arg + ": (status.Count != allmachines.Length)");
}
try
{
_EnsureAllStatus(status, allmachines, "STOPPED");
}
catch (Exception e)
{
throw new Exception("stopall " + xallargs + ": (stopped all services check) Qizmt servicestatusall " + arg + ": " + e.Message, e);
}
}
Console.WriteLine("Starting all services in cluster, including surrogate...");
Console.WriteLine(Exec.Shell("Qizmt startall " + xallargs));
Console.WriteLine("Confirm started status of all services...");
{
Dictionary <string, string> status = _GetStatusAll("");
if (status.Count != allmachines.Length)
{
throw new Exception("startall " + xallargs + ": (restarted all services check) Qizmt servicestatusall: (status.Count != allmachines.Length)");
}
try
{
_EnsureAllStatus(status, allmachines, "RUNNING");
}
catch (Exception e)
{
throw new Exception("startall " + xallargs + ": (restarted all services check) Qizmt servicestatusall: " + e.Message, e);
}
}
{
string arg = string.Join(",", allmachines);
Dictionary <string, string> status = _GetStatusAll(arg);
if (status.Count != allmachines.Length)
{
throw new Exception("startall " + xallargs + ": (restarted all services check) Qizmt servicestatusall " + arg + ": (status.Count != allmachines.Length)");
}
try
{
_EnsureAllStatus(status, allmachines, "RUNNING");
}
catch (Exception e)
{
throw new Exception("startall " + xallargs + ": (restarted all services check) Qizmt servicestatusall " + arg + ": " + e.Message, e);
}
}
{
string arg = "\"@" + hoststxtpath + "\"";
Dictionary <string, string> status = _GetStatusAll(arg);
if (status.Count != allmachines.Length)
{
throw new Exception("startall " + xallargs + ": (restarted all services check) Qizmt servicestatusall " + arg + ": (status.Count != allmachines.Length)");
}
try
{
_EnsureAllStatus(status, allmachines, "RUNNING");
}
catch (Exception e)
{
throw new Exception("startall " + xallargs + ": (restarted all services check) Qizmt servicestatusall " + arg + ": " + e.Message, e);
}
}
}
}
finally
{
try
{
System.IO.File.Delete(hoststxtpath);
}
catch
{
}
}
Console.WriteLine("[PASSED] - " + string.Join(" ", args));
}
/*
public static bool WildcardMatch(string wc, string str)
{
string srex = System.Text.RegularExpressions.Regex.Escape(wc).Replace(@"\*", @".*");
System.Text.RegularExpressions.Regex rex = new System.Text.RegularExpressions.Regex(srex, System.Text.RegularExpressions.RegexOptions.IgnoreCase);
return rex.IsMatch(str);
}
* */
public static List<string> SplitInputPaths(dfs dc, string pathlist, bool StripRecordInfo)
{
return dc.SplitInputPaths(pathlist, StripRecordInfo);
}
static void DfsUpdateStressTest(string[] args)
{
if (args.Length <= 1 || !System.IO.File.Exists(args[1]))
{
throw new Exception("Expected path to DFS.xml");
}
string dfsxmlpath = args[1];
string masterdir;
{
System.IO.FileInfo fi = new System.IO.FileInfo(dfsxmlpath);
masterdir = fi.DirectoryName; // Directory's full path.
}
Surrogate.SetNewMetaLocation(masterdir);
dfs dc = dfs.ReadDfsConfig_unlocked(dfsxmlpath);
string masterhost = System.Net.Dns.GetHostName();
string[] allmachines;
{
string[] sl = dc.Slaves.SlaveList.Split(';');
List <string> aml = new List <string>(sl.Length + 1);
aml.Add(masterhost);
foreach (string slave in sl)
{
if (0 != string.Compare(IPAddressUtil.GetName(slave), IPAddressUtil.GetName(masterhost), StringComparison.OrdinalIgnoreCase))
{
aml.Add(slave);
}
}
allmachines = aml.ToArray();
}
{
Console.WriteLine("Ensure cluster is perfectly healthy...");
EnsurePerfectQizmtHealtha();
Console.WriteLine("Stressing DFS updates...");
//System.Threading.Thread.Sleep(1000 * 8);
string exectempdir = @"\\" + System.Net.Dns.GetHostName() + @"\C$\temp\qizmt\regression_test_DfsUpdateStressTest-" + Guid.NewGuid().ToString();
if (!System.IO.Directory.Exists(exectempdir))
{
System.IO.Directory.CreateDirectory(exectempdir);
}
string execfp = exectempdir + @"\exec{8B8F731B-3BEC-4e99-B08F-BDEB81525172}";
const int NUMBER_OF_JOBS = 50; // <--STRESS-NUMBER--
for (int njob = 0; njob < NUMBER_OF_JOBS; njob++)
{
System.IO.File.WriteAllText(execfp + njob.ToString(), (@"<SourceCode><Jobs></Jobs></SourceCode>").Replace('`', '"'));
}
try
{
Exec.Shell("Qizmt importdirmt " + exectempdir);
Console.WriteLine("Confirming updates...");
{
string lsoutput = Exec.Shell("Qizmt ls");
int njobs = 0;
for (int i = 0; ;)
{
if (njobs == NUMBER_OF_JOBS)
{
break; // Good!
}
i = lsoutput.IndexOf("{8B8F731B-3BEC-4e99-B08F-BDEB81525172}", i);
if (-1 == i)
{
throw new Exception("Not all updates to DFS were written (only found " + njobs.ToString() + " jobs imported, expected " + NUMBER_OF_JOBS.ToString() + ")");
}
i += "{8B8F731B-3BEC-4e99-B08F-BDEB81525172}".Length;
njobs++;
}
}
}
finally
{
try
{
for (int njob = 0; njob < NUMBER_OF_JOBS; njob++)
{
System.IO.File.Delete(execfp + njob.ToString());
}
System.IO.Directory.Delete(exectempdir);
}
catch
{
}
try
{
Exec.Shell("Qizmt del exec{8B8F731B-3BEC-4e99-B08F-BDEB81525172}*");
}
catch
{
}
}
Console.WriteLine("[PASSED] - " + string.Join(" ", args));
}
}
static void RemoveMachine(string[] args)
{
bool two2one = -1 != args[0].IndexOf("2to1", StringComparison.OrdinalIgnoreCase);
bool three2two = -1 != args[0].IndexOf("3to2", StringComparison.OrdinalIgnoreCase);
if (args.Length <= 1 || !System.IO.File.Exists(args[1]))
{
throw new Exception("Expected path to DFS.xml");
}
string dfsxmlpath = args[1];
string dfsxmlpathbackup = dfsxmlpath + "$" + Guid.NewGuid().ToString();
string masterdir;
{
System.IO.FileInfo fi = new System.IO.FileInfo(dfsxmlpath);
masterdir = fi.DirectoryName; // Directory's full path.
}
Surrogate.SetNewMetaLocation(masterdir);
string masterslavedat = masterdir + @"\slave.dat";
dfs olddfs = dfs.ReadDfsConfig_unlocked(dfsxmlpath);
int iarg = 2;
string sreplication = "1";
if (args.Length > iarg)
{
if (args[iarg].StartsWith("#"))
{
sreplication = args[iarg++].Substring(1);
}
}
string masterhost = System.Net.Dns.GetHostName();
string[] allmachines;
{
string[] sl = olddfs.Slaves.SlaveList.Split(';');
List <string> aml = new List <string>(sl.Length + 1);
aml.Add(masterhost);
foreach (string slave in sl)
{
if (0 != string.Compare(IPAddressUtil.GetName(slave), IPAddressUtil.GetName(masterhost), StringComparison.OrdinalIgnoreCase))
{
aml.Add(slave);
}
}
allmachines = aml.ToArray();
}
if (two2one)
{
if (allmachines.Length < 2)
{
throw new Exception("Need >= 2 machines for 2to1");
}
allmachines = new string[] { allmachines[0], allmachines[1] };
}
if (three2two)
{
if (allmachines.Length < 3)
{
throw new Exception("Need >= 3 machines for 3to2");
}
allmachines = new string[] { allmachines[0], allmachines[1], allmachines[2] };
}
if (allmachines.Length < 2)
{
throw new Exception("Cluster needs at least 2 machines for this test");
}
Console.WriteLine("Backing up DFS.xml to: {0} ...", dfsxmlpathbackup);
try
{
System.IO.File.Delete(dfsxmlpathbackup);
}
catch
{
}
System.IO.File.Move(dfsxmlpath, dfsxmlpathbackup);
try
{
Console.WriteLine("Formatting DFS for test...");
{
string fmtcmd = "Qizmt @format Machines=" + string.Join(",", allmachines) + " Replication=" + sreplication;
Console.WriteLine(" {0}", fmtcmd);
Exec.Shell(fmtcmd);
}
Console.WriteLine("Adding some files to DFS...");
Console.Write(" ");
Exec.Shell("Qizmt bingen 1MB 1MB 50");
Console.Write("10%");
Exec.Shell("Qizmt examples");
Console.Write("..15%");
Exec.Shell("Qizmt wordgen 10MB 10MB 100");
Console.Write("..50%");
Exec.Shell("Qizmt asciigen 50MB 50MB 500");
Console.Write("..100%");
Console.WriteLine();
int ls_output_linecount = Exec.Shell("Qizmt ls").Split('\n').Length;
Console.WriteLine("Ensure the cluster is perfectly healthy...");
EnsurePerfectQizmtHealtha();
Console.WriteLine("Run test job, save output...");
string md5_10MB_output = Exec.Shell("Qizmt md5 10MB");
string rmachine = allmachines[allmachines.Length - 1];
Console.WriteLine("Removing machine (removemachine {0}) ...", rmachine);
Console.WriteLine(Exec.Shell("Qizmt removemachine " + rmachine));
Console.WriteLine("Ensure the cluster is perfectly healthy...");
EnsurePerfectQizmtHealtha();
Console.WriteLine("Run test job, confirm output...");
if (md5_10MB_output != Exec.Shell("Qizmt md5 10MB"))
{
throw new Exception("Test job output does not match previous run");
}
Console.WriteLine("[PASSED] - " + string.Join(" ", args));
}
finally
{
Console.WriteLine("Restoring DFS.xml backup...");
// Note: these are safe; the try/finally only wraps the new dfs.
try
{
Exec.Shell("Qizmt del *");
}
catch
{
}
try
{
// Delete temp dfs.xml, it's being replaced with the good one.
System.IO.File.Delete(dfsxmlpath);
}
catch
{
}
System.IO.File.Move(dfsxmlpathbackup, dfsxmlpath);
}
}
public static void CheckSorted(string dfsreader)
{
dfs dc = LoadDfsConfig();
if (null == DfsFindAny(dc, dfsreader))
{
Console.Error.WriteLine("Input file does not exist in DFS: {0}", dfsreader);
SetFailure();
return;
}
string tempfnpost = ".$" + Guid.NewGuid().ToString() + ".$" + System.Diagnostics.Process.GetCurrentProcess().Id.ToString();
string jobsfn = "checksorted-jobs.xml" + tempfnpost;
try
{
using (System.IO.StreamWriter sw = System.IO.File.CreateText(jobsfn))
{
sw.Write((@"<SourceCode>
<Jobs>
<Job>
<Narrative>
<Name>Check Sorted</Name>
<Custodian></Custodian>
<email></email>
</Narrative>
<IOSettings>
<JobType>remote</JobType>
<DFS_IO>
<DFSReader>" + dfsreader + @"</DFSReader>
<DFSWriter></DFSWriter>
</DFS_IO>
</IOSettings>
<Remote>
<![CDATA[
static int CompareBytes(IList<byte> x, IList<byte> y)
{
for (int i = 0;; i++)
{
if(i >= x.Count)
{
if(i >= y.Count)
{
return 0;
}
return -1;
}
if(i >= y.Count)
{
return 1;
}
int diff = x[i] - y[i];
if (0 != diff)
{
return diff;
}
}
return 0;
}
public virtual void Remote(RemoteInputStream dfsinput, RemoteOutputStream dfsoutput)
{
List<byte> prevline = new List<byte>();
List<byte> line = new List<byte>();
List<byte> templist;
//prevline.Clear();
if(!dfsinput.ReadLineAppend(prevline))
{
DSpace_Log(`No input data`);
return;
}
ulong linnum = 1;
for(;;)
{
linnum++;
line.Clear();
dfsinput.ReadLineAppend(line);
if(line.Count == 0 && dfsinput.EndOfStream)
{
break;
}
if(CompareBytes(prevline, line) > 0)
{
DSpace_Log(`Not sorted starting on line ` + linnum.ToString());
DSpace_Log(` ` + (linnum - 1).ToString() + `: \`` + Encoding.UTF8.GetString(prevline.ToArray()) + `\`\n ` + linnum.ToString() + `: \`` + Encoding.UTF8.GetString(line.ToArray()) + `\``);
return;
}
templist = prevline;
prevline = line;
line = templist;
}
DSpace_Log(`Sorted`);
}
]]>
</Remote>
</Job>
</Jobs>
</SourceCode>
").Replace("`", "\""));
}
Console.WriteLine("Checking if sorted...");
Exec("", LoadConfig(jobsfn), new string[] { }, false, false);
}
finally
{
try
{
System.IO.File.Delete(jobsfn);
}
catch
{
}
}
}
public static void Generate(List <string> xpaths, string outname, long sizeoutput, long rowsize, GenerateType type, int writersCount, GenerateRandom genrand)
{
int RecordLength = -1;
if (outname.StartsWith("dfs://", StringComparison.OrdinalIgnoreCase))
{
outname = outname.Substring(6);
}
{
int iat = outname.IndexOf('@');
if (-1 != iat)
{
try
{
RecordLength = Surrogate.GetRecordSize(outname.Substring(iat + 1));
}
catch (Exception e)
{
Console.Error.WriteLine(e.Message);
SetFailure();
return;
}
outname = outname.Substring(0, iat);
}
}
string reason = "";
if (dfs.IsBadFilename(outname, out reason))
{
Console.Error.WriteLine("Invalid output-dfsfile: {0}", reason);
SetFailure();
return;
}
dfs dc = LoadDfsConfig();
if (null != DfsFindAny(dc, outname))
{
Console.Error.WriteLine("Output file already exists in DFS: {0}", outname);
SetFailure();
return;
}
if (RecordLength > 0)
{
if (rowsize < 0)
{
rowsize = RecordLength;
}
else
{
if (rowsize > RecordLength)
{
Console.Error.WriteLine("Row data cannot be greater than DFS record length");
SetFailure();
return;
}
}
if (type != GenerateType.BINARY)
{
Console.Error.WriteLine("Error: must specify type=bin when generating a rectangular binary DFS file");
SetFailure();
return;
}
}
else
{
if (rowsize < 1)
{
rowsize = 100;
}
}
string tempfnpost = "." + Guid.NewGuid().ToString() + "." + System.Diagnostics.Process.GetCurrentProcess().Id.ToString();
string jobsfn = "gen-jobs.xml" + tempfnpost;
string rnd = "";
string nextrandBin = "";
string nextrandAscii = "";
string nextrandLen = "";
string nextrandLower = "";
string nextrandUpper = "";
if (genrand == GenerateRandom.RANDOM)
{
rnd = "Random rnd = new Random(unchecked(System.DateTime.Now.Millisecond + System.Diagnostics.Process.GetCurrentProcess().Id + DSpace_BlockID));";
nextrandBin = "rnd.Next()";
nextrandAscii = "rnd.Next((int)' ' + 1, (int)'~' + 1)";
nextrandLen = "rnd.Next(3, 9 + 1)";
nextrandLower = "rnd.Next((int)'a', (int)'z' + 1)";
nextrandUpper = "rnd.Next((int)'A', (int)'Z' + 1)";
}
else if (genrand == GenerateRandom.FRANDOM)
{
rnd = "FRandom.Seed(unchecked(System.DateTime.Now.Millisecond + System.Diagnostics.Process.GetCurrentProcess().Id + DSpace_BlockID));";
nextrandBin = "FRandom.Next()";
nextrandAscii = "FRandom.Next((int)' ' + 1, (int)'~' + 1)";
nextrandLen = "FRandom.Next(3, 9 + 1)";
nextrandLower = "FRandom.Next((int)'a', (int)'z' + 1)";
nextrandUpper = "FRandom.Next((int)'A', (int)'Z' + 1)";
}
else if (genrand == GenerateRandom.DRANDOM)
{
if (GenerateType.ASCII == type)
{
rnd = @"DRandom rnd = new DRandom(false, true, (int)' ' + 1, (int)'~' + 1);
DRandom rndLen = null;
DRandom rndLower = null;
DRandom rndUpper = null;";
nextrandBin = "rnd.Next()";
}
else if (GenerateType.WORDS == type)
{
rnd = @"DRandom rndLen = new DRandom(false, true, 3, 9 + 1);
DRandom rndLower = new DRandom(false, true, (int)'a', (int)'z' + 1);
DRandom rndUpper = new DRandom(false, true, (int)'A', (int)'Z' + 1);
DRandom rnd = null;";
}
else
{
rnd = @"DRandom rnd = new DRandom();
DRandom rndLen = null;
DRandom rndLower = null;
DRandom rndUpper = null;";
}
nextrandBin = "rnd.Next()";
nextrandAscii = "rnd.Next()";
nextrandLen = "rndLen.Next()";
nextrandLower = "rndLower.Next()";
nextrandUpper = "rndUpper.Next()";
}
try
{
string dline = "";
string drline = "";
#if DEBUG
dline = "#line 1 \"GEN.ds\"" + Environment.NewLine;
drline = "#line default" + Environment.NewLine;
#endif
string outsuffix = "";
if (RecordLength > 0)
{
outsuffix = "@" + RecordLength;
}
string[] slaves = dc.Slaves.SlaveList.Split(';');
int maxWriters = (writersCount > 0) ? writersCount : Surrogate.NumberOfProcessors * slaves.Length;
using (System.IO.StreamWriter sw = System.IO.File.CreateText(jobsfn))
{
sw.Write(@"<SourceCode>
<Jobs>
<Job>
<Narrative>
<Name>Generate Data</Name>
<Custodian></Custodian>
<email></email>
</Narrative>
<IOSettings>
<JobType>remote</JobType>");
for (int si = 0; si < maxWriters; si++)
{
sw.WriteLine(@" <DFS_IO>
<DFSReader></DFSReader>
<DFSWriter>dfs://" + outname + ".gen" + si.ToString() + tempfnpost + outsuffix + @"</DFSWriter>
</DFS_IO>");
}
sw.WriteLine((@" </IOSettings>
<Remote>
<![CDATA[
" + dline + @"
public virtual void Remote(RemoteInputStream dfsinput, RemoteOutputStream dfsoutput)
{
const bool IS_RBIN_FILE = " + ((RecordLength > 0) ? "true" : "false") + @"; // Is rectangular binary output file?
const bool IS_ASCII = " + ((GenerateType.ASCII == type) ? "true" : "false") + @";
const bool IS_WORDS = " + ((GenerateType.WORDS == type) ? "true" : "false") + @";
const long size = " + sizeoutput.ToString() + @";
const long rowsize = " + rowsize.ToString() + @";
long fullrecordsize = rowsize + " + Environment.NewLine.Length + @";
if(IS_RBIN_FILE)
{
fullrecordsize = Qizmt_OutputRecordLength;
}
long numrows = size / fullrecordsize;
if((size % fullrecordsize) != 0)
{
numrows++;
}
" + rnd + @"
List<byte> onerow = new List<byte>((rowsize > 16777216) ? 16777216 : (int)rowsize);
long numrowsPART = numrows / DSpace_BlocksTotalCount;
if(IS_RBIN_FILE)
{
if(DSpace_BlockID < (numrows % DSpace_BlocksTotalCount))
{
numrowsPART++;
}
}
else
{
if(0 == DSpace_BlockID)
{
numrowsPART += numrows % DSpace_BlocksTotalCount;
}
}
for(long rn = 0; rn < numrowsPART; rn++)
{
onerow.Clear();
byte b;
if(IS_WORDS)
{
bool fupper = true;
long remain = rowsize - 1; // Don't count trailing dot.
while(remain - 1 >= 3) // -1 here for word separator space.
{
if(onerow.Count > 0)
{
onerow.Add((byte)' ');
remain--;
}
long wlen = " + nextrandLen + @";
if(wlen > remain - 1)
{
wlen = remain - 1;
}
for(int wi = 0; wi < wlen; wi++)
{
if(fupper)
{
fupper = false;
b = (byte)" + nextrandUpper + @";
}
else
{
b = (byte)" + nextrandLower + @";
}
onerow.Add(b);
}
remain -= wlen;
}
onerow.Add((byte)'.'); // Don't count trailing dot.
while(remain > 0)
{
onerow.Add((byte)' ');
remain--;
}
}
else
{
for(int nb = 0; nb < rowsize; nb++)
{
if(IS_ASCII)
{
b = (byte)" + nextrandAscii + @";
}
else // Binary.
{
for(;;)
{
b = (byte)" + nextrandBin + @";
if(!IS_RBIN_FILE)
{
if(b == 0 || b == '\n' || b == '\r')
{
continue;
}
}
break;
}
}
onerow.Add(b);
}
}
if(IS_RBIN_FILE)
{
while(onerow.Count < DSpace_OutputRecordLength)
{
if(IS_ASCII || IS_WORDS)
{
//onerow.Add((byte)' ');
throw new NotImplementedException();
}
else
{
onerow.Add((byte)0);
}
}
dfsoutput.WriteRecord(onerow);
}
else
{
dfsoutput.WriteLine(onerow);
}
}
}
" + drline + @"
]]>
</Remote>
</Job>
</Jobs>
</SourceCode>
").Replace("`", "\""));
/*
* //dfsoutput.Write(onerow);
* //dfsoutput.Write(`" + rowsep.Replace("\\", "\\\\").Replace("\n", "\\n").Replace("\r", "\\r").Replace("\"", "\\\"") + @"`);
* */
}
{
// File jobsfn exists.
Console.WriteLine("Generating data...");
Exec("", LoadConfig(xpaths, jobsfn), new string[] { }, false, false);
if (null != DfsFindAny(dc, outname))
{
SetFailure();
return;
}
Shell("DSpace -dfs combine \"dfs://" + outname + ".gen*" + tempfnpost + "\" + \"" + outname + "\"");
if (RecordLength > 0)
{
Console.WriteLine("Done; file '{0}' written into DFS with {1} of random data per {2} record",
outname, GetFriendlyByteSize(rowsize), GetFriendlyByteSize(RecordLength));
}
else
{
Console.WriteLine("Done; file '{0}' written into DFS with {1} rows", outname, GetFriendlyByteSize(rowsize));
}
}
}
finally
{
try
{
System.IO.File.Delete(jobsfn);
}
catch
{
}
}
}
public static void GenerateHash(string hashname, string dfsreader)
{
dfs dc = LoadDfsConfig();
if (null == DfsFindAny(dc, dfsreader))
{
Console.Error.WriteLine("Input file does not exist in DFS: {0}", dfsreader);
SetFailure();
return;
}
string tempfnpost = ".$" + Guid.NewGuid().ToString() + ".$" + System.Diagnostics.Process.GetCurrentProcess().Id.ToString();
string jobsfn = "gen" + hashname + "-jobs.xml" + tempfnpost;
try
{
using (System.IO.StreamWriter sw = System.IO.File.CreateText(jobsfn))
{
sw.Write((@"<SourceCode>
<Jobs>
<Job>
<Narrative>
<Name>Generate " + hashname + @"</Name>
<Custodian></Custodian>
<email></email>
</Narrative>
<IOSettings>
<JobType>remote</JobType>
<DFS_IO>
<DFSReader>" + dfsreader + @"</DFSReader>
<DFSWriter></DFSWriter>
</DFS_IO>
</IOSettings>
<Remote>
<![CDATA[
public virtual void Remote(RemoteInputStream dfsinput, RemoteOutputStream dfsoutput)
{
string hashname = @`" + hashname + @"`;
if(0 == string.Compare(`MD5`, hashname, true))
{
System.Security.Cryptography.MD5 md5 = new System.Security.Cryptography.MD5CryptoServiceProvider();
byte[] hashresult = md5.ComputeHash(dfsinput);
StringBuilder sbresult = new StringBuilder(32);
sbresult.Append(hashname + ` of `);
sbresult.Append(@`dfs://" + dfsreader + @"`);
sbresult.Append(`: `);
foreach(byte hb in hashresult)
{
sbresult.Append(hb.ToString(`x2`));
}
DSpace_Log(sbresult.ToString());
}
else if(0 == string.Compare(`Sum`, hashname, true))
{
ulong lsum = 0;
for(;;)
{
int ib = dfsinput.ReadByte();
if(ib < 0)
{
break;
}
lsum += (byte)ib;
}
StringBuilder sbresult = new StringBuilder(32);
sbresult.Append(hashname + ` of `);
sbresult.Append(@`dfs://" + dfsreader + @"`);
sbresult.Append(`: `);
sbresult.Append(lsum);
DSpace_Log(sbresult.ToString());
}
else if(0 == string.Compare(`Sum2`, hashname, true))
{
ulong lsum = 0;
bool prevnl = true;
for(;;)
{
int ib = dfsinput.ReadByte();
if(ib < 0)
{
if(prevnl)
{
break;
}
prevnl = true;
ib = '\n';
}
if('\r' == ib || '\n' == ib)
{
if(prevnl)
{
continue;
}
prevnl = true;
ib = '\n';
}
else
{
prevnl = false;
}
lsum += (byte)ib;
}
StringBuilder sbresult = new StringBuilder(32);
sbresult.Append(hashname + ` of `);
sbresult.Append(@`dfs://" + dfsreader + @"`);
sbresult.Append(`: `);
sbresult.Append(lsum);
DSpace_Log(sbresult.ToString());
}
}
]]>
</Remote>
</Job>
</Jobs>
</SourceCode>
").Replace("`", "\""));
}
Console.WriteLine("Generating {0}...", hashname);
Exec("", LoadConfig(jobsfn), new string[] { }, false, false);
}
finally
{
try
{
System.IO.File.Delete(jobsfn);
}
catch
{
}
}
}
protected System.IO.Stream GetStreamFromDfsNode(dfs.DfsFile.FileNode node, bool skipdfschunkheader)
{
curpartheadersize = 0;
curpartpos = 0;
bool anydata = false;
System.IO.Stream stm = _OpenStream(node);
curpartfulllength = stm.Length;
anydata = 0 != curpartfulllength;
if (skipdfschunkheader && anydata)
{
if (null == _rb)
{
_rb = new byte[32];
}
_StreamReadExact(stm, _rb, 4);
int hlen = Entry.BytesToInt(_rb);
if (hlen > 4)
{
int hremain = hlen - 4;
if (hremain > _rb.Length)
{
_rb = new byte[hremain];
}
_StreamReadExact(stm, _rb, hremain);
}
curpartheadersize = hlen;
}
curpartpos = curpartheadersize;
return stm;
}
bool DoSave()
{
if (IsReadOnly)
{
SetStatus("Cannot save read-only");
return(false);
}
string DocText = Doc.Text;
{
try
{
string fn = PrettyFile;
if (fn.StartsWith("dfs://", StringComparison.OrdinalIgnoreCase))
{
fn = fn.Substring(6);
}
dfs dc = Surrogate.ReadMasterDfsConfig();
string[] slaves = dc.Slaves.SlaveList.Split(';');
if (0 == slaves.Length || dc.Slaves.SlaveList.Length == 0)
{
throw new Exception("DFS SlaveList error (machines)");
}
Random rnd = new Random(DateTime.Now.Millisecond / 2 + System.Diagnostics.Process.GetCurrentProcess().Id / 2);
string newactualfilehost = slaves[rnd.Next() % slaves.Length];
string newactualfilename = dfs.GenerateZdFileDataNodeName(fn);
string myActualFile = Surrogate.NetworkPathForHost(newactualfilehost) + @"\" + newactualfilename;
{
byte[] smallbuf = new byte[4];
MySpace.DataMining.DistributedObjects.Entry.ToBytes(4, smallbuf, 0);
using (System.IO.FileStream fs = new System.IO.FileStream(myActualFile, System.IO.FileMode.CreateNew, System.IO.FileAccess.Write))
{
fs.Write(smallbuf, 0, 4);
byte[] buf = Encoding.UTF8.GetBytes(Doc.Text);
fs.Write(buf, 0, buf.Length);
}
if (IsNewFile)
{
Console.Write(MySpace.DataMining.DistributedObjects.Exec.Shell(
"DSpace -dfsbind \"" + newactualfilehost + "\" \"" + newactualfilename + "\" \"" + fn + "\" " + DfsFileTypes.NORMAL
+ " -h4"));
}
else
{
string tempdfsfile = fn + Guid.NewGuid().ToString() + dfs.TEMP_FILE_MARKER;
Console.Write(MySpace.DataMining.DistributedObjects.Exec.Shell(
"DSpace -dfsbind \"" + newactualfilehost + "\" \"" + newactualfilename + "\" \"" + tempdfsfile + "\" " + DfsFileTypes.NORMAL
+ " -h4"));
MySpace.DataMining.DistributedObjects.Exec.Shell("DSpace swap \"" + tempdfsfile + "\" \"" + fn + "\"");
MySpace.DataMining.DistributedObjects.Exec.Shell("DSpace delete \"" + tempdfsfile + "\"", true); // suppresserrors=true
}
}
//ActualFile = myActualFile; // Only update this when fully committed to DFS!
IsNewFile = false;
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
MessageBox.Show(this, "Unable to save new file to DFS:\r\n\r\n" + e.Message, "Save-New Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
return(false); // Important! don't continue with any of the rest if this fails!
}
}
Doc.Modified = false; // !
return(true);
}
internal static void DoDfsCheck(string[] args, bool fix)
{
if (fix)
{
if (args.Length == 0)
{
Console.Error.WriteLine(" -all switch required for dfsfix");
SetFailure();
return;
}
}
string userspec = null;
bool userspecchunk = false;
bool userspecdfsfile = false;
if (args.Length > 0)
{
if (0 != string.Compare(args[0], "-all", true))
{
if ('-' == args[0][0])
{
Console.Error.WriteLine("Unknown switch: {0}", args[0]);
SetFailure();
return;
}
userspec = args[0];
if (userspec.StartsWith("dfs://", StringComparison.OrdinalIgnoreCase))
{
userspec = userspec.Substring(6);
userspecdfsfile = true;
}
else if (userspec.StartsWith("zd.", StringComparison.OrdinalIgnoreCase) &&
userspec.EndsWith(".zd", StringComparison.OrdinalIgnoreCase) &&
-1 != userspec.IndexOf('-') &&
userspec.Length > 45)
{
userspecchunk = true;
}
else
{
userspecdfsfile = true;
}
}
}
dfs dc = LoadDfsConfig();
if (dc.Files.Count == 0)
{
Console.WriteLine("No DFS files to scan");
return;
}
string tempfnpost = "." + Guid.NewGuid().ToString() + "." + System.Diagnostics.Process.GetCurrentProcess().Id.ToString();
string[] hosts = dc.Slaves.SlaveList.Split(';');
Dictionary <string, StringBuilder> sbdfsfiles = new Dictionary <string, StringBuilder>(StringComparer.OrdinalIgnoreCase);
foreach (string host in hosts)
{
StringBuilder sb = new StringBuilder(1024);
sbdfsfiles[host] = sb;
}
bool addedone = false;
foreach (dfs.DfsFile df in dc.Files)
{
if (0 == string.Compare(DfsFileTypes.NORMAL, df.Type, true) ||
0 == string.Compare(DfsFileTypes.BINARY_RECT, df.Type, true))
{
if (userspecdfsfile)
{
if (0 != string.Compare(df.Name, userspec, true))
{
continue;
}
}
foreach (dfs.DfsFile.FileNode fn in df.Nodes)
{
if (userspecchunk)
{
if (0 != string.Compare(fn.Name, userspec, true))
{
continue;
}
}
foreach (string rhost in fn.Host.Split(';'))
{
StringBuilder sb = sbdfsfiles[rhost];
sb.AppendLine(fn.Host + @"?" + df.Name + @"?" + fn.Name);
}
addedone = true;
}
}
}
if (!addedone)
{
Console.WriteLine("Nothing to scan");
return;
}
foreach (KeyValuePair <string, StringBuilder> kvp in sbdfsfiles)
{
try
{
System.IO.File.WriteAllText(Surrogate.NetworkPathForHost(kvp.Key) + @"\dfscheck" + tempfnpost, kvp.Value.ToString());
}
catch (Exception e)
{
Console.Error.WriteLine("Skipping host {0} due to error: {1}", kvp.Key, e.Message);
LogOutputToFile(e.ToString());
}
}
string jobsfn = "dfschecks-jobs.xml" + tempfnpost;
try
{
using (System.IO.StreamWriter sw = System.IO.File.CreateText(jobsfn))
{
sw.Write((@"<SourceCode>
<Jobs>
<Job Name=`DfsCheck` Custodian=`` Email=`` Description=``>
<IOSettings>
<JobType>remote</JobType>
<DFS_IO_Multi>
<DFSReader></DFSReader>
<DFSWriter></DFSWriter>
<Mode>ALL MACHINES</Mode>
</DFS_IO_Multi>
</IOSettings>
<Remote>
<![CDATA[
int numproblems = 0;
int numfixed = 0;
public virtual void Remote(RemoteInputStream dfsinput, RemoteOutputStream dfsoutput)
{
const bool fix = " + (fix ? "true" : "false") + @";
string dfsfilesinfofp = `dfscheck" + tempfnpost + @"`;
string[] dfsfilesinfos = System.IO.File.ReadAllLines(dfsfilesinfofp);
if(dfsfilesinfos.Length == 0)
{
return;
}
MySpace.DataMining.Threading.ThreadTools<string>.Parallel(
new Action<string>(
delegate(string dfsfilesinf)
{
string[] qq = dfsfilesinf.Split('?');
string chunkhost = qq[0]; // This machine is the first one.
string dfsfilename = qq[1];
string dfschunkname = qq[2];
try
{
using(System.IO.FileStream fs = new System.IO.FileStream(dfschunkname,
System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read))
{
for(;;)
{
if(-1 == fs.ReadByte())
{
break;
}
}
}
}
catch
{
lock(this)
{
numproblems++;
}
Qizmt_Log(` DFS file '` + dfsfilename + `' has problem with chunk '` + dfschunkname + `' on host ` + Qizmt_MachineHost);
if(fix)
{
string[] rhosts = chunkhost.Split(';');
if(rhosts.Length < 2)
{
Qizmt_Log(` Not enough replicates`);
}
else
{
for(int ir = 1; ir < rhosts.Length; ir++)
{
try
{
string newdfschunkname = `zd.dfscheck.` + Guid.NewGuid().ToString() + `.zd`;
System.IO.File.Copy(MySpace.DataMining.AELight.Surrogate.NetworkPathForHost(rhosts[ir]) + @`\` + dfschunkname,
newdfschunkname, false);
System.IO.File.Delete(dfschunkname);
System.IO.File.Move(newdfschunkname, dfschunkname);
numfixed++;
break;
}
catch(Exception e)
{
if(ir >= rhosts.Length - 1)
{
Qizmt_Log(` Unable to repair DFS file '` + dfsfilename + `' chunk '` + dfschunkname + `' on host ` + Qizmt_MachineHost + ` because: ` + e.ToString());
}
}
}
}
}
}
}), dfsfilesinfos);
if(fix)
{
Qizmt_Log(` ` + numfixed + ` of ` + numproblems + ` problems fixed on host ` + Qizmt_MachineHost);
}
else
{
Qizmt_Log(` ` + numproblems + ` problems found on host ` + Qizmt_MachineHost);
}
}
]]>
</Remote>
</Job>
</Jobs>
</SourceCode>").Replace('`', '"'));
}
if (userspecchunk)
{
Console.WriteLine("Checking DFS chunk file {0}...", userspec);
}
else if (userspecdfsfile)
{
Console.WriteLine("Checking DFS file dfs://{0}...", userspec);
}
else
{
Console.WriteLine("Checking DFS...");
}
Exec("", LoadConfig(jobsfn), new string[] { }, false, false);
}
finally
{
try
{
System.IO.File.Delete(jobsfn);
}
catch
{
}
foreach (KeyValuePair <string, StringBuilder> kvp in sbdfsfiles)
{
try
{
System.IO.File.Delete(Surrogate.NetworkPathForHost(kvp.Key) + @"\dfscheck" + tempfnpost);
}
catch
{
}
}
}
}
protected static void UpdateDfsXml(dfs dc)
{
UpdateDfsXml(dc, DFSXMLPATH, dc.GetMetaBackupLocation());
}
static void RemoveSurrogate(string[] args)
{
if (args.Length <= 1 || !System.IO.File.Exists(args[1]))
{
throw new Exception("Expected path to DFS.xml");
}
string dfsxmlpath = args[1];
string dfsxmlpathbackup = dfsxmlpath + "$" + Guid.NewGuid().ToString();
bool incluster;
if (args[2] == "incluster")
{
incluster = true;
}
else if (args[2] == "isolated")
{
incluster = false;
}
else
{
throw new Exception("Expected: incluster or isolated");
}
string masterdir;
{
System.IO.FileInfo fi = new System.IO.FileInfo(dfsxmlpath);
masterdir = fi.DirectoryName; // Directory's full path.
}
Surrogate.SetNewMetaLocation(masterdir);
string masterslavedat = masterdir + @"\slave.dat";
dfs olddfs = dfs.ReadDfsConfig_unlocked(dfsxmlpath);
string masterhost = System.Net.Dns.GetHostName();
List <string> otherhosts = new List <string>(); // Non-surrogate machines, reguardless if participating surrogate or not.
foreach (string slave in olddfs.Slaves.SlaveList.Split(';'))
{
if (0 != string.Compare(IPAddressUtil.GetName(slave), IPAddressUtil.GetName(masterhost), StringComparison.OrdinalIgnoreCase))
{
otherhosts.Add(slave);
}
}
string newmaster = otherhosts[0];
Console.WriteLine("Backing up DFS.xml to: {0} ...", dfsxmlpathbackup);
try
{
System.IO.File.Delete(dfsxmlpathbackup);
}
catch
{
}
System.IO.File.Move(dfsxmlpath, dfsxmlpathbackup);
try
{
StringBuilder sbmachines = new StringBuilder(1000);
if (incluster)
{
sbmachines.Append(masterhost);
}
foreach (string host in otherhosts)
{
if (0 != sbmachines.Length)
{
sbmachines.Append(',');
}
sbmachines.Append(host);
}
Console.WriteLine("Formatting DFS for test...");
Exec.Shell("Qizmt @format Machines=" + sbmachines.ToString());
Console.WriteLine("Adding some files to DFS...");
Console.Write(" ");
Exec.Shell("Qizmt bingen 1MB 1MB 50");
Console.Write("10%");
Exec.Shell("Qizmt examples");
Console.Write("..15%");
Exec.Shell("Qizmt wordgen 10MB 10MB 100");
Console.Write("..50%");
Exec.Shell("Qizmt asciigen 50MB 50MB 500");
Console.Write("..100%");
Console.WriteLine();
int ls_output_linecount = Exec.Shell("Qizmt ls").Split('\n').Length;
Console.WriteLine("Ensure the cluster is perfectly healthy...");
EnsurePerfectQizmtHealtha();
Console.WriteLine("Run test job, save output...");
string md5_10MB_output = Exec.Shell("Qizmt md5 10MB");
Console.WriteLine("Removing Surrogate (removemachine {0}) ...", masterhost);
Console.WriteLine(Exec.Shell("Qizmt removemachine " + masterhost));
Console.WriteLine("Interface with new surrogate...");
System.IO.File.WriteAllText(masterslavedat, "master=" + newmaster + Environment.NewLine);
{
// Not comparing contents because of the free disk space line.
int new_ls_output_linecount = Exec.Shell("Qizmt ls").Split('\n').Length;
if (ls_output_linecount != new_ls_output_linecount)
{
throw new Exception("Cluster does not contain the same files as before removemachine " + masterdir + ", or problem issuing commands on new surrogate");
}
}
Console.WriteLine("Ensure the cluster is perfectly healthy...");
EnsurePerfectQizmtHealtha();
Console.WriteLine("Run test job, confirm output...");
if (md5_10MB_output != Exec.Shell("Qizmt md5 10MB"))
{
throw new Exception("Test job output does not match previous run");
}
Console.WriteLine("[PASSED] - " + string.Join(" ", args));
}
finally
{
Console.WriteLine("Restoring DFS.xml backup...");
// Note: these are safe; the try/finally only wraps the new dfs.
try
{
Exec.Shell("Qizmt del *");
}
catch
{
}
try
{
System.IO.File.Delete(masterslavedat);
}
catch
{
}
for (int si = 0; si < otherhosts.Count; si++)
{
try
{
System.IO.File.Delete(Surrogate.NetworkPathForHost(otherhosts[si]) + @"\slave.dat");
// Deleting dfs.xml should go last because it'll usually fail.
System.IO.File.Delete(Surrogate.NetworkPathForHost(otherhosts[si]) + @"\dfs.xml");
}
catch
{
}
}
try
{
System.IO.File.Delete(dfsxmlpath);
}
catch
{
}
try
{
// Reformat the cluster so stuff like slave.dat is correct...
Exec.Shell("Qizmt @format Machines=" + string.Join(",", otherhosts.ToArray()));
}
catch (Exception exf)
{
Console.Error.WriteLine("Problem during reformat, there may be an issue with the cluster", exf);
}
try
{
// Delete the dfs.xml just written, it's being replaced with the good one.
System.IO.File.Delete(dfsxmlpath);
}
catch
{
}
System.IO.File.Move(dfsxmlpathbackup, dfsxmlpath);
}
}
static void EnableReplication(string[] args)
{
if (args.Length <= 1 || !System.IO.File.Exists(args[1]))
{
throw new Exception("Expected path to DFS.xml");
}
bool withcache = -1 != args[0].IndexOf("withcache", StringComparison.OrdinalIgnoreCase);
string dfsxmlpath = args[1];
string dfsxmlpathbackup = dfsxmlpath + "$" + Guid.NewGuid().ToString();
long bytes_to_add = 0;
if (args.Length > 2)
{
// To-do: ParseCapacity.
bytes_to_add = long.Parse(args[2]);
if (bytes_to_add < 0)
{
throw new Exception("Invalid bytes-to-add (" + bytes_to_add.ToString() + " bytes)");
}
if (bytes_to_add < 1048576)
{
throw new Exception("bytes-to-add must be at least 1 MB");
}
}
int num_files = 0;
if (args.Length > 3)
{
num_files = int.Parse(args[3]);
if (num_files < 0 || num_files > bytes_to_add / 20)
{
throw new Exception("Invalid #files");
}
}
string masterdir;
{
System.IO.FileInfo fi = new System.IO.FileInfo(dfsxmlpath);
masterdir = fi.DirectoryName; // Directory's full path.
}
Surrogate.SetNewMetaLocation(masterdir);
dfs olddfs = dfs.ReadDfsConfig_unlocked(dfsxmlpath);
string masterhost = System.Net.Dns.GetHostName();
string[] allmachines;
{
string[] sl = olddfs.Slaves.SlaveList.Split(';');
List <string> aml = new List <string>(sl.Length + 1);
aml.Add(masterhost);
foreach (string slave in sl)
{
if (0 != string.Compare(IPAddressUtil.GetName(slave), IPAddressUtil.GetName(masterhost), StringComparison.OrdinalIgnoreCase))
{
aml.Add(slave);
}
}
allmachines = aml.ToArray();
}
Console.WriteLine("Backing up DFS.xml to: {0} ...", dfsxmlpathbackup);
try
{
System.IO.File.Delete(dfsxmlpathbackup);
}
catch
{
}
System.IO.File.Move(dfsxmlpath, dfsxmlpathbackup);
try
{
{
Console.WriteLine("Formatting DFS for test...");
Exec.Shell("Qizmt @format Machines=" + string.Join(",", allmachines));
}
{
// Test logic:
Console.WriteLine("Adding some files to DFS...");
Console.Write(" ");
Exec.Shell("Qizmt bingen 1MB 1MB 50");
Console.Write("10%");
Exec.Shell("Qizmt examples");
Console.Write("..15%");
Exec.Shell("Qizmt wordgen 10MB 10MB 100"); // Note: also used by Cacher.
Console.Write("..50%");
Exec.Shell("Qizmt asciigen 50MB 50MB 500");
Console.Write("..100%");
Console.WriteLine();
if (bytes_to_add > 0)
{
Console.WriteLine("Adding {0} bytes as requested (bytes-to-add)...", bytes_to_add);
long bta10 = bytes_to_add / 10;
Console.Write(" ");
Exec.Shell("Qizmt gen bta10-" + Guid.NewGuid().ToString() + " " + bta10.ToString());
Console.Write("10%");
Exec.Shell("Qizmt gen bta20-" + Guid.NewGuid().ToString() + " " + (bta10 * 2).ToString());
Console.Write("..30%");
{
long totsz = (bta10 * 3);
if (num_files > 1)
{
long onesz = totsz / num_files;
//for (int inf = 0; inf < num_files; inf++)
MySpace.DataMining.Threading.ThreadTools.Parallel(
new Action <int>(
delegate(int inf)
{
Exec.Shell("Qizmt gen bta30." + inf.ToString() + "-" + Guid.NewGuid().ToString() + " " + onesz.ToString());
}), num_files, 15);
}
else
{
Exec.Shell("Qizmt gen bta30-" + Guid.NewGuid().ToString() + " " + totsz.ToString());
Console.Write("..60%");
}
}
Exec.Shell("Qizmt gen bta40-" + Guid.NewGuid().ToString() + " " + (bta10 * 4).ToString());
Console.Write("..100%");
Console.WriteLine();
}
if (withcache)
{
Console.WriteLine("Generating cache files...");
string cachertempdir = @"\\" + System.Net.Dns.GetHostName() + @"\C$\temp\qizmt\regression_test_Cacher-" + Guid.NewGuid().ToString();
if (!System.IO.Directory.Exists(cachertempdir))
{
System.IO.Directory.CreateDirectory(cachertempdir);
}
string cacherfp = cachertempdir + @"\Cacher";
System.IO.File.WriteAllText(cacherfp, (@"<?xml version=`1.0` encoding=`utf-8`?>
<SourceCode>
<Jobs>
<Job Name=`Cacher`>
<Delta>
<Name>Cacher_cache</Name>
<DFSInput>dfs://10MB</DFSInput>
</Delta>
<IOSettings>
<JobType>mapreduce</JobType>
<KeyLength>100</KeyLength>
<DFSInput></DFSInput>
<DFSOutput>dfs://Cacher_output</DFSOutput>
<KeyMajor>8</KeyMajor>
<OutputMethod>grouped</OutputMethod>
</IOSettings>
<MapReduce>
<Map>
<![CDATA[
public virtual void Map(ByteSlice line, MapOutput output)
{
output.Add(line, ByteSlice.Prepare());
}
]]>
</Map>
<Reduce>
<![CDATA[
public override void Reduce(ByteSlice key, RandomAccessEntries values, RandomAccessOutput output)
{
for(int i = 0; i < values.Length; i++)
{
output.Add(key);
}
}
]]>
</Reduce>
</MapReduce>
</Job>
</Jobs>
</SourceCode>
").Replace('`', '"'));
Exec.Shell("Qizmt importdir " + cachertempdir);
try
{
System.IO.File.Delete(cacherfp);
System.IO.Directory.Delete(cachertempdir);
}
catch
{
}
Exec.Shell("Qizmt exec Cacher"); // Creates cache file Cacher_cache
Exec.Shell("Qizmt del Cacher_output");
}
Console.WriteLine("Ensure the cluster is perfectly healthy...");
EnsurePerfectQizmtHealtha();
string ls_output = Exec.Shell("Qizmt ls");
int ls_output_linecount = ls_output.Split('\n').Length;
Console.WriteLine("*** ls output before replication:");
Console.WriteLine(ls_output);
Console.WriteLine("Updating Replication Factor...");
const int replicationfactor = 2;
Console.WriteLine(Exec.Shell("Qizmt replicationupdate " + replicationfactor.ToString()));
Console.WriteLine("Ensure the cluster is perfectly healthy...");
EnsurePerfectQizmtHealtha();
if (withcache)
{
Console.WriteLine("Validate cache files...");
Exec.Shell("Qizmt exec Cacher"); // Uses existing cache file Cacher_cache
Exec.Shell("Qizmt del Cacher_output");
}
Console.WriteLine("Ensure data is replicated...");
EnsureReplication(dfsxmlpath, replicationfactor);
{
// Not comparing contents because of the free disk space line.
string new_ls_output = Exec.Shell("Qizmt ls");
Console.WriteLine("*** ls output after replication:");
Console.WriteLine(new_ls_output);
int new_ls_output_linecount = new_ls_output.Split('\n').Length;
if (ls_output_linecount != new_ls_output_linecount)
{
throw new Exception("Cluster does not contain the same files as before replication");
}
}
}
Console.WriteLine("[PASSED] - " + string.Join(" ", args));
}
finally
{
Console.WriteLine("Restoring DFS.xml backup...");
// Note: these are safe; the try/finally only wraps the new dfs.
try
{
Exec.Shell("Qizmt del *");
}
catch
{
}
try
{
// Delete temp dfs.xml, it's being replaced with the good one.
System.IO.File.Delete(dfsxmlpath);
}
catch
{
}
System.IO.File.Move(dfsxmlpathbackup, dfsxmlpath);
}
}
public void placeRoads()
{
Debug.Log("dfs start");
// do a dfs procedure to ensure that every road sub-graph is connected to every other.
dfs DFS = new dfs();
DFS.initialise();
subGraphPoints = DFS.dfsUtil();
Debug.Log(subGraphPoints.Count);
Debug.Log("dfs end");
// if nodes have been marked as visited by the depth first search on the graph
// use them to construct edges/roads
Dictionary <Vector3, int> visitedInSubGraph = new Dictionary <Vector3, int> ();
Dictionary <Vector3, Vector3> newEdges = new Dictionary <Vector3, Vector3>();
for (int i = 0; i < subGraphPoints.Count; i++)
{
visitedInSubGraph.Add(subGraphPoints[i], 0);
newEdges.Add(subGraphPoints[i], new Vector3(-1f, -1f, -1f));
}
// prune some roads that are too long.
for (int i = 0; i < subGraphPoints.Count; i++)
{
float minimum = 10000f;
visitedInSubGraph [subGraphPoints [i]] = 1;
Vector3 minVertex = new Vector3();
for (int j = 0; j < subGraphPoints.Count; j++)
{
if (j != i && visitedInSubGraph[subGraphPoints[j]] != 1)
{
float distance = Vector3.Distance(subGraphPoints[i], subGraphPoints[j]);
if (distance < minimum)
{
minimum = distance;
minVertex = subGraphPoints [j];
newEdges [subGraphPoints [i]] = minVertex;
}
}
}
visitedInSubGraph [minVertex] = 1;
}
//add all the remaining edges to the final array of roads
for (int i = 0; i < subGraphPoints.Count; i++)
{
if (newEdges [subGraphPoints [i]] != new Vector3(-1f, -1f, -1f))
{
List <Vector3> edge = new List <Vector3> ();
edge.Add(subGraphPoints [i]);
edge.Add(newEdges[subGraphPoints[i]]);
edges.Add(edge);
}
}
// at all the edges place roads prefabs.
for (int i = 0; i < edges.Count; i++)
{
Vector3 midpoint = (edges[i][0] + edges[i][1]) / 2f;
GameObject road;
road = (GameObject)Instantiate(Resources.Load("preFabs/cubePrefab"));
road.transform.position = midpoint;
road.transform.LookAt(edges[i][1]);
float r = UnityEngine.Random.Range(0.1f, 0.7f);
road.transform.localScale = new Vector3(r, 0.1f, Vector3.Distance(edges[i][0], edges[i][1]));
road.name = "road_" + i.ToString();
roads.Add(road);
}
}
public void placeRoads()
{
Debug.Log("dfs start");
//do a dfs procedure to ensure that every road sub-graph is connected to every other.
dfs DFS = new dfs();
DFS.initialise();
subGraphPoints = DFS.dfsUtil();
Debug.Log(subGraphPoints.Count);
Debug.Log("dfs end");
Dictionary <Vector3, int> visitedInSubGraph = new Dictionary <Vector3, int> ();
Dictionary <Vector3, Vector3> newEdges = new Dictionary <Vector3, Vector3>();
for (int i = 0; i < subGraphPoints.Count; i++)
{
visitedInSubGraph.Add(subGraphPoints[i], 0);
newEdges.Add(subGraphPoints[i], new Vector3(-1f, -1f, -1f));
}
for (int i = 0; i < subGraphPoints.Count; i++)
{
float minimum = 10000f;
visitedInSubGraph [subGraphPoints [i]] = 1;
Vector3 minVertex = new Vector3();
for (int j = 0; j < subGraphPoints.Count; j++)
{
if (j != i && visitedInSubGraph[subGraphPoints[j]] != 1)
{
float distance = Vector3.Distance(subGraphPoints[i], subGraphPoints[j]);
if (distance < minimum)
{
minimum = distance;
minVertex = subGraphPoints [j];
newEdges [subGraphPoints [i]] = minVertex;
}
}
}
visitedInSubGraph [minVertex] = 1;
}
for (int i = 0; i < subGraphPoints.Count; i++)
{
if (newEdges [subGraphPoints [i]] != new Vector3(-1f, -1f, -1f))
{
List <Vector3> temp = new List <Vector3> ();
temp.Add(subGraphPoints [i]);
temp.Add(newEdges[subGraphPoints[i]]);
edges.Add(temp);
}
}
for (int i = 0; i < edges.Count; i++)
{
Vector3 midpoint = (edges[i][0] + edges[i][1]) / 2f;
GameObject road;
road = (GameObject)Instantiate(Resources.Load("preFabs/cubePrefab"));
road.transform.position = midpoint;
road.transform.LookAt(edges[i][1]);
float r = UnityEngine.Random.Range(0.1f, 0.7f);
road.transform.localScale = new Vector3(r, 0.1f, Vector3.Distance(edges[i][0], edges[i][1]));
road.name = "road_" + i.ToString();
roads.Add(road);
}
}
static string DfsCreateNewBaseFileName(dfs dc, string dfspath)
{
if (null != DfsFindAny(dc, dfspath))
{
throw new Exception("File already exists in DFS: " + dfspath);
}
return GenerateZdFileDataNodeName(dfspath);
}
static void ReplicationFailover(string[] args)
{
if (args.Length <= 1 || !System.IO.File.Exists(args[1]))
{
throw new Exception("Expected path to DFS.xml");
}
string dfsxmlpath = args[1];
string dfsxmlpathbackup = dfsxmlpath + "$" + Guid.NewGuid().ToString();
string masterdir;
{
System.IO.FileInfo fi = new System.IO.FileInfo(dfsxmlpath);
masterdir = fi.DirectoryName; // Directory's full path.
}
Surrogate.SetNewMetaLocation(masterdir);
dfs olddfs = dfs.ReadDfsConfig_unlocked(dfsxmlpath);
string masterhost = System.Net.Dns.GetHostName();
string[] allmachines;
{
string[] sl = olddfs.Slaves.SlaveList.Split(';');
List <string> aml = new List <string>(sl.Length + 1);
aml.Add(masterhost);
foreach (string slave in sl)
{
if (0 != string.Compare(IPAddressUtil.GetName(slave), IPAddressUtil.GetName(masterhost), StringComparison.OrdinalIgnoreCase))
{
aml.Add(slave);
}
}
allmachines = aml.ToArray();
}
Console.WriteLine("Backing up DFS.xml to: {0} ...", dfsxmlpathbackup);
try
{
System.IO.File.Delete(dfsxmlpathbackup);
}
catch
{
}
System.IO.File.Move(dfsxmlpath, dfsxmlpathbackup);
try
{
{
// Note: added replication and much lower DataNodeBaseSize!
Console.WriteLine("Formatting DFS with Replication=3 for test...");
Exec.Shell("Qizmt @format Machines=" + string.Join(",", allmachines) + " Replication=3 DataNodeBaseSize=1048576");
}
{
// Test logic:
if (allmachines.Length < 3)
{
throw new Exception("This test needs a cluster of at least 3 machines!");
}
long XBYTES = (long)4194304 * (long)allmachines.Length;
Console.WriteLine("Generating data...");
Console.Write(" ");
Exec.Shell("Qizmt gen data{476D6FE8-D645-41cc-83A1-3AB5E2DE23E7} " + (XBYTES / 4).ToString());
Console.Write("25%");
Exec.Shell("Qizmt gen data{61136275-16EC-4ff9-84CE-ACC967550181} " + (XBYTES / 4).ToString());
Console.Write("..50%");
Exec.Shell("Qizmt gen data{C76F6C06-EFC8-4808-B214-DB4D167171EB} " + (XBYTES / 2).ToString());
Console.Write("..100%");
Console.WriteLine();
Console.WriteLine("Ensure the cluster is perfectly healthy...");
EnsurePerfectQizmtHealtha();
Console.WriteLine("Running job on healthy cluster...");
string exec_md5;
{
string exectempdir = @"\\" + System.Net.Dns.GetHostName() + @"\C$\temp\qizmt\regression_test_ReplicationFailover-" + Guid.NewGuid().ToString();
if (!System.IO.Directory.Exists(exectempdir))
{
System.IO.Directory.CreateDirectory(exectempdir);
}
string execfp = exectempdir + @"\exec{FA19CAB0-5225-4cc8-8728-9BFC3A1B834C}";
System.IO.File.WriteAllText(execfp, (@"<?xml version=`1.0` encoding=`utf-8`?>
<SourceCode>
<Jobs>
<Job Name=`exec{FA19CAB0-5225-4cc8-8728-9BFC3A1B834C}`>
<IOSettings>
<JobType>mapreduce</JobType>
<KeyLength>100</KeyLength>
<DFSInput>dfs://data{*}</DFSInput>
<DFSOutput>dfs://output{04454992-E2CD-4342-AEEB-1D0607B32D84}</DFSOutput>
<KeyMajor>8</KeyMajor>
<OutputMethod>sorted</OutputMethod>
</IOSettings>
<MapReduce>
<Map>
<![CDATA[
public virtual void Map(ByteSlice line, MapOutput output)
{
output.Add(line, ByteSlice.Prepare());
}
]]>
</Map>
<Reduce>
<![CDATA[
public override void Reduce(ByteSlice key, RandomAccessEntries values, RandomAccessOutput output)
{
for(int i = 0; i < values.Length; i++)
{
output.Add(key);
}
}
]]>
</Reduce>
</MapReduce>
</Job>
</Jobs>
</SourceCode>
").Replace('`', '"'));
Exec.Shell("Qizmt importdir " + exectempdir);
try
{
System.IO.File.Delete(execfp);
System.IO.Directory.Delete(exectempdir);
}
catch
{
}
Exec.Shell("Qizmt exec exec{FA19CAB0-5225-4cc8-8728-9BFC3A1B834C}");
exec_md5 = DfsSum("md5", "output{04454992-E2CD-4342-AEEB-1D0607B32D84}");
Exec.Shell("Qizmt del output{04454992-E2CD-4342-AEEB-1D0607B32D84}");
}
try
{
Console.WriteLine("Disrupting 2 machines...");
{
string badmachine = allmachines[allmachines.Length - 1];
Console.WriteLine(" Bad disk on {0}", badmachine);
string netpath = Surrogate.NetworkPathForHost(badmachine);
foreach (System.IO.FileInfo fi in (new System.IO.DirectoryInfo(netpath)).GetFiles("zd.*.zd"))
{
if (!fi.Name.StartsWith("zd.!."))
{
System.IO.File.Move(fi.FullName, fi.DirectoryName + @"\zd.!." + fi.Name.Substring(3));
}
}
}
{
string badmachine = allmachines[allmachines.Length - 2];
Console.WriteLine(" Bad network connection on {0}", badmachine);
Exec.Shell(@"sc \\" + badmachine + @" stop DistributedObjects");
}
Console.WriteLine("Ensure the cluster is NOT perfectly healthy...");
{
bool healthy;
try
{
EnsurePerfectQizmtHealtha();
healthy = true;
}
catch
{
healthy = false;
}
if (healthy)
{
throw new Exception("Cluster is still healthy");
}
}
Console.WriteLine("Running job on unhealthy cluster...");
{
try
{
Exec.Shell("Qizmt exec exec{FA19CAB0-5225-4cc8-8728-9BFC3A1B834C}");
}
catch
{
// Replication will output a warning and throw an exception,
// so we need to ignore that exception.
// The MD5 check will ensure it ran fine.
}
string new_exec_md5 = DfsSum("md5", "output{04454992-E2CD-4342-AEEB-1D0607B32D84}");
Exec.Shell("Qizmt del output{04454992-E2CD-4342-AEEB-1D0607B32D84}");
if (new_exec_md5 != exec_md5)
{
throw new Exception("Output files from before and after disrupting cluster do not match");
}
}
}
finally
{
{
Console.WriteLine("Repairing disrupted disk");
string badmachine = allmachines[allmachines.Length - 1];
string netpath = Surrogate.NetworkPathForHost(badmachine);
foreach (System.IO.FileInfo fi in (new System.IO.DirectoryInfo(netpath)).GetFiles("zd.!.*.zd"))
{
System.IO.File.Move(fi.FullName, fi.DirectoryName + @"\zd." + fi.Name.Substring(5));
}
}
}
}
Console.WriteLine("[PASSED] - " + string.Join(" ", args));
}
finally
{
Console.WriteLine("Restoring DFS.xml backup...");
// Note: these are safe; the try/finally only wraps the new dfs.
try
{
Exec.Shell("Qizmt del *");
}
catch
{
}
try
{
// Delete temp dfs.xml, it's being replaced with the good one.
System.IO.File.Delete(dfsxmlpath);
}
catch
{
}
System.IO.File.Move(dfsxmlpathbackup, dfsxmlpath);
{
// Note: killall issued to fix disrupted machines
Console.WriteLine("Running killall to repair");
Exec.Shell("Qizmt killall -f");
}
}
}
protected static void UpdateDfsXml(dfs dc, string dfsxmlpath, string backupdir)
{
using (LockDfsMutex())
{
if (dc.UserModified)
{
throw new NotSupportedException("Cannot save user-modified " + dfs.DFSXMLNAME);
}
string oldpath = dfsxmlpath + ".old";
string newpath = dfsxmlpath + ".new";
//+++metabackup+++
string backupfile = null, oldbackupfile = null, newbackupfile = null;
if (null != backupdir)
{
//backupfile = backupdir + @"\dfs-" + Surrogate.SafeTextPath(System.Net.Dns.GetHostName()) + @".xml";
backupfile = backupdir + @"\dfs-backup.xml";
oldbackupfile = backupfile + ".old";
newbackupfile = backupfile + ".new";
}
//---metabackup---
// Write .new
for (; ; )
{
try
{
using (System.IO.FileStream fdc = new System.IO.FileStream(newpath, System.IO.FileMode.Create, System.IO.FileAccess.Write, System.IO.FileShare.None))
{
System.Xml.Serialization.XmlSerializer xs = new System.Xml.Serialization.XmlSerializer(typeof(dfs));
//xs.Serialize(fdc, dc);
System.Xml.XmlTextWriter xw = new System.Xml.XmlTextWriter(fdc, null); // null=UTF-8
#if DEBUG
xw.Formatting = System.Xml.Formatting.Indented;
#else
xw.Formatting = System.Xml.Formatting.None;
#endif
xs.Serialize(xw, dc);
}
//+++metabackup+++
if (null != backupdir)
{
System.IO.File.Copy(newpath, newbackupfile, true);
}
//---metabackup---
break;
}
catch (System.IO.IOException e)
{
_UpdateDfsXml_retryexception("write-new", e);
continue;
}
}
// Move current to .old
for (; ; )
{
try
{
try
{
System.IO.File.Delete(oldpath);
}
catch
{
}
// Check if dfs.xml exists first, because this might be the first save on format.
if (System.IO.File.Exists(dfsxmlpath)) // Not using dfs.DfsConfigExists()!
{
System.IO.File.Move(dfsxmlpath, oldpath);
}
break;
}
catch (System.IO.IOException e)
{
_UpdateDfsXml_retryexception("move-old", e);
continue;
}
}
//+++metabackup+++
if (null != backupdir)
{
for (; ; )
{
try
{
try
{
System.IO.File.Delete(oldbackupfile);
}
catch
{
}
// See if backup dfs.xml exists first, because metabackup might not have been on.
if (System.IO.File.Exists(backupfile))
{
System.IO.File.Move(backupfile, oldbackupfile);
}
break;
}
catch (System.IO.IOException e)
{
_UpdateDfsXml_retryexception("metabackup-move-old", e);
continue;
}
}
}
//---metabackup---
// Move .new to current.
for (; ; )
{
try
{
System.IO.File.Move(newpath, dfsxmlpath);
break;
}
catch (System.IO.IOException e)
{
_UpdateDfsXml_retryexception("move-new", e);
continue;
}
}
//+++metabackup+++
if (null != backupdir)
{
for (; ; )
{
try
{
System.IO.File.Move(newbackupfile, backupfile);
break;
}
catch (System.IO.IOException e)
{
_UpdateDfsXml_retryexception("metabackup-move-new", e);
continue;
}
}
}
//---metabackup---
// Delete .old
try
{
System.IO.File.Delete(oldpath);
}
catch
{
}
//+++metabackup+++
if (null != backupdir)
{
#if DEBUG
//System.Diagnostics.Debugger.Launch();
#endif
try
{
if (System.IO.File.Exists(oldbackupfile))
{
long lastprevid;
System.IO.FileInfo bfi = new System.IO.FileInfo(backupfile);
{
System.IO.FileInfo[] prevfiles = bfi.Directory.GetFiles(bfi.Name + ".*.prev");
Func<string, long> OrdFromXmlPrev = delegate(string fp)
{
int d = fp.LastIndexOf('.', fp.Length - 5 - 1) + 1;
string snum = fp.Substring(d, (fp.Length - 5) - d);
return long.Parse(snum);
};
Array.Sort<System.IO.FileInfo>(prevfiles, new Comparison<System.IO.FileInfo>(
delegate(System.IO.FileInfo fi1, System.IO.FileInfo fi2)
{
long ord1 = OrdFromXmlPrev(fi1.ToString());
long ord2 = OrdFromXmlPrev(fi2.ToString());
if (ord1 < ord2)
{
return -1;
}
if (ord1 > ord2)
{
return +1;
}
return 0;
}));
const int DFSXMLPREVMAXCOUNT = 10;
for (int i = 0; i < prevfiles.Length - (DFSXMLPREVMAXCOUNT - 1); i++)
{
try
{
System.IO.File.Delete(prevfiles[i].FullName);
}
catch
{
}
}
if (prevfiles.Length > 0)
{
lastprevid = OrdFromXmlPrev(prevfiles[prevfiles.Length - 1].ToString());
}
else
{
lastprevid = 0;
}
}
long newprevid;
if (lastprevid == long.MaxValue)
{
newprevid = 1;
}
else
{
newprevid = lastprevid + 1;
}
System.IO.File.Move(oldbackupfile, bfi.FullName + "." + newprevid + ".prev");
}
}
catch
{
}
try
{
System.IO.File.Delete(oldbackupfile);
}
catch
{
}
}
//---metabackup---
}
}
static void ReduceInitializeThrow(string[] args)
{
if (args.Length <= 1 || !System.IO.File.Exists(args[1]))
{
throw new Exception("Expected path to DFS.xml");
}
string dfsxmlpath = args[1];
string dfsxmlpathbackup = dfsxmlpath + "$" + Guid.NewGuid().ToString();
string masterdir;
{
System.IO.FileInfo fi = new System.IO.FileInfo(dfsxmlpath);
masterdir = fi.DirectoryName; // Directory's full path.
}
Surrogate.SetNewMetaLocation(masterdir);
dfs dc = dfs.ReadDfsConfig_unlocked(dfsxmlpath);
string masterhost = System.Net.Dns.GetHostName();
string[] allmachines;
{
string[] sl = dc.Slaves.SlaveList.Split(';');
List <string> aml = new List <string>(sl.Length + 1);
aml.Add(masterhost);
foreach (string slave in sl)
{
if (0 != string.Compare(IPAddressUtil.GetName(slave), IPAddressUtil.GetName(masterhost), StringComparison.OrdinalIgnoreCase))
{
aml.Add(slave);
}
}
allmachines = aml.ToArray();
}
{
// Test logic:
{
string exectempdir = @"\\" + System.Net.Dns.GetHostName() + @"\C$\temp\qizmt\regression_test_ReduceInitializeThrow-" + Guid.NewGuid().ToString();
if (!System.IO.Directory.Exists(exectempdir))
{
System.IO.Directory.CreateDirectory(exectempdir);
}
string execfp = exectempdir + @"\baderrordup{2E829DB6-E853-4fc2-B0B4-82ADAF637312}";
System.IO.File.WriteAllText(execfp, (@"<?xml version=`1.0` encoding=`utf-8`?>
<SourceCode>
<Jobs>
<Job Name=`PrepJob` Custodian=`` email=``>
<IOSettings>
<JobType>local</JobType>
</IOSettings>
<Local>
<![CDATA[
public virtual void Local()
{
Shell(@`Qizmt -dfs del baderrordup{2E829DB6-E853-4fc2-B0B4-82ADAF637312}_*.txt`,true); // Clean previous run.
}
]]>
</Local>
</Job>
<Job description=`Load sample data` Name=`baderrordup{2E829DB6-E853-4fc2-B0B4-82ADAF637312}_LoadData` Custodian=`` email=``>
<IOSettings>
<JobType>remote</JobType>
<DFS_IO>
<DFSReader></DFSReader>
<DFSWriter>dfs://baderrordup{2E829DB6-E853-4fc2-B0B4-82ADAF637312}_Input.txt</DFSWriter>
</DFS_IO>
</IOSettings>
<Remote>
<![CDATA[
public virtual void Remote(RemoteInputStream dfsinput, RemoteOutputStream dfsoutput)
{
dfsoutput.WriteLine(@`
MySpace is for everyone:
Friends who want to talk Online
Single people who want to meet other Singles
Matchmakers who want to connect their friends with other friends
Families who want to keep in touch--map your Family Tree
Business people and co-workers interested in networking
Classmates and study partners
Anyone looking for long lost friends!
`);
}
]]>
</Remote>
</Job>
<Job Name=`baderrordup{2E829DB6-E853-4fc2-B0B4-82ADAF637312}` Custodian=`` email=``>
<IOSettings>
<JobType>mapreduce</JobType>
<KeyLength>16</KeyLength>
<DFSInput>dfs://baderrordup{2E829DB6-E853-4fc2-B0B4-82ADAF637312}_Input.txt</DFSInput>
<DFSOutput>dfs://baderrordup{2E829DB6-E853-4fc2-B0B4-82ADAF637312}_Output.txt</DFSOutput>
</IOSettings>
<MapReduce>
<Map>
<![CDATA[
public virtual void Map(ByteSlice line, MapOutput output)
{
mstring sLine= mstring.Prepare(line);
mstringarray parts = sLine.SplitM(' ');
for(int i=0; i < parts.Length; i++)
{
mstring word = parts[i];
if(word.Length > 0 && word.Length <= 16) // Word cannot be longer than the KeyLength!
{
output.Add(word.ToLowerM(), mstring.Prepare(1));
}
}
}
]]>
</Map>
<ReduceInitialize>
<![CDATA[
public void ReduceInitialize()
{
Qizmt_Log(`ReduceInitialize {3AF77789-4D38-4fde-B5DD-DC5115A909F5}`);
throw new Exception(`Exception from ReduceInitialize {D5C5B240-537E-4b80-AD70-6463F632EE7B}`);
}
]]>
</ReduceInitialize>
<Reduce>
<![CDATA[
public override void Reduce(ByteSlice key, ByteSliceList values, ReduceOutput output)
{
mstring sLine = mstring.Prepare(UnpadKey(key));
sLine = sLine.AppendM(',').AppendM(values.Length);
output.Add(sLine);
}
]]>
</Reduce>
<ReduceFinalize>
<![CDATA[
public void ReduceFinalize()
{
Qizmt_Log(`ReduceFinalize {3C677456-22C5-46cd-A1E7-47383274C0C5}`);
}
]]>
</ReduceFinalize>
</MapReduce>
</Job>
</Jobs>
</SourceCode>
").Replace('`', '"'));
Exec.Shell("Qizmt importdir " + exectempdir);
try
{
System.IO.File.Delete(execfp);
System.IO.Directory.Delete(exectempdir);
}
catch
{
}
// Run it twice: first suppressing errors to get stdout, then again to get stderr exception.
string output = Exec.Shell("Qizmt exec baderrordup{2E829DB6-E853-4fc2-B0B4-82ADAF637312}", true);
try
{
Exec.Shell("Qizmt exec baderrordup{2E829DB6-E853-4fc2-B0B4-82ADAF637312}");
throw new Exception("<<< Job completed without errors; this is wrong! >>>");
}
catch (Exception e)
{
string err = e.ToString();
string badstr = "System.FormatException: Expected 16 hex digits, got \"-000000000000000\"";
if (-1 != err.IndexOf(badstr) || -1 != output.IndexOf(badstr))
{
throw new Exception("Test failed: broken protocol!", e);
}
if (-1 == err.IndexOf("{D5C5B240-537E-4b80-AD70-6463F632EE7B}"))
{
throw new Exception("Test failed: did not get expected exception from exec", e);
}
}
finally
{
Exec.Shell("Qizmt del baderrordup{2E829DB6-E853-4fc2-B0B4-82ADAF637312}*");
}
}
}
Console.WriteLine("[PASSED] - " + string.Join(" ", args));
}
// Note: metabackupdir can be null if no meta backup.
static void _DfsKillFileChunks_unlocked(dfs.DfsFile dfsf, string metabackupdir, bool verbose)
{
if (0 == string.Compare(DfsFileTypes.DELTA, dfsf.Type, true))
{
_KillSnowballFileChunks_unlocked(dfsf, verbose);
}
else if (0 == string.Compare(DfsFileTypes.DLL, dfsf.Type, true))
{
_KillDllFileChunks_unlocked(dfsf, verbose);
}
else if (0 == string.Compare(DfsFileTypes.JOB, dfsf.Type, true))
{
for (int dn = 0; dn < dfsf.Nodes.Count; dn++)
{
foreach (string chost in dfsf.Nodes[dn].Host.Split(';'))
{
try
{
System.IO.File.Delete(NetworkPathForHost(chost) + @"\" + dfsf.Nodes[dn].Name);
if (null != metabackupdir)
{
System.IO.File.Delete(metabackupdir + @"\" + dfsf.Nodes[dn].Name);
}
}
catch
{
}
}
}
if (verbose)
{
Console.WriteLine("Successfully deleted file '{0}' ({1} parts)", dfsf.Name, dfsf.Nodes.Count);
}
}
else if (dfsf.MemCache != null)
{
List<dfs.DfsFile> delfilesMC = new List<dfs.DfsFile>(1);
delfilesMC.Add(dfsf);
_KillMemCache_mt(delfilesMC, verbose);
}
else
{
for (int dn = 0; dn < dfsf.Nodes.Count; dn++)
{
foreach (string chost in dfsf.Nodes[dn].Host.Split(';'))
{
try
{
System.IO.File.Delete(NetworkPathForHost(chost) + @"\" + dfsf.Nodes[dn].Name);
System.IO.File.Delete(NetworkPathForHost(chost) + @"\" + dfsf.Nodes[dn].Name + ".zsa");
}
catch
{
}
}
}
if (verbose)
{
Console.WriteLine("Successfully deleted file '{0}' ({1} parts)", dfsf.Name, dfsf.Nodes.Count);
}
}
}
static void CacheWithRedundancy(string[] args)
{
if (args.Length <= 1 || !System.IO.File.Exists(args[1]))
{
throw new Exception("Expected path to DFS.xml");
}
string dfsxmlpath = args[1];
string dfsxmlpathbackup = dfsxmlpath + "$" + Guid.NewGuid().ToString();
string masterdir;
{
System.IO.FileInfo fi = new System.IO.FileInfo(dfsxmlpath);
masterdir = fi.DirectoryName; // Directory's full path.
}
Surrogate.SetNewMetaLocation(masterdir);
dfs olddfs = dfs.ReadDfsConfig_unlocked(dfsxmlpath);
string masterhost = System.Net.Dns.GetHostName();
string[] allmachines;
{
string[] sl = olddfs.Slaves.SlaveList.Split(';');
List <string> aml = new List <string>(sl.Length + 1);
aml.Add(masterhost);
foreach (string slave in sl)
{
if (0 != string.Compare(IPAddressUtil.GetName(slave), IPAddressUtil.GetName(masterhost), StringComparison.OrdinalIgnoreCase))
{
aml.Add(slave);
}
}
allmachines = aml.ToArray();
}
Console.WriteLine("Backing up DFS.xml to: {0} ...", dfsxmlpathbackup);
try
{
System.IO.File.Delete(dfsxmlpathbackup);
}
catch
{
}
System.IO.File.Move(dfsxmlpath, dfsxmlpathbackup);
try
{
{
// Note: added replication.
Console.WriteLine("Formatting DFS with Replication for test...");
Exec.Shell("Qizmt @format Machines=" + string.Join(",", allmachines) + " Replication=2");
}
{
// Test logic:
Console.WriteLine("Ensure the cluster is perfectly healthy...");
EnsurePerfectQizmtHealtha();
Console.WriteLine("Adding files to DFS...");
Exec.Shell("Qizmt wordgen Cacher_input 10MB 100");
Console.WriteLine("Generating cache files...");
string cachertempdir = @"\\" + System.Net.Dns.GetHostName() + @"\C$\temp\qizmt\regression_test_Cacher-" + Guid.NewGuid().ToString();
if (!System.IO.Directory.Exists(cachertempdir))
{
System.IO.Directory.CreateDirectory(cachertempdir);
}
string cacherfp = cachertempdir + @"\Cacher";
System.IO.File.WriteAllText(cacherfp, (@"<?xml version=`1.0` encoding=`utf-8`?>
<SourceCode>
<Jobs>
<Job Name=`Cacher`>
<Delta>
<Name>Cacher_cache</Name>
<DFSInput>dfs://Cacher_input</DFSInput>
</Delta>
<IOSettings>
<JobType>mapreduce</JobType>
<KeyLength>100</KeyLength>
<DFSInput></DFSInput>
<DFSOutput>dfs://Cacher_output</DFSOutput>
<KeyMajor>8</KeyMajor>
<OutputMethod>grouped</OutputMethod>
</IOSettings>
<MapReduce>
<Map>
<![CDATA[
public virtual void Map(ByteSlice line, MapOutput output)
{
output.Add(line, ByteSlice.Prepare());
}
]]>
</Map>
<Reduce>
<![CDATA[
public override void Reduce(ByteSlice key, RandomAccessEntries values, RandomAccessOutput output)
{
for(int i = 0; i < values.Length; i++)
{
output.Add(key);
}
}
]]>
</Reduce>
</MapReduce>
</Job>
</Jobs>
</SourceCode>
").Replace('`', '"'));
Exec.Shell("Qizmt importdir " + cachertempdir);
try
{
System.IO.File.Delete(cacherfp);
System.IO.Directory.Delete(cachertempdir);
}
catch
{
}
Exec.Shell("Qizmt exec Cacher"); // Creates cache file Cacher_cache
string cacher_output_sum = DfsSum("Cacher_output");
string cacher_output_md5 = DfsSum("md5", "Cacher_output");
if (cacher_output_sum != DfsSum("Cacher_input"))
{
throw new Exception("Output file does not have same checksum as input file");
}
Exec.Shell("Qizmt del Cacher_output");
Console.WriteLine("Ensure the cluster is perfectly healthy...");
EnsurePerfectQizmtHealtha();
Console.WriteLine("Validate cache files...");
Exec.Shell("Qizmt exec Cacher"); // Uses existing cache file Cacher_cache
if (cacher_output_sum != DfsSum("Cacher_output"))
{
throw new Exception("Output file not the same when using cache (sum)");
}
if (cacher_output_md5 != DfsSum("md5", "Cacher_output"))
{
throw new Exception("Output file not the same when using cache (md5)");
}
Exec.Shell("Qizmt del Cacher_output");
Console.WriteLine("Ensure the cluster is perfectly healthy...");
EnsurePerfectQizmtHealtha();
}
Console.WriteLine("[PASSED] - " + string.Join(" ", args));
}
finally
{
Console.WriteLine("Restoring DFS.xml backup...");
// Note: these are safe; the try/finally only wraps the new dfs.
try
{
Exec.Shell("Qizmt del *");
}
catch
{
}
try
{
// Delete temp dfs.xml, it's being replaced with the good one.
System.IO.File.Delete(dfsxmlpath);
}
catch
{
}
System.IO.File.Move(dfsxmlpathbackup, dfsxmlpath);
}
}
internal static void EnsureMetaBackupLocation(dfs dc)
{
// Make sure the backup location is writable and isn't the same as the primary store!
string mbl = dc.GetMetaBackupLocation();
if (!mbl.StartsWith(@"\\"))
{
throw new Exception("Must supply a network path for new metabackup location");
}
if (!System.IO.Path.IsPathRooted(mbl))
{
throw new Exception("MetaBackupLocation must be an absolute path");
}
try
{
System.IO.Directory.CreateDirectory(mbl);
}
catch
{
}
string mblfn = @"mbl-" + Guid.NewGuid().ToString() + @".test";
string mblfp = mbl + @"\" + mblfn;
try
{
System.IO.File.WriteAllText(mblfp, "MetaBackupLocation write test" + Environment.NewLine);
}
catch (Exception e)
{
throw new Exception("Unable to write to MetaBackupLocation " + mbl, e);
}
bool mblfexists = System.IO.File.Exists(AELight_Dir + @"\" + mblfn);
try
{
System.IO.File.Delete(mblfp);
}
catch
{
}
if (mblfexists)
{
throw new Exception("MetaBackupLocation is the same as the normal meta-data storage location: " + mbl);
}
}
public static void ExecOneLocal(SourceCode.Job cfgj, string[] ExecArgs, bool verbose)
{
if (verbose)
{
Console.WriteLine("[{0}] [Local: {2}]", System.DateTime.Now.ToString(), System.DateTime.Now.Millisecond, cfgj.NarrativeName);
}
int BlockID = 0;
string SlaveIP = null;
string logname = Surrogate.SafeTextPath(cfgj.NarrativeName) + "_" + Guid.NewGuid().ToString() + ".j" + sjid + "_log.txt";
bool aborting = false;
try
{
dfs dc = LoadDfsConfig();
string firstslave = dc.Slaves.GetFirstSlave();
string SlaveHost = cfgj.IOSettings.LocalHost;
if (SlaveHost == null || SlaveHost.Length == 0)
{
SlaveHost = firstslave;
}
SlaveIP = IPAddressUtil.GetIPv4Address(SlaveHost);
MySpace.DataMining.DistributedObjects5.Remote rem = new MySpace.DataMining.DistributedObjects5.Remote(cfgj.NarrativeName + "_local");
rem.OutputStartingPoint = BlockID;
rem.LocalCompile = true;
rem.CompilerOptions = cfgj.IOSettings.CompilerOptions;
rem.CompilerVersion = cfgj.IOSettings.CompilerVersion;
if (cfgj.OpenCVExtension != null)
{
rem.AddOpenCVExtension();
}
if (cfgj.MemCache != null)
{
rem.AddMemCacheExtension();
}
if (cfgj.Unsafe != null)
{
rem.AddUnsafe();
}
if (cfgj.AssemblyReferencesCount > 0)
{
cfgj.AddAssemblyReferences(rem.CompilerAssemblyReferences, Surrogate.NetworkPathForHost(firstslave));
}
rem.SetJID(jid, CurrentJobFileName + " Local: " + cfgj.NarrativeName);
rem.AddBlock(SlaveHost + @"|"
+ (cfgj.ForceStandardError != null ? "&" : "") + logname + @"|slaveid=0");
rem.Open();
string codectx = (@"
public const int DSpace_BlockID = " + BlockID.ToString() + @";
public const int DSpace_ProcessID = DSpace_BlockID;
public const int Qizmt_ProcessID = DSpace_ProcessID;
public const int DSpace_BlocksTotalCount = 1;
public const int DSpace_ProcessCount = DSpace_BlocksTotalCount;
public const int Qizmt_ProcessCount = DSpace_ProcessCount;
public const string DSpace_SlaveHost = `" + SlaveHost + @"`;
public const string DSpace_MachineHost = DSpace_SlaveHost;
public const string Qizmt_MachineHost = DSpace_MachineHost;
public const string DSpace_SlaveIP = `" + SlaveIP + @"`;
public const string DSpace_MachineIP = DSpace_SlaveIP;
public const string Qizmt_MachineIP = DSpace_MachineIP;
public static readonly string[] DSpace_ExecArgs = new string[] { " + ExecArgsCode(ExecArgs) + @" };
public static readonly string[] Qizmt_ExecArgs = DSpace_ExecArgs;
public const string DSpace_ExecDir = @`" + System.Environment.CurrentDirectory + @"`;
public const string Qizmt_ExecDir = DSpace_ExecDir;
static string Shell(string line, bool suppresserrors)
{
return MySpace.DataMining.DistributedObjects.Exec.Shell(line, suppresserrors);
}
static string Shell(string line)
{
return MySpace.DataMining.DistributedObjects.Exec.Shell(line, false);
}
const string _userlogname = `" + logname + @"`;
static System.Threading.Mutex _logmutex = new System.Threading.Mutex(false, `distobjlog`);
private static int userlogsremain = " + AELight.maxuserlogs.ToString() + @";
public static void Qizmt_Log(string line) { DSpace_Log(line); }
public static void DSpace_Log(string line)
{
if(--userlogsremain < 0)
{
return;
}
try
{
_logmutex.WaitOne();
}
catch (System.Threading.AbandonedMutexException)
{
}
try
{
using (System.IO.StreamWriter fstm = System.IO.File.AppendText(_userlogname))
{
fstm.WriteLine(`{0}`, line);
}
}
finally
{
_logmutex.ReleaseMutex();
}
}
public static void Qizmt_LogResult(string line, bool passed) { DSpace_LogResult(line, passed); }
public static void DSpace_LogResult(string name, bool passed)
{
if(passed)
{
DSpace_Log(`[\u00012PASSED\u00010] - ` + name);
}
else
{
DSpace_Log(`[\u00014FAILED\u00010] - ` + name);
}
}
").Replace('`', '"') + CommonDynamicCsCode;
rem.LocalExec(codectx + cfgj.Local, cfgj.Usings);
rem.Close();
if (verbose)
{
Console.Write('*');
ConsoleFlush();
}
}
catch (System.Threading.ThreadAbortException)
{
aborting = true;
}
finally
{
if (!aborting)
{
CheckUserLogs(new string[] { SlaveIP }, logname);
}
}
if (verbose)
{
Console.WriteLine();
Console.WriteLine("[{0}] Done", System.DateTime.Now.ToString(), System.DateTime.Now.Millisecond);
}
}
public static dfs.DfsFile.FileNode DfsFindFileNode(dfs dc, string nodename, out string ownerfilename)
{
return dc.FindFileNode(nodename, out ownerfilename);
}
public static void MemCacheCommand(string[] args)
{
if (args.Length < 1)
{
Console.Error.WriteLine("Expected memcache sub-command");
SetFailure();
return;
}
string act = args[0].ToLower();
switch (act)
{
case "create":
{
string mcname = null;
string mcschema = null;
int mcsegsize = -1;
EachArgument(args, 1,
new Action <string, string>(
delegate(string key, string value)
{
key = key.ToLower();
switch (key)
{
case "name":
mcname = value;
break;
case "schema":
mcschema = value;
break;
case "segment":
case "segsize":
case "segmentsize":
mcsegsize = ParseCapacity(value);
break;
}
}));
if (string.IsNullOrEmpty(mcname))
{
Console.Error.WriteLine("Expected name=<MemCacheName>");
SetFailure();
return;
}
if (string.IsNullOrEmpty(mcschema))
{
Console.Error.WriteLine("Expected schema=<schema>");
SetFailure();
return;
}
if (-1 != mcsegsize && mcsegsize < 1024)
{
Console.Error.WriteLine("Error: segment={0} is too small", mcsegsize);
SetFailure();
return;
}
if (mcname.StartsWith("dfs://", StringComparison.OrdinalIgnoreCase))
{
mcname = mcname.Substring(6);
}
{
string reason;
if (dfs.IsBadFilename(mcname, out reason))
{
Console.Error.WriteLine("MemCache cannot be named '{0}': {1}", mcname, reason);
SetFailure();
return;
}
}
dfs.DfsFile.ConfigMemCache cmc = new dfs.DfsFile.ConfigMemCache();
cmc.MetaFileName = "mcm." + Surrogate.SafeTextPath(mcname) + ".mcm";
cmc.Schema = mcschema;
List <int> offsets = new List <int>();
cmc.RowLength = Surrogate.GetRecordInfo(mcschema, out cmc.KeyOffset, out cmc.KeyLength, offsets);
/*if (0 == cmc.KeyOffset
* && cmc.RowLength == cmc.KeyLength
* && -1 == mcschema.IndexOf('['))
* {
* Console.WriteLine("Note: no key was specified, the key is the entire row");
* }*/
if (-1 == mcsegsize)
{
const int defsegsize = 0x400 * 0x400 * 64;
cmc.SegmentSize = defsegsize - (defsegsize % cmc.RowLength);
}
else
{
if (0 != (mcsegsize % cmc.RowLength))
{
Console.Error.WriteLine("Segment size must be a multiple of the row length");
Console.Error.WriteLine("Nearest segment size is {0} bytes",
mcsegsize - (mcsegsize % cmc.RowLength));
SetFailure();
return;
}
cmc.SegmentSize = mcsegsize;
}
{
StringBuilder sbFieldOffsets = new StringBuilder();
foreach (int offset in offsets)
{
if (sbFieldOffsets.Length != 0)
{
sbFieldOffsets.Append(',');
}
sbFieldOffsets.Append(offset);
}
cmc.FieldOffsets = sbFieldOffsets.ToString();
}
dfs.DfsFile df = new dfs.DfsFile();
df.Nodes = new List <dfs.DfsFile.FileNode>(0);
df.MemCache = cmc;
df.Name = mcname;
df.XFileType = DfsFileTypes.BINARY_RECT + "@" + cmc.RowLength;
df.Size = 0;
dfs dc = LoadDfsConfig();
{
dfs.DfsFile df2 = dc.FindAny(df.Name);
if (null != df2)
{
Console.Error.WriteLine("Error: a file named '{0}' already exists", df2.Name);
SetFailure();
return;
}
}
{
string startmeta = GetMemCacheMetaFileHeader(df);
string[] slaves = dc.Slaves.SlaveList.Split(';');
int totalworkercount = dc.Blocks.TotalCount; // Subprocess_TotalPrime
StringBuilder[] permachine = new StringBuilder[slaves.Length];
//byte[] HEADER = new byte[4];
//MySpace.DataMining.DistributedObjects.Entry.ToBytes(4, HEADER, 0);
for (int i = 0; i < permachine.Length; i++)
{
permachine[i] = new StringBuilder(256);
}
{
int si = -1;
for (int workerid = 0; workerid < totalworkercount; workerid++)
{
if (++si >= slaves.Length)
{
si = 0;
}
StringBuilder sb = permachine[si];
sb.AppendFormat("##{1}:{0}", Environment.NewLine, workerid);
// There's no segments, but write a dummy one for bookkeeping.
foreach (char snc in "MemCache_" + mcname + "_empty")
{
sb.Append(snc);
}
{
sb.Append(' ');
/*
* StringBuilder newchunkpath = new StringBuilder(100);
* newchunkpath.Append(Surrogate.NetworkPathForHost(slaves[si]));
* newchunkpath.Append('\\');
* */
// Make up a data node chunk name.
foreach (char ch in MakeMemCacheChunkName(mcname, workerid))
{
//newchunkpath.Append(ch);
sb.Append(ch);
}
// Write the empty chunk.
//System.IO.File.WriteAllBytes(newchunkpath.ToString(), HEADER);
}
//if (IsLastSegment) // true
{
sb.Append(' ');
string shexlen = string.Format("{0:x8}", 0); // Zero-length!
for (int i = 0; i < shexlen.Length; i++)
{
sb.Append(shexlen[i]);
}
}
sb.AppendLine();
}
}
for (int si = 0; si < slaves.Length; si++)
{
string slave = slaves[si];
string fp = Surrogate.NetworkPathForHost(slave) + @"\" + cmc.MetaFileName;
using (System.IO.StreamWriter sw = new System.IO.StreamWriter(fp))
{
sw.Write(startmeta);
sw.Write(permachine[si].ToString());
}
}
}
using (LockDfsMutex())
{
dc = LoadDfsConfig(); // Load again in update lock.
{
dfs.DfsFile df2 = dc.FindAny(df.Name);
if (null != df2)
{
Console.Error.WriteLine("Error: a file named '{0}' already exists", df2.Name);
SetFailure();
return;
}
}
dc.Files.Add(df);
UpdateDfsXml(dc);
}
try
{
// Need to commit it so that the empty chunks are in the metadata for bookkeeping.
// This has to be done after actually adding it to dfsxml.
MemCacheFlush(mcname);
}
catch (Exception e)
{
try
{
MemCacheDelete(mcname, false);
}
catch
{
}
Console.Error.WriteLine("Error: unable to commit newly created MemCache '{0}'; because:{1}{2}",
mcname, Environment.NewLine, e.ToString());
SetFailure();
return;
}
Console.WriteLine("Successfully created MemCache '{0}'", mcname);
}
break;
case "delete":
case "del":
case "rm":
{
string mcname = null;
EachArgument(args, 1,
new Action <string, string>(
delegate(string key, string value)
{
key = key.ToLower();
switch (key)
{
case "name":
mcname = value;
break;
}
}));
if (string.IsNullOrEmpty(mcname))
{
Console.Error.WriteLine("Expected name=<MemCacheName>");
SetFailure();
return;
}
MemCacheDelete(mcname, true);
}
break;
case "flush":
case "commit":
{
string mcname = null;
EachArgument(args, 1,
new Action <string, string>(
delegate(string key, string value)
{
key = key.ToLower();
switch (key)
{
case "name":
mcname = value;
break;
}
}));
if (string.IsNullOrEmpty(mcname))
{
Console.Error.WriteLine("Expected name=<MemCacheName>");
SetFailure();
return;
}
try
{
MemCacheFlush(mcname);
Console.WriteLine("Done");
}
catch (Exception e)
{
Console.WriteLine(" Commit was unsuccessful because: {0}", e.Message);
Console.WriteLine();
Console.Error.WriteLine(e.ToString());
SetFailure();
return;
}
}
break;
case "release":
case "rollback":
{
string mcname = null;
bool force = false;
EachArgument(args, 1,
new Action <string, string>(
delegate(string key, string value)
{
key = key.ToLower();
switch (key)
{
case "name":
mcname = value;
break;
case "-f":
force = true;
break;
}
}));
if (string.IsNullOrEmpty(mcname))
{
Console.Error.WriteLine("Expected name=<MemCacheName>");
SetFailure();
return;
}
try
{
MemCacheRelease(mcname, force);
Console.WriteLine("Done");
}
catch (Exception e)
{
string exception = e.ToString();
if (-1 != exception.IndexOf("MemCacheWarning"))
{
Console.WriteLine("Warning: " + exception);
}
else
{
Console.Error.WriteLine(exception);
string ioe = "InvalidOperationException:";
if (!force && -1 != exception.IndexOf(ioe))
{
try
{
string emsg = exception.Substring(exception.IndexOf(ioe) + ioe.Length)
.Split('\r', '\n')[0].Trim();
System.Threading.Thread.Sleep(100);
Console.WriteLine();
Console.WriteLine("{0}{2}{1}",
false ? "\u00014" : "", false ? "\u00010" : "",
emsg);
System.Threading.Thread.Sleep(100);
}
catch
{
}
Console.Error.WriteLine("Use rollback -f followed by killall to force rollback");
}
SetFailure();
return;
}
}
}
break;
case "load":
{
string mcname = null;
EachArgument(args, 1,
new Action <string, string>(
delegate(string key, string value)
{
key = key.ToLower();
switch (key)
{
case "name":
mcname = value;
break;
}
}));
if (string.IsNullOrEmpty(mcname))
{
Console.Error.WriteLine("Expected name=<MemCacheName>");
SetFailure();
return;
}
MemCacheLoad(mcname);
Console.WriteLine("Done");
}
break;
case "info":
case "information":
{
string mcname = null;
EachArgument(args, 1,
new Action <string, string>(
delegate(string key, string value)
{
key = key.ToLower();
switch (key)
{
case "name":
mcname = value;
break;
}
}));
if (string.IsNullOrEmpty(mcname))
{
Console.Error.WriteLine("Expected name=<MemCacheName>");
SetFailure();
return;
}
if (mcname.StartsWith("dfs://", StringComparison.OrdinalIgnoreCase))
{
mcname = mcname.Substring(6);
}
dfs dc = LoadDfsConfig();
dfs.DfsFile df = dc.FindAny(mcname);
if (null == df || df.MemCache == null)
{
Console.Error.WriteLine("Error: '{0}' is not a MemCache", (null == df ? mcname : df.Name));
SetFailure();
return;
}
Console.WriteLine(" MemCache: {0}", df.Name);
Console.WriteLine(" Segment size: {0} ({1})",
GetFriendlyByteSize(df.MemCache.SegmentSize),
df.MemCache.SegmentSize);
Console.WriteLine(" Schema: {0}", df.MemCache.Schema);
Console.WriteLine(" Row Length: {0}", df.MemCache.RowLength);
Console.WriteLine(" Key Offset: {0}", df.MemCache.KeyOffset);
Console.WriteLine(" Key Length: {0}", df.MemCache.KeyLength);
}
break;
default:
Console.Error.WriteLine("No such sub-command for memcache: {0}", act);
SetFailure();
return;
}
}
public static void ExecOneRemote(SourceCode.Job cfgj, string[] ExecArgs, bool verbose, bool verbosereplication)
{
if (verbose)
{
Console.WriteLine("[{0}] [Remote: {2}]", System.DateTime.Now.ToString(), System.DateTime.Now.Millisecond, cfgj.NarrativeName);
}
string logname = Surrogate.SafeTextPath(cfgj.NarrativeName) + "_" + Guid.NewGuid().ToString() + ".j" + sjid + "_log.txt";
//System.Threading.Thread.Sleep(8000);
/*if (cfgj.IOSettings.DFS_IOs == null || cfgj.IOSettings.DFS_IOs.Length == 0)
* {
* Console.Error.WriteLine("One or more IOSettings/DFS_IO needed in configuration for 'remote'");
* return;
* }*/
// Could provide BlockID here, which is just the n-th DFS_IO entry.
//cfgj.Remote
dfs dc = LoadDfsConfig();
string[] slaves = dc.Slaves.SlaveList.Split(',', ';');
if (dc.Slaves.SlaveList.Length == 0 || slaves.Length < 1)
{
throw new Exception("SlaveList expected in " + dfs.DFSXMLNAME);
}
if (dc.Replication > 1)
{
string[] slavesbefore = slaves;
slaves = ExcludeUnhealthySlaveMachines(slaves, true).ToArray();
if (slavesbefore.Length - slaves.Length >= dc.Replication)
{
throw new Exception("Not enough healthy machines to run job (hit replication count)");
}
}
if (cfgj.IOSettings.DFS_IO_Multis != null)
{
cfgj.ExpandDFSIOMultis(slaves.Length, MySpace.DataMining.DistributedObjects.MemoryUtils.NumberOfProcessors);
}
Dictionary <string, int> slaveIDs = new Dictionary <string, int>();
for (int si = 0; si < slaves.Length; si++)
{
slaveIDs.Add(slaves[si].ToUpper(), si);
}
bool aborting = false;
try
{
List <RemoteBlockInfo> blocks = new List <RemoteBlockInfo>(cfgj.IOSettings.DFS_IOs.Length);
if (verbose)
{
Console.WriteLine("{0} processes on {1} machines:", cfgj.IOSettings.DFS_IOs.Length, slaves.Length);
}
List <string> outputdfsdirs = new List <string>(slaves.Length);
{
for (int i = 0; i < slaves.Length; i++)
{
try
{
outputdfsdirs.Add(NetworkPathForHost(slaves[i]));
}
catch (Exception e)
{
Console.Error.WriteLine(" {0}", e.Message);
}
}
}
string slaveconfigxml = "";
{
System.Xml.XmlDocument pdoc = new System.Xml.XmlDocument();
{
System.IO.MemoryStream ms = new System.IO.MemoryStream();
System.Xml.Serialization.XmlSerializer xs = new System.Xml.Serialization.XmlSerializer(typeof(dfs));
xs.Serialize(ms, dc);
ms.Seek(0, System.IO.SeekOrigin.Begin);
pdoc.Load(ms);
}
string xml = pdoc.DocumentElement.SelectSingleNode("./slave").OuterXml;
//System.Threading.Thread.Sleep(8000);
slaveconfigxml = xml;
}
{
// Temporary:
for (int si = 0; si < slaves.Length; si++)
{
System.Threading.Mutex m = new System.Threading.Mutex(false, "AEL_SC_" + slaves[si]);
try
{
m.WaitOne();
}
catch (System.Threading.AbandonedMutexException)
{
}
try
{
System.IO.File.WriteAllText(NetworkPathForHost(slaves[si]) + @"\slaveconfig.j" + sjid + ".xml", slaveconfigxml);
}
catch
{
}
finally
{
m.ReleaseMutex();
m.Close();
}
}
}
int nextslave = (new Random(DateTime.Now.Millisecond / 2 + System.Diagnostics.Process.GetCurrentProcess().Id / 2)).Next() % slaves.Length;
int hosttypes = 0;
List <int> outputrecordlengths = new List <int>();
List <int> inputrecordlengths = new List <int>();
for (int BlockID = 0; BlockID < cfgj.IOSettings.DFS_IOs.Length; BlockID++)
{
int slaveHostID = 0;
RemoteBlockInfo bi = new RemoteBlockInfo();
bi.sampledist = dc.DataNodeBaseSize / dc.DataNodeSamples;
bi.BlockID = BlockID;
bi.blockcount = cfgj.IOSettings.DFS_IOs.Length;
if (string.IsNullOrEmpty(cfgj.IOSettings.DFS_IOs[BlockID].Host))
{
if (0 != hosttypes && 1 != hosttypes)
{
throw new Exception("DFS_IO/Host tag must be specified for all or none");
}
hosttypes = 1;
bi.SlaveHost = slaves[nextslave];
slaveHostID = nextslave;
bi.explicithost = false;
}
else
{
if (0 != hosttypes && 2 != hosttypes)
{
throw new Exception("DFS_IO/Host tag must be specified for all or none");
}
hosttypes = 2;
bi.SlaveHost = cfgj.IOSettings.DFS_IOs[BlockID].Host;
slaveHostID = slaveIDs[bi.SlaveHost.ToUpper()];
bi.explicithost = true;
}
bi.ExecArgs = ExecArgs;
if (++nextslave >= slaves.Length)
{
nextslave = 0;
}
bi.logname = logname;
bi.outputdfsdirs = outputdfsdirs;
bi.slaves = slaves;
bi.baseoutputfilesize = dc.DataNodeBaseSize;
bi.cfgj = cfgj;
bi.DFSWriter = cfgj.IOSettings.DFS_IOs[BlockID].DFSWriter.Trim();
bi.Meta = cfgj.IOSettings.DFS_IOs[BlockID].Meta;
List <string> dfswriters = new List <string>();
if (bi.DFSWriter.Length > 0)
{
string[] writers = bi.DFSWriter.Split(';');
for (int wi = 0; wi < writers.Length; wi++)
{
string thiswriter = writers[wi].Trim();
if (thiswriter.Length == 0)
{
continue;
}
int ic = thiswriter.IndexOf('@');
int reclen = -1;
if (-1 != ic)
{
try
{
reclen = Surrogate.GetRecordSize(thiswriter.Substring(ic + 1));
thiswriter = thiswriter.Substring(0, ic);
}
catch (FormatException e)
{
Console.Error.WriteLine("Error: remote output record length error: {0} ({1})", thiswriter, e.Message);
SetFailure();
return;
}
catch (OverflowException e)
{
Console.Error.WriteLine("Error: remote output record length error: {0} ({1})", thiswriter, e.Message);
SetFailure();
return;
}
}
string outfn = thiswriter;
if (outfn.StartsWith(@"dfs://", StringComparison.OrdinalIgnoreCase))
{
outfn = outfn.Substring(6);
}
string reason = "";
if (dfs.IsBadFilename(outfn, out reason))
{
Console.Error.WriteLine("Invalid output file: {0}", reason);
return;
}
if (null != DfsFindAny(dc, outfn))
{
Console.Error.WriteLine("Error: output file already exists in DFS: {0}", outfn);
return;
}
dfswriters.Add(thiswriter);
outputrecordlengths.Add(reclen);
}
}
else
{
dfswriters.Add("");
outputrecordlengths.Add(-1);
}
bi.DFSWriters = dfswriters;
bi.verbose = verbose;
bi.rem = new MySpace.DataMining.DistributedObjects5.Remote(cfgj.NarrativeName + "_remote");
bi.rem.CookRetries = dc.slave.CookRetries;
bi.rem.CookTimeout = dc.slave.CookTimeout;
bi.rem.DfsSampleDistance = bi.sampledist;
bi.rem.CompressFileOutput = dc.slave.CompressDfsChunks;
bi.rem.LocalCompile = true;
bi.rem.OutputStartingPoint = slaveHostID;
bi.rem.CompilerOptions = cfgj.IOSettings.CompilerOptions;
bi.rem.CompilerVersion = cfgj.IOSettings.CompilerVersion;
if (cfgj.AssemblyReferencesCount > 0)
{
cfgj.AddAssemblyReferences(bi.rem.CompilerAssemblyReferences, Surrogate.NetworkPathForHost(dc.Slaves.GetFirstSlave()));
}
if (cfgj.OpenCVExtension != null)
{
bi.rem.AddOpenCVExtension();
}
if (cfgj.MemCache != null)
{
bi.rem.AddMemCacheExtension();
}
if (cfgj.Unsafe != null)
{
bi.rem.AddUnsafe();
}
{
List <dfs.DfsFile.FileNode> nodes = new List <dfs.DfsFile.FileNode>();
List <string> mapfileswithnodes = null;
List <int> nodesoffsets = null;
IList <string> mapfiles = SplitInputPaths(dc, cfgj.IOSettings.DFS_IOs[BlockID].DFSReader);
if (mapfiles.Count > 0)
{
mapfileswithnodes = new List <string>(mapfiles.Count);
nodesoffsets = new List <int>(mapfiles.Count);
}
for (int i = 0; i < mapfiles.Count; i++)
{
string dp = mapfiles[i].Trim();
int inreclen = -1;
if (0 != dp.Length) // Allow empty entry where input isn't wanted.
{
if (dp.StartsWith("dfs://", StringComparison.OrdinalIgnoreCase))
{
dp = dp.Substring(6);
}
{
int ic = dp.IndexOf('@');
if (-1 != ic)
{
try
{
inreclen = Surrogate.GetRecordSize(dp.Substring(ic + 1));
dp = dp.Substring(0, ic);
}
catch (FormatException e)
{
Console.Error.WriteLine("Error: remote input record length error: {0} ({1})", dp, e.Message);
SetFailure();
return;
}
catch (OverflowException e)
{
Console.Error.WriteLine("Error: remote input record length error: {0} ({1})", dp, e.Message);
SetFailure();
return;
}
}
}
dfs.DfsFile df;
if (inreclen > 0 || inreclen == -2)
{
df = DfsFind(dc, dp, DfsFileTypes.BINARY_RECT);
if (null != df && inreclen != df.RecordLength)
{
Console.Error.WriteLine("Error: remote input file does not have expected record length of {0}: {1}@{2}", inreclen, dp, df.RecordLength);
SetFailure();
return;
}
}
else
{
df = DfsFind(dc, dp);
}
if (null == df)
{
//throw new Exception("Remote input file not found in DFS: " + dp);
Console.Error.WriteLine("Remote input file not found in DFS: {0}", dp);
return;
}
if (df.Nodes.Count > 0)
{
mapfileswithnodes.Add(dp);
nodesoffsets.Add(nodes.Count);
inputrecordlengths.Add(inreclen);
nodes.AddRange(df.Nodes);
}
}
}
bi.dfsinputpaths = new List <string>(nodes.Count);
//MapNodesToNetworkPaths(nodes, bi.dfsinputpaths);
dfs.MapNodesToNetworkStarPaths(nodes, bi.dfsinputpaths);
bi.dfsinputfilenames = mapfileswithnodes;
bi.dfsinputnodesoffsets = nodesoffsets;
}
blocks.Add(bi);
bi.thread = new System.Threading.Thread(new System.Threading.ThreadStart(bi.threadproc));
bi.thread.Name = "RemoteJobBlock" + bi.BlockID;
}
MySpace.DataMining.DistributedObjects.StaticGlobals.DSpace_InputRecordLength = inputrecordlengths.Count > 0 ? inputrecordlengths[0] : -1;
MySpace.DataMining.DistributedObjects.StaticGlobals.DSpace_OutputRecordLength = outputrecordlengths.Count > 0 ? outputrecordlengths[0] : -1;
// Need to start threads separately due to StaticGlobals being updated.
for (int BlockID = 0; BlockID < cfgj.IOSettings.DFS_IOs.Length; BlockID++)
{
RemoteBlockInfo bi = blocks[BlockID];
bi.rem.InputRecordLength = MySpace.DataMining.DistributedObjects.StaticGlobals.DSpace_InputRecordLength;
bi.rem.InputRecordLengths = inputrecordlengths;
bi.rem.OutputRecordLength = MySpace.DataMining.DistributedObjects.StaticGlobals.DSpace_OutputRecordLength;
bi.rem.OutputRecordLengths = outputrecordlengths;
AELight_StartTraceThread(bi.thread);
}
for (int BlockID = 0; BlockID < blocks.Count; BlockID++)
{
AELight_JoinTraceThread(blocks[BlockID].thread);
blocks[BlockID].rem.Close();
if (blocks[BlockID].blockfail)
{
Console.Error.WriteLine("BlockID {0} on host '{1}' did not complete successfully", BlockID, (blocks[BlockID].SlaveHost != null) ? blocks[BlockID].SlaveHost : "<null>");
continue;
}
}
List <string> dfsnames = new List <string>();
List <string> dfsnamesreplicating = new List <string>();
// Reload DFS config to make sure changes since starting get rolled in, and make sure the output file wasn't created in that time...
using (LockDfsMutex()) // Needed: change between load & save should be atomic.
{
dc = LoadDfsConfig();
for (int BlockID = 0; BlockID < blocks.Count; BlockID++)
{
if (blocks[BlockID].blockfail)
{
continue;
}
{
bool anyoutput = false;
bool nonemptyoutputpath = false;
for (int oi = 0; oi < blocks[BlockID].DFSWriters.Count; oi++)
{
string dfswriter = blocks[BlockID].DFSWriters[oi];
if (string.IsNullOrEmpty(dfswriter))
{
if (blocks[BlockID].outputdfsnodeses[oi].Count > 0)
{
Console.Error.WriteLine("Output data detected with no DFSWriter specified");
}
}
else
{
{
if (null != DfsFind(dc, dfswriter))
{
Console.Error.WriteLine("Error: output file was created during job: {0}", dfswriter);
continue;
}
string dfspath = dfswriter;
{
nonemptyoutputpath = true;
dfs.DfsFile df = new dfs.DfsFile();
if (blocks[BlockID].rem.OutputRecordLengths[oi] > 0)
{
df.XFileType = DfsFileTypes.BINARY_RECT + "@" + blocks[BlockID].rem.OutputRecordLengths[oi].ToString();
}
else if (blocks[BlockID].rem.OutputRecordLengths[oi] == -2)
{
df.XFileType = DfsFileTypes.BINARY_RECT + "@?";
}
df.Nodes = new List <dfs.DfsFile.FileNode>();
df.Size = -1; // Preset
if (dfspath.StartsWith("dfs://", StringComparison.OrdinalIgnoreCase))
{
dfspath = dfspath.Substring(6);
}
string dfspathreplicating = ".$" + dfspath + ".$replicating-" + Guid.NewGuid().ToString();
if (null != dc.FindAny(dfspathreplicating))
{
Console.Error.WriteLine("Error: file exists: file put into DFS from another location during job: " + dfspathreplicating);
SetFailure();
return;
}
dfsnames.Add(dfspath);
dfsnamesreplicating.Add(dfspathreplicating);
df.Name = dfspathreplicating;
bool anybad = false;
long totalsize = 0;
{
int i = BlockID;
for (int j = 0; j < blocks[i].outputdfsnodeses[oi].Count; j++)
{
dfs.DfsFile.FileNode fn = new dfs.DfsFile.FileNode();
fn.Host = blocks[i].slaves[(blocks[i].rem.OutputStartingPoint + j) % blocks[i].slaves.Count];
fn.Name = blocks[i].outputdfsnodeses[oi][j];
df.Nodes.Add(fn);
fn.Length = -1; // Preset
fn.Position = -1; // Preset
if (anybad)
{
continue;
}
fn.Length = blocks[i].outputsizeses[oi][j];
fn.Position = totalsize; // Position must be set before totalsize updated!
if (blocks[i].outputdfsnodeses[oi].Count != blocks[i].outputsizeses[oi].Count)
{
anybad = true;
continue;
}
totalsize += blocks[i].outputsizeses[oi][j];
}
}
if (!anybad)
{
df.Size = totalsize;
}
if (totalsize != 0)
{
anyoutput = true;
}
// Always add the file to DFS, even if blank!
dc.Files.Add(df);
}
}
}
}
if (!anyoutput && verbose && nonemptyoutputpath)
{
Console.Write(" (no DFS output) ");
ConsoleFlush();
}
}
}
UpdateDfsXml(dc);
}
ReplicationPhase(verbosereplication, blocks.Count, slaves, dfsnamesreplicating);
using (LockDfsMutex()) // Needed: change between load & save should be atomic.
{
dc = LoadDfsConfig(); // Reload in case of change or user modifications.
for (int nfile = 0; nfile < dfsnames.Count; nfile++)
{
string dfspath = dfsnames[nfile];
string dfspathreplicating = dfsnamesreplicating[nfile];
{
dfs.DfsFile dfu = dc.FindAny(dfspathreplicating);
if (null != dfu)
{
if (null != DfsFindAny(dc, dfspath))
{
Console.Error.WriteLine("Error: file exists: file put into DFS from another location during job");
SetFailure();
continue;
}
dfu.Name = dfspath;
}
}
}
UpdateDfsXml(dc);
}
if (verbose)
{
Console.WriteLine(); // Line after output chars.
}
}
catch (System.Threading.ThreadAbortException)
{
aborting = true;
}
finally
{
{
for (int si = 0; si < slaves.Length; si++)
{
System.Threading.Mutex m = new System.Threading.Mutex(false, "AEL_SC_" + slaves[si]);
try
{
m.WaitOne();
}
catch (System.Threading.AbandonedMutexException)
{
}
try
{
System.IO.File.Delete(NetworkPathForHost(slaves[si]) + @"\slaveconfig.j" + sjid + ".xml");
}
catch
{
}
finally
{
m.ReleaseMutex();
m.Close();
}
}
}
if (!aborting)
{
CheckUserLogs(slaves, logname);
}
}
if (verbose)
{
Console.WriteLine();
Console.WriteLine("[{0}] Done", System.DateTime.Now.ToString(), System.DateTime.Now.Millisecond);
for (int i = 0; i < cfgj.IOSettings.DFS_IOs.Length; i++)
{
Console.WriteLine("Output: {0}", cfgj.IOSettings.DFS_IOs[i].DFSWriter);
}
}
}
public static string MapNodeToNetworkPath(dfs.DfsFile.FileNode node)
{
return dfs.MapNodeToNetworkPath(node);
}
System.IO.Stream _OpenStream(dfs.DfsFile.FileNode node)
{
string[] nodehosts = node.Host.Split(';');
string[] fullnames = new string[nodehosts.Length];
int ReplicateCurrentIndex = ReplicateStartIndex;
for (int i = 0; i < fullnames.Length; i++)
{
fullnames[i] = Surrogate.NetworkPathForHost(
nodehosts[ReplicateCurrentIndex % nodehosts.Length])
+ @"\" + node.Name;
ReplicateCurrentIndex++;
}
return new MySpace.DataMining.AELight.DfsFileNodeStream(fullnames, true,
System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read, 0x400 * 4);
}
static void _MemCacheRelease(dfs.DfsFile df, bool force)
{
dfs dc = LoadDfsConfig();
string exception = "";
string[] slaves = dc.Slaves.SlaveList.Split(';');
{
//foreach (string slave in slaves)
MySpace.DataMining.Threading.ThreadTools<string>.Parallel(
new Action<string>(
delegate(string slave)
{
System.Net.Sockets.NetworkStream nstm = Surrogate.ConnectService(slave);
nstm.WriteByte((byte)'C');
nstm.WriteByte((byte)'r');
XContent.SendXContent(nstm, df.Name);
XContent.SendXContent(nstm, new byte[1] { (byte)(force ? 1 : 0) });
int ich = nstm.ReadByte();
if ('+' != ich)
{
string errmsg = null;
if ('-' == ich)
{
try
{
errmsg = XContent.ReceiveXString(nstm, null);
}
catch
{
}
}
lock (slaves)
{
string newexception;
if (null != errmsg)
{
newexception = ("Error received from DO service during MemCache rollback from " + slave + ": " + errmsg);
}
else
{
newexception = ("Did not receive a success signal from DO service during MemCache rollback from " + slave);
}
if (string.IsNullOrEmpty(exception)
|| -1 != exception.IndexOf("MemCacheWarning"))
{
exception = newexception;
}
}
}
}), slaves, slaves.Length);
}
if (!string.IsNullOrEmpty(exception))
{
throw new Exception(exception);
}
}
static void Kill(string[] args)
{
if (args.Length <= 1 || !System.IO.File.Exists(args[1]))
{
throw new Exception("Expected path to DFS.xml");
}
string dfsxmlpath = args[1];
string masterdir;
{
System.IO.FileInfo fi = new System.IO.FileInfo(dfsxmlpath);
masterdir = fi.DirectoryName; // Directory's full path.
}
Surrogate.SetNewMetaLocation(masterdir);
dfs dc = dfs.ReadDfsConfig_unlocked(dfsxmlpath);
string masterhost = System.Net.Dns.GetHostName();
string[] allmachines;
{
string[] sl = dc.Slaves.SlaveList.Split(';');
List <string> aml = new List <string>(sl.Length + 1);
aml.Add(masterhost);
foreach (string slave in sl)
{
if (0 != string.Compare(IPAddressUtil.GetName(slave), IPAddressUtil.GetName(masterhost), StringComparison.OrdinalIgnoreCase))
{
aml.Add(slave);
}
}
allmachines = aml.ToArray();
}
{
Console.WriteLine("Ensure cluster is perfectly healthy...");
EnsurePerfectQizmtHealtha();
Console.WriteLine("Run job in one thread, kill from another...");
//System.Threading.Thread.Sleep(1000 * 8);
string exectempdir = @"\\" + System.Net.Dns.GetHostName() + @"\C$\temp\qizmt\regression_test_Kill-" + Guid.NewGuid().ToString();
if (!System.IO.Directory.Exists(exectempdir))
{
System.IO.Directory.CreateDirectory(exectempdir);
}
string execfp = exectempdir + @"\exec{3EAE6884-28BB-4340-8F94-6E9421B68C92}";
System.IO.File.WriteAllText(execfp, (@"<SourceCode>
<Jobs>
<Job Name=`exec{3EAE6884-28BB-4340-8F94-6E9421B68C92}_Preprocessing 1`>
<IOSettings>
<JobType>local</JobType>
</IOSettings>
<Local>
<![CDATA[
public virtual void Local()
{
// Sleep forever so that kill will take it down.
System.Threading.Thread.Sleep(System.Threading.Timeout.Infinite);
}
]]>
</Local>
</Job>
<Job Name=`exec{3EAE6884-28BB-4340-8F94-6E9421B68C92}_Preprocessing 2`>
<IOSettings>
<JobType>local</JobType>
</IOSettings>
<Local>
<![CDATA[
public virtual void Local()
{
Qizmt_Log(`{DC46FA81-A69F-4d46-9A30-54869168916B}`);
}
]]>
</Local>
</Job>
</Jobs>
</SourceCode>
").Replace('`', '"'));
Exec.Shell("Qizmt importdir " + exectempdir);
try
{
try
{
System.IO.File.Delete(execfp);
System.IO.Directory.Delete(exectempdir);
}
catch
{
}
bool execdone = false;
bool execx = false;
System.Threading.Thread execthread = new System.Threading.Thread(
new System.Threading.ThreadStart(
delegate()
{
try
{
Console.WriteLine(" Running exec...");
string output = Exec.Shell("Qizmt exec exec{3EAE6884-28BB-4340-8F94-6E9421B68C92}");
Console.WriteLine("exec output: {0}", output.Trim());
if (-1 != output.IndexOf("{DC46FA81-A69F-4d46-9A30-54869168916B}"))
{
lock (typeof(Program))
{
execx = true;
}
}
}
catch (Exception e)
{
Console.Error.WriteLine("Warning: exec exception: {0}", e.ToString());
}
lock (typeof(Program))
{
execdone = true;
}
}));
execthread.Start();
// Wait a few seconds to give the exec a chance to get started.
System.Threading.Thread.Sleep(1000 * 5);
lock (typeof(Program))
{
if (execx)
{
throw new Exception("exec completed; problem with test");
}
if (execdone)
{
throw new Exception("exec finished too early, did not get a chance to call kill");
}
}
int execjid = 0;
string execsjid = "0";
string psexecline = "N/A";
{
foreach (string psln in Exec.Shell("qizmt ps").Split('\n'))
{
if (-1 != psln.IndexOf("exec{3EAE6884-28BB-4340-8F94-6E9421B68C92}"))
{
psexecline = psln.Trim();
{
int isp = psexecline.IndexOf(' ');
if (-1 != isp)
{
execsjid = psexecline.Substring(0, isp);
}
}
break;
}
}
}
if (!int.TryParse(execsjid, out execjid) || execjid < 1)
{
throw new Exception("JID for job not valid: " + execsjid);
}
execsjid = execjid.ToString(); // Normalize.
Console.WriteLine(" Running kill... ({0})", psexecline);
Exec.Shell("Qizmt kill " + execsjid);
lock (typeof(Program))
{
if (execx)
{
throw new Exception("exec completed; problem with test");
}
if (!execdone)
{
throw new Exception("kill completed but exec has not yet returned");
}
}
// Wait a couple seconds to give the services a chance to come back up.
System.Threading.Thread.Sleep(1000 * 2);
}
finally
{
try
{
Exec.Shell("Qizmt del exec{3EAE6884-28BB-4340-8F94-6E9421B68C92}");
}
catch
{
}
}
Console.WriteLine("[PASSED] - " + string.Join(" ", args));
}
}
public static void MemCacheFlush(string mcname)
{
if (mcname.StartsWith("dfs://", StringComparison.OrdinalIgnoreCase))
{
mcname = mcname.Substring(6);
}
dfs dc = LoadDfsConfig();
dfs.DfsFile df = dc.FindAny(mcname);
if (null == df || df.MemCache == null)
{
Console.Error.WriteLine("Error: '{0}' is not a MemCache", (null == df ? mcname : df.Name));
SetFailure();
return;
}
string tempdfsname = mcname + Guid.NewGuid() + dfs.TEMP_FILE_MARKER;
string[] slaves = dc.Slaves.SlaveList.Split(';');
long dfsfilesize = 0;
{
string tempfp = "bp-mcflush" + Guid.NewGuid() + ".tmp";
try
{
using (System.IO.StreamWriter sw = new System.IO.StreamWriter(tempfp))
{
//foreach (string slave in slaves)
MySpace.DataMining.Threading.ThreadTools <string> .Parallel(
new Action <string>(
delegate(string slave)
{
System.Net.Sockets.NetworkStream nstm = Surrogate.ConnectService(slave);
nstm.WriteByte((byte)'C');
nstm.WriteByte((byte)'f');
XContent.SendXContent(nstm, df.Name);
int ich = nstm.ReadByte();
if ('+' != ich)
{
string errmsg = null;
if ('-' == ich)
{
try
{
errmsg = XContent.ReceiveXString(nstm, null);
}
catch
{
}
}
if (null != errmsg)
{
throw new Exception("Error received from DO service during MemCache commit: " + errmsg);
}
throw new Exception("Did not receive a success signal from DO service during MemCache commit");
}
// flushinfos: chunk name, chunk size (without header)
string[] flushinfos = XContent.ReceiveXString(nstm, null).Split(
new char[] { '\r', '\n' },
StringSplitOptions.RemoveEmptyEntries);
foreach (string flushinfo in flushinfos)
{
lock (slaves)
{
sw.WriteLine("{0} {1}", slave, flushinfo);
dfsfilesize += int.Parse(flushinfo.Split(' ')[1]);
}
}
}), slaves, slaves.Length);
}
DfsBulkPut(new string[] { tempfp, tempdfsname, "rbin@" + df.RecordLength });
}
finally
{
try
{
System.IO.File.Delete(tempfp);
}
catch
{
}
}
}
Dictionary <string, bool> newchunknames = new Dictionary <string, bool>(StringComparer.OrdinalIgnoreCase);
using (LockDfsMutex())
{
dc = LoadDfsConfig();
dfs.DfsFile df2 = dc.FindAny(tempdfsname);
if (null == df2)
{
throw new Exception("DEBUG: Temp DFS file not found: " + tempdfsname);
}
for (int i = 0; i < dc.Files.Count; i++)
{
if (0 == string.Compare(dc.Files[i].Name, mcname, true))
{
dc.Files.RemoveAt(i);
break;
}
}
foreach (dfs.DfsFile.FileNode fn in df2.Nodes)
{
newchunknames[fn.Name] = true;
}
df2.MemCache = df.MemCache;
df2.Size = dfsfilesize;
df2.Name = df.Name;
UpdateDfsXml(dc);
}
{
// Just kill the old chunks, not the MemCache stuff.
List <string> delfnodes = new List <string>();
{
//Collect file node paths.
for (int dn = 0; dn < df.Nodes.Count; dn++)
{
if (newchunknames.ContainsKey(df.Nodes[dn].Name))
{
continue;
}
foreach (string chost in df.Nodes[dn].Host.Split(';'))
{
delfnodes.Add(NetworkPathForHost(chost) + @"\" + df.Nodes[dn].Name);
}
}
}
_KillDataFileChunksInternal_unlocked_mt(delfnodes);
}
}
public static List<string> SplitInputPaths(dfs dc, string pathlist)
{
return dc.SplitInputPaths(pathlist);
}
static void _MemCacheRelease(dfs.DfsFile df)
{
_MemCacheRelease(df, false);
}
