static void Main(string[] args)
{
string AzureaccountName = ConfigurationManager.AppSettings.Get("AccountName");
string AzureaccountKey = ConfigurationManager.AppSettings.Get("AccountSharedKey");
LocationInfo li = new LocationInfo(AzureaccountName, AzureaccountKey, SynchronizerType.Azure);
/*
* string dataFile = "D:\\b";
* int KB = 1024;
* int[] chunk_sizes = { 4*1024*KB , 8*1024*KB };
*
* for (int i = 1; i <= 1; i++)
* {
* for (int threads = 1; threads <= 1; threads++)
* {
* foreach (int csize in chunk_sizes)
* {
* Console.Write(">");
* File.Copy(dataFile, dataFile + threads + "," + csize);
*
* AzureHelper helper = new AzureHelper(AzureaccountName, AzureaccountKey, "foo123123", CompressionType.None, EncryptionType.None, null, null, new Logger(), csize, threads);
* long start = DateTime.Now.Ticks;
* helper.UploadFileAsChunks(dataFile + threads + "," + csize);
* long end = DateTime.Now.Ticks;
* Console.WriteLine(threads + "," + csize + "," + (((double)(end - start) / (double)10000000)) );
* }
*
* }
* }
*/
li = null;
FqStreamID fq_sid = new FqStreamID("1299-2716", "A", "TestBS");
CallerInfo ci = new CallerInfo(null, "A", "A", 1);
StreamFactory sf = StreamFactory.Instance;
sf.deleteStream(fq_sid, ci);
IStream dfs_byte_val = sf.openValueDataStream <StrKey, ByteValue>(fq_sid, ci, li,
StreamFactory.StreamSecurityType.Plain,
CompressionType.None,
StreamFactory.StreamOp.Write);
StrKey k1 = new StrKey("k1");
dfs_byte_val.Append(k1, new ByteValue(StreamFactory.GetBytes("k1-cmu")));
dfs_byte_val.Append(k1, new ByteValue(StreamFactory.GetBytes("k1-msr")));
dfs_byte_val.Seal(false);
dfs_byte_val.Append(k1, new ByteValue(StreamFactory.GetBytes("k1-uw")));
dfs_byte_val.Close();
Console.ReadKey();
dfs_byte_val = sf.openValueDataStream <StrKey, ByteValue>(fq_sid, ci, li,
StreamFactory.StreamSecurityType.Plain,
CompressionType.None,
StreamFactory.StreamOp.Write);
Console.WriteLine("Get in read : " + dfs_byte_val.Get(k1));
IEnumerable <IDataItem> data = dfs_byte_val.GetAll(k1, 0, StreamFactory.NowUtc());
foreach (IDataItem dataItem in data)
{
Console.WriteLine(dataItem.GetVal().ToString());
}
dfs_byte_val.Close();
Console.ReadKey();
/*
* ValueSerializerBase<StrKey> vsb = new ValueSerializerBase<StrKey>();
* Byte[] buffer1 = vsb.SerializeToByteStream().ToArray();
* Byte[] buffer2 = SerializerHelper<StrKey>.SerializeToProtoStream(k1).ToArray();
*
* FileStream fout = new FileStream("tmp.txt", FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite);
* BinaryWriter fs_bw = new BinaryWriter(fout);
* fs_bw.Write(buffer1);
* fs_bw.Write("-----W00t!-----");
* fs_bw.Write(buffer2);
* fs_bw.Write("-----W00t!-----");
* fs_bw.Close();
* fout.Close();
*/
}
static void Main(string[] args)
{
string dir = "";
List <string> summary = new List <string>();
foreach (StreamType type in types)
{
foreach (String address in streamListType)
{
foreach (int size in data_size)
{
foreach (StreamFactory.StreamDataType ptype in ptypes)
{
foreach (CompressionType ctype in ctypes)
{
foreach (SynchronizerType synctype in synctypes)
{
foreach (StreamOperation op in ops)
{
foreach (int chunksize in ChunkSize)
{
foreach (int ThreadPoolSize in num_threads)
{
foreach (int num_operations in n_operations)
{
for (int i = 0; i < n_iterations; ++i)
{
byte[] data = new byte[size];
Random rnd = new Random(DateTime.Now.Millisecond);
rnd.NextBytes(data);
Experiment e = new Experiment();
DateTime Date = new DateTime(DateTime.UtcNow.Ticks);
string HomeName = String.Format("ExpHome-{0}", Date.ToString("yyyy-MM-dd"));
string Caller = String.Format("{0}", Date.ToString("HH-mm-ss"));
string AppName = Caller;
string StreamName = String.Format("{0}", rnd.Next());
Byte[] value = data;
// int num_operations = n_operations;
string RandString = Experiment.RandomString(4);
long stime = 0;
long etime = 0;
// Run the experiment
if (op == StreamOperation.RandomKeyGet ||
op == StreamOperation.RandomKeyGetMultipleSegments ||
op == StreamOperation.RandomKeyGetAll)
{
// Need to create the stream before reading it
stime = StreamFactory.NowUtc();
Console.WriteLine("Stime: " + stime);
StreamOperation tmp = StreamOperation.RandomKeySameValueAppend;
if (op == StreamOperation.RandomKeyGetAll)
{
tmp = StreamOperation.SameKeySameValueAppend;
}
e.Run(Caller, HomeName, AppName, StreamName,
RandString,
stime, etime,
type, tmp,
ptype,
ctype, chunksize, ThreadPoolSize,
value, num_operations,
synctype, doCosts: true, address: address);
etime = StreamFactory.NowUtc();
Console.WriteLine("Etime: " + etime);
CallerInfo ci = new CallerInfo(null, AppName, AppName, 1);
string exp_directory = Path.GetFullPath((null != ci.workingDir) ? ci.workingDir : Directory.GetCurrentDirectory());
exp_directory = exp_directory + "/" + HomeName + "/" + AppName + "/" + StreamName;
File.Delete(exp_directory + "/log");
try
{
File.Delete(exp_directory + "/results");
File.Delete(exp_directory + "/exp");
}
catch
{
}
if (type == StreamType.Remote || type == StreamType.RemoteEnc)
{
Directory.Delete(exp_directory + "/0", true);
File.Delete(exp_directory + "/index_md.dat");
}
}
TimeSpan span = DateTime.Now.Subtract(new DateTime(1970, 1, 1, 0, 0, 0));
Console.WriteLine("Time: " + span.TotalSeconds);
if (op == StreamOperation.RandomKeyGetMultipleSegments || op == StreamOperation.RandomKeyGet)
{
e.Run(Caller, HomeName, AppName, StreamName, RandString, stime, etime, type, op, ptype, ctype, chunksize, ThreadPoolSize, value, num_multi_segment_operations, synctype, max_key: num_operations, doCosts: true, address: address);
}
else
{
e.Run(Caller, HomeName, AppName, StreamName, RandString, stime, etime, type, op, ptype, ctype, chunksize, ThreadPoolSize, value, num_operations, synctype, doCosts: true, address: address);
}
dir = e.exp_directory + "/../../../Results/";
Console.WriteLine("Completed: " + e.ToString());
span = DateTime.Now.Subtract(new DateTime(1970, 1, 1, 0, 0, 0));
Console.WriteLine("Time: " + span.TotalSeconds);
Console.WriteLine("====================================");
// Dump raw data from the experiment
e.Dump(Caller, HomeName, AppName, StreamName,
type, op, ptype,
value, num_operations, synctype);
// Get parsed data of the experiment
// if (i != 0)
//{
// Ignore the first iteration as warmup
summary.Add(e.exp_id);
List <string> ret = e.Analyze(type, value, num_operations);
summary.AddRange(ret);
//}
System.Threading.Thread.Sleep(2000);
e.Destroy();
}
}
}
}
}
}
}
}
}
}
}
dir = dir + StreamFactory.PrettyNowUtc();
if (!Directory.Exists(dir))
{
try
{
Directory.CreateDirectory(dir);
}
catch (Exception e)
{
Console.WriteLine(e.ToString());
}
}
dir = dir + "/";
File.WriteAllLines(dir + "Summary", summary);
// Dump .dat files
SummaryToDat(dir);
// Thats it!
Console.WriteLine("Done!");
Console.ReadKey();
}
protected void UpdateHelper(IKey key, IValue value, bool IsAppend)
{
if (!key.GetType().Equals(typeof(KeyType)))
{
throw new InvalidDataException("Invalid IKey Type");
}
if (!value.GetType().Equals(typeof(ValType)))
{
throw new InvalidDataException("Invalid IValue Type");
}
/*
* string keyHash = StreamFactory.GetString(
* sha1.ComputeHash(StreamFactory.GetBytes(key)));
*/
List <TS_Offset> offsets;
// check if the entry is present (and record offset)
if (index.ContainsKey(key))
{
// TODO: this could just have been a simple collision
// -- make sure that this is the same key
// -- if different, use a secondary hash function
offsets = index[key];
}
else
{
offsets = new List <TS_Offset>();
index[key] = offsets;
}
// get file offset; add <key_hash, offset> to index
fs_bw.BaseStream.Seek(0, SeekOrigin.End);
long offset = fs_bw.BaseStream.Position;
// write <op (1B), ts, key, val_len, val>
byte op;
if (IsAppend)
{
op = (byte)WriteOp.AppendOp;
}
else
{
op = (byte)WriteOp.UpdateOp;
}
long ts = StreamFactory.NowUtc();
DataBlock <KeyType, ValType> db = new DataBlock <KeyType, ValType>();
db.op = op;
db.timestamp = ts;
db.key = key;
db.value = value;
fs_bw.Write(db.SerializeToJsonStream());
fs_bw.Flush();
latest_tso = new TS_Offset(ts, offset);
// apply to index
if (IsAppend)
{
offsets.Add(latest_tso);
}
else
{
if (offsets.Count == 0)
{
offsets.Add(latest_tso);
}
else
{
offsets[offsets.Count - 1] = latest_tso;
}
}
/* ts_index.Add(new TS_Offset(ts, offset)); */
}
public static Logger doDiskRaw(StreamOperation op,
int num_operations, int val_size,
StreamFactory.StreamDataType ptype,
string exp_dir)
{
Logger logger = new Logger();
if (op == StreamOperation.RandomKeyRandomValueAppend ||
op == StreamOperation.SameKeyRandomValueAppend ||
op == StreamOperation.RandomKeySameValueAppend ||
op == StreamOperation.SameKeySameValueAppend)
{
// Populate the keys and the values
Random random = new Random(DateTime.Now.Millisecond);
StrKey[] keys = new StrKey[num_operations];
for (int i = 0; i < num_operations; ++i)
{
keys[i] = new StrKey("" + i);
}
Byte[] singleval = new Byte[val_size];
random.NextBytes(singleval);
Byte[][] tmp = new Byte[num_operations][];
if (!(op == StreamOperation.RandomKeySameValueAppend ||
op == StreamOperation.SameKeySameValueAppend))
{
for (int i = 0; i < num_operations; ++i)
{
tmp[i] = new Byte[val_size];
random.NextBytes(tmp[i]);
}
}
StrKey key = new StrKey("ExpKey");
if (ptype == StreamFactory.StreamDataType.Values)
{
// ValueDataStream type
logger.Log("Start Stream Open");
FileStream fout = new FileStream(exp_dir + "/DiskRaw", FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite);
fout.Seek(0, SeekOrigin.End);
BinaryWriter fs_bw = new BinaryWriter(fout);
logger.Log("End Stream Open");
// TimeSpan span = DateTime.Now.Subtract(new DateTime(1970, 1, 1, 0, 0, 0));
// Console.WriteLine("Time" + span.TotalSeconds);
logger.Log("Start Stream Append");
for (int i = 0; i < num_operations; ++i)
{
fs_bw.BaseStream.Seek(0, SeekOrigin.End);
/*
* if (op == StreamOperation.SameKeySameValueAppend || op == StreamOperation.SameKeyRandomValueAppend)
* {
* fs_bw.Write(key.key);
* }
* else
* {
*/
fs_bw.Write(keys[i].key);
/*
* }
*/
fs_bw.Write(StreamFactory.NowUtc());
/*
* if (!(op == StreamOperation.RandomKeySameValueAppend ||
* op == StreamOperation.SameKeySameValueAppend))
* fs_bw.Write(tmp[i]);
* else
*/
fs_bw.Write(singleval);
fs_bw.Flush();
}
fout.Flush(true);
fs_bw.Close();
// span = DateTime.Now.Subtract(new DateTime(1970, 1, 1, 0, 0, 0));
// Console.WriteLine("Time" + span.TotalSeconds);
logger.Log("End Stream Append");
}
else
{
string[] filenames = new string[num_operations];
for (int i = 0; i < num_operations; ++i)
{
filenames[i] = exp_dir + "/DiskRaw" + i;
}
// FileDataStream type
logger.Log("Start Stream Append");
for (int i = 0; i < num_operations; ++i)
{
FileStream fout = new FileStream(filenames[i], FileMode.OpenOrCreate, FileAccess.Write, FileShare.ReadWrite);
// fout.Seek(0, SeekOrigin.End);
BinaryWriter fs_bw = new BinaryWriter(fout);
// fs_bw.BaseStream.Seek(0, SeekOrigin.End);
/*
* if (op == StreamOperation.SameKeySameValueAppend || op == StreamOperation.SameKeyRandomValueAppend)
* {
* fs_bw.Write(key.key);
* }
* else
* {
*/
fs_bw.Write(keys[i].key);
// }
fs_bw.Write(StreamFactory.NowUtc());
/*
* if (!(op == StreamOperation.RandomKeySameValueAppend ||
* op == StreamOperation.SameKeySameValueAppend))
* fs_bw.Write(tmp[i]);
* else
*/
fs_bw.Write(singleval);
fout.Flush(true);
fs_bw.Close();
}
logger.Log("End Stream Append");
}
}
else
{
// Populate the keys and the values
Random random = new Random();
Byte[] val = new Byte[val_size];
if (ptype == StreamFactory.StreamDataType.Values)
{
long[] offsets = new long[num_operations];
for (int i = 0; i < num_operations; ++i)
{
offsets[i] = random.Next(0, num_operations - 1) * val_size;
}
// ValueDataStream type
logger.Log("Start Stream Open");
FileStream fin = new FileStream(exp_dir + "/DiskRaw", FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
BinaryReader fs_br = new BinaryReader(fin);
logger.Log("End Stream Open");
logger.Log("Start Stream Get");
logger.Log("Start Stream GetAll");
for (int i = 0; i < num_operations; ++i)
{
fs_br.BaseStream.Seek(offsets[i], SeekOrigin.Begin);
fs_br.Read(val, 0, val_size);
}
fs_br.Close();
logger.Log("End Stream Get");
logger.Log("End Stream GetAll");
}
else
{
string[] filenames = new string[num_operations];
for (int i = 0; i < num_operations; ++i)
{
filenames[i] = exp_dir + "/DiskRaw" + random.Next(0, num_operations);
}
// FileDataStream type
logger.Log("Start Stream Get");
logger.Log("Start Stream GetAll");
for (int i = 0; i < num_operations; ++i)
{
FileStream fin = new FileStream(filenames[i], FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
BinaryReader fs_br = new BinaryReader(fin);
fs_br.Read(val, 0, val_size);
fs_br.Close();
}
logger.Log("End Stream Get");
logger.Log("End Stream GetAll");
}
}
CostsHelper costhelper = new CostsHelper();
logger.Log(DateTime.UtcNow.Ticks + ": DataRelated Storage: " + costhelper.getStorageUsage(exp_dir, dataRelated: true) / 1000.0f);
logger.Log(DateTime.UtcNow.Ticks + ": Constant Storage: " + costhelper.getStorageUsage(exp_dir, dataRelated: false) / 1000.0f);
return(logger);
}
