void Awake ()
{
if (instance == default(FileDump))
{
instance = this;
string idFilePath = Application.dataPath + "/" + idFileName;
if (!File.Exists(idFilePath))
{
byte[] idKey = new byte[4];
byte[] idValue = new byte[4];
System.Random idGen = new System.Random((int)(DateTime.Now.ToOADate() * 10000));
StreamWriter sw = new StreamWriter(idFilePath, false, System.Text.Encoding.ASCII);
idGen.NextBytes(idKey);
idValue = System.Security.Cryptography.SHA256.Create().ComputeHash(idKey);
foreach (byte b in idValue)
{
sw.Write(b);
}
sw.Close();
}
isSessionFileOpen = false;
StreamReader idReader = new StreamReader(idFilePath, System.Text.Encoding.ASCII);
PlayerUniqueID = idReader.ReadLine();
GetSessionCount();
CreateSessionFile();
}
else
{
Destroy(this);
}
}
public void testShortRepeat()
{
ObjectInspector inspector = ObjectInspectorFactory.getReflectionObjectInspector(typeof(int));
using (Stream file = File.OpenWrite(TestFilePath))
using (Writer w = OrcFile.createWriter(TestFilePath, file, OrcFile.writerOptions(conf)
.compress(CompressionKind.NONE)
.inspector(inspector)
.rowIndexStride(0)
.encodingStrategy(OrcFile.EncodingStrategy.COMPRESSION)
.version(OrcFile.Version.V_0_12)))
{
for (int i = 0; i < 5; ++i)
{
w.addRow(10);
}
}
using (CaptureStdoutToMemory capture = new CaptureStdoutToMemory())
{
FileDump.Main(TestFilePath);
// 1 byte header + 1 byte value
Assert.True(capture.Text.Contains("Stream: column 0 section DATA start: 3 length 2"));
}
}
public void testDeltaUnknownSign()
{
ObjectInspector inspector = ObjectInspectorFactory.getReflectionObjectInspector(typeof(int));
using (Stream file = File.OpenWrite(TestFilePath))
using (Writer w = OrcFile.createWriter(TestFilePath, file, OrcFile.writerOptions(conf)
.compress(CompressionKind.NONE)
.inspector(inspector)
.rowIndexStride(0)
.encodingStrategy(OrcFile.EncodingStrategy.COMPRESSION)
.version(OrcFile.Version.V_0_12)))
{
w.addRow(0);
for (int i = 0; i < 511; ++i)
{
w.addRow(i);
}
}
using (CaptureStdoutToMemory capture = new CaptureStdoutToMemory())
{
FileDump.Main(TestFilePath);
// monotonicity will be undetermined for this sequence 0,0,1,2,3,...510. Hence DIRECT encoding
// will be used. 2 bytes for header and 640 bytes for data (512 values with fixed bit of 10 bits
// each, 5120/8 = 640). Total bytes 642
Assert.True(capture.Text.Contains("Stream: column 0 section DATA start: 3 length 642"));
}
}
public void testPatchedBase()
{
ObjectInspector inspector = ObjectInspectorFactory.getReflectionObjectInspector(typeof(int));
using (Stream file = File.OpenWrite(TestFilePath))
using (Writer w = OrcFile.createWriter(TestFilePath, file, OrcFile.writerOptions(conf)
.compress(CompressionKind.NONE)
.inspector(inspector)
.rowIndexStride(0)
.encodingStrategy(OrcFile.EncodingStrategy.COMPRESSION)
.version(OrcFile.Version.V_0_12)))
{
Random rand = new Random(123);
w.addRow(10000000);
for (int i = 0; i < 511; ++i)
{
w.addRow(rand.Next(i + 1));
}
}
using (CaptureStdoutToMemory capture = new CaptureStdoutToMemory())
{
FileDump.Main(TestFilePath);
// use PATCHED_BASE encoding
Assert.True(capture.Text.Contains("Stream: column 0 section DATA start: 3 length 583"));
}
}
public void testDump()
{
// conf.set(HiveConf.ConfVars.HIVE_ORC_ENCODING_STRATEGY.varname, "COMPRESSION");
using (Stream file = File.OpenWrite(TestFilePath))
{
OrcFile.WriterOptions options = new OrcFile.WriterOptions(new Properties(), conf);
options.inspector(ObjectInspectorFactory.getReflectionObjectInspector(typeof(MyRecord)));
options.stripeSize(100000);
options.compress(CompressionKind.ZLIB);
options.bufferSize(10000);
options.rowIndexStride(1000);
using (Writer writer = OrcFile.createWriter(TestFilePath, file, options))
{
Random r1 = new Random(1);
for (int i = 0; i < 21000; ++i)
{
writer.addRow(new MyRecord(r1.Next(), r1.NextLong(),
TestHelpers.words[r1.Next(TestHelpers.words.Length)]));
}
}
}
string outputFilename = "orc-file-dump.out";
using (CaptureStdout capture = new CaptureStdout(Path.Combine(workDir, outputFilename)))
{
FileDump.Main(new string[] { TestFilePath.ToString(), "--rowindex=1,2,3" });
}
TestHelpers.CompareFilesByLine(outputFilename, Path.Combine(workDir, outputFilename));
}
public void testFixedDeltaOneDescending()
{
ObjectInspector inspector = ObjectInspectorFactory.getReflectionObjectInspector(typeof(int));
using (Stream file = File.OpenWrite(TestFilePath))
using (Writer w = OrcFile.createWriter(TestFilePath, file, OrcFile.writerOptions(conf)
.compress(CompressionKind.NONE)
.inspector(inspector)
.rowIndexStride(0)
.encodingStrategy(OrcFile.EncodingStrategy.COMPRESSION)
.version(OrcFile.Version.V_0_12)))
{
for (int i = 0; i < 5120; ++i)
{
w.addRow(512 - (i % 512));
}
}
using (CaptureStdoutToMemory capture = new CaptureStdoutToMemory())
{
FileDump.Main(TestFilePath);
// 10 runs of 512 elements. Each run has 2 bytes header, 2 byte base (base = 512, zigzag + varint)
// and 1 byte delta (delta = 1). In total, 5 bytes per run.
Assert.True(capture.Text.Contains("Stream: column 0 section DATA start: 3 length 50"));
}
}
void OnDestroy()
{
if (instance == this)
{
Debug.Log("Destroying file dumper...");
FileDump.CloseSessionFile();
}
}
public void takeDamage(int dmg)
{
if (currentBounty > 0)
{
currentBounty -= dmg;
string[] message = { "Lost " + dmg + " bounty", currentBounty.ToString(), "DiffLvl" + SpawnScriptHandler.currentLevel.ToString() };
FileDump.LogData(message);
}
}
public void NextFlowState()
{
if (flow == FlowState.AfterGame)
{
return;
}
if (flow == FlowState.PostPlayQuestionsDual)
{
FileDump.CloseSessionFile();
}
flow = (FlowState)((int)flow + 1);
SaveFlowState();
GameOverScreen.deathCount = 0;
}
public void CreateAndUpload()
{
FileDump.CloseSessionFile();
string[] jsonFiles = Directory.GetFiles(Application.dataPath, "*.json");
Debug.Log("Create and upload!");
StreamReader FileReader = new StreamReader(Application.dataPath + "/id.txt");
FileReader.Close();
foreach (string path in jsonFiles)
{
fileToUpload = path;
StartCoroutine("Upload");
}
}
public void testDictionaryThreshold()
{
// conf.set(HiveConf.ConfVars.HIVE_ORC_ENCODING_STRATEGY.varname, "COMPRESSION");
// conf.setFloat(HiveConf.ConfVars.HIVE_ORC_DICTIONARY_KEY_SIZE_THRESHOLD.varname, 0.49f);
using (Stream file = File.OpenWrite(TestFilePath))
{
OrcFile.WriterOptions options = new OrcFile.WriterOptions(new Properties(), conf);
options.inspector(ObjectInspectorFactory.getReflectionObjectInspector(typeof(MyRecord)));
options.stripeSize(100000);
options.compress(CompressionKind.ZLIB);
options.bufferSize(10000);
options.rowIndexStride(1000);
using (Writer writer = OrcFile.createWriter(TestFilePath, file, options))
{
Random r1 = new Random(1);
int nextInt = 0;
for (int i = 0; i < 21000; ++i)
{
// Write out the same string twice, this guarantees the fraction of rows with
// distinct strings is 0.5
if (i % 2 == 0)
{
nextInt = r1.Next(TestHelpers.words.Length);
// Append the value of i to the word, this guarantees when an index or word is repeated
// the actual string is unique.
TestHelpers.words[nextInt] += "-" + i;
}
writer.addRow(new MyRecord(r1.Next(), r1.NextLong(), TestHelpers.words[nextInt]));
}
}
}
string outputFilename = "orc-file-dump-dictionary-threshold.out";
using (CaptureStdout capture = new CaptureStdout(Path.Combine(workDir, outputFilename)))
{
FileDump.Main(new string[] { TestFilePath.ToString(), "--rowindex=1,2,3" });
}
TestHelpers.CompareFilesByLine(outputFilename, Path.Combine(workDir, outputFilename));
}
public void testJsonDump()
{
ObjectInspector inspector;
inspector = ObjectInspectorFactory.getReflectionObjectInspector(typeof(MyRecord));
// conf.set(HiveConf.ConfVars.HIVE_ORC_ENCODING_STRATEGY.varname, "COMPRESSION");
OrcFile.WriterOptions options = OrcFile.writerOptions(conf)
.inspector(inspector)
.stripeSize(100000)
.compress(CompressionKind.ZLIB)
.bufferSize(10000)
.rowIndexStride(1000)
.bloomFilterColumns("s");
using (Stream file = File.OpenWrite(TestFilePath))
using (Writer writer = OrcFile.createWriter(TestFilePath, file, options))
{
Random r1 = new Random(1);
for (int i = 0; i < 21000; ++i)
{
if (i % 100 == 0)
{
writer.addRow(new MyRecord(r1.Next(), r1.NextLong(), null));
}
else
{
writer.addRow(new MyRecord(r1.Next(), r1.NextLong(),
TestHelpers.words[r1.Next(TestHelpers.words.Length)]));
}
}
}
const string outputFilename = "orc-file-dump.json";
using (CaptureStdout capture = new CaptureStdout(Path.Combine(workDir, outputFilename)))
{
FileDump.Main(new string[] { TestFilePath.ToString(), "-j", "-p", "--rowindex=3" });
}
TestHelpers.CompareFilesByLine(outputFilename, Path.Combine(workDir, outputFilename));
}
public void testDataDump()
{
using (Stream file = File.OpenWrite(TestFilePath))
{
OrcFile.WriterOptions options = new OrcFile.WriterOptions(new Properties(), conf);
options.inspector(ObjectInspectorFactory.getReflectionObjectInspector(typeof(AllTypesRecord)));
options.stripeSize(100000);
options.compress(CompressionKind.NONE);
options.bufferSize(10000);
options.rowIndexStride(1000);
using (Writer writer = OrcFile.createWriter(TestFilePath, file, options))
{
Dictionary <string, string> m = new Dictionary <string, string>(2);
m.Add("k1", "v1");
writer.addRow(new AllTypesRecord(
true,
(sbyte)10,
(short)100,
1000,
10000L,
4.0f,
20.0,
HiveDecimal.Parse("4.2222"),
new Timestamp(1416967764000L),
new Date(1416967764000L),
"string",
m,
new List <int> {
100, 200
},
new AllTypesRecord.Struct(10, "foo")));
m.Clear();
m.Add("k3", "v3");
writer.addRow(new AllTypesRecord(
false,
(sbyte)20,
(short)200,
2000,
20000L,
8.0f,
40.0,
HiveDecimal.Parse("2.2222"),
new Timestamp(1416967364000L),
new Date(1411967764000L),
"abcd",
m,
new List <int> {
200, 300
},
new AllTypesRecord.Struct(20, "bar")));
}
}
string[] lines;
using (CaptureStdoutToMemory capture = new CaptureStdoutToMemory())
{
FileDump.Main(TestFilePath, "-d");
lines = capture.Text.Split(new[] { '\n' }, StringSplitOptions.RemoveEmptyEntries);
}
Assert.Equal(2, lines.Length);
// Don't be fooled by the big space in the middle, this line is quite long
Assert.Equal("{\"b\":true,\"bt\":10,\"s\":100,\"i\":1000,\"l\":10000,\"f\":4,\"d\":20,\"de\":\"4.2222\",\"t\":\"2014-11-25 18:09:24\",\"dt\":\"2014-11-25\",\"str\":\"string\",\"c\":\"hello \",\"vc\":\"hello\",\"m\":[{\"_key\":\"k1\",\"_value\":\"v1\"}],\"a\":[100,200],\"st\":{\"i\":10,\"s\":\"foo\"}}", lines[0]);
Assert.Equal("{\"b\":false,\"bt\":20,\"s\":200,\"i\":2000,\"l\":20000,\"f\":8,\"d\":40,\"de\":\"2.2222\",\"t\":\"2014-11-25 18:02:44\",\"dt\":\"2014-09-28\",\"str\":\"abcd\",\"c\":\"world \",\"vc\":\"world\",\"m\":[{\"_key\":\"k3\",\"_value\":\"v3\"}],\"a\":[200,300],\"st\":{\"i\":20,\"s\":\"bar\"}}", lines[1]);
}
public void testHasNull()
{
ObjectInspector inspector = ObjectInspectorFactory.getReflectionObjectInspector(typeof(SimpleStruct));
using (Stream file = File.OpenWrite(TestFilePath))
using (Writer writer = OrcFile.createWriter(TestFilePath, file, OrcFile.writerOptions(conf)
.inspector(inspector)
.rowIndexStride(1000)
.stripeSize(10000)
.bufferSize(10000)))
{
// STRIPE 1
// RG1
for (int i = 0; i < 1000; i++)
{
writer.addRow(new SimpleStruct(bytes(1, 2, 3), "RG1"));
}
// RG2
for (int i = 0; i < 1000; i++)
{
writer.addRow(new SimpleStruct(bytes(1, 2, 3), null));
}
// RG3
for (int i = 0; i < 1000; i++)
{
writer.addRow(new SimpleStruct(bytes(1, 2, 3), "RG3"));
}
// RG4
for (int i = 0; i < 1000; i++)
{
writer.addRow(new SimpleStruct(bytes(1, 2, 3), null));
}
// RG5
for (int i = 0; i < 1000; i++)
{
writer.addRow(new SimpleStruct(bytes(1, 2, 3), null));
}
// STRIPE 2
for (int i = 0; i < 5000; i++)
{
writer.addRow(new SimpleStruct(bytes(1, 2, 3), null));
}
// STRIPE 3
for (int i = 0; i < 5000; i++)
{
writer.addRow(new SimpleStruct(bytes(1, 2, 3), "STRIPE-3"));
}
// STRIPE 4
for (int i = 0; i < 5000; i++)
{
writer.addRow(new SimpleStruct(bytes(1, 2, 3), null));
}
}
Reader reader = OrcFile.createReader(TestFilePath, OrcFile.readerOptions(conf));
// check the file level stats
ColumnStatistics[] stats = reader.getStatistics();
Assert.Equal(20000, stats[0].getNumberOfValues());
Assert.Equal(20000, stats[1].getNumberOfValues());
Assert.Equal(7000, stats[2].getNumberOfValues());
Assert.Equal(false, stats[0].hasNull());
Assert.Equal(false, stats[1].hasNull());
Assert.Equal(true, stats[2].hasNull());
// check the stripe level stats
List <StripeStatistics> stripeStats = reader.getStripeStatistics();
// stripe 1 stats
StripeStatistics ss1 = stripeStats[0];
ColumnStatistics ss1_cs1 = ss1.getColumnStatistics()[0];
ColumnStatistics ss1_cs2 = ss1.getColumnStatistics()[1];
ColumnStatistics ss1_cs3 = ss1.getColumnStatistics()[2];
Assert.Equal(false, ss1_cs1.hasNull());
Assert.Equal(false, ss1_cs2.hasNull());
Assert.Equal(true, ss1_cs3.hasNull());
// stripe 2 stats
StripeStatistics ss2 = stripeStats[1];
ColumnStatistics ss2_cs1 = ss2.getColumnStatistics()[0];
ColumnStatistics ss2_cs2 = ss2.getColumnStatistics()[1];
ColumnStatistics ss2_cs3 = ss2.getColumnStatistics()[2];
Assert.Equal(false, ss2_cs1.hasNull());
Assert.Equal(false, ss2_cs2.hasNull());
Assert.Equal(true, ss2_cs3.hasNull());
// stripe 3 stats
StripeStatistics ss3 = stripeStats[2];
ColumnStatistics ss3_cs1 = ss3.getColumnStatistics()[0];
ColumnStatistics ss3_cs2 = ss3.getColumnStatistics()[1];
ColumnStatistics ss3_cs3 = ss3.getColumnStatistics()[2];
Assert.Equal(false, ss3_cs1.hasNull());
Assert.Equal(false, ss3_cs2.hasNull());
Assert.Equal(false, ss3_cs3.hasNull());
// stripe 4 stats
StripeStatistics ss4 = stripeStats[3];
ColumnStatistics ss4_cs1 = ss4.getColumnStatistics()[0];
ColumnStatistics ss4_cs2 = ss4.getColumnStatistics()[1];
ColumnStatistics ss4_cs3 = ss4.getColumnStatistics()[2];
Assert.Equal(false, ss4_cs1.hasNull());
Assert.Equal(false, ss4_cs2.hasNull());
Assert.Equal(true, ss4_cs3.hasNull());
#if false
// Test file dump
TextWriter origOut = System.Console.Out;
string outputFilename = "orc-file-has-null.out";
FileOutputStream myOut = new FileOutputStream(workDir + File.separator + outputFilename);
// replace stdout and run command
System.Console.SetOut(new StreamWriter(myOut));
FileDump.main(new String[] { testFilePath.toString(), "--rowindex=2" });
System.Console.Out.Flush();
System.SetOut(origOut);
TestFileDump.checkOutput(outputFilename, workDir + File.separator + outputFilename);
#endif
}
void OnDisable()
{
FileDump.CloseSession();
}
void Start()
{
string isStatic = this.GetComponentInParent <SpawnScriptHandler>().bountyBased.ToString();
FileDump.OpenSession(isStatic);
}
public void addBounty(int b)
{
currentBounty += b;
string[] message = { "Gained " + b + " bounty", currentBounty.ToString(), "DiffLvl" + SpawnScriptHandler.currentLevel.ToString() };
FileDump.LogData(message);
}
