public void testOrcSerDeStatsMap()
{
ObjectInspector inspector = ObjectInspectorFactory.getReflectionObjectInspector(typeof(MapStruct));
using (Stream file = File.OpenWrite(TestFilePath))
using (Writer writer = OrcFile.createWriter(TestFilePath, file, OrcFile.writerOptions(conf)
.inspector(inspector)
.stripeSize(10000)
.bufferSize(10000)))
{
for (int row = 0; row < 1000; row++)
{
Dictionary <string, double> test = new Dictionary <string, double>();
for (int i = 0; i < 10; i++)
{
test.Add("hi" + i, 2.0);
}
writer.addRow(new MapStruct(test));
}
writer.close();
// stats from writer
Assert.Equal(1000, writer.getNumberOfRows());
Assert.Equal(950000, writer.getRawDataSize());
}
Reader reader = OrcFile.createReader(TestFilePath,
OrcFile.readerOptions(conf));
// stats from reader
Assert.Equal(1000, reader.getNumberOfRows());
Assert.Equal(950000, reader.getRawDataSize());
Assert.Equal(950000, reader.getRawDataSizeOfColumns(Lists.newArrayList("map1")));
}
VectorizedOrcAcidRowReader(AcidInputFormat.RowReader <OrcStruct> inner,
Configuration conf,
FileSplit split)
{
this.innerReader = inner;
this.key = inner.createKey();
this.rowBatchCtx = new VectorizedRowBatchCtx();
this.value = inner.createValue();
this.objectInspector = inner.getObjectInspector();
try
{
rowBatchCtx.init(conf, split);
}
catch (ClassNotFoundException e)
{
throw new IOException("Failed to initialize context", e);
}
catch (SerDeException e)
{
throw new IOException("Failed to initialize context", e);
}
catch (InstantiationException e)
{
throw new IOException("Failed to initialize context", e);
}
catch (IllegalAccessException e)
{
throw new IOException("Failed to initialize context", e);
}
catch (HiveException e)
{
throw new IOException("Failed to initialize context", e);
}
}
public void ActivateInspector(ResourceActivationContext context, ObjectInspector inspector)
{
if (inspector.TargetReferences != null)
{
string targetId = null;
string targetAction = null;
foreach (var reference in inspector.TargetReferences)
{
var id = context.GetInstanceId(reference.ObjectId);
if (targetId == null && m_objectActions.GetCount(id) > 0)
{
// TODO: inspectors should define this action
targetAction = m_objectActions[id].First();
targetId = id;
}
m_inspectors.Add(id, context);
}
// No current object or current object is the same as targetId
if (targetId != null)
{
if (m_currObjId == null ||
m_currObjId == targetId)
{
// Actions stored in m_objectActions are never one-shot
ApplyObjectAction(targetId, targetAction, false);
}
}
}
}
public void PerformanceTest()
{
lock (Runner.Locker)
{
var tObject = new TestObject {
Name = "Tony", Value = "Redondo"
};
var objInsp = new ObjectInspector();
var objData = objInsp.With(tObject);
if (objData.TryGetFetcher("Name", out var nameFetcher))
{
Runner.RunF("Property Fetcher", () => tObject.Name, () => nameFetcher.Fetch(tObject));
}
if (objData.TryGetFetcher("Value", out var valueFetcher))
{
Runner.RunF("Field Fetcher", () => tObject.Value, () => valueFetcher.Fetch(tObject));
}
if (objData.TryGetFetcher("Sum", out var sumFetcher))
{
var p = new object[] { 2, 2 };
Runner.RunF("Method Fetcher", () => tObject.Sum(2, 2), () => sumFetcher.Invoke(tObject, p) !);
}
}
}
private void OnEnable()
{
inspectorObject = null;
try {
foreach (Type t in Assembly.GetExecutingAssembly().GetTypes())
{
if (typeof(ObjectInspector).IsAssignableFrom(t) && !t.IsAbstract)
{
var constructorInfo = t.GetConstructor(Type.EmptyTypes);
inspectorObject = (ObjectInspector)constructorInfo.Invoke(noArgs);
var isValidInfo = t.GetMethod("IsValid", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public);
var targetInfo = t.GetField("target", BindingFlags.Instance | BindingFlags.NonPublic);
if (targetInfo != null && isValidInfo != null)
{
targetInfo.SetValue(inspectorObject, target);
if ((bool)isValidInfo.Invoke(inspectorObject, noArgs))
{
t.GetMethod("OnEnable", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public).Invoke(inspectorObject, noArgs);
}
else
{
inspectorObject = null;
}
}
else
{
inspectorObject = null;
}
break;
}
}
} catch (Exception) {
inspectorObject = null;
}
}
public void testShortRepeat()
{
ObjectInspector inspector = ObjectInspectorFactory.getReflectionObjectInspector(typeof(int));
using (Stream file = File.OpenWrite(TestFilePath))
using (Writer w = OrcFile.createWriter(TestFilePath, file, OrcFile.writerOptions(conf)
.compress(CompressionKind.NONE)
.inspector(inspector)
.rowIndexStride(0)
.encodingStrategy(OrcFile.EncodingStrategy.COMPRESSION)
.version(OrcFile.Version.V_0_12)))
{
for (int i = 0; i < 5; ++i)
{
w.addRow(10);
}
}
using (CaptureStdoutToMemory capture = new CaptureStdoutToMemory())
{
FileDump.Main(TestFilePath);
// 1 byte header + 1 byte value
Assert.True(capture.Text.Contains("Stream: column 0 section DATA start: 3 length 2"));
}
}
public void testPatchedBase()
{
ObjectInspector inspector = ObjectInspectorFactory.getReflectionObjectInspector(typeof(int));
using (Stream file = File.OpenWrite(TestFilePath))
using (Writer w = OrcFile.createWriter(TestFilePath, file, OrcFile.writerOptions(conf)
.compress(CompressionKind.NONE)
.inspector(inspector)
.rowIndexStride(0)
.encodingStrategy(OrcFile.EncodingStrategy.COMPRESSION)
.version(OrcFile.Version.V_0_12)))
{
Random rand = new Random(123);
w.addRow(10000000);
for (int i = 0; i < 511; ++i)
{
w.addRow(rand.Next(i + 1));
}
}
using (CaptureStdoutToMemory capture = new CaptureStdoutToMemory())
{
FileDump.Main(TestFilePath);
// use PATCHED_BASE encoding
Assert.True(capture.Text.Contains("Stream: column 0 section DATA start: 3 length 583"));
}
}
public void testDeltaUnknownSign()
{
ObjectInspector inspector = ObjectInspectorFactory.getReflectionObjectInspector(typeof(int));
using (Stream file = File.OpenWrite(TestFilePath))
using (Writer w = OrcFile.createWriter(TestFilePath, file, OrcFile.writerOptions(conf)
.compress(CompressionKind.NONE)
.inspector(inspector)
.rowIndexStride(0)
.encodingStrategy(OrcFile.EncodingStrategy.COMPRESSION)
.version(OrcFile.Version.V_0_12)))
{
w.addRow(0);
for (int i = 0; i < 511; ++i)
{
w.addRow(i);
}
}
using (CaptureStdoutToMemory capture = new CaptureStdoutToMemory())
{
FileDump.Main(TestFilePath);
// monotonicity will be undetermined for this sequence 0,0,1,2,3,...510. Hence DIRECT encoding
// will be used. 2 bytes for header and 640 bytes for data (512 values with fixed bit of 10 bits
// each, 5120/8 = 640). Total bytes 642
Assert.True(capture.Text.Contains("Stream: column 0 section DATA start: 3 length 642"));
}
}
public void testFixedDeltaOneDescending()
{
ObjectInspector inspector = ObjectInspectorFactory.getReflectionObjectInspector(typeof(int));
using (Stream file = File.OpenWrite(TestFilePath))
using (Writer w = OrcFile.createWriter(TestFilePath, file, OrcFile.writerOptions(conf)
.compress(CompressionKind.NONE)
.inspector(inspector)
.rowIndexStride(0)
.encodingStrategy(OrcFile.EncodingStrategy.COMPRESSION)
.version(OrcFile.Version.V_0_12)))
{
for (int i = 0; i < 5120; ++i)
{
w.addRow(512 - (i % 512));
}
}
using (CaptureStdoutToMemory capture = new CaptureStdoutToMemory())
{
FileDump.Main(TestFilePath);
// 10 runs of 512 elements. Each run has 2 bytes header, 2 byte base (base = 512, zigzag + varint)
// and 1 byte delta (delta = 1). In total, 5 bytes per run.
Assert.True(capture.Text.Contains("Stream: column 0 section DATA start: 3 length 50"));
}
}
// Find the record identifier column (if there) and return a possibly new ObjectInspector that
// will strain out the record id for the underlying writer.
private ObjectInspector findRecId(ObjectInspector inspector, int rowIdColNum)
{
if (!(inspector is StructObjectInspector))
{
throw new InvalidOperationException("Serious problem, expected a StructObjectInspector, but got a " +
inspector.GetType().FullName);
}
if (rowIdColNum < 0)
{
return(inspector);
}
else
{
RecIdStrippingObjectInspector newInspector =
new RecIdStrippingObjectInspector(inspector, rowIdColNum);
recIdField = newInspector.getRecId();
List <StructField> fields =
((StructObjectInspector)recIdField.getFieldObjectInspector()).getAllStructFieldRefs();
// Go by position, not field name, as field names aren't guaranteed. The order of fields
// in RecordIdentifier is transactionId, bucketId, rowId
originalTxnField = fields[0];
origTxnInspector = (LongObjectInspector)originalTxnField.getFieldObjectInspector();
rowIdField = fields[2];
rowIdInspector = (LongObjectInspector)rowIdField.getFieldObjectInspector();
recIdInspector = (StructObjectInspector)recIdField.getFieldObjectInspector();
return(newInspector);
}
}
/// <summary>
/// Draws the fields related to the inspector drawer.
/// </summary>
/// <param name="target">The object that is being drawn.</param>
/// <param name="parent">The Unity Object that the object belongs to.</param>
protected override void DrawInspectorDrawerFields(object target, Object parent)
{
ObjectInspector.DrawFields(target, true);
m_PlayAudioClip = (target as PlayAudioClip);
AudioClipSetInspector.DrawAudioClipSet(m_PlayAudioClip.AudioClipSet, null, ref m_AudioClipsList, OnAudioClipDraw, OnAudioClipListAdd, OnAudioClipListRemove);
}
/// <summary>
/// Draws the fields related to the inspector drawer.
/// </summary>
/// <param name="target">The object that is being drawn.</param>
/// <param name="parent">The Unity Object that the object belongs to.</param>
protected override void DrawInspectorDrawerFields(object target, Object parent)
{
ObjectInspector.DrawFields(target, true);
var shakeTarget = (Shake.ShakeTarget)EditorGUILayout.EnumFlagsField(new GUIContent("Shake Target", InspectorUtility.GetFieldTooltip(target, "m_Target")),
InspectorUtility.GetFieldValue <Shake.ShakeTarget>(target, "m_Target"));
InspectorUtility.SetFieldValue(target, "m_Target", shakeTarget);
}
public void CanSetNestedStaticPrivateField()
{
var dbg = new ObjectInspector();
dbg.StackAdd_OBJECT(333);
object result = dbg.SetStatic(testObjSubTypeName + ".privateIndexStat");
Assert.Equal(333, result);
}
public void CanSetNestedStaticPrivateProp()
{
var dbg = new ObjectInspector();
dbg.StackAdd_OBJECT(333);
object result = dbg.SetStatic(testObjSubTypeName + ".INDEXSTAT");
Assert.Equal(333, result);
}
/// <summary>
/// LoadContent will be called once per game and is the place to load
/// all of your content.
/// </summary>
protected override void LoadContent()
{
// Create a new SpriteBatch, which can be used to draw textures.
spriteBatch = new SpriteBatch(GraphicsDevice);
ContentContainer.LoadContent(Content);
objectInspector = new ObjectInspector(this);
gameEffect = Content.Load <Effect>("Effect\\effect");
tex = Content.Load <Texture2D>("character");
font = Content.Load <SpriteFont>("File");
Map = new Map(Content, GraphicsDevice);
SetUpVertices();
player = new Player(Vector2.Zero);
Engine.GameUtility.Physic.Point circle = new Engine.GameUtility.Physic.Point(new Vector2(positionOnPlane.X, positionOnPlane.Z));
player.CollisionObject = circle;
player.CollisionObject.OnCollision += new CollideDetected(delegate(ICollider item)
{
if (item.GetType() == typeof(Polygon))
{
MessageBox.Show("Collide", "Circle<>Linse", new string[] { "walsiw" });
}
});
player.CollisionObject.OnCollision += new CollideDetected(EventMethod);
Director.InstanceDirector.Camera.SetDevice(this.GraphicsDevice);
model = Content.Load <Model>("robot");
BiStableKey tempKey = new BiStableKey(Keys.Space);
tempKey.action += new ClickTrigger(delegate
{
IMapElement el = Map.GetMapElementByName <IMapElement>(objectInspector.selectedID);
if (el != null)
{
Map.GetMapElementByName <IMapElement>(objectInspector.selectedID).Position = new Vector2(positionOnPlane.X, positionOnPlane.Z);
}
else
{
IFloor floor = Map.GetMapElementByName <IFloor>(objectInspector.selectedID);
floor.FloorPolygon
.AddPoint(new VertexPositionColor(positionOnPlane + new Vector3(0, 1, 0), Color.BlueViolet));
}
});
keys.Add(tempKey);
// MapWriter.Write(jsonSerialize);
}
/// <summary>
/// CallInspect Method
/// This Method has been called in InspectTest Method
/// </summary>
/// <param name="obj">obj</param>
private void CallInspect(object obj)
{
string expected;
string results;
expected = ObjectInspector.Inspect(obj);
results = ObjectInspector.Inspect(obj);
Assert.AreEqual(expected, results);
}
public void CanHandleSettingReadOnlyProp()
{
var dbg = new ObjectInspector();
dbg.StackAdd_OBJECT(333);
object result = dbg.SetStatic(testObjTypeName + ".ReadOnlyName");
Assert.Equal("<error>", result);
}
private void ClearControls()
{
ObjectInspector.RemoveAllElements();
ObjectInspector.FocusedPropertiesNode = ObjectInspector.PropertiesRootNode;
DataSourceSurface.Clear();
ScriptSurface.Clear();
ParametersSurface.Clear();
ChildViewSurface.Clear();
}
/**
* A required option that sets the object inspector for the rows. If
* setSchema is not called, it also defines the schema.
*/
public WriterOptions inspector(ObjectInspector value)
{
_inspector = value;
if (!explicitSchema)
{
schema = OrcUtils.convertTypeInfo(
TypeInfoUtils.getTypeInfoFromObjectInspector(value));
}
return(this);
}
public void InspectTest(object obj)
{
try
{
if (obj is DateTime)
{
for (int i = 0; i < 2; i++)
{
if (i == 0)
{
ObjectInspector.DateTimeFormat = "dd/mm/yyyy";
CallInspect(obj);
}
if (i == 1)
{
ObjectInspector.DateTimeFormat = string.Empty;
CallInspect(obj);
}
}
}
if (obj != null)
{
if (obj.GetType().StructLayoutAttribute != null)
{
for (int i = 0; i < 2; i++)
{
if (i == 0)
{
ObjectInspector.ExclusionFullyQualifiedNameParts = new string[] { "a" };
CallInspect(obj);
}
if (i == 1)
{
ObjectInspector.ExclusionFullyQualifiedNameParts = new string[] { };
CallInspect(obj);
}
}
}
}
string expected = ObjectInspector.Inspect(obj);
string results = ObjectInspector.Inspect(obj);
Assert.AreEqual(expected, results);
//Assert.AreNotEqual(obj, results);
}
catch (Exception ex)
{
// Print a stack trace when an exception occurs.
Console.WriteLine(ex.StackTrace);
throw;
}
}
public void CanSetPublicProp()
{
var obj = new TestObject();
var dbg = new ObjectInspector();
dbg.StackAdd_OBJECT(obj);
dbg.StackAdd_OBJECT("new name");
object result = dbg.Set(testObjTypeName + ".NAME");
Assert.Equal("new name", result);
}
public void CanSetNestedPublicField()
{
var obj = new TestObjectSub();
var dbg = new ObjectInspector();
dbg.StackAdd_OBJECT(obj);
dbg.StackAdd_OBJECT("new name");
object result = dbg.Set(testObjSubTypeName + ".Name");
Assert.Equal("new name", result);
}
public void CanSetPrivateField()
{
var obj = new TestObject();
var dbg = new ObjectInspector();
dbg.StackAdd_OBJECT(obj);
dbg.StackAdd_OBJECT(777);
object result = dbg.Set(testObjTypeName + ".privateIndex");
Assert.Equal(777, result);
}
public void DeactivateInspector(ResourceActivationContext context, ObjectInspector inspector)
{
if (inspector.TargetReferences != null)
{
foreach (var reference in inspector.TargetReferences)
{
var id = context.GetInstanceId(reference.ObjectId);
m_inspectors.Remove(id, context);
}
}
}
/// <summary>１レコードを返すSELECTクエリを実行する</summary>
/// <param name="testParameter">引数クラス</param>
/// <param name="testReturn">戻り値クラス</param>
public void Select(TestParameterValue testParameter, TestReturnValue testReturn)
{
// ↓DBアクセス-----------------------------------------------------
string filename = "";
if ((testParameter.ActionType.Split('%'))[2] == "static")
{
// 静的SQL
filename = "ShipperSelect.sql";
}
else if ((testParameter.ActionType.Split('%'))[2] == "dynamic")
{
// 動的SQL
filename = "ShipperSelect.xml";
}
// -- ファイルから読み込む場合。
this.SetSqlByFile2(filename);
// パラメタ ライズド クエリのパラメタに対して、動的に値を設定する。
this.SetParameter("P1", testParameter.Shipper.ShipperID);
// 戻り値 dt
DataTable dt = new DataTable();
// -- １レコードを返すSELECTクエリを実行する
this.ExecSelectFill_DT(dt);
// ↑DBアクセス-----------------------------------------------------
// 一部、DataToPocoのテストコード
ShipperViweModel svm = DataToPoco.DataTableToPOCO <ShipperViweModel>(dt);
Debug.WriteLine("svm:" + ObjectInspector.Inspect(svm));
TestShipperViweModel tsvm = DataToPoco.DataTableToPOCO <TestShipperViweModel>(dt,
// mapの書き方は、Key-Valueでdst-srcのproperty field名を書く
new Dictionary <string, string>()
{
{ "_ShipperID", "ShipperID" },
{ "_CompanyName", "CompanyName" },
{ "_Phone", "Phone" }
});
Debug.WriteLine("tsvm:" + ObjectInspector.Inspect(tsvm));
testReturn.Obj = svm;
testReturn.Obj2 = tsvm;
}
public void SetDataSource(object datasource)
{
if (_dscontrol != null)
{
if (ObjectInspector.HasObjectDataSourceProperty(_dscontrol))
{
ObjectInspector.SetDataSource(_dscontrol, datasource);
}
}
else
{
throw new Exception("You must initialize list control first!!!");
}
}
public void SetDisplayMember(string dispmember)
{
if (_dscontrol != null)
{
if (ObjectInspector.HasObjectDisplayMemberProperty(_dscontrol))
{
ObjectInspector.SetDisplayMember(_dscontrol, dispmember);
}
}
else
{
throw new Exception("You must initialize list control first!!!");
}
}
public void testHalfDistinctCheckDisabled()
{
ObjectInspector inspector = ObjectInspectorFactory.getReflectionObjectInspector(typeof(string));
int[] input = new int[20000];
// conf.setBoolean(ConfVars.HIVE_ORC_ROW_INDEX_STRIDE_DICTIONARY_CHECK.varname, false);
using (Stream file = File.OpenWrite(TestFilePath))
using (Writer writer = OrcFile.createWriter(TestFilePath, file, OrcFile.writerOptions(conf)
.inspector(inspector)
.compress(CompressionKind.NONE)
.bufferSize(10000)))
{
Random rand = new Random(123);
for (int i = 0; i < 20000; i++)
{
input[i] = rand.Next(10000);
}
for (int i = 0; i < 20000; i++)
{
writer.addRow(input[i].ToString());
}
}
Reader reader = OrcFile.createReader(TestFilePath, OrcFile.readerOptions(conf));
using (RecordReader rows = reader.rows())
{
int idx = 0;
while (rows.hasNext())
{
object row = rows.next();
Assert.Equal(input[idx++].ToString(), row);
}
// make sure the encoding type is correct
foreach (StripeInformation stripe in reader.getStripes())
{
// hacky but does the job, this casting will work as long this test resides
// within the same package as ORC reader
OrcProto.StripeFooter footer = ((RecordReaderImpl)rows).readStripeFooter(stripe);
for (int i = 0; i < footer.ColumnsCount; ++i)
{
OrcProto.ColumnEncoding encoding = footer.GetColumns(i);
Assert.Equal(OrcProto.ColumnEncoding.Types.Kind.DICTIONARY_V2, encoding.Kind);
}
}
}
}
public override bool Equals(object o)
{
if (o == null || o.GetType() != GetType())
{
return(false);
}
else if (o == this)
{
return(true);
}
else
{
ObjectInspector other = ((OrcListObjectInspector)o).child;
return(other.Equals(child));
}
}
public void testTimestampWriter(string writerTimeZone, string readerTimeZone)
{
ObjectInspector inspector = ObjectInspectorFactory.getReflectionObjectInspector(typeof(Timestamp));
List <string> ts = new List <string>();
using (Stream file = File.OpenWrite(TestFilePath))
using (Writer writer = OrcFile.createWriter(TestFilePath, file, OrcFile.writerOptions(conf)
.inspector(inspector)
.stripeSize(100000)
.bufferSize(10000)))
using (TestHelpers.SetTimeZoneInfo(writerTimeZone))
{
ts.Add("2003-01-01 01:00:00.000000222");
ts.Add("1996-08-02 09:00:00.723100809");
ts.Add("1999-01-01 02:00:00.999999999");
ts.Add("1995-01-02 03:00:00.688888888");
ts.Add("2002-01-01 04:00:00.1");
ts.Add("2010-03-02 05:00:00.000009001");
ts.Add("2005-01-01 06:00:00.000002229");
ts.Add("2006-01-01 07:00:00.900203003");
ts.Add("2003-01-01 08:00:00.800000007");
ts.Add("1998-11-02 10:00:00.857340643");
ts.Add("2008-10-02 11:00:00.0");
ts.Add("2037-01-01 00:00:00.000999");
ts.Add("2014-03-28 00:00:00.0");
foreach (string t in ts)
{
writer.addRow(Timestamp.Parse(t));
}
}
using (TestHelpers.SetTimeZoneInfo(readerTimeZone))
{
Reader reader = OrcFile.createReader(TestFilePath, OrcFile.readerOptions(conf));
using (RecordReader rows = reader.rows(null))
{
int idx = 0;
while (rows.hasNext())
{
object row = rows.next();
Timestamp got = ((Timestamp)row);
Assert.Equal(ts[idx++], got.ToString());
}
}
}
}
public BinaryTreeWriter(
int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory writer,
bool nullable)
: base(columnId, inspector, schema, writer, nullable)
{
this.stream = writer.createStream(id,
OrcProto.Stream.Types.Kind.DATA);
this.isDirectV2 = isNewWriteFormat(writer);
this.length = createIntegerWriter(writer.createStream(id,
OrcProto.Stream.Types.Kind.LENGTH), false, isDirectV2, writer);
recordPosition(rowIndexPosition);
}
private static TreeWriter createTreeWriter(ObjectInspector inspector,
TypeDescription schema,
StreamFactory streamFactory,
bool nullable)
{
switch (schema.getCategory())
{
case Category.BOOLEAN:
return new BooleanTreeWriter(streamFactory.getNextColumnId(),
inspector, schema, streamFactory, nullable);
case Category.BYTE:
return new ByteTreeWriter(streamFactory.getNextColumnId(),
inspector, schema, streamFactory, nullable);
case Category.SHORT:
case Category.INT:
case Category.LONG:
return new IntegerTreeWriter(streamFactory.getNextColumnId(),
inspector, schema, streamFactory, nullable);
case Category.FLOAT:
return new FloatTreeWriter(streamFactory.getNextColumnId(),
inspector, schema, streamFactory, nullable);
case Category.DOUBLE:
return new DoubleTreeWriter(streamFactory.getNextColumnId(),
inspector, schema, streamFactory, nullable);
case Category.STRING:
return new StringTreeWriter(streamFactory.getNextColumnId(),
inspector, schema, streamFactory, nullable);
case Category.CHAR:
return new CharTreeWriter(streamFactory.getNextColumnId(),
inspector, schema, streamFactory, nullable);
case Category.VARCHAR:
return new VarcharTreeWriter(streamFactory.getNextColumnId(),
inspector, schema, streamFactory, nullable);
case Category.BINARY:
return new BinaryTreeWriter(streamFactory.getNextColumnId(),
inspector, schema, streamFactory, nullable);
case Category.TIMESTAMP:
return new TimestampTreeWriter(streamFactory.getNextColumnId(),
inspector, schema, streamFactory, nullable);
case Category.DATE:
return new DateTreeWriter(streamFactory.getNextColumnId(),
inspector, schema, streamFactory, nullable);
case Category.DECIMAL:
return new DecimalTreeWriter(streamFactory.getNextColumnId(),
inspector, schema, streamFactory, nullable);
case Category.STRUCT:
return new StructTreeWriter(streamFactory.getNextColumnId(),
inspector, schema, streamFactory, nullable);
case Category.MAP:
return new MapTreeWriter(streamFactory.getNextColumnId(), inspector,
schema, streamFactory, nullable);
case Category.LIST:
return new ListTreeWriter(streamFactory.getNextColumnId(), inspector,
schema, streamFactory, nullable);
case Category.UNION:
return new UnionTreeWriter(streamFactory.getNextColumnId(), inspector,
schema, streamFactory, nullable);
default:
throw new ArgumentException("Bad category: " +
schema.getCategory());
}
}
public BooleanTreeWriter(int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory writer,
bool nullable)
: base(columnId, inspector, schema, writer, nullable)
{
PositionedOutputStream @out = writer.createStream(id,
OrcProto.Stream.Types.Kind.DATA);
this.writer = new BitFieldWriter(@out, 1);
recordPosition(rowIndexPosition);
}
/**
* A required option that sets the object inspector for the rows. If
* setSchema is not called, it also defines the schema.
*/
public WriterOptions inspector(ObjectInspector value)
{
_inspector = value;
if (!explicitSchema)
{
schema = OrcUtils.convertTypeInfo(
TypeInfoUtils.getTypeInfoFromObjectInspector(value));
}
return this;
}
public OrcMapObjectInspector(MapTypeInfo info)
{
key = createObjectInspector(info.getMapKeyTypeInfo());
value = createObjectInspector(info.getMapValueTypeInfo());
}
public OrcListObjectInspector(ListTypeInfo info)
{
child = createObjectInspector(info.getListElementTypeInfo());
}
public StringTreeWriter(
int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory writer,
bool nullable)
: base(columnId, inspector, schema, writer, nullable)
{
}
public StringBaseTreeWriter(int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory writer,
bool nullable)
: base(columnId, inspector, schema, writer, nullable)
{
this.isDirectV2 = isNewWriteFormat(writer);
stringOutput = writer.createStream(id,
OrcProto.Stream.Types.Kind.DICTIONARY_DATA);
lengthOutput = createIntegerWriter(writer.createStream(id,
OrcProto.Stream.Types.Kind.LENGTH), false, isDirectV2, writer);
rowOutput = createIntegerWriter(writer.createStream(id,
OrcProto.Stream.Types.Kind.DATA), false, isDirectV2, writer);
recordPosition(rowIndexPosition);
rowIndexValueCount.Add(0L);
buildIndex = writer.buildIndex();
directStreamOutput = writer.createStream(id, OrcProto.Stream.Types.Kind.DATA);
directLengthOutput = createIntegerWriter(writer.createStream(id,
OrcProto.Stream.Types.Kind.LENGTH), false, isDirectV2, writer);
OrcFile.WriterOptions options = writer.getOptions();
dictionaryKeySizeThreshold = options.getDictionaryKeySizeThreshold();
strideDictionaryCheck = options.getStrideDictionaryCheck();
doneDictionaryCheck = false;
}
public MapTreeWriter(int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory writer,
bool nullable)
: base(columnId, inspector, schema, writer, nullable)
{
this.isDirectV2 = isNewWriteFormat(writer);
ObjectInspector keyInspector = null;
ObjectInspector valueInspector = null;
if (inspector != null)
{
MapObjectInspector insp = (MapObjectInspector)inspector;
keyInspector = insp.getMapKeyObjectInspector();
valueInspector = insp.getMapValueObjectInspector();
}
childrenWriters = new TreeWriter[2];
IList<TypeDescription> children = schema.getChildren();
childrenWriters[0] = createTreeWriter(keyInspector, children[0], writer, true);
childrenWriters[1] = createTreeWriter(valueInspector, children[1], writer, true);
lengths = createIntegerWriter(writer.createStream(columnId,
OrcProto.Stream.Types.Kind.LENGTH), false, isDirectV2, writer);
recordPosition(rowIndexPosition);
}
public ListTreeWriter(int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory writer,
bool nullable)
: base(columnId, inspector, schema, writer, nullable)
{
this.isDirectV2 = isNewWriteFormat(writer);
ObjectInspector childOI = inspector == null ? null :
((ListObjectInspector)inspector).getListElementObjectInspector();
childrenWriters = new TreeWriter[1];
childrenWriters[0] =
createTreeWriter(childOI, schema.getChildren()[0], writer, true);
lengths = createIntegerWriter(writer.createStream(columnId,
OrcProto.Stream.Types.Kind.LENGTH), false, isDirectV2, writer);
recordPosition(rowIndexPosition);
}
public IntegerTreeWriter(int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory writer,
bool nullable)
: base(columnId, inspector, schema, writer, nullable)
{
OutStream @out = writer.createStream(id,
OrcProto.Stream.Types.Kind.DATA);
this.isDirectV2 = isNewWriteFormat(writer);
this.writer = createIntegerWriter(@out, true, isDirectV2, writer);
if (inspector is IntObjectInspector)
{
intInspector = (IntObjectInspector)inspector;
shortInspector = null;
longInspector = null;
}
else
{
intInspector = null;
if (inspector is LongObjectInspector)
{
longInspector = (LongObjectInspector)inspector;
shortInspector = null;
}
else
{
shortInspector = (ShortObjectInspector)inspector;
longInspector = null;
}
}
recordPosition(rowIndexPosition);
}
public FloatTreeWriter(int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory writer,
bool nullable)
: base(columnId, inspector, schema, writer, nullable)
{
this.stream = writer.createStream(id,
OrcProto.Stream.Types.Kind.DATA);
this.utils = new SerializationUtils();
recordPosition(rowIndexPosition);
}
public RecIdStrippingObjectInspector(ObjectInspector oi, int rowIdColNum)
{
if (!(oi is StructObjectInspector))
{
throw new InvalidOperationException("Serious problem, expected a StructObjectInspector, " +
"but got a " + oi.GetType().Name);
}
wrapped = (StructObjectInspector)oi;
IList<StructField> wrappedFields = wrapped.getAllStructFieldRefs();
fields = new List<StructField>(wrapped.getAllStructFieldRefs().Count);
for (int i = 0; i < wrappedFields.Count; i++)
{
if (i == rowIdColNum)
{
recId = wrappedFields[i];
}
else
{
fields.Add(wrappedFields[i]);
}
}
}
public void Initialize()
{
_objectInspector = new ObjectInspector();
}
public OrcListObjectInspector(int columnId, IList<OrcProto.Type> types)
{
OrcProto.Type type = types[columnId];
child = createObjectInspector((int)type.SubtypesList[0], types);
}
public StructTreeWriter(int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory writer,
bool nullable)
: base(columnId, inspector, schema, writer, nullable)
{
IList<TypeDescription> children = schema.getChildren();
if (inspector != null)
{
StructObjectInspector structObjectInspector =
(StructObjectInspector)inspector;
fields = structObjectInspector.getAllStructFieldRefs();
}
childrenWriters = new TreeWriter[children.Count];
for (int i = 0; i < childrenWriters.Length; ++i)
{
ObjectInspector childOI;
if (fields != null && i < fields.Count)
{
childOI = fields[i].getFieldObjectInspector();
}
else
{
childOI = null;
}
childrenWriters[i] = createTreeWriter(
childOI, children[i], writer,
true);
}
recordPosition(rowIndexPosition);
}
BigRowField(int id, String fieldName, ObjectInspector inspector) {
this.id = id;
this.fieldName = fieldName;
this.inspector = inspector;
}
public ByteTreeWriter(int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory writer,
bool nullable)
: base(columnId, inspector, schema, writer, nullable)
{
this.writer = new RunLengthByteWriter(writer.createStream(id,
OrcProto.Stream.Types.Kind.DATA));
recordPosition(rowIndexPosition);
}
public Field(string name, ObjectInspector inspector, int offset)
{
this.name = name;
this.inspector = inspector;
this.offset = offset;
}
public CharTreeWriter(
int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory writer,
bool nullable)
: base(columnId, inspector, schema, writer, nullable)
{
itemLength = schema.getMaxLength();
padding = new byte[itemLength];
}
public TimestampTreeWriter(int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory writer,
bool nullable)
: base(columnId, inspector, schema, writer, nullable)
{
this.isDirectV2 = isNewWriteFormat(writer);
this.seconds = createIntegerWriter(writer.createStream(id,
OrcProto.Stream.Types.Kind.DATA), true, isDirectV2, writer);
this.nanos = createIntegerWriter(writer.createStream(id,
OrcProto.Stream.Types.Kind.SECONDARY), false, isDirectV2, writer);
recordPosition(rowIndexPosition);
TimeZoneInfo timeZone;
this.base_timestamp = TimeZones.GetBaseTimestamp(writer.Timezone, out timeZone);
}
public WriterImpl(
Stream stream,
string path,
OrcFile.WriterOptions options,
ObjectInspector inspector,
TypeDescription schema,
long stripeSize,
CompressionKind compress,
int bufferSize,
int rowIndexStride,
MemoryManager memoryManager,
bool addBlockPadding,
OrcFile.Version version,
OrcFile.WriterCallback callback,
OrcFile.EncodingStrategy encodingStrategy,
OrcFile.CompressionStrategy compressionStrategy,
double paddingTolerance,
long blockSizeValue,
string bloomFilterColumnNames,
double bloomFilterFpp)
{
this.baseStream = stream;
this.streamFactory = new StreamFactory(this);
this.path = path;
this.options = options;
this.callback = callback;
this.schema = schema;
this.adjustedStripeSize = stripeSize;
this.defaultStripeSize = stripeSize;
this.version = version;
this.encodingStrategy = encodingStrategy;
this.compressionStrategy = compressionStrategy;
this.addBlockPadding = addBlockPadding;
this.blockSize = blockSizeValue;
this.paddingTolerance = paddingTolerance;
this.compress = compress;
this.rowIndexStride = rowIndexStride;
this.memoryManager = memoryManager;
buildIndex = rowIndexStride > 0;
codec = createCodec(compress);
int numColumns = schema.getMaximumId() + 1;
this.bufferSize = getEstimatedBufferSize(defaultStripeSize, numColumns, bufferSize);
if (version == OrcFile.Version.V_0_11)
{
/* do not write bloom filters for ORC v11 */
this.bloomFilterColumns = new bool[schema.getMaximumId() + 1];
}
else
{
this.bloomFilterColumns =
OrcUtils.includeColumns(bloomFilterColumnNames, schema);
}
this.bloomFilterFpp = bloomFilterFpp;
treeWriter = createTreeWriter(inspector, schema, streamFactory, false);
if (buildIndex && rowIndexStride < MIN_ROW_INDEX_STRIDE)
{
throw new ArgumentException("Row stride must be at least " +
MIN_ROW_INDEX_STRIDE);
}
// ensure that we are able to handle callbacks before we register ourselves
memoryManager.addWriter(path, stripeSize, this);
}
/**
* Create a tree writer.
* @param columnId the column id of the column to write
* @param inspector the object inspector to use
* @param schema the row schema
* @param streamFactory limited access to the Writer's data.
* @param nullable can the value be null?
* @
*/
protected TreeWriter(
int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory streamFactory,
bool nullable)
{
this.streamFactory = streamFactory;
this.isCompressed = streamFactory.isCompressed();
this.id = columnId;
this.inspector = inspector;
if (nullable)
{
isPresentOutStream = streamFactory.createStream(id,
OrcProto.Stream.Types.Kind.PRESENT);
isPresent = new BitFieldWriter(isPresentOutStream, 1);
}
else
{
isPresent = null;
}
this.foundNulls = false;
createBloomFilter = streamFactory.getBloomFilterColumns()[columnId];
indexStatistics = ColumnStatisticsImpl.create(schema);
stripeColStatistics = ColumnStatisticsImpl.create(schema);
fileStatistics = ColumnStatisticsImpl.create(schema);
childrenWriters = new TreeWriter[0];
rowIndex = OrcProto.RowIndex.CreateBuilder();
rowIndexEntry = OrcProto.RowIndexEntry.CreateBuilder();
rowIndexPosition = new RowIndexPositionRecorder(rowIndexEntry);
stripeStatsBuilders = new List<OrcProto.StripeStatistics.Builder>();
if (streamFactory.buildIndex())
{
rowIndexStream = streamFactory.createStream(id, OrcProto.Stream.Types.Kind.ROW_INDEX);
}
else
{
rowIndexStream = null;
}
if (createBloomFilter)
{
bloomFilterEntry = OrcProto.BloomFilter.CreateBuilder();
bloomFilterIndex = OrcProto.BloomFilterIndex.CreateBuilder();
bloomFilterStream = streamFactory.createStream(id, OrcProto.Stream.Types.Kind.BLOOM_FILTER);
bloomFilter = new BloomFilter(streamFactory.getRowIndexStride(), streamFactory.getBloomFilterFPP());
}
else
{
bloomFilterEntry = null;
bloomFilterIndex = null;
bloomFilterStream = null;
bloomFilter = null;
}
}
public UnionTreeWriter(int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory writer,
bool nullable)
: base(columnId, inspector, schema, writer, nullable)
{
IList<ObjectInspector> choices = null;
if (inspector != null)
{
UnionObjectInspector insp = (UnionObjectInspector)inspector;
choices = insp.getObjectInspectors();
}
IList<TypeDescription> children = schema.getChildren();
childrenWriters = new TreeWriter[children.Count];
for (int i = 0; i < childrenWriters.Length; ++i)
{
childrenWriters[i] = createTreeWriter(
choices != null ? choices[i] : null, children[i], writer, true);
}
tags =
new RunLengthByteWriter(writer.createStream(columnId,
OrcProto.Stream.Types.Kind.DATA));
recordPosition(rowIndexPosition);
}
public FieldInfoField(FieldInfo field, ObjectInspector inspector)
{
this.field = field;
this.inspector = inspector;
}
public VarcharTreeWriter(
int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory writer,
bool nullable)
: base(columnId, inspector, schema, writer, nullable)
{
maxLength = schema.getMaxLength();
}
public DateTreeWriter(int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory writer,
bool nullable)
: base(columnId, inspector, schema, writer, nullable)
{
OutStream @out = writer.createStream(id,
OrcProto.Stream.Types.Kind.DATA);
this.isDirectV2 = isNewWriteFormat(writer);
this.writer = createIntegerWriter(@out, true, isDirectV2, writer);
recordPosition(rowIndexPosition);
}
public OrcMapObjectInspector(int columnId, IList<OrcProto.Type> types)
{
OrcProto.Type type = types[columnId];
key = createObjectInspector((int)type.SubtypesList[0], types);
value = createObjectInspector((int)type.SubtypesList[1], types);
}
public DecimalTreeWriter(int columnId,
ObjectInspector inspector,
TypeDescription schema,
StreamFactory writer,
bool nullable)
: base(columnId, inspector, schema, writer, nullable)
{
this.isDirectV2 = isNewWriteFormat(writer);
valueStream = writer.createStream(id, OrcProto.Stream.Types.Kind.DATA);
this.scaleStream = createIntegerWriter(writer.createStream(id,
OrcProto.Stream.Types.Kind.SECONDARY), true, isDirectV2, writer);
recordPosition(rowIndexPosition);
}
// Find the record identifier column (if there) and return a possibly new ObjectInspector that
// will strain out the record id for the underlying writer.
private ObjectInspector findRecId(ObjectInspector inspector, int rowIdColNum)
{
if (!(inspector is StructObjectInspector))
{
throw new InvalidOperationException("Serious problem, expected a StructObjectInspector, but got a " +
inspector.GetType().FullName);
}
if (rowIdColNum < 0)
{
return inspector;
}
else
{
RecIdStrippingObjectInspector newInspector =
new RecIdStrippingObjectInspector(inspector, rowIdColNum);
recIdField = newInspector.getRecId();
List<StructField> fields =
((StructObjectInspector)recIdField.getFieldObjectInspector()).getAllStructFieldRefs();
// Go by position, not field name, as field names aren't guaranteed. The order of fields
// in RecordIdentifier is transactionId, bucketId, rowId
originalTxnField = fields[0];
origTxnInspector = (LongObjectInspector)originalTxnField.getFieldObjectInspector();
rowIdField = fields[2];
rowIdInspector = (LongObjectInspector)rowIdField.getFieldObjectInspector();
recIdInspector = (StructObjectInspector)recIdField.getFieldObjectInspector();
return newInspector;
}
}
