/// <summary>
/// Chart의 Xml 정보를 HttpResponse 객체에 쓴다.
/// </summary>
/// <param name="dataXml">Chart의 Xml 정보</param>
protected virtual void WriteDataXml(string dataXml)
{
if (IsDebugEnabled)
{
log.Debug(@"Chart 정보를 응답스트림에 쓰기 시작합니다. 응답할 Chart Xml=[{0}]", dataXml.EllipsisChar(1024));
}
Response.Clear();
Response.ContentType = @"text/xml";
Response.Expires = -1;
byte[] output = ArrayTool.Combine(StringTool.MultiBytesPrefixBytes, Response.ContentEncoding.GetBytes(dataXml));
// 압축을 지원하고, 압축할 만큼 크기가 크다면...
var needCompress = CanCompression && dataXml.Length > CompressionThreshold;
if (needCompress)
{
// 압축했을 때에, 압축 방법을 응답헤더에 꼭 붙여줘야 Client에서 알아먹겠죠?
Response.AppendHeader(@"Content-Encoding", CompressionKind.ToString().ToLower());
output = Compressor.Compress(output);
}
Response.AppendHeader(@"Content-Length", output.Length.ToString());
Response.OutputStream.Write(output, 0, output.Length);
Response.Flush();
if (IsDebugEnabled)
{
log.Debug(@"Chart 정보를 응답스트림에 쓰기를 완료했습니다!!!");
}
}
/// <summary>
/// Provides a Stream that when read from, reads consecutive blocks of compressed data from an ORC Stream.
/// All data in the <paramref name="inputStream"/> will be consumed.
/// </summary>
public static IOStream GetDecompressingStream(IOStream inputStream, CompressionKind compressionKind)
{
if (compressionKind == CompressionKind.None)
{
return(inputStream);
}
else
{
return(new ConcatenatingStream(() =>
{
int blockLength;
bool isCompressed;
bool headerAvailable = ReadBlockHeader(inputStream, out blockLength, out isCompressed);
if (!headerAvailable)
{
return null;
}
var streamSegment = new StreamSegment(inputStream, blockLength, true);
if (!isCompressed)
{
return streamSegment;
}
else
{
return CompressionFactory.CreateDecompressorStream(compressionKind, streamSegment);
}
}, false));
}
}
/// <inheritdoc />
public ICompressor BuildCompressor(
CompressionKind compressionKind)
{
if (compressionKind == CompressionKind.Invalid)
{
throw new ArgumentOutOfRangeException(Invariant($"'{nameof(compressionKind)}' == '{CompressionKind.Invalid}'"), (Exception)null);
}
ICompressor result;
switch (compressionKind)
{
case CompressionKind.None:
result = new NullCompressor();
break;
case CompressionKind.DotNetZip:
result = new DotNetZipCompressor();
break;
default:
throw new NotSupportedException(Invariant($"{nameof(CompressionKind)} value {compressionKind} is not currently supported."));
}
return(result);
}
public OrcCompressedBuffer(int compressionBlockSize, CompressionKind compressionKind,
CompressionStrategy compressionStrategy)
{
_compressionBlockSize = compressionBlockSize;
_compressionKind = compressionKind;
_compressionStrategy = compressionStrategy;
}
internal StripeStreamReaderCollection(Stream inputStream, StripeFooter stripeFooter, long stripeOffset,
CompressionKind compressionKind)
{
var offset = stripeOffset;
foreach (var stream in stripeFooter.Streams)
{
if (_underlyingCollection.Exists(s => s.ColumnId == stream.Column && s.StreamKind == stream.Kind))
{
throw new InvalidOperationException(
$"More than one stream matching {nameof(stream.Column)} ({stream.Column}) and {nameof(stream.Kind)} ({stream.Kind}) found in {nameof(StripeFooter)}");
}
var column = stripeFooter.Columns[(int)stream.Column];
_underlyingCollection.Add(new StripeStreamReader(
inputStream,
stream.Column,
stream.Kind,
column.Kind,
offset,
stream.Length,
compressionKind
));
offset += (long)stream.Length;
}
}
/// <summary>
/// Builds the execute stored procedure operation.
/// </summary>
/// <param name="streamName">Name of the stream.</param>
/// <param name="configType">The serialization configuration type identifiers.</param>
/// <param name="serializationKind">The <see cref="SerializationKind"/>.</param>
/// <param name="serializationFormat">The <see cref="SerializationFormat"/>.</param>
/// <param name="compressionKind">The <see cref="CompressionKind"/>.</param>
/// <param name="unregisteredTypeEncounteredStrategy">The <see cref="UnregisteredTypeEncounteredStrategy"/>.</param>
/// <returns>Operation to execute stored procedure.</returns>
public static ExecuteStoredProcedureOp BuildExecuteStoredProcedureOp(
string streamName,
IdentifiedType configType,
SerializationKind serializationKind,
SerializationFormat serializationFormat,
CompressionKind compressionKind,
UnregisteredTypeEncounteredStrategy unregisteredTypeEncounteredStrategy)
{
var sprocName = Invariant($"[{streamName}].[{nameof(GetIdAddIfNecessarySerializerRepresentation)}]");
var parameters = new List <ParameterDefinitionBase>()
{
new InputParameterDefinition <int>(nameof(InputParamName.ConfigTypeWithoutVersionId), Tables.TypeWithoutVersion.Id.SqlDataType, configType.IdWithoutVersion),
new InputParameterDefinition <int>(nameof(InputParamName.ConfigTypeWithVersionId), Tables.TypeWithVersion.Id.SqlDataType, configType.IdWithVersion),
new InputParameterDefinition <string>(nameof(InputParamName.SerializationKind), Tables.SerializerRepresentation.SerializationKind.SqlDataType, serializationKind.ToString()),
new InputParameterDefinition <string>(nameof(InputParamName.SerializationFormat), Tables.SerializerRepresentation.SerializationFormat.SqlDataType, serializationFormat.ToString()),
new InputParameterDefinition <string>(nameof(InputParamName.CompressionKind), Tables.SerializerRepresentation.CompressionKind.SqlDataType, compressionKind.ToString()),
new InputParameterDefinition <string>(nameof(InputParamName.UnregisteredTypeEncounteredStrategy), Tables.SerializerRepresentation.SerializationKind.SqlDataType, unregisteredTypeEncounteredStrategy.ToString()),
new OutputParameterDefinition <int>(nameof(OutputParamName.Id), Tables.SerializerRepresentation.Id.SqlDataType),
};
var result = new ExecuteStoredProcedureOp(sprocName, parameters);
return(result);
}
public void CopyTo(Stream output, CompressionKind compression, Action <TagItem> callback)
{
using (var fOutput = new FlashWriter(output, true))
{
Assemble(fOutput, compression, callback);
}
}
protected ShockwaveFlash(FlashReader input)
: this(false)
{
Compression = (CompressionKind)input.ReadString(3)[0];
Version = input.ReadByte();
FileLength = input.ReadUInt32();
switch (Compression)
{
case CompressionKind.LZMA:
{
// Sup...
byte[] decompressed = LZMA.Decompress(input.BaseStream, ((int)FileLength - 8));
_input = new FlashReader(decompressed);
break;
}
case CompressionKind.ZLIB:
_input = ZLIB.WrapDecompressor(input.BaseStream);
break;
case CompressionKind.None:
_input = input;
break;
}
Frame = new FrameRecord(_input);
}
public OrcCompressedBufferFactory(int compressionBlockSize, CompressionKind compressionKind,
CompressionStrategy compressionStrategy)
{
CompressionBlockSize = compressionBlockSize;
CompressionKind = compressionKind;
CompressionStrategy = compressionStrategy;
}
/// <summary>
/// Compress the entirety of the MemoryStream to the destination Stream, or if compressing increases the size, just
/// copy the original data over
/// </summary>
/// <param name="uncompressedSource">A MemoryStream that will have its full contents compressed</param>
/// <param name="compressedDestination">Output Stream to write results to</param>
/// <param name="compressionKind">Compression Type</param>
/// <param name="compressionStrategy">The balance of speed vs size for the compressor</param>
public static void CompressCopyTo(MemoryStream uncompressedSource, IOStream compressedDestination,
CompressionKind compressionKind, CompressionStrategy compressionStrategy)
{
uncompressedSource.Seek(0, SeekOrigin.Begin);
if (compressionKind == CompressionKind.None)
{
uncompressedSource.CopyTo(compressedDestination);
}
else
{
var temporaryStream =
new MemoryStream(); //How can we avoid this?? We need to know the length of the compressed stream to write the header before we can write the stream itself...
using (var compressingStream =
CompressionFactory.CreateCompressorStream(compressionKind, compressionStrategy, temporaryStream))
{
uncompressedSource.CopyTo(compressingStream);
}
if (temporaryStream.Length >= uncompressedSource.Length)
{
//Write the original data rather than the compressed data
uncompressedSource.Seek(0, SeekOrigin.Begin);
WriteBlockHeader(compressedDestination, (int)uncompressedSource.Length, false);
uncompressedSource.CopyTo(compressedDestination);
}
else
{
temporaryStream.Seek(0, SeekOrigin.Begin);
WriteBlockHeader(compressedDestination, (int)temporaryStream.Length, true);
temporaryStream.CopyTo(compressedDestination);
}
}
}
public byte[] ToArray(CompressionKind compression)
{
using (var output = new MemoryStream((int)FileLength))
{
CopyTo(output, compression, null);
return(output.ToArray());
}
}
public ImageOptimizerOptions(CompressionKind kind, string sourceFile, string outFile, int quality = 80, bool progressive = true)
{
Quality = quality;
OutputFile = outFile;
CompressionKind = kind;
SourceFile = sourceFile;
Progressive = progressive;
FileType = Path.GetExtension(sourceFile).ToLowerInvariant();
}
public byte[] ToArray(CompressionKind compression)
{
using (var outputMem = new MemoryStream((int)FileLength))
using (var output = new FlashWriter(outputMem))
{
Assemble(output, compression);
return(outputMem.ToArray());
}
}
protected ShockwaveFlash(bool isCreatingTemplate)
{
Tags = new List <TagItem>();
if (isCreatingTemplate)
{
Frame = new FrameRecord();
Frame.Area = new RectangeRecord();
Compression = CompressionKind.ZLIB;
}
}
/// <summary>
/// Create a stream that when read from, decompresses data from the provided <paramref name="inputStream" />.
/// </summary>
/// <param name="compressionType">Type of compression to use</param>
/// <param name="inputStream">Stream to read compressed data from</param>
/// <returns>Readable decompressing stream</returns>
public static IOStream CreateDecompressorStream(CompressionKind compressionType, IOStream inputStream)
{
switch (compressionType)
{
case CompressionKind.Zlib: return(new ZLibStream(inputStream));
default:
throw new NotImplementedException($"Unimplemented {nameof(CompressionType)} {compressionType}");
}
}
public SerializerRepresentation DeepCloneWithCompressionKind(CompressionKind compressionKind)
{
var result = new SerializerRepresentation(
this.SerializationKind.DeepClone(),
this.SerializationConfigType?.DeepClone(),
compressionKind,
this.Metadata?.DeepClone());
return(result);
}
/** Ctor used when creating file info during init and when getting a new one. */
public FileMetaInfo(CompressionKind compressionKind, int bufferSize, int metadataSize,
ByteBuffer footerBuffer, IList <uint> versionList, OrcFile.WriterVersion writerVersion,
ByteBuffer fullFooterBuffer)
{
this.compressionKind = compressionKind;
this.bufferSize = bufferSize;
this.metadataSize = metadataSize;
this.footerBuffer = footerBuffer;
this.versionList = versionList;
this.writerVersion = writerVersion;
this.footerMetaAndPsBuffer = fullFooterBuffer;
}
/** Ctor used when creating file info during init and when getting a new one. */
public FileMetaInfo(CompressionKind compressionKind, int bufferSize, int metadataSize,
ByteBuffer footerBuffer, IList<uint> versionList, OrcFile.WriterVersion writerVersion,
ByteBuffer fullFooterBuffer)
{
this.compressionKind = compressionKind;
this.bufferSize = bufferSize;
this.metadataSize = metadataSize;
this.footerBuffer = footerBuffer;
this.versionList = versionList;
this.writerVersion = writerVersion;
this.footerMetaAndPsBuffer = fullFooterBuffer;
}
internal StripeStreamReader(Stream inputStream, uint columnId, StreamKind streamKind,
ColumnEncodingKind encodingKind, long inputStreamOffset, ulong compressedLength,
CompressionKind compressionKind)
{
_inputStream = inputStream;
ColumnId = columnId;
StreamKind = streamKind;
ColumnEncodingKind = encodingKind;
_inputStreamOffset = inputStreamOffset;
_compressedLength = compressedLength;
_compressionKind = compressionKind;
}
/// <summary>
/// Gets the serializer representation from identifier (first by cache and then by database).
/// </summary>
/// <param name="sqlServerLocator">The resource locator.</param>
/// <param name="serializerRepresentationId">The serializer representation identifier.</param>
/// <returns>IdentifiedSerializerRepresentation.</returns>
public IdentifiedSerializerRepresentation GetSerializerRepresentationFromId(
SqlServerLocator sqlServerLocator,
int serializerRepresentationId)
{
var found = this.serializerRepresentationIdToIdentifiedSerializerMap.TryGetValue(serializerRepresentationId, out var result);
if (found)
{
return(result);
}
else
{
var storedProcOp = StreamSchema.Sprocs.GetSerializerRepresentationFromId.BuildExecuteStoredProcedureOp(
this.Name,
serializerRepresentationId);
var sqlProtocol = this.BuildSqlOperationsProtocol(sqlServerLocator);
var sprocResult = sqlProtocol.Execute(storedProcOp);
SerializationKind serializationKind = sprocResult
.OutputParameters[nameof(StreamSchema
.Sprocs.GetSerializerRepresentationFromId.OutputParamName
.SerializationKind)]
.GetValueOfType <SerializationKind>();
int configTypeWithVersionId = sprocResult
.OutputParameters[nameof(StreamSchema
.Sprocs.GetSerializerRepresentationFromId.OutputParamName
.ConfigTypeWithVersionId)]
.GetValueOfType <int>();
CompressionKind compressionKind = sprocResult
.OutputParameters[nameof(StreamSchema
.Sprocs.GetSerializerRepresentationFromId.OutputParamName
.CompressionKind)]
.GetValueOfType <CompressionKind>();
SerializationFormat serializationFormat = sprocResult
.OutputParameters[nameof(StreamSchema
.Sprocs.GetSerializerRepresentationFromId.OutputParamName
.SerializationFormat)]
.GetValueOfType <SerializationFormat>();
var configType = this.GetTypeById(sqlServerLocator, configTypeWithVersionId, true);
var rep = new SerializerRepresentation(serializationKind, configType.WithVersion, compressionKind);
var item = new IdentifiedSerializerRepresentation(serializerRepresentationId, rep, serializationFormat);
this.serializerRepresentationIdToIdentifiedSerializerMap.TryAdd(item.Id, item);
this.serializerRepresentationToIdentifiedSerializerMap.TryAdd(item.SerializerRepresentation, item);
var newFound = this.serializerRepresentationIdToIdentifiedSerializerMap.TryGetValue(serializerRepresentationId, out var newResult);
newFound.MustForOp("failedToFindSerializationRepresentationAfterAdding").BeTrue();
return(newResult);
}
}
internal StripeReader(Stream inputStream, ulong indexOffset, ulong indexLength, ulong dataOffset,
ulong dataLength, ulong footerOffset, ulong footerLength, ulong numRows, CompressionKind compressionKind)
{
_inputStream = inputStream;
_indexOffset = indexOffset;
_indexLength = indexLength;
_dataOffset = dataOffset;
_dataLength = dataLength;
_footerOffset = footerOffset;
_footerLength = footerLength;
NumRows = numRows;
_compressionKind = compressionKind;
}
/// <inheritdoc />
public ICompressAndDecompress BuildCompressor(CompressionKind compressionKind)
{
new { compressionKind }.Must().NotBeEqualTo(CompressionKind.Invalid);
switch (compressionKind)
{
case CompressionKind.None: return(new NullCompressor());
case CompressionKind.DotNetZip: return(new DotNetZipCompressor());
default: throw new NotSupportedException(Invariant($"{nameof(CompressionKind)} value {compressionKind} is not currently supported."));
}
}
/// <summary>
/// Generate an appropriate wrapping decompressing stream for the codec.
/// </summary>
/// <param name="compression">The compression codec</param>
/// <param name="stream">The stream from which compressed data will be written</param>
/// <returns>A stream from which the user can read uncompressed data</returns>
public static Stream DecompressStream(this CompressionKind compression, Stream stream)
{
switch (compression)
{
case CompressionKind.None:
return(new SubsetStream(stream));
case CompressionKind.Deflate:
return(new DeflateStream(stream, CompressionMode.Decompress, true));
default:
throw Contracts.Except("unrecognized compression codec {0}", compression);
}
}
public virtual void Assemble(FlashWriter output, CompressionKind compression, Action <TagItem> callback)
{
output.Write(((char)compression) + "WS", true);
output.Write(Version);
output.Write(uint.MinValue);
int fileLength = 8;
FlashWriter compressor = null;
switch (compression)
{
case CompressionKind.LZMA:
{
compressor = new FlashWriter((int)FileLength);
break;
}
case CompressionKind.ZLIB:
{
compressor = ZLIB.WrapCompressor(output.BaseStream, true);
break;
}
}
/* Body Start */
Frame.WriteTo(compressor ?? output);
fileLength += (Frame.Area.GetByteSize() + 4);
for (int i = 0; i < Tags.Count; i++)
{
TagItem tag = Tags[i];
callback?.Invoke(tag);
WriteTag(tag, compressor ?? output);
fileLength += tag.Header.Length;
fileLength += (tag.Header.IsLongTag ? 6 : 2);
}
if (compression == CompressionKind.LZMA)
{
byte[] uncompressedBody = ((MemoryStream)compressor.BaseStream).ToArray();
byte[] compressedBody = LZMA.Compress(uncompressedBody);
output.Write(compressedBody);
}
compressor?.Dispose();
/* Body End */
output.Position = 4;
output.Write((uint)fileLength);
output.Position = output.Length;
}
public StripeReaderCollection(Stream inputStream, Footer footer, CompressionKind compressionKind)
{
foreach (var stripe in footer.Stripes)
{
_innerCollection.Add(new StripeReader(
inputStream,
stripe.Offset,
stripe.IndexLength,
stripe.Offset + stripe.IndexLength,
stripe.DataLength,
stripe.Offset + stripe.IndexLength + stripe.DataLength,
stripe.FooterLength,
stripe.NumberOfRows,
compressionKind
));
}
}
public MetaInfoObjExtractor(CompressionKind compressionKind, int bufferSize, int metadataSize,
ByteBuffer footerBuffer)
{
this.compressionKind = compressionKind;
this.bufferSize = bufferSize;
this.codec = WriterImpl.createCodec(compressionKind);
this.metadataSize = metadataSize;
int position = footerBuffer.position();
int footerBufferSize = footerBuffer.limit() - footerBuffer.position() - metadataSize;
this.metadata = extractMetadata(footerBuffer, position, metadataSize, codec, bufferSize);
this.footer = extractFooter(
footerBuffer, position + metadataSize, footerBufferSize, codec, bufferSize);
footerBuffer.position(position);
this.inspector = OrcStruct.createObjectInspector(0, footer.TypesList);
}
/// <summary>
/// 지정한 HttpContext에 리소스 파일을 씁니다.
/// </summary>
/// <param name="context"></param>
/// <param name="filename"></param>
protected void WriteChartFile(HttpContext context, string filename)
{
if (IsDebugEnabled)
{
log.Debug(@"리소스에서 해당 파일 정보를 읽어 반환합니다... filename=" + filename);
}
filename.ShouldNotBeWhiteSpace("filename");
var buffer = GetResourceFileContent(context, filename);
Guard.Assert(buffer != null && buffer.Length > 0, "지정된 파일이 리소스에 없습니다. filename=" + filename);
SetRepsonseHeaders(context, filename);
// file이 swf 처럼 이미 압축이 되어 있다면, 추가로 압축을 할 필요 없다.
//
var needCompress = CanCompression &&
WebTool.NoCompressionFileExtensions.Any(filename.Contains) == false &&
buffer.Length > CompressionThreshold;
if (needCompress)
{
if (IsDebugEnabled)
{
log.Debug("리소스 파일[{0}] 를 압축하여, 반환합니다. 압축방법[{1}]", filename, CompressionKind);
}
// 압축했을 때에, 압축 방법을 응답헤더에 꼭 붙여줘야 Client에서 알아먹겠죠?
context.Response.AppendHeader("Content-Encoding", CompressionKind.ToString().ToLower());
buffer = Compressor.Compress(buffer);
}
context.Response.AppendHeader(@"Content-Length", buffer.Length.ToString());
context.Response.OutputStream.Write(buffer, 0, buffer.Length);
context.Response.Flush();
if (IsDebugEnabled)
{
log.Debug(@"리소스 파일 내용을 Client에 전송했습니다!!! filename=" + filename);
}
}
/// <summary>
/// Constructs a saver for a data view.
/// </summary>
public BinarySaver(IHostEnvironment env, Arguments args)
{
Contracts.CheckValue(env, nameof(env));
_host = env.Register("BinarySaver");
_host.CheckUserArg(!args.MaxRowsPerBlock.HasValue || args.MaxRowsPerBlock > 0, nameof(args.MaxRowsPerBlock), "Must be positive.");
_host.CheckUserArg(!args.MaxBytesPerBlock.HasValue || args.MaxBytesPerBlock > 0, nameof(args.MaxBytesPerBlock), "Must be positive.");
_host.CheckUserArg(args.MaxRowsPerBlock.HasValue || args.MaxBytesPerBlock.HasValue, nameof(args.MaxBytesPerBlock),
"Either " + nameof(args.MaxRowsPerBlock) + " or " + nameof(args.MaxBytesPerBlock) + " must have a value.");
_memPool = new MemoryStreamPool();
_factory = new CodecFactory(_host, _memPool);
_compression = args.Compression;
_maxRowsPerBlock = args.MaxRowsPerBlock;
_maxBytesPerBlock = args.MaxBytesPerBlock;
_deterministicBlockOrder = args.DeterministicBlockOrder;
_silent = args.Silent;
}
/// <summary>
/// Initializes a new instance of the <see cref="SerializerRepresentation"/> class.
/// </summary>
/// <param name="serializationKind">The <see cref="SerializationKind" /> to serialize into.</param>
/// <param name="compressionKind">Optional <see cref="CompressionKind" /> to use; DEFAULT is None.</param>
/// <param name="serializationConfigType">Optional representation of the type of the serialization configuration to use; DEFAULT is null.</param>
/// <param name="metadata">Optional metadata to put, especially useful for custom serializer factory; DEFAULT is empty.</param>
public SerializerRepresentation(
SerializationKind serializationKind,
TypeRepresentation serializationConfigType = null,
CompressionKind compressionKind = CompressionKind.None,
IReadOnlyDictionary <string, string> metadata = null)
{
if (serializationKind == SerializationKind.Invalid)
{
throw new ArgumentOutOfRangeException(Invariant($"'{nameof(serializationKind)}' == '{SerializationKind.Invalid}'"), (Exception)null);
}
if (compressionKind == CompressionKind.Invalid)
{
throw new ArgumentOutOfRangeException(Invariant($"'{nameof(compressionKind)}' == '{CompressionKind.Invalid}'"), (Exception)null);
}
this.SerializationKind = serializationKind;
this.SerializationConfigType = serializationConfigType;
this.CompressionKind = compressionKind;
this.Metadata = metadata ?? new Dictionary <string, string>();
}
/** Extracts the necessary metadata from an externally store buffer (fullFooterBuffer). */
public static FooterInfo extractMetaInfoFromFooter(ByteBuffer bb, string srcPath)
{
// Read the PostScript. Be very careful as some parts of this historically use bb position
// and some use absolute offsets that have to take position into account.
int baseOffset = bb.position();
int lastByteAbsPos = baseOffset + bb.remaining() - 1;
int psLen = bb.get(lastByteAbsPos) & 0xff;
int psAbsPos = lastByteAbsPos - psLen;
OrcProto.PostScript ps = extractPostScript(bb, srcPath, psLen, psAbsPos);
Debug.Assert(baseOffset == bb.position());
// Extract PS information.
int footerSize = (int)ps.FooterLength, metadataSize = (int)ps.MetadataLength,
footerAbsPos = psAbsPos - footerSize, metadataAbsPos = footerAbsPos - metadataSize;
CompressionKind compressionKind = (CompressionKind)Enum.Parse(typeof(CompressionKind), ps.Compression.ToString(), true);
CompressionCodec codec = WriterImpl.createCodec(compressionKind);
int bufferSize = (int)ps.CompressionBlockSize;
bb.position(metadataAbsPos);
bb.mark();
// Extract metadata and footer.
OrcProto.Metadata metadata = extractMetadata(
bb, metadataAbsPos, metadataSize, codec, bufferSize);
List <StripeStatistics> stats = new List <StripeStatistics>(metadata.StripeStatsCount);
foreach (OrcProto.StripeStatistics ss in metadata.StripeStatsList)
{
stats.Add(new StripeStatistics(ss.ColStatsList));
}
OrcProto.Footer footer = extractFooter(bb, footerAbsPos, footerSize, codec, bufferSize);
bb.position(metadataAbsPos);
bb.limit(psAbsPos);
// TODO: do we need footer buffer here? FileInfo/FileMetaInfo is a mess...
FileMetaInfo fmi = new FileMetaInfo(
compressionKind, bufferSize, metadataSize, bb, extractWriterVersion(ps));
return(new FooterInfo(stats, footer, fmi));
}
public WriterOptions(Properties tableProperties, Configuration conf)
{
configuration = conf;
memoryManagerValue = getMemoryManager(conf);
stripeSizeValue = OrcConf.STRIPE_SIZE.getLong(tableProperties, conf);
blockSizeValue = OrcConf.BLOCK_SIZE.getLong(tableProperties, conf);
rowIndexStrideValue =
(int)OrcConf.ROW_INDEX_STRIDE.getLong(tableProperties, conf);
bufferSizeValue = (int)OrcConf.BUFFER_SIZE.getLong(tableProperties,
conf);
blockPaddingValue =
OrcConf.BLOCK_PADDING.getBoolean(tableProperties, conf);
compressValue = (CompressionKind)Enum.Parse(
typeof(CompressionKind),
OrcConf.COMPRESS.getString(tableProperties, conf),
true);
string versionName = OrcConf.WRITE_FORMAT.getString(tableProperties,
conf);
versionValue = VersionHelper.byName(versionName);
string enString = OrcConf.ENCODING_STRATEGY.getString(tableProperties,
conf);
_encodingStrategy = (EncodingStrategy)Enum.Parse(typeof(EncodingStrategy), enString, true);
string compString =
OrcConf.COMPRESSION_STRATEGY.getString(tableProperties, conf);
compressionStrategy = (CompressionStrategy)Enum.Parse(typeof(CompressionStrategy), compString, true);
_paddingTolerance =
OrcConf.BLOCK_PADDING_TOLERANCE.getDouble(tableProperties, conf);
_bloomFilterColumns = OrcConf.BLOOM_FILTER_COLUMNS.getString(tableProperties,
conf);
_bloomFilterFpp = OrcConf.BLOOM_FILTER_FPP.getDouble(tableProperties,
conf);
timeZone = TimeZoneInfo.Local.Id;
}
/** Ctor used when reading splits - no version list or full footer buffer. */
public FileMetaInfo(CompressionKind compressionKind, int bufferSize, int metadataSize,
ByteBuffer footerBuffer, OrcFile.WriterVersion writerVersion)
: this(compressionKind, bufferSize, metadataSize, footerBuffer, null,
writerVersion, null)
{
}
public WriterOptions(Properties tableProperties, Configuration conf)
{
configuration = conf;
memoryManagerValue = getMemoryManager(conf);
stripeSizeValue = OrcConf.STRIPE_SIZE.getLong(tableProperties, conf);
blockSizeValue = OrcConf.BLOCK_SIZE.getLong(tableProperties, conf);
rowIndexStrideValue =
(int)OrcConf.ROW_INDEX_STRIDE.getLong(tableProperties, conf);
bufferSizeValue = (int)OrcConf.BUFFER_SIZE.getLong(tableProperties,
conf);
blockPaddingValue =
OrcConf.BLOCK_PADDING.getBoolean(tableProperties, conf);
compressValue = (CompressionKind)Enum.Parse(
typeof(CompressionKind),
OrcConf.COMPRESS.getString(tableProperties, conf),
true);
string versionName = OrcConf.WRITE_FORMAT.getString(tableProperties,
conf);
versionValue = VersionHelper.byName(versionName);
string enString = OrcConf.ENCODING_STRATEGY.getString(tableProperties,
conf);
_encodingStrategy = (EncodingStrategy)Enum.Parse(typeof(EncodingStrategy), enString, true);
string compString =
OrcConf.COMPRESSION_STRATEGY.getString(tableProperties, conf);
compressionStrategy = (CompressionStrategy)Enum.Parse(typeof(CompressionStrategy), compString, true);
_paddingTolerance =
OrcConf.BLOCK_PADDING_TOLERANCE.getDouble(tableProperties, conf);
_bloomFilterColumns = OrcConf.BLOOM_FILTER_COLUMNS.getString(tableProperties,
conf);
_bloomFilterFpp = OrcConf.BLOOM_FILTER_FPP.getDouble(tableProperties,
conf);
timeZone = TimeZoneInfo.Local.Id;
}
/**
* Sets the generic compression that is used to compress the data.
*/
public WriterOptions compress(CompressionKind value)
{
compressValue = value;
return this;
}
private OrcProto.CompressionKind writeCompressionKind(CompressionKind kind)
{
switch (kind)
{
case CompressionKind.NONE: return OrcProto.CompressionKind.NONE;
case CompressionKind.ZLIB: return OrcProto.CompressionKind.ZLIB;
case CompressionKind.SNAPPY: return OrcProto.CompressionKind.SNAPPY;
case CompressionKind.LZO: return OrcProto.CompressionKind.LZO;
default:
throw new ArgumentException("Unknown compression " + kind);
}
}
public static CompressionCodec createCodec(CompressionKind kind)
{
switch (kind)
{
case CompressionKind.NONE:
return null;
case CompressionKind.ZLIB:
return new ZlibCodec();
case CompressionKind.SNAPPY:
case CompressionKind.LZO:
throw new NotSupportedException("Unsupported compression codec: " + kind);
default:
throw new ArgumentException("Unknown compression codec: " + kind);
}
}
public WriterImpl(
Stream stream,
string path,
OrcFile.WriterOptions options,
ObjectInspector inspector,
TypeDescription schema,
long stripeSize,
CompressionKind compress,
int bufferSize,
int rowIndexStride,
MemoryManager memoryManager,
bool addBlockPadding,
OrcFile.Version version,
OrcFile.WriterCallback callback,
OrcFile.EncodingStrategy encodingStrategy,
OrcFile.CompressionStrategy compressionStrategy,
double paddingTolerance,
long blockSizeValue,
string bloomFilterColumnNames,
double bloomFilterFpp)
{
this.baseStream = stream;
this.streamFactory = new StreamFactory(this);
this.path = path;
this.options = options;
this.callback = callback;
this.schema = schema;
this.adjustedStripeSize = stripeSize;
this.defaultStripeSize = stripeSize;
this.version = version;
this.encodingStrategy = encodingStrategy;
this.compressionStrategy = compressionStrategy;
this.addBlockPadding = addBlockPadding;
this.blockSize = blockSizeValue;
this.paddingTolerance = paddingTolerance;
this.compress = compress;
this.rowIndexStride = rowIndexStride;
this.memoryManager = memoryManager;
buildIndex = rowIndexStride > 0;
codec = createCodec(compress);
int numColumns = schema.getMaximumId() + 1;
this.bufferSize = getEstimatedBufferSize(defaultStripeSize, numColumns, bufferSize);
if (version == OrcFile.Version.V_0_11)
{
/* do not write bloom filters for ORC v11 */
this.bloomFilterColumns = new bool[schema.getMaximumId() + 1];
}
else
{
this.bloomFilterColumns =
OrcUtils.includeColumns(bloomFilterColumnNames, schema);
}
this.bloomFilterFpp = bloomFilterFpp;
treeWriter = createTreeWriter(inspector, schema, streamFactory, false);
if (buildIndex && rowIndexStride < MIN_ROW_INDEX_STRIDE)
{
throw new ArgumentException("Row stride must be at least " +
MIN_ROW_INDEX_STRIDE);
}
// ensure that we are able to handle callbacks before we register ourselves
memoryManager.addWriter(path, stripeSize, this);
}
