public void InitDictionary(DictionarySize size, params TextAsset[] dicts)
{
dictionaryLoaded = false;
if (size == DictionarySize.None)
{
return;
}
// must load basic first or else error...
if (size != DictionarySize.Dict82765)
{
InitDictionary(DictionarySize.Dict82765);
}
Loader loader = DictionaryLoader(size);
var pair = DictionaryTermAndCountIndices(size);
var termIndex = pair[0];
var countIndex = pair[1];
using (Stream corpusStream = DictionaryTextAsset(size, dicts).IntoMemoryStream())
{
if (!loader(corpusStream, termIndex, countIndex, defaultSeparatorChars))
{
throw new Exception("Could not load dictionary!");
}
else
{
Debug.LogWarning($"Dictionary Loaded! ({symSpell.EntryCount} entries)");
dictionaryLoaded = true;
}
}
}
private static Vector2Int DictionaryTermAndCountIndices(DictionarySize size)
{
switch (size)
{
case DictionarySize.Dict82765:
return(new Vector2Int(0, 1));
case DictionarySize.Dict243342:
return(new Vector2Int(0, 2));
}
throw new ArgumentException("Unknown size");
}
private TextAsset DictionaryTextAsset(DictionarySize size, params TextAsset[] dicts)
{
switch (size)
{
case DictionarySize.Dict82765:
return(dicts[0]);
case DictionarySize.Dict243342:
return(dicts[1]);
}
throw new ArgumentException("Unknown size");
}
private Loader DictionaryLoader(DictionarySize size)
{
switch (size)
{
case DictionarySize.Dict82765:
return(symSpell.LoadDictionary);
case DictionarySize.Dict243342:
return(symSpell.LoadBigramDictionary);
}
throw new ArgumentException("Unknown size");
}
public int CompareTo(object obj)
{
var objTable = (VpaTableViewModel)obj;
switch (SortColumn)
{
case "ColumnCardinality":
return(RowsCount.CompareTo(objTable.RowsCount) * SortDirection);
case "TotalSize":
return(TotalSize.CompareTo(objTable.TotalSize) * SortDirection);
case "DictionarySize":
return(DictionarySize.CompareTo(objTable.DictionarySize) * SortDirection);
case "DataSize":
return(DataSize.CompareTo(objTable.DataSize) * SortDirection);
case "HierarchiesSize":
return(HierarchiesSize.CompareTo(objTable.HierarchiesSize) * SortDirection);
case "TableSize":
return(TableSize.CompareTo(objTable.TableSize) * SortDirection);
case "PercentageDatabase":
return(PercentageDatabase.CompareTo(objTable.PercentageDatabase) * SortDirection);
case "SegmentsNumber":
return(SegmentsNumber.CompareTo(objTable.SegmentsNumber) * SortDirection);
case "PartitionsNumber":
return(PartitionsNumber.CompareTo(objTable.PartitionsNumber) * SortDirection);
case "ColumnsNumber":
return(ColumnsNumber.CompareTo(objTable.ColumnsNumber) * SortDirection);
case "ReferentialIntegrityViolationCount":
return(ReferentialIntegrityViolationCount.CompareTo(objTable.ReferentialIntegrityViolationCount) * SortDirection);
case "UserHierarchiesSize":
return(UserHierarchiesSize.CompareTo(objTable.UserHierarchiesSize) * SortDirection);
case "RelationshipSize":
return(RelationshipSize.CompareTo(objTable.RelationshipSize) * SortDirection);
case "RowsCount":
return(RowsCount.CompareTo(objTable.RowsCount) * SortDirection);
case "MissingKeys":
return(RelationshipFromMissingKeys.CompareTo(objTable.RelationshipFromMissingKeys) * SortDirection);
case "FromColumnCardinality":
return(RelationshipMaxFromCardinality.CompareTo(objTable.RelationshipMaxFromCardinality) * SortDirection);
case "ToColumnCardinality":
return(RelationshipMaxToCardinality.CompareTo(objTable.RelationshipMaxToCardinality) * SortDirection);
case "InvalidRows":
return(RelationshipInvalidRows.CompareTo(objTable.RelationshipInvalidRows) * SortDirection);
case "UsedSize":
return(RelationshipSize.CompareTo(objTable.RelationshipSize) * SortDirection);
default:
return(TableName.CompareTo(objTable.TableName) * SortDirection);
}
}
public static void Compress(Stream input, Stream output, LzmaSpeed speed = LzmaSpeed.Fastest, DictionarySize dictionarySize = DictionarySize.VerySmall, Action <long, long> onProgress = null)
{
int posStateBits = 2; // default: 2
int litContextBits = 3; // 3 for normal files, 0; for 32-bit data
int litPosBits = 0; // 0 for 64-bit data, 2 for 32-bit.
var numFastBytes = (int)speed;
string matchFinder = "BT4"; // default: BT4
bool endMarker = true;
CoderPropID[] propIDs =
{
CoderPropID.DictionarySize,
CoderPropID.PosStateBits, // (0 <= x <= 4).
CoderPropID.LitContextBits, // (0 <= x <= 8).
CoderPropID.LitPosBits, // (0 <= x <= 4).
CoderPropID.NumFastBytes,
CoderPropID.MatchFinder, // "BT2", "BT4".
CoderPropID.EndMarker
};
object[] properties =
{
(int)dictionarySize,
posStateBits,
litContextBits,
litPosBits,
numFastBytes,
matchFinder,
endMarker
};
var lzmaEncoder = new Encoder();
lzmaEncoder.SetCoderProperties(propIDs, properties);
lzmaEncoder.WriteCoderProperties(output);
var fileSize = input.Length;
for (int i = 0; i < 8; i++)
{
output.WriteByte((byte)(fileSize >> (8 * i)));
}
ICodeProgress prg = null;
if (onProgress != null)
{
prg = new DelegateCodeProgress(onProgress);
}
lzmaEncoder.Code(input, output, -1, -1, prg);
}
public static void CompressFile(string filename, string outfile, LzmaSpeed speed = LzmaSpeed.Fastest, DictionarySize dictionarySize = DictionarySize.VerySmall, Action <long, long> onProgress = null)
{
using (FileStream fs = File.OpenRead(filename))
using (FileStream target = File.Create(outfile)) {
Compress(fs, target, speed, dictionarySize, onProgress);
}
}