public void Write(CacheWriter f)
{
var flags = (UxlMethodFlags)EntityFlags;
if (Condition != null)
{
flags |= UxlMethodFlags.HasCondition;
}
if (IsDefault)
{
flags |= UxlMethodFlags.IsDefault;
}
if (Implementations.Count > 0)
{
flags |= UxlMethodFlags.Implementations;
}
f.Write((byte)flags);
f.WriteGlobal(Signature);
if (flags.HasFlag(UxlMethodFlags.HasCondition))
{
f.WriteGlobal(Condition.Value);
}
if (flags.HasFlag(UxlMethodFlags.Implementations))
{
f.WriteList(Implementations, (w, x) => x.Write(w));
}
WriteEntity(f, (UxlEntityFlags)flags);
}
private void AddWriters(Node node, NodeReader reader, BufferTree tree, QueueIndex queue)
{
if (node.Index.Count > 0)
{
FieldWriter writer = new FieldWriter(node)
{
BufferIndex = node.Index.ToArray(),
NamePart = this.GetNamePart(node, queue, tree),
Queue = queue,
};
reader.Writers.Add(writer);
tree.Fields.Add(writer);
if (queue != null && queue.Type == RelationQueueType.Cached)
{
CacheWriter cacheWriter = new CacheWriter(node)
{
BufferIndex = writer.BufferIndex.First(),
CacheIndex = queue.Cache.Count,
Queue = queue,
};
queue.Cache.Add(cacheWriter);
}
}
if (node.Item != null || node.Metadata.HasFlag(RelationMetadataFlags.List | RelationMetadataFlags.Recursive))
{
QueueIndex prevQueue = tree.Queues.LastOrDefault()?.Index;
QueueIndex nextQueue = this.CreateIndex(node, tree);
QueueWriter writer = new QueueWriter(node)
{
NamePart = this.GetNamePart(node.Item ?? node, queue, tree),
Queue = queue,
Next = nextQueue,
};
if ((node.Item ?? node).Metadata.HasFlag(RelationMetadataFlags.Recursive))
{
nextQueue.Type = RelationQueueType.Recursive;
}
else if (prevQueue != null && !prevQueue.List.Identity.Equals(nextQueue.List.Owner.Parent?.Identity))
{
nextQueue.Type = RelationQueueType.Cached;
}
else if (prevQueue != null && prevQueue.Type == RelationQueueType.Cached)
{
nextQueue.Type = RelationQueueType.Cached;
}
reader.Writers.Add(writer);
if (node.Item != null)
{
this.CreateQueue(node.Item, tree, nextQueue);
}
}
}
private Expression GetCacheReadExpression(CacheWriter writer)
{
Expression cacheIndex = this.GetCacheIndexExpression(writer);
Expression bufferIndex = Expression.ArrayAccess(Arguments.Fields, Expression.Constant(writer.BufferIndex));
return(Expression.Assign(bufferIndex, cacheIndex));
}
public static void Write(this CacheWriter f, CopyFile e)
{
var flags = (UxlCopyFileFlags)e.Flags & UxlCopyFileFlags.FlagsMask;
if (e.TargetName != null)
{
flags |= UxlCopyFileFlags.HasTargetName;
}
if (e.Condition != null)
{
flags |= UxlCopyFileFlags.HasCondition;
}
if (e.Type != null)
{
flags |= UxlCopyFileFlags.HasType;
}
f.Write((byte)flags);
f.WriteGlobal(e.SourceName);
if (flags.HasFlag(UxlCopyFileFlags.HasTargetName))
{
f.WriteGlobal(e.TargetName.Value);
}
if (flags.HasFlag(UxlCopyFileFlags.HasCondition))
{
f.WriteGlobal(e.Condition.Value);
}
if (flags.HasFlag(UxlCopyFileFlags.HasType))
{
f.WriteGlobal(e.Type.Value);
}
}
public void Write(CacheWriter f)
{
var flags = (UxlTypeFlags)EntityFlags;
if (Condition.HasValue)
{
flags |= UxlTypeFlags.HasCondition;
}
if (IsDefault)
{
flags |= UxlTypeFlags.IsDefault;
}
if (Methods.Count > 0)
{
flags |= UxlTypeFlags.Methods;
}
f.Write((byte)flags);
f.WriteGlobal(Name);
if (flags.HasFlag(UxlTypeFlags.HasCondition))
{
f.WriteGlobal(Condition.Value);
}
if (flags.HasFlag(UxlTypeFlags.Methods))
{
f.WriteList(Methods, (w, x) => x.Write(w));
}
WriteEntity(f, (UxlEntityFlags)flags);
}
private static BinaryReader CreateReaderFromDisk(ResolvePackageAssets task, byte[] settingsHash)
{
Debug.Assert(!task.DisablePackageAssetsCache);
BinaryReader reader = null;
try
{
if (File.GetLastWriteTimeUtc(task.ProjectAssetsCacheFile) > File.GetLastWriteTimeUtc(task.ProjectAssetsFile))
{
reader = OpenCacheFile(task.ProjectAssetsCacheFile, settingsHash);
}
}
catch (IOException) { }
catch (InvalidDataException) { }
catch (UnauthorizedAccessException) { }
if (reader == null)
{
using (var writer = new CacheWriter(task))
{
writer.Write();
}
reader = OpenCacheFile(task.ProjectAssetsCacheFile, settingsHash);
}
return(reader);
}
public CacheReader(ResolvePackageAssets task)
{
byte[] settingsHash = task.HashSettings();
BinaryReader reader = null;
try
{
if (File.GetLastWriteTimeUtc(task.ProjectAssetsCacheFile) > File.GetLastWriteTimeUtc(task.ProjectAssetsFile))
{
reader = OpenCacheFile(task.ProjectAssetsCacheFile, settingsHash);
}
}
catch (IOException) { }
catch (InvalidDataException) { }
catch (UnauthorizedAccessException) { }
if (reader == null)
{
using (var writer = new CacheWriter(task))
{
writer.Write();
}
reader = OpenCacheFile(task.ProjectAssetsCacheFile, settingsHash);
}
_reader = reader;
ReadMetadataStringTable();
}
internal static CacheWriter CreateWriter(SourcePackage upk, string filename)
{
if (upk.IsUnknown)
{
throw new InvalidOperationException("UxlDocument: Unknown source package");
}
var w = new CacheWriter(upk, filename);
w.Write(Magic);
return(w);
}
public ModuleDatabase(IServiceManager sm, string cacheFolder = null)
{
_services = sm;
_log = _services.GetService <ILogger>();
_fs = _services.GetService <IFileSystem>();
_defaultCachingLevel = AnalysisCachingLevel.Library;
var cfs = _services.GetService <ICacheFolderService>();
CacheFolder = cacheFolder ?? Path.Combine(cfs.CacheFolder, $"{CacheFolderBaseName}{DatabaseFormatVersion}");
_cacheWriter = new CacheWriter(_services.GetService <IPythonAnalyzer>(), _fs, _log, CacheFolder);
sm.AddService(this);
}
public void Write(CacheWriter f)
{
var flags = Flags;
f.Write((byte)flags);
f.WriteGlobal(Name);
if (flags.HasFlag(UxlTemplateFlags.HasCondition))
{
f.WriteGlobal(Condition.Value);
}
WriteEntity(f, (UxlEntityFlags)flags);
}
private static BinaryReader CreateReaderFromMemory(ResolvePackageAssets task, byte[] settingsHash)
{
Debug.Assert(task.DisablePackageAssetsCache);
var stream = new MemoryStream();
using (var writer = new CacheWriter(task, stream))
{
writer.Write();
}
stream.Position = 0;
return(OpenCacheStream(stream, settingsHash));
}
protected void WriteEntity(CacheWriter f, UxlEntityFlags flags)
{
if (flags.HasFlag(UxlEntityFlags.Elements))
{
f.WriteList(Elements, (w, x) => x.Write(w));
}
if (flags.HasFlag(UxlEntityFlags.CopyFiles))
{
f.WriteList(CopyFiles, (w, x) => w.Write(x));
}
if (flags.HasFlag(UxlEntityFlags.ImageFiles))
{
f.WriteList(ImageFiles, (w, x) => w.Write(x));
}
}
private static BinaryReader CreateReaderFromMemory(ResolvePackageAssets task, byte[] settingsHash)
{
if (!task.DisablePackageAssetsCache)
{
task.Log.LogMessage(MessageImportance.High, Strings.UnableToUsePackageAssetsCache);
}
var stream = new MemoryStream();
using (var writer = new CacheWriter(task, stream))
{
writer.Write();
}
stream.Position = 0;
return(OpenCacheStream(stream, settingsHash));
}
public void Write(CacheWriter f)
{
var flags = (UxlDeclareFlags)Type;
if (Condition != null)
{
flags |= UxlDeclareFlags.HasCondition;
}
f.Write((byte)flags);
f.Write(Source);
f.WriteGlobal(Key);
if (flags.HasFlag(UxlDeclareFlags.HasCondition))
{
f.WriteGlobal(Condition.Value);
}
}
public static void Write(this CacheWriter f, ImageFile e)
{
UxlImageFileFlags flags = 0;
if (e.Condition != null)
{
flags |= UxlImageFileFlags.HasCondition;
}
if (e.TargetName != null)
{
flags |= UxlImageFileFlags.HasTargetName;
}
if (e.TargetWidth != null)
{
flags |= UxlImageFileFlags.HasTargetWidth;
}
if (e.TargetHeight != null)
{
flags |= UxlImageFileFlags.HasTargetHeight;
}
f.Write((byte)flags);
f.WriteGlobal(e.SourceName);
if (flags.HasFlag(UxlImageFileFlags.HasCondition))
{
f.WriteGlobal(e.Condition.Value);
}
if (flags.HasFlag(UxlImageFileFlags.HasTargetName))
{
f.WriteGlobal(e.TargetName.Value);
}
if (flags.HasFlag(UxlImageFileFlags.HasTargetWidth))
{
f.WriteCompressed(e.TargetWidth.Value);
}
if (flags.HasFlag(UxlImageFileFlags.HasTargetHeight))
{
f.WriteCompressed(e.TargetHeight.Value);
}
}
public void Write(CacheWriter f)
{
var flags = (UxlImplementationFlags)Type & UxlImplementationFlags.TypeMask;
if (Condition != null)
{
flags |= UxlImplementationFlags.HasCondition;
}
if (IsDefault)
{
flags |= UxlImplementationFlags.IsDefault;
}
f.Write((byte)flags);
f.Write(Source);
f.WriteGlobal(Body);
if (flags.HasFlag(UxlImplementationFlags.HasCondition))
{
f.WriteGlobal(Condition.Value);
}
}
private Expression GetCacheIndexExpression(CacheWriter writer)
{
Expression cache = this.GetQueuePropertyExpression(writer.Queue, "Cache");
return(Expression.Property(cache, "Item", Expression.Constant(writer.CacheIndex)));
}
public KodiReader()
{
_http = new KodiHttpClient();
_cache = new CacheWriter();
}
public void Write(CacheWriter f)
{
UxlDocumentFlags flags = 0;
if (Condition != null)
{
flags |= UxlDocumentFlags.HasCondition;
}
if (Defines.Count > 0)
{
flags |= UxlDocumentFlags.Defines;
}
if (Usings.Count > 0)
{
flags |= UxlDocumentFlags.Usings;
}
if (Declarations.Count > 0)
{
flags |= UxlDocumentFlags.Declarations;
}
if (Deprecations.Count > 0)
{
flags |= UxlDocumentFlags.Deprecations;
}
if (Templates.Count > 0)
{
flags |= UxlDocumentFlags.Templates;
}
if (Types.Count > 0)
{
flags |= UxlDocumentFlags.Types;
}
f.Write((byte)Backend);
f.Write((byte)flags);
if (flags.HasFlag(UxlDocumentFlags.HasCondition))
{
f.WriteGlobal(Condition.Value);
}
if (flags.HasFlag(UxlDocumentFlags.Defines))
{
f.WriteList(Defines, (w, x) => x.Write(w));
}
if (flags.HasFlag(UxlDocumentFlags.Usings))
{
f.WriteList(Usings, (w, x) => w.WriteGlobal(x));
}
if (flags.HasFlag(UxlDocumentFlags.Declarations))
{
f.WriteList(Declarations, (w, x) => x.Write(w));
}
if (flags.HasFlag(UxlDocumentFlags.Deprecations))
{
f.WriteList(Deprecations, (w, x) => x.Write(w));
}
if (flags.HasFlag(UxlDocumentFlags.Templates))
{
f.WriteList(Templates, (w, x) => x.Write(w));
}
if (flags.HasFlag(UxlDocumentFlags.Types))
{
f.WriteList(Types, (w, x) => x.Write(w));
}
var entityFlags = EntityFlags;
f.Write((byte)entityFlags);
WriteEntity(f, entityFlags);
}
/// <summary>
/// End table caching by writing out remaining records & opening file reader
/// </summary>
public void EndCaching()
{
if (!AllowCaching)
{
return;
}
if (CacheStartPosition < 0 || CacheWriter == null)
{
return; // caching active
}
if (RowsRemovedFromDataTable == 0)
{
if (CacheWriter != null)
{
CacheWriter.Close();
CacheWriter = null;
}
CacheStartPosition = -1;
return;
}
if (RowRetrievalState == RowRetrievalState.Running || RowRetrievalState == RowRetrievalState.Paused)
{
CancelRowRetrieval(); // cancel if not complete
}
WriteRowsToCache(false); // write out remaining rows
CacheWriter.Close();
CacheWriter = null;
CacheReader = new StreamReader(CacheFile);
CacheStartPosition = -1;
//RowCount = CacheStartRowCount; // restore counts (todo: fix this: gives zero for export count with this code)
//KeyCount = CacheStartKeyCount;
RowRetrievalState = RowRetrievalState.Paused;
if (DebugDetails)
{
ClientLog.Message("RowRetrievalState.Paused 3");
}
if (QueryManager.MoleculeGrid != null)
{
QueryManager.MoleculeGrid.EndUpdate();
QueryManager.MoleculeGrid.Refresh();
Application.DoEvents();
}
if (StatusBarManager != null)
{ // update status bar, may process an event that generates a request for more rows
StatusBarManager.DisplayRetrievalProgressState(RowRetrievalState);
StatusBarManager.DisplayFilterCounts(); // update count display
}
if (DebugCaching)
{
ClientLog.Message("EndCaching: " + DataTableMx.Rows.Count);
}
return;
}
/// <summary>
/// Write out rows from table if caching has been activated
/// </summary>
internal void WriteRowsToCache(bool keepLastKeyValue)
{
DataRowMx dr;
string firstKey = "", lastKey = "";
int rowsRemovedInThisCall = 0, ri, ri2, srpi;
if (!AllowCaching)
{
return;
}
if (CacheStartPosition < 0 || CacheWriter == null)
{
return; // just return if caching not active
}
if (DataTableMx.Rows.Count < CacheMiminumRowsRequiredForWriting)
{
return;
}
lock (DataTransferLock) // lock the DataTable while changing
{
if (DataTableFetchPosition < CacheStartPosition)
{
return; // only cache if fetching beyond start position
}
if (DataTableFetchPosition < DataTableMx.Rows.Count - 1)
{
return; // can only cache out if at end of available rows
}
dr = DataTableMx.Rows[CacheStartPosition];
firstKey = dr[KeyValueVoPos] as string;
ri = DataTableMx.Rows.Count - 1;
dr = DataTableMx.Rows[ri];
lastKey = dr[KeyValueVoPos] as string; // key we want to keep
if (QueryManager.MoleculeGrid != null)
{
QueryManager.MoleculeGrid.BeginUpdate();
}
ri = CacheStartPosition; // start deleting here
while (ri < DataTableMx.Rows.Count)
{ // delete anything with key other than end value
dr = DataTableMx.Rows[ri];
string key2 = dr[KeyValueVoPos] as string; // end key with possibly partial data that we want to keep
if (keepLastKeyValue && key2 == lastKey)
{
break;
}
bool doCacheIO = !PurgeDataTableWithoutWritingToCacheFile;
if (doCacheIO)
{
object[] oa = dr.ItemArray;
StringBuilder sb = VoArray.SerializeToText(oa, KeyValueVoPos, oa.Length - KeyValueVoPos);
string lenStr = String.Format("{0,8:00000000}", sb.Length);
CacheWriter.Write(lenStr); // write length
CacheWriter.Write(sb); // write record
RowsWrittenToCache++;
}
DataTableMx.Rows.Remove(dr);
RowsRemovedFromDataTable++;
rowsRemovedInThisCall++;
}
DataTableFetchPosition -= rowsRemovedInThisCall; // adjust fetch position
for (ri2 = CacheStartPosition; ri2 < DataTableMx.Rows.Count; ri2++)
{ // adjust row indexes held the row attributes in rows below those paged out
DataRowAttributes dra = GetRowAttributes(ri2);
if (dra == null)
{
continue;
}
dra.FirstRowForKey -= rowsRemovedInThisCall;
if (dra.SubRowPos == null)
{
continue;
}
for (srpi = 0; srpi < dra.SubRowPos.Length; srpi++)
{
dra.SubRowPos[srpi] -= rowsRemovedInThisCall;
}
}
} // end of locked section
if (QueryManager.MoleculeGrid != null)
{
QueryManager.MoleculeGrid.EndUpdate();
QueryManager.MoleculeGrid.Refresh();
Application.DoEvents();
}
if (DebugCaching)
{
ClientLog.Message(
"CachedRows - DataTable.Rows.Count: " + DataTableMx.Rows.Count +
", FirstKey: " + firstKey + ", LastKey: " + lastKey +
", RowsRemovedFromDataTable (This Call): " + rowsRemovedInThisCall +
", RowsRemovedFromDataTable (Total): " + RowsRemovedFromDataTable +
", RowsWrittenToCache (Total): " + RowsWrittenToCache +
", DataTableFetchPosition: " + DataTableFetchPosition);
}
return;
}
public void Write(CacheWriter f)
{
f.Write(Source);
f.WriteGlobal(OldName);
f.WriteGlobal(NewName);
}
public void Write(CacheWriter f)
{
f.WriteGlobal(Name);
f.WriteGlobal(Condition);
}
