public static bool ContainsTable(this IStreamProvider streamProvider, string tableName)
{
// Don't allow trailing characters on table name which File I/O removes
if (tableName.EndsWith(".") || tableName.EndsWith("\\") || tableName.EndsWith("/"))
{
return(false);
}
if (streamProvider.Attributes(Path(streamProvider, LocationType.Source, tableName, CrawlType.Full)).Exists)
{
return(true);
}
if (streamProvider.Attributes(Path(streamProvider, LocationType.Table, tableName, CrawlType.Full)).Exists)
{
return(true);
}
if (streamProvider.Attributes(Path(streamProvider, LocationType.Config, tableName, ".xql")).Exists)
{
return(true);
}
if (streamProvider.Attributes(Path(streamProvider, LocationType.Query, tableName, ".xql")).Exists)
{
return(true);
}
return(false);
}
public VariableIntegerReader(IStreamProvider streamProvider, string columnPathPrefix, CachingOption option)
{
// Look for each potential size in descending order and build the right reader and converter
Type type = typeof(int);
string path = VariableIntegerWriter.PathForType(columnPathPrefix, typeof(int));
if (streamProvider.Attributes(path).Exists)
{
_reader = TypeProviderFactory.TryGetColumnReader(streamProvider, typeof(int), path, option, typeof(VariableIntegerReader));
_converter = null;
return;
}
path = VariableIntegerWriter.PathForType(columnPathPrefix, typeof(ushort));
if (streamProvider.Attributes(path).Exists)
{
_reader = TypeProviderFactory.TryGetColumnReader(streamProvider, typeof(ushort), path, option, typeof(VariableIntegerReader));
_converter = TypeConverterFactory.GetConverter(typeof(ushort), typeof(int));
return;
}
path = VariableIntegerWriter.PathForType(columnPathPrefix, typeof(byte));
_reader = TypeProviderFactory.TryGetColumnReader(streamProvider, typeof(byte), path, option, typeof(VariableIntegerReader));
_converter = TypeConverterFactory.GetConverter(typeof(byte), typeof(int));
}
public StreamAttributes Attributes(string logicalPath)
{
// If there's a wrapping zip, return the attributes for it with the base path
StreamAttributes zipAttributes = _inner.Attributes(logicalPath + ".zip");
if (zipAttributes.Exists)
{
zipAttributes.Path = logicalPath;
return(zipAttributes);
}
return(_inner.Attributes(logicalPath));
}
public static Type CheckIndicesType(IStreamProvider streamProvider, Type columnType, string columnPath)
{
if (streamProvider.Attributes(Path.Combine(columnPath, EnumWriter.RowIndexFileName)).Exists)
{
return(typeof(byte));
}
return(null);
}
public void Database_BranchedScenario()
{
SampleDatabase.EnsureBuilt();
// Make a branch of the database in "Database.Branched"
string branchedFolder = Path.Combine(s_RootPath, @"..\Database.Branched");
IStreamProvider branchedStreamProvider = new LocalFileStreamProvider(branchedFolder);
branchedStreamProvider.Delete(".");
IStreamProvider mainStreamProvider = SampleDatabase.XDatabaseContext.StreamProvider;
XDatabaseContext branchedContext = new XDatabaseContext(SampleDatabase.XDatabaseContext);
branchedContext.StreamProvider = new MultipleSourceStreamProvider(branchedStreamProvider, mainStreamProvider, MultipleSourceStreamConfiguration.LocalBranch);
branchedContext.Runner = new WorkflowRunner(branchedContext);
// Ask for WebRequest in the main database; verify built
XForm("build WebRequest", 0);
Assert.IsTrue(mainStreamProvider.Attributes("Table\\WebRequest").Exists);
// Ask for WebRequest in the branch. Verify the main one is loaded where it is
XForm("build WebRequest", 0, branchedContext);
Assert.IsFalse(branchedStreamProvider.Attributes("Table\\WebRequest").Exists);
// Ask for WebRequest.Authenticated in the main database; verify built
XForm("build WebRequest.Authenticated", 0);
Assert.IsTrue(mainStreamProvider.Attributes("Table\\WebRequest.Authenticated").Exists);
Assert.IsFalse(branchedStreamProvider.Attributes("Table\\WebRequest.Authenticated").Exists);
// Make a custom query in the branch. Verify the branched source has a copy with the new query, but it isn't published back
string webRequestAuthenticatedConfigNew = @"
read WebRequest
# Slightly different query
where [UserName] != ""
where [UserName] != null";
branchedStreamProvider.WriteAllText("Config\\WebRequest.Authenticated.xql", webRequestAuthenticatedConfigNew);
XForm("build WebRequest.Authenticated", 0, branchedContext);
Assert.IsTrue(branchedStreamProvider.Attributes("Table\\WebRequest.Authenticated").Exists);
Assert.AreEqual(webRequestAuthenticatedConfigNew, ((BinaryTableReader)branchedContext.Runner.Build("WebRequest.Authenticated", branchedContext)).Query);
Assert.AreNotEqual(webRequestAuthenticatedConfigNew, ((BinaryTableReader)SampleDatabase.XDatabaseContext.Runner.Build("WebRequest.Authenticated", SampleDatabase.XDatabaseContext)).Query);
}
public IColumnReader BinaryReader(IStreamProvider streamProvider, string columnPath, CachingOption option)
{
return(ColumnCache.Instance.GetOrBuild(columnPath, option, () =>
{
string filePath = ValuesFilePath(columnPath);
if (!streamProvider.Attributes(filePath).Exists)
{
return null;
}
return new PrimitiveArrayReader <T>(streamProvider.OpenRead(filePath));
}));
}
private static TableMetadata Build(IStreamProvider streamProvider, string tableRootPath)
{
TableMetadata metadata = new TableMetadata();
string schemaFilePath = Path.Combine(tableRootPath, SchemaFileName);
using (ITabularReader sr = TabularFactory.BuildReader(streamProvider.OpenRead(schemaFilePath), SchemaFileName))
{
int nameIndex = sr.ColumnIndex("Name");
int typeIndex = sr.ColumnIndex("Type");
while (sr.NextRow())
{
metadata.Schema.Add(new ColumnDetails(sr.Current(nameIndex).ToString(), TypeProviderFactory.Get(sr.Current(typeIndex).ToString()).Type));
}
}
using (ITabularReader mr = TabularFactory.BuildReader(streamProvider.OpenRead(Path.Combine(tableRootPath, MetadataFileName)), MetadataFileName))
{
int nameIndex = mr.ColumnIndex("Name");
int contextIndex = mr.ColumnIndex("Context");
int valueIndex = mr.ColumnIndex("Value");
while (mr.NextRow())
{
String8 name = mr.Current(nameIndex).ToString8();
String8 context = mr.Current(contextIndex).ToString8();
ITabularValue value = mr.Current(valueIndex);
if (name.Equals("RowCount"))
{
long rowCount;
if (value.ToString8().TryToLong(out rowCount))
{
metadata.RowCount = rowCount;
}
}
else
{
throw new NotImplementedException($"TableMetadataSerializer.Read doesn't know how to read Metadata '{name}'");
}
}
}
metadata.Query = streamProvider.ReadAllText(Path.Combine(tableRootPath, ConfigQueryPath));
string partitionsPath = Path.Combine(tableRootPath, PartitionsFileName);
if (streamProvider.Attributes(partitionsPath).Exists)
{
using (ITabularReader pr = TabularFactory.BuildReader(streamProvider.OpenRead(partitionsPath), PartitionsFileName))
{
int nameIndex = pr.ColumnIndex("Name");
while (pr.NextRow())
{
metadata.Partitions.Add(pr.Current(nameIndex).ToString());
}
}
}
return(metadata);
}
public static TableMetadata Read(IStreamProvider streamProvider, string tableRootPath)
{
return(s_Cache.GetOrBuild($"{streamProvider}|{tableRootPath}",
() => streamProvider.Attributes(Path.Combine(tableRootPath, MetadataFileName)).WhenModifiedUtc,
() => Build(streamProvider, tableRootPath)));
}
public static bool Exists(this IStreamProvider streamProvider, string logicalPath)
{
return(streamProvider.Attributes(logicalPath).Exists);
}
public StreamAttributes Attributes(string logicalPath)
{
return(_attributesCache.GetOrBuild(logicalPath, null, () => _inner.Attributes(logicalPath)));
}