public void ContainerIndex_Basics()
{
int threshold = 20000;
using (ContainerIndex index = ContainerIndex.OpenWrite("Sample.cdx"))
{
// Add three top level containers
index.Start(threshold - 3);
index.Start(threshold - 2);
index.Start(threshold - 1);
// Add 100 adjacent peers
for (int i = 1; i <= 100; ++i)
{
int start = i * threshold;
index.Start(start);
index.End(start + threshold - 1);
}
int end = (101 * threshold) - 1;
// Close the top level parents
index.End(end + 1);
index.End(end + 2);
index.End(end + 3);
}
using (ContainerIndex index = ContainerIndex.OpenRead("Sample.cdx"))
{
// Verify 103 entries read back
Assert.AreEqual(103, index.Count);
for (int i = 1; i < 100; ++i)
{
int start = i * threshold;
int end = start + threshold - 1;
// Verify positions inside each container return that container
Assert.AreEqual(start, index.NearestIndexedContainer(start).StartByteOffset);
Assert.AreEqual(start, index.NearestIndexedContainer(start + 1).StartByteOffset);
Assert.AreEqual(start, index.NearestIndexedContainer(end - 1).StartByteOffset);
Assert.AreEqual(start, index.NearestIndexedContainer(end).StartByteOffset);
// Verify the correct three parents are returned
ContainerEntry entry = index.NearestIndexedContainer(start);
entry = index.Parent(entry);
Assert.AreEqual(threshold - 1, entry.StartByteOffset);
entry = index.Parent(entry);
Assert.AreEqual(threshold - 2, entry.StartByteOffset);
entry = index.Parent(entry);
Assert.AreEqual(threshold - 3, entry.StartByteOffset);
entry = index.Parent(entry);
Assert.AreEqual(-1, entry.StartByteOffset);
}
}
}
public async Task RebuildIndex()
{
await Context.Channel.SendMessageAsync("**[Admin]** OK, building index");
// Create a dummy ContainerIndex using latest data
ContainerIndex containerIndex = new ContainerIndex();
containerIndex.Event = StrippedContainer.ConvertToStrippedContainer(ContainerCache.GetEvents().First());
containerIndex.LineNews = StrippedContainer.ConvertToStrippedContainer(ContainerCache.GetLineNews().First());
containerIndex.PopUpNews = StrippedContainer.ConvertToStrippedContainer(ContainerCache.GetPopUpNews().First());
containerIndex.Present = StrippedContainer.ConvertToStrippedContainer(ContainerCache.GetPresents().First());
// Acquire the WebFileHandler lock
lock (WebFileHandler.Lock)
{
// Connect
WebFileHandler.Connect(((SsbuBotConfiguration)Configuration.LoadedConfiguration).WebConfig);
// Write the file
WebFileHandler.WriteAllText(((SsbuBotConfiguration)Configuration.LoadedConfiguration).WebConfig.ContainerIndexPath, WebFileHandler.ToJson(containerIndex));
// Disconnect
WebFileHandler.Disconnect();
}
await Context.Channel.SendMessageAsync("**[Admin]** Done");
}
public BionSearcher(string bionFilePath, int runDepth)
{
_compressor = Memory.Log("Dictionary", () => WordCompressor.OpenRead(Path.ChangeExtension(bionFilePath, ".wdx")));
_containerIndex = Memory.Log("ContainerIndex", () => ContainerIndex.OpenRead(Path.ChangeExtension(bionFilePath, ".cdx")));
_searchIndexReader = Memory.Log("SearchIndex", () => new SearchIndexReader(Path.ChangeExtension(bionFilePath, ".idx")));
_bionReader = Memory.Log("BionReader", () => new BionReader(File.OpenRead(bionFilePath), containerIndex: _containerIndex, compressor: _compressor));
_runDepth = runDepth;
_termPositions = new long[256];
}
/// <summary>
/// Create certificate request
/// </summary>
/// <param name="db"></param>
public RequestDatabase(IItemContainerFactory db)
{
if (db == null)
{
throw new ArgumentNullException(nameof(db));
}
var container = db.OpenAsync("requests").Result;
_requests = container.AsDocuments();
_index = new ContainerIndex(db, container.Name);
}
/// <summary>
/// Create registry services
/// </summary>
/// <param name="db"></param>
/// <param name="logger"></param>
public ApplicationDatabase(IItemContainerFactory db, ILogger logger)
{
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
if (db == null)
{
throw new ArgumentNullException(nameof(db));
}
var container = db.OpenAsync("applications").Result;
_applications = container.AsDocuments();
_index = new ContainerIndex(db, container.Name);
}
public void Dispose()
{
_compressor?.Dispose();
_compressor = null;
_searchIndexReader?.Dispose();
_searchIndexReader = null;
_containerIndex?.Dispose();
_containerIndex = null;
_bionReader?.Dispose();
_bionReader = null;
}
public void ContainerIndex_EndToEnd()
{
string jsonFilePath = @"Content\Medium.json";
string bionFilePath = Path.ChangeExtension(jsonFilePath, ".bion");
string dictionaryPath = Path.ChangeExtension(bionFilePath, "dict.bion");
string comparePath = Path.ChangeExtension(jsonFilePath, "compare.json");
JsonBionConverter.JsonToBion(jsonFilePath, bionFilePath, dictionaryPath);
using (WordCompressor compressor = WordCompressor.OpenRead(dictionaryPath))
using (ContainerIndex cIndex = ContainerIndex.OpenRead(Path.ChangeExtension(bionFilePath, ".cdx")))
using (BionReader reader = new BionReader(File.OpenRead(bionFilePath), cIndex, compressor))
{
for (int i = 0; i < cIndex.Count; ++i)
{
ContainerEntry container = cIndex[i];
// Seek to container start
reader.Seek(container.StartByteOffset);
// Verify a container start is there
int depth = reader.Depth;
reader.Read();
bool isObject = (reader.TokenType == BionToken.StartObject);
Assert.AreEqual((isObject ? BionToken.StartObject : BionToken.StartArray), reader.TokenType);
// Read until the depth is back to the same value
while (reader.Depth != depth)
{
reader.Read();
}
// Verify this is the end container position
Assert.AreEqual((isObject ? BionToken.EndObject : BionToken.EndArray), reader.TokenType);
Assert.AreEqual(container.EndByteOffset, reader.BytesRead);
}
}
}
private VariableKeyBase getIndexStorage(Value arr, ContainerIndex index)
{
return(getIndex(arr, index.Identifier));
}
private VariableKeyBase getFieldStorage(Value obj, ContainerIndex field)
{
var name = string.Format("$obj{0}->{1}", obj.UID, field.Identifier);
return(getMeta(name));
}
public static void HandleContainer(Container container)
{
// Get the FileType
FileType fileType = FileTypeExtensions.GetTypeFromContainer(container);
// Format the destination S3 path
string s3Path = $"/smash/{FileTypeExtensions.GetNamePrefixFromType(fileType)}/{container.Id}";
// Convert the Container into a JSON string
byte[] json = Encoding.UTF8.GetBytes(WebFileHandler.ToJson(container));
// Write the data to S3
S3Api.TransferFile(json, s3Path, "data.json", "application/json");
// Check if this has an image
if (fileType == FileType.Event || fileType == FileType.PopUpNews || fileType == FileType.Present)
{
// Get the image
byte[] image = (byte[])container.GetType().GetProperty("Image").GetValue(container);
// Write the image to S3
S3Api.TransferFile(image, s3Path, "image.jpg", "image/jpeg");
// Create a new MagickImage
using (MagickImage magickImage = new MagickImage(image))
{
// Set the output format to WebP
magickImage.Format = MagickFormat.WebP;
// Create the raw WebP
byte[] webpImage = magickImage.ToByteArray();
// Upload to S3
S3Api.TransferFile(webpImage, s3Path, "image.webp", "image/webp");
}
}
lock (WebFileHandler.Lock)
{
// Connect to the remote server if needed
WebFileHandler.Connect(((SsbuBotConfiguration)Configuration.LoadedConfiguration).WebConfig);
// Convert the Container to a StrippedContainer
StrippedContainer strippedContainer = StrippedContainer.ConvertToStrippedContainer(container);
// Declare a variable to hold the container list
List <StrippedContainer> containerList;
// Format the container list path
string indexPath = string.Format(((SsbuBotConfiguration)Configuration.LoadedConfiguration).WebConfig.ContainerListPath, FileTypeExtensions.GetNamePrefixFromType(fileType));
// Check if the file exists
if (WebFileHandler.Exists(indexPath))
{
// Deserialize the List
containerList = WebFileHandler.ReadAllText <List <StrippedContainer> >(indexPath);
}
else
{
// Create a new List
containerList = new List <StrippedContainer>();
}
// Check if the Container already exists in the list
int index = containerList.FindIndex(x => x.Id == container.Id);
// Check the index
if (index == -1)
{
// Add the StrippedContainer to the List
containerList.Insert(0, strippedContainer);
}
else
{
// Replace the item at the index
containerList[index] = strippedContainer;
}
// Serialize and write the container list
WebFileHandler.WriteAllText(indexPath, WebFileHandler.ToJson(containerList));
// Declare a variable to hold the ContainerIndex
ContainerIndex containerIndex;
// Check if the ContainerIndex exists
if (!WebFileHandler.Exists(((SsbuBotConfiguration)Configuration.LoadedConfiguration).WebConfig.ContainerIndexPath))
{
// Create a dummy StrippedContainer
StrippedContainer dummyStrippedContainer = new StrippedContainer();
dummyStrippedContainer.Id = "-1";
dummyStrippedContainer.Text = new Dictionary <Nintendo.Bcat.Language, string>();
// Create a dummy ContainerIndex
containerIndex = new ContainerIndex();
containerIndex.Event = dummyStrippedContainer;
containerIndex.LineNews = dummyStrippedContainer;
containerIndex.PopUpNews = dummyStrippedContainer;
containerIndex.Present = dummyStrippedContainer;
}
else
{
// Read the file
containerIndex = WebFileHandler.ReadAllText <ContainerIndex>(((SsbuBotConfiguration)Configuration.LoadedConfiguration).WebConfig.ContainerIndexPath);
}
// Get the correct property
PropertyInfo propertyInfo = containerIndex.GetType().GetProperty(container.GetType().Name);
// Set the value
propertyInfo.SetValue(containerIndex, strippedContainer);
// Write out the ContainerIndex
WebFileHandler.WriteAllText(((SsbuBotConfiguration)Configuration.LoadedConfiguration).WebConfig.ContainerIndexPath, WebFileHandler.ToJson(containerIndex));
// Disconnect from the remote server
WebFileHandler.Disconnect();
}
}
internal void InternalExecuteTest(Modes mode)
{
string uploadOnlyDir = null;
string sasConnectionString = "https://liphistorageaccount.blob.core.windows.net/container1?sv=2015-04-05&sr=c&sig=w0tlW6%2BaeHOR%2FIE%2FxASnaPn%2B%2BWvuWD67%2BkmNgKEFLWE%3D&se=2017-02-18T07%3A17%3A41Z&sp=rwdl";
switch (mode)
{
case Modes.Collect:
TestUtility.TestProlog(new string[2] {
"-mode", "collect"
});
break;
case Modes.CollectAndUpload:
TestUtility.TestProlog(new string[4] {
"-mode", "collectandupload", "-StorageConnectionString", sasConnectionString
});
break;
case Modes.Upload:
uploadOnlyDir = Path.Combine(TestUtility.TestDirectory, "uploadonly");
Directory.CreateDirectory(uploadOnlyDir);
TestUtility.TestProlog(new string[6] {
"-mode", mode.ToString(), "-StorageConnectionString", sasConnectionString, "output", uploadOnlyDir
});
break;
}
string traceLogPath = Utility.GenerateProgramFilePath(Program.Config.WorkingDirectoryPath, ".log");
MainWorkflowTest workflow = null;
using (TraceLogger traceLogger = new TraceLogger(traceLogPath))
{
using (LogPackage logPackage = new LogPackage(Program.Config.OutputDirectoryPath, traceLogger))
{
workflow = new MainWorkflowTest(traceLogger);
workflow.Execute(logPackage);
}
}
if (mode != Modes.Upload)
{
string indexFilePath = Program.Config.OutputDirectoryPath + "\\" + ContainerIndex.IndexFileName;
using (StreamReader reader = new StreamReader(indexFilePath))
{
string content = reader.ReadToEnd();
ContainerIndex index;
Assert.IsTrue(ContainerIndex.TryDeserialize(content, out index));
Assert.IsNotNull(index.MiscellaneousLogsFileName);
Assert.IsNotNull(index.FabricLogDirectory);
Assert.IsNotNull(index.PerfCounterDirectory);
}
string extractedDirectory = Program.Config.WorkingDirectoryPath + "\\extracted";
string zipFilePath = Directory.GetFiles(Program.Config.OutputDirectoryPath, "*.zip", SearchOption.TopDirectoryOnly).First();
ZipFile.ExtractToDirectory(zipFilePath, extractedDirectory);
foreach (KeyValuePair <string, string> log in TestUtility.GeneratedTestLogs)
{
Assert.IsTrue(File.Exists(extractedDirectory + log.Key.Replace('/', '\\') + "\\" + log.Value));
}
foreach (KeyValuePair <string, string> log in TestUtility.GeneratedTestLogs)
{
Assert.IsTrue(File.Exists(extractedDirectory + log.Key.Replace('/', '\\') + "\\" + log.Value));
Assert.IsFalse(File.Exists(Path.Combine(TestUtility.TestDirectory, log.Value)));
}
Directory.Delete(extractedDirectory, recursive: true);
}
else
{
Assert.IsFalse(Directory.GetFiles(uploadOnlyDir, "*", SearchOption.AllDirectories).Any());
}
Assert.AreEqual(mode != Modes.Upload, (workflow.collector != null && workflow.collector.IsCollected));
Assert.AreEqual(mode != Modes.Collect, (workflow.uploader != null && workflow.uploader.IsUploaded));
File.Delete(traceLogPath);
TestUtility.TestEpilog();
}
