protected override void ExtractFromXML(Stream inputXmlFile, MultiStream outputStream, Action <MultiStream, IDContainer, bool> writeItem, bool withLocal = true)
{
var xmlDoc = new XmlDocument();
xmlDoc.Load(inputXmlFile);
XmlNode rootNode = xmlDoc.DocumentElement;
var first = true;
foreach (XmlNode node in rootNode.ChildNodes)
{
if (node.NodeType == XmlNodeType.Element)
{
var locID = node.Attributes["id"]?.Value;
if (string.IsNullOrEmpty(locID))
{
continue;
}
var locName = node.Attributes["displayname"].Value;
writeItem(outputStream, new IDContainer()
{
Index = locID,
UniqueName = locName
}, first);
if (first)
{
first = false;
}
}
}
}
protected override void ExtractFromXML(Stream inputXmlFile, MultiStream outputStream, Action <MultiStream, IDContainer, bool> writeItem, LocalizationData localizationData = default)
{
var xmlDoc = new XmlDocument();
xmlDoc.Load(inputXmlFile);
var rootNode = xmlDoc.LastChild.FirstChild;
var first = true;
foreach (XmlNode node in rootNode.ChildNodes)
{
if (node.NodeType == XmlNodeType.Element)
{
var locID = node.Attributes["id"].Value;
var locName = node.Attributes["displayname"].Value;
writeItem(outputStream, new IDContainer()
{
Index = locID,
UniqueName = locName
}, first);
if (first)
{
first = false;
}
}
}
}
private void WriteItem(MultiStream multiStream, IDContainer idContainer, bool first = false)
{
foreach (var streamType in multiStream.StreamTypes)
{
var output = new StringBuilder();
if (streamType.ExportType == ExportType.TextList)
{
output.AppendFormat("{0,4}: {1,-65}", idContainer.Index, idContainer.UniqueName);
if (idContainer is ItemContainer itemContainer && itemContainer.LocalizedNames != null)
{
var englishNames = itemContainer.LocalizedNames.Where(x => x.Key == "EN-US");
if (englishNames.Any())
{
output.AppendFormat(": {0}", englishNames.First().Value);
}
}
output.AppendLine();
}
else if (streamType.ExportType == ExportType.Json)
{
if (!first)
{
output.AppendLine(",");
}
output.Append(idContainer.ToJSON());
}
WriteString(streamType, output.ToString());
output.Clear();
}
}
public void RedirectsConsoleOutput()
{
var originalOutWriter = Console.Out;
try
{
var originalOut = Console.OpenStandardOutput();
var movingMemoryStream = new MovingMemoryStream();
var multiOut = new MultiStream(originalOut, movingMemoryStream);
var writer = new StreamWriter(multiOut)
{
AutoFlush = true
};
Console.SetOut(writer);
Console.WriteLine("Test");
var reader = new StreamReader(movingMemoryStream);
var line1 = reader.ReadLine();
Assert.Equal("Test", line1);
}
finally
{
Console.SetOut(originalOutWriter);
}
}
private void extractMultipleFromFolder(ulong outputStreamIndexOffset, bool[] matches, ulong folderIndex, ulong packIndex, Func <ulong, Stream> onStreamRequest, Action <ulong, Stream> onStreamClose)
{
using (var decoder = createDecoderStream(folderIndex, packIndex))
{
// define output stream
Stream outStream = null;
if (matches == null)
{
outStream = onStreamRequest(outputStreamIndexOffset);
}
else
{
ulong numStreams = streamsInfo.SubStreamsInfo.NumUnPackStreamsInFolders[folderIndex];
// create complex multistream
MultiStream multi = new MultiStream(numStreams,
(ulong innerIndex) =>
{
Stream innerStream = null;
if (matches[innerIndex])
{
innerStream = onStreamRequest(outputStreamIndexOffset + innerIndex);
if (innerStream == null)
{
matches[innerIndex] = false;
}
}
return(innerStream ??
new NullStream((long)streamsInfo.SubStreamsInfo.UnPackSizes[(int)(outputStreamIndexOffset + innerIndex)]));
},
(ulong innerIndex, Stream stream) =>
{
if (matches[innerIndex])
{
onStreamClose(outputStreamIndexOffset + innerIndex, stream);
}
});
// set sizes in multistream
for (ulong i = 0; i < numStreams; ++i)
{
multi.Sizes[i] = (long)streamsInfo.SubStreamsInfo.UnPackSizes[(int)(outputStreamIndexOffset + i)];
}
// set new stream as output stream
outStream = multi;
}
// actual extraction is done here (some decoder streams require knowing output size in advance, like PPMd)
Util.TransferTo(decoder, outStream, (long)streamsInfo.UnPackInfo.Folders[folderIndex].GetUnPackSize());
// call stream close delegate if only one stream and delegate present
if (matches == null)
{
onStreamClose?.Invoke(outputStreamIndexOffset, outStream);
}
}
}
public void SingleStreamWrite()
{
byte[] data = { 1, 2, 3, 4, 5 };
using (var stream1 = new MemoryStream())
{
using (var stream = new MultiStream(stream1))
{
stream.Write(data, 0, data.Length);
}
Assert.Equal(data, stream1.ToArray());
}
}
public MockMailReference(
string id,
string sender,
IImmutableList <string> recipients,
bool saved,
byte[] body,
IMailStore store)
{
Id = id;
Sender = sender;
Recipients = recipients;
BackupBodyStream = body == null ? new MemoryStream() : new MemoryStream(body);
BodyStream = new MultiStream(new Stream[] { BackupBodyStream }, true);
IsSaved = saved;
Store = store;
}
public MockMailboxItemReference(
string id,
string mailbox,
string folder,
MailboxFlags flags,
bool saved,
byte[] body,
IWriter store)
{
Id = id;
Flags = flags;
BackupBodyStream = body == null ? new MemoryStream() : new MemoryStream(body);
BodyStream = new MultiStream(new[] { BackupBodyStream }, true);
IsSaved = saved;
Store = store;
Mailbox = mailbox;
Folder = folder;
}
private async Task DispatchSingleMailReferenceAsync(
IMailReadReference readReference,
Stream bodyStream,
CancellationToken token)
{
string sender = readReference.Sender;
IDictionary <string, IEnumerable <string> > headers =
await MailUtilities.ParseHeadersAsync(bodyStream);
ISet <string> recipients =
AugmentRecipients(sender, readReference.Recipients, headers);
if (!recipients.Any())
{
_log.Warning($"{readReference.Id} had no recipients");
}
(bodyStream, sender) = await ReplaceSenderAsync(headers, bodyStream, sender, token);
IWritable[] dispatchReferences = await CreateDispatchesAsync(
readReference.Id,
recipients,
sender,
token);
if (!dispatchReferences.Any())
{
_log.Warning($"Failed to locate any processor for {readReference.Id}");
}
using (var targetStream = new MultiStream(dispatchReferences.Select(r => r.BodyStream)))
{
await bodyStream.CopyToAsync(targetStream, token);
}
await Task.WhenAll(dispatchReferences.Select(r => r.Store.SaveAsync(r, token)));
}
public void Extract(bool withLocal = true)
{
var xmlPath = DecryptBinFile(GetBinFilePath());
using (var inputFile = File.OpenRead(xmlPath))
{
var streamTypes = new List <StreamType>();
if (Program.ExportType == ExportType.TextList || Program.ExportType == ExportType.Both)
{
const ExportType exportType = ExportType.TextList;
streamTypes.Add(new StreamType
{
Stream = GetExportStream(exportType),
ExportType = exportType
});
}
if (Program.ExportType == ExportType.Json || Program.ExportType == ExportType.Both)
{
const ExportType exportType = ExportType.Json;
streamTypes.Add(new StreamType
{
Stream = GetExportStream(exportType),
ExportType = exportType
});
}
var multiStream = new MultiStream(streamTypes.ToArray());
ExtractFromXML(inputFile, multiStream, WriteItem, withLocal);
foreach (var streamType in streamTypes)
{
CloseExportStream(streamType.Stream, streamType.ExportType);
streamType.Stream.Close();
}
}
}
public void Extract(LocalizationData localizationData = default)
{
var xmlPath = DecryptBinFile(GetBinFilePath(), outputFolderPath);
using (var inputFile = File.OpenRead(xmlPath))
{
var streamTypes = new List <StreamType>();
if (exportType == ExportType.TextList || exportType == ExportType.Both)
{
const ExportType exportType = ExportType.TextList;
streamTypes.Add(new StreamType
{
Stream = GetExportStream(exportType),
ExportType = exportType
});
}
if (exportType == ExportType.Json || exportType == ExportType.Both)
{
const ExportType exportType = ExportType.Json;
streamTypes.Add(new StreamType
{
Stream = GetExportStream(exportType),
ExportType = exportType
});
}
var multiStream = new MultiStream(streamTypes.ToArray());
ExtractFromXML(inputFile, multiStream, WriteItem, localizationData);
foreach (var streamType in streamTypes)
{
CloseExportStream(streamType.Stream, streamType.ExportType);
streamType.Stream.Close();
}
}
}
protected abstract void ExtractFromXML(Stream inputXmlFile, MultiStream outputStream, Action <MultiStream, IDContainer, bool> writeItem, bool withLocal = true);
private void extractMultipleFromFolder(ulong outputStreamIndexOffset, bool[] matches, ulong folderIndex, ulong packIndex, Func <ulong, Stream> onStreamRequest, Action <ulong, Stream> onStreamClose, SevenZipProgressProvider progressProvider)
{
using (var decoder = createDecoderStream(folderIndex, packIndex))
{
ulong unPackSize = streamsInfo.UnPackInfo.Folders[folderIndex].GetUnPackSize();
ulong neededUnPackSize = 0;
// define output stream
Stream outStream = null;
if (matches == null)
{
// single stream
outStream = onStreamRequest(outputStreamIndexOffset);
neededUnPackSize = unPackSize;
}
else
{
// find actual number of needed streams
ulong numStreams = streamsInfo.SubStreamsInfo.NumUnPackStreamsInFolders[folderIndex];
ulong lastStream = checked ((ulong)Array.LastIndexOf(matches, true));
numStreams = lastStream + 1;
// create complex multistream
MultiStream multi = new MultiStream(numStreams,
(ulong innerIndex) =>
{
Stream innerStream = null;
if (matches[innerIndex])
{
innerStream = onStreamRequest(outputStreamIndexOffset + innerIndex);
if (innerStream == null)
{
matches[innerIndex] = false;
}
}
return(innerStream ??
new NullStream((long)streamsInfo.SubStreamsInfo.UnPackSizes[(int)(outputStreamIndexOffset + innerIndex)]));
},
(ulong innerIndex, Stream stream) =>
{
if (onStreamClose != null && matches[innerIndex])
{
onStreamClose(outputStreamIndexOffset + innerIndex, stream);
}
});
// set sizes in multistream and define needed data size
for (ulong i = 0; i < numStreams; i++)
{
ulong ss = streamsInfo.SubStreamsInfo.UnPackSizes[(int)(outputStreamIndexOffset + i)];
multi.Sizes[i] = (long)ss;
neededUnPackSize += ss;
}
// set new stream as output stream
outStream = multi;
}
// actual extraction is done here (some decoder streams require knowing output size in advance, like PPMd)
Util.TransferTo(decoder, outStream, (long)neededUnPackSize, progressProvider);
if (progressProvider != null)
{
progressProvider.IncreaseOffsetBy((long)unPackSize, 0);
progressProvider.SetProgress(0, 0);
}
// call stream close delegate if only one stream and delegate present
if (matches == null)
{
onStreamClose?.Invoke(outputStreamIndexOffset, outStream);
}
}
}
protected abstract void ExtractFromXML(Stream inputXmlFile, MultiStream outputStream, Action <MultiStream, IDContainer, bool> writeItem, LocalizationData localizationData = default);
protected override void ExtractFromXML(Stream inputXmlFile, MultiStream outputStream, Action <MultiStream, IDContainer, bool> writeItem, bool withLocal = true)
{
var journals = new List <IDContainer>();
var xmlDoc = new XmlDocument();
xmlDoc.Load(inputXmlFile);
var rootNode = xmlDoc.LastChild;
var localizationData = default(LocalizationData);
if (withLocal)
{
localizationData = ExtractLocalization();
}
var index = 0;
var first = true;
foreach (XmlNode node in rootNode.ChildNodes)
{
if (node.NodeType == XmlNodeType.Element && !string.IsNullOrEmpty(node.Attributes["uniquename"]?.Value))
{
var uniqueName = node.Attributes["uniquename"].Value;
var enchantmentLevel = node.Attributes["enchantmentlevel"];
var description = node.Attributes["descriptionlocatag"];
var name = node.Attributes["descvariable0"];
var enchantment = "";
if (enchantmentLevel != null && enchantmentLevel.Value != "0")
{
enchantment = "@" + enchantmentLevel.Value;
}
var localizationNameVariable = name != null ? name.Value : LocalizationItemPrefix + uniqueName;
if (uniqueName.Contains("ARTEFACT"))
{
localizationNameVariable = LocalizationItemPrefix + uniqueName;
}
var container = new ItemContainer()
{
Index = index.ToString(),
UniqueName = uniqueName + enchantment,
LocalizationDescriptionVariable = description != null ? description.Value : LocalizationItemPrefix + uniqueName + LocalizationItemDescPostfix,
LocalizationNameVariable = localizationNameVariable
};
SetLocalization(localizationData, container);
writeItem(outputStream, container, first);
if (first)
{
first = false;
}
index++;
if (node.Name == "journalitem")
{
journals.Add(new ItemContainer()
{
UniqueName = uniqueName
});
}
var element = FindElement(node, "enchantments");
if (element != null)
{
foreach (XmlElement childNode in element.ChildNodes)
{
var enchantmentName = node.Attributes["uniquename"].Value + "@" + childNode.Attributes["enchantmentlevel"].Value;
container = new ItemContainer()
{
Index = index.ToString(),
UniqueName = enchantmentName,
LocalizationDescriptionVariable = description != null ? description.Value : LocalizationItemPrefix + uniqueName + LocalizationItemDescPostfix,
LocalizationNameVariable = name != null ? name.Value : LocalizationItemPrefix + uniqueName
};
SetLocalization(localizationData, container);
writeItem(outputStream, container, false);
index++;
}
}
}
}
foreach (ItemContainer j in journals)
{
var container = new ItemContainer()
{
Index = index.ToString(),
UniqueName = j.UniqueName + "_EMPTY",
LocalizationDescriptionVariable = LocalizationItemPrefix + j.UniqueName + "_EMPTY" + LocalizationItemDescPostfix,
LocalizationNameVariable = LocalizationItemPrefix + j.UniqueName + "_EMPTY"
};
SetLocalization(localizationData, container);
writeItem(outputStream, container, false);
index++;
container = new ItemContainer()
{
Index = index.ToString(),
UniqueName = j.UniqueName + "_FULL",
LocalizationDescriptionVariable = LocalizationItemPrefix + j.UniqueName + "_FULL" + LocalizationItemDescPostfix,
LocalizationNameVariable = LocalizationItemPrefix + j.UniqueName + "_FULL"
};
SetLocalization(localizationData, container);
writeItem(outputStream, container, false);
index++;
}
}
public static Stream BuildMultipartData(IDictionary formData, string boundary, Encoding encoding = null)
{
AssertUtil.ArgumentNotNull(formData, nameof(formData));
AssertUtil.ArgumentNotEmpty(boundary, nameof(boundary));
encoding = encoding == null ? Encoding.UTF8 : encoding;
var stream = new MultiStream();
string keyValTemplate = $"--{boundary}\r\n"
+ "Content-Disposition: form-data; name=\"{0}\"\r\n"
+ "\r\n";
string fileValTemplate = $"--{boundary}\r\n"
+ "Content-Disposition: form-data; name=\"{0}\"; filename=\"{1}\"\r\n"
+ "Content-Type: application/octet-stream\r\n"
+ "\r\n";
foreach (var key in formData.Keys)
{
if (!(key is string))
{
throw new ArgumentException("Key isnot typeof string.");
}
if (string.IsNullOrWhiteSpace((string)key))
{
throw new ArgumentException("Key cannot be null or empty.");
}
if (formData[key] is IEnumerable)
{
foreach (var value in (IEnumerable)formData[key])
{
if (value is FileStream)
{
var fileValStrm = new MemoryStream();
using (var writer = new SmartStream(fileValStrm, false))
{
writer.Encoding = encoding;
writer.Write(string.Format(fileValTemplate, ((string)key).Trim(), Path.GetFileName(((FileStream)value).Name)));
writer.Flush();
}
fileValStrm.Position = 0;
stream.AppendStream(fileValStrm);
stream.AppendStream((FileStream)value);
stream.AppendStream(new MemoryStream(new byte[] { (byte)'\r', (byte)'\n' }));
}
else
{
var keyValStrm = new MemoryStream();
using (var writer = new SmartStream(keyValStrm, false))
{
writer.Encoding = encoding;
writer.Write(string.Format(keyValTemplate, ((string)key).Trim()));
writer.Write(value.ToString());
writer.Write("\r\n");
writer.Flush();
}
keyValStrm.Position = 0;
stream.AppendStream(keyValStrm);
}
}
}
else
{
if (formData[key] is FileStream)
{
var fileValStrm = new MemoryStream();
using (var writer = new SmartStream(fileValStrm, false))
{
writer.Encoding = encoding;
writer.Write(string.Format(fileValTemplate, ((string)key).Trim(), Path.GetFileName(((FileStream)formData[key]).Name)));
writer.Flush();
}
fileValStrm.Position = 0;
stream.AppendStream(fileValStrm);
stream.AppendStream((FileStream)formData[key]);
stream.AppendStream(new MemoryStream(new byte[] { (byte)'\r', (byte)'\n' }));
}
else
{
var keyValStrm = new MemoryStream();
using (var writer = new SmartStream(keyValStrm, false))
{
writer.Encoding = encoding;
writer.Write(string.Format(keyValTemplate, ((string)key).Trim()));
writer.Write(formData[key]?.ToString());
writer.Write("\r\n");
writer.Flush();
}
keyValStrm.Position = 0;
stream.AppendStream(keyValStrm);
}
}
}
// add end boundary string line.
stream.AppendStream(new MemoryStream(encoding.GetBytes($"--{boundary}--\r\n")));
return(stream);
}
public LogFile(string fileName)
{
_fileName = fileName;
_writer = new MultiStream(fileName);
_infoLogPrefix = new InfoLogPrefix();
}
