static void LoadContent(MimePart attachment, Stream stream)
{
var content = new MemoryBlockStream();
if (attachment.ContentType.IsMimeType("text", "*"))
{
var filter = new BestEncodingFilter();
var buf = new byte[4096];
int index, length;
int nread;
while ((nread = stream.Read(buf, 0, buf.Length)) > 0)
{
filter.Filter(buf, 0, nread, out index, out length);
content.Write(buf, 0, nread);
}
filter.Flush(buf, 0, 0, out index, out length);
attachment.ContentTransferEncoding = filter.GetBestEncoding(EncodingConstraint.SevenBit);
}
else
{
attachment.ContentTransferEncoding = ContentEncoding.Base64;
stream.CopyTo(content, 4096);
}
content.Position = 0;
attachment.Content = new MimeContent(content);
}
public MemoryBlockStreamTests()
{
var bytes = new byte[9 * 1024];
int position = 0;
random = new Random();
random.NextBytes(bytes);
// this is our master stream, all operations on the chained stream
// should match the results on this stream
master = new MemoryStream(bytes);
mbuf = new byte[4096];
buf = new byte[4096];
// write the content into the memory block stream in random chunks
blocks = new MemoryBlockStream();
while (position < bytes.Length)
{
int n = Math.Min(bytes.Length - position, random.Next() % 4096);
blocks.Write(bytes, position, n);
position += n;
}
blocks.Seek(0, SeekOrigin.Begin);
}
public void Setup()
{
var bytes = new byte[9 * 1024];
int position = 0;
random = new Random();
random.NextBytes(bytes);
// this is our master stream, all operations on the chained stream
// should match the results on this stream
master = new MemoryStream(bytes);
mbuf = new byte[4096];
buf = new byte[4096];
// write the content into the memory block stream in random chunks
blocks = new MemoryBlockStream();
Assert.IsTrue(blocks.CanRead, "Expected to be able to read from the memory block stream.");
Assert.IsTrue(blocks.CanWrite, "Expected to be able to write to the memory block stream.");
Assert.IsTrue(blocks.CanSeek, "Expected to be able to seek in the memory block stream.");
Assert.IsFalse(blocks.CanTimeout, "Did not expect to be able to set timeouts in the memory block stream.");
while (position < bytes.Length)
{
int n = Math.Min(bytes.Length - position, random.Next() % 4096);
blocks.Write(bytes, position, n);
position += n;
}
blocks.Seek(0, SeekOrigin.Begin);
}
void LoadContent(MimePart attachment, string fileName)
{
var content = new MemoryBlockStream();
var filter = new BestEncodingFilter();
using (var stream = File.OpenRead(fileName)) {
var buf = new byte[4096];
int index, length;
int nread;
while ((nread = stream.Read(buf, 0, buf.Length)) > 0)
{
filter.Filter(buf, 0, nread, out index, out length);
content.Write(buf, 0, nread);
}
filter.Flush(buf, 0, 0, out index, out length);
}
content.Position = 0;
if (linked)
{
attachment.ContentLocation = new Uri(Path.GetFileName(fileName), UriKind.Relative);
}
attachment.ContentTransferEncoding = filter.GetBestEncoding(EncodingConstraint.SevenBit);
attachment.ContentObject = new ContentObject(content, ContentEncoding.Default);
}
public void WriteZeroBytes()
{
byte[] buf = Array.Empty<byte>();
MemoryBlockStream ms = new MemoryBlockStream();
ms.Write(buf, 0, 0);
Assert.Equal(0, ms.Length);
IRandomSource rng = RandomDefaults.CreateRandomSource(1234567);
byte[] buf2 = new byte[100];
rng.NextBytes(buf2);
ms.Write(buf2);
Assert.Equal(ms.ToArray(), buf2);
ms.Write(buf, 0, 0);
Assert.Equal(ms.Length, buf2.Length);
}
void OnFieldsChanged(object sender, HeaderListChangedEventArgs e)
{
var stream = new MemoryBlockStream();
var options = FormatOptions.Default;
fields.WriteTo(options, stream);
stream.Write(options.NewLineBytes, 0, options.NewLineBytes.Length);
stream.Position = 0;
ContentObject = new ContentObject(stream);
}
void QueueCommand(SmtpCommand type, string command)
{
if (queue == null)
{
queue = new MemoryBlockStream();
}
var bytes = Encoding.UTF8.GetBytes(command + "\r\n");
queue.Write(bytes, 0, bytes.Length);
queued.Add(type);
}
private void PerformMutationOp_Write()
{
int len = _rng.Next(300);
Span <byte> buf = stackalloc byte[len];
_rng.NextBytes(buf);
_strmA.Write(buf);
_strmB.Write(buf);
Debug.WriteLine($"Write count={len}");
}
private bool StoreEntities(OutgoingEmail email, List <string> attachmentList)
{
foreach (string iter in attachmentList)
{
FileStream stream = File.OpenRead(iter);
if (!stream.CanRead)
{
return(false);
}
string mimeType = MimeTypes.GetMimeType(iter);
ContentType fileType = ContentType.Parse(mimeType);
MimePart attachment;
if (fileType.IsMimeType("text", "*"))
{
attachment = new TextPart(fileType.MediaSubtype);
foreach (var param in fileType.Parameters)
{
attachment.ContentType.Parameters.Add(param);
}
}
else
{
attachment = new MimePart(fileType);
}
attachment.FileName = Path.GetFileName(iter);
attachment.IsAttachment = true;
MemoryBlockStream memoryBlockStream = new MemoryBlockStream();
BestEncodingFilter encodingFilter = new BestEncodingFilter();
byte[] fileBuffer = new byte[4096];
int index, length, bytesRead;
while ((bytesRead = stream.Read(fileBuffer, 0, fileBuffer.Length)) > 0)
{
encodingFilter.Filter(fileBuffer, 0, bytesRead, out index, out length);
memoryBlockStream.Write(fileBuffer, 0, bytesRead);
}
encodingFilter.Flush(fileBuffer, 0, 0, out index, out length);
memoryBlockStream.Position = 0;
attachment.ContentTransferEncoding = encodingFilter.GetBestEncoding(EncodingConstraint.SevenBit);
attachment.ContentObject = new ContentObject(memoryBlockStream);
if (attachment != null)
{
email.AttachmentList.Add(attachment);
}
}
return(true);
}
public void TestWriteZeroBytes()
{
byte[] buf = new byte[0];
MemoryBlockStream ms = new MemoryBlockStream();
ms.Write(buf, 0, 0);
Assert.AreEqual(ms.Length, 0);
IRandomSource rng = RandomDefaults.CreateRandomSource(1234567);
byte[] buf2 = new byte[100];
rng.NextBytes(buf2);
ms.Write(buf2, 0, buf2.Length);
if (!Utils.AreEqual(ms.ToArray(), buf2))
{
Assert.Fail();
}
ms.Write(buf, 0, 0);
Assert.AreEqual(ms.Length, buf2.Length);
}
public void TestWriteZeroBytes()
{
byte[] buf = new byte[0];
MemoryBlockStream ms = new MemoryBlockStream();
ms.Write(buf, 0, 0);
Assert.AreEqual(ms.Length, 0);
XorShiftRandom rng = new XorShiftRandom(1234567);
byte[] buf2 = new byte[100];
rng.NextBytes(buf2);
ms.Write(buf2, 0, buf2.Length);
if (!Utils.AreEqual(ms.ToArray(), buf2))
{
Assert.Fail();
}
ms.Write(buf, 0, 0);
Assert.AreEqual(ms.Length, buf2.Length);
}
private void PerformMutationOp_Write()
{
int buffLen = _rng.Next(1000) + 1;
int offset = _rng.Next(buffLen);
int count = _rng.Next(buffLen - offset);
byte[] tmp = new byte[count];
_rng.NextBytes(tmp);
byte[] buff = new byte[buffLen];
Array.Copy(tmp, 0, buff, offset, count);
_strmA.Write(buff, offset, count);
_strmB.Write(buff, offset, count);
Debug.WriteLine(string.Format("Write offset={0}, count={1}", offset, count));
}
void TestMbox(ParserOptions options, string baseName)
{
var mbox = Path.Combine(MboxDataDir, baseName + ".mbox.txt");
var output = new MemoryBlockStream();
var builder = new StringBuilder();
List <MimeParserOffset> offsets;
NewLineFormat newLineFormat;
using (var stream = File.OpenRead(mbox)) {
var parser = options != null ? new CustomMimeParser(options, stream, MimeFormat.Mbox) : new CustomMimeParser(stream, MimeFormat.Mbox);
var format = FormatOptions.Default.Clone();
int count = 0;
format.NewLineFormat = newLineFormat = DetectNewLineFormat(mbox);
while (!parser.IsEndOfStream)
{
var message = parser.ParseMessage();
builder.AppendFormat("{0}", parser.MboxMarker).Append('\n');
if (message.From.Count > 0)
{
builder.AppendFormat("From: {0}", message.From).Append('\n');
}
if (message.To.Count > 0)
{
builder.AppendFormat("To: {0}", message.To).Append('\n');
}
builder.AppendFormat("Subject: {0}", message.Subject).Append('\n');
builder.AppendFormat("Date: {0}", DateUtils.FormatDate(message.Date)).Append('\n');
DumpMimeTree(builder, message);
builder.Append('\n');
var marker = Encoding.UTF8.GetBytes((count > 0 ? format.NewLine : string.Empty) + parser.MboxMarker + format.NewLine);
output.Write(marker, 0, marker.Length);
message.WriteTo(format, output);
count++;
}
offsets = parser.Offsets;
}
AssertMboxResults(baseName, builder.ToString(), output, offsets, newLineFormat);
}
public void TestJwzMbox(int blockSize)
{
var options = FormatOptions.Default.Clone();
options.NewLineFormat = NewLineFormat.Unix;
var builder = new StringBuilder();
var output = new MemoryBlockStream();
using (var stream = File.OpenRead(Path.Combine(MboxDataDir, "jwz.mbox.txt")))
{
var parser = new MimeParser(stream, MimeFormat.Mbox);
int count = 0;
parser.SetBlockSize(blockSize);
while (!parser.IsEndOfStream)
{
var message = parser.ParseMessage();
builder.AppendFormat("{0}", parser.MboxMarker).Append('\n');
if (message.From.Count > 0)
{
builder.AppendFormat("From: {0}", message.From).Append('\n');
}
if (message.To.Count > 0)
{
builder.AppendFormat("To: {0}", message.To).Append('\n');
}
builder.AppendFormat("Subject: {0}", message.Subject).Append('\n');
builder.AppendFormat("Date: {0}", DateUtils.FormatDate(message.Date)).Append('\n');
DumpMimeTree(builder, message);
builder.Append('\n');
var marker = Encoding.UTF8.GetBytes((count > 0 ? "\n" : string.Empty) + parser.MboxMarker + "\n");
output.Write(marker, 0, marker.Length);
message.WriteTo(options, output);
count++;
}
}
AssertJwzMboxResults(builder.ToString(), output);
}
void LoadContent(MimePart attachment, string fileName, byte[] data)
{
var content = new MemoryBlockStream();
var filter = new BestEncodingFilter();
int index, length;
filter.Flush(data, 0, data.Length, out index, out length);
content.Write(data, 0, data.Length);
content.Position = 0;
if (linked)
{
attachment.ContentLocation = new Uri(Path.GetFileName(fileName), UriKind.Relative);
}
attachment.ContentTransferEncoding = filter.GetBestEncoding(EncodingConstraint.SevenBit);
attachment.ContentObject = new ContentObject(content, ContentEncoding.Default);
}
static void LoadContent(MimePart attachment, Stream stream)
{
var content = new MemoryBlockStream();
var filter = new BestEncodingFilter();
var buf = new byte[4096];
int index, length;
int nread;
while ((nread = stream.Read(buf, 0, buf.Length)) > 0)
{
filter.Filter(buf, 0, nread, out index, out length);
content.Write(buf, 0, nread);
}
filter.Flush(buf, 0, 0, out index, out length);
content.Position = 0;
attachment.ContentTransferEncoding = filter.GetBestEncoding(EncodingConstraint.SevenBit);
attachment.ContentObject = new ContentObject(content);
}
public void TestWrite()
{
var bytes = new byte[9 * 1024];
int position = 0;
random.NextBytes(bytes);
blocks.Position = 0;
master.Position = 0;
while (position < bytes.Length)
{
int n = Math.Min(bytes.Length - position, random.Next() % 4096);
blocks.Write(bytes, position, n);
master.Write(bytes, position, n);
position += n;
}
blocks.Flush();
master.Flush();
}
/// <summary>
/// Writes a buffer to the archive.
/// </summary>
/// <param name="p">Pointer to the beginning of the buffer.</param>
/// <param name="count">Number of bytes to write.</param>
public unsafe void Write(IntPtr p, long count)
{
CloseArchiveStream();
// write payload type and size of the following buffer
mSerializer.mTempBuffer_Buffer[0] = (byte)PayloadType.Buffer;
int writtenBytes = Leb128EncodingHelper.Write(mSerializer.mTempBuffer_Buffer, 1, count);
mStream.Write(mSerializer.mTempBuffer_Buffer, 0, writtenBytes + 1);
if (mStream is MemoryBlockStream)
{
// the MemoryBlockStream provides a direct way to write to the underlying buffer more effiently
MemoryBlockStream mbs = mStream as MemoryBlockStream;
while (count > 0)
{
int bytesToCopy = (int)Math.Min(count, int.MaxValue);
mbs.Write(new ReadOnlySpan <byte>(p.ToPointer(), bytesToCopy));
count -= bytesToCopy;
p += bytesToCopy;
}
}
else
{
// some other stream
// => copying data to a temporary buffer is needed before passing it to the stream
if (mSerializer.mTempBuffer_BigBuffer.Length < 256 * 1024)
{
mSerializer.mTempBuffer_BigBuffer = new byte[256 * 1024];
}
while (count > 0)
{
int bytesToCopy = (int)Math.Min(count, mSerializer.mTempBuffer_BigBuffer.Length);
Marshal.Copy(p, mSerializer.mTempBuffer_BigBuffer, 0, bytesToCopy);
mStream.Write(mSerializer.mTempBuffer_BigBuffer, 0, bytesToCopy);
count -= bytesToCopy;
p += bytesToCopy;
}
}
}
static async Task LoadContentAsync(MimePart attachment, Stream stream, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var content = new MemoryBlockStream();
if (attachment.ContentType.IsMimeType("text", "*"))
{
var buf = ArrayPool <byte> .Shared.Rent(BufferLength);
var filter = new BestEncodingFilter();
int index, length;
int nread;
try {
while ((nread = await stream.ReadAsync(buf, 0, BufferLength, cancellationToken).ConfigureAwait(false)) > 0)
{
cancellationToken.ThrowIfCancellationRequested();
filter.Filter(buf, 0, nread, out index, out length);
content.Write(buf, 0, nread);
}
filter.Flush(buf, 0, 0, out index, out length);
} finally {
ArrayPool <byte> .Shared.Return(buf);
}
attachment.ContentTransferEncoding = filter.GetBestEncoding(EncodingConstraint.SevenBit);
}
else
{
attachment.ContentTransferEncoding = ContentEncoding.Base64;
await stream.CopyToAsync(content, 4096, cancellationToken).ConfigureAwait(false);
}
content.Position = 0;
attachment.Content = new MimeContent(content);
}
/// <summary>
/// Internal method handling writing the content of a stream.
/// </summary>
/// <param name="stream">Stream containing data to write (must be seekable).</param>
private void WriteInternal(Stream stream)
{
CloseArchiveStream();
// write payload type and size of the following buffer
long count = stream.Length;
mSerializer.mTempBuffer_Buffer[0] = (byte)PayloadType.Buffer;
int writtenBytes = Leb128EncodingHelper.Write(mSerializer.mTempBuffer_Buffer, 1, count);
mStream.Write(mSerializer.mTempBuffer_Buffer, 0, writtenBytes + 1);
if (mStream is MemoryBlockStream)
{
// the MemoryBlockStream provides a direct way to write to the underlying buffer more effiently
MemoryBlockStream mbs = mStream as MemoryBlockStream;
mbs.Write(stream);
}
else
{
// some other stream
// => copying data to a temporary buffer is needed before passing it to the stream
if (mSerializer.mTempBuffer_BigBuffer.Length < 256 * 1024)
{
mSerializer.mTempBuffer_BigBuffer = new byte[256 * 1024];
}
while (true)
{
int bytesRead = stream.Read(mSerializer.mTempBuffer_BigBuffer, 0, mSerializer.mTempBuffer_BigBuffer.Length);
if (bytesRead == 0)
{
break;
}
mStream.Write(mSerializer.mTempBuffer_BigBuffer, 0, bytesRead);
}
}
}
/// <summary>
/// Writes the contents of a stream to the archive (from the current position up to the end of the stream).
/// </summary>
/// <param name="s">Stream to write.</param>
public void Write(Stream s)
{
CloseArchiveStream();
if (s.CanSeek)
{
// stream can seek
// => stream is usable without additional preparation
WriteInternal(s);
}
else
{
// stream cannot seek
// => read data into memory to make it seekable
using (MemoryBlockStream bufferStream = new MemoryBlockStream())
{
// read stream into temporary buffer
if (mSerializer.mTempBuffer_BigBuffer.Length < 256 * 1024)
{
mSerializer.mTempBuffer_BigBuffer = new byte[256 * 1024];
}
while (true)
{
int bytesRead = s.Read(mSerializer.mTempBuffer_BigBuffer, 0, mSerializer.mTempBuffer_BigBuffer.Length);
if (bytesRead == 0)
{
break;
}
bufferStream.Write(mSerializer.mTempBuffer_BigBuffer, 0, bytesRead);
}
bufferStream.Position = 0;
WriteInternal(bufferStream);
}
}
}
/*
* Creates a ContentObject for MimePart from [content] supplied.
*/
void CreateContentObjectFromContent(Node contentNode, MimePart entity)
{
/*
* Creating stream to hold content.
* Notice, there is no need to add stream to list of streams, since it
* doesn't require closing or being disposed, since its implementation
* is a list of byte[].
*/
var stream = new MemoryBlockStream();
// Applying content object, but first checking type of object, special handling of blob/byte[].
if (contentNode.Value is byte [])
{
// This is byte[] array (blob).
byte [] value = contentNode.Value as byte [];
stream.Write(value, 0, value.Length);
stream.Position = 0;
}
else
{
/*
* Anything BUT byte[].
* Here we rely on conversion Active Events, making "everything else" serialise as strings.
* But first retrieving content, which might be in Hyperlambda format, or an expression.
*/
var content = contentNode.GetExValue <string> (_context, null);
if (content == null)
{
// Before we throw exception we check if contentNode has children, at which point it's Hyperlambda content.
if (contentNode.Count == 0)
{
throw new LambdaException("No [content] in your MIME envelope", _entityNode, _context);
}
// Converting lambda content specified to Hyperlambda and adding it as content.
_context.RaiseEvent("lambda2hyper", contentNode);
content = contentNode.Get <string> (_context);
}
/*
* Creating StreamWriter to make it easier to write content to stream.
* Notice, we cannot close StreamWriter since that'll close the underlaying stream.
*/
var streamWriter = new StreamWriter(stream);
// Writing content to streamWriter.
streamWriter.Write(content);
streamWriter.Flush();
stream.Position = 0;
}
// Retrieving ContentEncoding to use for reading stream.
ContentEncoding encoding = ContentEncoding.Default;
if (contentNode ["Content-Encoding"] != null)
{
encoding = (ContentEncoding)Enum.Parse(typeof(ContentEncoding), contentNode ["Content-Encoding"].GetExValue <string> (_context));
}
// Creating a ContentObject for MimePart from MemoryBlockStream.
entity.Content = new MimeContent(stream, encoding);
}
public void TestJwzMbox()
{
var summary = File.ReadAllText(Path.Combine(MboxDataDir, "jwz-summary.txt")).Replace("\r\n", "\n");
var options = FormatOptions.Default.Clone();
var original = new MemoryBlockStream();
var output = new MemoryBlockStream();
var builder = new StringBuilder();
var expected = new byte[4096];
var buffer = new byte[4096];
int nx, n;
options.NewLineFormat = NewLineFormat.Unix;
using (var stream = File.OpenRead(Path.Combine(MboxDataDir, "jwz.mbox.txt"))) {
var parser = new MimeParser(stream, MimeFormat.Mbox);
int count = 0;
while (!parser.IsEndOfStream)
{
var message = parser.ParseMessage();
builder.AppendFormat("{0}", parser.MboxMarker).Append('\n');
if (message.From.Count > 0)
{
builder.AppendFormat("From: {0}", message.From).Append('\n');
}
if (message.To.Count > 0)
{
builder.AppendFormat("To: {0}", message.To).Append('\n');
}
builder.AppendFormat("Subject: {0}", message.Subject).Append('\n');
builder.AppendFormat("Date: {0}", DateUtils.FormatDate(message.Date)).Append('\n');
DumpMimeTree(builder, message);
builder.Append('\n');
var marker = Encoding.UTF8.GetBytes((count > 0 ? "\n" : string.Empty) + parser.MboxMarker + "\n");
output.Write(marker, 0, marker.Length);
message.WriteTo(options, output);
count++;
}
}
string actual = builder.ToString();
// WORKAROUND: Mono's iso-2022-jp decoder breaks on this input in versions <= 3.2.3 but is fixed in 3.2.4+
string iso2022jp = Encoding.GetEncoding("iso-2022-jp").GetString(Convert.FromBase64String("GyRAOjRGI0stGyhK"));
if (iso2022jp != "佐藤豊")
{
actual = actual.Replace(iso2022jp, "佐藤豊");
}
Assert.AreEqual(summary, actual, "Summaries do not match for jwz.mbox");
using (var stream = File.OpenRead(Path.Combine(MboxDataDir, "jwz.mbox.txt"))) {
using (var filtered = new FilteredStream(original)) {
filtered.Add(new Dos2UnixFilter());
stream.CopyTo(filtered);
filtered.Flush();
}
}
original.Position = 0;
output.Position = 0;
Assert.AreEqual(original.Length, output.Length, "The length of the mbox did not match.");
do
{
var position = original.Position;
nx = original.Read(expected, 0, expected.Length);
n = output.Read(buffer, 0, buffer.Length);
if (nx == 0)
{
break;
}
for (int i = 0; i < nx; i++)
{
if (buffer[i] == expected[i])
{
continue;
}
var strExpected = CharsetUtils.Latin1.GetString(expected, 0, nx);
var strActual = CharsetUtils.Latin1.GetString(buffer, 0, n);
Assert.AreEqual(strExpected, strActual, "The mbox differs at position {0}", position + i);
}
} while (true);
}
