public AppendHelper(IList<ArraySegment<byte>> data,
SequenceNumber prev,
SequenceNumber next,
bool restartArea)
{
this.prev = prev;
this.next = next;
this.header = new FileLogRecordHeader(null);
this.header.IsRestartArea = restartArea;
this.header.PreviousLsn = prev;
this.header.NextUndoLsn = next;
this.blobs = new UnmanagedBlob[data.Count + 1];
this.handles = new GCHandle[data.Count + 1];
try
{
this.handles[0] = GCHandle.Alloc(header.Bits, GCHandleType.Pinned);
this.blobs[0].cbSize = (uint)FileLogRecordHeader.Size;
this.blobs[0].pBlobData = Marshal.UnsafeAddrOfPinnedArrayElement(header.Bits, 0);
for (int i = 0; i < data.Count; i++)
{
handles[i + 1] = GCHandle.Alloc(data[i].Array, GCHandleType.Pinned);
blobs[i + 1].cbSize = (uint)data[i].Count;
blobs[i + 1].pBlobData = Marshal.UnsafeAddrOfPinnedArrayElement(data[i].Array, data[i].Offset);
}
}
catch
{
Dispose();
throw;
}
}
public FileRecordSequenceCompletedAsyncResult(
SequenceNumber result,
AsyncCallback callback,
object userState,
Work work)
{
this.result = result;
this.callback = callback;
this.userState = userState;
this.work = work;
this.syncRoot = new object();
if (this.callback != null)
{
try
{
this.callback(this);
}
#pragma warning suppress 56500 // This is a callback exception
catch (Exception e)
{
if (Fx.IsFatal(e))
throw;
throw DiagnosticUtility.ExceptionUtility.ThrowHelperCallback(e);
}
}
}
public Data(EntityId readerId, EntityId writerId, long seqNum, ParameterList inlineQosParams, SerializedPayload payload)
: base(SubMessageKind.DATA)
{
this.readerId = readerId;
this.writerId = writerId;
this.writerSN = new SequenceNumber(seqNum);
if (inlineQosParams != null && inlineQosParams.Value.Count > 0)
{
Header.FlagsValue |= 0x2;
this.inlineQosParams = inlineQosParams;
}
if (payload.ContainsData())
{
Header.FlagsValue |= Flags.DataFlag; // dataFlag
}
else
{
Header.FlagsValue |= Flags.KeyFlag; // keyFlag
}
this.serializedPayload = payload;
}
public Task Initialize()
{
this.logger.Log("Initializing StateProvider.");
// Rebuild the local progress vector.
var records = this.serializer.Deserialize<Record>(this.progressVectorLog.GetAllRecords());
var progressUpdates = records.OfType<EpochUpdatedRecord>();
foreach (var progressUpdate in progressUpdates)
{
this.logger.Log(
$"{nameof(StateProvider)}.{nameof(this.Initialize)}: got progress {progressUpdate.Progress}");
this.progressVector.Update(progressUpdate.Progress);
}
this.highestCommittedLogSequenceNumber = this.progressVector.SequenceNumber;
// Update the highest committed sequence number with the persitent state.
var lastRecord = this.operationLog.GetLastRecord();
if (lastRecord != null)
{
this.prevRecord = lastRecord.SequenceNumber;
this.logger.Log($"LastRecord: {lastRecord.DetailedString()}");
var opLogEntry = this.serializer.Deserialize<Record>(lastRecord) as OperationCommittedRecord;
if (opLogEntry != null)
{
this.logger.Log($"LastRecord detail: {opLogEntry}");
this.UpdateHighestLogSequenceNumber(opLogEntry.Version.LogSequenceNumber);
}
}
this.logger.Log("Initialized StateProvider.");
return Task.FromResult(0);
#warning find a more scalable approach so we can avoid copying entire progress vector into memory.
}
public Data(EntityId readerId, EntityId writerId, long seqNum, ParameterList inlineQosParams, DataEncapsulation dEnc)
: base(SubMessageKind.DATA)
{
this.readerId = readerId;
this.writerId = writerId;
this.writerSN = new SequenceNumber(seqNum);
if (inlineQosParams != null && inlineQosParams.Count > 0)
{
Header.FlagsValue |= 0x2;
this.inlineQosParams = inlineQosParams;
}
if (dEnc.ContainsData())
{
Header.FlagsValue |= Flags.DataFlag; // dataFlag
}
else
{
Header.FlagsValue |= Flags.KeyFlag; // keyFlag
}
this.dataEncapsulation = dEnc;
}
private NetworkMessage(string RoundID, MessageType Type, SequenceNumber SequenceNo, string Value, Guid InResponseTo, Dictionary<string, string> DataSet = null)
{
this.RoundID = RoundID;
this.Type = Type;
this.SequenceNumber = SequenceNo;
this.Value = Value;
ReplyID = InResponseTo;
this.DataSet = DataSet;
}
internal FileRecordSequenceHelper(SimpleFileLog log)
{
this.log = log;
this.appendLock = new ReaderWriterLock();
this.truncateFailed = false;
this.lastRestartArea = SequenceNumber.Invalid;
this.newBaseSeqNum = SequenceNumber.Invalid;
Recover();
}
internal FileLogRecordStream(SimpleFileLog log, SequenceNumber recordSequenceNumber)
{
this.log = log;
this.recordSequenceNumber = recordSequenceNumber;
this.stream = null;
this.recordSize = 0;
this.entireRecordRead = false;
this.streamLock = new object();
CreateMemoryStream();
}
// Helper routine to convert the sequence number to a string representation.
public static string SequenceNumberToString(SequenceNumber sqn)
{
byte[] bytes = sqn.GetBytes();
long realLsnHigh = BitConverter.ToInt64(bytes, 0);
if (bytes.Length == 8)
{
return realLsnHigh.ToString();
}
else
{
long realLsnLow = BitConverter.ToInt64(bytes, 8);
return realLsnLow.ToString() + "-" + realLsnHigh.ToString();
}
}
static void Main(string[] args)
{
ImapClient client = new Email.Net.Imap.ImapClient();
client.Host = "host";
client.Username = "******";
client.Password = "******";
client.Port = (ushort)993;
client.AttachmentDirectory = "PathForAttachments";
client.SSLInteractionType = EInteractionType.SSLPort;
Mailbox inbox;
ImapMessage msg;
CompletionResponse response = client.Login();
if (response.CompletionResult == ECompletionResponseType.OK)
{
Console.WriteLine("We're In");
Mailbox folders = client.GetMailboxTree();
inbox = Mailbox.Find(folders, "INBOX");
Console.WriteLine("{0} Messages", client.GetMessageCount(inbox));
MessageCollection tmp = client.GetAllMessageHeaders(inbox);
foreach(ImapMessage x in tmp)
{
msg = client.GetMessageText(x.UID, inbox);
if (msg.Attachments.Count > 0)
{
for (int i = 0; i <msg.Attachments.Count; i++)
{
Attachment attach = msg.Attachments[i];
if (attach != null)
{
Attachment tmpAttach = new Attachment();
tmpAttach = client.GetAttachment(inbox, msg, msg.Attachments[i]);
string tmpName = tmpAttach.FullFilename.ToString();
string realName = client.AttachmentDirectory + msg.Attachments[i].TransferFilename.ToString();
File.Move(tmpName, realName);
Console.WriteLine("Attachment: {0}", realName);
}
}
ISequence squence = new SequenceNumber(x.UID);
CompletionResponse delResponse = client.MarkAsDeleted(squence, inbox);
delResponse = client.DeleteMarkedMessages(inbox);
}
}
}
}
unsafe internal LogLogRecord(
SequenceNumber sequenceNumber,
SequenceNumber user,
SequenceNumber previous,
byte* data,
long length)
{
this.sequenceNumber = sequenceNumber;
this.previous = previous;
this.user = user;
if (length < LogLogRecordHeader.Size)
{
throw DiagnosticUtility.ExceptionUtility.ThrowHelperError(Error.LogCorrupt());
}
byte[] headerBits = new byte[LogLogRecordHeader.Size];
Marshal.Copy(new IntPtr(data),
headerBits,
0,
LogLogRecordHeader.Size);
LogLogRecordHeader header = new LogLogRecordHeader(headerBits);
if (header.MajorVersion > LogLogRecordHeader.CurrentMajorVersion)
{
throw DiagnosticUtility.ExceptionUtility.ThrowHelperError(Error.IncompatibleVersion());
}
length -= LogLogRecordHeader.Size;
data += LogLogRecordHeader.Size;
if (header.Padding)
{
long paddingSize = LogLogRecordHeader.DecodePaddingSize(data, length);
if ((paddingSize < 0) || (paddingSize > length))
{
throw DiagnosticUtility.ExceptionUtility.ThrowHelperError(Error.LogCorrupt());
}
length -= paddingSize;
data += paddingSize;
}
this.stream = new RecordStream(
new UnmanagedMemoryStream(data, length));
}
public void TestSequenceNumbers()
{
var def = new SequenceNumber();
var small = new SequenceNumber(1, "B");
var big = new SequenceNumber(1000, "A");
var bigger = new SequenceNumber(1000, "B");
Assert.IsTrue(bigger > big);
Assert.IsTrue(big < bigger);
Assert.IsTrue(bigger > small);
Assert.IsTrue(small < bigger);
Assert.IsTrue(bigger > def);
Assert.IsTrue(def < bigger);
Assert.IsTrue(big > small);
Assert.IsTrue(small < big);
Assert.IsTrue(big > def);
Assert.IsTrue(def < big);
Assert.IsTrue(small > def);
Assert.IsTrue(def < small);
Assert.AreEqual("[1:B]", small.ToString(), "Should have a neat string representation");
Assert.AreEqual(small, new SequenceNumber("[1:B]"), "Should be able to parse string representation");
Assert.AreNotEqual(new SequenceNumber(), null);
Assert.AreNotEqual(null, new SequenceNumber());
Assert.AreNotEqual(new SequenceNumber(), new Object());
Assert.AreNotEqual(new Object(), new SequenceNumber());
Assert.AreEqual(small.GetHashCode(), new SequenceNumber(1, "B").GetHashCode());
Assert.IsTrue(small != big);
Assert.IsTrue(small <= big);
Assert.IsTrue(small <= new SequenceNumber(1, "B"));
Assert.IsFalse(big <= small);
Assert.IsTrue(big >= small);
Assert.IsTrue(small >= new SequenceNumber(1, "B"));
Assert.IsFalse(small >= big);
}
public static Task<SequenceNumber> AppendAsync(
this LogRecordSequence sequence,
ArraySegment<byte> data,
SequenceNumber nextUndoRecord,
SequenceNumber previousRecord,
RecordAppendOptions recordAppendOptions)
{
return
Task.Factory.FromAsync(
(callback, state) =>
((LogRecordSequence)state).BeginAppend(
data,
nextUndoRecord,
previousRecord,
recordAppendOptions,
callback,
state),
sequence.EndAppend,
sequence);
}
public static Task<SequenceNumber> AppendAsync(
this LogRecordSequence sequence,
IList<ArraySegment<byte>> data,
SequenceNumber userRecord,
SequenceNumber previousRecord,
RecordAppendOptions recordAppendOptions,
ReservationCollection reservationCollection)
{
return
Task.Factory.FromAsync(
(callback, state) =>
((LogRecordSequence)state).BeginAppend(
data,
userRecord,
previousRecord,
recordAppendOptions,
reservationCollection,
callback,
state),
sequence.EndAppend,
sequence);
}
public static void Encode(XdrDataOutputStream stream, SequenceNumber encodedSequenceNumber)
{
Int64.Encode(stream, encodedSequenceNumber.InnerValue);
}
internal SequenceNumber Flush(SequenceNumber sequenceNumber)
{
TruncateIfNecessary();
if (sequenceNumber == SequenceNumber.Invalid)
{
// Re-interpret... SimpleFileLog uses 0 to mean "flush
// entire log", not SequenceNumber.Invalid.
//
sequenceNumber = new SequenceNumber(0);
}
else
{
ValidateSequenceNumber(sequenceNumber);
}
this.log.Force(sequenceNumber);
return sequenceNumber;
}
internal SequenceNumber Append(IList<ArraySegment<byte>> data,
SequenceNumber nextUndoRecord,
SequenceNumber previousRecord,
bool forceFlush)
{
TruncateIfNecessary();
bool lockHeld = false;
try
{
try { }
finally
{
appendLock.AcquireReaderLock(-1);
lockHeld = true;
}
using (AppendHelper helper = new AppendHelper(
data,
previousRecord,
nextUndoRecord,
false))
{
return log.AppendRecord(helper.Blobs, forceFlush);
}
}
finally
{
if (lockHeld)
{
appendLock.ReleaseReaderLock();
}
}
}
public override int GetHashCode()
{
// why 397?
return((EventIndex * 397) ^ SequenceNumber.GetHashCode());
}
public static void PutSequenceNumber(this IoBuffer buffer, SequenceNumber obj)
{
buffer.PutInt32(obj.High);
buffer.PutInt32((int)obj.Low);
}
public static SequenceNumber GetSequenceNumber(this IoBuffer buffer)
{
SequenceNumber obj = new SequenceNumber();
buffer.GetSequenceNumber(ref obj);
return obj;
}
public SequenceNumber WriteRestartArea(
IList<ArraySegment<byte>> data,
SequenceNumber newBaseSeqNum,
ReservationCollection reservations)
{
long size = 0;
if (data == null)
{
throw DiagnosticUtility.ExceptionUtility.ThrowHelperError(Error.ArgumentNull("data"));
}
if (reservations == null)
return WriteRestartArea(data, newBaseSeqNum);
FileReservationCollection reservationCollection = reservations as FileReservationCollection;
if (reservationCollection == null || !reservationCollection.IsMyCollection(this))
{
throw DiagnosticUtility.ExceptionUtility.ThrowHelperError(Error.ArgumentInvalid(SR.LogRecSeq_InvalidReservationCollection));
}
for (int i = 0; i < data.Count; i++)
{
size = checked(size + data[i].Count);
}
long reservation = reservationCollection.GetBestMatchingReservation(size);
if (reservation < 0)
{
throw DiagnosticUtility.ExceptionUtility.ThrowHelperError(Error.ReservationNotFound());
}
bool throwing = true;
try
{
SequenceNumber returnValue = WriteRestartArea(data, newBaseSeqNum);
throwing = false;
return returnValue;
}
finally
{
if (throwing)
{
reservationCollection.Add(reservation);
}
}
}
public SequenceNumber WriteRestartArea(
IList<ArraySegment<byte>> data,
SequenceNumber newBaseSeqNum)
{
if (newBaseSeqNum == SequenceNumber.Invalid)
{
newBaseSeqNum = this.BaseSequenceNumber;
}
try
{
return this.frsHelper.WriteRestartAreaInternal(data, newBaseSeqNum);
}
catch (SequenceFullException)
{
RaiseTailPinnedEvent();
if (this.RetryAppend)
{
return this.frsHelper.WriteRestartAreaInternal(data, newBaseSeqNum);
}
else
{
throw;
}
}
}
internal SequenceNumber WriteRestartAreaInternal(
IList<ArraySegment<byte>> data,
SequenceNumber newBaseSeqNum)
{
if (data == null)
{
throw DiagnosticUtility.ExceptionUtility.ThrowHelperError(Error.ArgumentNull("data"));
}
TruncateIfNecessary();
bool lockHeld = false;
try
{
try { }
finally
{
appendLock.AcquireWriterLock(-1);
lockHeld = true;
}
SequenceNumber sn;
AppendHelper helper;
bool lastSeqNum, firstSeqNum;
lastSeqNum = newBaseSeqNum == this.LastSequenceNumber;
firstSeqNum = newBaseSeqNum == this.BaseSequenceNumber;
if (lastSeqNum)
{
// We dont know the new base sequence number.
// It will be the sequence number we get from append.
helper = new AppendHelper(data,
this.lastRestartArea,
SequenceNumber.Invalid,
true);
}
else
{
if (!firstSeqNum)
{
ValidateSequenceNumber(newBaseSeqNum);
// Validate newBaseSequenceNumber by reading the corresponding record.
int cbData = 1;
byte[] record;
int recordSize;
SequenceNumber prev, next;
// Sequence number validation checks if the Seq Num > BSN and Seq Num < LSN.
// Now we validate if Seq Number is a valid number in this sequence.
try
{
this.log.ReadRecordPrefix(newBaseSeqNum, out record, ref cbData, out recordSize, out prev, out next);
}
catch (ArgumentException exception)
{
throw DiagnosticUtility.ExceptionUtility.ThrowHelperError(Error.ArgumentOutOfRange(SR.Argument_TailInvalid, exception));
}
}
helper = new AppendHelper(data,
this.lastRestartArea,
newBaseSeqNum,
true);
}
// If there is a failure betweeen append and truncate,
// then the log will be in an inconsistent state. To
// solve this, we write the new base sequence number in
// the header. During recovery from failure, we will find
// the restart area and truncate the log.
using (helper)
{
// No need for a reader appendLock, since WriteRestartAreaInternal is under a lock.
sn = this.log.AppendRecord(helper.Blobs, true);
}
this.lastRestartArea = sn;
if (firstSeqNum)
{
return sn;
}
if (lastSeqNum)
{
newBaseSeqNum = sn;
}
try
{
log.TruncatePrefix(newBaseSeqNum);
}
#pragma warning suppress 56500
catch (Exception exception)
{
this.newBaseSeqNum = newBaseSeqNum;
this.truncateFailed = true;
if (Fx.IsFatal(exception)) throw;
}
return sn;
}
finally
{
if (lockHeld)
{
appendLock.ReleaseWriterLock();
}
}
}
// During WriteRestarArea, we truncate all records before the new base sequence number.
// The new base sequence number is recorded in the restart-area record.
// If truncate failed during WriteRestartArea, then the records before the new base seq number
// will still be present in the log. During recovery, we will cleanup the log by removing these records.
// Recovery steps -
// Scan the log backwards
// Stop when the last restart area is found
// Truncate the log if needed
private void Recover()
{
SequenceNumber first;
SequenceNumber last;
this.log.GetLogLimits(out first, out last);
// Internal knowledge - if last < first, log is empty
if (last < first)
return;
SequenceNumber sn = last;
while (sn != SequenceNumber.Invalid && first <= sn && sn <= last)
{
FileLogRecordStream stream = new FileLogRecordStream(log, sn);
if (stream.Header.IsRestartArea)
{
this.lastRestartArea = stream.RecordSequenceNumber;
// if the base sequence number is different from
// the next undo lsn, then we crashed during or
// before truncate. Perform the truncate now.
if (first < stream.Header.NextUndoLsn)
{
if (stream.Header.NextUndoLsn == SequenceNumber.Invalid)
{
// WriteRestartArea was called with LastSequenceNumber
if (first != stream.RecordSequenceNumber)
{
this.newBaseSeqNum = stream.RecordSequenceNumber;
}
}
else
{
this.newBaseSeqNum = stream.Header.NextUndoLsn;
}
// This method is called from the constructor. So no need to take a write lock.
if (this.newBaseSeqNum != SequenceNumber.Invalid)
{
try
{
log.TruncatePrefix(this.newBaseSeqNum);
this.newBaseSeqNum = SequenceNumber.Invalid;
}
#pragma warning suppress 56500
catch (Exception exception)
{
// Truncate failed again. We were unable to cleanup the log.
this.truncateFailed = true;
if (Fx.IsFatal(exception)) throw;
}
}
}
break;
}
sn = stream.PrevLsn;
}
}
internal override void OnAttach(uint remoteHandle, Attach attach)
{
if (role)
{
this.deliveryCount = attach.InitialDeliveryCount;
}
var container = ((ListenerConnection)this.Session.Connection).Listener.Container;
Error error = null;
try
{
bool done = container.AttachLink((ListenerConnection)this.Session.Connection, (ListenerSession)this.Session, this, attach);
if (!done)
{
return;
}
}
catch (AmqpException amqpException)
{
error = amqpException.Error;
}
catch (Exception exception)
{
error = new Error() { Condition = ErrorCode.InternalError, Description = exception.Message };
}
this.CompleteAttach(attach, error);
}
public static void GetSequenceNumber(this IoBuffer buffer, ref SequenceNumber obj)
{
obj.High = buffer.GetInt32();
obj.Low = (uint)buffer.GetInt32(); ;
}
static SequenceNumber AppendRecord(IRecordSequence sequence, string message, SequenceNumber user, SequenceNumber previous)
{
MemoryStream data = new MemoryStream();
BinaryWriter writer = new BinaryWriter(data);
writer.Write(message); ArraySegment <byte>[] segments;
segments = new ArraySegment <byte> [1];
segments[0] = new ArraySegment <byte>(data.GetBuffer(), 0, (int)data.Length);
return(sequence.Append(segments, user, previous, RecordAppendOptions.None));
}
public async Task AppendOperation(ArraySegment<byte> record, long logSequenceNumber)
{
this.prevRecord =
await
this.operationLog.Append(
record,
this.prevRecord,
SequenceNumber.Invalid,
RecordAppendOptions.ForceFlush);
this.UpdateHighestLogSequenceNumber(logSequenceNumber);
}
internal void ValidateSequenceNumber(SequenceNumber sequenceNumber)
{
if (sequenceNumber == SequenceNumber.Invalid || sequenceNumber.Low != 0)
{
throw DiagnosticUtility.ExceptionUtility.ThrowHelperError(Error.SequenceNumberInvalid());
}
if (sequenceNumber < this.BaseSequenceNumber || sequenceNumber > this.LastSequenceNumber)
{
throw DiagnosticUtility.ExceptionUtility.ThrowHelperError(Error.SequenceNumberNotActive("sequenceNumber"));
}
}
/// <summary>
/// Completes the link attach request. This should be called when the IContainer.AttachLink implementation returns false
/// and the asynchronous processing completes.
/// </summary>
/// <param name="attach">The attach to send back.</param>
/// <param name="error">The error, if any, for the link.</param>
public void CompleteAttach(Attach attach, Error error)
{
if (error != null)
{
this.SendAttach(!attach.Role, attach.InitialDeliveryCount, new Attach() { Target = null, Source = null });
}
else
{
if (!role)
{
this.deliveryCount = attach.InitialDeliveryCount;
}
this.SendAttach(!attach.Role, attach.InitialDeliveryCount, new Attach() { Target = attach.Target, Source = attach.Source });
}
base.OnAttach(attach.Handle, attach);
if (error != null)
{
this.Close(0, error);
}
else
{
if (this.credit > 0)
{
this.SendFlow(this.deliveryCount, this.credit, false);
}
}
}
internal void AdvanceBaseSequeceNumber(SequenceNumber newBaseSequenceNumber)
{
TruncateIfNecessary();
bool lockHeld = false;
try
{
try { }
finally
{
appendLock.AcquireWriterLock(-1);
lockHeld = true;
}
ValidateSequenceNumber(newBaseSequenceNumber);
this.log.TruncatePrefix(newBaseSequenceNumber);
}
finally
{
if (lockHeld)
{
appendLock.ReleaseWriterLock();
}
}
}
public void AdvanceBaseSequenceNumber(
SequenceNumber newBaseSequenceNumber)
{
this.frsHelper.AdvanceBaseSequeceNumber(newBaseSequenceNumber);
}
public override string ToString()
{
return(string.Format("{2} timer check at {0}, elapsed {1}{3}.", Timestamp.ToLongTimeString(), FromBeginning.ToString(), SequenceNumber.ToOrdinal(), EventName == null ? "" : " " + EventName));
}
public TailPinnedEventArgs(SequenceNumber sequenceNumber)
{
this.sequenceNumber = sequenceNumber;
}
public MessageId(TerminalId terminalId, SequenceNumber sequenceNumber, CheckNumber checkNumber)
{
TerminalId = terminalId;
SequenceNumber = sequenceNumber;
CheckNumber = checkNumber;
}