/// <summary>
/// Initializes a new instance of the <see cref="MoveDelayedRecordsCommandHandler" /> class.
/// </summary>
/// <param name="moveDelayedToPendingLua">The move delayed to pending lua.</param>
/// <param name="unixTimeFactory">The unix time factory.</param>
/// <param name="options">The options.</param>
/// <param name="queueContext">The queue context.</param>
public MoveDelayedRecordsCommandHandler(
MoveDelayedToPendingLua moveDelayedToPendingLua,
IUnixTimeFactory unixTimeFactory,
RedisQueueTransportOptions options,
QueueContext queueContext)
{
Guard.NotNull(() => moveDelayedToPendingLua, moveDelayedToPendingLua);
Guard.NotNull(() => unixTimeFactory, unixTimeFactory);
Guard.NotNull(() => options, options);
Guard.NotNull(() => queueContext, queueContext);
_moveDelayedToPendingLua = moveDelayedToPendingLua;
_unixTimeFactory = unixTimeFactory;
_options = options;
_rpcQueue = queueContext.Context == QueueContexts.RpcQueue;
}
public async Task Delay_Message_ThrowsWhenCountersThrows()
{
perfCounters.Setup(c => c.DelayMessage()).Throws(new ApplicationException());
var now = clock.Object.UtcNow;
var context = new QueueContext
{
MessageData = new QueueMessage
{
Id = 12345,
NotBefore = now,
},
SourceQueue = "SourceQueue"
};
await reader.DelayMessage(context, 1000);
}
private static DataProtocolVersion ReadDataProtocolVersion(QueueContext context, Stream inputStream)
{
int protocolVersion = inputStream.ReadByte();
if (protocolVersion == -1)
{
throw new DataFrameException(context, "Unable to read protocol version byte");
}
if (!_supportedProtocols.Contains(protocolVersion))
{
throw new UnsupportedProtocolVersionException(context, (byte)protocolVersion);
}
return((DataProtocolVersion)protocolVersion);
}
private static bool ReadStartByte(QueueContext context, Stream inputStream)
{
int startByte = inputStream.ReadByte();
if (startByte == -1)
{
return(false);
}
if (startByte != START_BYTE)
{
throw new InvalidBoundryByteException(context, START_BYTE, (byte)startByte);
}
return(true);
}
public Response ReturnMesagge(ref Response ret, int idmensje, string custom, Guid?id, string rute, HttpStatusCode status = HttpStatusCode.OK)
{
using (QueueContext ent = new QueueContext())
{
Agent_GenericError ge = (from g in ent.Agent_GenericError
where g.codigo_id == idmensje
select g).SingleOrDefault();
ret.status = status;
ret.response_code = ge.Codigo;
ret.message = ge.Message + " " + custom;
ret.rute = rute;
}
return(ret);
}
public async Task LoadSaga_InitializesWhenNew()
{
var context = new QueueContext
{
};
var testSaga = new TestSaga
{
};
dataAccess.Setup(d => d.GetSagaData(It.IsAny <string>(), It.IsAny <string>()))
.Returns(Task.FromResult <SagaData>(null));
await reader.LoadSaga(testSaga, context);
Assert.IsNotNull(testSaga.Data);
}
/// <summary>
/// Read QueueEqntry from DataFile frame stream. This method is thread-safe.
/// </summary>
/// <exception cref="InvalidCrcException">This exception is thrown if crc calculated from stream does nto equal crc written in the file</exception>
/// <exception cref="UnsupportedProtocolVersionException">This exception is thrown if stream data indicate to not implemented protocol</exception>
/// <exception cref="InvalidBoundryByteException">This exception is thrown if values of start or end byte have wrong values. This might indicate that while reading, application poiints to wrong place in the stream</exception>
/// <exception cref="DataFrameException">This exception is thrown if frame is not up to specification</exception>
public QueueEntry ReadEntry(QueueContext context, Stream inputStream)
{
#warning TEST
if (!ReadStartByte(context, inputStream))
{
return(null);
}
DataProtocolVersion dataProtocolVersion = ReadDataProtocolVersion(context, inputStream);
int messageLength = ReadMessageLength(context, inputStream);
byte[] messageBuffer = ReadMessageBuffer(context, inputStream, messageLength);
uint crc = ReadCrc(context, inputStream);
CheckCrc(context, messageBuffer, crc);
ReadEndByte(context, inputStream);
return(_dataProtocolFactory.GetProtocol(dataProtocolVersion).GetEntry(messageBuffer));
}
/// <summary>
/// Initializes a new instance of the <see cref="RollbackMessageCommandHandler" /> class.
/// </summary>
/// <param name="configuration">The configuration.</param>
/// <param name="resetHeartbeatLua">The reset heartbeat lua.</param>
/// <param name="unixTimeFactory">The unix time factory.</param>
/// <param name="options">The options.</param>
/// <param name="queueContext">The queue context.</param>
public ResetHeartBeatCommandHandler(IHeartBeatConfiguration configuration,
ResetHeartbeatLua resetHeartbeatLua,
IUnixTimeFactory unixTimeFactory,
RedisQueueTransportOptions options,
QueueContext queueContext)
{
Guard.NotNull(() => configuration, configuration);
Guard.NotNull(() => resetHeartbeatLua, resetHeartbeatLua);
Guard.NotNull(() => unixTimeFactory, unixTimeFactory);
Guard.NotNull(() => options, options);
Guard.NotNull(() => queueContext, queueContext);
_configuration = configuration;
_resetHeartbeatLua = resetHeartbeatLua;
_unixTimeFactory = unixTimeFactory;
_options = options;
}
public SaveSchemaQueueProcessor(QueueContext queueContext) : base(queueContext)
{
_uploadUrl = Config.Urls.First();
_mainMappingUploadRelativeUrl = queueContext.MainMappingUploadRelativeUrl;
_secondaryMappingUploadRelativeUrl = queueContext.SecondaryMappingUploadRelativeUrl;
_bulkUploadRelativeUrl = queueContext.BulkUploadRelativeUrl;
if (_uploadUrl.Last() == '/')
{
_uploadUrl = _uploadUrl.Substring(1, _uploadUrl.Length - 1);
}
if (_mainMappingUploadRelativeUrl.First() != '/')
{
_mainMappingUploadRelativeUrl = '/' + _mainMappingUploadRelativeUrl;
}
}
public ActionResult All()
{
ListViewModel mymodel = new ListViewModel();
using (var db = new QueueContext())
{
string id = User.Identity.GetUserId();
var query = (from tblQueue in db.Queues
orderby tblQueue.CreatedDate descending
where (tblQueue.UserID == id)
select tblQueue);
mymodel.Queue = query.ToList();
return(View(mymodel));
}
}
public async Task Delay_ThrowsWhenDataAccessThrows()
{
dataAccess.Setup(d => d.Update(It.IsAny <QueueMessage>(), It.IsAny <string>()))
.Throws(new ApplicationException());
var now = clock.Object.UtcNow;
var context = new QueueContext
{
MessageData = new QueueMessage
{
Id = 12345,
NotBefore = now,
},
SourceQueue = "SourceQueue"
};
await reader.DelayMessage(context, 1000);
}
public object Login(object key)
{
///cambios del dia 23/03/2019
using (QueueContext ent = new QueueContext())
{
Response rp = new Response();
try
{
Agent_Empresa ae = ent.Agent_Empresa.Where(a => a.Key == key && a.status == true).SingleOrDefault();
if (ae != null)
{
var token = tvh.GenerateToken(ae.Nombre, ae.Rut, ae.IdCompany.ToString());
ResponseConfigurationDTO responseConfigurationDTO = (from c in ent.Agent_Configuration
where c.Agent_Empresa.IdCompany == ae.IdCompany
select new ResponseConfigurationDTO
{
Id_Configuration = c.Id_Configuration,
InactivityPeriod = c.InactivityPeriod,
UploadFrecuency = c.UploadFrecuency,
CaptureFrecuency = c.CaptureFrecuency,
token = token,
IsLogged = true
}).FirstOrDefault();
rp.data = responseConfigurationDTO;
}
else
{
//login invalido
return(autil.ReturnMesagge(ref rp, (int)GenericErrors.ErrorLogin, string.Empty, null));;
}
//retorna un response, con el campo data lleno con la respuesta.
return(autil.ReturnMesagge(ref rp, (int)GenericErrors.OK, null, null, HttpStatusCode.OK));
}
catch (Exception ex)
{
//error general
return(autil.ReturnMesagge(ref rp, (int)GenericErrors.GeneralError, string.Empty, null, HttpStatusCode.InternalServerError));
}
}
}
public async Task Completee_Completes()
{
var now = clock.Object.UtcNow;
var context = new QueueContext
{
MessageData = new QueueMessage
{
Id = 12345,
NotBefore = now,
},
SourceQueue = "SourceQueue"
};
await reader.Complete(context);
Assert.IsTrue(ReferenceEquals(context.MessageData, CompletedMessage));
Assert.AreEqual(now, context.MessageData.Completed);
Assert.AreEqual(CompletedQueue, "SourceQueue");
}
public object GetConfiguration(ConfigurationModel model)
{
Response rp = new Response();
try
{
using (QueueContext ent = new QueueContext())
{
var agent_Configuration = ent.Agent_Configuration.Where(r => r.Agent_Empresa.IdCompany == model.Id_Empresa).FirstOrDefault();
rp.data = _mapper.Map <ResponseConfigurationDTO>(agent_Configuration);
}
//retorna un response, con el campo data lleno con la respuesta.
return(autil.ReturnMesagge(ref rp, (int)GenericErrors.OK, null, null, HttpStatusCode.OK));
}
catch (Exception ex)
{
//error general
return(autil.ReturnMesagge(ref rp, (int)GenericErrors.GeneralError, string.Empty, null, HttpStatusCode.InternalServerError));
}
}
public async Task Delay_DelaysMessage()
{
var now = clock.Object.UtcNow;
var context = new QueueContext
{
MessageData = new QueueMessage
{
Id = 12345,
NotBefore = now,
},
SourceQueue = "SourceQueue"
};
await reader.DelayMessage(context, 1000);
var expectedTime = now.AddMilliseconds(1000);
Assert.IsTrue(ReferenceEquals(context.MessageData, UpdatedMessage));
Assert.AreEqual(expectedTime, context.MessageData.NotBefore);
Assert.AreEqual(UpdatedQueue, "SourceQueue");
}
/// <summary>
/// A message handler that starts the sample saga.
/// </summary>
/// <param name="context"></param>
/// <param name="message"></param>
/// <returns></returns>
public async Task Handle(QueueContext context, SampleSagaStart message)
{
_log.Info($"Starting Tasks for SagaId {message.AppId}");
// record in our saga data how many distributed tasks that need to be compelted before
// the saga is regarded as complete.
Data.PendingTasks = 10;
// send a distributed task out to be worked.
await _queueWriter.WriteMessage(context.SourceQueue,
new SampleDistributedTaskRequest
{
AppId = message.AppId,
A = (Data.PendingTasks - 1) * 3,
B = (Data.PendingTasks - 1) * 4,
Operation = "+"
});
// interact with out application specific database.
await _dataAccess.Audit("Saga Started.");
}
private static int ReadMessageLength(QueueContext context, Stream inputStream)
{
const int BYTES_TO_READ = 4;
byte[] messageLengthBuffer = new byte[BYTES_TO_READ];
int readCount = inputStream.Read(messageLengthBuffer, 0, BYTES_TO_READ);
if (readCount != BYTES_TO_READ)
{
throw new DataFrameException(context, $"Unable to read {BYTES_TO_READ} bytes of message length");
}
int messageLength = BitConverter.ToInt32(messageLengthBuffer, 0);
if (messageLength == 0)
{
throw new DataFrameException(context, "Message length in frame is 0");
}
return(messageLength);
}
public async Task LoadSaga_ReturnsWhenBlocked()
{
var context = new QueueContext
{
};
var testSaga = new TestSaga
{
};
dataAccess.Setup(d => d.GetSagaData(It.IsAny <string>(), It.IsAny <string>()))
.Returns(Task.FromResult(new SagaData
{
Blocked = true,
Data = "{}",
Key = "SagaKey",
}));
await reader.LoadSaga(testSaga, context);
Assert.IsNull(testSaga.Data);
Assert.IsTrue(context.SagaData.Blocked);
}
public async Task SaveSaga_DeletesCompleteSaga()
{
var context = new QueueContext
{
MessageData = new QueueMessage
{
Id = 12345,
NotBefore = clock.Object.UtcNow,
},
SourceQueue = "SourceQueue",
SagaKey = "SagaKey"
};
var testSaga = new TestSaga
{
SagaComplete = true
};
await reader.SaveSaga(testSaga, context);
dataAccess.Verify(d => d.DeleteSagaData(testSaga.SagaName, context.SagaKey), Times.Once);
}
/// <summary>
/// Create queue entry from REST body request. Size check made.
/// </summary>
public static QueueEntry FromRequestString(QueueContext context, string bodyRequestString, int dataFileMaximumSizeB)
{
JObject request = JObject.Parse(bodyRequestString);
if (!request.TryGetValue(nameof(QueueEntry.Tag), StringComparison.OrdinalIgnoreCase, out JToken token))
{
throw new ArgumentException($"No {nameof(QueueEntry.Tag)} found in input JSON");
}
byte[] dataBytes = Encoding.UTF8.GetBytes(bodyRequestString);
int frameSize = DataProtocolAdapter.CalculateFrameSize(dataBytes.Length);
if (frameSize > dataFileMaximumSizeB)
{
throw new TooBigRequestException(context, dataFileMaximumSizeB, dataBytes.Length, frameSize);
}
return(new QueueEntry
{
Data = bodyRequestString,
Tag = token.Value <string>(),
DataBytes = Encoding.UTF8.GetBytes(bodyRequestString)
});
}
public void ContextsAreSingletons()
{
Fixture fixture = new Fixture();
string name1 = fixture.Create <string>();
string name2 = fixture.Create <string>();
QueueContextFactory factory = new QueueContextFactory();
QueueContext context1 = factory.GetContext(name1);
QueueContext context2 = factory.GetContext(name2);
QueueContext context3 = factory.GetContext(name2);
QueueContext context4 = factory.GetContext(name1);
QueueContext context5 = factory.GetContext(name1);
Assert.IsNotNull(context1);
Assert.IsNotNull(context2);
Assert.IsNotNull(context3);
Assert.IsNotNull(context4);
Assert.IsNotNull(context5);
Assert.AreSame(context1, context4);
Assert.AreSame(context1, context5);
Assert.AreSame(context2, context3);
}
public DataFrameException([NotNull] QueueContext context, string message)
: base(context, message)
{
}
public UserRepository(QueueContext context)
{
_context = context;
}
public Task Handle(QueueContext context, NotASagaStartMessage message)
{
throw new NotImplementedException();
}
public Task Handle(QueueContext context, TestSagaMessage1 message)
{
// do nothing.
return(Task.CompletedTask);
}
private IList <Task> RunAsync(Func <IBaseQueueProcessor> fnQueueProcessor, int count, QueueContext queueContext)
{
IList <Task> tasks = new List <Task>();
_stepNumber++;
var thisStepNumber = _stepNumber;
bool isFirst = true;
for (var i = 0; i < count; i++)
{
var queueProcessor = fnQueueProcessor();
if (isFirst)
{
queueProcessor.CreateOutQueue(thisStepNumber);
isFirst = false;
}
queueProcessor.InitializeWithStepNumber(thisStepNumber);
var task =
Task.Factory.StartNew((o) =>
{
queueProcessor.MonitorWorkQueue();
return(1);
}, thisStepNumber);
tasks.Add(task);
}
var taskArray = tasks.ToArray();
Task.WhenAll(taskArray).ContinueWith(a =>
fnQueueProcessor().MarkOutputQueueAsCompleted(thisStepNumber)
);
return(tasks);
}
public TerminalTicketRepository(QueueContext context) : base(context, context.TerminalTickets)
{
}
public InvalidBoundryByteException(QueueContext context, byte boundryByteExpected, byte boundryByteActual)
: base(context, $"Invalid boundry byte found")
{
BoundryByteExpected = boundryByteExpected;
BoundryByteActual = boundryByteActual;
}
/// <summary>
/// The run pipeline.
/// </summary>
/// <param name="job">
/// The job.
/// </param>
/// <param name="progressMonitor">
/// The progress monitor.
/// </param>
public void RunPipeline(Job job, IProgressMonitor progressMonitor)
{
var config = job.Config;
if (config.WriteTemporaryFilesToDisk)
{
FileSaveQueueProcessor.CleanOutputFolder(config.LocalSaveFolder);
}
var documentDictionary =
new MeteredConcurrentDictionary <string, IJsonObjectQueueItem>(MaximumDocumentsInQueue);
var stopwatch = new Stopwatch();
stopwatch.Start();
CancellationTokenSource cts = new CancellationTokenSource();
var queueContext = new QueueContext
{
Config = config,
QueueManager = new QueueManager(),
ProgressMonitor = progressMonitor,
BulkUploadRelativeUrl = $"/{config.Index}/{config.EntityType}/_bulk?pretty",
MainMappingUploadRelativeUrl = $"/{config.Index}",
SecondaryMappingUploadRelativeUrl = $"/{config.Index}/_mapping/{config.EntityType}",
PropertyTypes = job.Data.DataSources.Where(a => a.Path != null).ToDictionary(a => a.Path, a => a.PropertyType),
DocumentDictionary = documentDictionary
};
int loadNumber = 0;
foreach (var load in job.Data.DataSources)
{
load.SequenceNumber = ++loadNumber;
}
if (config.DropAndReloadIndex)
{
this.ReadAndSetSchema(config, queueContext, job);
}
var sqlBatchQueue = queueContext.QueueManager
.CreateInputQueue <SqlBatchQueueItem>(this.stepNumber + 1);
if (queueContext.Config.EntitiesPerBatch <= 0)
{
sqlBatchQueue.Add(new SqlBatchQueueItem
{
BatchNumber = 1,
Start = null,
End = null,
Loads = job.Data.DataSources,
});
}
else
{
var ranges = CalculateRanges(config, job);
int currentBatchNumber = 1;
foreach (var range in ranges)
{
sqlBatchQueue.Add(new SqlBatchQueueItem
{
BatchNumber = currentBatchNumber++,
Start = range.Item1,
End = range.Item2,
Loads = job.Data.DataSources,
});
}
}
sqlBatchQueue.CompleteAdding();
var tasks = new List <Task>();
// ReSharper disable once JoinDeclarationAndInitializer
IList <Task> newTasks;
newTasks = this.RunAsync(() => new SqlBatchQueueProcessor(queueContext), 1);
tasks.AddRange(newTasks);
newTasks = this.RunAsync(() => new SqlImportQueueProcessor(queueContext), 2);
tasks.AddRange(newTasks);
newTasks = this.RunAsync(() => new ConvertDatabaseRowToJsonQueueProcessor(queueContext), 1);
tasks.AddRange(newTasks);
newTasks = this.RunAsync(() => new JsonDocumentMergerQueueProcessor(queueContext), 1);
tasks.AddRange(newTasks);
newTasks = this.RunAsync(() => new CreateBatchItemsQueueProcessor(queueContext), 2);
tasks.AddRange(newTasks);
newTasks = this.RunAsync(() => new SaveBatchQueueProcessor(queueContext), 2);
tasks.AddRange(newTasks);
if (config.WriteTemporaryFilesToDisk)
{
newTasks = this.RunAsync(() => new FileSaveQueueProcessor(queueContext), 2);
tasks.AddRange(newTasks);
}
if (config.UploadToElasticSearch)
{
newTasks = this.RunAsync(() => new FileUploadQueueProcessor(queueContext), 1);
tasks.AddRange(newTasks);
}
Task.WaitAll(tasks.ToArray());
var stopwatchElapsed = stopwatch.Elapsed;
stopwatch.Stop();
Console.WriteLine(stopwatchElapsed);
}
public QueueController(QueueContext context)
{
_context = context;
}
