public Task Dispatch(DashboardContext context)
{
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
var consoleId = ConsoleId.Parse(context.UriMatch.Groups[1].Value);
var startArg = context.Request.GetQuery("start");
// try to parse offset at which we should start returning requests
int start;
if (string.IsNullOrEmpty(startArg) || !int.TryParse(startArg, out start))
{
// if not provided or invalid, fetch records from the very start
start = 0;
}
var buffer = new StringBuilder();
using (var data = new ConsoleStorage(context.Storage.GetConnection()))
{
ConsoleRenderer.RenderLineBuffer(buffer, data, consoleId, start);
}
context.Response.ContentType = "text/html";
return(context.Response.WriteAsync(buffer.ToString()));
}
static void Main(string[] args)
{
WeatherRequester weatherKyiv = new WeatherRequester("0867b08df07d2f5a09b6db1f31a7fffb", "Kyiv");
WeatherRequester weatherLviv = new WeatherRequester("0867b08df07d2f5a09b6db1f31a7fffb", "Lviv");
WeatherHandler weatherForecastKyiv = new WeatherHandler();
WeatherHandler weatherForecastLviv = new WeatherHandler();
string path = @"c:\temp\" + DateTime.Today.ToString("yyyyMMdd") + ".txt";
FileStorage file = new FileStorage(path);
ConsoleStorage console = new ConsoleStorage();
weatherForecastKyiv.Subscribe(file);
weatherForecastKyiv.Subscribe(console);
IDisposable unsubWeatherForecastKyiv = weatherForecastKyiv.Subscribe(file);
IDisposable unsubWeatherForecastKyiv2 = weatherForecastKyiv.Subscribe(console);
weatherForecastLviv.Subscribe(file);
weatherForecastLviv.Subscribe(console);
IDisposable unsubWeatherForecastLviv = weatherForecastKyiv.Subscribe(file);
IDisposable unsubWeatherForecastLviv2 = weatherForecastKyiv.Subscribe(console);
for (int i = 0; i < 10; i++)
{
weatherForecastKyiv.ChangeForecast(weatherKyiv.GetForecast());
weatherForecastLviv.ChangeForecast(weatherLviv.GetForecast());
// unsubWeatherForecastKyiv.Dispose();
//unsubWeatherForecastKyiv2.Dispose();
}
Console.ReadLine();
}
public NonEscapedString Render(HtmlHelper helper, IDictionary <string, string> stateData)
{
var builder = new StringBuilder();
builder.Append("<dl class=\"dl-horizontal\">");
string serverId = null;
if (stateData.ContainsKey("ServerId"))
{
serverId = stateData["ServerId"];
}
else if (stateData.ContainsKey("ServerName"))
{
serverId = stateData["ServerName"];
}
if (serverId != null)
{
builder.Append("<dt>Server:</dt>");
builder.Append($"<dd>{helper.ServerId(serverId)}</dd>");
}
if (stateData.ContainsKey("WorkerId"))
{
builder.Append("<dt>Worker:</dt>");
builder.Append($"<dd>{stateData["WorkerId"].Substring(0, 8)}</dd>");
}
else if (stateData.ContainsKey("WorkerNumber"))
{
builder.Append("<dt>Worker:</dt>");
builder.Append($"<dd>#{stateData["WorkerNumber"]}</dd>");
}
builder.Append("</dl>");
var page = helper.GetPage();
if (page.RequestPath.StartsWith("/jobs/details/"))
{
// We cannot cast page to an internal type JobDetailsPage to get jobId :(
var jobId = page.RequestPath.Substring("/jobs/details/".Length);
var startedAt = JobHelper.DeserializeDateTime(stateData["StartedAt"]);
var consoleId = new ConsoleId(jobId, startedAt);
builder.Append("<div class=\"console-area\">");
builder.AppendFormat("<div class=\"console\" data-id=\"{0}\">", consoleId);
using (var storage = new ConsoleStorage(page.Storage.GetConnection()))
{
ConsoleRenderer.RenderLineBuffer(builder, storage, consoleId, 0);
}
builder.Append("</div>");
builder.Append("</div>");
}
return(new NonEscapedString(builder.ToString()));
}
public void GetLines_ReturnsRangeFromOldSet_ForBackwardsCompatibility()
{
var lines = new[] {
new ConsoleLine {
TimeOffset = 0, Message = "line1"
},
new ConsoleLine {
TimeOffset = 1, Message = "line2"
},
new ConsoleLine {
TimeOffset = 2, Message = "line3"
},
new ConsoleLine {
TimeOffset = 3, Message = "line4"
},
};
_connection.Setup(x => x.GetRangeFromSet(_consoleId.GetOldConsoleKey(), It.IsAny <int>(), It.IsAny <int>()))
.Returns((string key, int start, int end) => lines.Where((x, i) => i >= start && i <= end).Select(JobHelper.ToJson).ToList());
var storage = new ConsoleStorage(_connection.Object);
var result = storage.GetLines(_consoleId, 1, 2).ToArray();
Assert.Equal(lines.Skip(1).Take(2).Select(x => x.Message), result.Select(x => x.Message));
}
public void GetProgress_ReturnsNull_IfProgressNotPresent()
{
var storage = new ConsoleStorage(_connection.Object);
var result = storage.GetProgress(_consoleId);
Assert.Null(result);
}
public void Dispose_ReallyDisposesConnection()
{
var storage = new ConsoleStorage(_connection.Object);
storage.Dispose();
_connection.Verify(x => x.Dispose());
}
public void InitConsole_JobIdIsAddedToHash()
{
var storage = new ConsoleStorage(_connection.Object);
storage.InitConsole(_consoleId);
_connection.Verify(x => x.SetRangeInHash(_consoleId.GetHashKey(), It.IsAny <IEnumerable <KeyValuePair <string, string> > >()));
}
public void AddLine_ThrowsException_IfLineIsReference()
{
var storage = new ConsoleStorage(_connection.Object);
Assert.Throws <ArgumentException>("line", () => storage.AddLine(_consoleId, new ConsoleLine()
{
IsReference = true
}));
}
public void Expire_ExpiresOldSetAndHashKeysEither_ForBackwardsCompatibility()
{
var storage = new ConsoleStorage(_connection.Object);
storage.Expire(_consoleId, TimeSpan.FromHours(1));
_transaction.Verify(x => x.ExpireSet(_consoleId.ToString(), It.IsAny <TimeSpan>()));
_transaction.Verify(x => x.ExpireHash(_consoleId.ToString(), It.IsAny <TimeSpan>()));
}
public void Expire_ExpiresSetAndHash()
{
var storage = new ConsoleStorage(_connection.Object);
storage.Expire(_consoleId, TimeSpan.FromHours(1));
_transaction.Verify(x => x.ExpireSet($"console:{_consoleId}", It.IsAny <TimeSpan>()));
_transaction.Verify(x => x.ExpireHash($"console:refs:{_consoleId}", It.IsAny <TimeSpan>()));
}
public void InitConsole_JobIdIsAddedToHash()
{
var storage = new ConsoleStorage(_connection.Object);
storage.InitConsole(_consoleId);
_connection.Verify(x => x.CreateWriteTransaction(), Times.Once);
_transaction.Verify(x => x.SetRangeInHash(_consoleId.GetHashKey(), It2.AnyIs <KVP>(p => p.Key == "jobId")));
_transaction.Verify(x => x.Commit(), Times.Once);
}
public void GetConsoleTtl_ReturnsTtlOfHash()
{
_connection.Setup(x => x.GetHashTtl(_consoleId.GetHashKey()))
.Returns(TimeSpan.FromSeconds(123));
var storage = new ConsoleStorage(_connection.Object);
var ttl = storage.GetConsoleTtl(_consoleId);
Assert.Equal(TimeSpan.FromSeconds(123), ttl);
}
public void Expire_ExpiresSetAndHash()
{
var storage = new ConsoleStorage(_connection.Object);
storage.Expire(_consoleId, TimeSpan.FromHours(1));
_connection.Verify(x => x.CreateWriteTransaction(), Times.Once);
_transaction.Verify(x => x.ExpireSet(_consoleId.GetSetKey(), It.IsAny <TimeSpan>()));
_transaction.Verify(x => x.ExpireHash(_consoleId.GetHashKey(), It.IsAny <TimeSpan>()));
_transaction.Verify(x => x.Commit(), Times.Once);
}
public void GetLineCount_ReturnsCountOfOldSet_WhenNewOneReturnsZero_ForBackwardsCompatibility()
{
_connection.Setup(x => x.GetSetCount(_consoleId.GetSetKey())).Returns(0);
_connection.Setup(x => x.GetSetCount(_consoleId.GetOldConsoleKey())).Returns(123);
var storage = new ConsoleStorage(_connection.Object);
var count = storage.GetLineCount(_consoleId);
Assert.Equal(123, count);
}
public void GetLineCount_ReturnsCountOfSet()
{
_connection.Setup(x => x.GetSetCount(_consoleId.GetSetKey()))
.Returns(123);
var storage = new ConsoleStorage(_connection.Object);
var count = storage.GetLineCount(_consoleId);
Assert.Equal(123, count);
}
public void InitConsole_ThrowsException_IfNotImplementsJobStorageTransaction()
{
var dummyTransaction = new Mock <IWriteOnlyTransaction>();
_connection.Setup(x => x.CreateWriteTransaction())
.Returns(dummyTransaction.Object);
var storage = new ConsoleStorage(_connection.Object);
Assert.Throws <NotSupportedException>(() => storage.InitConsole(_consoleId));
}
public void Expire_ExpiresOldSetAndHashKeysEither_ForBackwardsCompatibility()
{
var storage = new ConsoleStorage(_connection.Object);
storage.Expire(_consoleId, TimeSpan.FromHours(1));
_connection.Verify(x => x.CreateWriteTransaction(), Times.Once);
_transaction.Verify(x => x.ExpireSet(_consoleId.GetOldConsoleKey(), It.IsAny <TimeSpan>()));
_transaction.Verify(x => x.ExpireHash(_consoleId.GetOldConsoleKey(), It.IsAny <TimeSpan>()));
_transaction.Verify(x => x.Commit(), Times.Once);
}
public void GetProgress_ReturnsNull_IfValueIsInvalid()
{
_connection.Setup(x => x.GetValueFromHash(It.IsAny <string>(), It.IsIn("progress")))
.Returns("null");
var storage = new ConsoleStorage(_connection.Object);
var result = storage.GetProgress(_consoleId);
Assert.Null(result);
}
public void GetProgress_ReturnsProgressValue()
{
const double progress = 12.5;
_connection.Setup(x => x.GetValueFromHash(It.IsAny <string>(), It.IsIn("progress")))
.Returns(progress.ToString(CultureInfo.InvariantCulture));
var storage = new ConsoleStorage(_connection.Object);
var result = storage.GetProgress(_consoleId);
Assert.Equal(progress, result);
}
public void AddLine_ShortLineIsAddedToSet()
{
var storage = new ConsoleStorage(_connection.Object);
var line = new ConsoleLine()
{
Message = "test"
};
storage.AddLine(_consoleId, line);
Assert.False(line.IsReference);
_transaction.Verify(x => x.AddToSet($"console:{_consoleId}", It.IsAny <string>()));
_transaction.Verify(x => x.SetRangeInHash($"console:refs:{_consoleId}", It.IsAny <IEnumerable <KeyValuePair <string, string> > >()), Times.Never);
}
public void AddLine_ShortLineIsAddedToSet()
{
var storage = new ConsoleStorage(_connection.Object);
var line = new ConsoleLine()
{
Message = "test"
};
storage.AddLine(_consoleId, line);
Assert.False(line.IsReference);
_connection.Verify(x => x.CreateWriteTransaction(), Times.Once);
_transaction.Verify(x => x.AddToSet(_consoleId.GetSetKey(), It.IsAny <string>(), It.IsAny <double>()));
_transaction.Verify(x => x.SetRangeInHash(_consoleId.GetHashKey(), It.IsAny <IEnumerable <KVP> >()), Times.Never);
_transaction.Verify(x => x.Commit(), Times.Once);
}
public void GetState_ReturnsStateData()
{
var state = new StateData()
{
Name = ProcessingState.StateName,
Data = new Dictionary <string, string>()
};
_connection.Setup(x => x.GetStateData(It.IsAny <string>()))
.Returns(state);
var storage = new ConsoleStorage(_connection.Object);
var result = storage.GetState(_consoleId);
Assert.Same(state, result);
}
public void AddLine_ProgressBarIsAddedToSet_AndProgressIsUpdated()
{
var storage = new ConsoleStorage(_connection.Object);
var line = new ConsoleLine()
{
Message = "1",
ProgressValue = 10
};
storage.AddLine(_consoleId, line);
Assert.False(line.IsReference);
_connection.Verify(x => x.CreateWriteTransaction(), Times.Once);
_transaction.Verify(x => x.AddToSet(_consoleId.GetSetKey(), It.IsAny <string>(), It.IsAny <double>()));
_transaction.Verify(x => x.SetRangeInHash(_consoleId.GetHashKey(), It2.AnyIs <KVP>(p => p.Key == "progress")));
_transaction.Verify(x => x.Commit(), Times.Once);
}
public async Task Dispatch(DashboardContext context)
{
if (!"POST".Equals(context.Request.Method, StringComparison.OrdinalIgnoreCase))
{
context.Response.StatusCode = (int)HttpStatusCode.MethodNotAllowed;
return;
}
var result = new Dictionary <string, double>();
var jobIds = await context.Request.GetFormValuesAsync("jobs[]");
if (jobIds.Count > 0)
{
// there are some jobs to process
using (var connection = context.Storage.GetConnection())
using (var storage = new ConsoleStorage(connection))
{
foreach (var jobId in jobIds)
{
var state = connection.GetStateData(jobId);
if (state != null && string.Equals(state.Name, ProcessingState.StateName, StringComparison.OrdinalIgnoreCase))
{
var consoleId = new ConsoleId(jobId, JobHelper.DeserializeDateTime(state.Data["StartedAt"]));
var progress = storage.GetProgress(consoleId);
if (progress.HasValue)
{
result[jobId] = progress.Value;
}
}
else
{
// return -1 to indicate the job is not in Processing state
result[jobId] = -1;
}
}
}
}
var serialized = JsonConvert.SerializeObject(result, JsonSettings);
context.Response.ContentType = "application/json";
await context.Response.WriteAsync(serialized);
}
public void AddLine_LongLineIsAddedToHash_AndReferenceIsAddedToSet()
{
var storage = new ConsoleStorage(_connection.Object);
var line = new ConsoleLine()
{
Message = "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor " +
"incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud " +
"exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure " +
"dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. " +
"Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."
};
storage.AddLine(_consoleId, line);
Assert.True(line.IsReference);
_transaction.Verify(x => x.AddToSet($"console:{_consoleId}", It.IsAny <string>()));
_transaction.Verify(x => x.SetRangeInHash($"console:refs:{_consoleId}", It.IsAny <IEnumerable <KeyValuePair <string, string> > >()));
}
public void GetLines_ExpandsReferencesFromOldHash_ForBackwardsCompatibility()
{
var lines = new[] {
new ConsoleLine {
TimeOffset = 0, Message = "line1", IsReference = true
}
};
_connection.Setup(x => x.GetRangeFromSet(_consoleId.GetOldConsoleKey(), It.IsAny <int>(), It.IsAny <int>()))
.Returns((string key, int start, int end) => lines.Where((x, i) => i >= start && i <= end).Select(JobHelper.ToJson).ToList());
_connection.Setup(x => x.GetValueFromHash(_consoleId.GetOldConsoleKey(), It.IsAny <string>()))
.Returns("Dereferenced Line");
var storage = new ConsoleStorage(_connection.Object);
var result = storage.GetLines(_consoleId, 0, 1).Single();
Assert.False(result.IsReference);
Assert.Equal("Dereferenced Line", result.Message);
}
public void GetLines_HandlesHashException_WhenTryingToExpandReferences()
{
var lines = new[] {
new ConsoleLine {
TimeOffset = 0, Message = "line1", IsReference = true
}
};
_connection.Setup(x => x.GetRangeFromSet(_consoleId.GetOldConsoleKey(), It.IsAny <int>(), It.IsAny <int>()))
.Returns((string key, int start, int end) => lines.Where((x, i) => i >= start && i <= end).Select(JobHelper.ToJson).ToList());
_connection.Setup(x => x.GetValueFromHash(_consoleId.GetOldConsoleKey(), It.IsAny <string>()))
.Throws(new NotSupportedException());
var storage = new ConsoleStorage(_connection.Object);
var result = storage.GetLines(_consoleId, 0, 1).Single();
Assert.False(result.IsReference);
Assert.Equal("line1", result.Message);
}
static void Main(string[] args)
{
//string sourceFilePath = @"..\..\..\..\Files\StockPositions1.csv";
//string sourceFilePath = @"..\..\..\..\Files\StockPositions2.json";
//string sourceFilePath = @"http://solid.wincubate.net/stockpositions.json";
//string destinationFilePath = @"..\..\..\..\Files\Result.csv";
//string destinationFilePath = @"..\..\..\..\Files\Result.json";
//_writeStorage = new FileStorage( @"..\..\..\..\Files\StockPositions1.csv",@"..\..\..\..\Files\Result.json" );
//_storage = new FileStorage();
//_parser = new Parser();
//_serializer = new CsvSerializer();
IReadStorage readStorage = new WebStorage(@"http://solid.wincubate.net/stockpositions.json");
IWriteStorage writeStorage = new ConsoleStorage();
Parser parser = new JsonParser();
ISerializer serializer = new JsonSerializer();
StockAnalyzer analyzer = new StockAnalyzer(
readStorage, writeStorage, parser, serializer
);
analyzer.Process();
}
/// <summary>
/// 启动
/// </summary>
/// <param name="request">抓取请求</param>
/// <param name="parser">页面分析器</param>
/// <param name="storage">数据存储器,,默认显示在控制台</param>
public void Start(List <Request> request, List <IDataFlow> parser, IDataFlow storage = null)
{
Spider _spider = _provider.Create <Spider>();
if (null == storage)
{
storage = new ConsoleStorage();
}
_spider.NewGuidId(); // 设置任务标识
_spider.Name = "测试采集"; // 设置任务名称
_spider.Speed = 10; // 设置采集速度, 表示每秒下载多少个请求, 大于 1 时越大速度越快, 小于 1 时越小越慢, 不能为0.
_spider.Depth = 3; // 设置采集深度
if (parser != null)
{
foreach (IDataFlow item in parser)
{
_spider.AddDataFlow(item);
}
}
_spider.AddDataFlow(storage);
_spider.AddRequests(request.ToArray()); // 设置链接
_spider.RunAsync(); // 启动
}
public void AddLine_ThrowsException_IfLineIsNull()
{
var storage = new ConsoleStorage(_connection.Object);
Assert.Throws <ArgumentNullException>("line", () => storage.AddLine(_consoleId, null));
}