public static async Task Init(string storageAccountName, string storageAccountKey)
{
remoteFileIO = new RemoteFileIO(storageAccountName, storageAccountKey);
localFileIO = new LocalFileIO(storageAccountName, storageAccountKey);
// Open a connection to the Edge runtime
ModuleClient ioTHubModuleClient = await ModuleClient.CreateFromEnvironmentAsync(TransportType.Mqtt);
await ioTHubModuleClient.OpenAsync().ConfigureAwait(false);
// Register callback to be called when a message is received by the module
await ioTHubModuleClient.SetInputMessageHandlerAsync("UploadFile", UploadFileMessage, ioTHubModuleClient);
await ioTHubModuleClient.SetInputMessageHandlerAsync("DownloadFile", DownloadFileMessage, ioTHubModuleClient);
await ioTHubModuleClient.SetInputMessageHandlerAsync("RemoveFile", RemoveFileMessage, ioTHubModuleClient);
await ioTHubModuleClient.SetInputMessageHandlerAsync("WriteFile", WriteFileMessage, ioTHubModuleClient);
await ioTHubModuleClient.SetInputMessageHandlerAsync("GetFile", GetFileMessage, ioTHubModuleClient);
await ioTHubModuleClient.SetInputMessageHandlerAsync("ListFiles", ListFilesMessage, ioTHubModuleClient);
Console.WriteLine("EdgeHub logging module has been initialized");
}
public void ListFiles()
{
var io = new LocalFileIO("foo", "bar");
var files = io.ListFiles(".").Result;
Assert.IsTrue(files.Length > 0);
Assert.IsTrue(File.Exists(files[0]));
}
public void WriteAndDeleteFile_FileGone()
{
var io = new LocalFileIO("foo", "bar");
io.WriteFile(".", "test2.txt", GenerateTestBytes(), false).Wait();
io.RemoveFile(".", "test2.txt").Wait();
Assert.IsFalse(File.Exists(".\\test2.txt"));
}
public void WriteAndReadFile_RoundtripData()
{
var io = new LocalFileIO("foo", "bar");
io.WriteFile(".", "test1.txt", GenerateTestBytes(), false).Wait();
var outFileBytes = io.GetFile(".", "test1.txt").Result;
Assert.AreEqual("hello tests!", Encoding.UTF8.GetString(outFileBytes));
}
public async void Should_GetByteArray_When_FileIsFound()
{
// arrange
var io = new LocalFileIO("", "");
// act
var result = await io.GetFile("/", "test.txt");
// assert
Assert.Equal(2, result.Length);
}
public async void Should_GetException_When_FileIsNotFound()
{
// arrange
var io = new LocalFileIO("", "");
// act
try
{
var result = await io.GetFile("/", "file.txt");
Assert.Equal(0, result.Length);
}
catch (FileNotFoundException e)
{
Assert.Equal(null, null);
}
}
/// <summary>
/// 实际执行操作
/// </summary>
/// <param name="interval"></param>
public void Execute(int interval)
{
try
{
//休息片刻
if (interval > 0)
{
int step = (int)Math.Round(interval * 0.3);
int interval2 = new Random().Next(interval - step, interval + step);
Thread.Sleep(interval2);
}
Thread.Sleep(30 * 1000);
//采集图书基本信息数据/ id,标题,简介
currenttask.Url = BaseUrl;
currenttask.PageType = PageTypeEnum.StartupPage;
currenttask.Load(_CollectionModel.IsUTF8);
//采集完成后的操作--数据填充到ConfigModel中去
buildModel();
//写文件name和intro
LocalFileIO.writeName(this._CollectionModel.Name, this.sortName);
LocalFileIO.writeIntro(this._CollectionModel.Name, this._CollectionModel.Intr, this.sortName);
//如果存在章节列表地址并且当前章节列表数据为空的话,那么进行采集章节列表操作,说明上次采集到的数据已经被对比处理
while (!string.IsNullOrEmpty(this.CurrentUrl_ChapterList))
{
//采集章节列表数据--要一次性将章节列表数据采集完存放到CollectionModel中
currenttask.Url = CurrentUrl_ChapterList; //设置采集的章节列表页url--初始url
//开始执行采集操作
//currenttask.DoChapterList(_CollectionModel.IsUTF8);
currenttask.PageType = PageTypeEnum.ListPage1;
currenttask.Load(_CollectionModel.IsUTF8);
//采集完成后的操作--数据填充到ConfigModel中去
buildModel();
}
//章节列表数据采集完成---请求我们的图书数据,查看当前我们自己线上数据中图书章节状态信息
foreach (var item in this._CollectionModel.chapterList)
{
int intervalX = new Random().Next(10 * 1000, 60 * 1000); //每次休息一定时间
Thread.Sleep(intervalX);
//开始采集本章节内容数据
this.currentUrl_ChapterDetail = item.chapterUrl;
while (!string.IsNullOrEmpty(this.currentUrl_ChapterDetail))
{
currenttask.Url = this.currentUrl_ChapterDetail;
//currenttask.DoChapterDetail(_CollectionModel.IsUTF8);
currenttask.Load(_CollectionModel.IsUTF8);
//采集完成后的操作--数据填充到Model中去
buildModel(item);
}
//publishBiz.doSubmitChapter(nextChapter);
//保存本章数据
string realcontent = item.Content.Replace("<br>", System.Environment.NewLine)
.Replace("<p>", System.Environment.NewLine)
.Replace("</p>", System.Environment.NewLine)
.Replace(" ", " ")
;
LocalFileIO.writeChpater(this._CollectionModel.Name, item.Title, realcontent, this.sortName);
Log.Show(string.Format("<C> {0} 写入章节数据: {1} ", _CollectionModel.Name, item.Title), ConsoleColor.DarkGreen);
}
//一次完整的采集操作完成,现在要清理数据间隔一个比较长的时间之后再次
Log.Show(string.Format("<C{0}> {1} 完成一次完整流程,暂时没有更新,休息24小时再次检查 ", _CollectionModel.CollectionId, _CollectionModel.Name), ConsoleColor.DarkGreen);
//清理数据
this._CollectionModel.chapterList = new List <wxchapter>();
this.CurrentUrl_ChapterList = this._CollectionModel.Url;
}
catch (Exception ex)
{
Log.Show(string.Format("<C{0}> {1} error: {2} ", _CollectionModel.CollectionId, _CollectionModel.Name, ex.Message), ConsoleColor.DarkRed);
}
}
