public void Execute() { T lastCurrentPosition = default(T); var last = Manifest.GetLast(); foreach (var extract in Extracter.Extract(last)) { if (extract != null) { var current = extract.CurrentPosition; // avoid loops if ((!(current.Equals(lastCurrentPosition))) && (extract.IsSuccess)) { Loader.EnsureNext(current); foreach (var transformation in Transformer.Transform(extract)) { if (transformation != null) { Loader.Load(transformation); transformation.CleanUp(); } } extract.CleanUp(); Manifest.Update(current); lastCurrentPosition = current; } } } }
private void Test_Simple() { // Act PackTool.Process(new PackTool.Options { ProjectFile = TestHelper.GetDataPath("Simple.json"), OutputDirectory = TestHelper.GetOutputPath() }); // Assert var unpackPath = TestHelper.CreateOutputPath("Unpack"); Extracter.ExtractUnityPackage(TestHelper.GetOutputPath() + "/Simple.1.0.0.unitypackage", unpackPath, "Simple", true, true); var packagePath = Path.Combine(unpackPath, "Assets", "UnityPackages", "Simple"); AssertFileExistsWithMeta(packagePath, "../Simple.unitypackage.json"); AssertFileExistsWithMeta(packagePath, "Text1.txt"); AssertFileExistsWithMeta(packagePath, "Text2.txt"); AssertFileExistsWithMeta(packagePath, "SubDir", "TextInSubDir.txt"); AssertFileExists(unpackPath, "Assets", "UnityPackages.meta"); AssertFileExists(unpackPath, "Assets", "UnityPackages", "Simple.meta"); AssertFileExists(unpackPath, "Assets", "UnityPackages", "Simple", "SubDir.meta"); }
private void Test_FileItem() { // Act PackTool.Process(new PackTool.Options { ProjectFile = TestHelper.GetDataPath("FileItem.json"), OutputDirectory = TestHelper.GetOutputPath() }); // Assert var unpackPath = TestHelper.CreateOutputPath("Unpack"); Extracter.ExtractUnityPackage(TestHelper.GetOutputPath() + "/FileItem.1.0.0.unitypackage", unpackPath, "FileItem", true, true); var basePath = Path.Combine(unpackPath, "Assets", "UnityPackages"); AssertFileExistsWithMeta(basePath, "FileItem.unitypackage.json"); AssertFileExistsWithMeta(basePath, "1/Text.txt"); AssertFileExistsWithMeta(basePath, "2/Text.txt"); AssertFileExists(unpackPath, "Assets", "UnityPackages.meta"); AssertFileExists(unpackPath, "Assets", "UnityPackages", "1.meta"); AssertFileExists(unpackPath, "Assets", "UnityPackages", "2.meta"); }
public void Test_Extract_Default() { var source = new Mock <ISourceSpecification>(); source.Setup(x => x.GetDiscriminatingFilePairs()).Returns(() => { return(new [] { default(Pair) }); }); var downloader = new Mock <IDownloader>(); downloader.Setup(x => x.TryDownload(default(FormatObject))).Returns(default(TryDownloadResult)); downloader.SetupAllProperties(); var target = new Extracter() { TempDownloadLocation = TempLocation, Source = source.Object, Downloader = downloader.Object, }; var expected = new FileNameExtractionResult[] { }; var actual = target.Extract(default(FormatObject)); Assert.IsTrue(actual.SequenceEqual(expected.Cast <IExtractionResult <FormatObject> >())); // twice - as attempts to load the default datetime twice in a row downloader.Verify(x => x.TryDownload(default(FormatObject)), Times.Exactly(2)); }
//[WebApiCacheAttribute(Duration = 10)] public ExtractResult Extract([FromBody] string json) { var node = ServerManager.Get(Request.RequestUri.Authority); if (node.NodeType == Node.NodeTypeEnum.EXTRACTERPROXY) { var result = ExtracterManager.Instance.Elect(); if (result == null) { return(new ExtractResult()); } var client = new RestClient("http://" + result.BaseUrl); var restRequest = new RestRequest("api/extract"); restRequest.Method = Method.POST; restRequest.JsonSerializer = new NewtonJsonSerializer(); restRequest.AddJsonBody(json); restRequest.Timeout = 15000; var restResponse = client.Execute(restRequest); var response = JsonConvert.DeserializeObject <ExtractResult>(restResponse.Content); return(response); } else { var request = JsonConvert.DeserializeObject <ExtractRequest>(json); return(Extracter.Extract(request)); } }
private void Test_MergeDependencies() { // Arrange PackTool.Process(new PackTool.Options { ProjectFile = TestHelper.GetDataPath("DepA.json"), OutputDirectory = TestHelper.GetOutputPath() }); // Act PackTool.Process(new PackTool.Options { ProjectFile = TestHelper.GetDataPath("DepB-Full.json"), OutputDirectory = TestHelper.GetOutputPath(), LocalRepositoryDirectory = TestHelper.GetOutputPath() }); // Assert var unpackPath = TestHelper.CreateOutputPath("Unpack"); Extracter.ExtractUnityPackage(TestHelper.GetOutputPath() + "/DepB.1.0.0.unitypackage", unpackPath, "DepB", true, true); }
public void Extracter_Extract_Default() { var format = new FormatObject("test", DateTime.Now, TimeZoneInfo.Utc); var source = new Mock <ISourceSpecification>(); source.Setup(x => x.GetUpperBoundExclusive(It.IsAny <FormatObject>())).Returns <FormatObject>(y => y.DateTime.AddDays(1)); var downloader = new Mock <IDownloader>(); downloader.Setup(x => x.TryDownload(default(FormatObject))).Returns(default(TryDownloadResult)); downloader.SetupAllProperties(); var formatIterator = new Mock <IFormatIterator>(); formatIterator.Setup(x => x.GetNextCandidates(It.IsAny <FormatObject>())) .Returns <FormatObject>(y => { if (y.DateTime == default(FormatObject).DateTime) { return(new[] { format }); } else { var ret = y; ret.DateTime.AddDays(1); return(new[] { ret }); } }); var downloadAggregator = new DownloadAggregator() { NumberOfFilesToAggregate = 2, Source = source.Object, Downloader = downloader.Object, FormatIterator = formatIterator.Object }; downloadAggregator.Initialize(); var target = new Extracter() { Source = source.Object, TempDownloadLocation = TempLocation, Aggregator = downloadAggregator }; var expected = new FileNameExtractionResult[] { }; var actual = target.Extract(default(FormatObject)); AreEqual(expected, actual.Cast <FileNameExtractionResult>()); formatIterator.Verify(x => x.GetNextCandidates(default(FormatObject)), Times.Exactly(1)); downloader.Verify(x => x.TryDownload(format), Times.Exactly(1)); }
public void Test_Extract_OnlyStopsWhenCurrentDate() { var source = new Mock <ISourceSpecification>(); source.Setup(x => x.GetDiscriminatingFilePairs()).Returns(() => { return(new[] { default(Pair) }); }); var formatObject1 = new FormatObject("test", DateTime.Now.AddDays(-2)); var formatObject2 = new FormatObject("test", DateTime.Now.AddDays(-1)); var downloader = new Mock <IDownloader>(); downloader.Setup(x => x.TryDownload(default(FormatObject))).Returns( new TryDownloadResult() { IsSuccess = false, DestinationFile = new FileInfo(@"d:\test.test"), FormatObject = formatObject1, UpperBoundExclusive = DateTimeOffset.Now.AddDays(-1), }); downloader.Setup(x => x.TryDownload(It.Is <FormatObject>(y => y.LocalDateTime == formatObject1.LocalDateTime))).Returns( new TryDownloadResult() { IsSuccess = false, DestinationFile = new FileInfo(@"d:\test.test"), FormatObject = formatObject2, UpperBoundExclusive = DateTimeOffset.Now.AddDays(0), }); downloader.Setup(x => x.TryDownload(It.Is <FormatObject>(y => y.LocalDateTime == formatObject2.LocalDateTime))).Returns( new TryDownloadResult() { IsSuccess = false, DestinationFile = new FileInfo(@"d:\test.test"), FormatObject = formatObject2, UpperBoundExclusive = DateTimeOffset.Now.AddDays(1), }); var target = new Extracter() { TempDownloadLocation = TempLocation, Source = source.Object, Downloader = downloader.Object, }; var expected = new FileNameExtractionResult[] {}; var actual = target.Extract(default(FormatObject)); Assert.IsTrue(actual.SequenceEqual(expected.Cast <IExtractionResult <FormatObject> >())); downloader.Verify(x => x.TryDownload(default(FormatObject)), Times.Exactly(1)); downloader.Verify(x => x.TryDownload(formatObject1), Times.Exactly(1)); downloader.Verify(x => x.TryDownload(formatObject2), Times.Exactly(1)); }
public MainWindowViewModel() { AvailableCombinations = new ObservableCollection <Combination>(); var fp = new FileParser(); var ex = new Extracter(); Text = fp.GetFileContent(Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.Desktop) , "FusionsFile.txt")); ex.ExtractData(Text); AllCards = ex.CardsList; Associations = ex.Fusions; }
static void Main(string[] args) { string url1 = "http://www.baidu.com"; string protocol1 = Extracter.GetUrlPart(url1, UrlPart.PROTOCOL); string hostName1 = Extracter.GetUrlPart(url1, UrlPart.HOST_NAME); string port1 = Extracter.GetUrlPart(url1, UrlPart.PORT); Console.WriteLine("protocol1: " + string.Equals(protocol1, "http")); Console.WriteLine("hostName1: " + string.Equals(hostName1, "www.baidu.com")); Console.WriteLine("port1: " + string.Equals(port1, "80")); string url2 = "http://www.163.com/test.aspx"; string protocol2 = Extracter.GetUrlPart(url2, UrlPart.PROTOCOL); string hostName2 = Extracter.GetUrlPart(url2, UrlPart.HOST_NAME); string port2 = Extracter.GetUrlPart(url2, UrlPart.PORT); Console.WriteLine("protocol2: " + string.Equals(protocol2, "http")); Console.WriteLine("hostName2: " + string.Equals(hostName2, "www.163.com")); Console.WriteLine("port2: " + string.Equals(port2, "80")); string url3 = "file://www.163.com:808/test.aspx"; string protocol3 = Extracter.GetUrlPart(url3, UrlPart.PROTOCOL); string hostName3 = Extracter.GetUrlPart(url3, UrlPart.HOST_NAME); string port3 = Extracter.GetUrlPart(url3, UrlPart.PORT); Console.WriteLine("protocol3: " + string.Equals(protocol3, "file")); Console.WriteLine("hostName3: " + string.Equals(hostName3, "www.163.com")); Console.WriteLine("port3: " + string.Equals(port3, "808")); string url4 = "www.163.com:808/test.aspx"; string protocol4 = Extracter.GetUrlPart(url4, UrlPart.PROTOCOL); string hostName4 = Extracter.GetUrlPart(url4, UrlPart.HOST_NAME); string port4 = Extracter.GetUrlPart(url4, UrlPart.PORT); Console.WriteLine("protocol4: " + string.Equals(protocol4, "http")); Console.WriteLine("hostName4: " + string.Equals(hostName4, "www.163.com")); Console.WriteLine("port4: " + string.Equals(port4, "808")); string url5 = "192.168.10.12:808/test.aspx"; string protocol5 = Extracter.GetUrlPart(url5, UrlPart.PROTOCOL); string hostName5 = Extracter.GetUrlPart(url5, UrlPart.HOST_NAME); string port5 = Extracter.GetUrlPart(url5, UrlPart.PORT); Console.WriteLine("protocol5: " + string.Equals(protocol5, "http")); Console.WriteLine("hostName5: " + string.Equals(hostName5, "192.168.10.12")); Console.WriteLine("port5: " + string.Equals(port5, "808")); Console.Read(); }
public ExtractResult Extract([FromBody] string json) { var node = ServerManager.Get(Request.RequestUri.Authority); var request = JsonConvert.DeserializeObject <ExtractRequest>(json); if (node.NodeType == Node.NodeTypeEnum.EXTRACTER) { var result = RuiJiExtracter.Extract(request.Content, request.Block); return(result); } else { return(Extracter.Extract(request)); } }
public void TestNodeExtract() { ServerManager.StartServers(); var response = new Crawler().Request("http://www.ruijihg.com/%e5%bc%80%e5%8f%91/"); if (response.StatusCode != System.Net.HttpStatusCode.OK) { return; } var content = response.Data.ToString(); var block = new ExtractBlock(); block.Selectors = new List <ISelector> { new CssSelector(".entry-content", CssTypeEnum.InnerHtml) }; block.TileSelector = new ExtractTile { Selectors = new List <ISelector> { new CssSelector(".pt-cv-content-item", CssTypeEnum.InnerHtml) } }; block.TileSelector.Metas.AddMeta("title", new List <ISelector> { new CssSelector(".pt-cv-title") }); block.TileSelector.Metas.AddMeta("url", new List <ISelector> { new CssSelector(".pt-cv-readmore", "href") }); var r = Extracter.Extract(new ExtractRequest { Block = block, Content = content }); Assert.IsTrue(r.Content.Length > 0); Assert.IsTrue(r.Tiles.Count > 0); }
public void Test_Extract_ExtractBreaksOutOfLoopIfAttemptSameDownloadTwice() { var source = new Mock <ISourceSpecification>(); source.Setup(x => x.GetDiscriminatingFilePairs()).Returns(() => { return(new[] { default(Pair) }); }); var formatObject = new FormatObject("test", new DateTime(2003, 1, 1)); var downloader = new Mock <IDownloader>(); downloader.Setup(x => x.TryDownload(default(FormatObject))) .Returns(new TryDownloadResult() { IsSuccess = true, DestinationFile = new FileInfo(@"d:\test.test"), FormatObject = formatObject, }); var target = new Extracter() { TempDownloadLocation = TempLocation, Source = source.Object, Downloader = downloader.Object, }; var expected = new FileNameExtractionResult[] { new FileNameExtractionResult() { FileFullName = @"d:\test.test", Pair = default(Pair), CurrentPosition = formatObject, IsSuccess = true }, }; var actual = target.Extract(default(FormatObject)); Assert.IsTrue(actual.SequenceEqual(expected.Cast <IExtractionResult <FormatObject> >())); downloader.Verify(x => x.TryDownload(default(FormatObject)), Times.Exactly(1)); downloader.Verify(x => x.TryDownload(formatObject), Times.Exactly(1)); }
private void Test_InheritBaseProject() { // Act PackTool.Process(new PackTool.Options { ProjectFile = TestHelper.GetDataPath("InheritChild.json"), OutputDirectory = TestHelper.GetOutputPath() }); // Assert var unpackPath = TestHelper.CreateOutputPath("Unpack"); Extracter.ExtractUnityPackage(TestHelper.GetOutputPath() + "/InheritChild.1.0.0.unitypackage", unpackPath, "InheritChild", true, true); var packagePath = Path.Combine(unpackPath, "Assets", "UnityPackages", "InheritChild"); AssertFileExistsWithMeta(packagePath, "../InheritChild.unitypackage.json"); AssertFileExistsWithMeta(packagePath, "Text1.txt"); AssertFileExistsWithMeta(packagePath, "Text2.txt"); }
public void Test_Extract_EscapesIfDownloadSameTwice() { var source = new Mock <ISourceSpecification>(); source.Setup(x => x.GetDiscriminatingFilePairs()).Returns(() => { return(new[] { default(Pair) }); }); var downloader = new Mock <IDownloader>(); downloader.Setup(x => x.TryDownload(default(FormatObject))) .Returns(new TryDownloadResult() { IsSuccess = true, DestinationFile = new FileInfo(@"d:\test.test"), }); var target = new Extracter() { TempDownloadLocation = TempLocation, Source = source.Object, Downloader = downloader.Object, }; var expected = new FileNameExtractionResult[] { new FileNameExtractionResult() { FileFullName = @"d:\test.test", Pair = default(Pair), CurrentPosition = default(FormatObject) }, }; var actual = target.Extract(default(FormatObject)); Task.Factory.StartNew(() => actual.ToList()).Wait(TimeSpan.FromSeconds(1)); downloader.Verify(x => x.TryDownload(default(FormatObject)), Times.Exactly(2)); }
public void TestGetRule() { ServerManager.StartServers(); var response = new Crawler().Request("http://www.ruijihg.com/2018/05/20/ruiji-solr-net/"); if (response.StatusCode != System.Net.HttpStatusCode.OK) { return; } var content = response.Data.ToString(); var block = Feeder.GetExtractBlock("http://www.ruijihg.com/2018/05/20/ruiji-solr-net/").First(); var r = Extracter.Extract(new ExtractRequest { Block = block, Content = content }); Assert.IsTrue(r.Content.Length > 0); Assert.IsTrue(r.Tiles.Count > 0); }
public void Extracter_Extract_Sequence() { var firstDate = new DateTimeOffset(new DateTime(2001, 1, 1), TimeSpan.FromHours(0)); var formatObject1 = new FormatObject("test", DateTime.Now.AddDays(-2), TimeZoneInfo.FindSystemTimeZoneById("W. Europe Standard Time")); var formatObject2 = new FormatObject("test", DateTime.Now.AddDays(-1), TimeZoneInfo.FindSystemTimeZoneById("W. Europe Standard Time")); var formatObject3 = new FormatObject("test", DateTime.Now.AddDays(0), TimeZoneInfo.FindSystemTimeZoneById("W. Europe Standard Time")); var formatObject4 = new FormatObject("test", DateTime.Now.AddDays(1), TimeZoneInfo.FindSystemTimeZoneById("W. Europe Standard Time")); var source = new Mock <ISourceSpecification>(); source.Setup(x => x.GetUpperBoundExclusive(It.IsAny <FormatObject>())).Returns <FormatObject>(y => y.DateTime.AddDays(1)); source.SetupGet(x => x.FirstDateTimeOffset).Returns(firstDate); source.Setup(x => x.GetPossibleFormatObjects(firstDate)).Returns(new[] { formatObject1 }); var downloadAggregator = new Mock <IDownloadAggregator>(); downloadAggregator.Setup(x => x.DownloadFirst(It.Is <FormatObject>(y => y.DateTime == formatObject1.DateTime))).Returns(new DownloadAggregatorResult() { IsSuccess = false, CurrentPosition = formatObject1, }); downloadAggregator.Setup(x => x.DownloadNext(It.Is <FormatObject>(y => y.DateTime == formatObject1.DateTime))).Returns(new DownloadAggregatorResult() { IsSuccess = true, CurrentPosition = formatObject2, DestinationFiles = new[] { new FileInfo(@"d:\test1.test"), new FileInfo(@"d:\test2.test") } }); downloadAggregator.Setup(x => x.DownloadNext(It.Is <FormatObject>(y => y.DateTime == formatObject2.DateTime))).Returns(new DownloadAggregatorResult() { IsSuccess = true, CurrentPosition = formatObject3, DestinationFiles = new[] { new FileInfo(@"d:\test2.test"), new FileInfo(@"d:\test3.test") } }); downloadAggregator.Setup(x => x.DownloadNext(It.Is <FormatObject>(y => y.DateTime == formatObject3.DateTime))).Returns(new DownloadAggregatorResult() { IsSuccess = true, CurrentPosition = formatObject4, DestinationFiles = new[] { new FileInfo(@"d:\test3.test"), new FileInfo(@"d:\test4.test") } }); var target = new Extracter() { Source = source.Object, TempDownloadLocation = TempLocation, Aggregator = downloadAggregator.Object }; var expected = new FileNameExtractionResult[] { new FileNameExtractionResult() { FileFullNames = new [] { @"d:\test1.test", @"d:\test2.test" }, CurrentPosition = formatObject2, IsSuccess = true }, new FileNameExtractionResult() { FileFullNames = new [] { @"d:\test2.test", @"d:\test3.test" }, CurrentPosition = formatObject3, IsSuccess = true }, }; var actual = target.Extract(default(FormatObject)); AreEqual(expected, actual.Cast <FileNameExtractionResult>()); downloadAggregator.Verify(x => x.Initialize(), Times.Once()); source.Verify(x => x.GetPossibleFormatObjects(firstDate), Times.Once()); downloadAggregator.Verify(x => x.DownloadFirst(formatObject1), Times.Once()); downloadAggregator.Verify(x => x.DownloadNext(formatObject1), Times.Once()); downloadAggregator.Verify(x => x.DownloadNext(formatObject2), Times.Once()); downloadAggregator.Verify(x => x.DownloadNext(formatObject3), Times.Never()); downloadAggregator.Verify(x => x.DownloadNext(formatObject4), Times.Never()); }
public void Extracter_Extract_OnlyStopsWhenCurrentDateNotOnDownloadFail() { var source = new Mock <ISourceSpecification>(); source.Setup(x => x.GetUpperBoundExclusive(It.IsAny <FormatObject>())).Returns <FormatObject>((x) => x.DateTime.AddDays(1)); var formatObject1 = new FormatObject("test", DateTime.Now.AddDays(-2), TimeZoneInfo.FindSystemTimeZoneById("W. Europe Standard Time")); var formatObject2 = new FormatObject("test", DateTime.Now.AddDays(-1), TimeZoneInfo.FindSystemTimeZoneById("W. Europe Standard Time")); var formatObject3 = new FormatObject("test", DateTime.Now.AddDays(0), TimeZoneInfo.FindSystemTimeZoneById("W. Europe Standard Time")); var formatObject4 = new FormatObject("test", DateTime.Now.AddDays(1), TimeZoneInfo.FindSystemTimeZoneById("W. Europe Standard Time")); var downloader = new Mock <IDownloader>(); downloader.Setup(x => x.TryDownload(default(FormatObject))).Returns( new TryDownloadResult() { IsSuccess = false, DestinationFile = new FileInfo(@"d:\test.test"), }); downloader.Setup(x => x.TryDownload(It.Is <FormatObject>(y => y.DateTime == formatObject1.DateTime))).Returns( new TryDownloadResult() { IsSuccess = false, DestinationFile = new FileInfo(@"d:\test.test"), }); downloader.Setup(x => x.TryDownload(It.Is <FormatObject>(y => y.DateTime == formatObject2.DateTime))).Returns( new TryDownloadResult() { IsSuccess = false, DestinationFile = new FileInfo(@"d:\test.test"), }); downloader.Setup(x => x.TryDownload(It.Is <FormatObject>(y => y.DateTime == formatObject3.DateTime))).Returns( new TryDownloadResult() { IsSuccess = false, DestinationFile = new FileInfo(@"d:\test.test"), }); downloader.Setup(x => x.TryDownload(It.Is <FormatObject>(y => y.DateTime == formatObject4.DateTime))).Returns( new TryDownloadResult() { IsSuccess = false, DestinationFile = new FileInfo(@"d:\test.test"), }); var formatIterator = new Mock <IFormatIterator>(); formatIterator.Setup(x => x.GetNextCandidates(default(FormatObject))).Returns(new[] { formatObject1 }); formatIterator.Setup(x => x.GetNextCandidates(It.Is <FormatObject>(y => y.DateTime == formatObject1.DateTime))).Returns(new[] { formatObject2 }); formatIterator.Setup(x => x.GetNextCandidates(It.Is <FormatObject>(y => y.DateTime == formatObject2.DateTime))).Returns(new[] { formatObject3 }); formatIterator.Setup(x => x.GetNextCandidates(It.Is <FormatObject>(y => y.DateTime == formatObject3.DateTime))).Returns(new[] { formatObject4 }); var downloadAggregator = new DownloadAggregator() { NumberOfFilesToAggregate = 2, Source = source.Object, Downloader = downloader.Object, FormatIterator = formatIterator.Object }; downloadAggregator.Initialize(); var target = new Extracter() { Source = source.Object, TempDownloadLocation = TempLocation, Aggregator = downloadAggregator }; var expected = new FileNameExtractionResult[] { }; var actual = target.Extract(default(FormatObject)); AreEqual(expected, actual.Cast <FileNameExtractionResult>()); downloader.Verify(x => x.TryDownload(default(FormatObject)), Times.Exactly(0)); downloader.Verify(x => x.TryDownload(formatObject1), Times.Exactly(1)); downloader.Verify(x => x.TryDownload(formatObject2), Times.Exactly(1)); downloader.Verify(x => x.TryDownload(formatObject3), Times.Exactly(1)); downloader.Verify(x => x.TryDownload(formatObject4), Times.Exactly(0)); }