コード例 #1
0
ファイル: Crawler.cs プロジェクト: Kooldeji/WebSpy
        public async Task init()
        {
            Console.WriteLine("Initializing...");
            _indexer = new Indexer(_corpus);
            var repo = await _corpus.GetRepository();

            Console.WriteLine(repo);
            var files = crawl(repo).ToList();//All files in my repository

            foreach (var file in files)
            {
                Console.WriteLine(file);
                if (!isValidFile(file))
                {
                    continue;
                }
                Console.WriteLine(await _corpus.GetRepository());
                var sfile = file.Replace(await _corpus.GetRepository() + "\\", "");
                Console.WriteLine(await _corpus.GetRepository());
                //For each file in the repository, try to get their id from the corpus
                Console.WriteLine(sfile);
                string id = await _corpus.GetDocumentID(sfile);

                if (id == null)
                {
                    //File is not in the Corpus
                    //Index it and add it to the corpus.
                    Console.WriteLine("deji2");
                    await addFile(file);
                }
                else
                {
                    //File exists in the corpus
                    if (File.GetLastWriteTime(file).Ticks > await _corpus.GetLastCrawled())
                    {
                        //File has been edited since last checked, re-index it and update it into the corpus
                        await editFile(file);
                    }
                }
                //Console.WriteLine("done");
            }
            var kFiles = await _corpus.GetDocuments(); //Files that exist in my database

            foreach (var id in kFiles)
            {
                var file = await _corpus.GetDocumentPath(id);

                if (!files.Contains(Path.Combine(await _corpus.GetRepository(), file)))
                {
                    //File does not exist again in repo, delete it from database
                    await removeFile(file);
                }
            }

            //------------Watching for changes--------------------
            watcher = new FileSystemWatcher(await _corpus.GetRepository());
            Console.WriteLine("watch");
            watcher.NotifyFilter = NotifyFilters.LastWrite | NotifyFilters.FileName | NotifyFilters.DirectoryName | NotifyFilters.Attributes;
            watcher.Changed     += new FileSystemEventHandler(async(source, e) =>
            {
                Console.WriteLine("changed");
                if (!isValidFile(e.FullPath))
                {
                    return;
                }
                await editFile(e.FullPath);
            });
            watcher.Created += new FileSystemEventHandler(async(source, e) =>
            {
                Console.WriteLine("created");
                if (!isValidFile(e.FullPath))
                {
                    return;
                }
                await addFile(e.FullPath);
            });
            watcher.Deleted += new FileSystemEventHandler(async(source, e) =>
            {
                Console.WriteLine("remove");
                await removeFile(e.FullPath);
            });
            watcher.Renamed += new RenamedEventHandler(async(source, e) =>
            {
                if (!isValidFile(e.FullPath))
                {
                    return;
                }
                await renameFile(e.FullPath);
            });

            watcher.EnableRaisingEvents = true;
            Console.WriteLine("watcher: " + watcher.Path);
        }