NCrawler：基于.NET的高效多线程网页爬虫源码下载

【实例简介】
NCrawler是一款简单而高效的基于.NET的多线程网页爬虫，采用基于管道的处理方式编写，使用C#开发。它包含了HTML、文本、PDF以及IFilter文档处理器，并集成了语言检测功能（Google）。通过添加管道步骤，用户可以轻松地提取、使用和修改信息。
NCrawler是2010年NCrawler的全新重写版本，采用了更现代的编程方法。目前已更新至v4版本。

// 示例代码：初始化NCrawler
var crawler = new NCrawler();
// 配置爬虫参数
// 开始爬取网页

NCrawler不仅易于使用，还支持高度自定义，适合需要进行网页数据抓取的开发者。
【实例截图】
【核心代码】
文件清单
└── NCrawler-ee3e2af901adbcf452378ece673ffde3bb399496
    ├── LICENSE
    ├── README.md
    └── src
        ├── build.bat
        ├── build.cake
        ├── build.ps1
        ├── NCrawler
        │   ├── CrawlerConfiguration.cs
        │   ├── CrawlerConfiguration.Run.Cs
        │   ├── CrawlStep.cs
        │   ├── Extensions
        │   │   ├── AspectExtensions.cs
        │   │   ├── IEnumerableExtensions.cs
        │   │   ├── ObjectExtensions.cs
        │   │   ├── StringExtensions.cs
        │   │   ├── UriExtensions.cs
        │   │   └── UrlExtensions.cs
        │   ├── ICrawler.cs
        │   ├── IPipelineStep.cs
        │   ├── NCrawler.csproj
        │   ├── packages.config
        │   ├── Pipeline
        │   │   ├── DefaultDownloadPipelineStep.cs
        │   │   ├── EMailEntityExtractionProcessor.cs
        │   │   ├── LambdaFilterPipelineStep.cs
        │   │   └── LinkExtractionPipelineStep.cs
        │   ├── Properties
        │   │   ├── AssemblyInfo.cs
        │   │   └── SharedAssemblyInfo.cs
        │   ├── PropertyBag.cs
        │   └── Utils
        │       ├── AspectF.cs
        │       ├── DisposableBase.cs
        │       └── TempFile.cs
        ├── NCrawler.Console
        │   ├── app.config
        │   ├── Extensions
        │   │   └── AspectExtensions.cs
        │   ├── NConsoler.cs
        │   ├── NCrawler.Console.csproj
        │   ├── Program.cs
        │   └── Properties
        │       └── AssemblyInfo.cs
        ├── NCrawler.Flurl
        │   ├── app.config
        │   ├── CrawlerConfigurationExtensions.cs
        │   ├── FlurlDownloadPipelineStep.cs
        │   ├── NCrawler.Flurl.csproj
        │   ├── packages.config
        │   └── Properties
        │       └── AssemblyInfo.cs
        ├── NCrawler.Flurl.Tests
        │   ├── app.config
        │   ├── Content
        │   │   └── NCrawler - Home.html
        │   ├── FlurlDownloadPipelineStepTests.cs
        │   ├── NCrawler.Flurl.Tests.csproj
        │   ├── packages.config
        │   └── Properties
        │       └── AssemblyInfo.cs
        ├── NCrawler.HtmlProcessor
        │   ├── ContentCrawlerRules.cs
        │   ├── CrawlerConfigurationExtensions.cs
        │   ├── Extensions
        │   │   ├── ExtendedHtmlUtility.cs
        │   │   └── HtmlAgilityPackExtensions.cs
        │   ├── HtmlDocumentProcessorPipelineStep.cs
        │   ├── Interfaces
        │   │   └── ISubstitution.cs
        │   ├── NCrawler.HtmlProcessor.csproj
        │   ├── packages.config
        │   ├── Properties
        │   │   └── AssemblyInfo.cs
        │   ├── RegexSubstitution.cs
        │   └── TextDocumentProcessorPipelineStep.cs
        ├── NCrawler.HtmlProcessor.Tests
        │   ├── Content
        │   │   ├── NCrawler - Home.html
        │   │   └── NCrawler - Home.txt
        │   ├── HtmlDocumentProcessorPipelineStepTests.cs
        │   ├── NCrawler.HtmlProcessor.Tests.csproj
        │   ├── packages.config
        │   ├── Properties
        │   │   └── AssemblyInfo.cs
        │   └── TextDocumentProcessorPipelineStepTests.cs
        ├── NCrawler.LanguageDetection.Google
        │   ├── app.config
        │   ├── Core14.profile.xml
        │   ├── CrawlerConfigurationExtensions.cs
        │   ├── GoogleLanguageDetection.cs
        │   ├── NCrawler.LanguageDetection.Google.csproj
        │   ├── packages.config
        │   └── Properties
        │       └── AssemblyInfo.cs
        ├── NCrawler.LanguageDetection.Google.Tests
        │   ├── app.config
        │   ├── Content
        │   │   └── NCrawler - Home.txt
        │   ├── GoogleLanguageDetectionTests.cs
        │   ├── NCrawler.LanguageDetection.Google.Tests.csproj
        │   ├── packages.config
        │   └── Properties
        │       └── AssemblyInfo.cs
        ├── NCrawler.PDFBox
        │   ├── App.config
        │   ├── CrawlerConfigurationExtensions.cs
        │   ├── NCrawler.PDFBox.csproj
        │   ├── PdfBoxTextExtractorProcessorPipelineStep.cs
        │   └── Properties
        │       └── AssemblyInfo.cs
        ├── NCrawler.PDFBox.Tests
        │   ├── NCrawler.PDFBox.Tests.csproj
        │   ├── packages.config
        │   ├── PdfBoxTextExtractorProcessorPipelineStepTests.cs
        │   └── Properties
        │       └── AssemblyInfo.cs
        ├── NCrawler.proj
        ├── NCrawler.Robots
        │   ├── NCrawler.Robots.csproj
        │   ├── packages.config
        │   ├── Properties
        │   │   └── AssemblyInfo.cs
        │   ├── RobotsCrawlerConfigurationExtensions.cs
        │   └── RobotsPipelineStep.cs
        ├── NCrawler.Robots.Tests
        │   ├── Content
        │   │   └── Robots.txt
        │   ├── NCrawler.Robots.Tests.csproj
        │   ├── packages.config
        │   ├── Properties
        │   │   └── AssemblyInfo.cs
        │   └── RobotsPipelineStepTests.cs
        ├── NCrawler.SitemapProcessor
        │   ├── CrawlerConfigurationExtensions.cs
        │   ├── NCrawler.SitemapProcessor.csproj
        │   ├── Properties
        │   │   └── AssemblyInfo.cs
        │   └── SitemapProcessor.cs
        ├── NCrawler.SitemapProcessor.Tests
        │   ├── NCrawler.SitemapProcessor.Tests.csproj
        │   ├── packages.config
        │   ├── Properties
        │   │   └── AssemblyInfo.cs
        │   └── SitemapProcessorTests.cs
        ├── NCrawler.sln
        ├── NCrawler.Toxy
        │   ├── app.config
        │   ├── CrawlerConfigurationExtensions.cs
        │   ├── NCrawler.Toxy.csproj
        │   ├── packages.config
        │   ├── Properties
        │   │   └── AssemblyInfo.cs
        │   └── ToxyTextExtractorProcessorPipelineStep.cs
        ├── NCrawler.Toxy.Tests
        │   ├── App.config
        │   ├── Content
        │   │   ├── Georgia_opposition_NATO-Eng-F.doc
        │   │   └── SampleData.xls
        │   ├── NCrawler.Toxy.Tests.csproj
        │   ├── packages.config
        │   ├── Properties
        │   │   └── AssemblyInfo.cs
        │   └── ToxyTextExtractorProcessorPipelineStepTests.cs
        ├── NCrawler.WebServer
        │   ├── NCrawler.WebServer.csproj
        │   ├── packages.config
        │   ├── PortUtils.cs
        │   ├── Properties
        │   │   └── AssemblyInfo.cs
        │   └── SimpleWebServer.cs
        └── tools
            ├── Cake
            │   ├── Autofac.dll
            │   ├── Cake.Common.dll
            │   ├── Cake.Common.xml
            │   ├── Cake.Core.dll
            │   ├── Cake.Core.xml
            │   ├── Cake.exe
            │   ├── Cake.NuGet.dll
            │   ├── Cake.NuGet.xml
            │   ├── Mono.CSharp.dll
            │   ├── NuGet.Core.dll
            │   ├── Roslyn.Compilers.CSharp.dll
            │   └── Roslyn.Compilers.dll
            ├── nuget.exe
            ├── NUnit.ConsoleRunner
            │   ├── CHANGES.txt
            │   ├── LICENSE.txt
            │   ├── NOTICES.txt
            │   └── tools
            │       ├── Mono.Cecil.dll
            │       ├── nunit3-console.exe
            │       ├── nunit3-console.exe.config
            │       ├── nunit-agent.exe
            │       ├── nunit-agent.exe.config
            │       ├── nunit-agent-x86.exe
            │       ├── nunit-agent-x86.exe.config
            │       ├── nunit.engine.api.dll
            │       ├── nunit.engine.api.xml
            │       ├── nunit.engine.dll
            │       └── nunit.nuget.addins
            ├── packages.config
            └── PDFBox.NET-1.8.9
                └── license
                    ├── ikvm
                    │   ├── LICENSE
                    │   ├── THIRD_PARTY_README
                    │   └── TRADEMARK
                    └── pdfbox
                        ├── LICENSE.txt
                        └── NOTICE.txt

55 directories, 157 files