Java开源网络爬虫：crawler4j源码下载

【实例简介】
crawler4j是一个Java开源网络爬虫，提供了一个简单的接口来爬取互联网。它可以在几分钟内设置一个多线程的Web爬虫。这篇文章主要讲解了crawler4j的安装、快速入门、更多示例以及配置细节等内容。您可以在文章中找到完整的源代码，用于自己的项目中。
【实例截图】
【核心代码】
文件清单
└── crawler4j-68f5c1e4fb86542e74d31c0bcb4b1ae14ba2ea71
    ├── build.gradle
    ├── CHANGES.txt
    ├── config
    │   ├── checkstyle.xml
    │   └── intellij
    │       ├── gradle-template.xml
    │       └── reset.sh
    ├── crawler4j
    │   ├── build.gradle
    │   └── src
    │       ├── main
    │       │   └── java
    │       │       └── edu
    │       │           └── uci
    │       │               └── ics
    │       │                   └── crawler4j
    │       │                       ├── crawler
    │       │                       │   ├── authentication
    │       │                       │   │   ├── AuthInfo.java
    │       │                       │   │   ├── BasicAuthInfo.java
    │       │                       │   │   ├── FormAuthInfo.java
    │       │                       │   │   └── NtAuthInfo.java
    │       │                       │   ├── Configurable.java
    │       │                       │   ├── CrawlConfig.java
    │       │                       │   ├── CrawlController.java
    │       │                       │   ├── exceptions
    │       │                       │   │   ├── ContentFetchException.java
    │       │                       │   │   ├── PageBiggerThanMaxSizeException.java
    │       │                       │   │   └── ParseException.java
    │       │                       │   ├── Page.java
    │       │                       │   └── WebCrawler.java
    │       │                       ├── fetcher
    │       │                       │   ├── BasicAuthHttpRequestInterceptor.java
    │       │                       │   ├── IdleConnectionMonitorThread.java
    │       │                       │   ├── PageFetcher.java
    │       │                       │   ├── PageFetchResult.java
    │       │                       │   ├── SniPoolingHttpClientConnectionManager.java
    │       │                       │   └── SniSSLConnectionSocketFactory.java
    │       │                       ├── frontier
    │       │                       │   ├── Counters.java
    │       │                       │   ├── DocIDServer.java
    │       │                       │   ├── Frontier.java
    │       │                       │   ├── InProcessPagesDB.java
    │       │                       │   ├── WebURLTupleBinding.java
    │       │                       │   └── WorkQueues.java
    │       │                       ├── parser
    │       │                       │   ├── AllTagMapper.java
    │       │                       │   ├── BinaryParseData.java
    │       │                       │   ├── CssParseData.java
    │       │                       │   ├── ExtractedUrlAnchorPair.java
    │       │                       │   ├── HtmlContentHandler.java
    │       │                       │   ├── HtmlParseData.java
    │       │                       │   ├── HtmlParser.java
    │       │                       │   ├── NotAllowedContentException.java
    │       │                       │   ├── ParseData.java
    │       │                       │   ├── Parser.java
    │       │                       │   ├── TextParseData.java
    │       │                       │   └── TikaHtmlParser.java
    │       │                       ├── robotstxt
    │       │                       │   ├── HostDirectives.java
    │       │                       │   ├── PathRule.java
    │       │                       │   ├── RobotstxtConfig.java
    │       │                       │   ├── RobotstxtParser.java
    │       │                       │   ├── RobotstxtServer.java
    │       │                       │   └── UserAgentDirectives.java
    │       │                       ├── url
    │       │                       │   ├── TLDList.java
    │       │                       │   ├── URLCanonicalizer.java
    │       │                       │   ├── UrlResolver.java
    │       │                       │   └── WebURL.java
    │       │                       └── util
    │       │                           ├── IO.java
    │       │                           ├── Net.java
    │       │                           └── Util.java
    │       └── test
    │           ├── groovy
    │           │   └── edu
    │           │       └── uci
    │           │           └── ics
    │           │               └── crawler4j
    │           │                   ├── auth
    │           │                   │   ├── BasicAuthTest.groovy
    │           │                   │   └── FormAuthInfoTest.groovy
    │           │                   ├── config
    │           │                   │   └── CustomDnsResolverTest.groovy
    │           │                   ├── crawler
    │           │                   │   ├── CrawlerWithJSTest.groovy
    │           │                   │   ├── NoFollowTest.groovy
    │           │                   │   ├── NoIndexTest.groovy
    │           │                   │   ├── OnRedirectedToInvalidTest.groovy
    │           │                   │   ├── PageTest.groovy
    │           │                   │   ├── RedirectHandlerTest.groovy
    │           │                   │   ├── TimeoutTest.groovy
    │           │                   │   └── WebCrawlerTest.groovy
    │           │                   ├── parser
    │           │                   │   ├── CssParseDataTest.groovy
    │           │                   │   └── HtmlParserTest.groovy
    │           │                   ├── robotstxt
    │           │                   │   └── RobotstxtParserTest.groovy
    │           │                   ├── url
    │           │                   │   ├── PublicSuffixTest.groovy
    │           │                   │   └── TLDListOnlineTest.groovy
    │           │                   └── util
    │           │                       └── NetTest.groovy
    │           ├── java
    │           │   └── edu
    │           │       └── uci
    │           │           └── ics
    │           │               └── crawler4j
    │           │                   └── tests
    │           │                       ├── fetcher
    │           │                       │   ├── PageFetcherHtmlOnly.java
    │           │                       │   └── PageFetcherHtmlTest.java
    │           │                       └── URLCanonicalizerTest.java
    │           └── resources
    │               ├── css
    │               │   ├── absolute.css
    │               │   ├── quotes.css
    │               │   └── relative.css
    │               ├── html
    │               │   └── wiki.c2.com.html
    │               ├── logback.xml
    │               ├── public_suffix_list.dat
    │               └── robotstxt
    │                   └── he.wikipedia.org_robots.txt
    ├── crawler4j-examples
    │   ├── crawler4j-examples-base
    │   │   ├── build.gradle
    │   │   └── src
    │   │       ├── main
    │   │       │   └── resources
    │   │       │       └── logback.xml
    │   │       └── test
    │   │           └── java
    │   │               └── edu
    │   │                   └── uci
    │   │                       └── ics
    │   │                           └── crawler4j
    │   │                               └── examples
    │   │                                   ├── basic
    │   │                                   │   ├── BasicCrawlController.java
    │   │                                   │   └── BasicCrawler.java
    │   │                                   ├── imagecrawler
    │   │                                   │   ├── ImageCrawlController.java
    │   │                                   │   └── ImageCrawler.java
    │   │                                   ├── localdata
    │   │                                   │   ├── CrawlStat.java
    │   │                                   │   ├── LocalDataCollectorController.java
    │   │                                   │   └── LocalDataCollectorCrawler.java
    │   │                                   ├── multiple
    │   │                                   │   ├── BasicCrawler.java
    │   │                                   │   └── MultipleCrawlerController.java
    │   │                                   ├── shutdown
    │   │                                   │   ├── BasicCrawler.java
    │   │                                   │   └── ControllerWithShutdown.java
    │   │                                   └── statushandler
    │   │                                       ├── StatusHandlerCrawlController.java
    │   │                                       └── StatusHandlerCrawler.java
    │   └── crawler4j-examples-postgres
    │       ├── build.gradle
    │       ├── README.md
    │       └── src
    │           ├── main
    │           │   └── java
    │           │       └── edu
    │           │           └── uci
    │           │               └── ics
    │           │                   └── crawler4j
    │           │                       └── examples
    │           │                           ├── crawler
    │           │                           │   ├── PostgresCrawlerFactory.java
    │           │                           │   └── PostgresWebCrawler.java
    │           │                           ├── db
    │           │                           │   ├── impl
    │           │                           │   │   └── PostgresDBServiceImpl.java
    │           │                           │   └── PostgresDBService.java
    │           │                           └── SampleLauncher.java
    │           └── test
    │               ├── java
    │               │   └── edu
    │               │       └── uci
    │               │           └── ics
    │               │               └── crawler4j
    │               │                   └── examples
    │               │                       └── PgsqlTest.java
    │               └── resources
    │                   ├── db
    │                   │   └── migration
    │                   │       └── V1__initial_schema.sql
    │                   ├── docker-compose.yml
    │                   └── logback.xml
    ├── gradle
    │   └── wrapper
    │       ├── gradle-wrapper.jar
    │       └── gradle-wrapper.properties
    ├── gradle.properties
    ├── gradlew
    ├── gradlew.bat
    ├── LICENSE
    ├── README.md
    └── settings.gradle

86 directories, 116 files