crawler4j：轻量级多线程网络爬虫

<dependency>     <groupId>edu.uci.ics</groupId>     <artifactId>crawler4j</artifactId>     <version>4.1</version> </dependency>

<dependency>     <groupId>edu.uci.ics</groupId>     <artifactId>crawler4j</artifactId>     <version>4.1</version> </dependency>

public class MyCrawler extends WebCrawler {      private final static Pattern FILTERS = Pattern.compile(".*(//.(css|js|gif|jpg"                                                            + "|png|mp3|mp3|zip|gz))$");      /**      * This method receives two parameters. The first parameter is the page      * in which we have discovered this new url and the second parameter is      * the new url. You should implement this function to specify whether      * the given url should be crawled or not (based on your crawling logic).      * In this example, we are instructing the crawler to ignore urls that      * have css, js, git, ... extensions and to only accept urls that start      * with "http://www.ics.uci.edu/". In this case, we didn't need the      * referringPage parameter to make the decision.      */      @Override      public boolean shouldVisit(Page referringPage, WebURL url) {          String href = url.getURL().toLowerCase();          return !FILTERS.matcher(href).matches()                 && href.startsWith("http://www.ics.uci.edu/");      }       /**       * This function is called when a page is fetched and ready       * to be processed by your program.       */      @Override      public void visit(Page page) {          String url = page.getWebURL().getURL();          System.out.println("URL: " + url);           if (page.getParseData() instanceof HtmlParseData) {              HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();              String text = htmlParseData.getText();              String html = htmlParseData.getHtml();              Set<WebURL> links = htmlParseData.getOutgoingUrls();               System.out.println("Text length: " + text.length());              System.out.println("Html length: " + html.length());              System.out.println("Number of outgoing links: " + links.size());          }     } }

publicclassMyCrawlerextendsWebCrawler{       privatefinalstaticPatternFILTERS=Pattern.compile(".*(//.(css|js|gif|jpg"                                                            +"|png|mp3|mp3|zip|gz))$");       /**      * This method receives two parameters. The first parameter is the page      * in which we have discovered this new url and the second parameter is      * the new url. You should implement this function to specify whether      * the given url should be crawled or not (based on your crawling logic).      * In this example, we are instructing the crawler to ignore urls that      * have css, js, git, ... extensions and to only accept urls that start      * with "http://www.ics.uci.edu/". In this case, we didn't need the      * referringPage parameter to make the decision.      */      @Override      publicbooleanshouldVisit(PagereferringPage,WebURLurl){          Stringhref=url.getURL().toLowerCase();          return!FILTERS.matcher(href).matches()                 &&href.startsWith("http://www.ics.uci.edu/");      }        /**       * This function is called when a page is fetched and ready       * to be processed by your program.       */      @Override      publicvoidvisit(Pagepage){          Stringurl=page.getWebURL().getURL();          System.out.println("URL: "+url);            if(page.getParseData()instanceofHtmlParseData){              HtmlParseDatahtmlParseData=(HtmlParseData)page.getParseData();              Stringtext=htmlParseData.getText();              Stringhtml=htmlParseData.getHtml();              Set<WebURL>links=htmlParseData.getOutgoingUrls();                System.out.println("Text length: "+text.length());              System.out.println("Html length: "+html.length());              System.out.println("Number of outgoing links: "+links.size());          }     } }

public class Controller {     public static void main(String[] args) throws Exception {         String crawlStorageFolder = "/data/crawl/root";         int numberOfCrawlers = 7;          CrawlConfig config = new CrawlConfig();         config.setCrawlStorageFolder(crawlStorageFolder);          /*          * Instantiate the controller for this crawl.          */         PageFetcher pageFetcher = new PageFetcher(config);         RobotstxtConfig robotstxtConfig = new RobotstxtConfig();         RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);         CrawlController controller = new CrawlController(config, pageFetcher, robotstxtServer);          /*          * For each crawl, you need to add some seed urls. These are the first          * URLs that are fetched and then the crawler starts following links          * which are found in these pages          */         controller.addSeed("http://www.ics.uci.edu/~lopes/");         controller.addSeed("http://www.ics.uci.edu/~welling/");         controller.addSeed("http://www.ics.uci.edu/");          /*          * Start the crawl. This is a blocking operation, meaning that your code          * will reach the line after this only when crawling is finished.          */         controller.start(MyCrawler.class, numberOfCrawlers);     } }

publicclassController{     publicstaticvoidmain(String[]args)throwsException{         StringcrawlStorageFolder="/data/crawl/root";         intnumberOfCrawlers=7;           CrawlConfigconfig=newCrawlConfig();         config.setCrawlStorageFolder(crawlStorageFolder);           /*          * Instantiate the controller for this crawl.          */         PageFetcherpageFetcher=newPageFetcher(config);         RobotstxtConfigrobotstxtConfig=newRobotstxtConfig();         RobotstxtServerrobotstxtServer=newRobotstxtServer(robotstxtConfig,pageFetcher);         CrawlControllercontroller=newCrawlController(config,pageFetcher,robotstxtServer);           /*          * For each crawl, you need to add some seed urls. These are the first          * URLs that are fetched and then the crawler starts following links          * which are found in these pages          */         controller.addSeed("http://www.ics.uci.edu/~lopes/");         controller.addSeed("http://www.ics.uci.edu/~welling/");         controller.addSeed("http://www.ics.uci.edu/");           /*          * Start the crawl. This is a blocking operation, meaning that your code          * will reach the line after this only when crawling is finished.          */         controller.start(MyCrawler.class,numberOfCrawlers);     } }

A --> B --> C --> D

A-->B-->C-->D

crawlConfig.setMaxDepthOfCrawling(maxDepthOfCrawling);

crawlConfig.setMaxDepthOfCrawling(maxDepthOfCrawling);

crawlConfig.setMaxPagesToFetch(maxPagesToFetch);

crawlConfig.setMaxPagesToFetch(maxPagesToFetch);

crawlConfig.setPolitenessDelay(politenessDelay);

crawlConfig.setPolitenessDelay(politenessDelay);

crawlConfig.setProxyHost("proxyserver.example.com"); crawlConfig.setProxyPort(8080);

crawlConfig.setProxyHost("proxyserver.example.com"); crawlConfig.setProxyPort(8080);

crawlConfig.setProxyUsername(username); crawlConfig.getProxyPassword(password);

crawlConfig.setProxyUsername(username); crawlConfig.getProxyPassword(password);

crawlConfig.setResumableCrawling(true);

crawlConfig.setResumableCrawling(true);

crawlConfig.setUserAgentString(userAgentString);

crawlConfig.setUserAgentString(userAgentString);