Spider.php
/**
     * Function that crawls one webpage based on the give url.
     *
     * @param CrawlJob     $crawlJob
     * @param QueueFactory $queueFactory
     * @param string       $currentQueueType
     */
Function that crawls one webpage based on the give url.
Function that crawls one webpage based on the give url .
public function crawl(CrawlJob $crawlJob, QueueFactory $queueFactory, $currentQueueType)
    {
        $this->currentCrawlJob = $crawlJob;
        $uri = new Uri($crawlJob->getUrl());
        $this->curlClient->initClient();
        if ($this->curlClient->isDocument($uri) && $currentQueueType != QueueFactory::QUEUE_DOCUMENTS) {
            $message = new AMQPMessage(
                json_encode($crawlJob->toArray()),
                ['delivery_mode' => 1]
            );
            $queueFactory->getQueue(QueueFactory::QUEUE_DOCUMENTS)->publish($message);
        } else {
            $resource = $this->requestHandler->request($uri);
            $baseUrl = $resource->getUri()->toString();
            if (!trim($resource->getResponse()->getBody(true))) {
                $this->setClient($uri->toString());
                $resource = $this->requestHandler->request($uri);
            }
            $uris = [];
            $this->eventDispatcher->dispatch(SpiderEvents::SPIDER_CRAWL_PRE_DISCOVER);
            $crawler = $resource->getCrawler()->filterXPath('//a');
            foreach ($crawler as $node) {
                try {
                    if ($node->getAttribute("rel") === "nofollow") {
                        continue;
                    }
                    $href = $node->getAttribute('href');
                    $uri = new Uri($href, $baseUrl);
                    $uris[] = $uri;
                } catch (UriSyntaxException $e) {
                }
            }
            $crawler = $resource->getCrawler()->filterXPath('//loc');
            foreach ($crawler as $node) {
                try {
                    $href = $node->nodeValue;
                    $uri = new Uri($href, $baseUrl);
                    $uris[] = $uri;
                } catch (UriSyntaxException $e) {
                }
            }
            $this->eventDispatcher->dispatch(
                SpiderEvents::SPIDER_CRAWL_POST_DISCOVER,
                new GenericEvent($this, ['uris' => $uris])
            );
            $this->persistenceHandler->persist($resource, $crawlJob);
        }
    }
public function crawl ( CrawlJob $ crawlJob , QueueFactory $ queueFactory , $ currentQueueType ) { $ this -> currentCrawlJob = $ crawlJob ; $ uri = new Uri ( $ crawlJob -> getUrl ( ) ) ; $ this -> curlClient -> initClient ( ) ; if ( $ this -> curlClient -> isDocument ( $ uri ) && $ currentQueueType != QueueFactory :: QUEUE_DOCUMENTS ) { $ message = new AMQPMessage ( json_encode ( $ crawlJob -> toArray ( ) ) , [ 'delivery_mode' => 1 ] ) ; $ queueFactory -> getQueue ( QueueFactory :: QUEUE_DOCUMENTS ) -> publish ( $ message ) ; } else { $ resource = $ this -> requestHandler -> request ( $ uri ) ; $ baseUrl = $ resource -> getUri ( ) -> toString ( ) ; if ( ! trim ( $ resource -> getResponse ( ) -> getBody ( true ) ) ) { $ this -> setClient ( $ uri -> toString ( ) ) ; $ resource = $ this -> requestHandler -> request ( $ uri ) ; } $ uris = [ ] ; $ this -> eventDispatcher -> dispatch ( SpiderEvents :: SPIDER_CRAWL_PRE_DISCOVER ) ; $ crawler = $ resource -> getCrawler ( ) -> filterXPath ( '//a' ) ; foreach ( $ crawler as $ node ) { try { if ( $ node -> getAttribute ( " rel " ) === " nofollow " ) { continue ; } $ href = $ node -> getAttribute ( 'href' ) ; $ uri = new Uri ( $ href , $ baseUrl ) ; $ uris [ ] = $ uri ; } catch ( UriSyntaxException $ e ) { } } $ crawler = $ resource -> getCrawler ( ) -> filterXPath ( '//loc' ) ; foreach ( $ crawler as $ node ) { try { $ href = $ node -> nodeValue ; $ uri = new Uri ( $ href , $ baseUrl ) ; $ uris [ ] = $ uri ; } catch ( UriSyntaxException $ e ) { } } $ this -> eventDispatcher -> dispatch ( SpiderEvents :: SPIDER_CRAWL_POST_DISCOVER , new GenericEvent ( $ this , [ 'uris' => $ uris ] ) ) ; $ this -> persistenceHandler -> persist ( $ resource , $ crawlJob ) ; } }
Function that crawls one webpage based on the give url.
Function that crawls one webpage based on the give url .
//too bad //too bad