EventListener/DiscoverUrlListener.php
/**
     * Writes the found URL as a job on the queue.
     *
     * And URL is only persisted to the queue when it not has been indexed yet.
     *
     * @param \Symfony\Component\EventDispatcher\GenericEvent $event
     */
Writes the found URL as a job on the queue.
And URL is only persisted to the queue when it not has been indexed yet.


Writes the found URL as a job on the queue . And URL is only persisted to the queue when it not has been indexed yet .
public function onDiscoverUrl(Event $event)
    {
        $crawlJob = $event->getSubject()->getCurrentCrawlJob();
        $metadata = $crawlJob->getMetadata();
        $filteredUris = $this->indexer->filterIndexedAndNotExpired($event['uris'], $metadata);
        $setUriKey = sprintf('%s_%s',
            $metadata["core"],
            $this->queue->getName()
        );
        try {
            $this->smembers = $this->redis->smembers($setUriKey);
        } catch (ConnectionException $e) {
            $this->smembers = [];
        }
        foreach ($filteredUris as $uri) {
            $uri = UrlCheck::normalizeUri($uri);
            if (
                UrlCheck::isAllowedToCrawl(
                    UrlCheck::fixUrl($uri->normalize()->toString()),
                    (new Uri($crawlJob->getUrl()))->normalize()->toString(),
                    $crawlJob->getBlacklist(),
                    $crawlJob->getWhitelist()
                ) && !$this->isAlreadyInQueue($uri)
            ) {
                $job = new CrawlJob(
                    UrlCheck::fixUrl($uri->normalize()->toString()),
                    (new Uri($crawlJob->getUrl()))->normalize()->toString(),
                    $crawlJob->getBlacklist(),
                    $crawlJob->getMetadata(),
                    $crawlJob->getWhitelist(),
                    $crawlJob->getQueueName()
                );
                $this->queue->publishJob($job);
            }
        }
        if (count($this->smembers)) {
            try {
                $this->redis->sadd($setUriKey, $this->smembers);
                $this->redis->expire($setUriKey, $this->ttl);
            } catch (ConnectionException $e) {}
        }
    }
public function onDiscoverUrl ( Event $ event ) { $ crawlJob = $ event -> getSubject ( ) -> getCurrentCrawlJob ( ) ; $ metadata = $ crawlJob -> getMetadata ( ) ; $ filteredUris = $ this -> indexer -> filterIndexedAndNotExpired ( $ event [ 'uris' ] , $ metadata ) ; $ setUriKey = sprintf ( '%s_%s' , $ metadata [ " core " ] , $ this -> queue -> getName ( ) ) ; try { $ this -> smembers = $ this -> redis -> smembers ( $ setUriKey ) ; } catch ( ConnectionException $ e ) { $ this -> smembers = [ ] ; } foreach ( $ filteredUris as $ uri ) { $ uri = UrlCheck :: normalizeUri ( $ uri ) ; if ( UrlCheck :: isAllowedToCrawl ( UrlCheck :: fixUrl ( $ uri -> normalize ( ) -> toString ( ) ) , ( new Uri ( $ crawlJob -> getUrl ( ) ) ) -> normalize ( ) -> toString ( ) , $ crawlJob -> getBlacklist ( ) , $ crawlJob -> getWhitelist ( ) ) && ! $ this -> isAlreadyInQueue ( $ uri ) ) { $ job = new CrawlJob ( UrlCheck :: fixUrl ( $ uri -> normalize ( ) -> toString ( ) ) , ( new Uri ( $ crawlJob -> getUrl ( ) ) ) -> normalize ( ) -> toString ( ) , $ crawlJob -> getBlacklist ( ) , $ crawlJob -> getMetadata ( ) , $ crawlJob -> getWhitelist ( ) , $ crawlJob -> getQueueName ( ) ) ; $ this -> queue -> publishJob ( $ job ) ; } } if ( count ( $ this -> smembers ) ) { try { $ this -> redis -> sadd ( $ setUriKey , $ this -> smembers ) ; $ this -> redis -> expire ( $ setUriKey , $ this -> ttl ) ; } catch ( ConnectionException $ e ) { } } }
Writes the found URL as a job on the queue.
Writes the found URL as a job on the queue .