PersistenceHandler/RabbitMqPersistenceHandler.php
/**
     * Grabs the content from the crawled page and publishes a job on the queue.
     *
     * @param \VDB\Spider\Resource                      $resource
     * @param \Simgroep\ConcurrentSpiderBundle\CrawlJob $crawlJob
     *
     * @throws \Simgroep\ConcurrentSpiderBundle\InvalidContentException
     */
Grabs the content from the crawled page and publishes a job on the queue.


Grabs the content from the crawled page and publishes a job on the queue .
public function persist(Resource $resource, CrawlJob $crawlJob)
    {
        if (strlen($resource->getResponse()->getBody()) >= $this->maximumResourceSize) {
            throw new InvalidContentException(sprintf('Resource size exceeds limits (%s bytes)', $this->maximumResourceSize));
        }
        $document = $this->documentResolver->getDocumentByResource($resource);
        $persistenceEvent = new PersistenceEvent($document, $resource, $crawlJob->getMetadata());
        $this->eventDispatcher->dispatch(PersistenceEvents::PRE_PERSIST, $persistenceEvent);
        $message = new AMQPMessage(
            json_encode(
                array_merge(
                    ['document' => $document->toArray()],
                    ['metadata' => $crawlJob->getMetadata()]
                )
            ),
            ['delivery_mode' => 1]
        );
        $this->queue->publish($message);
    }
public function persist ( Resource $ resource , CrawlJob $ crawlJob ) { if ( strlen ( $ resource -> getResponse ( ) -> getBody ( ) ) >= $ this -> maximumResourceSize ) { throw new InvalidContentException ( sprintf ( 'Resource size exceeds limits (%s bytes)' , $ this -> maximumResourceSize ) ) ; } $ document = $ this -> documentResolver -> getDocumentByResource ( $ resource ) ; $ persistenceEvent = new PersistenceEvent ( $ document , $ resource , $ crawlJob -> getMetadata ( ) ) ; $ this -> eventDispatcher -> dispatch ( PersistenceEvents :: PRE_PERSIST , $ persistenceEvent ) ; $ message = new AMQPMessage ( json_encode ( array_merge ( [ 'document' => $ document -> toArray ( ) ] , [ 'metadata' => $ crawlJob -> getMetadata ( ) ] ) ) , [ 'delivery_mode' => 1 ] ) ; $ this -> queue -> publish ( $ message ) ; }
Grabs the content from the crawled page and publishes a job on the queue.
Grabs the content from the crawled page and publishes a job on the queue .