DocumentResolver/Type/Html.php
/**
     * Extracts all text content from the crawled resource exception javascript and style.
     *
     * @param \VDB\Spider\Resource $resource
     *
     * @return string
     */
Extracts all text content from the crawled resource exception javascript and style.
@param \VDB\Spider\Resource $resource

@return string
Extracts all text content from the crawled resource exception javascript and style . @param \ VDB \ Spider \ Resource $resource @return string
public function extractContentFromResource(Resource $resource)
    {
        $crawler = $resource->getCrawler();
        $content = $resource->getResponse()->getBody(true);
        if (preg_match('/["|\']<div(.*?)<\/div>["|\']/is', $content)) {
            $content = preg_replace('/<script(.*?)<\/script>/is', '', $content);
            $crawler->clear();
            $crawler->addContent($content);
        }
        if (null !== $this->cssBlacklist) {
            $crawler->filter($this->cssBlacklist)->each(function (Crawler $crawler) {
                foreach ($crawler as $node) {
                    $node->parentNode->removeChild($node);
                }
            });
        }
        $query = '//body//*[not(self::script|self::style)]/text()';
        $content = '';
        $crawler->filterXpath($query)->each(
            function (Crawler $crawler) use (&$content) {
                $text = trim($crawler->text());
                if (strlen($text) > 0) {
                    $content .= $text . ' ';
                }
            }
        );
        return trim($content);
    }
public function extractContentFromResource ( Resource $ resource ) { $ crawler = $ resource -> getCrawler ( ) ; $ content = $ resource -> getResponse ( ) -> getBody ( true ) ; if ( preg_match ( '/["|\']<div(.*?)<\/div>["|\']/is' , $ content ) ) { $ content = preg_replace ( '/<script(.*?)<\/script>/is' , '' , $ content ) ; $ crawler -> clear ( ) ; $ crawler -> addContent ( $ content ) ; } if ( null !== $ this -> cssBlacklist ) { $ crawler -> filter ( $ this -> cssBlacklist ) -> each ( function ( Crawler $ crawler ) { foreach ( $ crawler as $ node ) { $ node -> parentNode -> removeChild ( $ node ) ; } } ) ; } $ query = '//body//*[not(self::script|self::style)]/text()' ; $ content = '' ; $ crawler -> filterXpath ( $ query ) -> each ( function ( Crawler $ crawler ) use ( & $ content ) { $ text = trim ( $ crawler -> text ( ) ) ; if ( strlen ( $ text ) > 0 ) { $ content .= $ text . ' ' ; } } ) ; return trim ( $ content ) ; }
Extracts all text content from the crawled resource exception javascript and style.
Extracts all text content from the crawled resource exception javascript and style .
// Check if script block contains div tags