private ExtractResult extractHtmlTemplate(String url, HtmlTemplate htmlTemplate, Document doc) {
        //一个页面模板对应一个抽取结果
        ExtractResult extractResult = new ExtractResult();
        extractResult.setUrl(url);
        extractResult.setTableName(htmlTemplate.getTableName());
        List<CssPath> cssPaths = htmlTemplate.getCssPaths();
	    //页面模板中定义的所有CSS路径和抽取表达式全部抽取成功，才算抽取成功
        //只要有一个CSS路径或抽取表达式失败，就是抽取失败
        for (CssPath cssPath : cssPaths) {
            // 抽取一条CSS PATH
            Elements elements = doc.select(cssPath.getCssPath());
            // 如果CSS路径匹配多个元素，则抽取字段为多值
            for (Element element : elements) {
                String text = null;
                if(StringUtils.isBlank(cssPath.getAttr())){
                    //提取文本
                    text = element.text();
                }else{
                    //提取属性
                    text = element.attr(cssPath.getAttr());
                }
                if (StringUtils.isNotBlank(text)) {
                    // 成功提取文本
                    if (cssPath.hasExtractFunction()) {
                        //使用CSS路径下的抽取函数做进一步抽取
                        for (ExtractFunction pf : cssPath.getExtractFunctions()) {
                            text = ExtractFunctionExecutor.execute(text, doc, cssPath, pf.getExtractExpression());
                            if (text != null) {
                                ExtractResultItem extractResultItem = new ExtractResultItem();
                                extractResultItem.setField(pf.getFieldName());
                                extractResultItem.setValue(text);
                                extractResult.addExtractResultItem(extractResultItem);
                            } else {
                                ExtractFailLog extractFailLog = new ExtractFailLog();
                                extractFailLog.setUrl(url);
                                extractFailLog.setUrlPattern(htmlTemplate.getUrlPattern().getUrlPattern());
                                extractFailLog.setTemplateName(htmlTemplate.getTemplateName());
                                extractFailLog.setCssPath(cssPath.getCssPath());
                                extractFailLog.setExtractExpression(pf.getExtractExpression());
                                extractFailLog.setTableName(htmlTemplate.getTableName());
                                extractFailLog.setFieldName(pf.getFieldName());
                                extractFailLog.setFieldDescription(pf.getFieldDescription());
                                extractResult.addExtractFailLog(extractFailLog);
                                //未抽取到结果，保存抽取失败日志并停止抽取，抽取失败
                                //快速失败模式
                                //如果要记录所有失败日志，则去除下面一行返回的代码
                                return extractResult;
                            }
                        }
                    } else {
                        //使用CSS路径抽取的结果
                        ExtractResultItem extractResultItem = new ExtractResultItem();
                        extractResultItem.setField(cssPath.getFieldName());
                        extractResultItem.setValue(text);
                        extractResult.addExtractResultItem(extractResultItem);
                    }
                } else {
                    //未抽取到结果，保存抽取失败日志并停止抽取，抽取失败
                    ExtractFailLog extractFailLog = new ExtractFailLog();
                    extractFailLog.setUrl(url);
                    extractFailLog.setUrlPattern(htmlTemplate.getUrlPattern().getUrlPattern());
                    extractFailLog.setTemplateName(htmlTemplate.getTemplateName());
                    extractFailLog.setCssPath(cssPath.getCssPath());
                    extractFailLog.setExtractExpression("");
                    extractFailLog.setTableName(htmlTemplate.getTableName());
                    extractFailLog.setFieldName(cssPath.getFieldName());
                    extractFailLog.setFieldDescription(cssPath.getFieldDescription());
                    extractResult.addExtractFailLog(extractFailLog);
                    //未抽取到结果，保存抽取失败日志并停止抽取，抽取失败
                    //快速失败模式
                    //如果要记录所有失败日志，则去除下面一行返回的代码
                    return extractResult;
                }
            }
        }
        return extractResult;
    }
private ExtractResult extractHtmlTemplate ( String url , HtmlTemplate htmlTemplate , Document doc ) { ExtractResult extractResult = new ExtractResult ( ) ; extractResult . setUrl ( url ) ; extractResult . setTableName ( htmlTemplate . getTableName ( ) ) ; List < CssPath > cssPaths = htmlTemplate . getCssPaths ( ) ; for ( CssPath cssPath : cssPaths ) { Elements elements = doc . select ( cssPath . getCssPath ( ) ) ; for ( Element element : elements ) { String text = null ; if ( StringUtils . isBlank ( cssPath . getAttr ( ) ) ) { text = element . text ( ) ; } else { text = element . attr ( cssPath . getAttr ( ) ) ; } if ( StringUtils . isNotBlank ( text ) ) { if ( cssPath . hasExtractFunction ( ) ) { for ( ExtractFunction pf : cssPath . getExtractFunctions ( ) ) { text = ExtractFunctionExecutor . execute ( text , doc , cssPath , pf . getExtractExpression ( ) ) ; if ( text != null ) { ExtractResultItem extractResultItem = new ExtractResultItem ( ) ; extractResultItem . setField ( pf . getFieldName ( ) ) ; extractResultItem . setValue ( text ) ; extractResult . addExtractResultItem ( extractResultItem ) ; } else { ExtractFailLog extractFailLog = new ExtractFailLog ( ) ; extractFailLog . setUrl ( url ) ; extractFailLog . setUrlPattern ( htmlTemplate . getUrlPattern ( ) . getUrlPattern ( ) ) ; extractFailLog . setTemplateName ( htmlTemplate . getTemplateName ( ) ) ; extractFailLog . setCssPath ( cssPath . getCssPath ( ) ) ; extractFailLog . setExtractExpression ( pf . getExtractExpression ( ) ) ; extractFailLog . setTableName ( htmlTemplate . getTableName ( ) ) ; extractFailLog . setFieldName ( pf . getFieldName ( ) ) ; extractFailLog . setFieldDescription ( pf . getFieldDescription ( ) ) ; extractResult . addExtractFailLog ( extractFailLog ) ; return extractResult ; } } } else { ExtractResultItem extractResultItem = new ExtractResultItem ( ) ; extractResultItem . setField ( cssPath . getFieldName ( ) ) ; extractResultItem . setValue ( text ) ; extractResult . addExtractResultItem ( extractResultItem ) ; } } else { ExtractFailLog extractFailLog = new ExtractFailLog ( ) ; extractFailLog . setUrl ( url ) ; extractFailLog . setUrlPattern ( htmlTemplate . getUrlPattern ( ) . getUrlPattern ( ) ) ; extractFailLog . setTemplateName ( htmlTemplate . getTemplateName ( ) ) ; extractFailLog . setCssPath ( cssPath . getCssPath ( ) ) ; extractFailLog . setExtractExpression ( "" ) ; extractFailLog . setTableName ( htmlTemplate . getTableName ( ) ) ; extractFailLog . setFieldName ( cssPath . getFieldName ( ) ) ; extractFailLog . setFieldDescription ( cssPath . getFieldDescription ( ) ) ; extractResult . addExtractFailLog ( extractFailLog ) ; return extractResult ; } } } return extractResult ; }
根据模板的定义抽取信息
@param url html页面路径
@param htmlTemplate html页面模板
@param doc jsoup文档
@return 抽取结果