public ArrayList<Table> extract(File pdfFile, String outputDirPath) {
	m_docInfo = new DocInfo();
	xml_root = new Element("document");// create the root element:<document
					   // id="x.x.x.x.pdf"></document>
	xml_root.setAttribute("id",
		pdfFile.getName().substring(0, pdfFile.getName().length() - 4));
	Element xml_tables = new Element("tables");
	xml_root.addContent(xml_tables);
	xml_doc = new Document(xml_root); // prepare an empty XML doc with root
					  // <Tables>

	// System.out.printf("extracting tables from %s...\n",
	// pdfFile.getName());

	if (m_parser == null) {
	    System.out.printf("[Error] no parser is set\n");
	    return null;
	} else {
	    /*
	     * step 1: extracts words from the PDF document
	     */
	    ArrayList<ArrayList<TextPiece>> wordsByPage = m_parser
		    .getTextPiecesByPage(pdfFile);
	    tables = new ArrayList<Table>();

	    if (wordsByPage != null && wordsByPage.size() != 0) { // in order to
								  // keep the
								  // fast speed
								  // for large
								  // repositories,
								  // we do not
								  // process PDF
								  // files with
								  // 150+ pages
		int pageNum = wordsByPage.size();
		if (pageNum <= 150) {
		    /*
		     * Judges whether the file is in HTML code or other unknown
		     * codes
		     */
		    boolean unknownCodedDoc = unknownByPDFExtractor(wordsByPage);
		    if (unknownCodedDoc) {
			unknownCodedPDF++;
			m_docInfo
				.setErrorMsg("Sorry, this PDF document is coded with the special coders, which PDFBOX can not process successfully.");
			// System.out.println(unknownCodedPDF +
			// " PDF files are in an unknown code, which can not be processed by PDFBOX");
		    }
		    boolean isHTMLCode = true;
		    isHTMLCode = judgeHTMLCode(wordsByPage);
		    if (isHTMLCode) {
			HTMLCodePDF++;
			m_docInfo
				.setErrorMsg("Sorry, this PDF document is coded in HTML code. TableSeer does not process such files now.");
			// convertHTMLCode(wordsByPage);
			// System.out.println(HTMLCodePDF +
			// " PDF files are in HTML code, which can not be processed by PDFBOX");
		    }

		    /*
		     * currently, we only process those PDF documents that the
		     * PDF text extraction tools can process
		     */
		    // if (unknownCodedDoc==false && isHTMLCode==false) { //too
		    // much PDF files will be filtered out using this condition
		    if (isHTMLCode == false) {

			m_docInfo = computeParametersForDoc(wordsByPage);

			/*
			 * step 2: combines words into Lines, and calculates the
			 * averageLineWidth of docInfo
			 */
			ArrayList<ArrayList<TextPiece>> linesByPage = new ArrayList<ArrayList<TextPiece>>();
			float aveLineWidth = 0.0f;
			int ii = 0;
			for (ArrayList<TextPiece> wordsOfAPage : wordsByPage) {
			    ArrayList<TextPiece> linesOfAPage = combineLines(wordsOfAPage);

			    linesOfAPage = dataCleaning(linesOfAPage); // there
								       // are
								       // some
								       // noisy
								       // empty
								       // lines,
								       // we
								       // should
								       // clean
								       // them
								       // out

			    linesByPage.add(linesOfAPage);

			    for (int i = 0; i < linesOfAPage.size(); i++) {
				if (Math.max(linesOfAPage.get(i).getFontSize(),
					linesOfAPage.get(i).getXScale()) == m_docInfo
					.getBodyTextScale()) {
				    aveLineWidth = aveLineWidth
					    + linesOfAPage.get(i).getWidth();
				    ii++;
				}
			    }
			}
			aveLineWidth = aveLineWidth / (float) ii;
			if (aveLineWidth < 50.0f)
			    aveLineWidth = 175.0f; // TODO: manually fix the
						   // parameters here, we should
						   // fix it permanently
			m_docInfo.setAverageLineWidth(aveLineWidth);

			for (ArrayList<TextPiece> linesOfAPage : linesByPage) {
			    for (int i = 1; i < linesOfAPage.size() - 1; i++) {
				if ((linesOfAPage.get(i).getWidth() >= m_docInfo
					.getAverageLineWidth())
					&& (Math.abs(linesOfAPage.get(i).getY()
						- linesOfAPage.get(i - 1)
							.getY()) > 3.0)
					&& (Math.abs(linesOfAPage.get(i + 1)
						.getY()
						- linesOfAPage.get(i).getY()) > 3.0))
				    linesOfAPage.get(i).setSparseLine(false);
			    }
			}

			String meta = "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\"?>\n";
			meta = meta + "<Tables>\n";
			meta = meta + "<FileName>" + pdfFile.getName()
				+ "</FileName>\n";
			meta = meta + "<pageNum>" + pageNum + "</pageNum>\n";

			xml_root.addContent(new Element("FileName")
				.addContent(pdfFile.getName()));
			xml_root.addContent(new Element("pageNum")
				.addContent(Integer.toString(pageNum)));
			m_docInfo.setDocMeta(xml_doc);
			m_docInfo.setDocMeta(meta);
			m_docInfo.setTableMetadata(meta);
			if (Config.DEBUG_MODE)
			    Debug.printMiddleResults(wordsByPage, linesByPage,
				    outputDirPath, pdfFile);

			/*
			 * step 3: identifies tables, Loop over pages
			 */
			int i = 0;
			for (ArrayList<TextPiece> linesOfAPage : linesByPage) {
			    // System.out.println("Page: " + (i+1));
			    ArrayList<TextPiece> wordsOfAPage = wordsByPage
				    .get(i);
			    extractTablesFromAPage(linesOfAPage, tables,
				    wordsOfAPage, i, pdfFile, linesByPage);

			    // extractFiguresFromAPage(linesOfAPage,
			    // wordsOfAPage, i, pdfFile, linesByPage);
			    i++;
			}
			meta = m_docInfo.getTableMetaInThisDoc();
			meta = meta + "<errorMsg>" + m_docInfo.getErrorMsg()
				+ "</errorMsg>\n";
			meta = meta + "</Tables>\n";
			// System.out.println(meta);

			/*
			 * TODO [improve]: generates XML from tables here
			 */
			if (tables.size() != 0) {
			    for (int k = 0; k < tables.size(); k++) {
				Table eTable = tables.get(k);
				Element tabRec = new Element("tableRecord")
					.setAttribute("id",
						(Integer.toString(k)));
				Element table = new Element("table");
				if (eTable.getHeight() != 0
					&& eTable.getWidth() != 0) {
				    table.setAttribute("height", Integer
					    .toString((int) eTable.getHeight()));
				    table.setAttribute("width", Integer
					    .toString((int) eTable.getWidth()));
				} else {
				    table.setAttribute("height",
					    Integer.toString(300));
				    table.setAttribute("width",
					    Integer.toString(400));
				}
				table.setAttribute("border",
					Integer.toString(1));
				tabRec.addContent(new Element("pageNumInDoc")
					.addContent(Integer.toString(eTable
						.getPageNumber())));
				tabRec.addContent(new Element("caption")
					.addContent(eTable.getCaption()));

				// table column headings
				Element tHead = new Element("thead");
				Element tRc = new Element("tr");
				tHead.addContent(tRc);
				ArrayList<String> colHeading = eTable
					.getHeading();
				for (int cloop = 0; cloop < colHeading.size(); cloop++) {
				    String head = colHeading.get(cloop);
				    tRc.addContent(new Element("th")
					    .addContent(head));
				}

				// table row contents
				Element tBody = new Element("tbody");

				ArrayList<String> rows = eTable.getTableBody();
				for (int rloop = 0; rloop < rows.size(); rloop++) {
				    Element tRr = new Element("tr");
				    String[] cells = rows.get(rloop).split(";");
				    for (int inner = 0; inner < cells.length; inner++) {
					Element tD = new Element("td")
						.addContent(cells[inner]);
					tRr.addContent(tD);
				    }
				    tBody.addContent(tRr);
				}

				table.addContent(tHead);
				table.addContent(tBody);
				tabRec.addContent(table);
				tabRec.addContent(new Element("footnote")
					.addContent(eTable.getFootNote()));
				tabRec.addContent(new Element("referenceText")
					.addContent(eTable.getRefTextList()));
				xml_tables.addContent(tabRec);
			    }
			}
			xml_root.addContent(new Element("errorMsg")
				.addContent(m_docInfo.getErrorMsg()));

			/*
			 * standard XML finish generating here, ready to output
			 */
			m_docInfo.setTableMetadata(meta);
			// System.out.println(meta);
			if (Config.DEBUG_MODE) { // print table metadata ordered
						 // by documents
			    Debug.printTableMeta(outputDirPath, pdfFile, meta);
			    Debug.printStatInfo(outputDirPath, pdfFile,
				    pageNum, m_docInfo.getTableNum());
			}
		    }// end if( not unknowCode or HTML code)
		    return tables;

		}// end if (page Num<=150)
		else {
		    largeDocNum++;
		    // System.out.println("Sorry, for the scalability and speed reasons, we do not process the PDF documents that are TOO LARGE! "
		    // + largeDocNum);
		    m_docInfo
			    .setErrorMsg("Sorry, for the scalability and speed reasons, we do not process the PDF documents that are TOO LARGE!");
		    return null;
		}
	    } else {
		noTextDocNum++;
		// System.out.println("Sorry, PDFBOX can not extract any text from this PDF document! "
		// + noTextDocNum);
		m_docInfo
			.setErrorMsg("Sorry, PDFBOX can not extract any text from this PDF document!");
		return null;
	    }
	}// end else()
    }
public ArrayList < Table > extract ( File pdfFile , String outputDirPath ) { m_docInfo = new DocInfo ( ) ; xml_root = new Element ( "document" ) ; xml_root . setAttribute ( "id" , pdfFile . getName ( ) . substring ( 0 , pdfFile . getName ( ) . length ( ) - 4 ) ) ; Element xml_tables = new Element ( "tables" ) ; xml_root . addContent ( xml_tables ) ; xml_doc = new Document ( xml_root ) ; if ( m_parser == null ) { System . out . printf ( "[Error] no parser is set\n" ) ; return null ; } else { ArrayList < ArrayList < TextPiece > > wordsByPage = m_parser . getTextPiecesByPage ( pdfFile ) ; tables = new ArrayList < Table > ( ) ; if ( wordsByPage != null && wordsByPage . size ( ) != 0 ) { int pageNum = wordsByPage . size ( ) ; if ( pageNum <= 150 ) { boolean unknownCodedDoc = unknownByPDFExtractor ( wordsByPage ) ; if ( unknownCodedDoc ) { unknownCodedPDF ++ ; m_docInfo . setErrorMsg ( "Sorry, this PDF document is coded with the special coders, which PDFBOX can not process successfully." ) ; } boolean isHTMLCode = true ; isHTMLCode = judgeHTMLCode ( wordsByPage ) ; if ( isHTMLCode ) { HTMLCodePDF ++ ; m_docInfo . setErrorMsg ( "Sorry, this PDF document is coded in HTML code. TableSeer does not process such files now." ) ; } if ( isHTMLCode == false ) { m_docInfo = computeParametersForDoc ( wordsByPage ) ; ArrayList < ArrayList < TextPiece > > linesByPage = new ArrayList < ArrayList < TextPiece > > ( ) ; float aveLineWidth = 0.0f ; int ii = 0 ; for ( ArrayList < TextPiece > wordsOfAPage : wordsByPage ) { ArrayList < TextPiece > linesOfAPage = combineLines ( wordsOfAPage ) ; linesOfAPage = dataCleaning ( linesOfAPage ) ; linesByPage . add ( linesOfAPage ) ; for ( int i = 0 ; i < linesOfAPage . size ( ) ; i ++ ) { if ( Math . max ( linesOfAPage . get ( i ) . getFontSize ( ) , linesOfAPage . get ( i ) . getXScale ( ) ) == m_docInfo . getBodyTextScale ( ) ) { aveLineWidth = aveLineWidth + linesOfAPage . get ( i ) . getWidth ( ) ; ii ++ ; } } } aveLineWidth = aveLineWidth / ( float ) ii ; if ( aveLineWidth < 50.0f ) aveLineWidth = 175.0f ; m_docInfo . setAverageLineWidth ( aveLineWidth ) ; for ( ArrayList < TextPiece > linesOfAPage : linesByPage ) { for ( int i = 1 ; i < linesOfAPage . size ( ) - 1 ; i ++ ) { if ( ( linesOfAPage . get ( i ) . getWidth ( ) >= m_docInfo . getAverageLineWidth ( ) ) && ( Math . abs ( linesOfAPage . get ( i ) . getY ( ) - linesOfAPage . get ( i - 1 ) . getY ( ) ) > 3.0 ) && ( Math . abs ( linesOfAPage . get ( i + 1 ) . getY ( ) - linesOfAPage . get ( i ) . getY ( ) ) > 3.0 ) ) linesOfAPage . get ( i ) . setSparseLine ( false ) ; } } String meta = "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\"?>\n" ; meta = meta + "<Tables>\n" ; meta = meta + "<FileName>" + pdfFile . getName ( ) + "</FileName>\n" ; meta = meta + "<pageNum>" + pageNum + "</pageNum>\n" ; xml_root . addContent ( new Element ( "FileName" ) . addContent ( pdfFile . getName ( ) ) ) ; xml_root . addContent ( new Element ( "pageNum" ) . addContent ( Integer . toString ( pageNum ) ) ) ; m_docInfo . setDocMeta ( xml_doc ) ; m_docInfo . setDocMeta ( meta ) ; m_docInfo . setTableMetadata ( meta ) ; if ( Config . DEBUG_MODE ) Debug . printMiddleResults ( wordsByPage , linesByPage , outputDirPath , pdfFile ) ; int i = 0 ; for ( ArrayList < TextPiece > linesOfAPage : linesByPage ) { ArrayList < TextPiece > wordsOfAPage = wordsByPage . get ( i ) ; extractTablesFromAPage ( linesOfAPage , tables , wordsOfAPage , i , pdfFile , linesByPage ) ; i ++ ; } meta = m_docInfo . getTableMetaInThisDoc ( ) ; meta = meta + "<errorMsg>" + m_docInfo . getErrorMsg ( ) + "</errorMsg>\n" ; meta = meta + "</Tables>\n" ; if ( tables . size ( ) != 0 ) { for ( int k = 0 ; k < tables . size ( ) ; k ++ ) { Table eTable = tables . get ( k ) ; Element tabRec = new Element ( "tableRecord" ) . setAttribute ( "id" , ( Integer . toString ( k ) ) ) ; Element table = new Element ( "table" ) ; if ( eTable . getHeight ( ) != 0 && eTable . getWidth ( ) != 0 ) { table . setAttribute ( "height" , Integer . toString ( ( int ) eTable . getHeight ( ) ) ) ; table . setAttribute ( "width" , Integer . toString ( ( int ) eTable . getWidth ( ) ) ) ; } else { table . setAttribute ( "height" , Integer . toString ( 300 ) ) ; table . setAttribute ( "width" , Integer . toString ( 400 ) ) ; } table . setAttribute ( "border" , Integer . toString ( 1 ) ) ; tabRec . addContent ( new Element ( "pageNumInDoc" ) . addContent ( Integer . toString ( eTable . getPageNumber ( ) ) ) ) ; tabRec . addContent ( new Element ( "caption" ) . addContent ( eTable . getCaption ( ) ) ) ; Element tHead = new Element ( "thead" ) ; Element tRc = new Element ( "tr" ) ; tHead . addContent ( tRc ) ; ArrayList < String > colHeading = eTable . getHeading ( ) ; for ( int cloop = 0 ; cloop < colHeading . size ( ) ; cloop ++ ) { String head = colHeading . get ( cloop ) ; tRc . addContent ( new Element ( "th" ) . addContent ( head ) ) ; } Element tBody = new Element ( "tbody" ) ; ArrayList < String > rows = eTable . getTableBody ( ) ; for ( int rloop = 0 ; rloop < rows . size ( ) ; rloop ++ ) { Element tRr = new Element ( "tr" ) ; String [ ] cells = rows . get ( rloop ) . split ( ";" ) ; for ( int inner = 0 ; inner < cells . length ; inner ++ ) { Element tD = new Element ( "td" ) . addContent ( cells [ inner ] ) ; tRr . addContent ( tD ) ; } tBody . addContent ( tRr ) ; } table . addContent ( tHead ) ; table . addContent ( tBody ) ; tabRec . addContent ( table ) ; tabRec . addContent ( new Element ( "footnote" ) . addContent ( eTable . getFootNote ( ) ) ) ; tabRec . addContent ( new Element ( "referenceText" ) . addContent ( eTable . getRefTextList ( ) ) ) ; xml_tables . addContent ( tabRec ) ; } } xml_root . addContent ( new Element ( "errorMsg" ) . addContent ( m_docInfo . getErrorMsg ( ) ) ) ; m_docInfo . setTableMetadata ( meta ) ; if ( Config . DEBUG_MODE ) { Debug . printTableMeta ( outputDirPath , pdfFile , meta ) ; Debug . printStatInfo ( outputDirPath , pdfFile , pageNum , m_docInfo . getTableNum ( ) ) ; } } return tables ; } else { largeDocNum ++ ; m_docInfo . setErrorMsg ( "Sorry, for the scalability and speed reasons, we do not process the PDF documents that are TOO LARGE!" ) ; return null ; } } else { noTextDocNum ++ ; m_docInfo . setErrorMsg ( "Sorry, PDFBOX can not extract any text from this PDF document!" ) ; return null ; } } }
Extracts tables from a PDF document

@param pdfFile
PDF file to process
@param outputDirPath
the directory path to save the extracted tables, ONLY IN DEBUG
@return extracted table set / null (if error happens)