src/java/fr/jayasoft/ivy/url/ApacheURLLister.java
/**
     * Retrieves a {@link List} of {@link URL}s corresponding to the files
     * and/or directories found at the supplied base URL.
     *
     * @param url The base URL from which to retrieve the listing.
     * @param includeFiles If true include files in the returned list.
     * @param includeDirectories If true include directories in the returned
     *        list.
     *
     * @return A {@link List} of {@link URL}s.
     *
     * @throws IOException If an error occures retrieving the HTML.
     */
Retrieves a  List of  URLs corresponding to the files
and/or directories found at the supplied base URL.

@param url The base URL from which to retrieve the listing.
@param includeFiles If true include files in the returned list.
@param includeDirectories If true include directories in the returned
list.

@return A  List of  URLs.

@throws IOException If an error occures retrieving the HTML.
Retrieves a List of URLs corresponding to the files and / or directories found at the supplied base URL . @param url The base URL from which to retrieve the listing . @param includeFiles If true include files in the returned list . @param includeDirectories If true include directories in the returned list . @return A List of URLs . @throws IOException If an error occures retrieving the HTML .
public List retrieveListing(URL url, boolean includeFiles,
        boolean includeDirectories) throws IOException {
        List urlList = new ArrayList();
        if (!url.getPath().endsWith("/") && !url.getPath().endsWith(".html")) {
            url = new URL(url.getProtocol(), url.getHost(), url.getPort(),
                    url.getPath() + "/");
        }
        BufferedReader r =
            new BufferedReader(new InputStreamReader(URLHandlerRegistry.getDefault()
                                                                       .openStream(url)));
        String htmlText = FileUtil.readEntirely(r);
        Matcher matcher = PATTERN.matcher(htmlText);
        while (matcher.find()) {
            String href = matcher.group(1);
            String text = matcher.group(2);
            if ((href == null) || (text == null)) {
                continue;
            }
            text = text.trim(); 
            if (href.startsWith("/")) {
                int slashIndex = href.substring(0, href.length() - 1).lastIndexOf('/');
                href= href.substring(slashIndex+1);
            }
            int dotIndex = text.indexOf('.');
            if ( ((dotIndex != -1) && !href.startsWith(text.substring(0, dotIndex)))
                || ((dotIndex == -1) && !href.equalsIgnoreCase(text)) ) {
                continue;
            }
            boolean directory = href.endsWith("/");
            if ((directory && includeDirectories)
                || (!directory && includeFiles)) {
                URL child = new URL(url, href);
                urlList.add(child);
                Message.debug("ApacheURLLister found URL=[" + child + "].");
            }
        }
        return urlList;
    }
public List retrieveListing ( URL url , boolean includeFiles , boolean includeDirectories ) throws IOException { List urlList = new ArrayList ( ) ; if ( ! url . getPath ( ) . endsWith ( "/" ) && ! url . getPath ( ) . endsWith ( ".html" ) ) { url = new URL ( url . getProtocol ( ) , url . getHost ( ) , url . getPort ( ) , url . getPath ( ) + "/" ) ; } BufferedReader r = new BufferedReader ( new InputStreamReader ( URLHandlerRegistry . getDefault ( ) . openStream ( url ) ) ) ; String htmlText = FileUtil . readEntirely ( r ) ; Matcher matcher = PATTERN . matcher ( htmlText ) ; while ( matcher . find ( ) ) { String href = matcher . group ( 1 ) ; String text = matcher . group ( 2 ) ; if ( ( href == null ) || ( text == null ) ) { continue ; } text = text . trim ( ) ; if ( href . startsWith ( "/" ) ) { int slashIndex = href . substring ( 0 , href . length ( ) - 1 ) . lastIndexOf ( '/' ) ; href = href . substring ( slashIndex + 1 ) ; } int dotIndex = text . indexOf ( '.' ) ; if ( ( ( dotIndex != - 1 ) && ! href . startsWith ( text . substring ( 0 , dotIndex ) ) ) || ( ( dotIndex == - 1 ) && ! href . equalsIgnoreCase ( text ) ) ) { continue ; } boolean directory = href . endsWith ( "/" ) ; if ( ( directory && includeDirectories ) || ( ! directory && includeFiles ) ) { URL child = new URL ( url , href ) ; urlList . add ( child ) ; Message . debug ( "ApacheURLLister found URL=[" + child + "]." ) ; } } return urlList ; }
Retrieves a {@link List} of {@link URL}s corresponding to the files
 and/or directories found at the supplied base URL.
Retrieves a { @link List } of { @link URL } s corresponding to the files and / or directories found at the supplied base URL .
// add trailing slash for relative urls
 // get the href text and the displayed text
 // the groups were not found (shouldn't happen, really)
 // absolute href: convert to relative one
 // exclude those where they do not match
 // href will never be truncated, text may be truncated by apache
 // may have a '.' from either the extension (.jar) or "..&gt;"
 // the href and the text do not "match"
