Java提取网页树节点URL的三种高效方法

<dependency>
    <groupId>org.jsoup</groupId>
    <artifactId>jsoup</artifactId>
    <version>1.16.1</version>
</dependency>

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

public class TreeUrlParser {

    public static List<String> getChildUrls(String url, String targetNodeId) throws IOException {
        Document doc = Jsoup.connect(url).get();
        Element targetNode = doc.getElementById(targetNodeId);

        List<String> childUrls = new ArrayList<>();

        if (targetNode != null) {
            Elements links = targetNode.select("a[href]");
            for (Element link : links) {
                String href = link.absUrl("href"); // 获取绝对路径
                childUrls.add(href);
            }
        }

        return childUrls;
    }

    public static void main(String[] args) {
        try {
            List<String> urls = getChildUrls("https://example.com/tree-page", "tree-node-id");
            urls.forEach(System.out::println);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

<dependency>
    <groupId>org.seleniumhq.selenium</groupId>
    <artifactId>selenium-java</artifactId>
    <version>4.19.0</version>
</dependency>

import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.chrome.ChromeDriver;

import java.util.ArrayList;
import java.util.List;

public class DynamicTreeUrlParser {

    public static List<String> getChildUrls(String pageUrl, String targetNodeId) {
        System.setProperty("webdriver.chrome.driver", "path/to/chromedriver");

        WebDriver driver = new ChromeDriver();
        driver.get(pageUrl);

        WebElement targetNode = driver.findElement(By.id(targetNodeId));
        List<WebElement> links = targetNode.findElements(By.tagName("a"));

        List<String> childUrls = new ArrayList<>();
        for (WebElement link : links) {
            String href = link.getAttribute("href");
            if (href != null && !href.isEmpty()) {
                childUrls.add(href);
            }
        }

        driver.quit();
        return childUrls;
    }

    public static void main(String[] args) {
        List<String> urls = getChildUrls("https://example.com/dynamic-tree-page", "tree-node-id");
        urls.forEach(System.out::println);
    }
}

<dependency>
    <groupId>org.apache.httpcomponents.client5</groupId>
    <artifactId>httpclient5</artifactId>
    <version>5.2.1</version>
</dependency>
<dependency>
    <groupId>org.jsoup</groupId>
    <artifactId>jsoup</artifactId>
    <version>1.16.1</version>
</dependency>

import org.apache.hc.client5.http.classic.methods.HttpGet;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.core5.http.ClassicHttpResponse;
import org.apache.hc.core5.http.HttpEntity;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;

public class HttpClientJsoupParser {

    public static List<String> getChildUrls(String url, String targetNodeId) throws Exception {
        CloseableHttpClient httpClient = HttpClients.createDefault();
        HttpGet request = new HttpGet(url);

        ClassicHttpResponse response = httpClient.execute(request);
        HttpEntity entity = response.getEntity();

        List<String> childUrls = new ArrayList<>();

        if (entity != null) {
            BufferedReader reader = new BufferedReader(new InputStreamReader(entity.getContent()));
            StringBuilder content = new StringBuilder();
            String line;

            while ((line = reader.readLine()) != null) {
                content.append(line);
            }

            Document doc = Jsoup.parse(content.toString());
            org.jsoup.nodes.Element targetNode = doc.getElementById(targetNodeId);

            if (targetNode != null) {
                targetNode.select("a[href]").forEach(link -> {
                    childUrls.add(link.absUrl("href"));
                });
            }
        }

        return childUrls;
    }

    public static void main(String[] args) {
        try {
            List<String> urls = getChildUrls("https://example.com/tree-page", "tree-node-id");
            urls.forEach(System.out::println);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}