java selenium 获取 script java selenium 获取network

转载

技术博主 2023-10-04 09:18:14

文章标签 chrome java apache 文章分类 Java 后端开发

这两天遇到一个棘手的异常，时不时页面会弹出：“系统繁忙，请稍候再试！”，这时候我们去看网络请求数据，结果状态码全部都是 200，没有其它信息，这压根没法定位不了问题。

这就说明：网络出现异常的时候，仅靠状态码是不够的。我们最好能拿到 http 所有数据，包括：请求头、响应头、请求体、响应体。其中请求头、响应头，可以通过 PERFORMANCE_LOG 拿到，问题都不大。但是请求体与响应体，我们可以拿到么？

这个问题困扰了我整整一天的时间，终于解决了。为什么这么困难？

我们先来看 selenium，它为什么不直接支持这个功能呢？因为开发人员觉得这不是他们目标：

we will not be adding this feature to the WebDriver API as it falls outside of our current scope (emulating user actions).

然后我继续翻网络，发现谷歌的devtools-protocol明确是支持的

那到底有没有办法解决这个问题？

有！且不止一种方法....

第一种方法：我们使用BrowserMob Proxy

首先获取WebDriver及BrowserProxy

import net.lightbody.bmp.BrowserMobProxy;
import net.lightbody.bmp.BrowserMobProxyServer;
import net.lightbody.bmp.client.ClientUtil;
import net.lightbody.bmp.proxy.CaptureType;
import org.openqa.selenium.Proxy;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.logging.LogType;
import org.openqa.selenium.logging.LoggingPreferences;
import org.openqa.selenium.remote.CapabilityType;

import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;

/**
 * BrowserProxy 单例
 *
 * @author ant
 * @date 2019-11-21 10:38:20
 */
public class BrowserProxy {
    /**
     * BrowserMobProxy
     */
    private BrowserMobProxy proxy;

    /**
     * WebDriver
     */
    private WebDriver driver;


    public BrowserMobProxy getProxy() {
        return proxy;
    }

    private void setProxy(BrowserMobProxy proxy) {
        this.proxy = proxy;
    }

    public WebDriver getDriver() {
        return driver;
    }

    private void setDriver(WebDriver driver) {
        this.driver = driver;
    }

    /**
     * 无参构造 初始化参数
     * 私有构造 禁止外部new
     */
    private BrowserProxy() {
        String webDriverPath = System.getProperty("user.dir") + "/resources/chromedriver.exe";
        System.out.println("webDriverPath:"+webDriverPath);
        //String webDriverPath = "E:\\IDEA\\common\\src\\test\\java\\com\\autotest\\tools\\chromedriver.exe";
        System.setProperty("webdriver.chrome.driver", webDriverPath);

        ChromeOptions options = new ChromeOptions();
        // 禁用阻止弹出窗口
        options.addArguments("--disable-popup-blocking");
        // 启动无沙盒模式运行
        options.addArguments("no-sandbox");
        // 禁用扩展
        options.addArguments("disable-extensions");
        // 默认浏览器检查
        options.addArguments("no-default-browser-check");

        Map<String, Object> prefs = new HashMap();
        prefs.put("credentials_enable_service", false);
        prefs.put("profile.password_manager_enabled", false);
        // 禁用保存密码提示框
        options.setExperimentalOption("prefs", prefs);

        // set performance logger
        // this sends Network.enable to chromedriver
        LoggingPreferences logPrefs = new LoggingPreferences();
        logPrefs.enable(LogType.PERFORMANCE, Level.ALL);
        options.setCapability(CapabilityType.LOGGING_PREFS, logPrefs);

        // start the proxy
        BrowserMobProxy proxy = new BrowserMobProxyServer();
        // 端口号
        proxy.start(4566);

        // get the Selenium proxy object
        Proxy seleniumProxy = ClientUtil.createSeleniumProxy(proxy);
        options.setCapability(CapabilityType.PROXY, seleniumProxy);
        // start the browser up
        WebDriver driver = new ChromeDriver(options);
        // 等待10秒
        driver.manage().timeouts().implicitlyWait(10, TimeUnit.SECONDS);

        // enable more detailed HAR capture, if desired (see CaptureType for the complete list)
        proxy.enableHarCaptureTypes(CaptureType.REQUEST_CONTENT, CaptureType.RESPONSE_CONTENT);
        // enable cookies  --> REQUEST_COOKIES, RESPONSE_COOKIES
        proxy.enableHarCaptureTypes(CaptureType.getCookieCaptureTypes());
        // enable headers --> REQUEST_HEADERS, RESPONSE_HEADERS
        //proxy.enableHarCaptureTypes(CaptureType.getHeaderCaptureTypes());
        // 向对象赋值
        this.setProxy(proxy);
        this.setDriver(driver);
    }

    /**
     * 类级的内部类，也就是静态的成员式内部类，该内部类的实例与外部类的实例没有绑定关系，而且只有被调用到才会装载，从而实现了延迟加载
     */
    private static class BrowserProxyHolder {
        private final static BrowserProxy instance = new BrowserProxy();
    }

    /**
     * 提供公有调用方法
     *
     * @return
     */
    public static BrowserProxy getInstance() {
        return BrowserProxyHolder.instance;
    }

}

在这里需要注意的是

proxy.enableHarCaptureTypes(CaptureType.REQUEST_CONTENT, CaptureType.RESPONSE_CONTENT);

如果要获取request content或者response content则必需加上这行参数,否则下面获取不到数据。

那么，如果是获取cookie呢？

proxy.enableHarCaptureTypes(CaptureType.getCookieCaptureTypes());

换了一种写法，其实和上面获取content效果是一样的，当然还有其他写法，很灵活。

准备工作做好了，下面干正事。

import com.example.cat.selenium.BrowserProxy;
import net.lightbody.bmp.BrowserMobProxy;
import net.lightbody.bmp.core.har.*;
import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import java.util.List;


public class BrowserMobProxyTest {

    public void tearDown(WebDriver driver,BrowserMobProxy proxy) {
        if(proxy.isStarted()){
            proxy.stop();
        }
        // 关闭当前窗口
        driver.close();
    }

    public static void main(String[] args) {
        BrowserMobProxyTest browserMobProxyTest =  new BrowserMobProxyTest();
        BrowserProxy browserProxy = BrowserProxy.getInstance();
        WebDriver driver = browserProxy.getDriver();
        BrowserMobProxy proxy = browserProxy.getProxy();
        browserMobProxyTest.testMethod(driver,proxy);
        //browserMobProxyTest.tearDown(driver,proxy);
    }
    public void testMethod(WebDriver driver,BrowserMobProxy proxy) {
        // 这里必需在driver.get(url)之前
        proxy.newHar("test_har");

        driver.get("https://translate.google.cn/");

        String text = "selenium";

        driver.findElement(By.xpath("//*[@id=\"source\"]")).sendKeys(text);

        Har har = proxy.getHar();
        List<HarEntry> list =  har.getLog().getEntries();
        for (HarEntry harEntry : list){
            HarResponse harResponse = harEntry.getResponse();
            String responseBody = harResponse.getContent().getText();
            List<HarCookie> cookies = harResponse.getCookies();
            // network response
            System.out.println("responseBody:"+responseBody);
            // 可获取httpOnly 类型的cookie
            System.out.println("cookies:"+cookies);
        }
    }
}

如此，想获取network response的内容或是获取httpOnly类型的cookie也都ok了。

但是有一个问题，仍未解决

AnnotatedConnectException: Connection timed out: no further information: accounts.google.com/172.217.160.109:443

java selenium 获取 script java selenium 获取network_chrome

为什么会连接connection 这个ip呢？有发现的小伙伴留言交流！

接下来第二种方法我们通过requtesId去实现

package com.example.cat.test.hacpai;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.example.cat.utils.Hclient;
import org.apache.http.HttpResponse;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.util.EntityUtils;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeDriverService;
import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.logging.LogEntries;
import org.openqa.selenium.logging.LogEntry;
import org.openqa.selenium.logging.LogType;
import org.openqa.selenium.logging.Logs;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;


public class ChromeDriverProxy extends ChromeDriver {
    /**
     * method --> Network.responseReceived
     */
    public static final String NETWORK_RESPONSE_RECEIVED = "Network.responseReceived";

    /**
     * post 请求超时时间
     */
    private static final int COMMAND_TIMEOUT = 5000;

    /**
     * 必须固定端口，因为ChromeDriver没有实时获取端口的接口；
     */
    private static final int CHROME_DRIVER_PORT = 9102;

    private static ChromeDriverService driverService = new ChromeDriverService.Builder().usingPort(CHROME_DRIVER_PORT).build();

    public ChromeDriverProxy(ChromeOptions options) {
        super(driverService, options);
    }


    // 根据请求ID获取返回内容
    public String getResponseBody(String requestId) {
        try {
            // CHROME_DRIVER_PORT chromeDriver提供的端口
            String url = String.format("http://localhost:%s/session/%s/goog/cdp/execute",
                    CHROME_DRIVER_PORT, getSessionId());
            HttpPost httpPost = new HttpPost(url);
            JSONObject object = new JSONObject();
            JSONObject params = new JSONObject();
            params.put("requestId", requestId);
            object.put("cmd", "Network.getResponseBody");
            object.put("params", params);
            httpPost.setEntity(new StringEntity(object.toString()));

            RequestConfig requestConfig = RequestConfig
                    .custom()
                    .setSocketTimeout(COMMAND_TIMEOUT)
                    .setConnectTimeout(COMMAND_TIMEOUT).build();

            CloseableHttpClient httpClient = HttpClientBuilder.create()
                    .setDefaultRequestConfig(requestConfig).build();

            HttpResponse response = httpClient.execute(httpPost);
            String s = EntityUtils.toString(response.getEntity());
            return  this.getResponseValue(s);
        } catch (IOException e) {
            //logger.error("getResponseBody failed!", e);
        }
        return null;
    }



    // 根据请求ID获取返回内容
    // https://github.com/bayandin/chromedriver/blob/master/server/http_handler.cc
    public JSONArray getCookies(String requestId) {
        try {
            // CHROME_DRIVER_PORT chromeDriver提供的端口
            String url = String.format("http://localhost:%s/session/%s/cookie",
                    CHROME_DRIVER_PORT, getSessionId());
            String cookieStr = Hclient.doGet(url, null);
           return JSONArray.parseArray(getResponseValue(cookieStr));
        } catch (Exception e) {
            //logger.error("getResponseBody failed!", e);
        }
        return null;
    }
    /**
     * 获取响应结果{"sessionId": "","status": 0,"value": ""}的value
     *
     * @param data
     * @return
     */
    private String getResponseValue(String data){
        JSONObject json = JSONObject.parseObject(data);
        ResponseBodyVo responseBodyVo = JSONObject.toJavaObject(json,ResponseBodyVo.class);
        if(0 == responseBodyVo.getStatus()){
            return responseBodyVo.getValue();
        }else{
            System.out.println("status error:" + data);
            return "";
        }

    }

    public static void saveHttpTransferDataIfNecessary(ChromeDriverProxy driver) {
        Logs logs = driver.manage().logs();
        Set<String> availableLogTypes = logs.getAvailableLogTypes();

        if(availableLogTypes.contains(LogType.PERFORMANCE)) {
            LogEntries logEntries = logs.get(LogType.PERFORMANCE);
            List<ResponseReceivedEvent> responseReceivedEvents = new ArrayList<>();

            for(LogEntry entry : logEntries) {
                JSONObject jsonObj = JSON.parseObject(entry.getMessage()).getJSONObject("message");
                String method = jsonObj.getString("method");
                String params = jsonObj.getString("params");

                if (method.equals(NETWORK_RESPONSE_RECEIVED)) {
                    ResponseReceivedEvent response = JSON.parseObject(params, ResponseReceivedEvent.class);
                    responseReceivedEvents.add(response);
                }
            }
            doSaveHttpTransferDataIfNecessary(driver, responseReceivedEvents);
        }

    }

    // 保存网络请求
    private static void doSaveHttpTransferDataIfNecessary(ChromeDriverProxy driver, List<ResponseReceivedEvent> responses) {
        for(ResponseReceivedEvent responseReceivedEvent : responses) {
            String url = JSONObject.parseObject(responseReceivedEvent.getResponse()).getString("url");
            boolean staticFiles = url.endsWith(".png")
                    || url.endsWith(".jpg")
                    || url.endsWith(".css")
                    || url.endsWith(".ico")
                    || url.endsWith(".js")
                    || url.endsWith(".gif");

            if(!staticFiles && url.startsWith("http")) {
                // 使用上面开发的接口获取返回数据
                String  bodyx= driver.getResponseBody(responseReceivedEvent.getRequestId());

                System.out.println("url:"+url+" ,body-->"+bodyx);
                JSONArray cookies = driver.getCookies(responseReceivedEvent.getRequestId());
                System.out.println("url:"+url+" ,cookies-->"+cookies);
            }
        }
    }



}

附上几个实体类

ResponseBodyVo

import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;

@Data
@AllArgsConstructor
@NoArgsConstructor
public class ResponseBodyVo {
    private String sessionId;
    private String value;
    private int status;
}

ResponseReceivedEvent

import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;



@Data
@AllArgsConstructor
@NoArgsConstructor
public class ResponseReceivedEvent {
    private String frameId;
    private String requestId;
    private String response;
    private String loaderId;
    private String type;
    private String timestamp;
}

测试类

import org.openqa.selenium.chrome.ChromeOptions;
import org.openqa.selenium.logging.LogType;
import org.openqa.selenium.logging.LoggingPreferences;
import org.openqa.selenium.remote.CapabilityType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.HashMap;
import java.util.Map;
import java.util.logging.Level;


public class Test {
    private final static Logger logger = LoggerFactory.getLogger(Test.class);

    public static void main(String[] args) {
        String url = "https://xxxx";
        test(url);
    }

    private static void test(String url) {
        String webDriverPath = "E:\\IDEA\\common\\src\\test\\java\\com\\autotest\\tools\\chromedriver.exe";
        System.setProperty("webdriver.chrome.driver", webDriverPath);
        ChromeDriverProxy driver = null;
        try {
            ChromeOptions options = new ChromeOptions();
            options.addArguments("--disable-popup-blocking"); // 禁用阻止弹出窗口
            options.addArguments("no-sandbox"); // 启动无沙盒模式运行
            options.addArguments("disable-extensions"); // 禁用扩展
            options.addArguments("no-default-browser-check"); // 默认浏览器检查
            Map<String, Object> prefs = new HashMap();
            prefs.put("credentials_enable_service", false);
            prefs.put("profile.password_manager_enabled", false);
            options.setExperimentalOption("prefs", prefs);// 禁用保存密码提示框

            // set performance logger
            // this sends Network.enable to chromedriver
            LoggingPreferences logPrefs = new LoggingPreferences();
            logPrefs.enable(LogType.PERFORMANCE, Level.ALL);

            options.setCapability(CapabilityType.LOGGING_PREFS, logPrefs);

            driver = new ChromeDriverProxy(options);
            // do something

            ChromeDriverProxy.saveHttpTransferDataIfNecessary(driver);
        } finally {
            driver.close();
        }
    }
}

同样 需要注意的是logPrefs.enable(LogType.PERFORMANCE, Level.ALL);

以上两种方法基本获取network response的内容了。

大家有好的方法可以一起留言交流。

参考：

java+selenium+chromedriver 有什么办法能拿到 network 返回的数据吗，求大佬指教

selenium 获取请求返回内容的解决方案

本文章为转载内容，我们尊重原作者对文章享有的著作权。如有内容错误或侵权问题，欢迎原作者联系我们进行内容更正或删除文章。

上一篇：android 运行时权限方案安卓权限机制

下一篇：android中r有错误 android error

提问和评论都可以，用心的回复会被更多人看到评论

发布评论

相关文章

官方博客	全部文章	热门标签	班级博客
了解我们	网站地图	意见反馈

鸿蒙开发者社区	51CTO学堂
51CTO	软考资讯

java selenium 获取 script java selenium 获取network

java selenium 获取 script java selenium 获取network

51CTO博客