@Test
public void usingAProxyToTrackNetworkTrafficStep2() {
    BrowserMobProxy browserMobProxy = new BrowserMobProxyServer();
    browserMobProxy.start();
    Proxy seleniumProxyConfiguration = ClientUtil.createSeleniumProxy(browserMobProxy);

    FirefoxOptions firefoxOptions = new FirefoxOptions();
    firefoxOptions.setCapability(CapabilityType.PROXY, seleniumProxyConfiguration);
    driver = new FirefoxDriver(firefoxOptions);
    browserMobProxy.newHar();
    driver.get("https://www.google.co.uk");

    Har httpArchive = browserMobProxy.getHar();

    assertThat(getHTTPStatusCode("https://www.google.co.uk/", httpArchive))
            .isEqualTo(200);
}
 

private ProxyDriverIntegrator getProxyDriverIntegrator(RequestFilter recordRequestFilter,
                                                       WebDriverSupplier webDriverSupplier,
                                                       DriverServiceSupplier driverServiceSupplier,
                                                       @Named(PATH_TO_DRIVER) String pathToDriverExecutable,
                                                       @Named(SCREEN) String screen,
                                                       @Named(TIMEOUT) int timeout,
                                                       ResponseFilter responseFilter) throws IOException {
    BrowserMobProxy proxy = createBrowserMobProxy(timeout, recordRequestFilter, responseFilter);
    proxy.start(0);
    logger.info("Proxy running on port " + proxy.getPort());
    Proxy seleniumProxy = createSeleniumProxy(proxy);
    DesiredCapabilities desiredCapabilities = createDesiredCapabilities(seleniumProxy);
    DriverService driverService = driverServiceSupplier.getDriverService(pathToDriverExecutable, screen);
    WebDriver driver = webDriverSupplier.get(driverService, desiredCapabilities);

    return new ProxyDriverIntegrator(driver, proxy, driverService);
}
 

/**
 * EWnable HAR logging
 */
@When("^I enable HAR logging$")
public void enableHar() {
	final Optional<ProxyDetails<?>> proxy =
		State.getFeatureStateForThread().getProxyInterface(BrowsermobProxyUtilsImpl.PROXY_NAME);

	final EnumSet<CaptureType> captureTypes = CaptureType.getAllContentCaptureTypes();
	captureTypes.addAll(CaptureType.getCookieCaptureTypes());
	captureTypes.addAll(CaptureType.getHeaderCaptureTypes());

	proxy
		.flatMap(ProxyDetails::getInterface)
		.map(BrowserMobProxy.class::cast)
		.ifPresent(x -> {
			x.setHarCaptureTypes(captureTypes);
			x.newHar();
		});
}
 

@Disabled("TODO - some problems with timing (works in debug mode)")
@Test
public void shouldCallFilterByRegistry() throws IOException {
  // given
  BrowserMobProxy proxy = proxyController.startProxyServer(InetAddress.getLocalHost());
  proxyController.startAnalysis();
  requestFilterRegistry.add(requestFilter);
  // when
  DesiredCapabilities capabilities = proxyCapabilities(proxy);
  visitSamplePage(capabilities);
  proxyController.stopAnalysis();
  proxyController.stopProxyServer();
  // then
  verify(requestFilter, atLeastOnce()).filterRequest(any(HttpRequest.class),
      any(HttpMessageContents.class), any(HttpMessageInfo.class));
}
 

@Disabled("TODO - some problems with timing (works in debug mode)")
@Test
public void shouldCallFilterByRegistry() throws IOException {
  // given
  BrowserMobProxy browserMobProxy = new BrowserMobProxyServer();
  startProxyServer(browserMobProxy);
  requestFilterRegistry.add(requestFilter);
  browserMobProxy.addRequestFilter(requestFilterRegistry);
  // when
  DesiredCapabilities capabilities = proxyCapabilities(browserMobProxy);
  visitSamplePage(capabilities);
  browserMobProxy.stop();
  // then
  verify(requestFilter, atLeastOnce()).filterRequest(any(HttpRequest.class),
      any(HttpMessageContents.class), any(HttpMessageInfo.class));

}
 

/**
 * register custom BrowserMobProxy Server
 * 
 * @param proxy
 *            custom BrowserMobProxy
 * 
 */
public static void registerProxy(BrowserMobProxy proxy) {
    long threadId = Thread.currentThread().getId();
    if (proxies.containsKey(threadId)) {
        LOGGER.warn("Existing proxy is detected and will be overrwitten");
        // No sense to stop as it is not supported
        proxies.remove(threadId);
    }
    if (proxyPortsByThread.containsKey(threadId)) {
        LOGGER.warn("Existing proxyPort is detected and will be overwritten");
        proxyPortsByThread.remove(threadId);
    }
    
    LOGGER.info("Register custom proxy with thread: " + threadId);
    proxies.put(threadId, proxy);
}
 

@Test
public void testBrowserModProxyResponseFiltering() {
    List<String> content = new ArrayList<>();
    LocalTrustStoreBuilder localTrustStoreBuilder = new LocalTrustStoreBuilder();
    SSLContext sslContext = localTrustStoreBuilder.createSSLContext();
    SSLContext.setDefault(sslContext);

    ProxyPool.setupBrowserMobProxy();
    SystemProxy.setupProxy();
    BrowserMobProxy proxy = ProxyPool.getProxy();
    proxy.enableHarCaptureTypes(CaptureType.RESPONSE_CONTENT);
    proxy.newHar();

    proxy.addResponseFilter((request, contents, messageInfo) -> {
        LOGGER.info("Requested resource caught contents: " + contents.getTextContents());
        if (contents.getTextContents().contains(filterKey)) {
            content.add(contents.getTextContents());
        }
    });

    makeHttpRequest(testUrl, requestMethod);

    Assert.assertNotNull(proxy.getHar(), "Har is unexpectedly null!");
    Assert.assertEquals(content.size(), 1,"Filtered response number is not as expected!");
    Assert.assertTrue(content.get(0).contains(filterKey), "Response doesn't contain expected key!");
}
 

@Test
public void usingAProxyToTrackNetworkTrafficStep1() {
    BrowserMobProxy browserMobProxy = new BrowserMobProxyServer();
    browserMobProxy.start();
    Proxy seleniumProxyConfiguration = ClientUtil.createSeleniumProxy(browserMobProxy);

    FirefoxOptions firefoxOptions = new FirefoxOptions();
    firefoxOptions.setCapability(CapabilityType.PROXY, seleniumProxyConfiguration);
    driver = new FirefoxDriver(firefoxOptions);
    browserMobProxy.newHar();
    driver.get("https://www.google.co.uk");
}
 

public BrowserMobProxy createBrowserMobProxy(int timeout, RequestFilter requestFilter, ResponseFilter responseFilter) {
    BrowserMobProxyServer proxy = new BrowserMobProxyServer();
    proxy.enableHarCaptureTypes(CaptureType.REQUEST_CONTENT, CaptureType.RESPONSE_CONTENT);
    proxy.newHar("measurements");
    proxy.addRequestFilter(requestFilter);
    proxy.addResponseFilter(responseFilter);
    proxy.setIdleConnectionTimeout(timeout, TimeUnit.SECONDS);
    proxy.setRequestTimeout(timeout, TimeUnit.SECONDS);
    return proxy;
}
 

@Test
public void providesAccessToConnectedDriverProxyAndDriverService() {
    WebDriver driver = mock(WebDriver.class);
    BrowserMobProxy proxy = mock(BrowserMobProxy.class);
    DriverService driverService = mock(DriverService.class);

    ProxyDriverIntegrator proxyDriverIntegrator = new ProxyDriverIntegrator(driver, proxy, driverService);

    assertEquals(driver, proxyDriverIntegrator.getWebDriver());
    assertEquals(proxy, proxyDriverIntegrator.getProxy());
    assertEquals(driverService, proxyDriverIntegrator.getDriverService());
}
 

/**
 * Saves a HAR file with the details of the transactions that have passed through BrowserMob.
 * This step is only required if you wish to save har files at particular points during the test.
 * The HAR file is always saved when browsermob is shut down, meaning that once you begin the
 * capture of the HAR file it will be saved regardless of the success or failure of the test.
 * @param alias If this word is found in the step, it means the filename is found from the
 *              data set.
 * @param filename The optional filename to use for the HAR file
 */
@When("^I dump the HAR file(?: to( alias)? \"(.*?)\")?$")
public void saveHarFile(final String alias, final String filename) {

	final String fixedFilename = autoAliasUtils.getValue(
		StringUtils.defaultString(filename, Constants.HAR_FILE_NAME),
		StringUtils.isNotBlank(alias),
		State.getFeatureStateForThread());

	final Optional<ProxyDetails<?>> proxy =
		State.getFeatureStateForThread().getProxyInterface(BrowsermobProxyUtilsImpl.PROXY_NAME);

	proxy
		.flatMap(ProxyDetails::getInterface)
		.map(BrowserMobProxy.class::cast)
		.map(x -> Try.run(() -> {
			final Har har = x.getHar();

			checkState(
				har != null,
				"You need to add the step \"I enable HAR logging\" before saving the HAR file");

			final File file = new File(
				State.getFeatureStateForThread().getReportDirectory()
					+ "/"
					+ fixedFilename);
			har.writeTo(file);
		}));
}
 

/**
 * Block access to all urls that match the regex
 *
 * @param url      A regular expression that matches URLs to be blocked
 * @param response The response code to send back when a matching URL is accessed
 */
@When("^I block access to the URL regex \"(.*?)\" with response \"(\\d+)\"$")
public void blockUrl(final String url, final Integer response) {
	final Optional<ProxyDetails<?>> proxy =
		State.getFeatureStateForThread().getProxyInterface(BrowsermobProxyUtilsImpl.PROXY_NAME);

	proxy
		.flatMap(ProxyDetails::getInterface)
		.map(BrowserMobProxy.class::cast)
		.ifPresent(x -> {
			x.blacklistRequests(url, response);
		});
}
 

/**
 * Block access to all urls that match the regex
 *
 * @param url      A regular expression that matches URLs to be blocked
 * @param response The response code to send back when a matching URL is accessed
 * @param type     The http type of request to block (CONNECT, GET, PUT etc)
 */
@When("^I block access to the URL regex \"(.*?)\" of the type \"(.*?)\" with response \"(\\d+)\"$")
public void blockUrl(final String url, final String type, final Integer response) {
	final Optional<ProxyDetails<?>> proxy =
		State.getFeatureStateForThread().getProxyInterface(BrowsermobProxyUtilsImpl.PROXY_NAME);

	proxy
		.flatMap(ProxyDetails::getInterface)
		.map(BrowserMobProxy.class::cast)
		.ifPresent(x -> {
			x.blacklistRequests(url, response, type);
		});
}
 

/**
 * Allow access to all urls that match the regex
 *
 * @param response      The response code for all whitelisted urls
 */
@When("^I enable the whitelist with responding with \"(\\d+)\" for unmatched requests$")
public void enableWhitelist(final Integer response) {
	final Optional<ProxyDetails<?>> proxy =
		State.getFeatureStateForThread().getProxyInterface(BrowsermobProxyUtilsImpl.PROXY_NAME);

	proxy
		.flatMap(ProxyDetails::getInterface)
		.map(BrowserMobProxy.class::cast)
		.ifPresent(x -> {
			x.enableEmptyWhitelist(response);
		});
}
 

/**
 * Allow access to all urls that match the regex
 *
 * @param url      A regular expression that matches URLs to be allowed
 */
@When("^I allow access to the URL regex \"(.*?)\"$")
public void allowUrl(final String url) {
	final Optional<ProxyDetails<?>> proxy =
		State.getFeatureStateForThread().getProxyInterface(BrowsermobProxyUtilsImpl.PROXY_NAME);

	proxy
		.flatMap(ProxyDetails::getInterface)
		.map(BrowserMobProxy.class::cast)
		.ifPresent(x -> {
			x.addWhitelistPattern(url);
		});
}
 

/**
 * Set new or modify an existing HTTP header
 *
 * @param headerName HTTP header name
 * @param headerValue HTTP header value
 */
@When("^I set header \"([^\"]*)\" with value \"([^\"]*)\"$")
public void changeHeader(final String headerName, final String headerValue) {
	final Optional<ProxyDetails<?>> proxy =
		State.getFeatureStateForThread().getProxyInterface(BrowsermobProxyUtilsImpl.PROXY_NAME);

	proxy
		.flatMap(ProxyDetails::getInterface)
		.map(BrowserMobProxy.class::cast)
		.ifPresent(x -> x.addRequestFilter((request, contents, messageInfo) -> {
			request.headers().set(headerName, headerValue);
			return null;
		}));
}
 

/**
 * Remove HTTP header
 *
 * @param headerName HTTP header name
 */
@When("^I remove header \"([^\"]*)\"$")
public void removeHeader(final String headerName) {
	final Optional<ProxyDetails<?>> proxy =
		State.getFeatureStateForThread().getProxyInterface(BrowsermobProxyUtilsImpl.PROXY_NAME);

	proxy
		.flatMap(ProxyDetails::getInterface)
		.map(BrowserMobProxy.class::cast)
		.ifPresent(x -> x.addRequestFilter((request, contents, messageInfo) -> {
			request.headers().remove(headerName);
			return null;
		}));
}
 

@Override
public Optional<ProxyDetails<BrowserMobProxy>> initProxy(
		@NotNull final List<File> globalTempFiles,
		@NotNull final List<File> tempFolders,
		@NotNull final Optional<ProxySettings> upstreamProxy) {

	checkNotNull(globalTempFiles);
	checkNotNull(tempFolders);
	checkNotNull(upstreamProxy);

	try {
		final String proxyName =
			SYSTEM_PROPERTY_UTILS.getProperty(Constants.START_INTERNAL_PROXY);

		/*
			BrowserMob is enabled by default unless it is specifically
			disabled
		 */
		final boolean enabled = StringUtils.isBlank(proxyName)
			|| ENABLE_DISABLE_LIST_UTILS.enabled(
				proxyName,
				PROXY_NAME,
				true,
				true);

		if (enabled) {
			return Optional.of(startBrowsermobProxy(upstreamProxy));
		}

		return Optional.empty();
	} catch (final Exception ex) {
		throw new ProxyException(ex);
	}
}
 

@SuppressWarnings("unchecked")
private void trackErrorResponses(
	final BrowserMobProxy proxy,
	final ProxyDetails<BrowserMobProxy> proxyDetails) {

	proxy.addResponseFilter(new ResponseFilter() {
		@Override
		public void filterResponse(
			final HttpResponse response,
			final HttpMessageContents contents,
			final HttpMessageInfo messageInfo) {

			/*
				Track anything other than a 200 range response
			 */
			if (response.getStatus().code() >= START_HTTP_ERROR
				&& response.getStatus().code() <= END_HTTP_ERROR) {

				synchronized (proxyDetails) {
					final Map<String, Object> properties = proxyDetails.getProperties();
					if (!properties.containsKey(INVALID_REQUESTS)) {
						properties.put(
							INVALID_REQUESTS,
							new ArrayList<HttpMessageInfo>());
					}
					ArrayList.class.cast(properties.get(INVALID_REQUESTS)).add(messageInfo);
					proxyDetails.setProperties(properties);
				}
			}
		}
	});
}
 

@Override
public void stopProxies(@NotNull final List<ProxyDetails<?>> proxies, final String reportOutput) {
	checkNotNull(proxies);
	checkArgument(StringUtils.isNotBlank(reportOutput));

	proxies.stream()
		.filter(proxyDetails -> BrowsermobProxyUtilsImpl.PROXY_NAME.equals(proxyDetails.getProxyName()))
		.forEach(x -> x.getInterface()
			.map(BrowserMobProxy.class::cast)
			.ifPresent(proxy -> {
				/*
					Save the HAR file before the proxy is shut down. Doing this work
					here means that the HAR file is always available, even if the
					test failed and a step like "I dump the HAR file" was not executed.
				 */
				if (proxy.getHar() != null) {
					Try.run(() -> {
						final String filename = Constants.HAR_FILE_NAME_PREFIX
							+ new SimpleDateFormat(Constants.FILE_DATE_FORMAT).format(new Date())
							+ "."
							+ Constants.HAR_FILE_NAME_EXTENSION;

						final File file = new File(
							reportOutput
								+ "/"
								+ filename);
						proxy.getHar().writeTo(file);
					});
				}
				proxy.abort();
			}));
}
 

DesiredCapabilities proxyCapabilities(BrowserMobProxy browserMobProxy) {
  browserMobProxy.enableHarCaptureTypes(
      CaptureType.REQUEST_HEADERS,
      CaptureType.RESPONSE_HEADERS,
      CaptureType.REQUEST_CONTENT,
      CaptureType.RESPONSE_CONTENT);

  Proxy seleniumProxy = ClientUtil.createSeleniumProxy(browserMobProxy);
  DesiredCapabilities capabilities = new DesiredCapabilities();
  capabilities.setCapability(CapabilityType.PROXY, seleniumProxy);
  return capabilities;
}
 

public BrowserMobProxy startProxyServer(InetAddress proxyAddress) {
  if (!browserMobProxy.isStarted()) {
    try {
      browserMobProxy.start(port, proxyAddress);
      browserMobProxy.addRequestFilter(filterRegistry);
    } catch (Exception e) {
      LOG.error("Can't start proxy", e);
    }
  }
  return browserMobProxy;
}
 

private DesiredCapabilities enableProxy(Capabilities capabilities) {
  DesiredCapabilities caps = new DesiredCapabilities(capabilities);
  try {
    InetAddress proxyInetAddress = InetAddress.getByName(proxyIp);
    BrowserMobProxy browserMobProxy = proxyController.startProxyServer(proxyInetAddress);
    Proxy seleniumProxy = ClientUtil.createSeleniumProxy(browserMobProxy, proxyInetAddress);
    caps.setCapability(CapabilityType.PROXY, seleniumProxy);
  } catch (UnknownHostException e) {
    throw new IllegalStateException(e);
  }
  return caps;
}
 

public static void changeHost(BrowserMobProxy browserMobProxy,String newValue){
    AdvancedHostResolver advancedHostResolver = browserMobProxy.getHostNameResolver();
    advancedHostResolver.clearHostRemappings();
    for (String temp : newValue.split("\\n")) {
        if (temp.split(" ").length == 2) {
            advancedHostResolver.remapHost(temp.split(" ")[1], temp.split(" ")[0]);
            Log.e("~~~~remapHost ", temp.split(" ")[1] + " " + temp.split(" ")[0]);
        }
    }


    browserMobProxy.setHostNameResolver(advancedHostResolver);
}
 

public Set<String> getPageSet() {
    BrowserMobProxy proxy = ((SysApplication) getApplication()).proxy;

    Set<String> pageSet = new HashSet<>();
    for (HarPage harPage : proxy.getHar().getLog().getPages()) {
        if (!disablePages.contains(harPage.getId())) {
            pageSet.add(harPage.getId());
        }
    }

    return pageSet;
}
 

public String getHost() {
    String result = "";
    BrowserMobProxy browserMobProxy = ((SysApplication) getApplication()).proxy;
    AdvancedHostResolver advancedHostResolver = browserMobProxy.getHostNameResolver();
    for (String key : advancedHostResolver.getHostRemappings().keySet()) {
        result += key + " " + advancedHostResolver.getHostRemappings().get(key) + "\n";
    }
    return result.length() > 1 ? result.substring(0, result.length() - 1) : "无";
}
 

/**
 * create BrowserMobProxy Server object
 * @return BrowserMobProxy
 * 
 */
public static BrowserMobProxy createProxy() {
    BrowserMobProxyServer proxy = new BrowserMobProxyServer();
    proxy.setTrustAllServers(true);
    //System.setProperty("jsse.enableSNIExtension", "false");
    
    // disable MITM in case we do not need it
    proxy.setMitmDisabled(Configuration.getBoolean(Parameter.BROWSERMOB_MITM));
    
    return proxy;
}
 

public static void setupBrowserMobProxy()
{
    if (Configuration.getBoolean(Parameter.BROWSERMOB_PROXY)) {
        long threadId = Thread.currentThread().getId();
        BrowserMobProxy proxy = startProxy();

        Integer port = proxy.getPort();
        proxyPortsByThread.put(threadId, port);
        
        String currentIP = "";
        if (!Configuration.get(Parameter.BROWSERMOB_HOST).isEmpty()) {
        	// reuse "browsermob_host" to be able to share valid publicly available host. 
        	// it is useful when java and web tests are executed absolutely in different containers/networks. 
        	currentIP = Configuration.get(Parameter.BROWSERMOB_HOST);
        } else {
        	currentIP = NetworkUtil.getIpAddress();
        }
        
        LOGGER.warn("Set http/https proxy settings only to use with BrowserMobProxy host: " + currentIP + "; port: " + proxyPortsByThread.get(threadId));
        
        R.CONFIG.put("proxy_host", currentIP);
        R.CONFIG.put(Parameter.PROXY_PORT.getKey(), port.toString());
        
        R.CONFIG.put("proxy_protocols", "http,https");

        // follow steps to configure https traffic sniffering: https://github.com/lightbody/browsermob-proxy#ssl-support
        // the most important are:
        // download - https://github.com/lightbody/browsermob-proxy/blob/master/browsermob-core/src/main/resources/sslSupport/ca-certificate-rsa.cer
        // import to system (for android we should use certifiate installer in system settings->security...)
    }
}
 

public static synchronized BrowserMobProxy startProxy(int proxyPort) {
    if (!Configuration.getBoolean(Parameter.BROWSERMOB_PROXY)) {
        LOGGER.debug("Proxy is disabled.");
        return null;
    }
    // integrate browserMob proxy if required here
    BrowserMobProxy proxy = null;
    long threadId = Thread.currentThread().getId();
    if (proxies.containsKey(threadId)) {
        proxy = proxies.get(threadId);
    }

    if (proxyPortsByThread.containsKey(threadId)) {
        proxyPort = proxyPortsByThread.get(threadId);
    }

    // case when proxy was already instantiated but port doesn't correspond to current device
    if (null == proxy || proxy.getPort() != proxyPort) {
        proxy = ProxyPool.createProxy();
        proxies.put(Thread.currentThread().getId(), proxy);
    }
    
    if (!proxy.isStarted()) {
        LOGGER.info("Starting BrowserMob proxy...");
    	// TODO: [VD] confirmed with MB that restart was added just in case. Maybe comment/remove?
        killProcessByPort(proxyPort);
        proxy.start(proxyPort);
    } else {
        LOGGER.info("BrowserMob proxy is already started on port " + proxy.getPort());
    }

    return proxy;
}
 

/**
 * Stop single proxy instance by id
 * @param threadId
 */
private static void stopProxyByThread(long threadId) {
    if (proxies.containsKey(threadId)) {
        LOGGER.debug("stopProxy starting...");
        setProxyPortToAvailable(threadId);
        BrowserMobProxy proxy = proxies.get(threadId);
        if (proxy != null) {
            LOGGER.debug("Found registered proxy by thread: " + threadId);

            // isStarted returns true even if proxy was already stopped
            if (proxy.isStarted()) {
                LOGGER.info("Stopping BrowserMob proxy...");
                try {
                    proxy.stop();
                } catch (IllegalStateException e) {
                    LOGGER.info("Seems like proxy was already stopped.");
                    LOGGER.info(e.getMessage());
                }
                
            } else {
                LOGGER.info("Stopping BrowserMob proxy skipped as it is not started.");
            }
        }
        proxies.remove(threadId);
        //proxyPortsByThread.remove(threadId);
        LOGGER.debug("stopProxy finished...");
    }
}