youshang/erp_sb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1

Browse Source
This commit is contained in:
ZiJIe
2025-09-23 17:34:09 +08:00
parent 41c24a2147
commit 3c6c29c64c
5 changed files with 70 additions and 274 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
erp_client_sb/src/main/java/com/tashow/erp/service/impl/Alibaba1688ServiceImpl.java
View File

@@ -9,7 +9,6 @@ import com.tashow.erp.service.Alibaba1688Service;
import com.tashow.erp.utils.Alibaba1688CookieUtil;
import com.tashow.erp.utils.ErrorReporter;
import com.tashow.erp.utils.QiniuUtil;
import com.tashow.erp.utils.PersistentDriverManager;
import org.openqa.selenium.chrome.ChromeDriver;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -43,8 +42,7 @@ public class Alibaba1688ServiceImpl implements Alibaba1688Service {
private final RestTemplate noSslRestTemplate = createNoSslRestTemplate();
@Autowired
private ErrorReporter errorReporter;
@Autowired
private PersistentDriverManager driverManager;
private RestTemplate createNoSslRestTemplate() {
try {
TrustManager[] trustManagers = new TrustManager[] {
@@ -282,42 +280,42 @@ public class Alibaba1688ServiceImpl implements Alibaba1688Service {
}
private String getWeight(List<String> ids) {
Pattern WEIGHT_PATTERN = Pattern.compile("\"(?:weight|unitWeight)\":\\s*([1-9]\\d*(?:\\.\\d+)?)");
List<String> weightList = new ArrayList<>();
ChromeDriver driver = driverManager.getCurrentDriver();
Set<String> weightSet = new HashSet<>();
try {
for (int i = 0; weightSet.size() <= 2; i++) {
driver.get("https://detail.1688.com/offer/" + ids.get(i) + ".html");
if(i==0) driver.navigate().refresh();
if (Objects.equals(driver.getTitle(), "验证码拦截")) {
driver = driverManager.switchToHeadful();
while (Objects.equals(driver.getTitle(), "验证码拦截")) {
Thread.sleep(1000);
}
}else if(Objects.equals(driver.getTitle(), "淘宝网 - 淘！我喜欢")){
}
System.out.println("标题"+driver.getTitle());
String source = driver.getPageSource();
Matcher weightMatcher = WEIGHT_PATTERN.matcher(source);
if (weightMatcher.find()) {
String weightValue = weightMatcher.group(1);
String width = weightValue.contains(".") ? (int) (Float.parseFloat(weightValue) * 1000) + "g" : weightValue + "g";
weightSet.add(width);
System.out.println("重量"+width);
}
Thread.sleep(2000+random.nextInt(3000));
}
} catch (InterruptedException e) {
e.printStackTrace();
errorReporter.reportDataCollectError("获取重量出错",e);
}
weightList.addAll(weightSet);
weightList.sort((a, b) -> Integer.compare(Integer.parseInt(a.replace("g", "")), Integer.parseInt(b.replace("g", ""))));
System.out.println("weightList: " +ids+"::::::"+ weightList);
return weightList.get(1);
}
// private String getWeight(List<String> ids) {
// Pattern WEIGHT_PATTERN = Pattern.compile("\"(?:weight|unitWeight)\":\\s*([1-9]\\d*(?:\\.\\d+)?)");
// List<String> weightList = new ArrayList<>();
// ChromeDriver driver = driverManager.getCurrentDriver();
// Set<String> weightSet = new HashSet<>();
// try {
// for (int i = 0; weightSet.size() <= 2; i++) {
// driver.get("https://detail.1688.com/offer/" + ids.get(i) + ".html");
// if(i==0) driver.navigate().refresh();
// if (Objects.equals(driver.getTitle(), "验证码拦截")) {
// driver = driverManager.switchToHeadful();
// while (Objects.equals(driver.getTitle(), "验证码拦截")) {
// Thread.sleep(1000);
// }
// }else if(Objects.equals(driver.getTitle(), "淘宝网 - 淘！我喜欢")){
//
// }
// System.out.println("标题"+driver.getTitle());
// String source = driver.getPageSource();
// Matcher weightMatcher = WEIGHT_PATTERN.matcher(source);
// if (weightMatcher.find()) {
// String weightValue = weightMatcher.group(1);
// String width = weightValue.contains(".") ? (int) (Float.parseFloat(weightValue) * 1000) + "g" : weightValue + "g";
// weightSet.add(width);
// System.out.println("重量"+width);
// }
// Thread.sleep(2000+random.nextInt(3000));
// }
// } catch (InterruptedException e) {
// e.printStackTrace();
// errorReporter.reportDataCollectError("获取重量出错",e);
// }
// weightList.addAll(weightSet);
// weightList.sort((a, b) -> Integer.compare(Integer.parseInt(a.replace("g", "")), Integer.parseInt(b.replace("g", ""))));
// System.out.println("weightList: " +ids+"::::::"+ weightList);
// return weightList.get(1);
// }
}

4
erp_client_sb/src/main/java/com/tashow/erp/test/SeleniumWithProfile.java
View File

@@ -1,11 +1,7 @@
package com.tashow.erp.test;
import com.tashow.erp.utils.StealthSelenium;
import org.openqa.selenium.chrome.ChromeDriver;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class SeleniumWithProfile {

63
erp_client_sb/src/main/java/com/tashow/erp/utils/PersistentDriverManager.java
View File

@@ -1,63 +0,0 @@
package com.tashow.erp.utils;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import org.openqa.selenium.Cookie;
import org.openqa.selenium.chrome.ChromeDriver;
import org.springframework.stereotype.Component;
import java.util.Set;
@Component
public class PersistentDriverManager {
private ChromeDriver headlessDriver;
private ChromeDriver headfulDriver;
private boolean usingHeadless = true;
@PostConstruct
public void init() {
headlessDriver = StealthSelenium.createDriver(true);
}
public ChromeDriver getCurrentDriver() {
return usingHeadless ? headlessDriver : headfulDriver;
}
public ChromeDriver switchToHeadful() {
if (usingHeadless) {
if (headfulDriver == null) {
headfulDriver = StealthSelenium.createDriver(false);
}
transferState(headlessDriver, headfulDriver);
usingHeadless = false;
}
return headfulDriver;
}
private void transferState(ChromeDriver from, ChromeDriver to) {
try {
String currentUrl = from.getCurrentUrl();
Set<Cookie> cookies = from.manage().getCookies();
to.get(currentUrl);
cookies.forEach(cookie -> {
try {
to.manage().addCookie(cookie);
} catch (Exception ignored) {}
});
to.navigate().refresh();
} catch (Exception ignored) {}
}
@PreDestroy
public void cleanup() {
if (headlessDriver != null) {
headlessDriver.quit();
}
if (headfulDriver != null) {
headfulDriver.quit();
}
}
}

143
erp_client_sb/src/main/java/com/tashow/erp/utils/StealthSelenium.java
View File

@@ -1,143 +0,0 @@
package com.tashow.erp.utils;
import io.github.bonigarcia.wdm.WebDriverManager;
import org.openqa.selenium.Dimension;
import org.openqa.selenium.Point;
import org.openqa.selenium.chrome.ChromeDriver;
import org.openqa.selenium.chrome.ChromeOptions;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.TimeUnit;
/**
* Selenium反检测工具类
* 使用方法：ChromeDriver driver = StealthSelenium.createDriver();
*/
public class StealthSelenium {
private static final List<String> USER_AGENTS = Arrays.asList(
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36"
);
private static final Random random = new Random();
/**
* 创建反检测ChromeDriver
* @return 配置好的ChromeDriver实例
*/
public static ChromeDriver createDriver() {
return createDriver(false);
}
/**
* 创建反检测ChromeDriver
* @param headless 是否无头模式
* @return 配置好的ChromeDriver实例
*/
public static ChromeDriver createDriver(boolean headless) {
WebDriverManager.chromedriver().setup();
ChromeOptions options = createStealthOptions(headless);
ChromeDriver driver = new ChromeDriver(options);
applyAntiDetection(driver);
setRandomWindowProperties(driver);
driver.manage().timeouts().implicitlyWait(10, TimeUnit.SECONDS);
return driver;
}
private static ChromeOptions createStealthOptions(boolean headless) {
ChromeOptions options = new ChromeOptions();
options.addArguments(
"--disable-blink-features=AutomationControlled",
"--disable-extensions",
"--disable-gpu",
"--disable-dev-shm-usage",
"--no-sandbox",
"--disable-background-timer-throttling",
"--disable-backgrounding-occluded-windows",
"--disable-renderer-backgrounding",
"--disable-features=TranslateUI",
"--disable-ipc-flooding-protection",
"--exclude-switches=enable-automation",
"--disable-component-extensions-with-background-pages",
"--disable-web-security",
"--disable-client-side-phishing-detection",
"--lang=zh-CN"
);
if (headless) {
options.addArguments("--headless");
}
String userAgent = USER_AGENTS.get(random.nextInt(USER_AGENTS.size()));
options.addArguments("--user-agent=" + userAgent);
Map<String, Object> prefs = new HashMap<>();
prefs.put("profile.default_content_setting_values.notifications", 2);
prefs.put("profile.default_content_settings.popups", 0);
options.setExperimentalOption("prefs", prefs);
options.setExperimentalOption("excludeSwitches", Arrays.asList("enable-automation"));
options.setExperimentalOption("useAutomationExtension", false);
return options;
}
private static void applyAntiDetection(ChromeDriver driver) {
String stealthScript = """
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
delete Object.getPrototypeOf(navigator).webdriver;
window.chrome = { runtime: {}, loadTimes: function() {}, csi: function() {}, app: {} };
Object.defineProperty(navigator, 'permissions', {
get: () => ({ query: () => Promise.resolve({state: 'granted'}) })
});
Object.defineProperty(navigator, 'plugins', {
get: () => Array.from({length: Math.floor(Math.random() * 5) + 2}, (_, i) => ({
name: `Plugin ${i}`, description: `Plugin Description ${i}`,
filename: `plugin${i}.dll`, version: `${Math.floor(Math.random() * 10)}.0.0`
}))
});
Object.defineProperty(navigator, 'languages', {
get: () => ['zh-CN', 'zh', 'en-US', 'en']
});
const screenWidth = 1920 + Math.floor(Math.random() * 100);
const screenHeight = 1080 + Math.floor(Math.random() * 100);
Object.defineProperty(screen, 'width', { get: () => screenWidth });
Object.defineProperty(screen, 'height', { get: () => screenHeight });
Object.defineProperty(screen, 'availWidth', { get: () => screenWidth });
Object.defineProperty(screen, 'availHeight', { get: () => screenHeight - 40 });
if (window.document) {
['$cdc_asdjflasutopfhvcZLmcfl_', '$chrome_asyncScriptInfo'].forEach(prop => {
Object.defineProperty(window.document, prop, { get: () => undefined, set: () => {} });
});
}
""";
Map<String, Object> params = new HashMap<>();
params.put("source", stealthScript);
driver.executeCdpCommand("Page.addScriptToEvaluateOnNewDocument", params);
String userAgent = USER_AGENTS.get(random.nextInt(USER_AGENTS.size()));
Map<String, Object> networkParams = new HashMap<>();
networkParams.put("userAgent", userAgent);
networkParams.put("acceptLanguage", "zh-CN,zh;q=0.9,en;q=0.8");
networkParams.put("platform", "Win32");
driver.executeCdpCommand("Network.setUserAgentOverride", networkParams);
}
private static void setRandomWindowProperties(ChromeDriver driver) {
int width = 1200 + random.nextInt(400);
int height = 800 + random.nextInt(300);
driver.manage().window().setSize(new Dimension(width, height));
driver.manage().window().setPosition(new Point(random.nextInt(100), random.nextInt(100)));
}
}