diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 9101fad..99cf9ef 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -5,31 +5,11 @@ - - - - - - - - - - - - - - - - - - - - - - - + + + @@ -239,4 +235,16 @@ + + + + + file://$PROJECT_DIR$/electron-vue-template/src/renderer/components/common/AccountManager.vue + 34 + + + + + \ No newline at end of file diff --git a/erp_client_sb/src/main/java/com/tashow/erp/service/impl/Alibaba1688ServiceImpl.java b/erp_client_sb/src/main/java/com/tashow/erp/service/impl/Alibaba1688ServiceImpl.java index 7ccfc61..337637b 100644 --- a/erp_client_sb/src/main/java/com/tashow/erp/service/impl/Alibaba1688ServiceImpl.java +++ b/erp_client_sb/src/main/java/com/tashow/erp/service/impl/Alibaba1688ServiceImpl.java @@ -9,7 +9,6 @@ import com.tashow.erp.service.Alibaba1688Service; import com.tashow.erp.utils.Alibaba1688CookieUtil; import com.tashow.erp.utils.ErrorReporter; import com.tashow.erp.utils.QiniuUtil; -import com.tashow.erp.utils.PersistentDriverManager; import org.openqa.selenium.chrome.ChromeDriver; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,8 +42,7 @@ public class Alibaba1688ServiceImpl implements Alibaba1688Service { private final RestTemplate noSslRestTemplate = createNoSslRestTemplate(); @Autowired private ErrorReporter errorReporter; - @Autowired - private PersistentDriverManager driverManager; + private RestTemplate createNoSslRestTemplate() { try { TrustManager[] trustManagers = new TrustManager[] { @@ -282,42 +280,42 @@ public class Alibaba1688ServiceImpl implements Alibaba1688Service { } - private String getWeight(List ids) { - Pattern WEIGHT_PATTERN = Pattern.compile("\"(?:weight|unitWeight)\":\\s*([1-9]\\d*(?:\\.\\d+)?)"); - List weightList = new ArrayList<>(); - ChromeDriver driver = driverManager.getCurrentDriver(); - Set weightSet = new HashSet<>(); - try { - for (int i = 0; weightSet.size() <= 2; i++) { - driver.get("https://detail.1688.com/offer/" + ids.get(i) + ".html"); - if(i==0) driver.navigate().refresh(); - if (Objects.equals(driver.getTitle(), "验证码拦截")) { - driver = driverManager.switchToHeadful(); - while (Objects.equals(driver.getTitle(), "验证码拦截")) { - Thread.sleep(1000); - } - }else if(Objects.equals(driver.getTitle(), "淘宝网 - 淘！我喜欢")){ - - } - System.out.println("标题"+driver.getTitle()); - String source = driver.getPageSource(); - Matcher weightMatcher = WEIGHT_PATTERN.matcher(source); - if (weightMatcher.find()) { - String weightValue = weightMatcher.group(1); - String width = weightValue.contains(".") ? (int) (Float.parseFloat(weightValue) * 1000) + "g" : weightValue + "g"; - weightSet.add(width); - System.out.println("重量"+width); - } - Thread.sleep(2000+random.nextInt(3000)); - } - } catch (InterruptedException e) { - e.printStackTrace(); - errorReporter.reportDataCollectError("获取重量出错",e); - } - weightList.addAll(weightSet); - weightList.sort((a, b) -> Integer.compare(Integer.parseInt(a.replace("g", "")), Integer.parseInt(b.replace("g", "")))); - System.out.println("weightList: " +ids+"::::::"+ weightList); - return weightList.get(1); - } +// private String getWeight(List ids) { +// Pattern WEIGHT_PATTERN = Pattern.compile("\"(?:weight|unitWeight)\":\\s*([1-9]\\d*(?:\\.\\d+)?)"); +// List weightList = new ArrayList<>(); +// ChromeDriver driver = driverManager.getCurrentDriver(); +// Set weightSet = new HashSet<>(); +// try { +// for (int i = 0; weightSet.size() <= 2; i++) { +// driver.get("https://detail.1688.com/offer/" + ids.get(i) + ".html"); +// if(i==0) driver.navigate().refresh(); +// if (Objects.equals(driver.getTitle(), "验证码拦截")) { +// driver = driverManager.switchToHeadful(); +// while (Objects.equals(driver.getTitle(), "验证码拦截")) { +// Thread.sleep(1000); +// } +// }else if(Objects.equals(driver.getTitle(), "淘宝网 - 淘！我喜欢")){ +// +// } +// System.out.println("标题"+driver.getTitle()); +// String source = driver.getPageSource(); +// Matcher weightMatcher = WEIGHT_PATTERN.matcher(source); +// if (weightMatcher.find()) { +// String weightValue = weightMatcher.group(1); +// String width = weightValue.contains(".") ? (int) (Float.parseFloat(weightValue) * 1000) + "g" : weightValue + "g"; +// weightSet.add(width); +// System.out.println("重量"+width); +// } +// Thread.sleep(2000+random.nextInt(3000)); +// } +// } catch (InterruptedException e) { +// e.printStackTrace(); +// errorReporter.reportDataCollectError("获取重量出错",e); +// } +// weightList.addAll(weightSet); +// weightList.sort((a, b) -> Integer.compare(Integer.parseInt(a.replace("g", "")), Integer.parseInt(b.replace("g", "")))); +// System.out.println("weightList: " +ids+"::::::"+ weightList); +// return weightList.get(1); +// } } \ No newline at end of file diff --git a/erp_client_sb/src/main/java/com/tashow/erp/test/SeleniumWithProfile.java b/erp_client_sb/src/main/java/com/tashow/erp/test/SeleniumWithProfile.java index 33c45cb..4b66440 100644 --- a/erp_client_sb/src/main/java/com/tashow/erp/test/SeleniumWithProfile.java +++ b/erp_client_sb/src/main/java/com/tashow/erp/test/SeleniumWithProfile.java @@ -1,11 +1,7 @@ package com.tashow.erp.test; -import com.tashow.erp.utils.StealthSelenium; -import org.openqa.selenium.chrome.ChromeDriver; - import java.io.BufferedReader; import java.io.InputStreamReader; -import java.util.regex.Matcher; import java.util.regex.Pattern; public class SeleniumWithProfile { diff --git a/erp_client_sb/src/main/java/com/tashow/erp/utils/PersistentDriverManager.java b/erp_client_sb/src/main/java/com/tashow/erp/utils/PersistentDriverManager.java deleted file mode 100644 index 5295458..0000000 --- a/erp_client_sb/src/main/java/com/tashow/erp/utils/PersistentDriverManager.java +++ /dev/null @@ -1,63 +0,0 @@ -package com.tashow.erp.utils; - -import jakarta.annotation.PostConstruct; -import jakarta.annotation.PreDestroy; -import org.openqa.selenium.Cookie; -import org.openqa.selenium.chrome.ChromeDriver; -import org.springframework.stereotype.Component; - -import java.util.Set; - -@Component -public class PersistentDriverManager { - - private ChromeDriver headlessDriver; - private ChromeDriver headfulDriver; - private boolean usingHeadless = true; - - @PostConstruct - public void init() { - headlessDriver = StealthSelenium.createDriver(true); - } - - public ChromeDriver getCurrentDriver() { - return usingHeadless ? headlessDriver : headfulDriver; - } - - public ChromeDriver switchToHeadful() { - if (usingHeadless) { - if (headfulDriver == null) { - headfulDriver = StealthSelenium.createDriver(false); - } - transferState(headlessDriver, headfulDriver); - usingHeadless = false; - } - return headfulDriver; - } - - - private void transferState(ChromeDriver from, ChromeDriver to) { - try { - String currentUrl = from.getCurrentUrl(); - Set cookies = from.manage().getCookies(); - - to.get(currentUrl); - cookies.forEach(cookie -> { - try { - to.manage().addCookie(cookie); - } catch (Exception ignored) {} - }); - to.navigate().refresh(); - } catch (Exception ignored) {} - } - - @PreDestroy - public void cleanup() { - if (headlessDriver != null) { - headlessDriver.quit(); - } - if (headfulDriver != null) { - headfulDriver.quit(); - } - } -} \ No newline at end of file diff --git a/erp_client_sb/src/main/java/com/tashow/erp/utils/StealthSelenium.java b/erp_client_sb/src/main/java/com/tashow/erp/utils/StealthSelenium.java deleted file mode 100644 index e13eede..0000000 --- a/erp_client_sb/src/main/java/com/tashow/erp/utils/StealthSelenium.java +++ /dev/null @@ -1,143 +0,0 @@ -package com.tashow.erp.utils; - -import io.github.bonigarcia.wdm.WebDriverManager; -import org.openqa.selenium.Dimension; -import org.openqa.selenium.Point; -import org.openqa.selenium.chrome.ChromeDriver; -import org.openqa.selenium.chrome.ChromeOptions; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.concurrent.TimeUnit; - -/** - * Selenium反检测工具类 - * 使用方法：ChromeDriver driver = StealthSelenium.createDriver(); - */ -public class StealthSelenium { - private static final List USER_AGENTS = Arrays.asList( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36", - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36" - ); - - private static final Random random = new Random(); - /** - * 创建反检测ChromeDriver - * @return 配置好的ChromeDriver实例 - */ - public static ChromeDriver createDriver() { - return createDriver(false); - } - - /** - * 创建反检测ChromeDriver - * @param headless 是否无头模式 - * @return 配置好的ChromeDriver实例 - */ - public static ChromeDriver createDriver(boolean headless) { - WebDriverManager.chromedriver().setup(); - ChromeOptions options = createStealthOptions(headless); - ChromeDriver driver = new ChromeDriver(options); - applyAntiDetection(driver); - setRandomWindowProperties(driver); - driver.manage().timeouts().implicitlyWait(10, TimeUnit.SECONDS); - return driver; - } - private static ChromeOptions createStealthOptions(boolean headless) { - ChromeOptions options = new ChromeOptions(); - - options.addArguments( - "--disable-blink-features=AutomationControlled", - "--disable-extensions", - "--disable-gpu", - "--disable-dev-shm-usage", - "--no-sandbox", - "--disable-background-timer-throttling", - "--disable-backgrounding-occluded-windows", - "--disable-renderer-backgrounding", - "--disable-features=TranslateUI", - "--disable-ipc-flooding-protection", - "--exclude-switches=enable-automation", - "--disable-component-extensions-with-background-pages", - "--disable-web-security", - "--disable-client-side-phishing-detection", - "--lang=zh-CN" - ); - - if (headless) { - options.addArguments("--headless"); - } - - String userAgent = USER_AGENTS.get(random.nextInt(USER_AGENTS.size())); - options.addArguments("--user-agent=" + userAgent); - - Map prefs = new HashMap<>(); - prefs.put("profile.default_content_setting_values.notifications", 2); - prefs.put("profile.default_content_settings.popups", 0); - options.setExperimentalOption("prefs", prefs); - options.setExperimentalOption("excludeSwitches", Arrays.asList("enable-automation")); - options.setExperimentalOption("useAutomationExtension", false); - - return options; - } - - private static void applyAntiDetection(ChromeDriver driver) { - String stealthScript = """ - Object.defineProperty(navigator, 'webdriver', { get: () => undefined }); - delete Object.getPrototypeOf(navigator).webdriver; - - window.chrome = { runtime: {}, loadTimes: function() {}, csi: function() {}, app: {} }; - - Object.defineProperty(navigator, 'permissions', { - get: () => ({ query: () => Promise.resolve({state: 'granted'}) }) - }); - - Object.defineProperty(navigator, 'plugins', { - get: () => Array.from({length: Math.floor(Math.random() * 5) + 2}, (_, i) => ({ - name: `Plugin ${i}`, description: `Plugin Description ${i}`, - filename: `plugin${i}.dll`, version: `${Math.floor(Math.random() * 10)}.0.0` - })) - }); - - Object.defineProperty(navigator, 'languages', { - get: () => ['zh-CN', 'zh', 'en-US', 'en'] - }); - - const screenWidth = 1920 + Math.floor(Math.random() * 100); - const screenHeight = 1080 + Math.floor(Math.random() * 100); - Object.defineProperty(screen, 'width', { get: () => screenWidth }); - Object.defineProperty(screen, 'height', { get: () => screenHeight }); - Object.defineProperty(screen, 'availWidth', { get: () => screenWidth }); - Object.defineProperty(screen, 'availHeight', { get: () => screenHeight - 40 }); - - if (window.document) { - ['$cdc_asdjflasutopfhvcZLmcfl_', '$chrome_asyncScriptInfo'].forEach(prop => { - Object.defineProperty(window.document, prop, { get: () => undefined, set: () => {} }); - }); - } - """; - - Map params = new HashMap<>(); - params.put("source", stealthScript); - driver.executeCdpCommand("Page.addScriptToEvaluateOnNewDocument", params); - - String userAgent = USER_AGENTS.get(random.nextInt(USER_AGENTS.size())); - Map networkParams = new HashMap<>(); - networkParams.put("userAgent", userAgent); - networkParams.put("acceptLanguage", "zh-CN,zh;q=0.9,en;q=0.8"); - networkParams.put("platform", "Win32"); - driver.executeCdpCommand("Network.setUserAgentOverride", networkParams); - } - - private static void setRandomWindowProperties(ChromeDriver driver) { - int width = 1200 + random.nextInt(400); - int height = 800 + random.nextInt(300); - driver.manage().window().setSize(new Dimension(width, height)); - driver.manage().window().setPosition(new Point(random.nextInt(100), random.nextInt(100))); - } -} \ No newline at end of file