diff --git a/src/main/java/com/tem/bocai/entity/LoginInfoResult.java b/src/main/java/com/tem/bocai/entity/LoginInfoResult.java index e26ecab..b8a4f4a 100644 --- a/src/main/java/com/tem/bocai/entity/LoginInfoResult.java +++ b/src/main/java/com/tem/bocai/entity/LoginInfoResult.java @@ -52,6 +52,10 @@ public class LoginInfoResult { @Column(name = "cookie") private String cookie; + //余额 + @Column(name = "balance") + private String balance; + @Column(name = "create_time", nullable = false, updatable = false) @CreationTimestamp @Convert(converter = SQLiteDateConverter.class) diff --git a/src/main/java/com/tem/bocai/schedules/CrawlerSchedule.java b/src/main/java/com/tem/bocai/schedules/CrawlerSchedule.java index bf76568..cd38639 100644 --- a/src/main/java/com/tem/bocai/schedules/CrawlerSchedule.java +++ b/src/main/java/com/tem/bocai/schedules/CrawlerSchedule.java @@ -6,19 +6,14 @@ import com.tem.bocai.repository.LoginInfoRepository; import com.tem.bocai.repository.LotteryResultRepository; import com.tem.bocai.util.*; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.ObjectUtils; -import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import us.codecraft.webmagic.Spider; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; + import java.util.List; import java.util.Map; -import java.util.Optional; @Component @Slf4j @@ -174,7 +169,7 @@ public class CrawlerSchedule { // 从7:00分30秒起每5分钟执行一次爬取今日已经结算 - //@Scheduled(cron = "35 0/5 * * * ?") + @Scheduled(cron = "35 0/5 * * * ?") public void executeSettlement() { log.info("开始爬取今日已经结算..."); int retryCount = 0; @@ -237,23 +232,109 @@ public class CrawlerSchedule { } } + /** + * 每5分钟执行一次获取余额信息 + * 从7:00分40秒起每5分钟执行一次 + */ + @Scheduled(cron = "40 0/5 * * * ?") + //@Scheduled(cron = "*/9 * * * * ?") + public void executeGetBalance() { + log.info("开始获取余额信息"); - /*public void executeSettlement() { + // 获取最新的登录信息 + LoginInfoResult firstByOrderByCreateTimeDesc = loginInfoRepository.findFirstByOrderByCreateTimeDesc() + .orElse(null); + + if (firstByOrderByCreateTimeDesc == null) { + log.warn("未找到登录信息，跳过余额获取"); + return; + } + + if (firstByOrderByCreateTimeDesc.getOnOff() == ONOFF) { + log.info("开关关闭，跳过余额获取"); + return; + } + + // 获取token String token = tokenCacheService.getToken(); - System.out.println("得到token = " + token); - if (token != null && !token.isEmpty()) { - // 2. 创建爬虫实例，传入token - CompletedTodayCrawler crawler = new CompletedTodayCrawler(token); - LoginInfoResult firstByOrderByCreateTimeDesc = loginInfoRepository.findFirstByOrderByCreateTimeDesc() - .orElse(null); - // 4. 执行爬虫 - String url = firstByOrderByCreateTimeDesc.getLoginUrl()+"/member/bets?settled=true"; + if (token == null || token.isEmpty()) { + return; + } + + int retryCount = 0; + boolean success = false; + + while (!success && retryCount < MAX_CRA) { + log.info("=== 第 {} 次尝试获取余额信息 ===", retryCount + 1); + + // 重新获取token（如果重试） + if (retryCount > 0) { + token = tokenCacheService.getTokenSqlite(); + if (token == null) { + log.error("重试时无法获取有效token"); + retryCount++; + continue; + } + } + + log.info("使用token: " + (token.length() > 20 ? token.substring(0, 20) + "..." : token)); + + // 创建爬虫实例 + BalanceWebMagicCrawler crawler = new BalanceWebMagicCrawler(token); + + // 构建URL + String url = firstByOrderByCreateTimeDesc.getLoginUrl() + "/member/index"; + + // 执行爬虫 Spider.create(crawler) .addUrl(url) .thread(1) .run(); - } - }*/ + // 检查是否成功解析数据 + success = BalanceWebMagicCrawler.isLastParseSuccess(); + + if (success) { + // 获取并处理余额信息 + Map balanceInfo = BalanceWebMagicCrawler.getBalanceInfo(); + if (!balanceInfo.isEmpty()) { + log.info("成功获取余额信息:"); + for (Map.Entry entry : balanceInfo.entrySet()) { + if (entry.getKey().contains("快开彩额度")) { + firstByOrderByCreateTimeDesc.setBalance(entry.getValue()); + if (entry.getValue()!=null) { + loginInfoRepository.save(firstByOrderByCreateTimeDesc); + } + + } + + } + } else { + log.warn("解析到空余额信息"); + success = false; + } + } + + if (!success) { + log.info("本次尝试未解析到余额信息"); + retryCount++; + + // 等待一下再重试 + if (retryCount < MAX_CRA) { + try { + Thread.sleep(2000 * retryCount); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } + } + + if (!success) { + log.error("获取余额信息失败，所有重试均未成功"); + } else { + log.info("余额信息获取完成"); + } + } } diff --git a/src/main/java/com/tem/bocai/util/BalanceWebMagicCrawler.java b/src/main/java/com/tem/bocai/util/BalanceWebMagicCrawler.java new file mode 100644 index 0000000..3804d13 --- /dev/null +++ b/src/main/java/com/tem/bocai/util/BalanceWebMagicCrawler.java @@ -0,0 +1,225 @@ +package com.tem.bocai.util; + +import lombok.extern.slf4j.Slf4j; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import us.codecraft.webmagic.Page; +import us.codecraft.webmagic.Site; +import us.codecraft.webmagic.Spider; +import us.codecraft.webmagic.processor.PageProcessor; +import us.codecraft.webmagic.selector.Html; +import us.codecraft.webmagic.selector.Selectable; + +import java.util.HashMap; +import java.util.Map; + +@Slf4j +public class BalanceWebMagicCrawler implements PageProcessor { + + private final String token; + private Site site; + private static volatile boolean lastParseSuccess = false; + private static Map balanceInfo = new HashMap<>(); + + public BalanceWebMagicCrawler(String token) { + this.token = token; + initSite(); + } + + /** + * 初始化Site配置 + */ + private void initSite() { + site = Site.me() + .setRetryTimes(3) + .setSleepTime(1000) + .setTimeOut(10000) + .setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"); + + // 设置cookie + if (token != null && !token.isEmpty()) { + site.addHeader("cookie", "token=" + token); + } + + } + + @Override + public void process(Page page) { + try { + // 获取页面HTML + Html html = page.getHtml(); + String htmlContent = html.toString(); + + log.info("访问余额页面URL: " + page.getUrl()); + log.info("页面标题: " + html.xpath("//title/text()").get()); + + // 解析HTML获取余额信息 + Map result = parseBalanceInfo(htmlContent); + + if (!result.isEmpty()) { + lastParseSuccess = true; + balanceInfo = result; + + // 打印解析到的余额信息 + log.info("========== 余额信息解析成功 =========="); + for (Map.Entry entry : result.entrySet()) { + log.info("{}: {}", entry.getKey(), entry.getValue()); + } + log.info("====================================="); + + // 将数据存入结果 + page.putField("balanceInfo", result); + page.putField("html", htmlContent.substring(0, Math.min(500, htmlContent.length())) + "..."); + + } else { + lastParseSuccess = false; + log.warn("未解析到余额信息"); + } + + } catch (Exception e) { + lastParseSuccess = false; + log.error("解析余额页面时发生错误: " + e.getMessage(), e); + } + } + + /** + * 解析余额信息 + * @param htmlContent HTML内容 + * @return 余额信息Map + */ + private Map parseBalanceInfo(String htmlContent) { + Map result = new HashMap<>(); + + try { + Document doc = Jsoup.parse(htmlContent); + + // 查找账户信息部分 + Element userInfoDiv = doc.selectFirst("div.user_info"); + if (userInfoDiv == null) { + log.warn("未找到账户信息div"); + return result; + } + + // 查找所有账户类型 + Elements accountDivs = userInfoDiv.select("div.accounts"); + + for (Element accountDiv : accountDivs) { + // 检查是否显示（有些账户类型可能不显示） + String style = accountDiv.attr("style"); + if (style.contains("display:none")) { + continue; + } + + // 获取账户类型标签和余额 + Elements infoDivs = accountDiv.select("div.info"); + for (Element infoDiv : infoDivs) { + Element label = infoDiv.selectFirst("label"); + Element value = infoDiv.selectFirst("span"); + + if (label != null && value != null) { + String labelText = label.text().trim(); + String valueText = value.text().trim(); + + // 特别关注快开彩额度 + if (labelText.contains("快开彩额度")) { + result.put("快开彩额度", valueText); + log.info("找到快开彩额度: {}", valueText); + } else if (labelText.contains("未结算金额")) { + result.put("未结算金额", valueText); + } else if (labelText.contains("全国彩额度")) { + result.put("全国彩额度", valueText); + } else if (labelText.contains("香港彩额度")) { + result.put("香港彩额度", valueText); + } else if (labelText.contains("第三方额度")) { + result.put("第三方额度", valueText); + } + } + } + } + + // 如果没找到账户信息，尝试其他方式查找 + if (result.isEmpty()) { + // 查找账号信息 + Element accountSpan = doc.selectFirst("div.inline-name span"); + if (accountSpan != null) { + result.put("账号", accountSpan.text().trim()); + } + + // 查找所有包含"额度"的元素 + Elements balanceElements = doc.select("span.balance"); + for (Element balance : balanceElements) { + Element parent = balance.parent(); + if (parent != null) { + Element label = parent.selectFirst("label"); + if (label != null && label.text().contains("额度")) { + result.put(label.text().trim(), balance.text().trim()); + } + } + } + } + + } catch (Exception e) { + log.error("解析余额信息时发生错误: " + e.getMessage(), e); + } + + return result; + } + + @Override + public Site getSite() { + return site; + } + + /** + * 获取解析状态 + */ + public static boolean isLastParseSuccess() { + return lastParseSuccess; + } + + /** + * 获取解析到的余额信息 + */ + public static Map getBalanceInfo() { + return new HashMap<>(balanceInfo); + } + + /** + * 清除余额信息缓存 + */ + public static void clearBalanceInfo() { + balanceInfo.clear(); + } + + /** + * 测试方法 + */ + public static void main(String[] args) { + // 测试用的token + String testToken = "31ea12c0a75f1f17dc2004ea5f7501aed73c00fa"; + String url = "https://4701268539-esh.qdk63ayw8g.com/member/index"; + + log.info("开始测试余额爬虫..."); + log.info("使用URL: " + url); + + // 创建爬虫 + Spider.create(new BalanceWebMagicCrawler(testToken)) + .addUrl(url) + .thread(1) + .run(); + + // 打印结果 + if (isLastParseSuccess()) { + Map info = getBalanceInfo(); + System.out.println("\n========== 余额信息 =========="); + for (Map.Entry entry : info.entrySet()) { + System.out.println(entry.getKey() + ": " + entry.getValue()); + } + System.out.println("=============================="); + } else { + System.out.println("获取余额信息失败"); + } + } +} \ No newline at end of file diff --git a/src/main/java/com/tem/bocai/util/HttpClientExample.java b/src/main/java/com/tem/bocai/util/HttpClientExample.java index 20abfdb..a06ce33 100644 --- a/src/main/java/com/tem/bocai/util/HttpClientExample.java +++ b/src/main/java/com/tem/bocai/util/HttpClientExample.java @@ -4,8 +4,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import lombok.extern.slf4j.Slf4j; - -import javax.net.ssl.SSLContext; import java.io.BufferedReader; import java.io.File; import java.io.IOException; @@ -14,11 +12,6 @@ import java.net.HttpURLConnection; import java.net.InetSocketAddress; import java.net.Proxy; import java.net.URL; -import java.security.KeyManagementException; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.security.cert.CertificateException; -import java.security.cert.X509Certificate; import java.text.SimpleDateFormat; import java.util.*;