liwq/bocai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master'

Browse Source 
# Conflicts:
#	bocai.db
#	src/main/java/com/tem/bocai/util/HttpClientExample.java
This commit is contained in:
liwq
2026-02-04 09:55:37 +08:00
parent 3f4e0b8f8f 9460367dc1
commit 9376a85673
4 changed files with 328 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/com/tem/bocai/entity/LoginInfoResult.java
View File

@@ -52,6 +52,10 @@ public class LoginInfoResult {
@Column(name = "cookie")
private String cookie;
//余额
@Column(name = "balance")
private String balance;
@Column(name = "create_time", nullable = false, updatable = false)
@CreationTimestamp
@Convert(converter = SQLiteDateConverter.class)

115
src/main/java/com/tem/bocai/schedules/CrawlerSchedule.java
View File

@@ -6,19 +6,14 @@ import com.tem.bocai.repository.LoginInfoRepository;
import com.tem.bocai.repository.LotteryResultRepository;
import com.tem.bocai.util.*;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Spider;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@Component
@Slf4j
@@ -174,7 +169,7 @@ public class CrawlerSchedule {
// 从7:00分30秒起每5分钟执行一次爬取今日已经结算
//@Scheduled(cron = "35 0/5 * * * ?")
@Scheduled(cron = "35 0/5 * * * ?")
public void executeSettlement() {
log.info("开始爬取今日已经结算...");
int retryCount = 0;
@@ -237,23 +232,109 @@ public class CrawlerSchedule {
}
}
/**
* 每5分钟执行一次获取余额信息
* 从7:00分40秒起每5分钟执行一次
*/
@Scheduled(cron = "40 0/5 * * * ?")
//@Scheduled(cron = "*/9 * * * * ?")
public void executeGetBalance() {
log.info("开始获取余额信息");
/*public void executeSettlement() {
String token = tokenCacheService.getToken();
System.out.println("得到token = " + token);
if (token != null && !token.isEmpty()) {
// 2. 创建爬虫实例，传入token
CompletedTodayCrawler crawler = new CompletedTodayCrawler(token);
// 获取最新的登录信息
LoginInfoResult firstByOrderByCreateTimeDesc = loginInfoRepository.findFirstByOrderByCreateTimeDesc()
.orElse(null);
// 4. 执行爬虫
String url = firstByOrderByCreateTimeDesc.getLoginUrl()+"/member/bets?settled=true";
if (firstByOrderByCreateTimeDesc == null) {
log.warn("未找到登录信息，跳过余额获取");
return;
}
if (firstByOrderByCreateTimeDesc.getOnOff() == ONOFF) {
log.info("开关关闭，跳过余额获取");
return;
}
// 获取token
String token = tokenCacheService.getToken();
if (token == null || token.isEmpty()) {
return;
}
int retryCount = 0;
boolean success = false;
while (!success && retryCount < MAX_CRA) {
log.info("=== 第 {} 次尝试获取余额信息 ===", retryCount + 1);
// 重新获取token（如果重试）
if (retryCount > 0) {
token = tokenCacheService.getTokenSqlite();
if (token == null) {
log.error("重试时无法获取有效token");
retryCount++;
continue;
}
}
log.info("使用token: " + (token.length() > 20 ? token.substring(0, 20) + "..." : token));
// 创建爬虫实例
BalanceWebMagicCrawler crawler = new BalanceWebMagicCrawler(token);
// 构建URL
String url = firstByOrderByCreateTimeDesc.getLoginUrl() + "/member/index";
// 执行爬虫
Spider.create(crawler)
.addUrl(url)
.thread(1)
.run();
}
}*/
// 检查是否成功解析数据
success = BalanceWebMagicCrawler.isLastParseSuccess();
if (success) {
// 获取并处理余额信息
Map<String, String> balanceInfo = BalanceWebMagicCrawler.getBalanceInfo();
if (!balanceInfo.isEmpty()) {
log.info("成功获取余额信息:");
for (Map.Entry<String, String> entry : balanceInfo.entrySet()) {
if (entry.getKey().contains("快开彩额度")) {
firstByOrderByCreateTimeDesc.setBalance(entry.getValue());
if (entry.getValue()!=null) {
loginInfoRepository.save(firstByOrderByCreateTimeDesc);
}
}
}
} else {
log.warn("解析到空余额信息");
success = false;
}
}
if (!success) {
log.info("本次尝试未解析到余额信息");
retryCount++;
// 等待一下再重试
if (retryCount < MAX_CRA) {
try {
Thread.sleep(2000 * retryCount);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
}
}
if (!success) {
log.error("获取余额信息失败，所有重试均未成功");
} else {
log.info("余额信息获取完成");
}
}
}

225
src/main/java/com/tem/bocai/util/BalanceWebMagicCrawler.java Normal file
View File

@@ -0,0 +1,225 @@
package com.tem.bocai.util;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;
import java.util.HashMap;
import java.util.Map;
@Slf4j
public class BalanceWebMagicCrawler implements PageProcessor {
private final String token;
private Site site;
private static volatile boolean lastParseSuccess = false;
private static Map<String, String> balanceInfo = new HashMap<>();
public BalanceWebMagicCrawler(String token) {
this.token = token;
initSite();
}
/**
* 初始化Site配置
*/
private void initSite() {
site = Site.me()
.setRetryTimes(3)
.setSleepTime(1000)
.setTimeOut(10000)
.setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36");
// 设置cookie
if (token != null && !token.isEmpty()) {
site.addHeader("cookie", "token=" + token);
}
}
@Override
public void process(Page page) {
try {
// 获取页面HTML
Html html = page.getHtml();
String htmlContent = html.toString();
log.info("访问余额页面URL: " + page.getUrl());
log.info("页面标题: " + html.xpath("//title/text()").get());
// 解析HTML获取余额信息
Map<String, String> result = parseBalanceInfo(htmlContent);
if (!result.isEmpty()) {
lastParseSuccess = true;
balanceInfo = result;
// 打印解析到的余额信息
log.info("========== 余额信息解析成功 ==========");
for (Map.Entry<String, String> entry : result.entrySet()) {
log.info("{}: {}", entry.getKey(), entry.getValue());
}
log.info("=====================================");
// 将数据存入结果
page.putField("balanceInfo", result);
page.putField("html", htmlContent.substring(0, Math.min(500, htmlContent.length())) + "...");
} else {
lastParseSuccess = false;
log.warn("未解析到余额信息");
}
} catch (Exception e) {
lastParseSuccess = false;
log.error("解析余额页面时发生错误: " + e.getMessage(), e);
}
}
/**
* 解析余额信息
* @param htmlContent HTML内容
* @return 余额信息Map
*/
private Map<String, String> parseBalanceInfo(String htmlContent) {
Map<String, String> result = new HashMap<>();
try {
Document doc = Jsoup.parse(htmlContent);
// 查找账户信息部分
Element userInfoDiv = doc.selectFirst("div.user_info");
if (userInfoDiv == null) {
log.warn("未找到账户信息div");
return result;
}
// 查找所有账户类型
Elements accountDivs = userInfoDiv.select("div.accounts");
for (Element accountDiv : accountDivs) {
// 检查是否显示（有些账户类型可能不显示）
String style = accountDiv.attr("style");
if (style.contains("display:none")) {
continue;
}
// 获取账户类型标签和余额
Elements infoDivs = accountDiv.select("div.info");
for (Element infoDiv : infoDivs) {
Element label = infoDiv.selectFirst("label");
Element value = infoDiv.selectFirst("span");
if (label != null && value != null) {
String labelText = label.text().trim();
String valueText = value.text().trim();
// 特别关注快开彩额度
if (labelText.contains("快开彩额度")) {
result.put("快开彩额度", valueText);
log.info("找到快开彩额度: {}", valueText);
} else if (labelText.contains("未结算金额")) {
result.put("未结算金额", valueText);
} else if (labelText.contains("全国彩额度")) {
result.put("全国彩额度", valueText);
} else if (labelText.contains("香港彩额度")) {
result.put("香港彩额度", valueText);
} else if (labelText.contains("第三方额度")) {
result.put("第三方额度", valueText);
}
}
}
}
// 如果没找到账户信息，尝试其他方式查找
if (result.isEmpty()) {
// 查找账号信息
Element accountSpan = doc.selectFirst("div.inline-name span");
if (accountSpan != null) {
result.put("账号", accountSpan.text().trim());
}
// 查找所有包含"额度"的元素
Elements balanceElements = doc.select("span.balance");
for (Element balance : balanceElements) {
Element parent = balance.parent();
if (parent != null) {
Element label = parent.selectFirst("label");
if (label != null && label.text().contains("额度")) {
result.put(label.text().trim(), balance.text().trim());
}
}
}
}
} catch (Exception e) {
log.error("解析余额信息时发生错误: " + e.getMessage(), e);
}
return result;
}
@Override
public Site getSite() {
return site;
}
/**
* 获取解析状态
*/
public static boolean isLastParseSuccess() {
return lastParseSuccess;
}
/**
* 获取解析到的余额信息
*/
public static Map<String, String> getBalanceInfo() {
return new HashMap<>(balanceInfo);
}
/**
* 清除余额信息缓存
*/
public static void clearBalanceInfo() {
balanceInfo.clear();
}
/**
* 测试方法
*/
public static void main(String[] args) {
// 测试用的token
String testToken = "31ea12c0a75f1f17dc2004ea5f7501aed73c00fa";
String url = "https://4701268539-esh.qdk63ayw8g.com/member/index";
log.info("开始测试余额爬虫...");
log.info("使用URL: " + url);
// 创建爬虫
Spider.create(new BalanceWebMagicCrawler(testToken))
.addUrl(url)
.thread(1)
.run();
// 打印结果
if (isLastParseSuccess()) {
Map<String, String> info = getBalanceInfo();
System.out.println("\n========== 余额信息 ==========");
for (Map.Entry<String, String> entry : info.entrySet()) {
System.out.println(entry.getKey() + ": " + entry.getValue());
}
System.out.println("==============================");
} else {
System.out.println("获取余额信息失败");
}
}
}

7
src/main/java/com/tem/bocai/util/HttpClientExample.java
View File

@@ -4,8 +4,6 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import lombok.extern.slf4j.Slf4j;
import javax.net.ssl.SSLContext;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
@@ -14,11 +12,6 @@ import java.net.HttpURLConnection;
import java.net.InetSocketAddress;
import java.net.Proxy;
import java.net.URL;
import java.security.KeyManagementException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.text.SimpleDateFormat;
import java.util.*;