liwq/bocai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

添加日志

Browse Source
This commit is contained in:
xuelijun
2026-01-31 09:54:53 +08:00
parent f7db42d060
commit 7f9885253f
3 changed files with 31 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
src/main/java/com/tem/bocai/schedules/CrawlerSchedule.java
View File

@@ -3,6 +3,7 @@ package com.tem.bocai.schedules;
import com.tem.bocai.entity.LoginInfoResult; import com.tem.bocai.entity.LoginInfoResult;
import com.tem.bocai.repository.LoginInfoRepository; import com.tem.bocai.repository.LoginInfoRepository;
import com.tem.bocai.util.*; import com.tem.bocai.util.*;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
@@ -16,6 +17,7 @@ import java.io.IOException;
import java.io.InputStreamReader; import java.io.InputStreamReader;
@Component @Component
@Slf4j
public class CrawlerSchedule { public class CrawlerSchedule {
@Autowired @Autowired
@@ -35,7 +37,7 @@ public class CrawlerSchedule {
// 从7:00分30秒起每5分钟执行一次 // 从7:00分30秒起每5分钟执行一次
@Scheduled(cron = "30 0/5 * * * ?") @Scheduled(cron = "30 0/5 * * * ?")
public void executeLotteryDraw() { public void executeLotteryDraw() {
System.out.println("开始爬取开奖结果..."); log.info("开始爬取开奖结果");
int retryCount = 0; int retryCount = 0;
boolean success = false; boolean success = false;
LoginInfoResult firstByOrderByCreateTimeDesc = loginInfoRepository.findFirstByOrderByCreateTimeDesc() LoginInfoResult firstByOrderByCreateTimeDesc = loginInfoRepository.findFirstByOrderByCreateTimeDesc()
@@ -46,17 +48,15 @@ public class CrawlerSchedule {
String token = tokenCacheService.getToken(); String token = tokenCacheService.getToken();
while (!success && retryCount < MAX_CRA) { while (!success && retryCount < MAX_CRA) {
System.out.println("\n=== 第 " + (retryCount + 1) + " 次尝试获取开奖结果 ==="); log.info("\n=== 第 " + (retryCount + 1) + " 次尝试获取开奖结果 ===");
if (token == null || token.isEmpty()) { if (token == null || token.isEmpty()) {
System.out.println("token为空，从数据库重新获取"); log.info("token为空，从数据库重新获取");
token = tokenCacheService.getTokenSqlite(); token = tokenCacheService.getTokenSqlite();
if (token == null) { if (token == null) {
System.err.println("无法获取有效token"); log.error("无法获取有效token");
} }
} }
System.out.println("使用token: " + (token.length() > 20 ? token.substring(0, 20) + "..." : token)); log.info("使用token: " + (token.length() > 20 ? token.substring(0, 20) + "..." : token));
// 创建爬虫实例，传入token // 创建爬虫实例，传入token
LotteryWebMagicCrawler crawler = new LotteryWebMagicCrawler(token,pypath); LotteryWebMagicCrawler crawler = new LotteryWebMagicCrawler(token,pypath);
@@ -72,8 +72,7 @@ public class CrawlerSchedule {
success = LotteryWebMagicCrawler.isLastParseSuccess(); success = LotteryWebMagicCrawler.isLastParseSuccess();
if (!success) { if (!success) {
System.out.println("本次尝试未解析到数据"); log.info("本次尝试未解析到数据");
// 重新获取token（下次重试用） // 重新获取token（下次重试用）
token = tokenCacheService.getTokenSqlite(); token = tokenCacheService.getTokenSqlite();
retryCount++; retryCount++;
@@ -87,12 +86,12 @@ public class CrawlerSchedule {
} }
} }
} else { } else {
System.out.println("成功解析到数据"); log.info("成功解析到数据");
} }
} }
if (!success) { if (!success) {
System.err.println("获取开奖结果失败，所有重试均未成功"); log.error("获取开奖结果失败，所有重试均未成功");
} }
} }
@@ -120,14 +119,13 @@ public class CrawlerSchedule {
// 从7:00分30秒起每5分钟执行一次爬取今日已经结算 // 从7:00分30秒起每5分钟执行一次爬取今日已经结算
@Scheduled(cron = "30 0/5 * * * ?") @Scheduled(cron = "30 0/5 * * * ?")
public void executeSettlement() { public void executeSettlement() {
System.out.println("开始爬取今日已经结算..."); log.info("开始爬取今日已经结算...");
int retryCount = 0; int retryCount = 0;
boolean success = false; boolean success = false;
String token = tokenCacheService.getToken(); String token = tokenCacheService.getToken();
while (!success && retryCount < MAX_CRA) { while (!success && retryCount < MAX_CRA) {
System.out.println("\n=== 第 " + (retryCount + 1) + " 次尝试获取今日注单 ==="); log.info("\n=== 第 " + (retryCount + 1) + " 次尝试获取今日注单 ===");
if (token == null || token.isEmpty()) { if (token == null || token.isEmpty()) {
System.out.println("token为空，从数据库重新获取"); System.out.println("token为空，从数据库重新获取");
token = tokenCacheService.getTokenSqlite(); token = tokenCacheService.getTokenSqlite();
@@ -153,8 +151,7 @@ public class CrawlerSchedule {
success = CompletedTodayCrawler.isLastParseSuccess(); success = CompletedTodayCrawler.isLastParseSuccess();
if (!success) { if (!success) {
System.out.println("本次尝试未解析到数据"); log.info("本次尝试未解析到数据");
// 重新获取token（下次重试用） // 重新获取token（下次重试用）
token = tokenCacheService.getTokenSqlite(); token = tokenCacheService.getTokenSqlite();
retryCount++; retryCount++;
@@ -168,12 +165,12 @@ public class CrawlerSchedule {
} }
} }
} else { } else {
System.out.println("成功解析到数据"); log.info("成功解析到数据");
} }
} }
if (!success) { if (!success) {
System.err.println("获取今日注单失败，所有重试均未成功"); log.error("获取今日注单失败，所有重试均未成功");
} }
} }

18
src/main/java/com/tem/bocai/util/CompletedTodayCrawler.java
View File

@@ -5,6 +5,7 @@ import com.tem.bocai.entity.CompletedToday;
import com.tem.bocai.repository.CompletedTodayRepository; import com.tem.bocai.repository.CompletedTodayRepository;
import com.tem.bocai.repository.LoginInfoRepository; import com.tem.bocai.repository.LoginInfoRepository;
import com.tem.bocai.service.CompletedTodayService; import com.tem.bocai.service.CompletedTodayService;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
@@ -26,7 +27,7 @@ import java.util.*;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
@Slf4j
public class CompletedTodayCrawler implements PageProcessor { public class CompletedTodayCrawler implements PageProcessor {
private final String token; private final String token;
@@ -66,36 +67,33 @@ public class CompletedTodayCrawler implements PageProcessor {
@Override @Override
public void process(Page page) { public void process(Page page) {
String url = page.getUrl().toString(); String url = page.getUrl().toString();
System.out.println("处理页面: " + url); log.info("处理页面: " + url);
Html html = page.getHtml(); Html html = page.getHtml();
String content = html.toString(); String content = html.toString();
// 打印一些基本信息 // 打印一些基本信息
System.out.println("页面标题: " + html.xpath("//title/text()").get()); log.info("页面标题: " + html.xpath("//title/text()").get());
System.out.println("页面大小: " + content.length() + " 字符");
// 检查是否有"暂无数据"提示 // 检查是否有"暂无数据"提示
if (content.contains("暂无数据")) { if (content.contains("暂无数据")) {
System.out.println("警告: 页面显示'暂无数据'"); log.info("警告: 页面显示'暂无数据'");
lastParseSuccess = true; // 标记失败 lastParseSuccess = true; // 标记失败
return; return;
} }
if (content.contains("可赢金额")) { if (content.contains("可赢金额")) {
System.out.println("未结明细不用爬"); log.info("未结明细不用爬");
lastParseSuccess = false; // 标记失败 lastParseSuccess = false; // 标记失败
return; return;
} }
// 解析注单数据 // 解析注单数据
List<Map<String, Object>> betList = parseBetHtml(content); List<Map<String, Object>> betList = parseBetHtml(content);
if (betList.isEmpty()) { if (betList.isEmpty()) {
System.out.println("未解析到注单数据"); log.info("未解析到注单数据");
lastParseSuccess = false; // 标记失败 lastParseSuccess = false; // 标记失败
// 尝试从其他可能的位置解析 // 尝试从其他可能的位置解析
//extractDebugInfo(html); //extractDebugInfo(html);
} else { } else {
lastParseSuccess = true; // 标记成功 lastParseSuccess = true; // 标记成功
System.out.println("解析到 " + betList.size() + " 条注单数据"); log.info("解析到 " + betList.size() + " 条注单数据");
List<CompletedToday> completedTodayList = convertForDatabase(betList); List<CompletedToday> completedTodayList = convertForDatabase(betList);
SQLiteUtil.saveCompletedToday(completedTodayList); SQLiteUtil.saveCompletedToday(completedTodayList);
} }

15
src/main/java/com/tem/bocai/util/LotteryWebMagicCrawler.java
View File

@@ -1,6 +1,7 @@
package com.tem.bocai.util; package com.tem.bocai.util;
import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.SerializationFeature;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import org.jsoup.nodes.Document; import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element; import org.jsoup.nodes.Element;
@@ -25,6 +26,7 @@ import java.io.IOException;
import java.util.*; import java.util.*;
//开奖的历史结果 //开奖的历史结果
@Slf4j
public class LotteryWebMagicCrawler implements PageProcessor { public class LotteryWebMagicCrawler implements PageProcessor {
private final String token; private final String token;
@@ -65,12 +67,11 @@ public class LotteryWebMagicCrawler implements PageProcessor {
Html html = page.getHtml(); Html html = page.getHtml();
// 打印页面基本信息 // 打印页面基本信息
System.out.println("页面URL: " + page.getUrl()); log.info("页面URL: " + page.getUrl());
System.out.println("页面标题: " + html.xpath("//title/text()").get()); log.info("页面标题: " + html.xpath("//title/text()").get());
// 示例：提取所有表格数据 // 示例：提取所有表格数据
Selectable tables = html.xpath("//table"); Selectable tables = html.xpath("//table");
System.out.println("找到 " + tables.nodes().size() + " 个表格"); log.info("找到 " + tables.nodes().size() + " 个表格");
if(tables.nodes().isEmpty()){ if(tables.nodes().isEmpty()){
lastParseSuccess = false; lastParseSuccess = false;
}else { }else {
@@ -238,7 +239,7 @@ public class LotteryWebMagicCrawler implements PageProcessor {
SQLiteUtil.writeToSQLite(resultList); SQLiteUtil.writeToSQLite(resultList);
// 将数据写入JSON文件（保留原有功能） // 将数据写入JSON文件（保留原有功能）
writeToJsonFile(resultList); writeToJsonFile(resultList);
System.out.println("打印结果===" + resultList); log.info("历史爬虫打印结果===" + resultList);
return resultList; return resultList;
} }
@@ -279,10 +280,10 @@ public class LotteryWebMagicCrawler implements PageProcessor {
// 将 List 写入 JSON 文件 // 将 List 写入 JSON 文件
objectMapper.writeValue(outputFile, resultList); objectMapper.writeValue(outputFile, resultList);
System.out.println("数据已成功写入文件: " + outputFile.getAbsolutePath()); log.info("数据已成功写入文件: " + outputFile.getAbsolutePath());
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
log.error("写入 JSON 文件失败: " + e.getMessage(), e);
throw new RuntimeException("写入 JSON 文件失败: " + e.getMessage(), e); throw new RuntimeException("写入 JSON 文件失败: " + e.getMessage(), e);
} }
} }