liwq/bocai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

爬取保存今日已结算

Browse Source
This commit is contained in:
xuelijun
2026-01-22 11:53:44 +08:00
parent 6e0e4b684f
commit 3c8d81bd7b
7 changed files with 191 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/com/tem/bocai/controller/LoginCrawler.java
View File

@@ -15,7 +15,7 @@ public class LoginCrawler {
private final LoginService loginService;
// 登入
// 登入爬取历史开奖结果
public LoginCrawler(LoginService loginService) {
this.loginService = loginService;
}

44
src/main/java/com/tem/bocai/entity/CompletedToday.java Normal file
View File

@@ -0,0 +1,44 @@
package com.tem.bocai.entity;
import jakarta.persistence.*;
import lombok.Data;
import java.util.Date;
@Entity //今日完结
@Table(name = "completed_today")
@Data
public class CompletedToday {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Column(name = "bet_id", nullable = false, unique = true)
private String betId; //
@Column(name = "time", nullable = false)
private String time; //
//下注金额
@Column(name = "bet_amount", nullable = false)
private Double betAmount;
//输赢
@Column(name = "result", nullable = false)
private String result;
//输赢结果金额
@Column(name = "result_amount", nullable = false)
private Double resultAmount;
/* @Column(name = "current_num", nullable = false)
private Integer currentNum;*/
@Column(name = "create_time", nullable = false, updatable = false)
@Temporal(TemporalType.TIMESTAMP)
private Date createTime;
@Column(name = "update_time", nullable = false)
@Temporal(TemporalType.TIMESTAMP)
private Date updateTime;
}

14
src/main/java/com/tem/bocai/repository/CompletedTodayRepository.java Normal file
View File

@@ -0,0 +1,14 @@
package com.tem.bocai.repository;
import com.tem.bocai.entity.CompletedToday;
import com.tem.bocai.entity.LoginInfoResult;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.stereotype.Repository;
import java.util.Optional;
@Repository
public interface CompletedTodayRepository extends JpaRepository<CompletedToday, Long> {
}

11
src/main/java/com/tem/bocai/service/CompletedTodayService.java Normal file
View File

@@ -0,0 +1,11 @@
package com.tem.bocai.service;
import com.tem.bocai.entity.CompletedToday;
import com.tem.bocai.param.LoginInfoParam;
import java.util.List;
public interface CompletedTodayService {
void saveCompletedToday(List<CompletedToday> list);
}

31
src/main/java/com/tem/bocai/service/impl/CompletedTodayServiceImpl.java Normal file
View File

@@ -0,0 +1,31 @@
package com.tem.bocai.service.impl;
import com.tem.bocai.entity.CompletedToday;
import com.tem.bocai.entity.LoginInfoResult;
import com.tem.bocai.param.LoginInfoParam;
import com.tem.bocai.repository.CompletedTodayRepository;
import com.tem.bocai.repository.LoginInfoRepository;
import com.tem.bocai.service.CompletedTodayService;
import com.tem.bocai.service.LoginService;
import com.tem.bocai.util.*;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import us.codecraft.webmagic.Spider;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.util.Map;
@Service
public class CompletedTodayServiceImpl implements CompletedTodayService {
@Autowired
private CompletedTodayRepository completedTodayRepository;
@Override
public void saveCompletedToday(List<CompletedToday> list) {
completedTodayRepository.saveAll(list);
}
}

55
src/main/java/com/tem/bocai/util/CompletedTodayCrawler.java
View File

@@ -1,10 +1,16 @@
package com.tem.bocai.util;
import com.tem.bocai.entity.CompletedToday;
import com.tem.bocai.repository.CompletedTodayRepository;
import com.tem.bocai.repository.LoginInfoRepository;
import com.tem.bocai.service.CompletedTodayService;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
@@ -17,12 +23,15 @@ import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class CompletedTodayCrawler implements PageProcessor {
private final String token;
private Site site;
private static final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
// 或者提供带参数的构造函数
public CompletedTodayCrawler(String token) {
this.token = token;
initSite();
@@ -76,11 +85,9 @@ public class CompletedTodayCrawler implements PageProcessor {
extractDebugInfo(html);
} else {
System.out.println("解析到 " + betList.size() + " 条注单数据");
// 打印部分数据示例
printSampleData(betList);
List<CompletedToday> completedTodayList = convertForDatabase(betList);
SQLiteUtil.saveCompletedToday(completedTodayList);
}
// 保存原始HTML用于调试
saveHtmlForDebug(content, url);
}
@@ -279,29 +286,21 @@ public class CompletedTodayCrawler implements PageProcessor {
/**
* 转换数据结构以适应数据库
*/
private List<Map<String, Object>> convertForDatabase(List<Map<String, Object>> betList) {
List<Map<String, Object>> dbData = new ArrayList<>();
private List<CompletedToday> convertForDatabase(List<Map<String, Object>> betList) {
List<CompletedToday> completedTodayList = new ArrayList<>();
for (Map<String, Object> bet : betList) {
Map<String, Object> dbRecord = new HashMap<>();
CompletedToday completedToday = new CompletedToday();
dbRecord.put("id", bet.get("bet_id"));
dbRecord.put("bet_id", bet.get("bet_id"));
dbRecord.put("period", bet.get("period"));
dbRecord.put("bet_time", bet.get("time"));
dbRecord.put("game_type", bet.get("game_type"));
dbRecord.put("plate", bet.get("plate"));
dbRecord.put("bet_amount", bet.get("bet_amount"));
dbRecord.put("rebate_rate", bet.get("rebate_rate"));
dbRecord.put("result", bet.get("result"));
dbRecord.put("result_amount", bet.get("result_amount"));
dbRecord.put("parse_time", bet.get("parse_time"));
dbRecord.put("source", bet.get("source"));
completedToday.setBetId((String) bet.get("bet_id"));
completedToday.setTime((String) bet.get("time"));
completedToday.setBetAmount((Double)bet.get("bet_amount"));
completedToday.setResultAmount((Double) bet.get("result_amount"));
completedToday.setResult((String) bet.get("result"));
dbData.add(dbRecord);
completedTodayList.add(completedToday);
}
return dbData;
return completedTodayList;
}
@@ -378,6 +377,18 @@ public class CompletedTodayCrawler implements PageProcessor {
}
}
/**
* 打印示例数据
*/
/* private void saveCompletedToday(List<Map<String, Object>> betList) {
for (Map<String, Object> rowData : betList) {
// 期数
pstmt.setString(1, rowData.get("id").toString());
}*/
@Override
public Site getSite() {
return site;

57
src/main/java/com/tem/bocai/util/SQLiteUtil.java
View File

@@ -1,5 +1,6 @@
package com.tem.bocai.util;
import com.tem.bocai.entity.CompletedToday;
import com.tem.bocai.entity.LoginInfoResult;
import java.sql.*;
@@ -187,6 +188,62 @@ public class SQLiteUtil {
}
}
/**
* 批量插入今日完结数据
*/
public static boolean saveCompletedToday(List<CompletedToday> list) {
if (list == null || list.isEmpty()) return false;
String sql = """
INSERT OR IGNORE INTO completed_today
(bet_id, time, bet_amount, result, result_amount, create_time, update_time)
VALUES (?, ?, ?, ?, ?, ?, ?)
""";
try (Connection conn = getConnection();
PreparedStatement pstmt = conn.prepareStatement(sql)) {
conn.setAutoCommit(false);
for (CompletedToday data : list) {
if (data == null || data.getBetId() == null) continue;
pstmt.setString(1, data.getBetId());
pstmt.setString(2, data.getTime());
pstmt.setDouble(3, data.getBetAmount() != null ? data.getBetAmount() : 0.0);
pstmt.setString(4, data.getResult() != null ? data.getResult() : "未知");
pstmt.setDouble(5, data.getResultAmount() != null ? data.getResultAmount() : 0.0);
// 使用ISO8601格式: yyyy-MM-dd HH:mm:ss
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
// 处理create_time
if (data.getCreateTime() != null) {
pstmt.setString(6, sdf.format(data.getCreateTime()));
} else {
pstmt.setString(6, sdf.format(new Date()));
}
// 处理update_time
if (data.getUpdateTime() != null) {
pstmt.setString(7, sdf.format(data.getUpdateTime()));
} else {
pstmt.setString(7, sdf.format(new Date()));
}
pstmt.addBatch();
}
pstmt.executeBatch();
conn.commit();
return true;
} catch (SQLException e) {
e.printStackTrace();
return false;
}
}
/**
* 关闭数据库资源
*/