This commit is contained in:
2025-09-25 16:04:00 +08:00
parent 05b923b1ac
commit bb997857fd
14 changed files with 185 additions and 182 deletions

View File

@@ -7,6 +7,7 @@ import { amazonApi } from '../../api/amazon'
const loading = ref(false) // 主加载状态 const loading = ref(false) // 主加载状态
const tableLoading = ref(false) // 表格加载状态 const tableLoading = ref(false) // 表格加载状态
const progressPercentage = ref(0) // 进度百分比 const progressPercentage = ref(0) // 进度百分比
const progressVisible = ref(false) // 进度条是否显示(完成后仍保留)
const localProductData = ref<any[]>([]) // 本地产品数据 const localProductData = ref<any[]>([]) // 本地产品数据
const currentAsin = ref('') // 当前处理的ASIN const currentAsin = ref('') // 当前处理的ASIN
const genmaiLoading = ref(false) // Genmai Spirit加载状态 const genmaiLoading = ref(false) // Genmai Spirit加载状态
@@ -43,7 +44,7 @@ const regionOptions = [
] ]
const pendingAsins = ref<string[]>([]) const pendingAsins = ref<string[]>([])
// 通用消息提示 // 通用消息提示Element Plus
function showMessage(message: string, type: 'success' | 'warning' | 'error' | 'info' = 'info') { function showMessage(message: string, type: 'success' | 'warning' | 'error' | 'info' = 'info') {
ElMessage({ message, type }) ElMessage({ message, type })
} }
@@ -55,6 +56,7 @@ async function processExcelFile(file: File) {
tableLoading.value = true tableLoading.value = true
localProductData.value = [] localProductData.value = []
progressPercentage.value = 0 progressPercentage.value = 0
progressVisible.value = false
const response = await amazonApi.importAsinFromExcel(file) const response = await amazonApi.importAsinFromExcel(file)
const asinList = response.data.asinList const asinList = response.data.asinList
@@ -63,10 +65,7 @@ async function processExcelFile(file: File) {
showMessage('文件中未找到有效的ASIN数据', 'warning') showMessage('文件中未找到有效的ASIN数据', 'warning')
return return
} }
// 存入待采集队列,等待用户点击“获取数据”再开始
pendingAsins.value = asinList pendingAsins.value = asinList
showMessage(`成功解析 ${asinList.length} 个ASIN点击“获取数据”开始采集`, 'success')
} catch (error: any) { } catch (error: any) {
showMessage(error.message || '处理文件失败', 'error') showMessage(error.message || '处理文件失败', 'error')
} finally { } finally {
@@ -90,8 +89,8 @@ async function onDrop(e: DragEvent) {
dragActive.value = false dragActive.value = false
const file = e.dataTransfer?.files?.[0] const file = e.dataTransfer?.files?.[0]
if (!file) return if (!file) return
const ok = /(\.csv|\.txt|\.xls|\.xlsx)$/i.test(file.name) const ok = /\.xlsx?$/i.test(file.name)
if (!ok) return showMessage('仅支持 .csv/.txt/.xls/.xlsx 文件', 'warning') if (!ok) return showMessage('仅支持 .xls/.xlsx 文件', 'warning')
await processExcelFile(file) await processExcelFile(file)
} }
@@ -170,6 +169,7 @@ async function startQueuedFetch() {
return return
} }
loading.value = true loading.value = true
progressVisible.value = true
tableLoading.value = true tableLoading.value = true
try { try {
await batchGetProductInfo(pendingAsins.value) await batchGetProductInfo(pendingAsins.value)
@@ -218,6 +218,13 @@ function getSellerShipperText(product: any) {
return text return text
} }
// 判定无货(用于标红 ASIN
function isOutOfStock(product: any) {
const sellerEmpty = !product?.seller || product.seller === '无货'
const priceEmpty = !product?.price || product.price === '无货'
return sellerEmpty || priceEmpty
}
// 停止获取操作 // 停止获取操作
function stopFetch() { function stopFetch() {
loading.value = false loading.value = false
@@ -248,10 +255,26 @@ function handleCurrentChange(page: number) {
} }
// 使用 Element Plus 的 jumper不再需要手动跳转函数 // 使用 Element Plus 的 jumper不再需要手动跳转函数
// 示例弹窗
const amazonExampleVisible = ref(false)
function openAmazonUpload() { function openAmazonUpload() {
amazonUpload.value?.click() amazonUpload.value?.click()
} }
function viewAmazonExample() { amazonExampleVisible.value = true }
function downloadAmazonTemplate() {
const html = '<table><tr><th>ASIN</th></tr><tr><td>B0XXXXXXX1</td></tr><tr><td>B0XXXXXXX2</td></tr></table>'
const blob = new Blob([html], { type: 'application/vnd.ms-excel' })
const url = URL.createObjectURL(blob)
const a = document.createElement('a')
a.href = url
a.download = 'amazon_asin_template.xls'
document.body.appendChild(a)
a.click()
document.body.removeChild(a)
URL.revokeObjectURL(url)
}
// 组件挂载时获取最新数据 // 组件挂载时获取最新数据
onMounted(async () => { onMounted(async () => {
try { try {
@@ -276,18 +299,18 @@ onMounted(async () => {
<div class="step-index">1</div> <div class="step-index">1</div>
<div class="step-card"> <div class="step-card">
<div class="step-header"><div class="title">导入ASIN</div></div> <div class="step-header"><div class="title">导入ASIN</div></div>
<div class="desc">仅支持包含 ASIN 列的 CSV/Excel 文档</div> <div class="desc">仅支持包含 ASIN 列的 Excel 文档</div>
<div class="links"> <div class="links">
<a class="link" @click.prevent>点击查看示例</a> <a class="link" @click.prevent="viewAmazonExample">点击查看示例</a>
<span class="sep">|</span> <span class="sep">|</span>
<a class="link" @click.prevent>点击下载模板</a> <a class="link" @click.prevent="downloadAmazonTemplate">点击下载模板</a>
</div> </div>
<div class="dropzone" @dragover.prevent="onDragOver" @dragleave="onDragLeave" @drop="onDrop" @click="openAmazonUpload"> <div class="dropzone" @dragover.prevent="onDragOver" @dragleave="onDragLeave" @drop="onDrop" @click="openAmazonUpload">
<div class="dz-el-icon">📤</div> <div class="dz-el-icon">📤</div>
<div class="dz-text">点击或将文件拖拽到这里上传</div> <div class="dz-text">点击或将文件拖拽到这里上传</div>
<div class="dz-sub">支持 .csv .txt .xls .xlsx</div> <div class="dz-sub">支持 .xls .xlsx</div>
</div> </div>
<input ref="amazonUpload" style="display:none" type="file" accept=".csv,.txt,.xls,.xlsx" @change="handleExcelUpload" :disabled="loading" /> <input ref="amazonUpload" style="display:none" type="file" accept=".xls,.xlsx" @change="handleExcelUpload" :disabled="loading" />
</div> </div>
</div> </div>
<!-- 2 网站地区 --> <!-- 2 网站地区 -->
@@ -311,6 +334,16 @@ onMounted(async () => {
<div class="desc">导入表格后点击下方按钮开始获取ASIN数据</div> <div class="desc">导入表格后点击下方按钮开始获取ASIN数据</div>
<el-button size="small" class="w100 btn-blue" :disabled="!pendingAsins.length || loading" @click="startQueuedFetch">{{ loading ? '处理中...' : '获取数据' }}</el-button> <el-button size="small" class="w100 btn-blue" :disabled="!pendingAsins.length || loading" @click="startQueuedFetch">{{ loading ? '处理中...' : '获取数据' }}</el-button>
<div class="mini-hint" v-if="pendingAsins.length">已导入 {{ pendingAsins.length }} ASIN</div> <div class="mini-hint" v-if="pendingAsins.length">已导入 {{ pendingAsins.length }} ASIN</div>
<div class="progress-section" v-if="progressVisible">
<div class="progress-box">
<div class="progress-container">
<div class="progress-bar">
<div class="progress-fill" :style="{ width: progressPercentage + '%' }"></div>
</div>
<div class="progress-text">{{ progressPercentage }}%</div>
</div>
</div>
</div>
<!-- 左侧不再显示进度条 --> <!-- 左侧不再显示进度条 -->
</div> </div>
</div> </div>
@@ -337,17 +370,18 @@ onMounted(async () => {
<!-- 数据显示区域 --> <!-- 数据显示区域 -->
<div class="table-container"> <div class="table-container">
<div class="table-section"> <div class="table-section">
<!-- 表格上方进度条与乐天一致 --> <el-dialog v-model="amazonExampleVisible" title="示例 - ASIN文档格式" width="480px">
<div class="progress-section" v-if="loading"> <div>
<div class="progress-box"> <div style="margin:8px 0;color:#606266;font-size:13px;">Excel 示例</div>
<div class="progress-container"> <el-table :data="[{asin:'B0XXXXXXX1'},{asin:'B0XXXXXXX2'}]" size="small" border>
<div class="progress-bar"> <el-table-column prop="asin" label="ASIN" />
<div class="progress-fill" :style="{ width: progressPercentage + '%' }"></div> </el-table>
</div>
<div class="progress-text">{{ progressPercentage }}%</div>
</div>
</div> </div>
</div> <template #footer>
<el-button type="primary" class="btn-blue" @click="amazonExampleVisible = false">我知道了</el-button>
</template>
</el-dialog>
<div class="table-wrapper"> <div class="table-wrapper">
<table class="table"> <table class="table">
<thead> <thead>
@@ -359,7 +393,7 @@ onMounted(async () => {
</thead> </thead>
<tbody> <tbody>
<tr v-for="row in paginatedData" :key="row.asin"> <tr v-for="row in paginatedData" :key="row.asin">
<td>{{ row.asin }}</td> <td><span :class="{ 'asin-out': isOutOfStock(row) }">{{ row.asin }}</span></td>
<td> <td>
<div class="seller-info"> <div class="seller-info">
<span class="seller">{{ row.seller || '无货' }}</span> <span class="seller">{{ row.seller || '无货' }}</span>
@@ -419,6 +453,7 @@ onMounted(async () => {
.step-header { display: flex; align-items: center; gap: 8px; margin-bottom: 6px; } .step-header { display: flex; align-items: center; gap: 8px; margin-bottom: 6px; }
.title { font-size: 13px; font-weight: 600; color: #303133; text-align: left; } .title { font-size: 13px; font-weight: 600; color: #303133; text-align: left; }
.desc { font-size: 12px; color: #909399; margin-bottom: 8px; text-align: left; } .desc { font-size: 12px; color: #909399; margin-bottom: 8px; text-align: left; }
.mini-hint { font-size: 12px; color: #909399; margin-top: 6px; }
.links { display: flex; align-items: center; gap: 6px; margin-bottom: 8px; } .links { display: flex; align-items: center; gap: 6px; margin-bottom: 8px; }
.link { color: #409EFF; cursor: pointer; font-size: 12px; } .link { color: #409EFF; cursor: pointer; font-size: 12px; }
.sep { color: #dcdfe6; } .sep { color: #dcdfe6; }
@@ -452,7 +487,7 @@ onMounted(async () => {
.text:focus { border-color: #409EFF; } .text:focus { border-color: #409EFF; }
.text:disabled { background: #f5f7fa; color: #c0c4cc; } .text:disabled { background: #f5f7fa; color: #c0c4cc; }
.action-buttons { display: flex; gap: 10px; flex-wrap: wrap; } .action-buttons { display: flex; gap: 10px; flex-wrap: wrap; }
.progress-section { margin: 12px 12px 6px 12px; } .progress-section { margin: 0px 12px 0px 12px; }
.progress-box { padding: 4px 0; } .progress-box { padding: 4px 0; }
.progress-container { display: flex; align-items: center; gap: 8px; } .progress-container { display: flex; align-items: center; gap: 8px; }
.progress-bar { flex: 1; height: 6px; background: #e3eeff; border-radius: 999px; overflow: hidden; } .progress-bar { flex: 1; height: 6px; background: #e3eeff; border-radius: 999px; overflow: hidden; }
@@ -474,6 +509,7 @@ onMounted(async () => {
.table th:nth-child(1), .table td:nth-child(1) { width: 33.33%; } .table th:nth-child(1), .table td:nth-child(1) { width: 33.33%; }
.table th:nth-child(2), .table td:nth-child(2) { width: 33.33%; } .table th:nth-child(2), .table td:nth-child(2) { width: 33.33%; }
.table th:nth-child(3), .table td:nth-child(3) { width: 33.33%; } .table th:nth-child(3), .table td:nth-child(3) { width: 33.33%; }
.asin-out { color: #f56c6c; font-weight: 600; }
.seller-info { display: flex; align-items: center; gap: 4px; } .seller-info { display: flex; align-items: center; gap: 4px; }
.seller { color: #303133; font-weight: 500; } .seller { color: #303133; font-weight: 500; }
.shipper { color: #909399; font-size: 12px; } .shipper { color: #909399; font-size: 12px; }

View File

@@ -1,6 +1,7 @@
<script setup lang="ts"> <script setup lang="ts">
import { ref, onMounted, computed } from 'vue' import { ref, onMounted, computed } from 'vue'
import { zebraApi, type BanmaAccount } from '../../api/zebra' import { zebraApi, type BanmaAccount } from '../../api/zebra'
import { ElMessageBox, ElMessage } from 'element-plus'
type PlatformKey = 'zebra' | 'shopee' | 'rakuten' | 'amazon' type PlatformKey = 'zebra' | 'shopee' | 'rakuten' | 'amazon'
@@ -36,10 +37,12 @@ function formatDate(a: any) {
async function onDelete(a: any) { async function onDelete(a: any) {
const id = a?.id const id = a?.id
const ok = confirm(`确定删除账号 “${a?.name || a?.username || id}” 吗?`) try {
if (!ok) return await ElMessageBox.confirm(`确定删除账号 “${a?.name || a?.username || id}” 吗?`, '提示', { type: 'warning' })
await zebraApi.removeAccount(id) } catch { return }
await load() await zebraApi.removeAccount(id)
ElMessage({ message: '删除成功', type: 'success' })
await load()
} }
</script> </script>
@@ -117,13 +120,12 @@ export default defineComponent({ name: 'AccountManager' })
.dot { width:6px; height:6px; border-radius:50%; justify-self: center; } .dot { width:6px; height:6px; border-radius:50%; justify-self: center; }
.dot.on { background:#52c41a; } .dot.on { background:#52c41a; }
.dot.off { background:#ff4d4f; } .dot.off { background:#ff4d4f; }
.user-info { display: flex; align-items: center; gap: 8px; } .user-info { display: flex; align-items: center; gap: 8px; min-width: 0; }
.avatar { width:22px; height:22px; border-radius:50%; object-fit: cover; } .avatar { width:22px; height:22px; border-radius:50%; object-fit: cover; }
.name { font-weight:500; font-size: 13px; color:#303133; } .name { font-weight:500; font-size: 13px; color:#303133; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
.date { color:#999; font-size:11px; text-align: center; } .date { color:#999; font-size:11px; text-align: center; }
.footer { display:flex; justify-content:center; padding-top: 10px; } .footer { display:flex; justify-content:center; padding-top: 10px; }
.btn { width: 180px; height: 32px; font-size: 13px; } .btn { width: 180px; height: 32px; font-size: 13px; }
.el-button--danger.is-link { font-size: 11px; padding: 0; height: auto; }
</style> </style>

View File

@@ -1,4 +1,5 @@
package com.tashow.erp.controller; package com.tashow.erp.controller;
import com.tashow.erp.entity.AmazonProductEntity;
import com.tashow.erp.repository.AmazonProductRepository; import com.tashow.erp.repository.AmazonProductRepository;
import com.tashow.erp.service.IAmazonScrapingService; import com.tashow.erp.service.IAmazonScrapingService;
import com.tashow.erp.utils.ExcelParseUtil; import com.tashow.erp.utils.ExcelParseUtil;
@@ -11,7 +12,6 @@ import org.springframework.web.multipart.MultipartFile;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.stream.Collectors;
@RestController @RestController
@RequestMapping("/api/amazon") @RequestMapping("/api/amazon")
public class AmazonController { public class AmazonController {
@@ -29,7 +29,11 @@ public class AmazonController {
Map<String, Object> requestMap = (Map<String, Object>) request; Map<String, Object> requestMap = (Map<String, Object>) request;
List<String> asinList = (List<String>) requestMap.get("asinList"); List<String> asinList = (List<String>) requestMap.get("asinList");
String batchId = (String) requestMap.get("batchId"); String batchId = (String) requestMap.get("batchId");
return JsonData.buildSuccess(amazonScrapingService.batchGetProductInfo(asinList, batchId)); List<AmazonProductEntity> products = amazonScrapingService.batchGetProductInfo(asinList, batchId);
Map<String, Object> result = new HashMap<>();
result.put("products", products);
result.put("total", products.size());
return JsonData.buildSuccess(result);
} }
/** /**
@@ -37,25 +41,7 @@ public class AmazonController {
*/ */
@GetMapping("/products/latest") @GetMapping("/products/latest")
public JsonData getLatestProducts() { public JsonData getLatestProducts() {
List<Map<String, Object>> products = amazonProductRepository.findLatestProducts() List<AmazonProductEntity> products = amazonProductRepository.findLatestProducts();
.parallelStream()
.map(entity -> {
Map<String, Object> map = new HashMap<>();
map.put("asin", entity.getAsin());
map.put("title", entity.getTitle());
map.put("price", entity.getPrice());
map.put("imageUrl", entity.getImageUrl());
map.put("productUrl", entity.getProductUrl());
map.put("brand", entity.getBrand());
map.put("category", entity.getCategory());
map.put("rating", entity.getRating());
map.put("reviewCount", entity.getReviewCount());
map.put("availability", entity.getAvailability());
map.put("seller", entity.getSeller());
map.put("shipper", entity.getSeller());
return map;
})
.collect(Collectors.toList());
Map<String, Object> result = new HashMap<>(); Map<String, Object> result = new HashMap<>();
result.put("products", products); result.put("products", products);
result.put("total", products.size()); result.put("total", products.size());

View File

@@ -25,9 +25,6 @@ public class AuthController {
public ResponseEntity<?> login(@RequestBody Map<String, Object> loginData) { public ResponseEntity<?> login(@RequestBody Map<String, Object> loginData) {
String username = (String) loginData.get("username"); String username = (String) loginData.get("username");
String password = (String) loginData.get("password"); String password = (String) loginData.get("password");
if (username == null || password == null) {
return ResponseEntity.ok(Map.of("code", 400, "message", "用户名和密码不能为空"));
}
Map<String, Object> result = authService.login(username, password); Map<String, Object> result = authService.login(username, password);
Object success = result.get("success"); Object success = result.get("success");
Object tokenObj = result.get("token"); Object tokenObj = result.get("token");

View File

@@ -23,10 +23,9 @@ public class BanmaOrderController {
BanmaOrderRepository banmaOrderRepository; BanmaOrderRepository banmaOrderRepository;
@Autowired @Autowired
JavaBridge javaBridge; JavaBridge javaBridge;
@Autowired
RestTemplate restTemplate;
@GetMapping("/orders") @GetMapping("/orders")
public ResponseEntity<Map<String, Object>> getOrders( public ResponseEntity<Map<String, Object>> getOrders(
@RequestParam(required = false, name = "accountId") Long accountId,
@RequestParam(required = false, name = "startDate") String startDate, @RequestParam(required = false, name = "startDate") String startDate,
@RequestParam(required = false, name = "endDate") String endDate, @RequestParam(required = false, name = "endDate") String endDate,
@RequestParam(defaultValue = "1", name = "page") int page, @RequestParam(defaultValue = "1", name = "page") int page,
@@ -34,16 +33,16 @@ public class BanmaOrderController {
@RequestParam(required = false, name = "batchId") String batchId, @RequestParam(required = false, name = "batchId") String batchId,
@RequestParam(required = false, name = "shopIds") String shopIds) { @RequestParam(required = false, name = "shopIds") String shopIds) {
List<String> shopIdList = shopIds != null ? java.util.Arrays.asList(shopIds.split(",")) : null; List<String> shopIdList = shopIds != null ? java.util.Arrays.asList(shopIds.split(",")) : null;
Map<String, Object> result = banmaOrderService.getOrdersByPage(startDate, endDate, page, pageSize, batchId, shopIdList); Map<String, Object> result = banmaOrderService.getOrdersByPage(accountId, startDate, endDate, page, pageSize, batchId, shopIdList);
return ResponseEntity.ok(result); return ResponseEntity.ok(result);
} }
/** /**
* 获取店铺列表 * 获取店铺列表
*/ */
@GetMapping("/shops") @GetMapping("/shops")
public JsonData getShops() { public JsonData getShops(@RequestParam(required = false, name = "accountId") Long accountId) {
try { try {
Map<String, Object> response = banmaOrderService.getShops(); Map<String, Object> response = banmaOrderService.getShops(accountId);
return JsonData.buildSuccess(response); return JsonData.buildSuccess(response);
} catch (Exception e) { } catch (Exception e) {
logger.error("获取店铺列表失败: {}", e.getMessage(), e); logger.error("获取店铺列表失败: {}", e.getMessage(), e);
@@ -51,19 +50,6 @@ public class BanmaOrderController {
} }
} }
/**
* 刷新斑马认证Token
*/
@PostMapping("/refresh-token")
public JsonData refreshToken(){
try {
banmaOrderService.refreshToken();
return JsonData.buildSuccess("Token刷新成功");
} catch (Exception e) {
logger.error("刷新Token失败: {}", e.getMessage(), e);
return JsonData.buildError("Token刷新失败: " + e.getMessage());
}
}
/** /**
* 获取最新订单数据 * 获取最新订单数据
*/ */

View File

@@ -21,33 +21,9 @@ public class AmazonProductEntity {
@Column(unique = true, nullable = false) @Column(unique = true, nullable = false)
private String asin; private String asin;
@Column(name = "title", length = 1000)
private String title;
@Column(name = "price") @Column(name = "price")
private String price; private String price;
@Column(name = "image_url", length = 1000)
private String imageUrl;
@Column(name = "product_url", length = 1000)
private String productUrl;
@Column(name = "brand")
private String brand;
@Column(name = "category")
private String category;
@Column(name = "rating")
private String rating;
@Column(name = "review_count")
private String reviewCount;
@Column(name = "availability")
private String availability;
@Column(name = "seller") @Column(name = "seller")
private String seller; private String seller;

View File

@@ -152,6 +152,12 @@ public class Alibaba1688ServiceImpl implements Alibaba1688Service {
} }
System.out.println("url"+uploadedUrl); System.out.println("url"+uploadedUrl);
System.out.println("skuPrices:"+skuPrices); System.out.println("skuPrices:"+skuPrices);
// 检查并上报空数据
if (skuPrices.isEmpty()) errorReporter.reportDataEmpty("alibaba1688", uploadedUrl, skuPrices);
if (median == null || median == 0.0) errorReporter.reportDataEmpty("alibaba1688", uploadedUrl, median);
if (freightFee.isEmpty()) errorReporter.reportDataEmpty("alibaba1688", uploadedUrl, freightFee);
result.setSkuPrice(skuPrices); result.setSkuPrice(skuPrices);
result.setMedian( median); result.setMedian( median);
result.setMapRecognitionLink( uploadImageBase64(imageUrl)); result.setMapRecognitionLink( uploadImageBase64(imageUrl));

View File

@@ -1,8 +1,10 @@
package com.tashow.erp.service.impl; package com.tashow.erp.service.impl;
import com.tashow.erp.entity.AmazonProductEntity; import com.tashow.erp.entity.AmazonProductEntity;
import com.tashow.erp.repository.AmazonProductRepository; import com.tashow.erp.repository.AmazonProductRepository;
import com.tashow.erp.service.IAmazonScrapingService; import com.tashow.erp.service.IAmazonScrapingService;
import com.tashow.erp.utils.DataReportUtil; import com.tashow.erp.utils.DataReportUtil;
import com.tashow.erp.utils.ErrorReporter;
import com.tashow.erp.utils.RakutenProxyUtil; import com.tashow.erp.utils.RakutenProxyUtil;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@@ -13,6 +15,7 @@ import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider; import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor; import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html; import us.codecraft.webmagic.selector.Html;
import java.time.LocalDateTime; import java.time.LocalDateTime;
import java.util.*; import java.util.*;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
@@ -29,50 +32,59 @@ public class AmazonScrapingServiceImpl implements IAmazonScrapingService, PagePr
private AmazonProductRepository amazonProductRepository; private AmazonProductRepository amazonProductRepository;
@Autowired @Autowired
private DataReportUtil dataReportUtil; private DataReportUtil dataReportUtil;
@Autowired
private ErrorReporter errorReporter;
private final Random random = new Random(); private final Random random = new Random();
private static volatile Spider activeSpider = null; private static volatile Spider activeSpider = null;
private static final Object spiderLock = new Object(); private static final Object spiderLock = new Object();
private final Map<String, Map<String, Object>> resultCache = new ConcurrentHashMap<>(); private final Map<String, AmazonProductEntity> resultCache = new ConcurrentHashMap<>();
private final Site site = Site.me().setRetryTimes(3).setSleepTime(2000 + random.nextInt(2000)) private final Site site = Site.me().setRetryTimes(3).setSleepTime(2000 + random.nextInt(2000)).setTimeOut(15000).setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/128.0.0.0 Safari/537.36").addHeader("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9").addHeader("accept-language", "ja,en;q=0.9,zh-CN;q=0.8,zh;q=0.7").addHeader("cache-control", "max-age=0").addHeader("upgrade-insecure-requests", "1").addHeader("sec-ch-ua", "\"Chromium\";v=\"128\", \"Not=A?Brand\";v=\"24\"").addHeader("sec-ch-ua-mobile", "?0").addHeader("sec-ch-ua-platform", "\"Windows\"").addHeader("sec-fetch-site", "none").addHeader("sec-fetch-mode", "navigate").addHeader("sec-fetch-user", "?1").addHeader("sec-fetch-dest", "document").addCookie("i18n-prefs", "JPY").addCookie("session-id", "358-1261309-0483141").addCookie("session-id-time", "2082787201l").addCookie("i18n-prefs", "JPY").addCookie("lc-acbjp", "zh_CN").addCookie("ubid-acbjp", "357-8224002-9668932");
.setTimeOut(15000).setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/128.0.0.0 Safari/537.36").addHeader("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9").addHeader("accept-language", "ja,en;q=0.9,zh-CN;q=0.8,zh;q=0.7").addHeader("cache-control", "max-age=0").addHeader("upgrade-insecure-requests", "1").addHeader("sec-ch-ua", "\"Chromium\";v=\"128\", \"Not=A?Brand\";v=\"24\"").addHeader("sec-ch-ua-mobile", "?0").addHeader("sec-ch-ua-platform", "\"Windows\"").addHeader("sec-fetch-site", "none").addHeader("sec-fetch-mode", "navigate").addHeader("sec-fetch-user", "?1").addHeader("sec-fetch-dest", "document").addCookie("i18n-prefs", "JPY").addCookie("session-id", "358-1261309-0483141").addCookie("session-id-time", "2082787201l").addCookie("i18n-prefs", "JPY").addCookie("lc-acbjp", "zh_CN").addCookie("ubid-acbjp", "357-8224002-9668932");
/** /**
* 处理亚马逊页面数据提取 * 处理亚马逊页面数据提取
*/ */
@Override @Override
public void process(Page page) { public void process(Page page) {
Html html = page.getHtml(); Html html = page.getHtml();
Map<String, Object> resultMap = new HashMap<>(); String url = page.getUrl().toString();
// 提取ASIN
String asin = html.xpath("//input[@id='ASIN']/@value").toString();
if (isEmpty(asin)) {
String[] parts = url.split("/dp/");
if (parts.length > 1) asin = parts[1].split("/")[0].split("\\?")[0];
}
// 提取价格
String priceSymbol = html.xpath("//span[@class='a-price-symbol']/text()").toString(); String priceSymbol = html.xpath("//span[@class='a-price-symbol']/text()").toString();
String priceWhole = html.xpath("//span[@class='a-price-whole']/text()").toString(); String priceWhole = html.xpath("//span[@class='a-price-whole']/text()").toString();
String price = priceSymbol + priceWhole; String price = priceSymbol + priceWhole;
if (price.isEmpty()) { if (isEmpty(price)) {
price = html.xpath("//span[@class='a-price-range']/text()").toString(); price = html.xpath("//span[@class='a-price-range']/text()").toString();
} }
// 提取卖家
String seller = html.xpath("//a[@id='sellerProfileTriggerId']/text()").toString(); String seller = html.xpath("//a[@id='sellerProfileTriggerId']/text()").toString();
if (seller == null || seller.isEmpty()) { if (isEmpty(seller)) {
seller = html.xpath("//span[@class='a-size-small offer-display-feature-text-message']/text()").toString(); seller = html.xpath("//span[@class='a-size-small offer-display-feature-text-message']/text()").toString();
} }
resultMap.put("seller", seller);
if (price != null || seller != null) { // 关键数据为空时重试
resultMap.put("price", price); if (isEmpty(price) && isEmpty(seller)) {
} else {
throw new RuntimeException("Retry this page"); throw new RuntimeException("Retry this page");
} }
String asin = html.xpath("//input[@id='ASIN']/@value").toString(); // 检查并上报空数据
if (asin == null || asin.isEmpty()) { if (isEmpty(price)) errorReporter.reportDataEmpty("amazon", asin, price);
String[] parts = page.getUrl().toString().split("/dp/"); if (isEmpty(seller)) errorReporter.reportDataEmpty("amazon", asin, seller);
if (parts.length > 1) asin = parts[1].split("/")[0].split("\\?")[0];
}
String title = html.xpath("//span[@id='productTitle']/text()").toString();
if (title == null || title.isEmpty())
title = html.xpath("//h1[@class='a-size-large a-spacing-none']/text()").toString();
resultMap.put("asin", asin != null ? asin : "");
resultMap.put("title", (title == null || title.isEmpty()) ? "未获取" : title.trim());
resultCache.put(asin, resultMap); AmazonProductEntity entity = new AmazonProductEntity();
page.putField("resultMap", resultMap); entity.setAsin(asin != null ? asin : "");
entity.setPrice(price);
entity.setSeller(seller);
resultCache.put(asin, entity);
page.putField("entity", entity);
} }
/** /**
@@ -87,77 +99,45 @@ public class AmazonScrapingServiceImpl implements IAmazonScrapingService, PagePr
* 批量获取产品信息 * 批量获取产品信息
*/ */
@Override @Override
public Map<String, Object> batchGetProductInfo(List<String> asinList, String batchId) { public List<AmazonProductEntity> batchGetProductInfo(List<String> asinList, String batchId) {
String sessionId = (batchId != null) ? batchId : "SINGLE_" + UUID.randomUUID(); String sessionId = (batchId != null) ? batchId : "SINGLE_" + UUID.randomUUID();
List<Map<String, Object>> products = new ArrayList<>(); List<AmazonProductEntity> products = new ArrayList<>();
for (String asin : asinList) { for (String asin : asinList) {
if (asin == null || asin.trim().isEmpty()) continue; if (asin == null || asin.trim().isEmpty()) continue;
String cleanAsin = asin.replaceAll("[^a-zA-Z0-9]", ""); String cleanAsin = asin.replaceAll("[^a-zA-Z0-9]", "");
Map<String, Object> result = new HashMap<>(); AmazonProductEntity product = amazonProductRepository.findByAsin(cleanAsin).filter(entity -> entity.getCreatedAt().isAfter(LocalDateTime.now().minusHours(1)) && !isEmpty(entity.getPrice()) && !isEmpty(entity.getSeller())).orElseGet(() -> {
// 采集新数据
amazonProductRepository.findByAsin(cleanAsin).ifPresentOrElse(entity -> {
if (entity.getCreatedAt().isAfter(LocalDateTime.now().minusHours(1))) {
result.put("asin", entity.getAsin());
result.put("title", entity.getTitle());
result.put("price", entity.getPrice());
result.put("seller", entity.getSeller());
result.put("imageUrl", entity.getImageUrl());
result.put("productUrl", entity.getProductUrl());
result.put("brand", entity.getBrand());
result.put("category", entity.getCategory());
result.put("rating", entity.getRating());
result.put("reviewCount", entity.getReviewCount());
result.put("availability", entity.getAvailability());
products.add(result);
}
}, () -> {
// 数据库没有或过期 -> 爬取
String url = "https://www.amazon.co.jp/dp/" + cleanAsin; String url = "https://www.amazon.co.jp/dp/" + cleanAsin;
RakutenProxyUtil proxyUtil = new RakutenProxyUtil(); RakutenProxyUtil proxyUtil = new RakutenProxyUtil();
synchronized (spiderLock) { synchronized (spiderLock) {
activeSpider = Spider.create(this) activeSpider = Spider.create(this).addUrl(url).setDownloader(proxyUtil.createProxyDownloader(proxyUtil.detectSystemProxy(url))).thread(1);
.addUrl(url)
.setDownloader(proxyUtil.createProxyDownloader(proxyUtil.detectSystemProxy(url)))
.thread(1);
activeSpider.run(); activeSpider.run();
activeSpider = null; activeSpider = null;
} }
result.putAll(resultCache.getOrDefault(cleanAsin, Map.of("asin", cleanAsin, "price", "", "seller", "", "title", "")));
// 存库 AmazonProductEntity entity = resultCache.getOrDefault(cleanAsin, new AmazonProductEntity());
AmazonProductEntity entity = new AmazonProductEntity();
entity.setAsin(cleanAsin); entity.setAsin(cleanAsin);
entity.setTitle((String) result.get("title"));
entity.setPrice((String) result.get("price"));
entity.setSeller((String) result.get("seller"));
entity.setImageUrl((String) result.get("imageUrl"));
entity.setProductUrl((String) result.get("productUrl"));
entity.setBrand((String) result.get("brand"));
entity.setCategory((String) result.get("category"));
entity.setRating((String) result.get("rating"));
entity.setReviewCount((String) result.get("reviewCount"));
entity.setAvailability((String) result.get("availability"));
entity.setSessionId(sessionId); entity.setSessionId(sessionId);
entity.setCreatedAt(LocalDateTime.now());
try { try {
amazonProductRepository.save(entity); amazonProductRepository.save(entity);
dataReportUtil.reportDataCollection("AMAZON", 1, "0"); dataReportUtil.reportDataCollection("AMAZON", 1, "0");
} catch (Exception e) { } catch (Exception e) {
logger.warn("保存商品数据失败: {}", cleanAsin); logger.warn("保存商品数据失败: {}", cleanAsin);
} }
products.add(result); return entity;
}); });
products.add(product);
} }
long failedCount = products.stream().filter(p -> p.get("price").toString().isEmpty()).count(); return products;
return Map.of(
"products", products,
"total", products.size(),
"success", true,
"failedCount", failedCount
);
} }
private boolean isEmpty(String str) {
return str == null || str.trim().isEmpty();
}
} }

View File

@@ -50,7 +50,6 @@ public class ClientMonitorController extends BaseController {
startPage(); startPage();
return getDataTable(clientMonitorService.selectClientEventLogList(clientEventLog)); return getDataTable(clientMonitorService.selectClientEventLogList(clientEventLog));
} }
/** /**
* 获取客户端数据采集报告列表 * 获取客户端数据采集报告列表
*/ */
@@ -108,7 +107,6 @@ public class ClientMonitorController extends BaseController {
} }
/** /**
* 客户端错误上报API * 客户端错误上报API
*/ */
@@ -151,10 +149,6 @@ public class ClientMonitorController extends BaseController {
return AjaxResult.success(clientMonitorService.getVersionDistribution()); return AjaxResult.success(clientMonitorService.getVersionDistribution());
} }
/** /**
* 清理过期数据 * 清理过期数据
*/ */

View File

@@ -1,5 +1,4 @@
package com.ruoyi.web.controller.system; package com.ruoyi.web.controller.system;
import com.ruoyi.common.annotation.Anonymous; import com.ruoyi.common.annotation.Anonymous;
import com.ruoyi.common.core.domain.AjaxResult; import com.ruoyi.common.core.domain.AjaxResult;
import com.ruoyi.common.utils.ip.IpUtils; import com.ruoyi.common.utils.ip.IpUtils;
@@ -8,7 +7,6 @@ import com.ruoyi.system.mapper.ClientDeviceMapper;
import com.ruoyi.web.sse.SseHubService; import com.ruoyi.web.sse.SseHubService;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*; import org.springframework.web.bind.annotation.*;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@@ -42,7 +40,6 @@ public class ClientDeviceController {
map.put("used", used); map.put("used", used);
return AjaxResult.success(map); return AjaxResult.success(map);
} }
/** /**
* 按用户名查询设备列表(最近活动优先) * 按用户名查询设备列表(最近活动优先)
* @param username 用户名,必需参数 * @param username 用户名,必需参数
@@ -61,7 +58,7 @@ public class ClientDeviceController {
* 设备注册(幂等) * 设备注册(幂等)
* *
* 根据 deviceId 判断: * 根据 deviceId 判断:
* - 不存在:插入新记录(后端生成设备名称、IP等信息 * - 不存在:插入新记录(检查设备数量限制
* - 已存在:更新设备信息 * - 已存在:更新设备信息
*/ */
@PostMapping("/register") @PostMapping("/register")
@@ -70,6 +67,15 @@ public class ClientDeviceController {
String ip = IpUtils.getIpAddr(request); String ip = IpUtils.getIpAddr(request);
String deviceName = request.getHeader("X-Client-User") + "@" + ip + " (" + request.getHeader("X-Client-OS") + ")"; String deviceName = request.getHeader("X-Client-User") + "@" + ip + " (" + request.getHeader("X-Client-OS") + ")";
if (exists == null) { if (exists == null) {
// 检查设备数量限制
List<ClientDevice> userDevices = clientDeviceMapper.selectByUsername(device.getUsername());
int activeDeviceCount = 0;
for (ClientDevice d : userDevices) {
if (!"removed".equals(d.getStatus())) activeDeviceCount++;
}
if (activeDeviceCount >= DEFAULT_LIMIT) {
return AjaxResult.error("设备数量已达上限(" + DEFAULT_LIMIT + "个),请先移除其他设备");
}
device.setIp(ip); device.setIp(ip);
device.setStatus("online"); device.setStatus("online");
device.setLastActiveAt(new java.util.Date()); device.setLastActiveAt(new java.util.Date());
@@ -153,6 +159,15 @@ public class ClientDeviceController {
String ip = IpUtils.getIpAddr(request); String ip = IpUtils.getIpAddr(request);
String deviceName = request.getHeader("X-Client-User") + "@" + ip + " (" + request.getHeader("X-Client-OS") + ")"; String deviceName = request.getHeader("X-Client-User") + "@" + ip + " (" + request.getHeader("X-Client-OS") + ")";
if (exists == null) { if (exists == null) {
// 检查设备数量限制
List<ClientDevice> userDevices = clientDeviceMapper.selectByUsername(device.getUsername());
int activeDeviceCount = 0;
for (ClientDevice d : userDevices) {
if (!"removed".equals(d.getStatus())) activeDeviceCount++;
}
if (activeDeviceCount >= DEFAULT_LIMIT) {
return AjaxResult.error("设备数量已达上限(" + DEFAULT_LIMIT + "个),请先移除其他设备");
}
device.setIp(ip); device.setIp(ip);
device.setStatus("online"); device.setStatus("online");
device.setLastActiveAt(new java.util.Date()); device.setLastActiveAt(new java.util.Date());

View File

@@ -23,7 +23,7 @@ public class BanmaOrderController extends BaseController {
private IBanmaAccountService accountService; private IBanmaAccountService accountService;
/** /**
* 查询账号列表(仅返回必要字段) * 查询账号列表(
*/ */
@GetMapping("/accounts") @GetMapping("/accounts")
public R<?> listAccounts() { public R<?> listAccounts() {
@@ -37,7 +37,9 @@ public class BanmaOrderController extends BaseController {
@PostMapping("/accounts") @PostMapping("/accounts")
public R<?> saveAccount(@RequestBody BanmaAccount body) { public R<?> saveAccount(@RequestBody BanmaAccount body) {
Long id = accountService.saveOrUpdate(body); Long id = accountService.saveOrUpdate(body);
return R.ok(Map.of("id", id)); boolean ok = false;
try { ok = accountService.refreshToken(id); } catch (Exception ignore) {}
return ok ? R.ok(Map.of("id", id)) : R.fail("账号或密码错误无法获取Token");
} }
/** /**
@@ -49,4 +51,18 @@ public class BanmaOrderController extends BaseController {
return R.ok(); return R.ok();
} }
/** 手动刷新单个账号 Token */
@PostMapping("/accounts/{id}/refresh-token")
public R<?> refreshOne(@PathVariable Long id) {
accountService.refreshToken(id);
return R.ok();
}
/** 手动刷新全部启用账号 Token */
@PostMapping("/refresh-all")
public R<?> refreshAll() {
accountService.refreshAllTokens();
return R.ok();
}
} }

View File

@@ -714,8 +714,8 @@ public class ClientMonitorServiceImpl implements IClientMonitorService {
@Override @Override
public void cleanExpiredData() { public void cleanExpiredData() {
try { try {
// 清理过期的客户端(设置为离线状态) // // 清理过期的客户端(设置为离线状态)
clientMonitorMapper.updateExpiredClientsOffline(); // clientMonitorMapper.updateExpiredClientsOffline();
// 清理过期的设备(设置为离线状态) // 清理过期的设备(设置为离线状态)
clientMonitorMapper.updateExpiredDevicesOffline(); clientMonitorMapper.updateExpiredDevicesOffline();
@@ -725,8 +725,6 @@ public class ClientMonitorServiceImpl implements IClientMonitorService {
// 删除过期的事件日志 // 删除过期的事件日志
clientMonitorMapper.deleteExpiredEventLogs(); clientMonitorMapper.deleteExpiredEventLogs();
logger.info("过期数据清理完成");
} catch (Exception e) { } catch (Exception e) {
logger.error("清理过期数据失败: {}", e.getMessage(), e); logger.error("清理过期数据失败: {}", e.getMessage(), e);
throw new RuntimeException("清理过期数据失败", e); throw new RuntimeException("清理过期数据失败", e);

View File

@@ -6,8 +6,10 @@ import java.util.List;
public interface ClientDeviceMapper { public interface ClientDeviceMapper {
ClientDevice selectByDeviceId(String deviceId); ClientDevice selectByDeviceId(String deviceId);
List<ClientDevice> selectByUsername(String username); List<ClientDevice> selectByUsername(String username);
List<ClientDevice> selectOnlineDevices();
int insert(ClientDevice device); int insert(ClientDevice device);
int updateByDeviceId(ClientDevice device); int updateByDeviceId(ClientDevice device);
int updateExpiredDevicesOffline();
int deleteByDeviceId(String deviceId); int deleteByDeviceId(String deviceId);
int countByUsername(String username); int countByUsername(String username);
} }

View File

@@ -48,6 +48,15 @@
<select id="countByUsername" resultType="int"> <select id="countByUsername" resultType="int">
select count(1) from client_device where username = #{username} select count(1) from client_device where username = #{username}
</select> </select>
<select id="selectOnlineDevices" resultMap="ClientDeviceMap">
select * from client_device where status = 'online' order by last_active_at desc
</select>
<update id="updateExpiredDevicesOffline">
update client_device set status = 'offline'
where status = 'online' and (last_active_at is null or last_active_at &lt; date_sub(now(), interval 2 minute))
</update>
</mapper> </mapper>