1
This commit is contained in:
@@ -7,6 +7,7 @@ import { amazonApi } from '../../api/amazon'
|
||||
const loading = ref(false) // 主加载状态
|
||||
const tableLoading = ref(false) // 表格加载状态
|
||||
const progressPercentage = ref(0) // 进度百分比
|
||||
const progressVisible = ref(false) // 进度条是否显示(完成后仍保留)
|
||||
const localProductData = ref<any[]>([]) // 本地产品数据
|
||||
const currentAsin = ref('') // 当前处理的ASIN
|
||||
const genmaiLoading = ref(false) // Genmai Spirit加载状态
|
||||
@@ -43,7 +44,7 @@ const regionOptions = [
|
||||
]
|
||||
const pendingAsins = ref<string[]>([])
|
||||
|
||||
// 通用消息提示
|
||||
// 通用消息提示(Element Plus)
|
||||
function showMessage(message: string, type: 'success' | 'warning' | 'error' | 'info' = 'info') {
|
||||
ElMessage({ message, type })
|
||||
}
|
||||
@@ -55,6 +56,7 @@ async function processExcelFile(file: File) {
|
||||
tableLoading.value = true
|
||||
localProductData.value = []
|
||||
progressPercentage.value = 0
|
||||
progressVisible.value = false
|
||||
|
||||
const response = await amazonApi.importAsinFromExcel(file)
|
||||
const asinList = response.data.asinList
|
||||
@@ -63,10 +65,7 @@ async function processExcelFile(file: File) {
|
||||
showMessage('文件中未找到有效的ASIN数据', 'warning')
|
||||
return
|
||||
}
|
||||
|
||||
// 存入待采集队列,等待用户点击“获取数据”再开始
|
||||
pendingAsins.value = asinList
|
||||
showMessage(`成功解析 ${asinList.length} 个ASIN,点击“获取数据”开始采集`, 'success')
|
||||
} catch (error: any) {
|
||||
showMessage(error.message || '处理文件失败', 'error')
|
||||
} finally {
|
||||
@@ -90,8 +89,8 @@ async function onDrop(e: DragEvent) {
|
||||
dragActive.value = false
|
||||
const file = e.dataTransfer?.files?.[0]
|
||||
if (!file) return
|
||||
const ok = /(\.csv|\.txt|\.xls|\.xlsx)$/i.test(file.name)
|
||||
if (!ok) return showMessage('仅支持 .csv/.txt/.xls/.xlsx 文件', 'warning')
|
||||
const ok = /\.xlsx?$/i.test(file.name)
|
||||
if (!ok) return showMessage('仅支持 .xls/.xlsx 文件', 'warning')
|
||||
await processExcelFile(file)
|
||||
}
|
||||
|
||||
@@ -170,6 +169,7 @@ async function startQueuedFetch() {
|
||||
return
|
||||
}
|
||||
loading.value = true
|
||||
progressVisible.value = true
|
||||
tableLoading.value = true
|
||||
try {
|
||||
await batchGetProductInfo(pendingAsins.value)
|
||||
@@ -218,6 +218,13 @@ function getSellerShipperText(product: any) {
|
||||
return text
|
||||
}
|
||||
|
||||
// 判定无货(用于标红 ASIN)
|
||||
function isOutOfStock(product: any) {
|
||||
const sellerEmpty = !product?.seller || product.seller === '无货'
|
||||
const priceEmpty = !product?.price || product.price === '无货'
|
||||
return sellerEmpty || priceEmpty
|
||||
}
|
||||
|
||||
// 停止获取操作
|
||||
function stopFetch() {
|
||||
loading.value = false
|
||||
@@ -248,10 +255,26 @@ function handleCurrentChange(page: number) {
|
||||
}
|
||||
|
||||
// 使用 Element Plus 的 jumper,不再需要手动跳转函数
|
||||
|
||||
// 示例弹窗
|
||||
const amazonExampleVisible = ref(false)
|
||||
function openAmazonUpload() {
|
||||
amazonUpload.value?.click()
|
||||
}
|
||||
|
||||
function viewAmazonExample() { amazonExampleVisible.value = true }
|
||||
|
||||
function downloadAmazonTemplate() {
|
||||
const html = '<table><tr><th>ASIN</th></tr><tr><td>B0XXXXXXX1</td></tr><tr><td>B0XXXXXXX2</td></tr></table>'
|
||||
const blob = new Blob([html], { type: 'application/vnd.ms-excel' })
|
||||
const url = URL.createObjectURL(blob)
|
||||
const a = document.createElement('a')
|
||||
a.href = url
|
||||
a.download = 'amazon_asin_template.xls'
|
||||
document.body.appendChild(a)
|
||||
a.click()
|
||||
document.body.removeChild(a)
|
||||
URL.revokeObjectURL(url)
|
||||
}
|
||||
// 组件挂载时获取最新数据
|
||||
onMounted(async () => {
|
||||
try {
|
||||
@@ -276,18 +299,18 @@ onMounted(async () => {
|
||||
<div class="step-index">1</div>
|
||||
<div class="step-card">
|
||||
<div class="step-header"><div class="title">导入ASIN</div></div>
|
||||
<div class="desc">仅支持包含 ASIN 列的 CSV/Excel 文档</div>
|
||||
<div class="desc">仅支持包含 ASIN 列的 Excel 文档</div>
|
||||
<div class="links">
|
||||
<a class="link" @click.prevent>点击查看示例</a>
|
||||
<a class="link" @click.prevent="viewAmazonExample">点击查看示例</a>
|
||||
<span class="sep">|</span>
|
||||
<a class="link" @click.prevent>点击下载模板</a>
|
||||
<a class="link" @click.prevent="downloadAmazonTemplate">点击下载模板</a>
|
||||
</div>
|
||||
<div class="dropzone" @dragover.prevent="onDragOver" @dragleave="onDragLeave" @drop="onDrop" @click="openAmazonUpload">
|
||||
<div class="dz-el-icon">📤</div>
|
||||
<div class="dz-text">点击或将文件拖拽到这里上传</div>
|
||||
<div class="dz-sub">支持 .csv .txt .xls .xlsx</div>
|
||||
<div class="dz-sub">支持 .xls .xlsx</div>
|
||||
</div>
|
||||
<input ref="amazonUpload" style="display:none" type="file" accept=".csv,.txt,.xls,.xlsx" @change="handleExcelUpload" :disabled="loading" />
|
||||
<input ref="amazonUpload" style="display:none" type="file" accept=".xls,.xlsx" @change="handleExcelUpload" :disabled="loading" />
|
||||
</div>
|
||||
</div>
|
||||
<!-- 2 网站地区 -->
|
||||
@@ -311,6 +334,16 @@ onMounted(async () => {
|
||||
<div class="desc">导入表格后,点击下方按钮开始获取ASIN数据</div>
|
||||
<el-button size="small" class="w100 btn-blue" :disabled="!pendingAsins.length || loading" @click="startQueuedFetch">{{ loading ? '处理中...' : '获取数据' }}</el-button>
|
||||
<div class="mini-hint" v-if="pendingAsins.length">已导入 {{ pendingAsins.length }} 个 ASIN</div>
|
||||
<div class="progress-section" v-if="progressVisible">
|
||||
<div class="progress-box">
|
||||
<div class="progress-container">
|
||||
<div class="progress-bar">
|
||||
<div class="progress-fill" :style="{ width: progressPercentage + '%' }"></div>
|
||||
</div>
|
||||
<div class="progress-text">{{ progressPercentage }}%</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<!-- 左侧不再显示进度条 -->
|
||||
</div>
|
||||
</div>
|
||||
@@ -337,17 +370,18 @@ onMounted(async () => {
|
||||
<!-- 数据显示区域 -->
|
||||
<div class="table-container">
|
||||
<div class="table-section">
|
||||
<!-- 表格上方进度条(与乐天一致) -->
|
||||
<div class="progress-section" v-if="loading">
|
||||
<div class="progress-box">
|
||||
<div class="progress-container">
|
||||
<div class="progress-bar">
|
||||
<div class="progress-fill" :style="{ width: progressPercentage + '%' }"></div>
|
||||
</div>
|
||||
<div class="progress-text">{{ progressPercentage }}%</div>
|
||||
</div>
|
||||
<el-dialog v-model="amazonExampleVisible" title="示例 - ASIN文档格式" width="480px">
|
||||
<div>
|
||||
<div style="margin:8px 0;color:#606266;font-size:13px;">Excel 示例:</div>
|
||||
<el-table :data="[{asin:'B0XXXXXXX1'},{asin:'B0XXXXXXX2'}]" size="small" border>
|
||||
<el-table-column prop="asin" label="ASIN" />
|
||||
</el-table>
|
||||
</div>
|
||||
</div>
|
||||
<template #footer>
|
||||
<el-button type="primary" class="btn-blue" @click="amazonExampleVisible = false">我知道了</el-button>
|
||||
</template>
|
||||
</el-dialog>
|
||||
|
||||
<div class="table-wrapper">
|
||||
<table class="table">
|
||||
<thead>
|
||||
@@ -359,7 +393,7 @@ onMounted(async () => {
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr v-for="row in paginatedData" :key="row.asin">
|
||||
<td>{{ row.asin }}</td>
|
||||
<td><span :class="{ 'asin-out': isOutOfStock(row) }">{{ row.asin }}</span></td>
|
||||
<td>
|
||||
<div class="seller-info">
|
||||
<span class="seller">{{ row.seller || '无货' }}</span>
|
||||
@@ -419,6 +453,7 @@ onMounted(async () => {
|
||||
.step-header { display: flex; align-items: center; gap: 8px; margin-bottom: 6px; }
|
||||
.title { font-size: 13px; font-weight: 600; color: #303133; text-align: left; }
|
||||
.desc { font-size: 12px; color: #909399; margin-bottom: 8px; text-align: left; }
|
||||
.mini-hint { font-size: 12px; color: #909399; margin-top: 6px; }
|
||||
.links { display: flex; align-items: center; gap: 6px; margin-bottom: 8px; }
|
||||
.link { color: #409EFF; cursor: pointer; font-size: 12px; }
|
||||
.sep { color: #dcdfe6; }
|
||||
@@ -452,7 +487,7 @@ onMounted(async () => {
|
||||
.text:focus { border-color: #409EFF; }
|
||||
.text:disabled { background: #f5f7fa; color: #c0c4cc; }
|
||||
.action-buttons { display: flex; gap: 10px; flex-wrap: wrap; }
|
||||
.progress-section { margin: 12px 12px 6px 12px; }
|
||||
.progress-section { margin: 0px 12px 0px 12px; }
|
||||
.progress-box { padding: 4px 0; }
|
||||
.progress-container { display: flex; align-items: center; gap: 8px; }
|
||||
.progress-bar { flex: 1; height: 6px; background: #e3eeff; border-radius: 999px; overflow: hidden; }
|
||||
@@ -474,6 +509,7 @@ onMounted(async () => {
|
||||
.table th:nth-child(1), .table td:nth-child(1) { width: 33.33%; }
|
||||
.table th:nth-child(2), .table td:nth-child(2) { width: 33.33%; }
|
||||
.table th:nth-child(3), .table td:nth-child(3) { width: 33.33%; }
|
||||
.asin-out { color: #f56c6c; font-weight: 600; }
|
||||
.seller-info { display: flex; align-items: center; gap: 4px; }
|
||||
.seller { color: #303133; font-weight: 500; }
|
||||
.shipper { color: #909399; font-size: 12px; }
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
<script setup lang="ts">
|
||||
import { ref, onMounted, computed } from 'vue'
|
||||
import { zebraApi, type BanmaAccount } from '../../api/zebra'
|
||||
import { ElMessageBox, ElMessage } from 'element-plus'
|
||||
|
||||
type PlatformKey = 'zebra' | 'shopee' | 'rakuten' | 'amazon'
|
||||
|
||||
@@ -36,10 +37,12 @@ function formatDate(a: any) {
|
||||
|
||||
async function onDelete(a: any) {
|
||||
const id = a?.id
|
||||
const ok = confirm(`确定删除账号 “${a?.name || a?.username || id}” 吗?`)
|
||||
if (!ok) return
|
||||
await zebraApi.removeAccount(id)
|
||||
await load()
|
||||
try {
|
||||
await ElMessageBox.confirm(`确定删除账号 “${a?.name || a?.username || id}” 吗?`, '提示', { type: 'warning' })
|
||||
} catch { return }
|
||||
await zebraApi.removeAccount(id)
|
||||
ElMessage({ message: '删除成功', type: 'success' })
|
||||
await load()
|
||||
}
|
||||
</script>
|
||||
|
||||
@@ -117,13 +120,12 @@ export default defineComponent({ name: 'AccountManager' })
|
||||
.dot { width:6px; height:6px; border-radius:50%; justify-self: center; }
|
||||
.dot.on { background:#52c41a; }
|
||||
.dot.off { background:#ff4d4f; }
|
||||
.user-info { display: flex; align-items: center; gap: 8px; }
|
||||
.user-info { display: flex; align-items: center; gap: 8px; min-width: 0; }
|
||||
.avatar { width:22px; height:22px; border-radius:50%; object-fit: cover; }
|
||||
.name { font-weight:500; font-size: 13px; color:#303133; }
|
||||
.name { font-weight:500; font-size: 13px; color:#303133; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.date { color:#999; font-size:11px; text-align: center; }
|
||||
.footer { display:flex; justify-content:center; padding-top: 10px; }
|
||||
.btn { width: 180px; height: 32px; font-size: 13px; }
|
||||
.el-button--danger.is-link { font-size: 11px; padding: 0; height: auto; }
|
||||
</style>
|
||||
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
package com.tashow.erp.controller;
|
||||
import com.tashow.erp.entity.AmazonProductEntity;
|
||||
import com.tashow.erp.repository.AmazonProductRepository;
|
||||
import com.tashow.erp.service.IAmazonScrapingService;
|
||||
import com.tashow.erp.utils.ExcelParseUtil;
|
||||
@@ -11,7 +12,6 @@ import org.springframework.web.multipart.MultipartFile;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
@RestController
|
||||
@RequestMapping("/api/amazon")
|
||||
public class AmazonController {
|
||||
@@ -29,7 +29,11 @@ public class AmazonController {
|
||||
Map<String, Object> requestMap = (Map<String, Object>) request;
|
||||
List<String> asinList = (List<String>) requestMap.get("asinList");
|
||||
String batchId = (String) requestMap.get("batchId");
|
||||
return JsonData.buildSuccess(amazonScrapingService.batchGetProductInfo(asinList, batchId));
|
||||
List<AmazonProductEntity> products = amazonScrapingService.batchGetProductInfo(asinList, batchId);
|
||||
Map<String, Object> result = new HashMap<>();
|
||||
result.put("products", products);
|
||||
result.put("total", products.size());
|
||||
return JsonData.buildSuccess(result);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -37,25 +41,7 @@ public class AmazonController {
|
||||
*/
|
||||
@GetMapping("/products/latest")
|
||||
public JsonData getLatestProducts() {
|
||||
List<Map<String, Object>> products = amazonProductRepository.findLatestProducts()
|
||||
.parallelStream()
|
||||
.map(entity -> {
|
||||
Map<String, Object> map = new HashMap<>();
|
||||
map.put("asin", entity.getAsin());
|
||||
map.put("title", entity.getTitle());
|
||||
map.put("price", entity.getPrice());
|
||||
map.put("imageUrl", entity.getImageUrl());
|
||||
map.put("productUrl", entity.getProductUrl());
|
||||
map.put("brand", entity.getBrand());
|
||||
map.put("category", entity.getCategory());
|
||||
map.put("rating", entity.getRating());
|
||||
map.put("reviewCount", entity.getReviewCount());
|
||||
map.put("availability", entity.getAvailability());
|
||||
map.put("seller", entity.getSeller());
|
||||
map.put("shipper", entity.getSeller());
|
||||
return map;
|
||||
})
|
||||
.collect(Collectors.toList());
|
||||
List<AmazonProductEntity> products = amazonProductRepository.findLatestProducts();
|
||||
Map<String, Object> result = new HashMap<>();
|
||||
result.put("products", products);
|
||||
result.put("total", products.size());
|
||||
|
||||
@@ -25,9 +25,6 @@ public class AuthController {
|
||||
public ResponseEntity<?> login(@RequestBody Map<String, Object> loginData) {
|
||||
String username = (String) loginData.get("username");
|
||||
String password = (String) loginData.get("password");
|
||||
if (username == null || password == null) {
|
||||
return ResponseEntity.ok(Map.of("code", 400, "message", "用户名和密码不能为空"));
|
||||
}
|
||||
Map<String, Object> result = authService.login(username, password);
|
||||
Object success = result.get("success");
|
||||
Object tokenObj = result.get("token");
|
||||
|
||||
@@ -23,10 +23,9 @@ public class BanmaOrderController {
|
||||
BanmaOrderRepository banmaOrderRepository;
|
||||
@Autowired
|
||||
JavaBridge javaBridge;
|
||||
@Autowired
|
||||
RestTemplate restTemplate;
|
||||
@GetMapping("/orders")
|
||||
public ResponseEntity<Map<String, Object>> getOrders(
|
||||
@RequestParam(required = false, name = "accountId") Long accountId,
|
||||
@RequestParam(required = false, name = "startDate") String startDate,
|
||||
@RequestParam(required = false, name = "endDate") String endDate,
|
||||
@RequestParam(defaultValue = "1", name = "page") int page,
|
||||
@@ -34,16 +33,16 @@ public class BanmaOrderController {
|
||||
@RequestParam(required = false, name = "batchId") String batchId,
|
||||
@RequestParam(required = false, name = "shopIds") String shopIds) {
|
||||
List<String> shopIdList = shopIds != null ? java.util.Arrays.asList(shopIds.split(",")) : null;
|
||||
Map<String, Object> result = banmaOrderService.getOrdersByPage(startDate, endDate, page, pageSize, batchId, shopIdList);
|
||||
Map<String, Object> result = banmaOrderService.getOrdersByPage(accountId, startDate, endDate, page, pageSize, batchId, shopIdList);
|
||||
return ResponseEntity.ok(result);
|
||||
}
|
||||
/**
|
||||
* 获取店铺列表
|
||||
*/
|
||||
@GetMapping("/shops")
|
||||
public JsonData getShops() {
|
||||
public JsonData getShops(@RequestParam(required = false, name = "accountId") Long accountId) {
|
||||
try {
|
||||
Map<String, Object> response = banmaOrderService.getShops();
|
||||
Map<String, Object> response = banmaOrderService.getShops(accountId);
|
||||
return JsonData.buildSuccess(response);
|
||||
} catch (Exception e) {
|
||||
logger.error("获取店铺列表失败: {}", e.getMessage(), e);
|
||||
@@ -51,19 +50,6 @@ public class BanmaOrderController {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 刷新斑马认证Token
|
||||
*/
|
||||
@PostMapping("/refresh-token")
|
||||
public JsonData refreshToken(){
|
||||
try {
|
||||
banmaOrderService.refreshToken();
|
||||
return JsonData.buildSuccess("Token刷新成功");
|
||||
} catch (Exception e) {
|
||||
logger.error("刷新Token失败: {}", e.getMessage(), e);
|
||||
return JsonData.buildError("Token刷新失败: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
/**
|
||||
* 获取最新订单数据
|
||||
*/
|
||||
|
||||
@@ -21,33 +21,9 @@ public class AmazonProductEntity {
|
||||
@Column(unique = true, nullable = false)
|
||||
private String asin;
|
||||
|
||||
@Column(name = "title", length = 1000)
|
||||
private String title;
|
||||
|
||||
@Column(name = "price")
|
||||
private String price;
|
||||
|
||||
@Column(name = "image_url", length = 1000)
|
||||
private String imageUrl;
|
||||
|
||||
@Column(name = "product_url", length = 1000)
|
||||
private String productUrl;
|
||||
|
||||
@Column(name = "brand")
|
||||
private String brand;
|
||||
|
||||
@Column(name = "category")
|
||||
private String category;
|
||||
|
||||
@Column(name = "rating")
|
||||
private String rating;
|
||||
|
||||
@Column(name = "review_count")
|
||||
private String reviewCount;
|
||||
|
||||
@Column(name = "availability")
|
||||
private String availability;
|
||||
|
||||
@Column(name = "seller")
|
||||
private String seller;
|
||||
|
||||
|
||||
@@ -152,6 +152,12 @@ public class Alibaba1688ServiceImpl implements Alibaba1688Service {
|
||||
}
|
||||
System.out.println("url"+uploadedUrl);
|
||||
System.out.println("skuPrices:"+skuPrices);
|
||||
|
||||
// 检查并上报空数据
|
||||
if (skuPrices.isEmpty()) errorReporter.reportDataEmpty("alibaba1688", uploadedUrl, skuPrices);
|
||||
if (median == null || median == 0.0) errorReporter.reportDataEmpty("alibaba1688", uploadedUrl, median);
|
||||
if (freightFee.isEmpty()) errorReporter.reportDataEmpty("alibaba1688", uploadedUrl, freightFee);
|
||||
|
||||
result.setSkuPrice(skuPrices);
|
||||
result.setMedian( median);
|
||||
result.setMapRecognitionLink( uploadImageBase64(imageUrl));
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
package com.tashow.erp.service.impl;
|
||||
|
||||
import com.tashow.erp.entity.AmazonProductEntity;
|
||||
import com.tashow.erp.repository.AmazonProductRepository;
|
||||
import com.tashow.erp.service.IAmazonScrapingService;
|
||||
import com.tashow.erp.utils.DataReportUtil;
|
||||
import com.tashow.erp.utils.ErrorReporter;
|
||||
import com.tashow.erp.utils.RakutenProxyUtil;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
@@ -13,6 +15,7 @@ import us.codecraft.webmagic.Site;
|
||||
import us.codecraft.webmagic.Spider;
|
||||
import us.codecraft.webmagic.processor.PageProcessor;
|
||||
import us.codecraft.webmagic.selector.Html;
|
||||
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
@@ -29,50 +32,59 @@ public class AmazonScrapingServiceImpl implements IAmazonScrapingService, PagePr
|
||||
private AmazonProductRepository amazonProductRepository;
|
||||
@Autowired
|
||||
private DataReportUtil dataReportUtil;
|
||||
@Autowired
|
||||
private ErrorReporter errorReporter;
|
||||
private final Random random = new Random();
|
||||
private static volatile Spider activeSpider = null;
|
||||
private static final Object spiderLock = new Object();
|
||||
private final Map<String, Map<String, Object>> resultCache = new ConcurrentHashMap<>();
|
||||
private final Site site = Site.me().setRetryTimes(3).setSleepTime(2000 + random.nextInt(2000))
|
||||
.setTimeOut(15000).setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/128.0.0.0 Safari/537.36").addHeader("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9").addHeader("accept-language", "ja,en;q=0.9,zh-CN;q=0.8,zh;q=0.7").addHeader("cache-control", "max-age=0").addHeader("upgrade-insecure-requests", "1").addHeader("sec-ch-ua", "\"Chromium\";v=\"128\", \"Not=A?Brand\";v=\"24\"").addHeader("sec-ch-ua-mobile", "?0").addHeader("sec-ch-ua-platform", "\"Windows\"").addHeader("sec-fetch-site", "none").addHeader("sec-fetch-mode", "navigate").addHeader("sec-fetch-user", "?1").addHeader("sec-fetch-dest", "document").addCookie("i18n-prefs", "JPY").addCookie("session-id", "358-1261309-0483141").addCookie("session-id-time", "2082787201l").addCookie("i18n-prefs", "JPY").addCookie("lc-acbjp", "zh_CN").addCookie("ubid-acbjp", "357-8224002-9668932");
|
||||
private final Map<String, AmazonProductEntity> resultCache = new ConcurrentHashMap<>();
|
||||
private final Site site = Site.me().setRetryTimes(3).setSleepTime(2000 + random.nextInt(2000)).setTimeOut(15000).setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/128.0.0.0 Safari/537.36").addHeader("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9").addHeader("accept-language", "ja,en;q=0.9,zh-CN;q=0.8,zh;q=0.7").addHeader("cache-control", "max-age=0").addHeader("upgrade-insecure-requests", "1").addHeader("sec-ch-ua", "\"Chromium\";v=\"128\", \"Not=A?Brand\";v=\"24\"").addHeader("sec-ch-ua-mobile", "?0").addHeader("sec-ch-ua-platform", "\"Windows\"").addHeader("sec-fetch-site", "none").addHeader("sec-fetch-mode", "navigate").addHeader("sec-fetch-user", "?1").addHeader("sec-fetch-dest", "document").addCookie("i18n-prefs", "JPY").addCookie("session-id", "358-1261309-0483141").addCookie("session-id-time", "2082787201l").addCookie("i18n-prefs", "JPY").addCookie("lc-acbjp", "zh_CN").addCookie("ubid-acbjp", "357-8224002-9668932");
|
||||
|
||||
/**
|
||||
* 处理亚马逊页面数据提取
|
||||
*/
|
||||
@Override
|
||||
public void process(Page page) {
|
||||
Html html = page.getHtml();
|
||||
Map<String, Object> resultMap = new HashMap<>();
|
||||
String url = page.getUrl().toString();
|
||||
|
||||
// 提取ASIN
|
||||
String asin = html.xpath("//input[@id='ASIN']/@value").toString();
|
||||
if (isEmpty(asin)) {
|
||||
String[] parts = url.split("/dp/");
|
||||
if (parts.length > 1) asin = parts[1].split("/")[0].split("\\?")[0];
|
||||
}
|
||||
|
||||
// 提取价格
|
||||
String priceSymbol = html.xpath("//span[@class='a-price-symbol']/text()").toString();
|
||||
String priceWhole = html.xpath("//span[@class='a-price-whole']/text()").toString();
|
||||
String price = priceSymbol + priceWhole;
|
||||
if (price.isEmpty()) {
|
||||
if (isEmpty(price)) {
|
||||
price = html.xpath("//span[@class='a-price-range']/text()").toString();
|
||||
}
|
||||
|
||||
// 提取卖家
|
||||
String seller = html.xpath("//a[@id='sellerProfileTriggerId']/text()").toString();
|
||||
if (seller == null || seller.isEmpty()) {
|
||||
if (isEmpty(seller)) {
|
||||
seller = html.xpath("//span[@class='a-size-small offer-display-feature-text-message']/text()").toString();
|
||||
}
|
||||
resultMap.put("seller", seller);
|
||||
if (price != null || seller != null) {
|
||||
resultMap.put("price", price);
|
||||
} else {
|
||||
|
||||
// 关键数据为空时重试
|
||||
if (isEmpty(price) && isEmpty(seller)) {
|
||||
throw new RuntimeException("Retry this page");
|
||||
}
|
||||
|
||||
String asin = html.xpath("//input[@id='ASIN']/@value").toString();
|
||||
if (asin == null || asin.isEmpty()) {
|
||||
String[] parts = page.getUrl().toString().split("/dp/");
|
||||
if (parts.length > 1) asin = parts[1].split("/")[0].split("\\?")[0];
|
||||
}
|
||||
String title = html.xpath("//span[@id='productTitle']/text()").toString();
|
||||
if (title == null || title.isEmpty())
|
||||
title = html.xpath("//h1[@class='a-size-large a-spacing-none']/text()").toString();
|
||||
resultMap.put("asin", asin != null ? asin : "");
|
||||
resultMap.put("title", (title == null || title.isEmpty()) ? "未获取" : title.trim());
|
||||
// 检查并上报空数据
|
||||
if (isEmpty(price)) errorReporter.reportDataEmpty("amazon", asin, price);
|
||||
if (isEmpty(seller)) errorReporter.reportDataEmpty("amazon", asin, seller);
|
||||
|
||||
resultCache.put(asin, resultMap);
|
||||
page.putField("resultMap", resultMap);
|
||||
AmazonProductEntity entity = new AmazonProductEntity();
|
||||
entity.setAsin(asin != null ? asin : "");
|
||||
entity.setPrice(price);
|
||||
entity.setSeller(seller);
|
||||
|
||||
resultCache.put(asin, entity);
|
||||
page.putField("entity", entity);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -87,77 +99,45 @@ public class AmazonScrapingServiceImpl implements IAmazonScrapingService, PagePr
|
||||
* 批量获取产品信息
|
||||
*/
|
||||
@Override
|
||||
public Map<String, Object> batchGetProductInfo(List<String> asinList, String batchId) {
|
||||
public List<AmazonProductEntity> batchGetProductInfo(List<String> asinList, String batchId) {
|
||||
String sessionId = (batchId != null) ? batchId : "SINGLE_" + UUID.randomUUID();
|
||||
List<Map<String, Object>> products = new ArrayList<>();
|
||||
List<AmazonProductEntity> products = new ArrayList<>();
|
||||
|
||||
for (String asin : asinList) {
|
||||
if (asin == null || asin.trim().isEmpty()) continue;
|
||||
String cleanAsin = asin.replaceAll("[^a-zA-Z0-9]", "");
|
||||
Map<String, Object> result = new HashMap<>();
|
||||
|
||||
amazonProductRepository.findByAsin(cleanAsin).ifPresentOrElse(entity -> {
|
||||
if (entity.getCreatedAt().isAfter(LocalDateTime.now().minusHours(1))) {
|
||||
result.put("asin", entity.getAsin());
|
||||
result.put("title", entity.getTitle());
|
||||
result.put("price", entity.getPrice());
|
||||
result.put("seller", entity.getSeller());
|
||||
result.put("imageUrl", entity.getImageUrl());
|
||||
result.put("productUrl", entity.getProductUrl());
|
||||
result.put("brand", entity.getBrand());
|
||||
result.put("category", entity.getCategory());
|
||||
result.put("rating", entity.getRating());
|
||||
result.put("reviewCount", entity.getReviewCount());
|
||||
result.put("availability", entity.getAvailability());
|
||||
products.add(result);
|
||||
}
|
||||
}, () -> {
|
||||
// 数据库没有或过期 -> 爬取
|
||||
AmazonProductEntity product = amazonProductRepository.findByAsin(cleanAsin).filter(entity -> entity.getCreatedAt().isAfter(LocalDateTime.now().minusHours(1)) && !isEmpty(entity.getPrice()) && !isEmpty(entity.getSeller())).orElseGet(() -> {
|
||||
// 采集新数据
|
||||
String url = "https://www.amazon.co.jp/dp/" + cleanAsin;
|
||||
RakutenProxyUtil proxyUtil = new RakutenProxyUtil();
|
||||
|
||||
synchronized (spiderLock) {
|
||||
activeSpider = Spider.create(this)
|
||||
.addUrl(url)
|
||||
.setDownloader(proxyUtil.createProxyDownloader(proxyUtil.detectSystemProxy(url)))
|
||||
.thread(1);
|
||||
activeSpider = Spider.create(this).addUrl(url).setDownloader(proxyUtil.createProxyDownloader(proxyUtil.detectSystemProxy(url))).thread(1);
|
||||
activeSpider.run();
|
||||
activeSpider = null;
|
||||
}
|
||||
result.putAll(resultCache.getOrDefault(cleanAsin, Map.of("asin", cleanAsin, "price", "", "seller", "", "title", "")));
|
||||
|
||||
// 存库
|
||||
AmazonProductEntity entity = new AmazonProductEntity();
|
||||
AmazonProductEntity entity = resultCache.getOrDefault(cleanAsin, new AmazonProductEntity());
|
||||
entity.setAsin(cleanAsin);
|
||||
entity.setTitle((String) result.get("title"));
|
||||
entity.setPrice((String) result.get("price"));
|
||||
entity.setSeller((String) result.get("seller"));
|
||||
entity.setImageUrl((String) result.get("imageUrl"));
|
||||
entity.setProductUrl((String) result.get("productUrl"));
|
||||
entity.setBrand((String) result.get("brand"));
|
||||
entity.setCategory((String) result.get("category"));
|
||||
entity.setRating((String) result.get("rating"));
|
||||
entity.setReviewCount((String) result.get("reviewCount"));
|
||||
entity.setAvailability((String) result.get("availability"));
|
||||
entity.setSessionId(sessionId);
|
||||
entity.setCreatedAt(LocalDateTime.now());
|
||||
|
||||
try {
|
||||
amazonProductRepository.save(entity);
|
||||
dataReportUtil.reportDataCollection("AMAZON", 1, "0");
|
||||
} catch (Exception e) {
|
||||
logger.warn("保存商品数据失败: {}", cleanAsin);
|
||||
}
|
||||
products.add(result);
|
||||
return entity;
|
||||
});
|
||||
|
||||
products.add(product);
|
||||
}
|
||||
|
||||
long failedCount = products.stream().filter(p -> p.get("price").toString().isEmpty()).count();
|
||||
return Map.of(
|
||||
"products", products,
|
||||
"total", products.size(),
|
||||
"success", true,
|
||||
"failedCount", failedCount
|
||||
);
|
||||
return products;
|
||||
}
|
||||
|
||||
private boolean isEmpty(String str) {
|
||||
return str == null || str.trim().isEmpty();
|
||||
}
|
||||
|
||||
}
|
||||
@@ -50,7 +50,6 @@ public class ClientMonitorController extends BaseController {
|
||||
startPage();
|
||||
return getDataTable(clientMonitorService.selectClientEventLogList(clientEventLog));
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取客户端数据采集报告列表
|
||||
*/
|
||||
@@ -108,7 +107,6 @@ public class ClientMonitorController extends BaseController {
|
||||
}
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 客户端错误上报API
|
||||
*/
|
||||
@@ -151,10 +149,6 @@ public class ClientMonitorController extends BaseController {
|
||||
return AjaxResult.success(clientMonitorService.getVersionDistribution());
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* 清理过期数据
|
||||
*/
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
package com.ruoyi.web.controller.system;
|
||||
|
||||
import com.ruoyi.common.annotation.Anonymous;
|
||||
import com.ruoyi.common.core.domain.AjaxResult;
|
||||
import com.ruoyi.common.utils.ip.IpUtils;
|
||||
@@ -8,7 +7,6 @@ import com.ruoyi.system.mapper.ClientDeviceMapper;
|
||||
import com.ruoyi.web.sse.SseHubService;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
@@ -42,7 +40,6 @@ public class ClientDeviceController {
|
||||
map.put("used", used);
|
||||
return AjaxResult.success(map);
|
||||
}
|
||||
|
||||
/**
|
||||
* 按用户名查询设备列表(最近活动优先)
|
||||
* @param username 用户名,必需参数
|
||||
@@ -61,7 +58,7 @@ public class ClientDeviceController {
|
||||
* 设备注册(幂等)
|
||||
*
|
||||
* 根据 deviceId 判断:
|
||||
* - 不存在:插入新记录(后端生成设备名称、IP等信息)
|
||||
* - 不存在:插入新记录(检查设备数量限制)
|
||||
* - 已存在:更新设备信息
|
||||
*/
|
||||
@PostMapping("/register")
|
||||
@@ -70,6 +67,15 @@ public class ClientDeviceController {
|
||||
String ip = IpUtils.getIpAddr(request);
|
||||
String deviceName = request.getHeader("X-Client-User") + "@" + ip + " (" + request.getHeader("X-Client-OS") + ")";
|
||||
if (exists == null) {
|
||||
// 检查设备数量限制
|
||||
List<ClientDevice> userDevices = clientDeviceMapper.selectByUsername(device.getUsername());
|
||||
int activeDeviceCount = 0;
|
||||
for (ClientDevice d : userDevices) {
|
||||
if (!"removed".equals(d.getStatus())) activeDeviceCount++;
|
||||
}
|
||||
if (activeDeviceCount >= DEFAULT_LIMIT) {
|
||||
return AjaxResult.error("设备数量已达上限(" + DEFAULT_LIMIT + "个),请先移除其他设备");
|
||||
}
|
||||
device.setIp(ip);
|
||||
device.setStatus("online");
|
||||
device.setLastActiveAt(new java.util.Date());
|
||||
@@ -153,6 +159,15 @@ public class ClientDeviceController {
|
||||
String ip = IpUtils.getIpAddr(request);
|
||||
String deviceName = request.getHeader("X-Client-User") + "@" + ip + " (" + request.getHeader("X-Client-OS") + ")";
|
||||
if (exists == null) {
|
||||
// 检查设备数量限制
|
||||
List<ClientDevice> userDevices = clientDeviceMapper.selectByUsername(device.getUsername());
|
||||
int activeDeviceCount = 0;
|
||||
for (ClientDevice d : userDevices) {
|
||||
if (!"removed".equals(d.getStatus())) activeDeviceCount++;
|
||||
}
|
||||
if (activeDeviceCount >= DEFAULT_LIMIT) {
|
||||
return AjaxResult.error("设备数量已达上限(" + DEFAULT_LIMIT + "个),请先移除其他设备");
|
||||
}
|
||||
device.setIp(ip);
|
||||
device.setStatus("online");
|
||||
device.setLastActiveAt(new java.util.Date());
|
||||
|
||||
@@ -23,7 +23,7 @@ public class BanmaOrderController extends BaseController {
|
||||
private IBanmaAccountService accountService;
|
||||
|
||||
/**
|
||||
* 查询账号列表(仅返回必要字段)
|
||||
* 查询账号列表(
|
||||
*/
|
||||
@GetMapping("/accounts")
|
||||
public R<?> listAccounts() {
|
||||
@@ -37,7 +37,9 @@ public class BanmaOrderController extends BaseController {
|
||||
@PostMapping("/accounts")
|
||||
public R<?> saveAccount(@RequestBody BanmaAccount body) {
|
||||
Long id = accountService.saveOrUpdate(body);
|
||||
return R.ok(Map.of("id", id));
|
||||
boolean ok = false;
|
||||
try { ok = accountService.refreshToken(id); } catch (Exception ignore) {}
|
||||
return ok ? R.ok(Map.of("id", id)) : R.fail("账号或密码错误,无法获取Token");
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -49,4 +51,18 @@ public class BanmaOrderController extends BaseController {
|
||||
return R.ok();
|
||||
}
|
||||
|
||||
/** 手动刷新单个账号 Token */
|
||||
@PostMapping("/accounts/{id}/refresh-token")
|
||||
public R<?> refreshOne(@PathVariable Long id) {
|
||||
accountService.refreshToken(id);
|
||||
return R.ok();
|
||||
}
|
||||
|
||||
/** 手动刷新全部启用账号 Token */
|
||||
@PostMapping("/refresh-all")
|
||||
public R<?> refreshAll() {
|
||||
accountService.refreshAllTokens();
|
||||
return R.ok();
|
||||
}
|
||||
|
||||
}
|
||||
@@ -714,8 +714,8 @@ public class ClientMonitorServiceImpl implements IClientMonitorService {
|
||||
@Override
|
||||
public void cleanExpiredData() {
|
||||
try {
|
||||
// 清理过期的客户端(设置为离线状态)
|
||||
clientMonitorMapper.updateExpiredClientsOffline();
|
||||
// // 清理过期的客户端(设置为离线状态)
|
||||
// clientMonitorMapper.updateExpiredClientsOffline();
|
||||
|
||||
// 清理过期的设备(设置为离线状态)
|
||||
clientMonitorMapper.updateExpiredDevicesOffline();
|
||||
@@ -725,8 +725,6 @@ public class ClientMonitorServiceImpl implements IClientMonitorService {
|
||||
|
||||
// 删除过期的事件日志
|
||||
clientMonitorMapper.deleteExpiredEventLogs();
|
||||
|
||||
logger.info("过期数据清理完成");
|
||||
} catch (Exception e) {
|
||||
logger.error("清理过期数据失败: {}", e.getMessage(), e);
|
||||
throw new RuntimeException("清理过期数据失败", e);
|
||||
|
||||
@@ -6,8 +6,10 @@ import java.util.List;
|
||||
public interface ClientDeviceMapper {
|
||||
ClientDevice selectByDeviceId(String deviceId);
|
||||
List<ClientDevice> selectByUsername(String username);
|
||||
List<ClientDevice> selectOnlineDevices();
|
||||
int insert(ClientDevice device);
|
||||
int updateByDeviceId(ClientDevice device);
|
||||
int updateExpiredDevicesOffline();
|
||||
int deleteByDeviceId(String deviceId);
|
||||
int countByUsername(String username);
|
||||
}
|
||||
|
||||
@@ -48,6 +48,15 @@
|
||||
<select id="countByUsername" resultType="int">
|
||||
select count(1) from client_device where username = #{username}
|
||||
</select>
|
||||
|
||||
<select id="selectOnlineDevices" resultMap="ClientDeviceMap">
|
||||
select * from client_device where status = 'online' order by last_active_at desc
|
||||
</select>
|
||||
|
||||
<update id="updateExpiredDevicesOffline">
|
||||
update client_device set status = 'offline'
|
||||
where status = 'online' and (last_active_at is null or last_active_at < date_sub(now(), interval 2 minute))
|
||||
</update>
|
||||
</mapper>
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user