xuqmGroup/XuqmGroup-Server
0
0
派生 0
代码 工单 合并请求 版本发布 百科 动态

feat(update-service,file-service): 按占用空间阈值清理历史APK，下载页兼容"从未点击发布"的真实数据

浏览代码 
- 下载页/公开接口重新定义"曾上架"：不再要求内部 PublishStatus=PUBLISHED，
  实际生产数据显示大多数租户只提交应用商店从不点击"发布"，现改为
  PublishStatus 已发布 或 任一厂商 storeReviewStatus 显示 liveOnStore
- 新增 file-service DELETE /api/file/{hash}（X-Internal-Token 保护，销毁性操作不能像
  pin/GET 一样完全公开）
- 新增 ApkRetentionService：按 appKey 分组，从最新版本往回累加安装包大小，
  超过阈值（默认 5GB，可通过发布配置 JSON 里 apkRetentionMaxTotalSizeMb 按应用覆盖）
  即清理更旧版本的安装包，但永远保留最新版本
- AppVersionEntity 新增 fileSize 字段，上传时记录（本地存储路径直接取
  MultipartFile.getSize()，file-service 路径由前端一并上传时的 size 回传）
- 新增每日 02:00 自动重新 pin 所有历史 APK 的兜底任务，保护本次修复之前已上传、
  尚未被 pin 保护的历史版本

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
这个提交包含在：
XuqmGroup 2026-07-04 02:38:29 +08:00
父节点 acad056e55
当前提交 9be2e71b82
共有 9 个文件被更改，包括 277 次插入 和 20 次删除
显示统计 下载 Patch 文件 下载 Diff 文件 展开所有文件 折叠所有文件

3
file-service/src/main/java/com/xuqm/file/config/SecurityConfig.java
查看文件

@ -40,6 +40,9 @@ public class SecurityConfig {
.requestMatchers(new AntPathRequestMatcher("/api/file/*/thumbnail", "GET")).permitAll()
.requestMatchers(new AntPathRequestMatcher("/api/file/*", "GET")).permitAll()
.requestMatchers(new AntPathRequestMatcher("/api/file/*/pin", "POST")).permitAll()
// DELETE is permitAll at the Spring Security layer but enforces its own
// X-Internal-Token shared-secret check in FileController — see delete().
.requestMatchers(new AntPathRequestMatcher("/api/file/*", "DELETE")).permitAll()
.requestMatchers(new AntPathRequestMatcher("/actuator/**", "GET")).permitAll()
.requestMatchers(new AntPathRequestMatcher("/error")).permitAll()
.anyRequest().authenticated()

22
file-service/src/main/java/com/xuqm/file/controller/FileController.java
查看文件

@ -4,7 +4,9 @@ import com.xuqm.common.model.ApiResponse;
import com.xuqm.file.entity.FileEntity;
import com.xuqm.file.service.FileStorageService;
import jakarta.servlet.http.HttpServletResponse;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
@ -22,6 +24,9 @@ public class FileController {
private final FileStorageService fileStorageService;
@Value("${file.internal-token:xuqm-internal-token}")
private String internalToken;
public FileController(FileStorageService fileStorageService) {
this.fileStorageService = fileStorageService;
}
@ -57,6 +62,23 @@ public class FileController {
return ApiResponse.success(null);
}
/**
* Permanently delete a file (DB record + disk file + thumbnail), regardless of pinned
* status. Destructive and shared across every service that uses file-service (IM
* attachments, avatars, APK packages, ...) — gated by an internal shared-secret header
* rather than left open like GET/upload/pin, since a wrong call here is unrecoverable.
* Called by update-service's APK space-based retention cleanup.
*/
@DeleteMapping("/{hash}")
public ResponseEntity<ApiResponse<Void>> delete(@PathVariable String hash,
@RequestHeader(value = "X-Internal-Token", required = false) String token) {
if (token == null || !internalToken.equals(token)) {
return ResponseEntity.status(403).body(ApiResponse.error(403, "Forbidden"));
}
fileStorageService.delete(hash);
return ResponseEntity.ok(ApiResponse.success(null));
}
/**
* Stream the thumbnail for a given hash. Falls back to the original if no thumbnail exists.
*/

13
file-service/src/main/java/com/xuqm/file/service/FileStorageService.java
查看文件

@ -177,6 +177,19 @@ public class FileStorageService {
}
}
@Transactional
public void delete(String hash) {
FileEntity entity = fileRepository.findByHash(hash).orElse(null);
if (entity == null) {
return;
}
deleteDiskFile(entity.getStoragePath());
if (entity.getThumbnailPath() != null) {
deleteDiskFile(entity.getThumbnailPath());
}
fileRepository.delete(entity);
}
@Scheduled(cron = "0 0 3 * * *")
@Transactional
public void reclaimExpiredFiles() {

3
update-service/src/main/java/com/xuqm/update/controller/AppVersionController.java
查看文件

@ -152,6 +152,7 @@ public class AppVersionController {
@RequestParam(required = false) String changeLog,
@RequestParam(defaultValue = "false") boolean forceUpdate,
@RequestParam(required = false) String apkUrl,
@RequestParam(required = false) Long apkSize,
@RequestParam(required = false) MultipartFile apkFile,
@RequestParam(required = false) String scheduledPublishAt,
@RequestParam(required = false) String webhookUrl,
@ -193,6 +194,7 @@ public class AppVersionController {
if (platform == AppVersionEntity.Platform.ANDROID) {
if (hasText(apkUrl)) {
entity.setDownloadUrl(apkUrl);
entity.setFileSize(apkSize != null ? apkSize : 0L);
// apkUrl was uploaded to file-service directly by the client; pin it so the
// shared 30-day unused-file reclaim job never deletes a published app version's APK.
updateAssetService.pinRemoteFile(apkUrl);
@ -200,6 +202,7 @@ public class AppVersionController {
UpdateAssetService.StoreResult stored = updateAssetService.storeAppPackage(apkFile);
entity.setDownloadUrl(stored != null ? stored.url() : null);
entity.setApkHash(stored != null ? stored.hash() : null);
entity.setFileSize(apkFile.getSize());
}
}
entity.setChangeLog(changeLog);

59
update-service/src/main/java/com/xuqm/update/controller/PublicDownloadController.java
查看文件

@ -1,5 +1,6 @@
package com.xuqm.update.controller;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.xuqm.common.model.ApiResponse;
import com.xuqm.update.entity.AppStoreConfigEntity;
import com.xuqm.update.entity.AppVersionEntity;
@ -15,7 +16,6 @@ import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
/**
* Public, unauthenticated endpoint backing the per-app download page. Returns only what a
@ -27,10 +27,13 @@ public class PublicDownloadController {
private final AppVersionRepository versionRepository;
private final AppStoreService appStoreService;
private final ObjectMapper objectMapper;
public PublicDownloadController(AppVersionRepository versionRepository, AppStoreService appStoreService) {
public PublicDownloadController(AppVersionRepository versionRepository, AppStoreService appStoreService,
ObjectMapper objectMapper) {
this.versionRepository = versionRepository;
this.appStoreService = appStoreService;
this.objectMapper = objectMapper;
}
@GetMapping("/download-info")
@ -42,14 +45,41 @@ public class PublicDownloadController {
return ResponseEntity.ok(ApiResponse.success(response));
}
private static final List<AppVersionEntity.PublishStatus> EVER_PUBLISHED = List.of(
AppVersionEntity.PublishStatus.PUBLISHED, AppVersionEntity.PublishStatus.DEPRECATED);
/**
* "曾上架" (ever released) means either update-service's own publish workflow was used
* (PublishStatus PUBLISHED/DEPRECATED), OR — the far more common real-world case — the
* tenant submits straight to vendor app stores and never touches the internal "发布"
* button at all, leaving publishStatus stuck at DRAFT forever even though the version is
* fully live on vivo/华为/小米/etc. Treat a version confirmed live on any vendor store
* (storeReviewStatus liveOnStore=true or state=APPROVED) as released too.
*/
private boolean everReleased(AppVersionEntity v) {
if (v.getPublishStatus() == AppVersionEntity.PublishStatus.PUBLISHED
|| v.getPublishStatus() == AppVersionEntity.PublishStatus.DEPRECATED) {
return true;
}
String reviewJson = v.getStoreReviewStatus();
if (reviewJson == null || reviewJson.isBlank()) {
return false;
}
try {
Map<String, Object> reviewMap = objectMapper.readValue(reviewJson,
new com.fasterxml.jackson.core.type.TypeReference<Map<String, Object>>() {});
for (Object entry : reviewMap.values()) {
if (!(entry instanceof Map<?, ?> m)) continue;
if (Boolean.TRUE.equals(m.get("liveOnStore"))) return true;
if ("APPROVED".equals(m.get("state"))) return true;
}
} catch (Exception ignored) {
// malformed JSON — treat as not released
}
return false;
}
private Map<String, Object> androidInfo(String appKey) {
// "Ever published" = currently PUBLISHED or since superseded (DEPRECATED). DRAFT
// versions were never actually released and must not appear on the public page.
List<AppVersionEntity> versions = versionRepository.findByAppKeyAndPlatformAndPublishStatusInOrderByVersionCodeDesc(
appKey, AppVersionEntity.Platform.ANDROID, EVER_PUBLISHED);
List<AppVersionEntity> all = versionRepository.findByAppKeyAndPlatformOrderByVersionCodeDesc(
appKey, AppVersionEntity.Platform.ANDROID);
List<AppVersionEntity> versions = all.stream().filter(this::everReleased).toList();
if (versions.isEmpty()) {
return null;
}
@ -92,17 +122,16 @@ public class PublicDownloadController {
if (storeUrl == null || storeUrl.isBlank()) {
return null;
}
Optional<AppVersionEntity> latest = versionRepository.findTopByAppKeyAndPlatformAndPublishStatusOrderByVersionCodeDesc(
appKey, platform, AppVersionEntity.PublishStatus.PUBLISHED);
if (latest.isEmpty()) {
List<AppVersionEntity> all = versionRepository.findByAppKeyAndPlatformOrderByVersionCodeDesc(appKey, platform);
AppVersionEntity latest = all.stream().filter(this::everReleased).findFirst().orElse(null);
if (latest == null) {
return null;
}
AppVersionEntity v = latest.get();
Map<String, Object> info = new LinkedHashMap<>();
info.put("versionName", v.getVersionName());
info.put("versionCode", v.getVersionCode());
info.put("versionName", latest.getVersionName());
info.put("versionCode", latest.getVersionCode());
info.put("storeUrl", storeUrl);
info.put("publishedAt", v.getCreatedAt() != null ? v.getCreatedAt().toString() : "");
info.put("publishedAt", latest.getCreatedAt() != null ? latest.getCreatedAt().toString() : "");
return info;
}
}

7
update-service/src/main/java/com/xuqm/update/entity/AppVersionEntity.java
查看文件

@ -132,6 +132,10 @@ public class AppVersionEntity {
@Column(length = 64)
private String apkHash;
/** APK file size in bytes. Used by the size-based retention cleanup to evict old versions. */
@Column(nullable = false, columnDefinition = "BIGINT NOT NULL DEFAULT 0")
private long fileSize;
@Column(nullable = false)
private LocalDateTime createdAt;
@ -186,6 +190,9 @@ public class AppVersionEntity {
public String getApkHash() { return apkHash; }
public void setApkHash(String apkHash) { this.apkHash = apkHash; }
public long getFileSize() { return fileSize; }
public void setFileSize(long fileSize) { this.fileSize = fileSize; }
public LocalDateTime getCreatedAt() { return createdAt; }
public void setCreatedAt(LocalDateTime createdAt) { this.createdAt = createdAt; }

3
update-service/src/main/java/com/xuqm/update/repository/AppVersionRepository.java
查看文件

@ -59,4 +59,7 @@ public interface AppVersionRepository extends JpaRepository<AppVersionEntity, St
List<AppVersionEntity> findByAppKeyAndPlatformAndPublishStatusInOrderByVersionCodeDesc(
String appKey, AppVersionEntity.Platform platform, List<AppVersionEntity.PublishStatus> statuses);
@Query(value = "SELECT * FROM update_app_version WHERE download_url LIKE 'https://file.dev.xuqinmin.com%'", nativeQuery = true)
List<AppVersionEntity> findAllWithFileServiceDownloadUrl();
}

118
update-service/src/main/java/com/xuqm/update/service/ApkRetentionService.java 普通文件
查看文件

@ -0,0 +1,118 @@
package com.xuqm.update.service;
import com.xuqm.update.entity.AppVersionEntity;
import com.xuqm.update.repository.AppVersionRepository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
* Evicts old Android APK packages once an app's total install-package storage exceeds a
* configured budget, instead of relying on file-service's generic 30-day-unused reclaim
* (which has no notion of "keep every historical release downloadable" and, worse, has no
* notion of "app" at all). The currently-latest version per app is never evicted regardless
* of total size, so the app always stays installable/updatable.
*/
@Service
public class ApkRetentionService {
private static final Logger log = LoggerFactory.getLogger(ApkRetentionService.class);
private final AppVersionRepository versionRepository;
private final UpdateAssetService updateAssetService;
private final PublishConfigService publishConfigService;
private final UpdateOperationLogService operationLogService;
@Value("${update.apk-retention.default-max-total-size-mb:5120}")
private long defaultMaxTotalSizeMb;
public ApkRetentionService(AppVersionRepository versionRepository,
UpdateAssetService updateAssetService,
PublishConfigService publishConfigService,
UpdateOperationLogService operationLogService) {
this.versionRepository = versionRepository;
this.updateAssetService = updateAssetService;
this.publishConfigService = publishConfigService;
this.operationLogService = operationLogService;
}
/** Runs after the daily re-pin (02:00) and before file-service's reclaim job (03:00). */
@Scheduled(cron = "0 30 2 * * *")
public void enforceRetention() {
List<AppVersionEntity> all = versionRepository.findAllWithFileServiceDownloadUrl();
Map<String, List<AppVersionEntity>> byApp = all.stream()
.collect(Collectors.groupingBy(AppVersionEntity::getAppKey));
int evictedTotal = 0;
for (Map.Entry<String, List<AppVersionEntity>> entry : byApp.entrySet()) {
evictedTotal += enforceForApp(entry.getKey(), entry.getValue());
}
if (evictedTotal > 0) {
log.info("APK retention: evicted {} old version(s) across {} app(s) over their size budget",
evictedTotal, byApp.size());
}
}
private int enforceForApp(String appKey, List<AppVersionEntity> unsorted) {
List<AppVersionEntity> versions = new ArrayList<>(unsorted);
versions.sort(Comparator.comparingInt(AppVersionEntity::getVersionCode).reversed());
long maxBytes = resolveMaxTotalBytes(appKey);
long running = 0;
int evicted = 0;
for (int i = 0; i < versions.size(); i++) {
AppVersionEntity v = versions.get(i);
running += v.getFileSize();
if (i == 0) {
// Always keep the latest version downloadable, regardless of total size.
continue;
}
if (running > maxBytes) {
if (evict(v)) {
evicted++;
}
}
}
return evicted;
}
private long resolveMaxTotalBytes(String appKey) {
long mb = defaultMaxTotalSizeMb;
try {
long override = publishConfigService.getConfigNode(appKey)
.path("apkRetentionMaxTotalSizeMb").asLong(0);
if (override > 0) {
mb = override;
}
} catch (Exception ignored) {
// fall back to the default
}
return mb * 1024 * 1024;
}
private boolean evict(AppVersionEntity v) {
String downloadUrl = v.getDownloadUrl();
if (!updateAssetService.deleteRemoteFile(downloadUrl)) {
return false;
}
v.setDownloadUrl(null);
v.setApkHash(null);
long freedBytes = v.getFileSize();
v.setFileSize(0);
versionRepository.save(v);
operationLogService.record(v.getAppKey(), "APP_VERSION", v.getId(), "APK_RETENTION_EVICT",
"安装包占用空间超过阈值，自动清理旧版本安装包",
Map.of("versionName", v.getVersionName(), "versionCode", v.getVersionCode(),
"freedBytes", freedBytes));
log.info("APK retention: evicted {}/{} versionCode={} freedBytes={}",
v.getAppKey(), v.getPlatform(), v.getVersionCode(), freedBytes);
return true;
}
}

69
update-service/src/main/java/com/xuqm/update/service/UpdateAssetService.java
查看文件

@ -2,12 +2,15 @@ package com.xuqm.update.service;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.xuqm.update.entity.AppVersionEntity;
import com.xuqm.update.model.AppPackageInspectResult;
import com.xuqm.update.model.RnBundleInspectResult;
import com.xuqm.update.repository.AppVersionRepository;
import net.dongliu.apk.parser.ApkFile;
import net.dongliu.apk.parser.bean.ApkMeta;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.ContentDisposition;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import org.slf4j.Logger;
@ -25,6 +28,7 @@ import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.util.List;
import java.util.Map;
import java.util.Locale;
import java.util.HexFormat;
@ -43,6 +47,7 @@ public class UpdateAssetService {
private static final Logger log = LoggerFactory.getLogger(UpdateAssetService.class);
private final ObjectMapper objectMapper;
private final AppVersionRepository versionRepository;
@Value("${update.upload-dir:/tmp/xuqm-update}")
private String uploadDir;
@ -56,8 +61,29 @@ public class UpdateAssetService {
@Value("${FILE_BASE_URL:}")
private String fileBaseUrl;
public UpdateAssetService(ObjectMapper objectMapper) {
@Value("${file.internal-token:xuqm-internal-token}")
private String internalToken;
public UpdateAssetService(ObjectMapper objectMapper, AppVersionRepository versionRepository) {
this.objectMapper = objectMapper;
this.versionRepository = versionRepository;
}
/**
* Daily safety net: re-pin every app version whose APK lives on file-service, so
* historical versions already uploaded before the pin-on-upload fix (or any that
* transiently failed to pin) don't get silently deleted by file-service's 30-day
* unused-file reclaim job. Runs well before that job's 03:00 cron.
*/
@Scheduled(cron = "0 0 2 * * *")
public void rePinAllFileServiceAssets() {
List<AppVersionEntity> versions = versionRepository.findAllWithFileServiceDownloadUrl();
for (AppVersionEntity v : versions) {
pinRemoteFile(v.getDownloadUrl());
}
if (!versions.isEmpty()) {
log.info("Re-pinned {} file-service-hosted APK(s) against 30-day reclaim", versions.size());
}
}
/** 存储结果：下载 URL 和文件 SHA-256 哈希 */
@ -400,16 +426,19 @@ public class UpdateAssetService {
return packageUrl;
}
private boolean isFileServiceUrl(String packageUrl) {
return packageUrl != null && (
packageUrl.contains("file.dev.xuqinmin.com")
|| (hasText(fileBaseUrl) && packageUrl.startsWith(fileBaseUrl)));
}
/**
* Mark an APK hosted on file-service as permanently retained so it's excluded from
* file-service's 30-day unused-file reclaim job. Best-effort — failures are logged and
* swallowed so they never block version creation/publishing.
*/
public void pinRemoteFile(String packageUrl) {
boolean isFileServiceUrl = packageUrl != null && (
packageUrl.contains("file.dev.xuqinmin.com")
|| (hasText(fileBaseUrl) && packageUrl.startsWith(fileBaseUrl)));
if (!isFileServiceUrl) {
if (!isFileServiceUrl(packageUrl)) {
return;
}
try {
@ -427,6 +456,36 @@ public class UpdateAssetService {
}
}
/**
* Permanently delete an APK hosted on file-service. Used by the size-based retention
* cleanup when evicting old versions past the configured total-size budget. Returns
* true only on a confirmed successful delete — callers should not clear their own
* downloadUrl reference unless this returns true, to avoid orphaning a version whose
* file deletion actually failed.
*/
public boolean deleteRemoteFile(String packageUrl) {
if (!isFileServiceUrl(packageUrl)) {
return false;
}
try {
String deleteUrl = rewriteToInternalUrl(packageUrl);
HttpURLConnection connection = (HttpURLConnection) new URL(deleteUrl).openConnection();
connection.setRequestMethod("DELETE");
connection.setRequestProperty("X-Internal-Token", internalToken);
connection.setConnectTimeout(10_000);
connection.setReadTimeout(15_000);
int status = connection.getResponseCode();
if (status >= 400) {
log.warn("Failed to delete file-service asset {}: HTTP {}", packageUrl, status);
return false;
}
return true;
} catch (Exception e) {
log.warn("Failed to delete file-service asset {}: {}", packageUrl, e.getMessage());
return false;
}
}
private RemotePackage downloadRemotePackage(String packageUrl, boolean tempFile) throws IOException {
String internalUrl = rewriteToInternalUrl(packageUrl);
HttpURLConnection connection = (HttpURLConnection) new URL(internalUrl).openConnection();