perf:爬虫数据拉取优化

script/src/main/java/com/xiang/xservice/fwd/schedule/PullDataFromFWDJob.java

@@ -2,6 +2,7 @@ package com.xiang.xservice.fwd.schedule;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
+import com.xiang.xservice.basic.common.req.BaseRequest;
 import com.xiang.xservice.basic.config.MyThreadFactory;
 import com.xiang.xservice.basic.utils.DateUtils;
 import com.xiang.xservice.fwd.entity.pojo.FPerformProjectInfo;
@@ -69,15 +70,24 @@ public class PullDataFromFWDJob {
     @Scheduled(cron = "0 0 1 1/1 * ?")
     @PostMapping("/pullProjectsDataJob")
     public void pullProjectsDataJob() {
-        ProjectsResp showProjectsFromHttp = performServiceHttp.getShowProjectsFromHttp();
+        int i = 0;
+        while (true) {
+            i++;
+            BaseRequest request = new BaseRequest();
+            request.setCurrent(i);
+            request.setPageSize(10 * i);
+            ProjectsResp showProjectsFromHttp = performServiceHttp.getShowProjectsFromHttp(request);
             if (Objects.isNull(showProjectsFromHttp)) {
                 return;
             }
+            if (CollectionUtils.isEmpty(showProjectsFromHttp.getList())) {
+                break;
+            }
             List<ProjectList> projectList = showProjectsFromHttp.getList();
 
             List<CompletableFuture> futureList = Lists.newArrayList();
             for (ProjectList project : projectList) {
-            CompletableFuture future = CompletableFuture.runAsync(() -> {
+                CompletableFuture<Void> future = CompletableFuture.runAsync(() -> {
                     ProjectInfoResp projectInfoFromHttp = performServiceHttp.getProjectInfoFromHttp(project.getProjectId());
                     Perform performsByProjectIdFromHttp = performServiceHttp.getPerformsByProjectIdFromHttp(project.getProjectId());
                     if (Objects.nonNull(projectInfoFromHttp) && Objects.nonNull(performsByProjectIdFromHttp)) {
@@ -123,9 +133,6 @@ public class PullDataFromFWDJob {
                                                     if (!seatMap.containsKey(seatPlan.getSeatPlanId())) {
                                                         continue;
                                                     }
-                                                if (!seatMap.get(seatPlan.getSeatPlanId())) {
-                                                    continue;
-                                                }
                                                     FPerformSeatInfo fPerformSeatInfo = new FPerformSeatInfo();
                                                     fPerformSeatInfo.setSeatPlanId(seatPlan.getSeatPlanId());
                                                     fPerformSeatInfo.setSeatPlanName(seatPlan.getSeatPlanName());
@@ -152,6 +159,7 @@ public class PullDataFromFWDJob {
                                                         fPerformSeatInfo.setSaleTime(DateUtils.getTimeFromStr(seatPlan.getSaleTime()));
                                                     }
                                                     fPerformSeatInfo.setProjectId(project.getProjectId());
+                                                    fPerformSeatInfo.setSoldOut(seatMap.get(seatPlan.getSeatPlanId()) ? 1 : 0);
                                                     FPerformSeatInfo seatInfo = performSeatInfoMapper.getPerformSeatInfoBySeatIdAndPerformIdAndProjectId(fPerformSeatInfo.getSeatPlanId(), fPerformSeatInfo.getPerformId(), project.getProjectId());
                                                     if (Objects.nonNull(seatInfo)) {
                                                         fPerformSeatInfo.setId(seatInfo.getId());
@@ -213,9 +221,6 @@ public class PullDataFromFWDJob {
                                                     if (!seatMap.containsKey(seatPlan.getSeatPlanId())) {
                                                         continue;
                                                     }
-                                                if (!seatMap.get(seatPlan.getSeatPlanId())) {
-                                                    continue;
-                                                }
                                                     FPerformSeatInfo fPerformSeatInfo = new FPerformSeatInfo();
                                                     fPerformSeatInfo.setSeatPlanId(seatPlan.getSeatPlanId());
                                                     fPerformSeatInfo.setSeatPlanName(seatPlan.getSeatPlanName());
@@ -242,6 +247,7 @@ public class PullDataFromFWDJob {
                                                         fPerformSeatInfo.setSaleTime(DateUtils.getTimeFromStr(seatPlan.getSaleTime()));
                                                     }
                                                     fPerformSeatInfo.setProjectId(project.getProjectId());
+                                                    fPerformSeatInfo.setSoldOut(seatMap.get(seatPlan.getSeatPlanId()) ? 1 : 0);
                                                 }
                                             }
                                         }
@@ -260,6 +266,7 @@ public class PullDataFromFWDJob {
             CompletableFuture[] futureArr = futureList.toArray(futureList.toArray(new CompletableFuture[0]));
             CompletableFuture.allOf(futureArr).join();
         }
+    }
 
     private Map<Long, Boolean> getSeatMap(List<Long> seatPlanIds) {
         List<SeatPlanStatus> statusFromHttp = performServiceHttp.getSeatPlanStatusFromHttp(seatPlanIds);

script/src/main/java/com/xiang/xservice/fwd/server/FwdOuterController.java

@@ -1,5 +1,6 @@
 package com.xiang.xservice.fwd.server;
 
+import com.xiang.xservice.basic.common.req.BaseRequest;
 import com.xiang.xservice.basic.common.resp.Result;
 import com.xiang.xservice.fwd.entity.resp.http.perform.Perform;
 import com.xiang.xservice.fwd.entity.resp.http.perform.ProjectsResp;
@@ -30,7 +31,7 @@ public class FwdOuterController {
 
     @GetMapping("/getShowProjects")
     public Result<ProjectsResp> getShowProjects() {
-        ProjectsResp showProjectsFromHttp = performServiceHttp.getShowProjectsFromHttp();
+        ProjectsResp showProjectsFromHttp = performServiceHttp.getShowProjectsFromHttp(new BaseRequest());
         return Result.success("查询成功!", showProjectsFromHttp);
     }
 

script/src/main/java/com/xiang/xservice/fwd/service/IPerformServiceHttp.java

@@ -1,5 +1,6 @@
 package com.xiang.xservice.fwd.service;
 
+import com.xiang.xservice.basic.common.req.BaseRequest;
 import com.xiang.xservice.fwd.entity.req.ProjectOrderCreateReq;
 import com.xiang.xservice.fwd.entity.resp.http.perform.Perform;
 import com.xiang.xservice.fwd.entity.resp.http.perform.ProjectInfoResp;
@@ -15,7 +16,7 @@ public interface IPerformServiceHttp {
      * http请求查询全部演出列表
      * @return ProjectsResp
      */
-    ProjectsResp getShowProjectsFromHttp();
+    ProjectsResp getShowProjectsFromHttp(BaseRequest request);
 
     /**
      * http请求根据projectId查询演出详情

script/src/main/java/com/xiang/xservice/fwd/service/impl/PerformServiceHttpServiceImpl.java

@@ -4,6 +4,7 @@ import com.alibaba.fastjson.JSON;
 import com.alibaba.fastjson.JSONObject;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
+import com.xiang.xservice.basic.common.req.BaseRequest;
 import com.xiang.xservice.basic.utils.HttpUtils;
 import com.xiang.xservice.basic.utils.JsonUtils;
 import com.xiang.xservice.fwd.constants.CodeConstants;
@@ -31,13 +32,19 @@ import java.util.Objects;
 public class PerformServiceHttpServiceImpl implements IPerformServiceHttp {
 
     @Override
-    public ProjectsResp getShowProjectsFromHttp() {
+    public ProjectsResp getShowProjectsFromHttp(BaseRequest request) {
-        int pageNum = 1;
-        int pageSize = 9999;
         Map<String, String> params = Maps.newHashMap();
         params.put("projectModuleId", "7");
-        params.put("pageNum", String.valueOf(pageNum));
+        if (Objects.isNull(request.getCurrent())) {
-        params.put("pageSize", String.valueOf(pageSize));
+            params.put("pageNum", String.valueOf(1));
+        } else {
+            params.put("pageNum", String.valueOf(request.getCurrent()));
+        }
+        if (Objects.isNull(request.getPageSize())) {
+            params.put("pageSize", String.valueOf(10));
+        } else {
+            params.put("pageSize", String.valueOf(request.getPageSize()));
+        }
         String respStr = HttpUtils.doGet(UrlConstants.PROJECTS_URL, buildFWDHeaders(null), params);
         if (StringUtils.isBlank(respStr)) {
             return new ProjectsResp();

script/src/main/java/com/xiang/xservice/fwd/service/impl/PerformServiceImpl.java

@@ -62,6 +62,9 @@ public class PerformServiceImpl implements IPerformService {
             return Boolean.FALSE;
         }
         for (FPerformSeatInfo fPerformSeatInfo : seatInfoByProjectId) {
+            if (1 == fPerformSeatInfo.getSoldOut()) {
+                continue;
+            }
             ProjectOrderCreateReq projectOrderCreateReq = new ProjectOrderCreateReq();
             projectOrderCreateReq.setDeliveryType(1);
             projectOrderCreateReq.setContactName("朱吉祥");