perf:爬虫数据拉取优化
This commit is contained in:
Zhujx
@@ -2,6 +2,7 @@ package com.xiang.xservice.fwd.schedule;
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.collect.Maps;
|
||||
import com.xiang.xservice.basic.common.req.BaseRequest;
|
||||
import com.xiang.xservice.basic.config.MyThreadFactory;
|
||||
import com.xiang.xservice.basic.utils.DateUtils;
|
||||
import com.xiang.xservice.fwd.entity.pojo.FPerformProjectInfo;
|
||||
@@ -69,15 +70,24 @@ public class PullDataFromFWDJob {
|
||||
@Scheduled(cron = "0 0 1 1/1 * ?")
|
||||
@PostMapping("/pullProjectsDataJob")
|
||||
public void pullProjectsDataJob() {
|
||||
ProjectsResp showProjectsFromHttp = performServiceHttp.getShowProjectsFromHttp();
|
||||
int i = 0;
|
||||
while (true) {
|
||||
i++;
|
||||
BaseRequest request = new BaseRequest();
|
||||
request.setCurrent(i);
|
||||
request.setPageSize(10 * i);
|
||||
ProjectsResp showProjectsFromHttp = performServiceHttp.getShowProjectsFromHttp(request);
|
||||
if (Objects.isNull(showProjectsFromHttp)) {
|
||||
return;
|
||||
}
|
||||
if (CollectionUtils.isEmpty(showProjectsFromHttp.getList())) {
|
||||
break;
|
||||
}
|
||||
List<ProjectList> projectList = showProjectsFromHttp.getList();
|
||||
|
||||
List<CompletableFuture> futureList = Lists.newArrayList();
|
||||
for (ProjectList project : projectList) {
|
||||
CompletableFuture future = CompletableFuture.runAsync(() -> {
|
||||
CompletableFuture<Void> future = CompletableFuture.runAsync(() -> {
|
||||
ProjectInfoResp projectInfoFromHttp = performServiceHttp.getProjectInfoFromHttp(project.getProjectId());
|
||||
Perform performsByProjectIdFromHttp = performServiceHttp.getPerformsByProjectIdFromHttp(project.getProjectId());
|
||||
if (Objects.nonNull(projectInfoFromHttp) && Objects.nonNull(performsByProjectIdFromHttp)) {
|
||||
@@ -123,9 +133,6 @@ public class PullDataFromFWDJob {
|
||||
if (!seatMap.containsKey(seatPlan.getSeatPlanId())) {
|
||||
continue;
|
||||
}
|
||||
if (!seatMap.get(seatPlan.getSeatPlanId())) {
|
||||
continue;
|
||||
}
|
||||
FPerformSeatInfo fPerformSeatInfo = new FPerformSeatInfo();
|
||||
fPerformSeatInfo.setSeatPlanId(seatPlan.getSeatPlanId());
|
||||
fPerformSeatInfo.setSeatPlanName(seatPlan.getSeatPlanName());
|
||||
@@ -152,6 +159,7 @@ public class PullDataFromFWDJob {
|
||||
fPerformSeatInfo.setSaleTime(DateUtils.getTimeFromStr(seatPlan.getSaleTime()));
|
||||
}
|
||||
fPerformSeatInfo.setProjectId(project.getProjectId());
|
||||
fPerformSeatInfo.setSoldOut(seatMap.get(seatPlan.getSeatPlanId()) ? 1 : 0);
|
||||
FPerformSeatInfo seatInfo = performSeatInfoMapper.getPerformSeatInfoBySeatIdAndPerformIdAndProjectId(fPerformSeatInfo.getSeatPlanId(), fPerformSeatInfo.getPerformId(), project.getProjectId());
|
||||
if (Objects.nonNull(seatInfo)) {
|
||||
fPerformSeatInfo.setId(seatInfo.getId());
|
||||
@@ -213,9 +221,6 @@ public class PullDataFromFWDJob {
|
||||
if (!seatMap.containsKey(seatPlan.getSeatPlanId())) {
|
||||
continue;
|
||||
}
|
||||
if (!seatMap.get(seatPlan.getSeatPlanId())) {
|
||||
continue;
|
||||
}
|
||||
FPerformSeatInfo fPerformSeatInfo = new FPerformSeatInfo();
|
||||
fPerformSeatInfo.setSeatPlanId(seatPlan.getSeatPlanId());
|
||||
fPerformSeatInfo.setSeatPlanName(seatPlan.getSeatPlanName());
|
||||
@@ -242,6 +247,7 @@ public class PullDataFromFWDJob {
|
||||
fPerformSeatInfo.setSaleTime(DateUtils.getTimeFromStr(seatPlan.getSaleTime()));
|
||||
}
|
||||
fPerformSeatInfo.setProjectId(project.getProjectId());
|
||||
fPerformSeatInfo.setSoldOut(seatMap.get(seatPlan.getSeatPlanId()) ? 1 : 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -260,6 +266,7 @@ public class PullDataFromFWDJob {
|
||||
CompletableFuture[] futureArr = futureList.toArray(futureList.toArray(new CompletableFuture[0]));
|
||||
CompletableFuture.allOf(futureArr).join();
|
||||
}
|
||||
}
|
||||
|
||||
private Map<Long, Boolean> getSeatMap(List<Long> seatPlanIds) {
|
||||
List<SeatPlanStatus> statusFromHttp = performServiceHttp.getSeatPlanStatusFromHttp(seatPlanIds);
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package com.xiang.xservice.fwd.server;
|
||||
|
||||
import com.xiang.xservice.basic.common.req.BaseRequest;
|
||||
import com.xiang.xservice.basic.common.resp.Result;
|
||||
import com.xiang.xservice.fwd.entity.resp.http.perform.Perform;
|
||||
import com.xiang.xservice.fwd.entity.resp.http.perform.ProjectsResp;
|
||||
@@ -30,7 +31,7 @@ public class FwdOuterController {
|
||||
|
||||
@GetMapping("/getShowProjects")
|
||||
public Result<ProjectsResp> getShowProjects() {
|
||||
ProjectsResp showProjectsFromHttp = performServiceHttp.getShowProjectsFromHttp();
|
||||
ProjectsResp showProjectsFromHttp = performServiceHttp.getShowProjectsFromHttp(new BaseRequest());
|
||||
return Result.success("查询成功!", showProjectsFromHttp);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
package com.xiang.xservice.fwd.service;
|
||||
|
||||
import com.xiang.xservice.basic.common.req.BaseRequest;
|
||||
import com.xiang.xservice.fwd.entity.req.ProjectOrderCreateReq;
|
||||
import com.xiang.xservice.fwd.entity.resp.http.perform.Perform;
|
||||
import com.xiang.xservice.fwd.entity.resp.http.perform.ProjectInfoResp;
|
||||
@@ -15,7 +16,7 @@ public interface IPerformServiceHttp {
|
||||
* http请求查询全部演出列表
|
||||
* @return ProjectsResp
|
||||
*/
|
||||
ProjectsResp getShowProjectsFromHttp();
|
||||
ProjectsResp getShowProjectsFromHttp(BaseRequest request);
|
||||
|
||||
/**
|
||||
* http请求根据projectId查询演出详情
|
||||
|
||||
@@ -4,6 +4,7 @@ import com.alibaba.fastjson.JSON;
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.collect.Maps;
|
||||
import com.xiang.xservice.basic.common.req.BaseRequest;
|
||||
import com.xiang.xservice.basic.utils.HttpUtils;
|
||||
import com.xiang.xservice.basic.utils.JsonUtils;
|
||||
import com.xiang.xservice.fwd.constants.CodeConstants;
|
||||
@@ -31,13 +32,19 @@ import java.util.Objects;
|
||||
public class PerformServiceHttpServiceImpl implements IPerformServiceHttp {
|
||||
|
||||
@Override
|
||||
public ProjectsResp getShowProjectsFromHttp() {
|
||||
int pageNum = 1;
|
||||
int pageSize = 9999;
|
||||
public ProjectsResp getShowProjectsFromHttp(BaseRequest request) {
|
||||
Map<String, String> params = Maps.newHashMap();
|
||||
params.put("projectModuleId", "7");
|
||||
params.put("pageNum", String.valueOf(pageNum));
|
||||
params.put("pageSize", String.valueOf(pageSize));
|
||||
if (Objects.isNull(request.getCurrent())) {
|
||||
params.put("pageNum", String.valueOf(1));
|
||||
} else {
|
||||
params.put("pageNum", String.valueOf(request.getCurrent()));
|
||||
}
|
||||
if (Objects.isNull(request.getPageSize())) {
|
||||
params.put("pageSize", String.valueOf(10));
|
||||
} else {
|
||||
params.put("pageSize", String.valueOf(request.getPageSize()));
|
||||
}
|
||||
String respStr = HttpUtils.doGet(UrlConstants.PROJECTS_URL, buildFWDHeaders(null), params);
|
||||
if (StringUtils.isBlank(respStr)) {
|
||||
return new ProjectsResp();
|
||||
|
||||
@@ -62,6 +62,9 @@ public class PerformServiceImpl implements IPerformService {
|
||||
return Boolean.FALSE;
|
||||
}
|
||||
for (FPerformSeatInfo fPerformSeatInfo : seatInfoByProjectId) {
|
||||
if (1 == fPerformSeatInfo.getSoldOut()) {
|
||||
continue;
|
||||
}
|
||||
ProjectOrderCreateReq projectOrderCreateReq = new ProjectOrderCreateReq();
|
||||
projectOrderCreateReq.setDeliveryType(1);
|
||||
projectOrderCreateReq.setContactName("朱吉祥");
|
||||
|
||||
Reference in New Issue
Block a user