feat(dify): 合并爬虫1和3
This commit is contained in:
parent
1de71042f2
commit
736673ede8
@ -0,0 +1,12 @@
|
|||||||
|
package com.zsc.edu.dify.common.strategy;
|
||||||
|
|
||||||
|
import com.alibaba.fastjson.JSONObject;
|
||||||
|
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||||
|
import com.zsc.edu.dify.modules.dify.dto.SpiderDto;
|
||||||
|
|
||||||
|
public interface SpiderStrategy {
|
||||||
|
|
||||||
|
JSONObject run(SpiderDto dto) throws JsonProcessingException;
|
||||||
|
|
||||||
|
String getUrl();
|
||||||
|
}
|
@ -0,0 +1,35 @@
|
|||||||
|
package com.zsc.edu.dify.common.strategy.factory;
|
||||||
|
|
||||||
|
import com.zsc.edu.dify.common.strategy.SpiderStrategy;
|
||||||
|
import com.zsc.edu.dify.exception.NotExistException;
|
||||||
|
import jakarta.annotation.PostConstruct;
|
||||||
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
@Component
|
||||||
|
public class SpiderStrategyFactory {
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
private List<SpiderStrategy> spiderUrlStrategyList;
|
||||||
|
|
||||||
|
private final Map<String, SpiderStrategy> spiderUrlStrategyMap = new HashMap<>();
|
||||||
|
|
||||||
|
@PostConstruct
|
||||||
|
public void init() {
|
||||||
|
spiderUrlStrategyMap.put("1", spiderUrlStrategyList.get(0));
|
||||||
|
spiderUrlStrategyMap.put("3", spiderUrlStrategyList.get(1));
|
||||||
|
}
|
||||||
|
|
||||||
|
public SpiderStrategy getSpiderStrategy(String strategyName) {
|
||||||
|
SpiderStrategy spiderUrlStrategy = spiderUrlStrategyMap.get(strategyName);
|
||||||
|
if (spiderUrlStrategy == null) {
|
||||||
|
throw new NotExistException(SpiderStrategy.class);
|
||||||
|
}
|
||||||
|
return spiderUrlStrategy;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
@ -0,0 +1,43 @@
|
|||||||
|
package com.zsc.edu.dify.common.strategy.impl;
|
||||||
|
|
||||||
|
import com.alibaba.fastjson.JSONObject;
|
||||||
|
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.zsc.edu.dify.common.strategy.SpiderStrategy;
|
||||||
|
import com.zsc.edu.dify.modules.dify.dto.SpiderDto;
|
||||||
|
import jakarta.annotation.Resource;
|
||||||
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
|
import org.springframework.http.MediaType;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import org.springframework.web.reactive.function.client.WebClient;
|
||||||
|
|
||||||
|
@Component("spider1")
|
||||||
|
public class Spider1StrategyImpl implements SpiderStrategy {
|
||||||
|
|
||||||
|
@Value("${quanguo.url}")
|
||||||
|
private String SPIDER_URL;
|
||||||
|
|
||||||
|
@Value("${quanguo.api-key}")
|
||||||
|
private String API_KEY;
|
||||||
|
|
||||||
|
@Resource
|
||||||
|
private ObjectMapper objectMapper;
|
||||||
|
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public JSONObject run(SpiderDto dto) throws JsonProcessingException {
|
||||||
|
dto.setLlm_api_key(API_KEY);
|
||||||
|
String body = objectMapper.writeValueAsString(dto);
|
||||||
|
return WebClient.create(SPIDER_URL).post().uri("/start_crawl")
|
||||||
|
.contentType(MediaType.APPLICATION_JSON)
|
||||||
|
.bodyValue(body)
|
||||||
|
.retrieve()
|
||||||
|
.bodyToMono(JSONObject.class)
|
||||||
|
.block();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getUrl() {
|
||||||
|
return SPIDER_URL;
|
||||||
|
}
|
||||||
|
}
|
@ -0,0 +1,29 @@
|
|||||||
|
package com.zsc.edu.dify.common.strategy.impl;
|
||||||
|
|
||||||
|
import com.alibaba.fastjson.JSONObject;
|
||||||
|
import com.zsc.edu.dify.common.strategy.SpiderStrategy;
|
||||||
|
import com.zsc.edu.dify.modules.dify.dto.SpiderDto;
|
||||||
|
import org.springframework.beans.factory.annotation.Value;
|
||||||
|
import org.springframework.http.MediaType;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import org.springframework.web.reactive.function.client.WebClient;
|
||||||
|
|
||||||
|
@Component("spider3")
|
||||||
|
public class Spider3StrategyImpl implements SpiderStrategy {
|
||||||
|
@Value("${spider3.url}")
|
||||||
|
private String SPIDER_URL;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public JSONObject run(SpiderDto dto) {
|
||||||
|
return WebClient.create(SPIDER_URL).post().uri("/start_crawl")
|
||||||
|
.contentType(MediaType.APPLICATION_JSON)
|
||||||
|
.retrieve()
|
||||||
|
.bodyToMono(JSONObject.class)
|
||||||
|
.block();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String getUrl() {
|
||||||
|
return SPIDER_URL;
|
||||||
|
}
|
||||||
|
}
|
@ -2,63 +2,51 @@ package com.zsc.edu.dify.modules.dify.controller;
|
|||||||
|
|
||||||
import com.alibaba.fastjson.JSONObject;
|
import com.alibaba.fastjson.JSONObject;
|
||||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.zsc.edu.dify.common.strategy.SpiderStrategy;
|
||||||
|
import com.zsc.edu.dify.common.strategy.factory.SpiderStrategyFactory;
|
||||||
import com.zsc.edu.dify.modules.dify.dto.SpiderDto;
|
import com.zsc.edu.dify.modules.dify.dto.SpiderDto;
|
||||||
import jakarta.annotation.Resource;
|
|
||||||
import org.springframework.beans.factory.annotation.Autowired;
|
import org.springframework.beans.factory.annotation.Autowired;
|
||||||
import org.springframework.beans.factory.annotation.Value;
|
|
||||||
import org.springframework.http.HttpHeaders;
|
|
||||||
import org.springframework.http.MediaType;
|
|
||||||
import org.springframework.security.access.prepost.PreAuthorize;
|
|
||||||
import org.springframework.web.bind.annotation.*;
|
import org.springframework.web.bind.annotation.*;
|
||||||
import org.springframework.web.reactive.function.client.WebClient;
|
import org.springframework.web.reactive.function.client.WebClient;
|
||||||
|
|
||||||
import java.util.Collections;
|
|
||||||
|
|
||||||
@RestController
|
@RestController
|
||||||
@RequestMapping("/api/spider")
|
@RequestMapping("/api/spider")
|
||||||
public class SpiderController {
|
public class SpiderController {
|
||||||
|
|
||||||
@Resource
|
@Autowired
|
||||||
private ObjectMapper objectMapper;
|
private SpiderStrategyFactory spiderStrategyFactory;
|
||||||
|
|
||||||
@Value("${quanguo.url}")
|
|
||||||
private String SPIDER_URL;
|
|
||||||
|
|
||||||
@Value("${quanguo.api-key}")
|
|
||||||
private String API_KEY;
|
|
||||||
|
|
||||||
@PostMapping("/run")
|
@PostMapping("/run")
|
||||||
public JSONObject run(@RequestBody SpiderDto dto) throws JsonProcessingException {
|
public JSONObject run(@RequestBody(required = false) SpiderDto dto, @RequestParam String spiderId) throws JsonProcessingException {
|
||||||
dto.setLlm_api_key(API_KEY);
|
SpiderStrategy spiderStrategy = spiderStrategyFactory.getSpiderStrategy(spiderId);
|
||||||
String body = objectMapper.writeValueAsString(dto);
|
return spiderStrategy.run(dto);
|
||||||
return WebClient.create(SPIDER_URL).post().uri("/start_crawl")
|
|
||||||
.contentType(MediaType.APPLICATION_JSON)
|
|
||||||
.bodyValue(body)
|
|
||||||
.retrieve()
|
|
||||||
.bodyToMono(JSONObject.class)
|
|
||||||
.block();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@PostMapping("/status")
|
@PostMapping("/status")
|
||||||
public JSONObject status() {
|
public JSONObject status(@RequestParam String spiderId) {
|
||||||
return WebClient.create(SPIDER_URL).post().uri("/crawl_status")
|
SpiderStrategy spiderStrategy = spiderStrategyFactory.getSpiderStrategy(spiderId);
|
||||||
|
String url = spiderStrategy.getUrl();
|
||||||
|
return WebClient.create(url).post().uri("/crawl_status")
|
||||||
.retrieve()
|
.retrieve()
|
||||||
.bodyToMono(JSONObject.class)
|
.bodyToMono(JSONObject.class)
|
||||||
.block();
|
.block();
|
||||||
}
|
}
|
||||||
|
|
||||||
@PostMapping("/logs")
|
@PostMapping("/logs")
|
||||||
public JSONObject logs() {
|
public JSONObject logs(@RequestParam String spiderId) {
|
||||||
return WebClient.create(SPIDER_URL).post().uri("/logs")
|
SpiderStrategy spiderStrategy = spiderStrategyFactory.getSpiderStrategy(spiderId);
|
||||||
|
String url = spiderStrategy.getUrl();
|
||||||
|
return WebClient.create(url).post().uri("/logs")
|
||||||
.retrieve()
|
.retrieve()
|
||||||
.bodyToMono(JSONObject.class)
|
.bodyToMono(JSONObject.class)
|
||||||
.block();
|
.block();
|
||||||
}
|
}
|
||||||
|
|
||||||
@PostMapping("/stop")
|
@PostMapping("/stop")
|
||||||
public JSONObject stop() {
|
public JSONObject stop(@RequestParam String spiderId) {
|
||||||
return WebClient.create(SPIDER_URL).post().uri("/stop_crawl")
|
SpiderStrategy spiderStrategy = spiderStrategyFactory.getSpiderStrategy(spiderId);
|
||||||
|
String url = spiderStrategy.getUrl();
|
||||||
|
return WebClient.create(url).post().uri("/stop_crawl")
|
||||||
.retrieve()
|
.retrieve()
|
||||||
.bodyToMono(JSONObject.class)
|
.bodyToMono(JSONObject.class)
|
||||||
.block();
|
.block();
|
||||||
|
Loading…
Reference in New Issue
Block a user