vertoryao/dify-proxy-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: vertoryao:feature/update-dify
Branches Tags
vertoryao:feature/update-dify
vertoryao:feature/email
vertoryao:dev
..
compare: vertoryao:dev
Branches Tags
vertoryao:feature/update-dify
vertoryao:feature/email
vertoryao:dev

No commits in common. "feature/update-dify" and "dev" have entirely different histories.
feature/up ... dev

18 changed files with 74 additions and 891 deletions
Show Stats Expand all files Collapse all files

8
pom.xml
View File

@ -32,7 +32,6 @@
<mapstruct.version>1.6.2</mapstruct.version>
<fastjson.version>2.0.53</fastjson.version>
<docker-maven-plugin.version>0.44.0</docker-maven-plugin.version>
<poi-tl.version>1.12.2</poi-tl.version>
</properties>
<dependencies>
<!-- <dependency>-->
@ -132,13 +131,6 @@
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.deepoove</groupId>
<artifactId>poi-tl</artifactId>
<version>${poi-tl.version}</version>
</dependency>
<!-- <dependency>-->
<!-- <groupId>org.springframework.boot</groupId>-->
<!-- <artifactId>spring-boot-starter-data-r2dbc</artifactId>-->

555
src/main/java/com/zsc/edu/dify/common/util/RedisUtils.java
View File

@ -1,555 +0,0 @@
package com.zsc.edu.dify.common.util;
import jakarta.annotation.Resource;
import org.springframework.data.redis.core.Cursor;
import org.springframework.data.redis.core.RedisCallback;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.core.ScanOptions;
import org.springframework.stereotype.Component;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
/**
* @author harry_yao
*/
@Component
public class RedisUtils {
@Resource
private RedisTemplate<String, Object> redisTemplate;
/**
* 指定缓存失效时间
* @param key 键
* @param time 时间（秒）
* @return true / false
*/
public boolean expire(String key, long time) {
try {
if (time > 0) {
redisTemplate.expire(key, time, TimeUnit.SECONDS);
}
return true;
} catch (Exception e) {
e.printStackTrace();
return false;
}
}
/**
* 根据 key 获取过期时间
* @param key 键
* @return
*/
public long getExpire(String key) {
return redisTemplate.getExpire(key, TimeUnit.SECONDS);
}
/**
* 判断 key 是否存在
* @param key 键
* @return true / false
*/
public boolean hasKey(String key) {
try {
return redisTemplate.hasKey(key);
} catch (Exception e) {
e.printStackTrace();
return false;
}
}
/**
* 删除缓存
* @SuppressWarnings("unchecked") 忽略类型转换警告
* @param key 键（一个或者多个）
*/
@SuppressWarnings("unchecked")
public void del(String... key) {
if (key != null && key.length > 0) {
if (key.length == 1) {
redisTemplate.delete(key[0]);
} else {
// 传入一个 Collection<String> 集合
// redisTemplate.delete(CollectionUtils.arrayToList(key));
redisTemplate.delete(Arrays.stream(key).collect(Collectors.toList()));
}
}
}
// ============================== String ==============================
/**
* 普通缓存获取
* @param key 键
* @return 值
*/
public Object get(String key) {
return key == null ? null : redisTemplate.opsForValue().get(key);
}
/**
* 普通缓存放入
* @param key 键
* @param value 值
* @return true / false
*/
public boolean set(String key, Object value) {
try {
redisTemplate.opsForValue().set(key, value);
return true;
} catch (Exception e) {
e.printStackTrace();
return false;
}
}
/**
* 普通缓存放入并设置时间
* @param key 键
* @param value 值
* @param time 时间（秒），如果 time < 0 则设置无限时间
* @return true / false
*/
public boolean set(String key, Object value, long time) {
try {
if (time > 0) {
redisTemplate.opsForValue().set(key, value, time, TimeUnit.SECONDS);
} else {
set(key, value);
}
return true;
} catch (Exception e) {
e.printStackTrace();
return false;
}
}
/**
* 递增
* @param key 键
* @param delta 递增大小
* @return
*/
public long incr(String key, long delta) {
if (delta < 0) {
throw new RuntimeException("递增因子必须大于 0");
}
return redisTemplate.opsForValue().increment(key, delta);
}
/**
* 递减
* @param key 键
* @param delta 递减大小
* @return
*/
public long decr(String key, long delta) {
if (delta < 0) {
throw new RuntimeException("递减因子必须大于 0");
}
return redisTemplate.opsForValue().increment(key, delta);
}
// ============================== Map ==============================
/**
* HashGet
* @param key 键（no null）
* @param item 项（no null）
* @return 值
*/
public Object hget(String key, String item) {
return redisTemplate.opsForHash().get(key, item);
}
/**
* 获取 key 对应的 map
* @param key 键（no null）
* @return 对应的多个键值
*/
public Map<Object, Object> hmget(String key) {
return redisTemplate.opsForHash().entries(key);
}
/**
* HashSet
* @param key 键
* @param map 值
* @return true / false
*/
public boolean hmset(String key, Map<Object, Object> map) {
try {
redisTemplate.opsForHash().putAll(key, map);
return true;
} catch (Exception e) {
e.printStackTrace();
return false;
}
}
/**
* HashSet 并设置时间
* @param key 键
* @param map 值
* @param time 时间
* @return true / false
*/
public boolean hmset(String key, Map<Object, Object> map, long time) {
try {
redisTemplate.opsForHash().putAll(key, map);
if (time > 0) {
expire(key, time);
}
return true;
} catch (Exception e) {
e.printStackTrace();
return false;
}
}
/**
* 向一张 Hash表 中放入数据，如不存在则创建
* @param key 键
* @param item 项
* @param value 值
* @return true / false
*/
public boolean hset(String key, String item, Object value) {
try {
redisTemplate.opsForHash().put(key, item, value);
return true;
} catch (Exception e) {
e.printStackTrace();
return false;
}
}
/**
* 向一张 Hash表 中放入数据，并设置时间，如不存在则创建
* @param key 键
* @param item 项
* @param value 值
* @param time 时间（如果原来的 Hash表 设置了时间，这里会覆盖）
* @return true / false
*/
public boolean hset(String key, String item, Object value, long time) {
try {
redisTemplate.opsForHash().put(key, item, value);
if (time > 0) {
expire(key, time);
}
return true;
} catch (Exception e) {
e.printStackTrace();
return false;
}
}
/**
* 删除 Hash表 中的值
* @param key 键
* @param item 项（可以多个，no null）
*/
public void hdel(String key, Object... item) {
redisTemplate.opsForHash().delete(key, item);
}
/**
* 判断 Hash表 中是否有该键的值
* @param key 键（no null）
* @param item 值（no null）
* @return true / false
*/
public boolean hHasKey(String key, String item) {
return redisTemplate.opsForHash().hasKey(key, item);
}
/**
* Hash递增，如果不存在则创建一个，并把新增的值返回
* @param key 键
* @param item 项
* @param by 递增大小 > 0
* @return
*/
public Double hincr(String key, String item, Double by) {
return redisTemplate.opsForHash().increment(key, item, by);
}
/**
* Hash递减
* @param key 键
* @param item 项
* @param by 递减大小
* @return
*/
public Double hdecr(String key, String item, Double by) {
return redisTemplate.opsForHash().increment(key, item, -by);
}
// ============================== Set ==============================
/**
* 根据 key 获取 set 中的所有值
* @param key 键
* @return 值
*/
public Set<Object> sGet(String key) {
try {
return redisTemplate.opsForSet().members(key);
} catch (Exception e) {
e.printStackTrace();
return null;
}
}
/**
* 从键为 key 的 set 中，根据 value 查询是否存在
* @param key 键
* @param value 值
* @return true / false
*/
public boolean sHasKey(String key, Object value) {
try {
return redisTemplate.opsForSet().isMember(key, value);
} catch (Exception e) {
e.printStackTrace();
return false;
}
}
/**
* 将数据放入 set缓存
* @param key 键值
* @param values 值（可以多个）
* @return 成功个数
*/
public long sSet(String key, Object... values) {
try {
return redisTemplate.opsForSet().add(key, values);
} catch (Exception e) {
e.printStackTrace();
return 0;
}
}
/**
* 将数据放入 set缓存，并设置时间
* @param key 键
* @param time 时间
* @param values 值（可以多个）
* @return 成功放入个数
*/
public long sSett(String key, long time, Object... values) {
try {
long count = redisTemplate.opsForSet().add(key, values);
if (time > 0) {
expire(key, time);
}
return count;
} catch (Exception e) {
e.printStackTrace();
return 0;
}
}
/**
* 获取 set缓存的长度
* @param key 键
* @return 长度
*/
public long sGetSetSize(String key) {
try {
return redisTemplate.opsForSet().size(key);
} catch (Exception e) {
e.printStackTrace();
return 0;
}
}
/**
* 移除 set缓存中，值为 value 的
* @param key 键
* @param values 值
* @return 成功移除个数
*/
public long setRemove(String key, Object... values) {
try {
return redisTemplate.opsForSet().remove(key, values);
} catch (Exception e) {
e.printStackTrace();
return 0;
}
}
// ============================== List ==============================
/**
* 获取 list缓存的内容
* @param key 键
* @param start 开始
* @param end 结束（0 到 -1 代表所有值）
* @return
*/
public List<Object> lGet(String key, long start, long end) {
try {
return redisTemplate.opsForList().range(key, start, end);
} catch (Exception e) {
e.printStackTrace();
return null;
}
}
/**
* 获取 list缓存的长度
* @param key 键
* @return 长度
*/
public long lGetListSize(String key) {
try {
return redisTemplate.opsForList().size(key);
} catch (Exception e) {
e.printStackTrace();
return 0;
}
}
/**
* 根据索引 index 获取键为 key 的 list 中的元素
* @param key 键
* @param index 索引
* 当 index >= 0 时 {0:表头, 1:第二个元素}
* 当 index < 0 时 {-1:表尾, -2:倒数第二个元素}
* @return 值
*/
public Object lGetIndex(String key, long index) {
try {
return redisTemplate.opsForList().index(key, index);
} catch (Exception e) {
e.printStackTrace();
return null;
}
}
/**
* 将值 value 插入键为 key 的 list 中，如果 list 不存在则创建空 list
* @param key 键
* @param value 值
* @return true / false
*/
public boolean lSet(String key, Object value) {
try {
redisTemplate.opsForList().rightPush(key, value);
return true;
} catch (Exception e) {
e.printStackTrace();
return false;
}
}
/**
* 将值 value 插入键为 key 的 list 中，并设置时间
* @param key 键
* @param value 值
* @param time 时间
* @return true / false
*/
public boolean lSet(String key, Object value, long time) {
try {
redisTemplate.opsForList().rightPush(key, value);
if (time > 0) {
expire(key, time);
}
return true;
} catch (Exception e) {
e.printStackTrace();
return false;
}
}
/**
* 将 values 插入键为 key 的 list 中
* @param key 键
* @param values 值
* @return true / false
*/
public boolean lSetList(String key, List<Object> values) {
try {
redisTemplate.opsForList().rightPushAll(key, values);
return true;
} catch (Exception e) {
e.printStackTrace();
return false;
}
}
/**
* 将 values 插入键为 key 的 list 中，并设置时间
* @param key 键
* @param values 值
* @param time 时间
* @return true / false
*/
public boolean lSetList(String key, List<Object> values, long time) {
try {
redisTemplate.opsForList().rightPushAll(key, values);
if (time > 0) {
expire(key, time);
}
return true;
} catch (Exception e) {
e.printStackTrace();
return false;
}
}
/**
* 根据索引 index 修改键为 key 的值
* @param key 键
* @param index 索引
* @param value 值
* @return true / false
*/
public boolean lUpdateIndex(String key, long index, Object value) {
try {
redisTemplate.opsForList().set(key, index, value);
return true;
} catch (Exception e) {
e.printStackTrace();
return false;
}
}
/**
* 在键为 key 的 list 中删除值为 value 的元素
* @param key 键
* @param count 如果 count == 0 则删除 list 中所有值为 value 的元素
* 如果 count > 0 则删除 list 中最左边那个值为 value 的元素
* 如果 count < 0 则删除 list 中最右边那个值为 value 的元素
* @param value
* @return
*/
public long lRemove(String key, long count, Object value) {
try {
return redisTemplate.opsForList().remove(key, count, value);
} catch (Exception e) {
e.printStackTrace();
return 0;
}
}
public Set<String> scan(String matchKey) {
ScanOptions options = ScanOptions.scanOptions().match("*" + matchKey + "*").count(3000).build();
return redisTemplate.execute((RedisCallback<Set<String>>) connection -> {
Set<String> keysTmp = new HashSet<>();
Cursor<byte[]> cursor = connection.scan(options);
while (cursor.hasNext()) {
keysTmp.add(new String(cursor.next()));
}
return keysTmp;
});
}
}

25
src/main/java/com/zsc/edu/dify/framework/event/EventListener.java
View File

@ -1,25 +0,0 @@
package com.zsc.edu.dify.framework.event;
import lombok.RequiredArgsConstructor;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Component;
/**
* @description 事件监听者
* @author vivid
*
* */
@Component
@RequiredArgsConstructor
public class EventListener {
private final SpiderPollingService spiderPollingService;
/**
* 监听爬虫启动事件
* */
@org.springframework.context.event.EventListener
public void handleStartPollingEvent(SpiderStartPollingEvent event) {
spiderPollingService.startPolling(event.getSpiderId(), event.getEmail());
}
}

73
src/main/java/com/zsc/edu/dify/framework/event/SpiderPollingService.java
View File

@ -1,73 +0,0 @@
package com.zsc.edu.dify.framework.event;
import com.alibaba.fastjson.JSONObject;
import com.zsc.edu.dify.common.util.RedisUtils;
import com.zsc.edu.dify.framework.message.email.EmailSender;
import com.zsc.edu.dify.framework.spider.SpiderConfig;
import com.zsc.edu.dify.framework.spider.SpiderProperty;
import com.zsc.edu.dify.modules.dify.service.SpiderService;
import com.zsc.edu.dify.modules.message.entity.Notice;
import com.zsc.edu.dify.modules.message.entity.NoticeType;
import jakarta.annotation.PostConstruct;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import java.util.HashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
/**
* @description 轮询查看爬虫结果
* @author vivid
* */
@Service
@RequiredArgsConstructor
public class SpiderPollingService {
private ScheduledExecutorService scheduler;
private final RedisUtils redisUtils;
private final SpiderService spiderService;
private final EmailSender emailSender;
private final SpiderConfig spiderConfig;
private static final HashMap<String, SpiderProperty> PROPERTY_MAP = new HashMap<>();
@PostConstruct
public void init() {
for (SpiderProperty property : spiderConfig.getConfigs()) {
PROPERTY_MAP.put(property.getId(), property);
}
}
public void startPolling(String spiderId, String email) {
scheduler = Executors.newSingleThreadScheduledExecutor();
scheduler.scheduleAtFixedRate(() -> {
executeBusinessLogic(spiderId, email);
}, 400, 3000, TimeUnit.MILLISECONDS);
}
private void executeBusinessLogic(String spiderId, String email) {
JSONObject jsonObject = spiderService.status(spiderId);
JSONObject data = jsonObject.getJSONObject("data");
if (!data.getBoolean("is_running")) {
//获取文件
String url = data.getString("download_url");
//添加爬虫请求头，并且将前缀去掉
String path = PROPERTY_MAP.get(spiderId).getUrl().replace("/api/v1", "") + url;
//发送邮件给用户
emailSender.send(email, new Notice(NoticeType.MESSAGE, false, true, false, false, "文件已到达", path, 0L));
//将整个对象写入redis
redisUtils.set(spiderId + ":result", jsonObject, 60 * 60 * 24);
//停止任务
scheduler.shutdown();
}
}
public void stopPolling() {
scheduler.shutdown();
}
}

23
src/main/java/com/zsc/edu/dify/framework/event/SpiderStartPollingEvent.java
View File

@ -1,23 +0,0 @@
package com.zsc.edu.dify.framework.event;
import lombok.Getter;
import org.springframework.context.ApplicationEvent;
/**
*
* @description 定义爬虫启动的事件
* @author vivid
*
* */
@Getter
public class SpiderStartPollingEvent extends ApplicationEvent {
private String spiderId;
private String email;
public SpiderStartPollingEvent(Object source, String spiderId, String email) {
super(source);
this.spiderId = spiderId;
this.email = email;
}
}

35
src/main/java/com/zsc/edu/dify/framework/generateDocx/generateDocx.java
View File

@ -1,35 +0,0 @@
package com.zsc.edu.dify.framework.generateDocx;
import com.deepoove.poi.XWPFTemplate;
import com.zsc.edu.dify.framework.generateDocx.styleConfig.styleDataWrapper;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
/**
* @author Ding
*/
public class generateDocx {
public String generateSpider4(List<Map<String, Object>> rawData) {
UUID uuid = UUID.randomUUID();
String target = "storage/temp/" + uuid + ".docx";
String resource = "src/main/resources/docxTemplate/model.docx";
String back = uuid + ".docx";
Map<String, Object> data = new HashMap<>();
List<Map<String, Object>> styledData = styleDataWrapper.wrap(rawData);
data.put("context", styledData);
try (XWPFTemplate template = XWPFTemplate.compile(resource).render(data)) {
template.writeToFile(target);
return back;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}

32
src/main/java/com/zsc/edu/dify/framework/generateDocx/styleConfig/labelStyles.java
View File

@ -1,32 +0,0 @@
package com.zsc.edu.dify.framework.generateDocx.styleConfig;
import com.deepoove.poi.data.style.Style;
import lombok.Data;
/**
* @description 创建特定样式
* @author Ding
*/
@Data
public class labelStyles {
private final Style titleStyle;
private final Style textStyle;
// 私有构造函数，强制通过工厂方法创建
private labelStyles(Style titleStyle, Style textStyle) {
this.titleStyle = titleStyle;
this.textStyle = textStyle;
}
// 工厂方法：初始化默认样式
public static labelStyles createDefault() {
Style titleStyle = new Style();
titleStyle.setBold(true);
titleStyle.setFontSize(20);
Style textStyle = new Style();
textStyle.setFontSize(11);
return new labelStyles(titleStyle, textStyle);
}
}

43
src/main/java/com/zsc/edu/dify/framework/generateDocx/styleConfig/styleDataWrapper.java
View File

@ -1,43 +0,0 @@
package com.zsc.edu.dify.framework.generateDocx.styleConfig;
import com.deepoove.poi.data.ParagraphRenderData;
import com.deepoove.poi.data.Paragraphs;
import com.deepoove.poi.data.TextRenderData;
import com.deepoove.poi.data.style.Style;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
* @description 用于将样式与字段绑定
* @author Ding
*/
public class styleDataWrapper {
static labelStyles styles = labelStyles.createDefault();
// 定义标签与样式映射规则。标题用一个样式，其他标签用一个样式
private static final Map<String, Style> STYLE_MAPPING = Map.of(
"title", styles.getTitleStyle(),
"*", styles.getTextStyle()
);
public static List<Map<String, Object>> wrap(List<Map<String, Object>> rawData) {
return rawData.stream()
.map(styleDataWrapper::processMap)
.collect(Collectors.toList());
}
// 为每个字段绑定特定的样式
private static Map<String, Object> processMap(Map<String, Object> rawMap) {
Map<String, Object> styledMap = new HashMap<>();
rawMap.forEach((key, value) -> {
Style style = STYLE_MAPPING.getOrDefault(key, STYLE_MAPPING.get("*"));
TextRenderData textRenderData = new TextRenderData(value.toString(), style);
styledMap.put(key, textRenderData);
});
return styledMap;
}
}

2
src/main/java/com/zsc/edu/dify/framework/message/email/EmailSender.java
View File

@ -42,7 +42,7 @@ public class EmailSender {
@Async
public void send(String email, Notice notice) {
if (!StringUtils.hasText(email)) {
if (StringUtils.hasText(email)) {
return;
}
InternetAddress to;

34
src/main/java/com/zsc/edu/dify/framework/redis/RedisConfig.java
View File

@ -1,34 +0,0 @@
package com.zsc.edu.dify.framework.redis;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.serializer.GenericJackson2JsonRedisSerializer;
import org.springframework.data.redis.serializer.StringRedisSerializer;
/**
* @description redis配置类
* @author vivid
* */
@Configuration
public class RedisConfig {
// 自定义 RedisTemplate Bean 配置
@Bean
public RedisTemplate<String, Object> redisTemplate(RedisConnectionFactory factory) {
RedisTemplate<String, Object> template = new RedisTemplate<>();
template.setConnectionFactory(factory);
// 设置键的序列化方式为 String
template.setKeySerializer(new StringRedisSerializer());
template.setHashKeySerializer(new StringRedisSerializer());
// 设置值的序列化方式为 JSON
template.setValueSerializer(new GenericJackson2JsonRedisSerializer());
template.setHashValueSerializer(new GenericJackson2JsonRedisSerializer());
template.afterPropertiesSet();
return template;
}
}

62
src/main/java/com/zsc/edu/dify/modules/dify/controller/Spider3Controller.java Normal file
View File

@ -0,0 +1,62 @@
package com.zsc.edu.dify.modules.dify.controller;
import com.alibaba.fastjson.JSONObject;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.zsc.edu.dify.modules.dify.dto.SpiderDto;
import jakarta.annotation.Resource;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.reactive.function.client.WebClient;
@RestController
@RequestMapping("/api/spider3")
public class Spider3Controller {
@Resource
private ObjectMapper objectMapper;
@Value("${spider3.url}")
private String SPIDER_URL;
@Value("${spider3.api-key}")
private String API_KEY;
@PostMapping("/run")
public JSONObject run() throws JsonProcessingException {
return WebClient.create(SPIDER_URL).post().uri("/start_crawl")
.contentType(MediaType.APPLICATION_JSON)
.retrieve()
.bodyToMono(JSONObject.class)
.block();
}
@PostMapping("/status")
public JSONObject status() {
return WebClient.create(SPIDER_URL).post().uri("/crawl_status")
.retrieve()
.bodyToMono(JSONObject.class)
.block();
}
@PostMapping("/logs")
public JSONObject logs() {
return WebClient.create(SPIDER_URL).post().uri("/logs")
.retrieve()
.bodyToMono(JSONObject.class)
.block();
}
@PostMapping("/stop")
public JSONObject stop() {
return WebClient.create(SPIDER_URL).post().uri("/stop_crawl")
.retrieve()
.bodyToMono(JSONObject.class)
.block();
}
}

22
src/main/java/com/zsc/edu/dify/modules/dify/controller/SpiderController.java
View File

@ -2,15 +2,10 @@ package com.zsc.edu.dify.modules.dify.controller;
import com.alibaba.fastjson.JSONObject;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.zsc.edu.dify.framework.event.SpiderPollingService;
import com.zsc.edu.dify.framework.event.SpiderStartPollingEvent;
import com.zsc.edu.dify.modules.dify.dto.SpiderDto;
import com.zsc.edu.dify.modules.dify.service.SpiderService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.web.bind.annotation.*;
/**
* @description: 自定义爬虫
* @author: yao
@ -22,23 +17,9 @@ public class SpiderController {
@Autowired
private SpiderService spiderService;
@Autowired
private ApplicationEventPublisher applicationEventPublisher;
@Autowired
private SpiderPollingService spiderPollingService;
@PostMapping("/run/{spiderId}")
public JSONObject run(@RequestBody(required = false) SpiderDto dto, @PathVariable String spiderId) throws JsonProcessingException {
JSONObject data = spiderService.isExistCache(spiderId, dto.getEmail());
if (data != null) {
// 如果缓存存在，直接返回，不执行 spiderService.run()
return data;
} else {
// 如果缓存不存在，正常执行，并且发布事件
applicationEventPublisher.publishEvent(new SpiderStartPollingEvent(this, spiderId, dto.getEmail()));
return spiderService.run(dto, spiderId);
}
return spiderService.run(dto, spiderId);
}
@PostMapping("/status/{spiderId}")
@ -53,7 +34,6 @@ public class SpiderController {
@PostMapping("/stop/{spiderId}")
public JSONObject stop(@PathVariable String spiderId) {
spiderPollingService.stopPolling();
return spiderService.stop(spiderId);
}

3
src/main/java/com/zsc/edu/dify/modules/dify/dto/SpiderDto.java
View File

@ -1,6 +1,5 @@
package com.zsc.edu.dify.modules.dify.dto;
import jakarta.validation.constraints.Email;
import jakarta.validation.constraints.NotBlank;
import lombok.Data;
@ -13,6 +12,4 @@ public class SpiderDto {
private String keyword;
private Integer[] site_codes;
private String llm_api_key;
@Email
private String email;
}

30
src/main/java/com/zsc/edu/dify/modules/dify/service/Impl/SpiderServiceImpl.java
View File

@ -3,15 +3,10 @@ package com.zsc.edu.dify.modules.dify.service.Impl;
import com.alibaba.fastjson.JSONObject;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.zsc.edu.dify.common.util.RedisUtils;
import com.zsc.edu.dify.framework.event.SpiderPollingService;
import com.zsc.edu.dify.framework.message.email.EmailSender;
import com.zsc.edu.dify.framework.spider.SpiderConfig;
import com.zsc.edu.dify.framework.spider.SpiderProperty;
import com.zsc.edu.dify.modules.dify.dto.SpiderDto;
import com.zsc.edu.dify.modules.dify.service.SpiderService;
import com.zsc.edu.dify.modules.message.entity.Notice;
import com.zsc.edu.dify.modules.message.entity.NoticeType;
import jakarta.annotation.PostConstruct;
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils;
@ -31,12 +26,6 @@ public class SpiderServiceImpl implements SpiderService {
private static final HashMap<String, SpiderProperty> PROPERTY_MAP = new HashMap<>();
private final RedisUtils redisUtils;
private final EmailSender emailSender;
@PostConstruct
public void init() {
for (SpiderProperty property : spiderConfig.getConfigs()) {
@ -85,23 +74,4 @@ public class SpiderServiceImpl implements SpiderService {
.bodyToMono(JSONObject.class)
.block();
}
@Override
public JSONObject isExistCache(String spiderId, String email) {
JSONObject result = (JSONObject) redisUtils.get(spiderId + ":result");
if (result != null) {
//获取文件路径
String url = result.getJSONObject("data").getString("download_url");
//拼接爬虫路径
String path = PROPERTY_MAP.get(spiderId).getUrl().replace("/api/v1", "") + url;
//发送邮件给用户
emailSender.send(email, new Notice(NoticeType.MESSAGE, false, true, false, false, "文件已到达", path, 0L));
//构造响应体
JSONObject response = new JSONObject();
response.put("msg", "爬虫任务已接受");
response.put("code", "0");
return response;
}
return null;
}
}

3
src/main/java/com/zsc/edu/dify/modules/dify/service/SpiderService.java
View File

@ -13,7 +13,4 @@ public interface SpiderService {
JSONObject logs(String spiderId);
JSONObject stop(String spiderId);
JSONObject isExistCache(String spiderId, String email);
}

15
src/main/resources/application-dev.yml
View File

@ -83,6 +83,16 @@ dify:
dataset:
api-key: dataset-kN5WTJ8jR877YfN1A34JceVg # 请替换为实际的知识库api-key, 若不需要调用知识库可不填
quanguo: &quanguo
spider-id: ${QUANGUO_ID:77c068fd-d5b6-4c33-97d8-db5511a09b26}
url: http://${QUANGUO_HOST:47.112.173.8:6806/api/v1}
api-key: ${QUANGUO_API_KEY:77c068fd-d5b6-4c33-97d8-db5511a09b26}
spider3: &spider3
spider-id: ${SPIDER3_ID:f3a7b9c2-5d6e-4b8f-9c1a-2d3e4f5a6b7c}
url: http://${SPIDER3_HOST:47.112.173.8:6257/api/v1}
api-key:
spider:
configs:
# 全国爬虫
@ -93,10 +103,5 @@ spider:
- id: ${SPIDER3_ID:f3a7b9c2-5d6e-4b8f-9c1a-2d3e4f5a6b7c}
url: http://${SPIDER3_HOST:47.112.173.8:6257/api/v1}
api-key:
# 爬虫D
- id: ${SPIDERD_ID:e1f90005-8c42-4736-a598-9Bebe49c8e12}
url: http://${SPIDER3_HOST:47.112.173.8:6258/api/v1}
api-key:
# - *quanguo
# - *spider3
# - *spiderD

BIN
src/main/resources/docxTemplate/model.docx
View File

Binary file not shown.

0
src/main/resources/docxTemplate/resultDocx.docx
View File