feat(generateDocx): 优化 generateSpider4 方法

- 添加返回值，返回生成文档的 UUID- 修改目标文件路径为动态生成的临时文件

pom.xml

@@ -32,6 +32,7 @@
 		<mapstruct.version>1.6.2</mapstruct.version>
 		<fastjson.version>2.0.53</fastjson.version>
 		<docker-maven-plugin.version>0.44.0</docker-maven-plugin.version>
+		<poi-tl.version>1.12.2</poi-tl.version>
 	</properties>
 	<dependencies>
 <!--		<dependency>-->
@@ -131,6 +132,13 @@
 			<scope>provided</scope>
 		</dependency>
 
+		<dependency>
+			<groupId>com.deepoove</groupId>
+			<artifactId>poi-tl</artifactId>
+			<version>${poi-tl.version}</version>
+		</dependency>
+
+
 <!--		<dependency>-->
 <!--			<groupId>org.springframework.boot</groupId>-->
 <!--			<artifactId>spring-boot-starter-data-r2dbc</artifactId>-->

src/main/java/com/zsc/edu/dify/common/util/RedisUtils.java

@@ -0,0 +1,555 @@
+package com.zsc.edu.dify.common.util;
+
+import jakarta.annotation.Resource;
+import org.springframework.data.redis.core.Cursor;
+import org.springframework.data.redis.core.RedisCallback;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.data.redis.core.ScanOptions;
+import org.springframework.stereotype.Component;
+
+import java.util.*;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+/**
+ * @author harry_yao
+ */
+@Component
+public class RedisUtils {
+
+    @Resource
+    private RedisTemplate<String, Object> redisTemplate;
+
+    /**
+     * 指定缓存失效时间
+     * @param key 键
+     * @param time 时间（秒）
+     * @return true / false
+     */
+    public boolean expire(String key, long time) {
+        try {
+            if (time > 0) {
+                redisTemplate.expire(key, time, TimeUnit.SECONDS);
+            }
+            return true;
+        } catch (Exception e) {
+            e.printStackTrace();
+            return false;
+        }
+    }
+
+    /**
+     * 根据 key 获取过期时间
+     * @param key 键
+     * @return
+     */
+    public long getExpire(String key) {
+        return redisTemplate.getExpire(key, TimeUnit.SECONDS);
+    }
+
+    /**
+     * 判断 key 是否存在
+     * @param key 键
+     * @return true / false
+     */
+    public boolean hasKey(String key) {
+        try {
+            return redisTemplate.hasKey(key);
+        } catch (Exception e) {
+            e.printStackTrace();
+            return false;
+        }
+    }
+
+    /**
+     * 删除缓存
+     * @SuppressWarnings("unchecked") 忽略类型转换警告
+     * @param key 键（一个或者多个）
+     */
+    @SuppressWarnings("unchecked")
+    public void del(String... key) {
+        if (key != null && key.length > 0) {
+            if (key.length == 1) {
+                redisTemplate.delete(key[0]);
+            } else {
+//                传入一个 Collection<String> 集合
+//                redisTemplate.delete(CollectionUtils.arrayToList(key));
+                redisTemplate.delete(Arrays.stream(key).collect(Collectors.toList()));
+            }
+        }
+    }
+
+//    ============================== String ==============================
+
+    /**
+     * 普通缓存获取
+     * @param key 键
+     * @return 值
+     */
+    public Object get(String key) {
+        return key == null ? null : redisTemplate.opsForValue().get(key);
+    }
+
+    /**
+     * 普通缓存放入
+     * @param key 键
+     * @param value 值
+     * @return true / false
+     */
+    public boolean set(String key, Object value) {
+        try {
+            redisTemplate.opsForValue().set(key, value);
+            return true;
+        } catch (Exception e) {
+            e.printStackTrace();
+            return false;
+        }
+    }
+
+    /**
+     * 普通缓存放入并设置时间
+     * @param key 键
+     * @param value 值
+     * @param time 时间（秒），如果 time < 0 则设置无限时间
+     * @return true / false
+     */
+    public boolean set(String key, Object value, long time) {
+        try {
+            if (time > 0) {
+                redisTemplate.opsForValue().set(key, value, time, TimeUnit.SECONDS);
+            } else {
+                set(key, value);
+            }
+            return true;
+        } catch (Exception e) {
+            e.printStackTrace();
+            return false;
+        }
+    }
+
+    /**
+     * 递增
+     * @param key 键
+     * @param delta 递增大小
+     * @return
+     */
+    public long incr(String key, long delta) {
+        if (delta < 0) {
+            throw new RuntimeException("递增因子必须大于 0");
+        }
+        return redisTemplate.opsForValue().increment(key, delta);
+    }
+
+    /**
+     * 递减
+     * @param key 键
+     * @param delta 递减大小
+     * @return
+     */
+    public long decr(String key, long delta) {
+        if (delta < 0) {
+            throw new RuntimeException("递减因子必须大于 0");
+        }
+        return redisTemplate.opsForValue().increment(key, delta);
+    }
+
+//    ============================== Map ==============================
+
+    /**
+     * HashGet
+     * @param key 键（no null）
+     * @param item 项（no null）
+     * @return 值
+     */
+    public Object hget(String key, String item) {
+        return redisTemplate.opsForHash().get(key, item);
+    }
+
+    /**
+     * 获取 key 对应的 map
+     * @param key 键（no null）
+     * @return 对应的多个键值
+     */
+    public Map<Object, Object> hmget(String key) {
+        return redisTemplate.opsForHash().entries(key);
+    }
+
+    /**
+     * HashSet
+     * @param key 键
+     * @param map 值
+     * @return true / false
+     */
+    public boolean hmset(String key, Map<Object, Object> map) {
+        try {
+            redisTemplate.opsForHash().putAll(key, map);
+            return true;
+        } catch (Exception e) {
+            e.printStackTrace();
+            return false;
+        }
+    }
+
+    /**
+     * HashSet 并设置时间
+     * @param key 键
+     * @param map 值
+     * @param time 时间
+     * @return true / false
+     */
+    public boolean hmset(String key, Map<Object, Object> map, long time) {
+        try {
+            redisTemplate.opsForHash().putAll(key, map);
+            if (time > 0) {
+                expire(key, time);
+            }
+            return true;
+        } catch (Exception e) {
+            e.printStackTrace();
+            return false;
+        }
+    }
+
+    /**
+     * 向一张 Hash表 中放入数据，如不存在则创建
+     * @param key 键
+     * @param item 项
+     * @param value 值
+     * @return true / false
+     */
+    public boolean hset(String key, String item, Object value) {
+        try {
+            redisTemplate.opsForHash().put(key, item, value);
+            return true;
+        } catch (Exception e) {
+            e.printStackTrace();
+            return false;
+        }
+    }
+
+    /**
+     * 向一张 Hash表 中放入数据，并设置时间，如不存在则创建
+     * @param key 键
+     * @param item 项
+     * @param value 值
+     * @param time 时间（如果原来的 Hash表 设置了时间，这里会覆盖）
+     * @return true / false
+     */
+    public boolean hset(String key, String item, Object value, long time) {
+        try {
+            redisTemplate.opsForHash().put(key, item, value);
+            if (time > 0) {
+                expire(key, time);
+            }
+            return true;
+        } catch (Exception e) {
+            e.printStackTrace();
+            return false;
+        }
+    }
+
+    /**
+     * 删除 Hash表 中的值
+     * @param key 键
+     * @param item 项（可以多个，no null）
+     */
+    public void hdel(String key, Object... item) {
+        redisTemplate.opsForHash().delete(key, item);
+    }
+
+    /**
+     * 判断 Hash表 中是否有该键的值
+     * @param key 键（no null）
+     * @param item 值（no null）
+     * @return true / false
+     */
+    public boolean hHasKey(String key, String item) {
+        return redisTemplate.opsForHash().hasKey(key, item);
+    }
+
+    /**
+     * Hash递增，如果不存在则创建一个，并把新增的值返回
+     * @param key 键
+     * @param item 项
+     * @param by 递增大小 > 0
+     * @return
+     */
+    public Double hincr(String key, String item, Double by) {
+        return redisTemplate.opsForHash().increment(key, item, by);
+    }
+
+    /**
+     * Hash递减
+     * @param key 键
+     * @param item 项
+     * @param by 递减大小
+     * @return
+     */
+    public Double hdecr(String key, String item, Double by) {
+        return redisTemplate.opsForHash().increment(key, item, -by);
+    }
+
+//    ============================== Set ==============================
+
+    /**
+     * 根据 key 获取 set 中的所有值
+     * @param key 键
+     * @return 值
+     */
+    public Set<Object> sGet(String key) {
+        try {
+            return redisTemplate.opsForSet().members(key);
+        } catch (Exception e) {
+            e.printStackTrace();
+            return null;
+        }
+    }
+
+    /**
+     * 从键为 key 的 set 中，根据 value 查询是否存在
+     * @param key 键
+     * @param value 值
+     * @return true / false
+     */
+    public boolean sHasKey(String key, Object value) {
+        try {
+            return redisTemplate.opsForSet().isMember(key, value);
+        } catch (Exception e) {
+            e.printStackTrace();
+            return false;
+        }
+    }
+
+    /**
+     * 将数据放入 set缓存
+     * @param key 键值
+     * @param values 值（可以多个）
+     * @return 成功个数
+     */
+    public long sSet(String key, Object... values) {
+        try {
+            return redisTemplate.opsForSet().add(key, values);
+        } catch (Exception e) {
+            e.printStackTrace();
+            return 0;
+        }
+    }
+
+    /**
+     * 将数据放入 set缓存，并设置时间
+     * @param key 键
+     * @param time 时间
+     * @param values 值（可以多个）
+     * @return 成功放入个数
+     */
+    public long sSett(String key, long time, Object... values) {
+        try {
+            long count = redisTemplate.opsForSet().add(key, values);
+            if (time > 0) {
+                expire(key, time);
+            }
+            return count;
+        } catch (Exception e) {
+            e.printStackTrace();
+            return 0;
+        }
+    }
+
+    /**
+     * 获取 set缓存的长度
+     * @param key 键
+     * @return 长度
+     */
+    public long sGetSetSize(String key) {
+        try {
+            return redisTemplate.opsForSet().size(key);
+        } catch (Exception e) {
+            e.printStackTrace();
+            return 0;
+        }
+    }
+
+    /**
+     * 移除 set缓存中，值为 value 的
+     * @param key 键
+     * @param values 值
+     * @return 成功移除个数
+     */
+    public long setRemove(String key, Object... values) {
+        try {
+            return redisTemplate.opsForSet().remove(key, values);
+        } catch (Exception e) {
+            e.printStackTrace();
+            return 0;
+        }
+    }
+
+//    ============================== List ==============================
+
+    /**
+     * 获取 list缓存的内容
+     * @param key 键
+     * @param start 开始
+     * @param end 结束（0 到 -1 代表所有值）
+     * @return
+     */
+    public List<Object> lGet(String key, long start, long end) {
+        try {
+            return redisTemplate.opsForList().range(key, start, end);
+        } catch (Exception e) {
+            e.printStackTrace();
+            return null;
+        }
+    }
+
+    /**
+     * 获取 list缓存的长度
+     * @param key 键
+     * @return 长度
+     */
+    public long lGetListSize(String key) {
+        try {
+            return redisTemplate.opsForList().size(key);
+        } catch (Exception e) {
+            e.printStackTrace();
+            return 0;
+        }
+    }
+
+    /**
+     * 根据索引 index 获取键为 key 的 list 中的元素
+     * @param key 键
+     * @param index 索引
+     *              当 index >= 0 时 {0:表头, 1:第二个元素}
+     *              当 index < 0 时 {-1:表尾, -2:倒数第二个元素}
+     * @return 值
+     */
+    public Object lGetIndex(String key, long index) {
+        try {
+            return redisTemplate.opsForList().index(key, index);
+        } catch (Exception e) {
+            e.printStackTrace();
+            return null;
+        }
+    }
+
+    /**
+     * 将值 value 插入键为 key 的 list 中，如果 list 不存在则创建空 list
+     * @param key 键
+     * @param value 值
+     * @return true / false
+     */
+    public boolean lSet(String key, Object value) {
+        try {
+            redisTemplate.opsForList().rightPush(key, value);
+            return true;
+        } catch (Exception e) {
+            e.printStackTrace();
+            return false;
+        }
+    }
+
+    /**
+     * 将值 value 插入键为 key 的 list 中，并设置时间
+     * @param key 键
+     * @param value 值
+     * @param time 时间
+     * @return true / false
+     */
+    public boolean lSet(String key, Object value, long time) {
+        try {
+            redisTemplate.opsForList().rightPush(key, value);
+            if (time > 0) {
+                expire(key, time);
+            }
+            return true;
+        } catch (Exception e) {
+            e.printStackTrace();
+            return false;
+        }
+    }
+
+    /**
+     * 将 values 插入键为 key 的 list 中
+     * @param key 键
+     * @param values 值
+     * @return true / false
+     */
+    public boolean lSetList(String key, List<Object> values) {
+        try {
+            redisTemplate.opsForList().rightPushAll(key, values);
+            return true;
+        } catch (Exception e) {
+            e.printStackTrace();
+            return false;
+        }
+    }
+
+    /**
+     * 将 values 插入键为 key 的 list 中，并设置时间
+     * @param key 键
+     * @param values 值
+     * @param time 时间
+     * @return true / false
+     */
+    public boolean lSetList(String key, List<Object> values, long time) {
+        try {
+            redisTemplate.opsForList().rightPushAll(key, values);
+            if (time > 0) {
+                expire(key, time);
+            }
+            return true;
+        } catch (Exception e) {
+            e.printStackTrace();
+            return false;
+        }
+    }
+
+    /**
+     * 根据索引 index 修改键为 key 的值
+     * @param key 键
+     * @param index 索引
+     * @param value 值
+     * @return true / false
+     */
+    public boolean lUpdateIndex(String key, long index, Object value) {
+        try {
+            redisTemplate.opsForList().set(key, index, value);
+            return true;
+        } catch (Exception e) {
+            e.printStackTrace();
+            return false;
+        }
+    }
+
+    /**
+     * 在键为 key 的 list 中删除值为 value 的元素
+     * @param key 键
+     * @param count 如果 count == 0 则删除 list 中所有值为 value 的元素
+     *              如果 count > 0 则删除 list 中最左边那个值为 value 的元素
+     *              如果 count < 0 则删除 list 中最右边那个值为 value 的元素
+     * @param value
+     * @return
+     */
+    public long lRemove(String key, long count, Object value) {
+        try {
+            return redisTemplate.opsForList().remove(key, count, value);
+        } catch (Exception e) {
+            e.printStackTrace();
+            return 0;
+        }
+    }
+
+    public Set<String> scan(String matchKey) {
+        ScanOptions options = ScanOptions.scanOptions().match("*" + matchKey + "*").count(3000).build();
+        return redisTemplate.execute((RedisCallback<Set<String>>) connection -> {
+            Set<String> keysTmp = new HashSet<>();
+            Cursor<byte[]> cursor = connection.scan(options);
+            while (cursor.hasNext()) {
+                keysTmp.add(new String(cursor.next()));
+            }
+            return keysTmp;
+        });
+    }
+
+}

src/main/java/com/zsc/edu/dify/framework/event/EventListener.java

@@ -0,0 +1,25 @@
+package com.zsc.edu.dify.framework.event;
+
+
+import lombok.RequiredArgsConstructor;
+import org.springframework.data.redis.core.StringRedisTemplate;
+import org.springframework.stereotype.Component;
+/**
+ * @description 事件监听者
+ * @author vivid
+ *
+ * */
+@Component
+@RequiredArgsConstructor
+public class EventListener {
+
+    private final SpiderPollingService spiderPollingService;
+
+    /**
+     * 监听爬虫启动事件
+     * */
+    @org.springframework.context.event.EventListener
+    public void handleStartPollingEvent(SpiderStartPollingEvent event) {
+        spiderPollingService.startPolling(event.getSpiderId(), event.getEmail());
+    }
+}

src/main/java/com/zsc/edu/dify/framework/event/SpiderPollingService.java

@@ -0,0 +1,73 @@
+package com.zsc.edu.dify.framework.event;
+
+import com.alibaba.fastjson.JSONObject;
+import com.zsc.edu.dify.common.util.RedisUtils;
+import com.zsc.edu.dify.framework.message.email.EmailSender;
+import com.zsc.edu.dify.framework.spider.SpiderConfig;
+import com.zsc.edu.dify.framework.spider.SpiderProperty;
+import com.zsc.edu.dify.modules.dify.service.SpiderService;
+import com.zsc.edu.dify.modules.message.entity.Notice;
+import com.zsc.edu.dify.modules.message.entity.NoticeType;
+import jakarta.annotation.PostConstruct;
+import lombok.RequiredArgsConstructor;
+import org.springframework.stereotype.Service;
+
+import java.util.HashMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+/**
+ * @description 轮询查看爬虫结果
+ * @author vivid
+ * */
+@Service
+@RequiredArgsConstructor
+public class SpiderPollingService {
+
+    private ScheduledExecutorService scheduler;
+
+    private final RedisUtils redisUtils;
+
+    private final SpiderService spiderService;
+
+    private final EmailSender emailSender;
+
+    private final SpiderConfig spiderConfig;
+
+    private static final HashMap<String, SpiderProperty> PROPERTY_MAP = new HashMap<>();
+
+    @PostConstruct
+    public void init() {
+        for (SpiderProperty property : spiderConfig.getConfigs()) {
+            PROPERTY_MAP.put(property.getId(), property);
+        }
+    }
+
+    public void startPolling(String spiderId, String email) {
+        scheduler = Executors.newSingleThreadScheduledExecutor();
+        scheduler.scheduleAtFixedRate(() -> {
+            executeBusinessLogic(spiderId, email);
+        }, 400, 3000, TimeUnit.MILLISECONDS);
+    }
+
+    private void executeBusinessLogic(String spiderId, String email) {
+        JSONObject jsonObject = spiderService.status(spiderId);
+        JSONObject data = jsonObject.getJSONObject("data");
+        if (!data.getBoolean("is_running")) {
+            //获取文件
+            String url = data.getString("download_url");
+            //添加爬虫请求头，并且将前缀去掉
+            String path = PROPERTY_MAP.get(spiderId).getUrl().replace("/api/v1", "") + url;
+            //发送邮件给用户
+            emailSender.send(email, new Notice(NoticeType.MESSAGE, false, true, false, false, "文件已到达", path, 0L));
+            //将整个对象写入redis
+            redisUtils.set(spiderId + ":result", jsonObject, 60 * 60 * 24);
+            //停止任务
+            scheduler.shutdown();
+        }
+    }
+
+    public void stopPolling() {
+        scheduler.shutdown();
+    }
+}

src/main/java/com/zsc/edu/dify/framework/event/SpiderStartPollingEvent.java

@@ -0,0 +1,23 @@
+package com.zsc.edu.dify.framework.event;
+
+import lombok.Getter;
+import org.springframework.context.ApplicationEvent;
+/**
+ *
+ * @description 定义爬虫启动的事件
+ * @author vivid
+ *
+ * */
+@Getter
+public class SpiderStartPollingEvent extends ApplicationEvent {
+
+    private String spiderId;
+
+    private String email;
+
+    public SpiderStartPollingEvent(Object source, String spiderId, String email) {
+        super(source);
+        this.spiderId = spiderId;
+        this.email = email;
+    }
+}

src/main/java/com/zsc/edu/dify/framework/generateDocx/generateDocx.java

@@ -0,0 +1,35 @@
+package com.zsc.edu.dify.framework.generateDocx;
+
+import com.deepoove.poi.XWPFTemplate;
+import com.zsc.edu.dify.framework.generateDocx.styleConfig.styleDataWrapper;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+/**
+ * @author Ding
+ */
+public class generateDocx {
+    public String generateSpider4(List<Map<String, Object>> rawData) {
+        UUID uuid = UUID.randomUUID();
+        String target = "storage/temp/" + uuid + ".docx";
+        String resource = "src/main/resources/docxTemplate/model.docx";
+        String back = uuid + ".docx";
+
+        Map<String, Object> data = new HashMap<>();
+
+        List<Map<String, Object>> styledData = styleDataWrapper.wrap(rawData);
+
+        data.put("context", styledData);
+
+        try (XWPFTemplate template = XWPFTemplate.compile(resource).render(data)) {
+            template.writeToFile(target);
+            return back;
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

src/main/java/com/zsc/edu/dify/framework/generateDocx/styleConfig/labelStyles.java

@@ -0,0 +1,32 @@
+package com.zsc.edu.dify.framework.generateDocx.styleConfig;
+
+import com.deepoove.poi.data.style.Style;
+import lombok.Data;
+
+/**
+ * @description 创建特定样式
+ * @author Ding
+ */
+@Data
+public class labelStyles {
+    private final Style titleStyle;
+    private final Style textStyle;
+
+    // 私有构造函数，强制通过工厂方法创建
+    private labelStyles(Style titleStyle, Style textStyle) {
+        this.titleStyle = titleStyle;
+        this.textStyle = textStyle;
+    }
+
+    // 工厂方法：初始化默认样式
+    public static labelStyles createDefault() {
+        Style titleStyle = new Style();
+        titleStyle.setBold(true);
+        titleStyle.setFontSize(20);
+
+        Style textStyle = new Style();
+        textStyle.setFontSize(11);
+
+        return new labelStyles(titleStyle, textStyle);
+    }
+}

src/main/java/com/zsc/edu/dify/framework/generateDocx/styleConfig/styleDataWrapper.java

@@ -0,0 +1,43 @@
+package com.zsc.edu.dify.framework.generateDocx.styleConfig;
+
+import com.deepoove.poi.data.ParagraphRenderData;
+import com.deepoove.poi.data.Paragraphs;
+import com.deepoove.poi.data.TextRenderData;
+import com.deepoove.poi.data.style.Style;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+
+/**
+ * @description 用于将样式与字段绑定
+ * @author Ding
+ */
+public class styleDataWrapper {
+    static labelStyles styles = labelStyles.createDefault();
+
+    // 定义标签与样式映射规则。标题用一个样式，其他标签用一个样式
+    private static final Map<String, Style> STYLE_MAPPING = Map.of(
+            "title", styles.getTitleStyle(),
+            "*", styles.getTextStyle()
+    );
+
+    public static List<Map<String, Object>> wrap(List<Map<String, Object>> rawData) {
+        return rawData.stream()
+                .map(styleDataWrapper::processMap)
+                .collect(Collectors.toList());
+    }
+
+    // 为每个字段绑定特定的样式
+    private static Map<String, Object> processMap(Map<String, Object> rawMap) {
+        Map<String, Object> styledMap = new HashMap<>();
+        rawMap.forEach((key, value) -> {
+            Style style = STYLE_MAPPING.getOrDefault(key, STYLE_MAPPING.get("*"));
+            TextRenderData textRenderData = new TextRenderData(value.toString(), style);
+            styledMap.put(key, textRenderData);
+        });
+        return styledMap;
+    }
+}

src/main/java/com/zsc/edu/dify/framework/message/email/EmailSender.java

@@ -42,7 +42,7 @@ public class EmailSender {
 
     @Async
     public void send(String email, Notice notice) {
-        if (StringUtils.hasText(email)) {
+        if (!StringUtils.hasText(email)) {
             return;
         }
         InternetAddress to;

src/main/java/com/zsc/edu/dify/framework/redis/RedisConfig.java

@@ -0,0 +1,34 @@
+package com.zsc.edu.dify.framework.redis;
+
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.data.redis.connection.RedisConnectionFactory;
+import org.springframework.data.redis.core.RedisTemplate;
+import org.springframework.data.redis.serializer.GenericJackson2JsonRedisSerializer;
+import org.springframework.data.redis.serializer.StringRedisSerializer;
+/**
+ * @description redis配置类
+ * @author vivid
+ * */
+@Configuration
+public class RedisConfig {
+
+    // 自定义 RedisTemplate Bean 配置
+    @Bean
+    public RedisTemplate<String, Object> redisTemplate(RedisConnectionFactory factory) {
+        RedisTemplate<String, Object> template = new RedisTemplate<>();
+        template.setConnectionFactory(factory);
+
+        // 设置键的序列化方式为 String
+        template.setKeySerializer(new StringRedisSerializer());
+        template.setHashKeySerializer(new StringRedisSerializer());
+
+        // 设置值的序列化方式为 JSON
+        template.setValueSerializer(new GenericJackson2JsonRedisSerializer());
+        template.setHashValueSerializer(new GenericJackson2JsonRedisSerializer());
+
+        template.afterPropertiesSet();
+        return template;
+    }
+
+}

src/main/java/com/zsc/edu/dify/modules/dify/controller/Spider3Controller.java

@@ -1,62 +0,0 @@
-package com.zsc.edu.dify.modules.dify.controller;
-
-
-import com.alibaba.fastjson.JSONObject;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.zsc.edu.dify.modules.dify.dto.SpiderDto;
-import jakarta.annotation.Resource;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.http.MediaType;
-import org.springframework.web.bind.annotation.*;
-import org.springframework.web.reactive.function.client.WebClient;
-
-
-@RestController
-@RequestMapping("/api/spider3")
-public class Spider3Controller {
-    @Resource
-    private ObjectMapper objectMapper;
-
-    @Value("${spider3.url}")
-    private String SPIDER_URL;
-
-    @Value("${spider3.api-key}")
-    private String API_KEY;
-
-    @PostMapping("/run")
-    public JSONObject run() throws JsonProcessingException {
-        return WebClient.create(SPIDER_URL).post().uri("/start_crawl")
-                .contentType(MediaType.APPLICATION_JSON)
-                .retrieve()
-                .bodyToMono(JSONObject.class)
-                .block();
-    }
-
-    @PostMapping("/status")
-    public JSONObject status() {
-        return WebClient.create(SPIDER_URL).post().uri("/crawl_status")
-                .retrieve()
-                .bodyToMono(JSONObject.class)
-                .block();
-    }
-
-    @PostMapping("/logs")
-    public JSONObject logs() {
-        return WebClient.create(SPIDER_URL).post().uri("/logs")
-                .retrieve()
-                .bodyToMono(JSONObject.class)
-                .block();
-    }
-
-    @PostMapping("/stop")
-    public JSONObject stop() {
-        return WebClient.create(SPIDER_URL).post().uri("/stop_crawl")
-                .retrieve()
-                .bodyToMono(JSONObject.class)
-                .block();
-    }
-
-
-
-}

src/main/java/com/zsc/edu/dify/modules/dify/controller/SpiderController.java

@@ -2,10 +2,15 @@ package com.zsc.edu.dify.modules.dify.controller;
 
 import com.alibaba.fastjson.JSONObject;
 import com.fasterxml.jackson.core.JsonProcessingException;
+import com.zsc.edu.dify.framework.event.SpiderPollingService;
+import com.zsc.edu.dify.framework.event.SpiderStartPollingEvent;
 import com.zsc.edu.dify.modules.dify.dto.SpiderDto;
 import com.zsc.edu.dify.modules.dify.service.SpiderService;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.ApplicationEventPublisher;
 import org.springframework.web.bind.annotation.*;
+
+
 /**
  * @description: 自定义爬虫
  * @author: yao
@@ -17,9 +22,23 @@ public class SpiderController {
     @Autowired
     private SpiderService spiderService;
 
+    @Autowired
+    private ApplicationEventPublisher applicationEventPublisher;
+
+    @Autowired
+    private SpiderPollingService spiderPollingService;
+
     @PostMapping("/run/{spiderId}")
     public JSONObject run(@RequestBody(required = false) SpiderDto dto, @PathVariable String spiderId) throws JsonProcessingException {
-        return spiderService.run(dto, spiderId);
+        JSONObject data = spiderService.isExistCache(spiderId, dto.getEmail());
+        if (data != null) {
+            // 如果缓存存在，直接返回，不执行 spiderService.run()
+            return data;
+        } else {
+            // 如果缓存不存在，正常执行，并且发布事件
+            applicationEventPublisher.publishEvent(new SpiderStartPollingEvent(this, spiderId, dto.getEmail()));
+            return spiderService.run(dto, spiderId);
+        }
     }
 
     @PostMapping("/status/{spiderId}")
@@ -34,6 +53,7 @@ public class SpiderController {
 
     @PostMapping("/stop/{spiderId}")
     public JSONObject stop(@PathVariable String spiderId) {
+        spiderPollingService.stopPolling();
         return spiderService.stop(spiderId);
     }
 

src/main/java/com/zsc/edu/dify/modules/dify/dto/SpiderDto.java

@@ -1,5 +1,6 @@
 package com.zsc.edu.dify.modules.dify.dto;
 
+import jakarta.validation.constraints.Email;
 import jakarta.validation.constraints.NotBlank;
 import lombok.Data;
 
@@ -12,4 +13,6 @@ public class SpiderDto {
     private String keyword;
     private Integer[] site_codes;
     private String llm_api_key;
+    @Email
+    private String email;
 }

src/main/java/com/zsc/edu/dify/modules/dify/service/Impl/SpiderServiceImpl.java

@@ -3,10 +3,15 @@ package com.zsc.edu.dify.modules.dify.service.Impl;
 import com.alibaba.fastjson.JSONObject;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.zsc.edu.dify.common.util.RedisUtils;
+import com.zsc.edu.dify.framework.event.SpiderPollingService;
+import com.zsc.edu.dify.framework.message.email.EmailSender;
 import com.zsc.edu.dify.framework.spider.SpiderConfig;
 import com.zsc.edu.dify.framework.spider.SpiderProperty;
 import com.zsc.edu.dify.modules.dify.dto.SpiderDto;
 import com.zsc.edu.dify.modules.dify.service.SpiderService;
+import com.zsc.edu.dify.modules.message.entity.Notice;
+import com.zsc.edu.dify.modules.message.entity.NoticeType;
 import jakarta.annotation.PostConstruct;
 import lombok.RequiredArgsConstructor;
 import org.apache.commons.lang3.StringUtils;
@@ -26,6 +31,12 @@ public class SpiderServiceImpl implements SpiderService {
 
     private static final HashMap<String, SpiderProperty> PROPERTY_MAP = new HashMap<>();
 
+    private final RedisUtils redisUtils;
+
+    private final EmailSender emailSender;
+
+
+
     @PostConstruct
     public void init() {
         for (SpiderProperty property : spiderConfig.getConfigs()) {
@@ -74,4 +85,23 @@ public class SpiderServiceImpl implements SpiderService {
                 .bodyToMono(JSONObject.class)
                 .block();
     }
+
+    @Override
+    public JSONObject isExistCache(String spiderId, String email) {
+        JSONObject result = (JSONObject) redisUtils.get(spiderId + ":result");
+        if (result != null) {
+            //获取文件路径
+            String url = result.getJSONObject("data").getString("download_url");
+            //拼接爬虫路径
+            String path = PROPERTY_MAP.get(spiderId).getUrl().replace("/api/v1", "") + url;
+            //发送邮件给用户
+            emailSender.send(email, new Notice(NoticeType.MESSAGE, false, true, false, false, "文件已到达", path, 0L));
+            //构造响应体
+            JSONObject response = new JSONObject();
+            response.put("msg", "爬虫任务已接受");
+            response.put("code", "0");
+            return response;
+        }
+        return null;
+    }
 }

src/main/java/com/zsc/edu/dify/modules/dify/service/SpiderService.java

@@ -13,4 +13,7 @@ public interface SpiderService {
     JSONObject logs(String spiderId);
 
     JSONObject stop(String spiderId);
+
+
+    JSONObject isExistCache(String spiderId, String email);
 }

src/main/resources/application-dev.yml

@@ -83,16 +83,6 @@ dify:
   dataset:
     api-key: dataset-kN5WTJ8jR877YfN1A34JceVg # 请替换为实际的知识库api-key, 若不需要调用知识库可不填
 
-quanguo: &quanguo
-  spider-id: ${QUANGUO_ID:77c068fd-d5b6-4c33-97d8-db5511a09b26}
-  url: http://${QUANGUO_HOST:47.112.173.8:6806/api/v1}
-  api-key: ${QUANGUO_API_KEY:77c068fd-d5b6-4c33-97d8-db5511a09b26}
-
-spider3: &spider3
-  spider-id: ${SPIDER3_ID:f3a7b9c2-5d6e-4b8f-9c1a-2d3e4f5a6b7c}
-  url: http://${SPIDER3_HOST:47.112.173.8:6257/api/v1}
-  api-key:
-
 spider:
   configs:
     # 全国爬虫
@@ -103,5 +93,10 @@ spider:
     - id: ${SPIDER3_ID:f3a7b9c2-5d6e-4b8f-9c1a-2d3e4f5a6b7c}
       url: http://${SPIDER3_HOST:47.112.173.8:6257/api/v1}
       api-key:
+    # 爬虫D
+    - id: ${SPIDERD_ID:e1f90005-8c42-4736-a598-9Bebe49c8e12}
+      url: http://${SPIDER3_HOST:47.112.173.8:6258/api/v1}
+      api-key:
 #  - *quanguo
 #  - *spider3
+#  - *spiderD