2.0.0项目初始化

datax-common/datax-common-core/src/main/java/cn/datax/common/utils/MD5Util.java

 package cn.datax.common.utils;
 
-import sun.misc.BASE64Decoder;
-import sun.misc.BASE64Encoder;
-
 import javax.crypto.Cipher;
 import javax.crypto.SecretKeyFactory;
 import javax.crypto.spec.DESKeySpec;
@@ -11,6 +8,7 @@ import java.security.Key;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.security.spec.AlgorithmParameterSpec;
+import java.util.Base64;
 
 public class MD5Util {
 
@@ -68,8 +66,7 @@ public class MD5Util {
         Cipher enCipher = Cipher.getInstance("DES/CBC/PKCS5Padding");// 得到加密对象Cipher
         enCipher.init(Cipher.ENCRYPT_MODE, key, iv);// 设置工作模式为加密模式，给出密钥和向量
         byte[] pasByte = enCipher.doFinal(data.getBytes(this.charset));
-        BASE64Encoder base64Encoder = new BASE64Encoder();
-        return base64Encoder.encode(pasByte);
+        return Base64.getEncoder().encodeToString(pasByte);
     }
 
     /**
@@ -81,10 +78,9 @@ public class MD5Util {
     public String decode(String data) throws Exception {
         Cipher deCipher = Cipher.getInstance("DES/CBC/PKCS5Padding");
         deCipher.init(Cipher.DECRYPT_MODE, key, iv);
-        BASE64Decoder base64Decoder = new BASE64Decoder();
         //此处注意doFinal()的参数的位数必须是8的倍数，否则会报错（通过encode加密的字符串读出来都是8的倍数位，但写入文件再读出来，就可能因为读取的方式的问题，导致最后此处的doFinal()的参数的位数不是8的倍数）
         //此处必须用base64Decoder，若用data。getBytes()则获取的字符串的byte数组的个数极可能不是8的倍数，而且不与上面的BASE64Encoder对应（即使解密不报错也不会得到正确结果）
-        byte[] pasByte = deCipher.doFinal(base64Decoder.decodeBuffer(data));
+        byte[] pasByte = deCipher.doFinal(Base64.getDecoder().decode(data));
         return new String(pasByte, this.charset);
     }
 

datax-common/datax-common-log/src/main/java/cn/datax/common/log/annotation/EnableDataLog.java

-package cn.datax.common.log.annotation;
-
-import cn.datax.common.log.config.AutoConfiguration;
-import org.springframework.context.annotation.Import;
-
-import java.lang.annotation.*;
-
-@Target({ElementType.TYPE})
-@Retention(RetentionPolicy.RUNTIME)
-@Documented
-@Inherited
-@Import({AutoConfiguration.class})
-public @interface EnableDataLog {
-}

datax-common/datax-common-log/src/main/java/cn/datax/common/log/aspectj/LogAspect.java

@@ -3,10 +3,10 @@ package cn.datax.common.log.aspectj;
 import java.lang.reflect.Method;
 
 import cn.datax.common.log.annotation.LogAop;
-import cn.datax.common.log.async.AsyncTask;
 import cn.datax.common.utils.RequestHolder;
 import cn.datax.common.utils.SecurityUtil;
 import cn.datax.service.system.api.dto.LogDto;
+import cn.datax.service.system.api.feign.LogServiceFeign;
 import cn.hutool.core.util.URLUtil;
 import cn.hutool.extra.servlet.ServletUtil;
 import cn.hutool.http.HttpUtil;
@@ -28,7 +28,7 @@ import javax.servlet.http.HttpServletRequest;
 public class LogAspect {
 
     @Autowired
-    private AsyncTask asyncTask;
+    private LogServiceFeign logServiceFeign;
 
     // 配置织入点
     @Pointcut("@annotation(cn.datax.common.log.annotation.LogAop)")
@@ -89,8 +89,8 @@ public class LogAspect {
         logDto.setModule(logAop.module()).setTitle(logAop.value())
                 .setClassName(className).setMethodName(methodName);
 
-        // 异步保存数据库
-        asyncTask.doTask(logDto);
+        // 保存数据库
+        logServiceFeign.saveLog(logDto);
     }
 
     /**

datax-common/datax-common-log/src/main/java/cn/datax/common/log/config/AutoConfiguration.java

-package cn.datax.common.log.config;
-
-import cn.datax.common.log.aspectj.LogAspect;
-import cn.datax.common.log.async.AsyncTask;
-import org.springframework.context.annotation.Import;
-
-/**
- * 扫描注入bean
- *
- * @author yuwei
- * @since 2019/10/30
- */
-@Import({LogAspect.class, AsyncTask.class, LogAsyncConfig.class})
-public class AutoConfiguration {
-
-}

datax-common/datax-common-log/src/main/resources/META-INF/spring.factories

+org.springframework.boot.autoconfigure.EnableAutoConfiguration=\
+  cn.datax.common.log.aspectj.LogAspect

datax-common/datax-common-redis/src/main/java/cn/datax/common/redis/config/RedisConfig.java

@@ -2,53 +2,34 @@ package cn.datax.common.redis.config;
 
 import cn.datax.common.redis.service.RedisService;
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import com.fasterxml.jackson.annotation.PropertyAccessor;
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
 import com.fasterxml.jackson.databind.jsontype.impl.LaissezFaireSubTypeValidator;
 import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
-import com.fasterxml.jackson.datatype.jsr310.deser.LocalDateDeserializer;
-import com.fasterxml.jackson.datatype.jsr310.deser.LocalDateTimeDeserializer;
-import com.fasterxml.jackson.datatype.jsr310.deser.LocalTimeDeserializer;
-import com.fasterxml.jackson.datatype.jsr310.ser.LocalDateSerializer;
-import com.fasterxml.jackson.datatype.jsr310.ser.LocalDateTimeSerializer;
-import com.fasterxml.jackson.datatype.jsr310.ser.LocalTimeSerializer;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
 import org.springframework.context.annotation.Bean;
 import org.springframework.data.redis.connection.RedisConnectionFactory;
 import org.springframework.data.redis.core.*;
-import org.springframework.data.redis.serializer.Jackson2JsonRedisSerializer;
+import org.springframework.data.redis.serializer.GenericJackson2JsonRedisSerializer;
+import org.springframework.data.redis.serializer.RedisSerializer;
 import org.springframework.data.redis.serializer.StringRedisSerializer;
 
-import java.time.LocalDate;
-import java.time.LocalDateTime;
-import java.time.LocalTime;
-import java.time.format.DateTimeFormatter;
-
 @ConditionalOnClass(RedisOperations.class)
 public class RedisConfig {
 
     /**
-     * 实例化 RedisTemplate 对象
+     * 自定义redis序列化的机制,重新定义一个ObjectMapper.防止和MVC的冲突
      *
      * @return
      */
-    @Bean(name = "redisTemplate")
-    public RedisTemplate<String, Object> redisTemplate(RedisConnectionFactory redisConnectionFactory) {
+    @Bean
+    public RedisSerializer<Object> redisSerializer() {
         ObjectMapper objectMapper = new ObjectMapper();
         objectMapper.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.ANY);
-        //使用JSR310提供的序列化类,里面包含了大量的JDK8时间序列化类
-        objectMapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
-        JavaTimeModule javaTimeModule = new JavaTimeModule();
-        javaTimeModule.addSerializer(LocalDateTime.class,new LocalDateTimeSerializer(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")));
-        javaTimeModule.addSerializer(LocalDate.class,new LocalDateSerializer(DateTimeFormatter.ofPattern("yyyy-MM-dd")));
-        javaTimeModule.addSerializer(LocalTime.class,new LocalTimeSerializer(DateTimeFormatter.ofPattern("HH:mm:ss")));
-        javaTimeModule.addDeserializer(LocalDateTime.class,new LocalDateTimeDeserializer(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")));
-        javaTimeModule.addDeserializer(LocalDate.class,new LocalDateDeserializer(DateTimeFormatter.ofPattern("yyyy-MM-dd")));
-        javaTimeModule.addDeserializer(LocalTime.class,new LocalTimeDeserializer(DateTimeFormatter.ofPattern("HH:mm:ss")));
-        objectMapper.registerModule(javaTimeModule);
         //反序列化时候遇到不匹配的属性并不抛出异常
         objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
         //序列化时候遇到空对象不抛出异常
@@ -57,15 +38,27 @@ public class RedisConfig {
         objectMapper.configure(DeserializationFeature.FAIL_ON_INVALID_SUBTYPE, false);
         //不使用默认的dateTime进行序列化,
         objectMapper.configure(SerializationFeature.WRITE_DATE_KEYS_AS_TIMESTAMPS, false);
+        //使用JSR310提供的序列化类,里面包含了大量的JDK8时间序列化类
+        objectMapper.registerModule(new JavaTimeModule());
         //启用反序列化所需的类型信息,在属性中添加@class
-        objectMapper.activateDefaultTyping(LaissezFaireSubTypeValidator.instance, ObjectMapper.DefaultTyping.NON_FINAL);
-        Jackson2JsonRedisSerializer<Object> serializer = new Jackson2JsonRedisSerializer<Object>(Object.class);
-        serializer.setObjectMapper(objectMapper);
+        objectMapper.activateDefaultTyping(LaissezFaireSubTypeValidator.instance, ObjectMapper.DefaultTyping.NON_FINAL, JsonTypeInfo.As.PROPERTY);
+        //配置null值的序列化器
+        GenericJackson2JsonRedisSerializer.registerNullValueSerializer(objectMapper, null);
+        return new GenericJackson2JsonRedisSerializer(objectMapper);
+    }
+
+    /**
+     * 实例化 RedisTemplate 对象
+     *
+     * @return
+     */
+    @Bean(name = "redisTemplate")
+    public RedisTemplate<String, Object> redisTemplate(RedisConnectionFactory redisConnectionFactory, RedisSerializer<Object> redisSerializer) {
         RedisTemplate<String, Object> redisTemplate = new RedisTemplate<>();
         redisTemplate.setConnectionFactory(redisConnectionFactory);
-        redisTemplate.setDefaultSerializer(serializer);
-        redisTemplate.setValueSerializer(serializer);
-        redisTemplate.setHashValueSerializer(serializer);
+        redisTemplate.setDefaultSerializer(redisSerializer);
+        redisTemplate.setValueSerializer(redisSerializer);
+        redisTemplate.setHashValueSerializer(redisSerializer);
         redisTemplate.setKeySerializer(StringRedisSerializer.UTF_8);
         redisTemplate.setHashKeySerializer(StringRedisSerializer.UTF_8);
         redisTemplate.afterPropertiesSet();

datax-modules/data-factory-service-parent/data-factory-service-sql-console/src/main/java/cn/datax/service/data/factory/sql/console/DataSqlConsoleApplication.java → datax-modules/data-factory-service-parent/data-factory-service-sql-console/src/main/java/cn/datax/service/data/factory/sql/console/DataxSqlConsoleApplication.java

 package cn.datax.service.data.factory.sql.console;
 
-import cn.datax.common.log.annotation.EnableDataLog;
 import org.springframework.boot.SpringApplication;
 import org.springframework.cloud.client.SpringCloudApplication;
 import org.springframework.cloud.openfeign.EnableFeignClients;
 
-@EnableDataLog
 @EnableFeignClients(basePackages = {"cn.datax.service.system.api.feign", "cn.datax.service.data.factory.api.feign"})
 @SpringCloudApplication
-public class DataSqlConsoleApplication {
+public class DataxSqlConsoleApplication {
 
     public static void main(String[] args) {
-        SpringApplication.run(DataSqlConsoleApplication.class);
+        SpringApplication.run(DataxSqlConsoleApplication.class);
     }
 }

datax-modules/data-factory-service-parent/data-factory-service/src/main/java/cn/datax/service/data/factory/DataFactoryApplication.java → datax-modules/data-factory-service-parent/data-factory-service/src/main/java/cn/datax/service/data/factory/DataxFactoryApplication.java

 package cn.datax.service.data.factory;
 
-import cn.datax.common.log.annotation.EnableDataLog;
 import org.springframework.boot.SpringApplication;
 import org.springframework.cloud.client.SpringCloudApplication;
 import org.springframework.cloud.openfeign.EnableFeignClients;
 
-@EnableDataLog
 @EnableFeignClients(basePackages = {"cn.datax.service.system.api.feign", "cn.datax.service.data.factory.api.feign"})
 @SpringCloudApplication
-public class DataFactoryApplication {
+public class DataxFactoryApplication {
 
     public static void main(String[] args) {
-        SpringApplication.run(DataFactoryApplication.class);
+        SpringApplication.run(DataxFactoryApplication.class);
     }
 }

datax-modules/data-market-service-parent/data-market-service-api-call/src/main/java/cn/datax/service/data/market/api/call/DataApiCallApplication.java → datax-modules/data-market-service-parent/data-market-service-api-call/src/main/java/cn/datax/service/data/market/api/call/DataxApiCallApplication.java

 package cn.datax.service.data.market.api.call;
 
-import cn.datax.common.log.annotation.EnableDataLog;
 import org.springframework.boot.SpringApplication;
 import org.springframework.cloud.client.SpringCloudApplication;
 import org.springframework.cloud.openfeign.EnableFeignClients;
 
-@EnableDataLog
 @EnableFeignClients(basePackages = {"cn.datax.service.system.api.feign", "cn.datax.service.data.factory.api.feign", "cn.datax.service.data.market.api.feign"})
 @SpringCloudApplication
-public class DataApiCallApplication {
+public class DataxApiCallApplication {
 
     public static void main(String[] args) {
-        SpringApplication.run(DataApiCallApplication.class);
+        SpringApplication.run(DataxApiCallApplication.class);
     }
 }

datax-modules/data-market-service-parent/data-market-service/src/main/java/cn/datax/service/data/market/DataMarketApplication.java → datax-modules/data-market-service-parent/data-market-service/src/main/java/cn/datax/service/data/market/DataxMarketApplication.java

 package cn.datax.service.data.market;
 
-import cn.datax.common.log.annotation.EnableDataLog;
 import org.springframework.boot.SpringApplication;
 import org.springframework.cloud.client.SpringCloudApplication;
 import org.springframework.cloud.openfeign.EnableFeignClients;
 
-@EnableDataLog
 @EnableFeignClients(basePackages = {"cn.datax.service.system.api.feign"})
 @SpringCloudApplication
-public class DataMarketApplication {
+public class DataxMarketApplication {
 
     public static void main(String[] args) {
-        SpringApplication.run(DataMarketApplication.class);
+        SpringApplication.run(DataxMarketApplication.class);
     }
 }

datax-modules/email-service-parent/email-service/src/main/java/cn/datax/service/email/DataxMailApplication.java

 package cn.datax.service.email;
 
-import cn.datax.common.log.annotation.EnableDataLog;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 
-@EnableDataLog
 @SpringBootApplication
 public class DataxMailApplication {
 

datax-modules/file-service-parent/file-service/src/main/java/cn/datax/service/file/DataxFileApplication.java

 package cn.datax.service.file;
 
-import cn.datax.common.log.annotation.EnableDataLog;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 
-@EnableDataLog
 @SpringBootApplication
 public class DataxFileApplication {
 

datax-modules/system-service-parent/system-service/src/main/java/cn/datax/service/system/DataxSystemApplication.java

 package cn.datax.service.system;
 
-import cn.datax.common.log.annotation.EnableDataLog;
 import org.springframework.boot.SpringApplication;
 import org.springframework.cloud.client.SpringCloudApplication;
 import org.springframework.cloud.openfeign.EnableFeignClients;
 
-@EnableDataLog
 @EnableFeignClients(basePackages = {"cn.datax.service.system.api.feign"})
 @SpringCloudApplication
 public class DataxSystemApplication {

datax-common/datax-common-log/src/main/java/cn/datax/common/log/async/AsyncTask.java → datax-modules/system-service-parent/system-service/src/main/java/cn/datax/service/system/async/AsyncTask.java

-package cn.datax.common.log.async;
+package cn.datax.service.system.async;
 
 import cn.datax.service.system.api.dto.LogDto;
-import cn.datax.service.system.api.feign.LogServiceFeign;
+import cn.datax.service.system.service.LogService;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.scheduling.annotation.Async;
 import org.springframework.stereotype.Component;
 
-import lombok.extern.slf4j.Slf4j;
-
 /**
  * 异步处理 分布式获取请求头有问题
  */
@@ -16,11 +15,10 @@ import lombok.extern.slf4j.Slf4j;
 public class AsyncTask {
 
     @Autowired
-    private LogServiceFeign logServiceFeign;
+    private LogService logService;
 
-    @Async("dataLogExecutor")
+    @Async("dataxAsyncThreadPool")
     public void doTask(LogDto logDto) {
-        logServiceFeign.saveLog(logDto);
+        logService.saveLog(logDto);
     }
-
 }
\ No newline at end of file

datax-common/datax-common-log/src/main/java/cn/datax/common/log/config/LogAsyncConfig.java → datax-modules/system-service-parent/system-service/src/main/java/cn/datax/service/system/config/LogAsyncConfig.java

-package cn.datax.common.log.config;
-
-import java.util.concurrent.Executor;
-import java.util.concurrent.ThreadPoolExecutor;
+package cn.datax.service.system.config;
 
 import org.springframework.context.annotation.Bean;
 import org.springframework.scheduling.annotation.EnableAsync;
 import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
 
+import java.util.concurrent.ThreadPoolExecutor;
+
 @EnableAsync
 public class LogAsyncConfig {
 
-    private static int corePoolSize = 5;
-    private static int maxPoolSize = 10;
-    private static int queueCapacity = 20;
-
-    @Bean("dataLogExecutor")
-    public Executor dataLogExecutor() {
+    @Bean("dataxAsyncThreadPool")
+    public ThreadPoolTaskExecutor asyncThreadPoolTaskExecutor() {
         ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
-        executor.setCorePoolSize(corePoolSize);
-        executor.setMaxPoolSize(maxPoolSize);
-        executor.setQueueCapacity(queueCapacity);
-        executor.setKeepAliveSeconds(60);
-        executor.setThreadNamePrefix("dataLogExecutor-");
+        executor.setCorePoolSize(5);
+        executor.setMaxPoolSize(20);
+        executor.setQueueCapacity(100);
+        executor.setKeepAliveSeconds(30);
+        executor.setThreadNamePrefix("Datax-Async-Thread");
         executor.setWaitForTasksToCompleteOnShutdown(true);
         executor.setAwaitTerminationSeconds(60);
         executor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());

datax-modules/system-service-parent/system-service/src/main/java/cn/datax/service/system/controller/InnerController.java

@@ -4,7 +4,7 @@ import cn.datax.common.base.BaseController;
 import cn.datax.common.core.R;
 import cn.datax.common.security.annotation.DataInner;
 import cn.datax.service.system.api.dto.LogDto;
-import cn.datax.service.system.service.LogService;
+import cn.datax.service.system.async.AsyncTask;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
@@ -16,12 +16,12 @@ import org.springframework.web.bind.annotation.RestController;
 public class InnerController extends BaseController {
 
     @Autowired
-    private LogService logService;
+    private AsyncTask asyncTask;
 
     @DataInner
     @PostMapping("/logs")
     public R saveLog(@RequestBody LogDto log) {
-        logService.saveLog(log);
+        asyncTask.doTask(log);
         return R.ok();
     }
 }