添加tlog日志配置,excel导入数据库采用多线程导入

admin-api/src/main/java/com/tbyf/his/web/dataImport/service/impl/ExcelDataServiceImpl.java

 package com.tbyf.his.web.dataImport.service.impl;
 
+import cn.hutool.core.collection.CollectionUtil;
+import cn.hutool.core.util.ArrayUtil;
+import cn.hutool.core.util.StrUtil;
 import com.baomidou.mybatisplus.core.toolkit.Wrappers;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
 import com.tbyf.his.adapter.task.StringColumnRowMapper;
@@ -22,6 +25,7 @@ import com.tbyf.his.web.dataImport.service.DataDictService;
 import com.tbyf.his.web.dataImport.service.DataSourceService;
 import com.tbyf.his.web.dataImport.service.DataTemplateService;
 import com.tbyf.his.web.dataImport.service.ExcelDataService;
+import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.compress.archivers.ArchiveEntry;
 import org.apache.commons.compress.archivers.zip.Zip64Mode;
@@ -33,6 +37,7 @@ import org.apache.poi.ss.util.CellReference;
 import org.apache.poi.util.IOUtils;
 import org.springframework.beans.BeanUtils;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.stereotype.Service;
 import org.springframework.util.CollectionUtils;
@@ -45,6 +50,7 @@ import java.io.InputStream;
 import java.math.BigDecimal;
 import java.math.RoundingMode;
 import java.util.*;
+import java.util.concurrent.*;
 import java.util.stream.Collectors;
 
 /**
@@ -65,6 +71,10 @@ public class ExcelDataServiceImpl extends ServiceImpl<ExcelDataMapper, ExcelData
     @Autowired
     private DataDictService dataDictService;
 
+    @Autowired
+    @Qualifier("threadPoolTaskExecutor")
+    private Executor executor;
+
     @Override
     public void analyzeExport(ExcelData excelData, HttpServletResponse response) {
         // 查询出所有字段配置
@@ -461,6 +471,7 @@ public class ExcelDataServiceImpl extends ServiceImpl<ExcelDataMapper, ExcelData
 
     @Override
     public List<String> importData(ExcelData excelData) {
+        log.info("开始解析excel数据");
         List<String> errorList = new ArrayList<>();
         long startTime = System.currentTimeMillis();
         final DataTemplate template = dataTemplateService.getById(excelData.getTemplateId());
@@ -542,13 +553,14 @@ public class ExcelDataServiceImpl extends ServiceImpl<ExcelDataMapper, ExcelData
                     }
                 }
             }
-            log.info("解析数据用时:[{}]", System.currentTimeMillis() - startTime);
+            log.info("解析Excel数据用时:[{}]", System.currentTimeMillis() - startTime);
             startTime = System.currentTimeMillis();
             // 直接导入临时表
             String insertSql = "INSERT INTO {}(" + verifyList.stream()
                     .map(VerifyVO::getFieldName).collect(Collectors.joining(","))
                     + ",DORDER,YEAROOFDATARECORD,ROWCODE ) VALUES ({})";
-            String[] sqlArr = new String[length];
+            //String[] sqlArr = new String[length];
+            List<String> sqlList = new ArrayList<>(length + 100);
             for (int i = 0; i < length; i++) {
                 StringBuilder sb = new StringBuilder();
                 for (VerifyVO v : verifyList) {
@@ -557,7 +569,8 @@ public class ExcelDataServiceImpl extends ServiceImpl<ExcelDataMapper, ExcelData
                 sb.append("'").append(DataImportUtils.getNextId()).append("','")
                         .append(template.getYear()).append("年").append("','")
                         .append(DataImportUtils.getNextId()).append("'");
-                sqlArr[i] = StrFormatter.format(insertSql, template.getTableName() + "_TEMP", sb);
+                sqlList.add( StrFormatter.format(insertSql, template.getTableName() + "_TEMP", sb));
+                //sqlArr[i] = StrFormatter.format(insertSql, template.getTableName() + "_TEMP", sb);
             }
             try {
                 if(!CollectionUtils.isEmpty(errorList)){
@@ -567,8 +580,19 @@ public class ExcelDataServiceImpl extends ServiceImpl<ExcelDataMapper, ExcelData
                 // 需要先清空临时表的数据,按照年份
                 jdbcTemplate.execute(StrFormatter.format("DELETE FROM {} WHERE YEAROOFDATARECORD = '{}'"
                         , template.getTableName() + "_TEMP", template.getYear() + "年"));
-
-                jdbcTemplate.batchUpdate(sqlArr);
+                List<List<String>> split = CollectionUtil.split(sqlList, 500);
+                CompletableFuture<?>[] array = split.stream()
+                        .map(list -> CompletableFuture.runAsync(() -> {
+                            try {
+                                DataSourceService.switchDb(template.getDataSourceId());
+                                jdbcTemplate.batchUpdate(ArrayUtil.toArray(list, String.class));
+                            }finally {
+                                DataSourceService.switchDefault();
+                            }
+                        }, executor))
+                        .toArray(CompletableFuture<?>[]::new);
+                CompletableFuture.allOf(array).join();
+                //jdbcTemplate.batchUpdate(sqlArr);
                 log.info("导入数据用时:[{}]", System.currentTimeMillis() - startTime);
             } catch (Throwable e) {
                 log.error("数据导入错误", e);
@@ -583,6 +607,7 @@ public class ExcelDataServiceImpl extends ServiceImpl<ExcelDataMapper, ExcelData
         return null;
     }
 
+
     /**
      * 获取逻辑校验公式
      *
@@ -611,8 +636,24 @@ public class ExcelDataServiceImpl extends ServiceImpl<ExcelDataMapper, ExcelData
         return DiConfig.getWord(columnStart + index) + "," + coords[1];
     }
 
+    @SneakyThrows
     public static void main(String[] args) {
-        BigDecimal bigDecimal = new BigDecimal("1234567.123");
-        System.out.println(bigDecimal.setScale(5, BigDecimal.ROUND_DOWN).intValue());
+        ExecutorService executorService = Executors.newFixedThreadPool(10);
+        List<Integer> taskList = Arrays.asList(1, 1, 3, 3, 5, 6, 9, 9, 9, 9);
+        CompletableFuture<?>[] array = taskList.stream()
+                .map(i -> CompletableFuture.runAsync(() -> {
+                    try {
+                        TimeUnit.SECONDS.sleep(i);
+                    } catch (InterruptedException e) {
+                        throw new RuntimeException("休眠异常:"+ i);
+                    }
+                    System.out.println(StrUtil.format("当前线程:{}:当前数据:{}",Thread.currentThread().getName(),i));
+                    if (i == 6){
+                        throw new RuntimeException("指定错误:"+ i);
+                    }
+                }, executorService))
+                .toArray(CompletableFuture<?>[]::new);
+        CompletableFuture.allOf(array).get();
+        System.out.println("结束");
     }
 }

admin-api/src/main/resources/application.yml

@@ -47,7 +47,7 @@ spring:
         # 国际化资源文件路径
         basename: i18n/messages
     profiles:
-        active: druid
+        active: zyy
     # 文件上传
     servlet:
         multipart:
@@ -216,3 +216,7 @@ tx:
 adapter:
     syncPath: http://192.168.0.153:15001/datacenter/rpc/webservice/SubscriptionService?wsdl
     sourceId: Adapt
+
+tlog:
+    pattern: $traceId
+    enable-invoke-time-print: true #打印请求参数与时间
\ No newline at end of file

admin-service/pom.xml

@@ -57,6 +57,11 @@
             <artifactId>commons-compress</artifactId>
             <version>1.12</version>
         </dependency>-->
+        <dependency>
+            <groupId>com.yomahub</groupId>
+            <artifactId>tlog-all-spring-boot-starter</artifactId>
+            <version>1.5.0</version>
+        </dependency>
     </dependencies>
 
 </project>