Skip to content

E
emport-api
Commit bd0a1925 by 刘泽志
Options

1:自动生成表 √ 

2:导入四舍五入测试            √
3:插入sql更新                √
4:导入数据要先清除本年度数据    √
5:机构字典
parent b0f6c5ac
Showing with 198 additions and 36 deletions
admin-api/src/main/java/com/tbyf/his/web/controller/dataImport/DataTemplateController.java
......@@ -88,7 +88,7 @@ public class DataTemplateController {
}
@IgnoreWebSecurity
@PostMapping("/template/update")
@PostMapping("/update")
@ApiOperation("修改模板")
public AjaxResult updateTemplate(@RequestBody @Validated UpdateTemplateParam param) {
DataTemplate template = new DataTemplate();
......@@ -99,7 +99,7 @@ public class DataTemplateController {
}
@IgnoreWebSecurity
@GetMapping("/template/delete")
@GetMapping("/delete")
@ApiOperation("删除模板")
public AjaxResult deleteTemplate(@RequestParam String templateId) {
dataTemplateService.removeById(templateId);
......
admin-api/src/main/java/com/tbyf/his/web/dataImport/DataImportUtils.java
package com.tbyf.his.web.dataImport;
import com.tbyf.his.common.core.text.StrFormatter;
import com.tbyf.his.common.utils.DateUtils;
import com.tbyf.his.common.utils.StringUtils;
import com.tbyf.his.web.dataImport.domain.vo.VerifyVO;
import java.util.Date;
import java.util.concurrent.atomic.AtomicLong;
/**
* @author lzz
* @date 2023/2/17 16:01
......@@ -74,13 +82,45 @@ public class DataImportUtils {
return 1 - (float) compare(str, target) / max;
}
public static void main(String[] args) {
System.out.println(getSimilarityRatio("123", "21321"));
System.out.println(getSimilarityRatio("123", "123"));
System.out.println(getSimilarityRatio("中医院", "省中医院"));
System.out.println(getSimilarityRatio("name", "age"));
System.out.println(getSimilarityRatio("指标", "指标%"));
public static final AtomicLong NEXT = new AtomicLong(100000);
/**
* 获取16位唯一id 一秒少于100000条就可以
*
* @return
*/
public static synchronized String getNextId() {
if (NEXT.get() >= 999998) {
NEXT.set(100000);
}
return System.currentTimeMillis() / 1000 + "" + NEXT.getAndIncrement();
}
/**
* 获取插入值的格式化形式
*
* @param vo
* @return
*/
public static String getInsertContent(VerifyVO vo, int index) {
if (StringUtils.containsIgnoreCase(vo.getFieldType(), "VARCHAR")) {
return "'" + vo.getValues().get(index) + "'";
} else if (StringUtils.containsIgnoreCase(vo.getFieldType(), "NUMBER")) {
return "'" + vo.getValues().get(index) + "'";
} else if (StringUtils.containsIgnoreCase(vo.getFieldType(), "DATE")) {
try {
final Date date = DateUtils.parseDate(vo.getValues().get(index), "yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss", "yyyy-MM-dd HH:mm", "yyyy-MM",
"yyyy/MM/dd", "yyyy/MM/dd HH:mm:ss", "yyyy/MM/dd HH:mm", "yyyy/MM",
"yyyy.MM.dd", "yyyy.MM.dd HH:mm:ss", "yyyy.MM.dd HH:mm", "yyyy.MM", "HH:mm");
if (date != null) {
return StrFormatter.format("to_date('{}','YYYY-MM-DD HH24:MI:SS')", DateUtils.parseDateToStr("yyyy-MM-dd HH:mm:ss", date));
}
} catch (Exception ignore) {
}
return "null";
}
return "'" + vo.getValues().get(index) + "'";
}
}
admin-api/src/main/java/com/tbyf/his/web/dataImport/core/DiConstants.java
......@@ -72,4 +72,27 @@ public class DiConstants {
*/
public static final List<String> WHITE = Arrays.asList("-");
/**
* 建表的默认字段
*/
public static final List<String> DEFAULT_FIELD_TEMPLATE = Arrays.asList(
" DORDER VARCHAR2(16) NOT NULL,",
" YEAROOFDATARECORD VARCHAR2(16),",
" SOURCEID VARCHAR2(32),",
" SOURCEDATE DATE,",
" SOURCETYPE VARCHAR2(2),",
" ENTERDATA DATE DEFAULT sysdate,",
" UPLOADDATE DATE,",
" ROWCODE VARCHAR2(32) NOT NULL,"
);
/**
* 建表的默认注释
*/
public static final List<String> DEFAULT_FIELD_COMMENT_TEMPLATE = Arrays.asList(
" COMMENT ON COLUMN @tableName.DORDER IS '顺序号' ",
" COMMENT ON COLUMN @tableName.YEAROOFDATARECORD IS '数据记录年份' ",
" ALTER TABLE @tableName ADD CONSTRAINT PK_@tableName_ROWCODE PRIMARY KEY (ROWCODE) "
);
}
admin-api/src/main/java/com/tbyf/his/web/dataImport/domain/vo/CreateFieldVO.java 0 → 100644
package com.tbyf.his.web.dataImport.domain.vo;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import lombok.experimental.Accessors;
import java.io.Serializable;
/**
* @author lzz
* @date 2023/1/10 15:36
*/
@Data
@Accessors(chain = true)
@ApiModel(value = "建表抽象字段", description = "建表抽象字段")
public class CreateFieldVO implements Serializable {
private static final long serialVersionUID = 1L;
@ApiModelProperty(value = "字段ID")
private String id;
@ApiModelProperty(value = "代码")
private String code;
@ApiModelProperty(value = "指标名称")
private String title;
@ApiModelProperty(value = "排序字段")
private Integer sort;
@ApiModelProperty(value = "元字段ID")
private String fieldId;
@ApiModelProperty(value = "元字段名称")
private String fieldName;
@ApiModelProperty(value = "元字段类型")
private String fieldType;
}
admin-api/src/main/java/com/tbyf/his/web/dataImport/domain/vo/VerifyVO.java
......@@ -39,7 +39,10 @@ public class VerifyVO {
private Integer sort;
@ApiModelProperty(value = "数据库字段名")
private String field;
private String fieldName;
@ApiModelProperty(value = "数据库字段类型")
private String fieldType;
@ApiModelProperty(value = "待校验的值列表")
private List<String> values;
......
admin-api/src/main/java/com/tbyf/his/web/dataImport/mapper/DataFieldMapper.java
......@@ -3,6 +3,7 @@ package com.tbyf.his.web.dataImport.mapper;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.tbyf.his.web.dataImport.domain.param.QueryFieldParam;
import com.tbyf.his.web.dataImport.domain.vo.CreateFieldVO;
import com.tbyf.his.web.dataImport.domain.vo.DataFieldVO;
import com.tbyf.his.web.dataImport.entity.DataField;
import org.apache.ibatis.annotations.Param;
......@@ -32,4 +33,12 @@ public interface DataFieldMapper extends BaseMapper<DataField> {
* @return
*/
List<DataField> listFieldMatchList(String templateId);
/**
* 查询指定模板可以用于建表的字段
*
* @param templateId
* @return
*/
List<CreateFieldVO> getCreateFields(String templateId);
}
admin-api/src/main/java/com/tbyf/his/web/dataImport/service/DataFieldService.java
......@@ -3,6 +3,7 @@ package com.tbyf.his.web.dataImport.service;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.baomidou.mybatisplus.extension.service.IService;
import com.tbyf.his.web.dataImport.domain.param.QueryFieldParam;
import com.tbyf.his.web.dataImport.domain.vo.CreateFieldVO;
import com.tbyf.his.web.dataImport.domain.vo.DataFieldVO;
import com.tbyf.his.web.dataImport.entity.DataField;
import com.tbyf.his.web.dataImport.entity.MetaField;
......@@ -40,4 +41,11 @@ public interface DataFieldService extends IService<DataField> {
*/
List<DataField> listFieldMatchList(String templateId);
/**
* 查询指定模板可以用于建表的字段
*
* @param templateId
* @return
*/
List<CreateFieldVO> getCreateFields(String templateId);
}
admin-api/src/main/java/com/tbyf/his/web/dataImport/service/impl/DataFieldServiceImpl.java
......@@ -7,6 +7,7 @@ import com.tbyf.his.common.enums.DataSourceType;
import com.tbyf.his.common.utils.StringUtils;
import com.tbyf.his.web.dataImport.DataImportUtils;
import com.tbyf.his.web.dataImport.domain.param.QueryFieldParam;
import com.tbyf.his.web.dataImport.domain.vo.CreateFieldVO;
import com.tbyf.his.web.dataImport.domain.vo.DataFieldVO;
import com.tbyf.his.web.dataImport.entity.DataField;
import com.tbyf.his.web.dataImport.entity.MetaField;
......@@ -16,9 +17,7 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import org.springframework.util.ObjectUtils;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.Optional;
......@@ -75,18 +74,26 @@ public class DataFieldServiceImpl extends ServiceImpl<DataFieldMapper, DataField
}
}
if (metaField == null) {
if (dataField != null && dataField.getRatio() > 0.3) {
if (dataField != null && dataField.getRatio() >= 0.5) {
log.debug("title:[{}],匹配指标名称:[{}],达成字段:[{}],相似度:[{}]",
field.getTitle(), dataField.getTitle(), dataField.getField(), dataField.getRatio());
field.setField(dataField.getField());
}
} else {
if (dataField == null) {
if (metaField.getRatio() > 0.3) {
if (metaField.getRatio() >= 0.5) {
log.debug("title:[{}],匹配指标名称:[{}],达成字段:[{}],相似度:[{}]",
field.getTitle(), metaField.getFieldComment(), metaField.getFieldName(), metaField.getRatio());
field.setField(metaField.getId());
}
} else {
if (dataField.getRatio() >= metaField.getRatio()) {
if (dataField.getRatio() >= metaField.getRatio() && dataField.getRatio() >= 0.5) {
log.debug("title:[{}],匹配指标名称:[{}],达成字段:[{}],相似度:[{}]",
field.getTitle(), dataField.getTitle(), dataField.getField(), dataField.getRatio());
field.setField(dataField.getField());
} else {
} else if (metaField.getRatio() >= 0.5) {
log.debug("title:[{}],匹配指标名称:[{}],达成字段:[{}],相似度:[{}]",
field.getTitle(), metaField.getFieldComment(), metaField.getFieldName(), metaField.getRatio());
field.setField(metaField.getId());
}
}
......@@ -97,4 +104,9 @@ public class DataFieldServiceImpl extends ServiceImpl<DataFieldMapper, DataField
public List<DataField> listFieldMatchList(String templateId) {
return dataFieldMapper.listFieldMatchList(templateId);
}
@Override
public List<CreateFieldVO> getCreateFields(String templateId) {
return dataFieldMapper.getCreateFields(templateId);
}
}
admin-api/src/main/java/com/tbyf/his/web/dataImport/service/impl/ExcelDataServiceImpl.java
......@@ -6,7 +6,7 @@ import com.tbyf.his.common.core.text.StrFormatter;
import com.tbyf.his.common.enums.DataSourceType;
import com.tbyf.his.common.utils.StringUtils;
import com.tbyf.his.common.utils.poi.ExcelUtil;
import com.tbyf.his.framework.datasource.DataSourceUtil;
import com.tbyf.his.web.dataImport.DataImportUtils;
import com.tbyf.his.web.dataImport.core.DiConfig;
import com.tbyf.his.web.dataImport.core.DiConstants;
import com.tbyf.his.web.dataImport.core.RuleVO;
......@@ -17,6 +17,7 @@ import com.tbyf.his.web.dataImport.entity.DataRule;
import com.tbyf.his.web.dataImport.entity.DataTemplate;
import com.tbyf.his.web.dataImport.entity.ExcelData;
import com.tbyf.his.web.dataImport.mapper.ExcelDataMapper;
import com.tbyf.his.web.dataImport.service.DataSourceService;
import com.tbyf.his.web.dataImport.service.DataTemplateService;
import com.tbyf.his.web.dataImport.service.ExcelDataService;
import lombok.extern.slf4j.Slf4j;
......@@ -32,6 +33,7 @@ import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
......@@ -96,15 +98,15 @@ public class ExcelDataServiceImpl extends ServiceImpl<ExcelDataMapper, ExcelData
excels[e] = new ExcelVO();
excels[e].setOrgType(template.getOrgName());
for (VerifyVO v : verifyList) {
if (StringUtils.equals("ORG_NAME", v.getField())) {
if (StringUtils.equals("ORG_NAME", v.getFieldName())) {
excels[e].setOrgName(v.getValues().get(e));
continue;
}
if (StringUtils.equals("ORG_CODE", v.getField())) {
if (StringUtils.equals("ORG_CODE", v.getFieldName())) {
excels[e].setOrgCode(v.getValues().get(e));
continue;
}
if (StringUtils.equals("UNIFIED_CODE", v.getField())) {
if (StringUtils.equals("UNIFORM_CREDIT_NUMBER", v.getFieldName())) {
excels[e].setUnifiedCode(v.getValues().get(e));
continue;
}
......@@ -175,10 +177,10 @@ public class ExcelDataServiceImpl extends ServiceImpl<ExcelDataMapper, ExcelData
}
}
// 这里需要进行四舍五入的精确位数处理
if(StringUtils.isNotBlank(rule.getRemarks())){
if(StringUtils.isNotBlank(rule.getRemarks())) {
int scale = Integer.parseInt(rule.getRemarks());
left.setScale(scale,BigDecimal.ROUND_UP);
right.setScale(scale,BigDecimal.ROUND_UP);
left = left.setScale(scale, RoundingMode.UP);
right = right.setScale(scale, RoundingMode.UP);
}
if (left.compareTo(right) != 0) {
ExcelVO excelVO = new ExcelVO();
......@@ -225,27 +227,30 @@ public class ExcelDataServiceImpl extends ServiceImpl<ExcelDataMapper, ExcelData
}
}
// 第三步导入临时表
String sql = "INSERT INTO {}(" + verifyList.stream()
.map(VerifyVO::getField).collect(Collectors.joining(","))
+ ") VALUES ({})";
String insertSql = "INSERT INTO {}(" + verifyList.stream()
.map(VerifyVO::getFieldName).collect(Collectors.joining(","))
+ ",DORDER,YEAROOFDATARECORD,ROWCODE ) VALUES ({})";
String[] sqlArr = new String[length];
for (int i = 0; i < length; i++) {
StringBuilder sb = new StringBuilder();
for (VerifyVO v : verifyList) {
sb.append("'").append(v.getValues().get(i)).append("',");
sb.append(DataImportUtils.getInsertContent(v, i)).append(",");
}
sb.deleteCharAt(sb.length() - 1);
sqlArr[i] = StrFormatter.format(sql, template.getTableName() + "_TEMP", sb);
sb.append("'").append(DataImportUtils.getNextId()).append("','")
.append(template.getYear()).append("年").append("','")
.append(DataImportUtils.getNextId()).append("'");
sqlArr[i] = StrFormatter.format(insertSql, template.getTableName() + "_TEMP", sb);
}
try {
DataSourceUtil.switchDs(template.getDataSourceId());
// 需要先清空临时表的数据
jdbcTemplate.execute("DELETE FROM " + template.getTableName() + "_TEMP");
DataSourceService.switchDb(template.getDataSourceId());
// 需要先清空临时表的数据,按照年份
jdbcTemplate.execute(StrFormatter.format("DELETE FROM {} WHERE YEAROOFDATARECORD = '{}'"
, template.getTableName() + "_TEMP", template.getYear() + "年"));
jdbcTemplate.batchUpdate(sqlArr);
} catch (Exception e) {
throw new RuntimeException(e);
} finally {
DataSourceUtil.switchDefaultDs();
DataSourceService.switchDefault();
}
// 第四步导出异常excel
ExcelUtil<ExcelVO> util = new ExcelUtil<>(ExcelVO.class);
......
admin-api/src/main/resources/mapper/DataFieldMapper.xml
......@@ -29,7 +29,7 @@
and df.coordinate like concat('%', #{param.coordinate}, '%')
</if>
<if test="param.field != null and param.field != ''">
and df.field like concat('%', #{param.field}, '%')
and mf.field_name like concat('%', #{param.field}, '%')
</if>
</where>
</select>
......@@ -48,5 +48,21 @@
WHERE dt.org_name = (SELECT org_name FROM data_template WHERE id = #{templateId})
AND dt.id != #{templateId}
AND df.field is not null
AND df.field != ''
</select>
<select id="getCreateFields" resultType="com.tbyf.his.web.dataImport.domain.vo.CreateFieldVO">
select df.id,
df.code,
df.title,
df.sort,
mf.id as fieldId,
mf.field_name as fieldName,
mf.field_type as fieldType
from data_field df
left join meta_field mf on df.field = mf.id
where df.template_id = #{templateId}
and mf.id is not null
order by df.sort
</select>
</mapper>
admin-api/src/main/resources/mapper/DataTemplateMapper.xml
......@@ -43,7 +43,8 @@
<result property="unit" column="unit"/>
<result property="coordinate" column="coordinate"/>
<result property="sort" column="sort"/>
<result property="field" column="field"/>
<result property="fieldName" column="fieldName"/>
<result property="fieldType" column="fieldType"/>
<collection property="rules" ofType="com.tbyf.his.web.dataImport.core.RuleVO">
<id property="id" column="ruleId"/>
<result property="name" column="name"/>
......@@ -60,17 +61,19 @@
df.unit as unit,
df.coordinate as coordinate,
df.sort as sort,
df.field as field,
mf.field_name as fieldName,
mf.field_type as fieldType,
dr.id as ruleId,
dr.name as name,
dr.type as type,
dr.mode as mode,
dr.content as content
from data_field df
left join meta_field mf on df.field = mf.id
left join bind_rule br on df.id = br.data_id
left join data_rule dr on br.rule_id = dr.id
where df.template_id = #{id}
and df.field is not null
and mf.id is not null
order by df.sort
</select>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment