Skip to content

E
emport-api
Commit 9bae4a21 by 刘泽志
Options

批量异常导出

parent be7c11ba
Showing with 87 additions and 99 deletions
admin-api/src/main/java/com/tbyf/his/web/controller/dataImport/DataFieldController.java
......@@ -9,6 +9,7 @@ import com.tbyf.his.common.annotation.IgnoreWebSecurity;
import com.tbyf.his.common.core.domain.AjaxResult;
import com.tbyf.his.common.core.page.TableDataInfo;
import com.tbyf.his.common.core.text.StrFormatter;
import com.tbyf.his.common.exception.base.BaseException;
import com.tbyf.his.common.utils.StringUtils;
import com.tbyf.his.common.utils.bean.BeanUtils;
import com.tbyf.his.web.dataImport.DataImportUtils;
......@@ -65,6 +66,9 @@ public class DataFieldController {
private MetaFieldService metaFieldService;
@Autowired
private DataRuleService dataRuleService;
@Autowired
private JdbcTemplate jdbcTemplate;
@IgnoreWebSecurity
......@@ -136,15 +140,14 @@ public class DataFieldController {
@ApiOperation("根据基础模板重置字段")
public AjaxResult resetField(@RequestParam String excelId, @RequestParam(required = false) String year) {
final ExcelData excelData = excelDataService.getById(excelId);
// 获取元字段列表
final List<MetaField> metaFields = metaFieldService.list();
// 获取需要同步的字段列表
List<DataField> fieldMatchList = null;
if (StringUtils.isNotBlank(year)) {
DataTemplate template = dataTemplateService.getOne(Wrappers.lambdaQuery(DataTemplate.class)
.eq(DataTemplate::getYear, year).eq(DataTemplate::getOrgName, excelData.getOrgName()), false);
fieldMatchList = dataFieldService.list(Wrappers.lambdaQuery(DataField.class)
.eq(DataField::getTemplateId, template.getId()).isNotNull(DataField::getField));
.eq(DataField::getTemplateId, template.getId()).isNotNull(DataField::getField)
.ne(DataField::getField, ""));
}
// 删除掉之前的模板信息
final LambdaQueryWrapper<DataField> wrapper = Wrappers.lambdaQuery(DataField.class).eq(DataField::getTemplateId, excelData.getTemplateId());
......@@ -152,6 +155,12 @@ public class DataFieldController {
if (!CollectionUtils.isEmpty(list)) {
final List<String> fieldIdList = list.stream().map(DataField::getId).collect(Collectors.toList());
bindRuleService.remove(Wrappers.lambdaQuery(BindRule.class).in(BindRule::getDataId, fieldIdList));
// 模板规则也要删除
List<BindRule> templateRuleList = bindRuleService.list(Wrappers.lambdaQuery(BindRule.class).eq(BindRule::getDataId, excelData.getTemplateId()));
if (!CollectionUtils.isEmpty(templateRuleList)) {
dataRuleService.remove(Wrappers.lambdaQuery(DataRule.class).in(DataRule::getId, templateRuleList.stream().map(BindRule::getRuleId).collect(Collectors.toList())));
}
bindRuleService.remove(Wrappers.lambdaQuery(BindRule.class).in(BindRule::getDataId, Collections.singletonList(excelData.getTemplateId())));
}
dataFieldService.remove(Wrappers.lambdaQuery(DataField.class).eq(DataField::getTemplateId, excelData.getTemplateId()));
try (InputStream is = new ByteArrayInputStream(excelData.getFile()); Workbook workbook = WorkbookFactory.create(is)) {
......@@ -177,8 +186,8 @@ public class DataFieldController {
// TODO 这里坐标修改为i+1是因为excel文件中展示的第一行号是1,符合直觉
dataField.setCoordinate("F," + (i + 1));
dataField.setSort(i + 1);
// 字段绑定需要进行模糊匹配
dataFieldService.fieldMatch(metaFields, dataField, fieldMatchList);
// 字段绑定确定进行精确匹配
dataFieldService.fieldMatch(dataField, fieldMatchList);
fieldList.add(dataField);
}
dataFieldService.saveBatch(fieldList);
......@@ -289,20 +298,20 @@ public class DataFieldController {
@ApiOperation("相同字段检查")
public AjaxResult sameFieldCheck(@RequestParam String templateId) {
List<CreateFieldVO> fieldList = dataFieldService.getCreateFields(templateId);
if (CollectionUtils.isEmpty(fieldList)){
if (CollectionUtils.isEmpty(fieldList)) {
return AjaxResult.success(Collections.emptyList());
}
Map<String, Integer> map = new HashMap<>();
List<CreateFieldVO> list = new ArrayList<>();
for (CreateFieldVO field : fieldList){
if (map.containsKey(field.getFieldName())){
map.put(field.getFieldName(),map.get(field.getFieldName())+1);
}else {
map.put(field.getFieldName(),1);
for (CreateFieldVO field : fieldList) {
if (map.containsKey(field.getFieldName())) {
map.put(field.getFieldName(), map.get(field.getFieldName()) + 1);
} else {
map.put(field.getFieldName(), 1);
}
}
map.forEach((key,value)->{
if (value > 1){
map.forEach((key, value) -> {
if (value > 1) {
Optional<CreateFieldVO> first = fieldList.stream().filter(item -> StringUtils.equals(item.getFieldName(), key)).findFirst();
first.ifPresent(list::add);
}
......
admin-api/src/main/java/com/tbyf/his/web/controller/dataImport/DataRuleController.java
......@@ -182,26 +182,33 @@ public class DataRuleController {
@GetMapping("/sync/year")
@ApiOperation("根据年份同步规则")
public AjaxResult syncRuleByYear(@RequestParam String templateId, @RequestParam String year) {
// 查询两个模板
// 查询两个模板 source-本模板 target-对比同步的模板
DataTemplate sourceTemplate = dataTemplateService.getById(templateId);
DataTemplate targetTemplate = dataTemplateService.getOne(Wrappers.lambdaQuery(DataTemplate.class).eq(DataTemplate::getYear, year).eq(DataTemplate::getOrgName, sourceTemplate.getOrgName()), false);
// 查询本模板的字段
List<DataField> fieldList = dataFieldService.list(Wrappers.lambdaQuery(DataField.class)
.eq(DataField::getTemplateId, sourceTemplate.getId())
.isNotNull(DataField::getField)
.select(DataField::getField, DataField::getId));
// 查询被同步的模板的字段
List<DataField> targetFieldList = dataFieldService.list(Wrappers.lambdaQuery(DataField.class)
.eq(DataField::getTemplateId, targetTemplate.getId())
.isNotNull(DataField::getField)
.select(DataField::getField, DataField::getId));
Set<String> fieldNameSet = fieldList.stream().map(DataField::getField).collect(Collectors.toSet());
if (!CollectionUtils.isEmpty(fieldList)) {
List<BindRule> rules = new ArrayList<>();
// 查询两个模板各自的字段规则
List<BindRule> sourceBindList = bindRuleService.queryRuleList(sourceTemplate.getId());
List<BindRule> targetBindList = bindRuleService.queryRuleList(targetTemplate.getId());
// 字段规则同步
if (!CollectionUtils.isEmpty(targetBindList)) {
// 遍历字段规则
for (BindRule targetRule : targetBindList) {
// 匹配此规则对应的字段ID是否在本模板中存在
Optional<DataField> first = fieldList.stream().filter(df -> StringUtils.equals(df.getField(), targetRule.getDataId())).findFirst();
if (first.isPresent()) {
// 判断本模板是否存在相同的字段及规则
boolean match = sourceBindList.stream()
.anyMatch(br -> StringUtils.equals(br.getDataId(), targetRule.getDataId()) && StringUtils.equals(br.getRuleId(), targetRule.getRuleId()));
if (!match) {
......@@ -218,40 +225,31 @@ public class DataRuleController {
root:
for (DataRule targetRule : targetRules) {
String[] split = targetRule.getContent().split("=");
String[] split1 = split[0].split(",");
String[] split2 = split[1].split(",");
for (int i = 0; i < split1.length; i++) {
int finalI = i;
Optional<DataField> first = targetFieldList.stream().filter(item -> StringUtils.equals(item.getId(), split1[finalI])).findFirst();
if (first.isPresent()) {
Optional<DataField> first1 = fieldList.stream().filter(item -> StringUtils.equals(item.getField(), first.get().getField())).findFirst();
if (first1.isPresent()) {
split1[i] = first1.get().getId();
} else {
continue root;
}
} else {
continue root;
}
}
for (int j = 0; j < split2.length; j++) {
int finalI = j;
Optional<DataField> first = targetFieldList.stream().filter(item -> StringUtils.equals(item.getId(), split2[finalI])).findFirst();
if (first.isPresent()) {
Optional<DataField> first1 = fieldList.stream().filter(item -> StringUtils.equals(item.getField(), first.get().getField())).findFirst();
if (first1.isPresent()) {
split2[j] = first1.get().getId();
} else {
continue root;
}
} else {
// 查询出对应的两边的字段ID
List<String> list1 = Arrays.asList(split[0].split(","));
List<String> list2 = Arrays.asList(split[1].split(","));
// 先判断其中的每个字段是否在源表中存在
boolean match1 = fieldNameSet.containsAll(list1);
boolean match2 = fieldNameSet.containsAll(list2);
if (!match1 || !match2){
continue root;
}
// 再判断此规则是否已经在本模板中存在
boolean match = sourceRules.stream().anyMatch(item -> {
String[] temp = item.getContent().split("=");
// 查询出对应的两边的字段ID
List<String> temp1 = Arrays.asList(temp[0].split(","));
List<String> temp2 = Arrays.asList(temp[1].split(","));
if (temp1.containsAll(list1) && temp2.containsAll(list2)){
return true;
}
return temp1.containsAll(list2) && temp2.containsAll(list1);
});
if (!match){
targetRule.setId(null);
targetRule.setContent(String.join(",", split1) + "=" + String.join(",", split2));
datas.add(targetRule);
}
}
if (!CollectionUtils.isEmpty(datas)) {
dataRuleService.saveBatch(datas);
datas.forEach(item -> {
......
admin-api/src/main/java/com/tbyf/his/web/controller/dataImport/ExcelDataController.java
......@@ -25,6 +25,7 @@ import org.apache.poi.ss.usermodel.*;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.BeanPropertyRowMapper;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.ObjectUtils;
import org.springframework.web.bind.annotation.*;
......@@ -95,6 +96,7 @@ public class ExcelDataController {
@SneakyThrows
@IgnoreWebSecurity
@PostMapping("/upload")
//@Transactional(rollbackFor = Exception.class)
@ApiOperation("文件导入")
public AjaxResult bindRule(UploadExcelParam param) {
if (StringUtils.equals(param.getType(), "1")) {
......@@ -110,6 +112,8 @@ public class ExcelDataController {
}
excelData.setFile(param.getFile().getBytes());
excelData.setFileName(param.getFile().getOriginalFilename());
// 保存后导入文件数据
excelDataService.importData(excelData);
excelDataService.saveOrUpdate(excelData);
} else if (StringUtils.equals(param.getType(), "2")) {
ExcelData excelData = excelDataService.getOne(Wrappers.lambdaQuery(ExcelData.class)
......@@ -203,7 +207,7 @@ public class ExcelDataController {
final int rowStart = Integer.parseInt(orgCodeCo.split(",")[1]);
try (InputStream is = new ByteArrayInputStream(excelData.getFile()); Workbook workbook = WorkbookFactory.create(is)) {
final Sheet sheet = workbook.getSheetAt(0);
String sql = "SELECT ORG_CODE,ORG_NAME,MAILING_ADDRESS,AREA_NUMBER_CODE FROM {} WHERE YEAROOFDATARECORD='{}年'";
String sql = "SELECT ORG_CODE,ORG_NAME,MAILING_ADDRESS,AREA_NUMBER_CODE FROM {} WHERE YEAROOFDATARECORD='{}年' ORDER BY ROWCODE";
try {
DataSourceService.switchDb(template.getDataSourceId());
final List<Map<String, Object>> mapList = jdbcTemplate.queryForList(
......
admin-api/src/main/java/com/tbyf/his/web/dataImport/domain/vo/ExcelVO.java
......@@ -22,7 +22,7 @@ public class ExcelVO {
@Excel(name = "机构类型")
private String orgType;
@Excel(name = "医疗机构名称", width = 20)
@Excel(name = "医疗机构名称", width = 30)
private String orgName;
@Excel(name = "医疗机构代码")
......@@ -43,6 +43,9 @@ public class ExcelVO {
@Excel(name = "错误原因", width = 30)
private String message;
@Excel(isExport=false)
private String areaCode;
public ExcelVO(String message, String value) {
this.message = message;
this.value = value;
......
admin-api/src/main/java/com/tbyf/his/web/dataImport/domain/vo/VerifyVO.java
......@@ -38,6 +38,9 @@ public class VerifyVO {
@ApiModelProperty(value = "排序字段")
private Integer sort;
@ApiModelProperty(value = "元字段ID")
private String metaFieldId;
@ApiModelProperty(value = "数据库字段名")
private String fieldName;
......
admin-api/src/main/java/com/tbyf/his/web/dataImport/service/DataFieldService.java
......@@ -29,11 +29,10 @@ public interface DataFieldService extends IService<DataField> {
/**
* 字段匹配
*
* @param list
* @param field
* @param fieldMatchList
*/
void fieldMatch(List<MetaField> list, DataField field, List<DataField> fieldMatchList);
void fieldMatch(DataField field, List<DataField> fieldMatchList);
/**
* 获取待匹配的字段
......
admin-api/src/main/java/com/tbyf/his/web/dataImport/service/ExcelDataService.java
......@@ -20,4 +20,9 @@ public interface ExcelDataService extends IService<ExcelData> {
*/
void analyzeExport(ExcelData excelData, HttpServletResponse response);
/**
* 导入文件数据
* @param excelData
*/
void importData(ExcelData excelData);
}
admin-api/src/main/java/com/tbyf/his/web/dataImport/service/impl/DataFieldServiceImpl.java
......@@ -18,6 +18,7 @@ import com.tbyf.his.web.dataImport.mapper.DataFieldMapper;
import com.tbyf.his.web.dataImport.service.DataFieldService;
import com.tbyf.his.web.dataImport.service.DataTemplateService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.RegExUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
......@@ -59,63 +60,22 @@ public class DataFieldServiceImpl extends ServiceImpl<DataFieldMapper, DataField
}
@Override
public void fieldMatch(List<MetaField> list, DataField field, List<DataField> fieldMatchList) {
public void fieldMatch(DataField field, List<DataField> fieldMatchList) {
if (StringUtils.isBlank(field.getTitle())) {
return;
}
MetaField metaField = null;
DataField dataField = null;
if (!CollectionUtils.isEmpty(list)) {
list.forEach(item -> {
if (StringUtils.isNotBlank(item.getFieldComment())) {
item.setRatio(DataImportUtils.getSimilarityRatio(field.getTitle(), item.getFieldComment()));
} else {
item.setRatio(0);
}
});
Optional<MetaField> max = list.stream().max(Comparator.comparing(MetaField::getRatio));
if (max.isPresent()) {
metaField = max.get();
}
}
if (!CollectionUtils.isEmpty(fieldMatchList)) {
fieldMatchList.forEach(item -> {
if (StringUtils.isNotBlank(item.getTitle())) {
item.setRatio(DataImportUtils.getSimilarityRatio(field.getTitle(), item.getTitle()));
} else {
item.setRatio(0);
}
Optional<DataField> fieldOptional = fieldMatchList.stream().filter(item -> {
// 1.先判断两个是否相等
if (StringUtils.equals(item.getTitle().trim(), field.getTitle().trim())) {
return true;
}
// 去除五角星号再判断
return StringUtils.equals(item.getTitle().replaceAll("★","").replaceAll("：","").trim(),
field.getTitle().replaceAll("★","").replaceAll("：","").trim());
}).findFirst();
fieldOptional.ifPresent(f ->{
field.setField(f.getField());
});
Optional<DataField> max = fieldMatchList.stream().max(Comparator.comparing(DataField::getRatio));
if (max.isPresent()) {
dataField = max.get();
}
}
if (metaField == null) {
if (dataField != null && dataField.getRatio() >= 0.5) {
log.debug("title:[{}],匹配指标名称:[{}],达成字段:[{}],相似度:[{}]",
field.getTitle(), dataField.getTitle(), dataField.getField(), dataField.getRatio());
field.setField(dataField.getField());
}
} else {
if (dataField == null) {
if (metaField.getRatio() >= 0.5) {
log.debug("title:[{}],匹配指标名称:[{}],达成字段:[{}],相似度:[{}]",
field.getTitle(), metaField.getFieldComment(), metaField.getFieldName(), metaField.getRatio());
field.setField(metaField.getId());
}
} else {
if (dataField.getRatio() >= metaField.getRatio() && dataField.getRatio() >= 0.5) {
log.debug("title:[{}],匹配指标名称:[{}],达成字段:[{}],相似度:[{}]",
field.getTitle(), dataField.getTitle(), dataField.getField(), dataField.getRatio());
field.setField(dataField.getField());
} else if (metaField.getRatio() >= 0.5) {
log.debug("title:[{}],匹配指标名称:[{}],达成字段:[{}],相似度:[{}]",
field.getTitle(), metaField.getFieldComment(), metaField.getFieldName(), metaField.getRatio());
field.setField(metaField.getId());
}
}
}
}
@Override
......
admin-api/src/main/resources/mapper/DataTemplateMapper.xml
......@@ -50,6 +50,7 @@
<result property="sort" column="sort"/>
<result property="fieldName" column="fieldName"/>
<result property="fieldType" column="fieldType"/>
<result property="metaFieldId" column="metaFieldId"/>
<collection property="rules" ofType="com.tbyf.his.web.dataImport.core.RuleVO">
<id property="id" column="ruleId"/>
<result property="name" column="name"/>
......@@ -66,6 +67,7 @@
df.unit as unit,
df.coordinate as coordinate,
df.sort as sort,
df.field as metaFieldId,
mf.field_name as fieldName,
mf.field_type as fieldType,
dr.id as ruleId,
......
admin-service/pom.xml
......@@ -52,6 +52,11 @@
<artifactId>pinyin4j</artifactId>
<version>2.5.0</version>
</dependency>
<!--<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
<version>1.12</version>
</dependency>-->
</dependencies>
</project>
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment