Skip to content

D
datax-cloud
Commit 7554d4e3 by yuwei
Options

项目初始化

parent e72e349f
Showing with 750 additions and 185 deletions
datax-modules/data-quality-service-parent/data-quality-service-api/src/main/java/cn/datax/service/data/quality/api/dto/Accuracy.java 0 → 100644
package cn.datax.service.data.quality.api.dto;
import lombok.Data;
import java.io.Serializable;
/**
* 准确性
*/
@Data
public class Accuracy implements Serializable {
private static final long serialVersionUID = 1L;
/**
* 最大长度
*/
private Integer maxLength;
}
datax-modules/data-quality-service-parent/data-quality-service-api/src/main/java/cn/datax/service/data/quality/api/dto/CheckRuleDto.java
...@@ -35,6 +35,8 @@ public class CheckRuleDto implements Serializable { ...@@ -35,6 +35,8 @@ public class CheckRuleDto implements Serializable {
private String ruleItemId; private String ruleItemId;
@ApiModelProperty(value = "规则级别（3高、2中、1低）") @ApiModelProperty(value = "规则级别（3高、2中、1低）")
private String ruleLevelId; private String ruleLevelId;
@ApiModelProperty(value = "数据源类型")
private String ruleDbType;
@ApiModelProperty(value = "数据源主键") @ApiModelProperty(value = "数据源主键")
private String ruleSourceId; private String ruleSourceId;
@ApiModelProperty(value = "数据源") @ApiModelProperty(value = "数据源")
......
datax-modules/data-quality-service-parent/data-quality-service-api/src/main/java/cn/datax/service/data/quality/api/dto/Consistent.java 0 → 100644
package cn.datax.service.data.quality.api.dto;
import lombok.Data;
import java.io.Serializable;
/**
* 一致性
*/
@Data
public class Consistent implements Serializable {
private static final long serialVersionUID = 1L;
private String gbTypeId;
private String gbTypeCode;
private String gbTypeName;
private String bindGbColumn;
}
datax-modules/data-quality-service-parent/data-quality-service-api/src/main/java/cn/datax/service/data/quality/api/dto/Relevance.java 0 → 100644
package cn.datax.service.data.quality.api.dto;
import lombok.Data;
import java.io.Serializable;
/**
* 关联性
*/
@Data
public class Relevance implements Serializable {
private static final long serialVersionUID = 1L;
private String relatedTableId;
private String relatedTable;
private String relatedTableComment;
private String relatedColumnId;
private String relatedColumn;
private String relatedColumnComment;
}
datax-modules/data-quality-service-parent/data-quality-service-api/src/main/java/cn/datax/service/data/quality/api/dto/RuleConfig.java
package cn.datax.service.data.quality.api.dto; package cn.datax.service.data.quality.api.dto;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data; import lombok.Data;
import java.io.Serializable; import java.io.Serializable;
...@@ -9,5 +10,26 @@ public class RuleConfig implements Serializable { ...@@ -9,5 +10,26 @@ public class RuleConfig implements Serializable {
private static final long serialVersionUID=1L; private static final long serialVersionUID=1L;
@ApiModelProperty(value = "核查类型编码")
private String ruleItemCode; private String ruleItemCode;
/**
* 一致性
*/
private Consistent consistent;
/**
* 关联性
*/
private Relevance relevance;
/**
* 及时性
*/
private Timeliness timeliness;
/**
* 准确性
*/
private Accuracy accuracy;
} }
datax-modules/data-quality-service-parent/data-quality-service-api/src/main/java/cn/datax/service/data/quality/api/dto/Timeliness.java 0 → 100644
package cn.datax.service.data.quality.api.dto;
import lombok.Data;
import java.io.Serializable;
/**
* 及时性
*/
@Data
public class Timeliness implements Serializable {
private static final long serialVersionUID = 1L;
/**
* 判定阀值 当前时间-业务时间>阀值
*/
private Integer threshold;
}
datax-modules/data-quality-service-parent/data-quality-service-api/src/main/java/cn/datax/service/data/quality/api/entity/CheckRuleEntity.java
...@@ -55,6 +55,11 @@ public class CheckRuleEntity extends DataScopeBaseEntity { ...@@ -55,6 +55,11 @@ public class CheckRuleEntity extends DataScopeBaseEntity {
private String ruleLevel; private String ruleLevel;
/** /**
* 数据源类型
*/
private String ruleDbType;
/**
* 数据源主键 * 数据源主键
*/ */
private String ruleSourceId; private String ruleSourceId;
......
datax-modules/data-quality-service-parent/data-quality-service-api/src/main/java/cn/datax/service/data/quality/api/enums/RuleItem.java
...@@ -3,7 +3,7 @@ package cn.datax.service.data.quality.api.enums; ...@@ -3,7 +3,7 @@ package cn.datax.service.data.quality.api.enums;
public enum RuleItem { public enum RuleItem {
Unique("unique_key", "验证用户指定的字段是否具有唯一性"), Unique("unique_key", "验证用户指定的字段是否具有唯一性"),
AccuracyLlength("accuracy_key_length", "验证长度是否符合规定"), AccuracyLength("accuracy_key_length", "验证长度是否符合规定"),
Integrity("integrity_key", "验证表中必须出现的字段非空"), Integrity("integrity_key", "验证表中必须出现的字段非空"),
Relevance("relevance_key", "验证关联性"), Relevance("relevance_key", "验证关联性"),
Timeliness("timeliness_key", "验证及时性"), Timeliness("timeliness_key", "验证及时性"),
...@@ -21,4 +21,13 @@ public enum RuleItem { ...@@ -21,4 +21,13 @@ public enum RuleItem {
public String getCode() { public String getCode() {
return code; return code;
} }
public static RuleItem getRuleItem(String code) {
for (RuleItem item : RuleItem.values()) {
if (item.code.equals(code)) {
return item;
}
}
return null;
}
} }
datax-modules/data-quality-service-parent/data-quality-service-api/src/main/java/cn/datax/service/data/quality/api/vo/CheckRuleVo.java
...@@ -31,6 +31,7 @@ public class CheckRuleVo implements Serializable { ...@@ -31,6 +31,7 @@ public class CheckRuleVo implements Serializable {
private String ruleType; private String ruleType;
private String ruleLevelId; private String ruleLevelId;
private String ruleLevel; private String ruleLevel;
private String ruleDbType;
private String ruleSourceId; private String ruleSourceId;
private String ruleSource; private String ruleSource;
private String ruleTableId; private String ruleTableId;
......
datax-modules/data-quality-service-parent/data-quality-service/pom.xml
...@@ -79,6 +79,11 @@ ...@@ -79,6 +79,11 @@
<artifactId>data-metadata-service-api</artifactId> <artifactId>data-metadata-service-api</artifactId>
<version>2.0.0</version> <version>2.0.0</version>
</dependency> </dependency>
<dependency>
<groupId>cn.datax</groupId>
<artifactId>data-standard-service-api</artifactId>
<version>2.0.0</version>
</dependency>
</dependencies> </dependencies>
<build> <build>
......
datax-modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/controller/CheckReportController.java
...@@ -76,7 +76,7 @@ public class CheckReportController extends BaseController { ...@@ -76,7 +76,7 @@ public class CheckReportController extends BaseController {
queryWrapper.like(StrUtil.isNotBlank(checkReportQuery.getRuleTable()), "r.rule_table", checkReportQuery.getRuleTable()); queryWrapper.like(StrUtil.isNotBlank(checkReportQuery.getRuleTable()), "r.rule_table", checkReportQuery.getRuleTable());
queryWrapper.like(StrUtil.isNotBlank(checkReportQuery.getRuleColumn()), "r.rule_column", checkReportQuery.getRuleColumn()); queryWrapper.like(StrUtil.isNotBlank(checkReportQuery.getRuleColumn()), "r.rule_column", checkReportQuery.getRuleColumn());
// 确定唯一核查报告 // 确定唯一核查报告
queryWrapper.eq("c.check_batch", "r.last_check_batch"); queryWrapper.apply("c.check_batch = r.last_check_batch");
IPage<CheckReportEntity> page = checkReportService.page(new Page<>(checkReportQuery.getPageNum(), checkReportQuery.getPageSize()), queryWrapper); IPage<CheckReportEntity> page = checkReportService.page(new Page<>(checkReportQuery.getPageNum(), checkReportQuery.getPageSize()), queryWrapper);
List<CheckReportVo> collect = page.getRecords().stream().map(checkReportMapper::toVO).collect(Collectors.toList()); List<CheckReportVo> collect = page.getRecords().stream().map(checkReportMapper::toVO).collect(Collectors.toList());
JsonPage<CheckReportVo> jsonPage = new JsonPage<>(page.getCurrent(), page.getSize(), page.getTotal(), collect); JsonPage<CheckReportVo> jsonPage = new JsonPage<>(page.getCurrent(), page.getSize(), page.getTotal(), collect);
......
datax-modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/controller/RuleItemController.java
...@@ -59,8 +59,10 @@ public class RuleItemController extends BaseController { ...@@ -59,8 +59,10 @@ public class RuleItemController extends BaseController {
@ApiOperation(value = "获取列表", notes = "") @ApiOperation(value = "获取列表", notes = "")
@GetMapping("/list") @GetMapping("/list")
public R getRuleTypeList() { public R getRuleTypeList(RuleItemQuery ruleItemQuery) {
List<RuleItemEntity> list = ruleItemService.list(Wrappers.emptyWrapper()); QueryWrapper<RuleItemEntity> queryWrapper = new QueryWrapper<>();
queryWrapper.eq(StrUtil.isNotBlank(ruleItemQuery.getRuleTypeId()), "rule_type_id", ruleItemQuery.getRuleTypeId());
List<RuleItemEntity> list = ruleItemService.list(queryWrapper);
return R.ok().setData(list); return R.ok().setData(list);
} }
......
datax-modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/schedule/rules/AccuracyLlengthRule.java → datax-modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/schedule/rules/AccuracyLengthRule.java
package cn.datax.service.data.quality.schedule.rules; package cn.datax.service.data.quality.schedule.rules;
import cn.datax.common.database.constants.DbType;
import java.util.Map;
/** /**
* 准确性核查 * 准确性核查
* 核查项:长度 * 核查项:最大长度
* select sum(case when length(column) > 15 then 1 else 0 end), count(*) from table; * select sum(case when length(column) > 15 then 1 else 0 end), count(*) from table;
*/ */
public class AccuracyLlengthRule implements RuleItem { public class AccuracyLengthRule implements RuleItem {
private static String MAX_LENGTH = "max_length";
@Override @Override
public String parse(String table, String column) { public String parse(DbType dbType, String table, String column, Map<String, Object> map) {
final StringBuilder builder = new StringBuilder(); final StringBuilder builder = new StringBuilder();
builder.append("SELECT SUM(CASE WHEN LENGTH(").append(column).append(") > 15 THEN 1 ELSE 0 END), COUNT(*) FROM ").append(table); builder.append("SELECT SUM(CASE WHEN LENGTH(").append(column).append(") > ").append(map.get(MAX_LENGTH)).append(" THEN 1 ELSE 0 END), COUNT(*) FROM ").append(table);
return builder.toString(); return builder.toString();
} }
......
datax-modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/schedule/rules/ConsistentRule.java
package cn.datax.service.data.quality.schedule.rules; package cn.datax.service.data.quality.schedule.rules;
import cn.datax.common.database.constants.DbType;
import java.util.Map;
/** /**
* 一致性核查 * 一致性核查
* 核查项:字典 * 核查项:字典
...@@ -7,10 +11,12 @@ package cn.datax.service.data.quality.schedule.rules; ...@@ -7,10 +11,12 @@ package cn.datax.service.data.quality.schedule.rules;
*/ */
public class ConsistentRule implements RuleItem { public class ConsistentRule implements RuleItem {
private static String GB_ITEM = "gb_item";
@Override @Override
public String parse(String table, String column) { public String parse(DbType dbType, String table, String column, Map<String, Object> map) {
final StringBuilder builder = new StringBuilder(); final StringBuilder builder = new StringBuilder();
builder.append("SELECT SUM(CASE WHEN ").append(column).append(" NOT IN ('0', '1') THEN 1 ELSE 0 END), COUNT(*) FROM ").append(table); builder.append("SELECT SUM(CASE WHEN ").append(column).append(" NOT IN (").append(map.get(GB_ITEM)).append(") THEN 1 ELSE 0 END), COUNT(*) FROM ").append(table);
return builder.toString(); return builder.toString();
} }
......
datax-modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/schedule/rules/IntegrityRule.java
package cn.datax.service.data.quality.schedule.rules; package cn.datax.service.data.quality.schedule.rules;
import cn.datax.common.database.constants.DbType;
import java.util.Map;
/** /**
* 完整性核查 * 完整性核查
* 核查项:非空 * 核查项:非空
...@@ -8,7 +12,7 @@ package cn.datax.service.data.quality.schedule.rules; ...@@ -8,7 +12,7 @@ package cn.datax.service.data.quality.schedule.rules;
public class IntegrityRule implements RuleItem { public class IntegrityRule implements RuleItem {
@Override @Override
public String parse(String table, String column) { public String parse(DbType dbType, String table, String column, Map<String, Object> map) {
final StringBuilder builder = new StringBuilder(); final StringBuilder builder = new StringBuilder();
builder.append("SELECT SUM(CASE WHEN ").append(column).append(" IS NOT NULL AND TRIM(").append(column).append(") != '' THEN 1 ELSE 0 END), COUNT(*) FROM ").append(table); builder.append("SELECT SUM(CASE WHEN ").append(column).append(" IS NOT NULL AND TRIM(").append(column).append(") != '' THEN 1 ELSE 0 END), COUNT(*) FROM ").append(table);
return builder.toString(); return builder.toString();
......
datax-modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/schedule/rules/RelevanceRule.java
package cn.datax.service.data.quality.schedule.rules; package cn.datax.service.data.quality.schedule.rules;
import cn.datax.common.database.constants.DbType;
import java.util.Map;
/** /**
* 关联性核查 * 关联性核查
* select SUM(errorCount) errorCount, SUM(totalCount) totalCount
* FROM (
* select count(*) errorCount, 0 as totalCount from MAIN_TABLE a where not exists(select 1 from FOLLOW_TWO b where a.NAME = b.NAME)
* union select 0 as errorCount, count(*) totalCount from MAIN_TABLE
* ) temp;
*/ */
public class RelevanceRule implements RuleItem { public class RelevanceRule implements RuleItem {
private static String RELATED_TABLE = "related_table";
private static String RELATED_COLUMN = "related_column";
@Override @Override
public String parse(String table, String column) { public String parse(DbType dbType, String table, String column, Map<String, Object> map) {
return null; final StringBuilder builder = new StringBuilder();
builder.append("SELECT SUM(errorCount) AS errorCount, SUM(totalCount) AS totalCount FROM (")
.append("SELECT COUNT(*) AS errorCount, 0 AS totalCount FROM ")
.append(table).append(" a WHERE NOT EXISTS (SELECT 1 FROM ").append(map.get(RELATED_TABLE)).append(" b WHERE a.").append(column).append(" = b.").append(map.get(RELATED_COLUMN)).append(")")
.append("UNION SELECT 0 AS errorCount, COUNT(*) AS totalCount FROM ").append(table).append(") TEMP");
return builder.toString();
} }
@Override @Override
......
datax-modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/schedule/rules/RuleItem.java
package cn.datax.service.data.quality.schedule.rules; package cn.datax.service.data.quality.schedule.rules;
import cn.datax.common.database.constants.DbType;
import java.util.Map;
public interface RuleItem { public interface RuleItem {
String parse(String table, String column); String parse(DbType dbType, String table, String column, Map<String, Object> map);
String code(); String code();
} }
datax-modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/schedule/rules/RuleItemRegistry.java
...@@ -13,7 +13,7 @@ public class RuleItemRegistry { ...@@ -13,7 +13,7 @@ public class RuleItemRegistry {
this.rule_item_map.put("integrity_key", new IntegrityRule()); this.rule_item_map.put("integrity_key", new IntegrityRule());
this.rule_item_map.put("relevance_key", new RelevanceRule()); this.rule_item_map.put("relevance_key", new RelevanceRule());
this.rule_item_map.put("timeliness_key", new TimelinessRule()); this.rule_item_map.put("timeliness_key", new TimelinessRule());
this.rule_item_map.put("accuracy_key_length", new AccuracyLlengthRule()); this.rule_item_map.put("accuracy_key_length", new AccuracyLengthRule());
} }
public RuleItem getRuleItem(String code) { public RuleItem getRuleItem(String code) {
......
datax-modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/schedule/rules/TimelinessRule.java
package cn.datax.service.data.quality.schedule.rules; package cn.datax.service.data.quality.schedule.rules;
import cn.datax.common.database.constants.DbType;
import java.util.Map;
/** /**
* 及时性核查 * 及时性核查
*/ */
public class TimelinessRule implements RuleItem { public class TimelinessRule implements RuleItem {
private static String THRESHOLD = "threshold";
@Override @Override
public String parse(String table, String column) { public String parse(DbType dbType, String table, String column, Map<String, Object> map) {
return null; final StringBuilder builder = new StringBuilder();
switch (dbType) {
case ORACLE:
case ORACLE_12C:
builder.append("SELECT SUM(CASE WHEN ROUND(TO_NUMBER(SYSDATE - ").append(column).append(")) >= ").append(map.get(THRESHOLD)).append(" THEN 1 ELSE 0 END), COUNT(*) FROM ").append(table);
break;
case MYSQL:
case MARIADB:
builder.append("SELECT SUM(CASE WHEN DATEDIFF(NOW(), ").append(column).append(") >= ").append(map.get(THRESHOLD)).append(" THEN 1 ELSE 0 END), COUNT(*) FROM ").append(table);
break;
case SQL_SERVER:
case SQL_SERVER2008:
builder.append("SELECT SUM(CASE WHEN DATEDIFF(DAY, ").append(column).append(", GETDATE()) >= ").append(map.get(THRESHOLD)).append(" THEN 1 ELSE 0 END), COUNT(*) FROM ").append(table);
break;
case POSTGRE_SQL:
case OTHER:
default:
break;
}
return builder.toString();
} }
@Override @Override
......
datax-modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/schedule/rules/UniqueRule.java
package cn.datax.service.data.quality.schedule.rules; package cn.datax.service.data.quality.schedule.rules;
import cn.datax.common.database.constants.DbType;
import java.util.Map;
/** /**
* 唯一性核查 * 唯一性核查
* 核查项:主键 * 核查项:主键
...@@ -8,11 +12,11 @@ package cn.datax.service.data.quality.schedule.rules; ...@@ -8,11 +12,11 @@ package cn.datax.service.data.quality.schedule.rules;
public class UniqueRule implements RuleItem { public class UniqueRule implements RuleItem {
@Override @Override
public String parse(String table, String column) { public String parse(DbType dbType, String table, String column, Map<String, Object> map) {
final StringBuilder builder = new StringBuilder(); final StringBuilder builder = new StringBuilder();
builder.append("SELECT totalCount-count AS errorCount, totalCount FROM ("); builder.append("SELECT totalCount - errorCount AS errorCount, totalCount FROM (");
builder.append("SELECT COUNT(DISTINCT ").append(column).append(") AS count, COUNT(*) AS totalCount FROM ").append(table); builder.append("SELECT COUNT(DISTINCT ").append(column).append(") AS errorCount, COUNT(*) AS totalCount FROM ").append(table);
builder.append(") AS TEMP"); builder.append(") TEMP");
return builder.toString(); return builder.toString();
} }
......
datax-modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/schedule/task/QualityTask.java
...@@ -2,17 +2,18 @@ package cn.datax.service.data.quality.schedule.task; ...@@ -2,17 +2,18 @@ package cn.datax.service.data.quality.schedule.task;
import cn.datax.common.core.DataConstant; import cn.datax.common.core.DataConstant;
import cn.datax.common.database.DataSourceFactory; import cn.datax.common.database.DataSourceFactory;
import cn.datax.common.database.DbQuery;
import cn.datax.common.database.constants.DbQueryProperty;
import cn.datax.common.exception.DataException;
import cn.datax.common.utils.SpringContextHolder;
import cn.datax.service.data.metadata.api.dto.DbSchema;
import cn.datax.service.data.metadata.api.entity.MetadataSourceEntity;
import cn.datax.service.data.metadata.api.feign.MetadataSourceServiceFeign; import cn.datax.service.data.metadata.api.feign.MetadataSourceServiceFeign;
import cn.datax.service.data.quality.api.entity.CheckReportEntity; import cn.datax.service.data.quality.api.entity.CheckReportEntity;
import cn.datax.service.data.quality.api.entity.CheckRuleEntity; import cn.datax.service.data.quality.api.entity.CheckRuleEntity;
import cn.datax.service.data.quality.api.entity.RuleItemEntity;
import cn.datax.service.data.quality.api.entity.ScheduleLogEntity; import cn.datax.service.data.quality.api.entity.ScheduleLogEntity;
import cn.datax.service.data.quality.schedule.exception.ChildThreadException;
import cn.datax.service.data.quality.schedule.thread.MultiThreadHandler;
import cn.datax.service.data.quality.schedule.thread.parallel.ParallelTaskWithThreadPool;
import cn.datax.service.data.quality.service.CheckReportService; import cn.datax.service.data.quality.service.CheckReportService;
import cn.datax.service.data.quality.service.CheckRuleService; import cn.datax.service.data.quality.service.CheckRuleService;
import cn.datax.service.data.quality.service.RuleItemService;
import cn.datax.service.data.quality.service.ScheduleLogService; import cn.datax.service.data.quality.service.ScheduleLogService;
import cn.hutool.core.util.StrUtil; import cn.hutool.core.util.StrUtil;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
...@@ -22,9 +23,15 @@ import org.apache.commons.lang3.concurrent.BasicThreadFactory; ...@@ -22,9 +23,15 @@ import org.apache.commons.lang3.concurrent.BasicThreadFactory;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.time.LocalDateTime;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Optional;
import java.util.concurrent.*; import java.util.concurrent.*;
@Slf4j @Slf4j
...@@ -35,15 +42,6 @@ public class QualityTask { ...@@ -35,15 +42,6 @@ public class QualityTask {
private CheckRuleService checkRuleService; private CheckRuleService checkRuleService;
@Autowired @Autowired
private RuleItemService ruleItemService;
@Autowired
private DataSourceFactory dataSourceFactory;
@Autowired
private MetadataSourceServiceFeign metadataSourceServiceFeign;
@Autowired
private CheckReportService checkReportService; private CheckReportService checkReportService;
@Autowired @Autowired
...@@ -54,23 +52,29 @@ public class QualityTask { ...@@ -54,23 +52,29 @@ public class QualityTask {
List<CheckReportEntity> result = new ArrayList<>(); List<CheckReportEntity> result = new ArrayList<>();
// 获取可执行的核查规则 // 获取可执行的核查规则
List<CheckRuleEntity> list = checkRuleService.list(Wrappers.<CheckRuleEntity>lambdaQuery().eq(CheckRuleEntity::getStatus, DataConstant.TrueOrFalse.TRUE.getKey())); List<CheckRuleEntity> list = checkRuleService.list(Wrappers.<CheckRuleEntity>lambdaQuery().eq(CheckRuleEntity::getStatus, DataConstant.TrueOrFalse.TRUE.getKey()));
// 获取核查类型
List<RuleItemEntity> ruleItemList = ruleItemService.list(Wrappers.emptyWrapper());
// 定义固定长度的线程池 // 定义固定长度的线程池
ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(5, 5, 0L, TimeUnit.MILLISECONDS, ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(5, 5, 0L, TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<Runnable>(50), new LinkedBlockingQueue<Runnable>(50),
new BasicThreadFactory.Builder().namingPattern("executor-schedule-pool-%d").daemon(true).build()); new BasicThreadFactory.Builder().namingPattern("executor-schedule-pool-%d").daemon(true).build());
MultiThreadHandler handler = new ParallelTaskWithThreadPool(threadPoolExecutor); // 定义计数器
// 启动子线程作为要处理的并行任务 final CountDownLatch latch = new CountDownLatch(list.size());
// Callable用于产生结果
List<TaskHander> tasks = new ArrayList<>();
list.stream().forEach(rule -> { list.stream().forEach(rule -> {
RuleItemEntity ruleItem = ruleItemList.stream().filter(item -> item.getId().equals(rule.getRuleItemId())).findFirst().get(); TaskHander task = new TaskHander(latch, rule);
TaskHander task = new TaskHander(metadataSourceServiceFeign, dataSourceFactory, ruleItem, rule, result); tasks.add(task);
handler.addTask(task);
}); });
List<Future<CheckReportEntity>> futures;
try { try {
handler.run(); futures = threadPoolExecutor.invokeAll(tasks);
} catch (ChildThreadException e) { // 处理线程返回结果
System.out.println(e.getAllStackTraceMessage()); for (Future<CheckReportEntity> future : futures) {
result.add(future.get());
}
// 主线程阻塞，等待所有子线程执行完成
latch.await();
} catch (Exception e) {
e.printStackTrace();
} }
// 关闭线程池 // 关闭线程池
threadPoolExecutor.shutdown(); threadPoolExecutor.shutdown();
...@@ -80,12 +84,12 @@ public class QualityTask { ...@@ -80,12 +84,12 @@ public class QualityTask {
String status = StrUtil.isBlank(s.getCheckResult()) ? DataConstant.TrueOrFalse.TRUE.getKey() : DataConstant.TrueOrFalse.FALSE.getKey(); String status = StrUtil.isBlank(s.getCheckResult()) ? DataConstant.TrueOrFalse.TRUE.getKey() : DataConstant.TrueOrFalse.FALSE.getKey();
if (StrUtil.isBlank(s.getCheckResult())) { if (StrUtil.isBlank(s.getCheckResult())) {
s.setCheckBatch((String) map.get("batch")); s.setCheckBatch((String) map.get("batch"));
// checkReportService.save(s); checkReportService.save(s);
// 更新最近核查批次号 // 更新最近核查批次号
LambdaUpdateWrapper<CheckRuleEntity> updateWrapper = new LambdaUpdateWrapper<>(); LambdaUpdateWrapper<CheckRuleEntity> updateWrapper = new LambdaUpdateWrapper<>();
updateWrapper.set(CheckRuleEntity::getLastCheckBatch, (String) map.get("batch")); updateWrapper.set(CheckRuleEntity::getLastCheckBatch, (String) map.get("batch"));
updateWrapper.eq(CheckRuleEntity::getId, s.getCheckRuleId()); updateWrapper.eq(CheckRuleEntity::getId, s.getCheckRuleId());
// checkRuleService.update(updateWrapper); checkRuleService.update(updateWrapper);
} }
// 定时任务日志 // 定时任务日志
ScheduleLogEntity scheduleLogEntity = new ScheduleLogEntity(); ScheduleLogEntity scheduleLogEntity = new ScheduleLogEntity();
...@@ -95,7 +99,67 @@ public class QualityTask { ...@@ -95,7 +99,67 @@ public class QualityTask {
scheduleLogEntity.setExecuteRuleId(s.getCheckRuleId()); scheduleLogEntity.setExecuteRuleId(s.getCheckRuleId());
scheduleLogEntity.setExecuteResult(s.getCheckResult()); scheduleLogEntity.setExecuteResult(s.getCheckResult());
scheduleLogEntity.setStatus(status); scheduleLogEntity.setStatus(status);
// scheduleLogService.save(scheduleLogEntity); scheduleLogService.save(scheduleLogEntity);
}); });
} }
static class TaskHander implements Callable<CheckReportEntity> {
private CountDownLatch latch;
private CheckRuleEntity checkRuleEntity;
public TaskHander(CountDownLatch latch, CheckRuleEntity checkRuleEntity) {
super();
this.latch = latch;
this.checkRuleEntity = checkRuleEntity;
}
@Override
public CheckReportEntity call() {
log.info("任务 - 规则id：{}，规则名称：{}， 时间：{}", checkRuleEntity.getId(), checkRuleEntity.getRuleName(), System.currentTimeMillis());
CheckReportEntity checkReportEntity = new CheckReportEntity();
checkReportEntity.setCheckRuleId(checkRuleEntity.getId());
checkReportEntity.setCheckDate(LocalDateTime.now());
Connection conn = null;
Statement stmt = null;
ResultSet rs = null;
try {
MetadataSourceServiceFeign metadataSourceServiceFeign = SpringContextHolder.getBean(MetadataSourceServiceFeign.class);
MetadataSourceEntity dataSource = Optional.ofNullable(metadataSourceServiceFeign.getMetadataSourceById(checkRuleEntity.getRuleSourceId())).orElseThrow(() -> new DataException("获取数据源接口出错"));
DbSchema dbSchema = dataSource.getDbSchema();
DbQueryProperty dbQueryProperty = new DbQueryProperty(dataSource.getDbType(), dbSchema.getHost(),
dbSchema.getUsername(), dbSchema.getPassword(), dbSchema.getPort(), dbSchema.getDbName(), dbSchema.getSid());
DataSourceFactory dataSourceFactory = SpringContextHolder.getBean(DataSourceFactory.class);
DbQuery dbQuery = Optional.ofNullable(dataSourceFactory.createDbQuery(dbQueryProperty)).orElseThrow(() -> new DataException("创建数据查询接口出错"));
conn = dbQuery.getConnection();
stmt = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
rs = stmt.executeQuery(checkRuleEntity.getRuleSql());
while (rs.next()) {
Integer checkErrorCount = rs.getInt(1);
checkReportEntity.setCheckErrorCount(checkErrorCount);
Integer checkTotalCount = rs.getInt(2);
checkReportEntity.setCheckTotalCount(checkTotalCount);
}
} catch (Exception e) {
checkReportEntity.setCheckResult(e.getMessage());
} finally {
latch.countDown();
try {
if(rs != null){
rs.close();
}
if (stmt != null) {
stmt.close();
}
if (conn != null) {
conn.close();
}
} catch (SQLException e) {
checkReportEntity.setCheckResult("释放数据库连接出错");
}
return checkReportEntity;
}
}
}
} }
datax-modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/schedule/task/TaskHander.java deleted 100644 → 0
package cn.datax.service.data.quality.schedule.task;
import cn.datax.common.database.DataSourceFactory;
import cn.datax.common.database.DbQuery;
import cn.datax.common.database.constants.DbQueryProperty;
import cn.datax.common.exception.DataException;
import cn.datax.service.data.metadata.api.dto.DbSchema;
import cn.datax.service.data.metadata.api.entity.MetadataSourceEntity;
import cn.datax.service.data.metadata.api.feign.MetadataSourceServiceFeign;
import cn.datax.service.data.quality.api.entity.CheckReportEntity;
import cn.datax.service.data.quality.api.entity.CheckRuleEntity;
import cn.datax.service.data.quality.api.entity.RuleItemEntity;
import cn.datax.service.data.quality.schedule.CheckRuleFactory;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Optional;
public class TaskHander implements Runnable {
private MetadataSourceServiceFeign metadataSourceServiceFeign;
private DataSourceFactory dataSourceFactory;
private RuleItemEntity ruleItemEntity;
private CheckRuleEntity checkRuleEntity;
private List<CheckReportEntity> list;
public TaskHander(MetadataSourceServiceFeign metadataSourceServiceFeign, DataSourceFactory dataSourceFactory, RuleItemEntity ruleItemEntity, CheckRuleEntity checkRuleEntity, List<CheckReportEntity> list) {
super();
this.metadataSourceServiceFeign = metadataSourceServiceFeign;
this.dataSourceFactory = dataSourceFactory;
this.ruleItemEntity = ruleItemEntity;
this.checkRuleEntity = checkRuleEntity;
this.list = list;
}
@Override
public void run() {
CheckReportEntity checkReportEntity = new CheckReportEntity();
checkReportEntity.setCheckRuleId(checkRuleEntity.getId());
checkReportEntity.setCheckDate(LocalDateTime.now());
MetadataSourceEntity dataSource = Optional.ofNullable(metadataSourceServiceFeign.getMetadataSourceById(checkRuleEntity.getRuleSourceId())).orElseThrow(() -> {
checkReportEntity.setCheckResult("获取数据源接口出错");
list.add(checkReportEntity);
return new DataException("获取数据源接口出错");
});
// if (dataSourceOptional.isPresent()) {
// dataSource = dataSourceOptional.get();
// } else {
// checkReportEntity.setCheckResult("获取数据源接口出错");
// list.add(checkReportEntity);
// dataSourceOptional.orElseThrow(DataException::new);
// }
DbSchema dbSchema = dataSource.getDbSchema();
DbQueryProperty dbQueryProperty = new DbQueryProperty(dataSource.getDbType(), dbSchema.getHost(),
dbSchema.getUsername(), dbSchema.getPassword(), dbSchema.getPort(), dbSchema.getDbName(), dbSchema.getSid());
DbQuery dbQuery = Optional.ofNullable(dataSourceFactory.createDbQuery(dbQueryProperty)).orElseThrow(() -> {
checkReportEntity.setCheckResult("创建数据查询接口出错");
list.add(checkReportEntity);
return new DataException("创建数据查询接口出错");
});
// if (dbQueryOptional.isPresent()) {
// dbQuery = dbQueryOptional.get();
// } else {
// checkReportEntity.setCheckResult("创建数据查询接口出错");
// list.add(checkReportEntity);
// dbQueryOptional.orElseThrow(DataException::new);
// }
String sql = CheckRuleFactory.getRuleItem(ruleItemEntity.getItemCode()).parse(checkRuleEntity.getRuleTable(), checkRuleEntity.getRuleColumn());
System.out.println(sql);
Connection conn = dbQuery.getConnection();
Statement stmt = null;
ResultSet rs = null;
try {
stmt = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
rs = stmt.executeQuery(checkRuleEntity.getRuleSql());
while (rs.next()) {
}
list.add(checkReportEntity);
} catch (Exception e) {
checkReportEntity.setCheckResult(e.getMessage());
list.add(checkReportEntity);
throw new DataException(e);
} finally {
try {
if(rs != null){
rs.close();
}
if (stmt != null) {
stmt.close();
}
if (conn != null) {
conn.close();
}
} catch (SQLException e) {
checkReportEntity.setCheckResult("释放数据库连接出错");
list.add(checkReportEntity);
throw new DataException("释放数据库连接出错");
}
}
}
}
datax-modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/service/impl/CheckRuleServiceImpl.java
package cn.datax.service.data.quality.service.impl; package cn.datax.service.data.quality.service.impl;
import cn.datax.common.core.RedisConstant;
import cn.datax.common.database.constants.DbType;
import cn.datax.common.redis.service.RedisService;
import cn.datax.service.data.metadata.api.entity.MetadataTableEntity;
import cn.datax.service.data.quality.api.dto.*;
import cn.datax.service.data.quality.api.entity.CheckRuleEntity; import cn.datax.service.data.quality.api.entity.CheckRuleEntity;
import cn.datax.service.data.quality.api.dto.CheckRuleDto; import cn.datax.service.data.quality.api.enums.RuleItem;
import cn.datax.service.data.quality.schedule.CheckRuleFactory;
import cn.datax.service.data.quality.service.CheckRuleService; import cn.datax.service.data.quality.service.CheckRuleService;
import cn.datax.service.data.quality.mapstruct.CheckRuleMapper; import cn.datax.service.data.quality.mapstruct.CheckRuleMapper;
import cn.datax.service.data.quality.dao.CheckRuleDao; import cn.datax.service.data.quality.dao.CheckRuleDao;
import cn.datax.common.base.BaseServiceImpl; import cn.datax.common.base.BaseServiceImpl;
import cn.datax.service.data.standard.api.entity.DictEntity;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/** /**
* <p> * <p>
...@@ -31,10 +41,18 @@ public class CheckRuleServiceImpl extends BaseServiceImpl<CheckRuleDao, CheckRul ...@@ -31,10 +41,18 @@ public class CheckRuleServiceImpl extends BaseServiceImpl<CheckRuleDao, CheckRul
@Autowired @Autowired
private CheckRuleMapper checkRuleMapper; private CheckRuleMapper checkRuleMapper;
@Autowired
private RedisService redisService;
private static String BIND_GB_CODE = "gb_code";
private static String BIND_GB_NAME = "gb_name";
@Override @Override
@Transactional(rollbackFor = Exception.class) @Transactional(rollbackFor = Exception.class)
public CheckRuleEntity saveCheckRule(CheckRuleDto checkRuleDto) { public CheckRuleEntity saveCheckRule(CheckRuleDto checkRuleDto) {
CheckRuleEntity checkRule = checkRuleMapper.toEntity(checkRuleDto); CheckRuleEntity checkRule = checkRuleMapper.toEntity(checkRuleDto);
String sql = parseSql(checkRule);
checkRule.setRuleSql(sql);
checkRuleDao.insert(checkRule); checkRuleDao.insert(checkRule);
return checkRule; return checkRule;
} }
...@@ -43,6 +61,8 @@ public class CheckRuleServiceImpl extends BaseServiceImpl<CheckRuleDao, CheckRul ...@@ -43,6 +61,8 @@ public class CheckRuleServiceImpl extends BaseServiceImpl<CheckRuleDao, CheckRul
@Transactional(rollbackFor = Exception.class) @Transactional(rollbackFor = Exception.class)
public CheckRuleEntity updateCheckRule(CheckRuleDto checkRuleDto) { public CheckRuleEntity updateCheckRule(CheckRuleDto checkRuleDto) {
CheckRuleEntity checkRule = checkRuleMapper.toEntity(checkRuleDto); CheckRuleEntity checkRule = checkRuleMapper.toEntity(checkRuleDto);
String sql = parseSql(checkRule);
checkRule.setRuleSql(sql);
checkRuleDao.updateById(checkRule); checkRuleDao.updateById(checkRule);
return checkRule; return checkRule;
} }
...@@ -64,4 +84,49 @@ public class CheckRuleServiceImpl extends BaseServiceImpl<CheckRuleDao, CheckRul ...@@ -64,4 +84,49 @@ public class CheckRuleServiceImpl extends BaseServiceImpl<CheckRuleDao, CheckRul
public void deleteCheckRuleBatch(List<String> ids) { public void deleteCheckRuleBatch(List<String> ids) {
checkRuleDao.deleteBatchIds(ids); checkRuleDao.deleteBatchIds(ids);
} }
private String parseSql(CheckRuleEntity checkRule) {
RuleConfig ruleConfig = checkRule.getRuleConfig();
Map<String, Object> map = new HashMap<>();
RuleItem ruleItem = RuleItem.getRuleItem(ruleConfig.getRuleItemCode());
switch (ruleItem) {
case Unique:
case Integrity:
break;
// 一致性参数处理
case Consistent:
Consistent consistent = ruleConfig.getConsistent();
List<DictEntity> dictEntityList = (List<DictEntity>) redisService.hget(RedisConstant.STANDARD_DICT_KEY, consistent.getGbTypeId());
String collect = dictEntityList.stream().map(s -> {
if (BIND_GB_CODE.equals(consistent.getBindGbColumn())) {
return "\'" + s.getGbCode() + "\'";
} else {
return "\'" + s.getGbName() + "\'";
}
}).collect(Collectors.joining(","));
map.put("gb_item", collect);
break;
// 关联性参数处理
case Relevance:
Relevance relevance = ruleConfig.getRelevance();
map.put("related_table", relevance.getRelatedTable());
map.put("related_column", relevance.getRelatedColumn());
break;
// 及时性参数处理
case Timeliness:
Timeliness timeliness = ruleConfig.getTimeliness();
map.put("threshold", timeliness.getThreshold());
break;
// 准确性参数处理
case AccuracyLength:
Accuracy accuracy = ruleConfig.getAccuracy();
map.put("max_length", accuracy.getMaxLength());
break;
default:
return null;
}
DbType dbType = DbType.getDbType(checkRule.getRuleDbType());
String sql = CheckRuleFactory.getRuleItem(ruleConfig.getRuleItemCode()).parse(dbType, checkRule.getRuleTable(), checkRule.getRuleColumn(), map);
return sql;
}
} }
datax-modules/data-quality-service-parent/data-quality-service/src/main/resources/mapper/CheckRuleMapper.xml
...@@ -16,6 +16,7 @@ ...@@ -16,6 +16,7 @@
<result column="rule_type_id" property="ruleTypeId" /> <result column="rule_type_id" property="ruleTypeId" />
<result column="rule_item_id" property="ruleItemId" /> <result column="rule_item_id" property="ruleItemId" />
<result column="rule_level_id" property="ruleLevelId" /> <result column="rule_level_id" property="ruleLevelId" />
<result column="rule_db_type" property="ruleDbType" />
<result column="rule_source_id" property="ruleSourceId" /> <result column="rule_source_id" property="ruleSourceId" />
<result column="rule_source" property="ruleSource" /> <result column="rule_source" property="ruleSource" />
<result column="rule_table_id" property="ruleTableId" /> <result column="rule_table_id" property="ruleTableId" />
...@@ -44,7 +45,7 @@ ...@@ -44,7 +45,7 @@
update_by, update_by,
update_time, update_time,
remark, remark,
rule_name, rule_type_id, rule_item_id, rule_level_id, rule_source_id, rule_source, rule_table_id, rule_table, rule_table_comment rule_name, rule_type_id, rule_item_id, rule_level_id, rule_db_type, rule_source_id, rule_source, rule_table_id, rule_table, rule_table_comment
rule_column_id, rule_column, rule_column_comment, last_check_batch rule_column_id, rule_column, rule_column_comment, last_check_batch
</sql> </sql>
...@@ -57,12 +58,12 @@ ...@@ -57,12 +58,12 @@
${alias}.update_by, ${alias}.update_by,
${alias}.update_time, ${alias}.update_time,
${alias}.remark, ${alias}.remark,
${alias}.rule_name, ${alias}.rule_type_id, ${alias}.rule_item_id, ${alias}.rule_level_id, ${alias}.rule_source_id, ${alias}.rule_source, ${alias}.rule_name, ${alias}.rule_type_id, ${alias}.rule_item_id, ${alias}.rule_level_id, ${alias}.rule_db_type, ${alias}.rule_source_id, ${alias}.rule_source,
${alias}.rule_table_id, ${alias}.rule_table, ${alias}.rule_table_comment, ${alias}.rule_column_id, ${alias}.rule_column, ${alias}.rule_column_comment, ${alias}.last_check_batch ${alias}.rule_table_id, ${alias}.rule_table, ${alias}.rule_table_comment, ${alias}.rule_column_id, ${alias}.rule_column, ${alias}.rule_column_comment, ${alias}.last_check_batch
</sql> </sql>
<select id="selectPage" resultMap="ExtendResultMap"> <select id="selectPage" resultMap="ExtendResultMap">
SELECT t.name as rule_type, l.name as rule_level SELECT t.name as rule_type, l.name as rule_level,
<include refid="Rule_Column_List"><property name="alias" value="r"/></include> <include refid="Rule_Column_List"><property name="alias" value="r"/></include>
FROM quality_check_rule r FROM quality_check_rule r
LEFT JOIN quality_rule_type t ON t.id = r.rule_type_id LEFT JOIN quality_rule_type t ON t.id = r.rule_type_id
...@@ -72,7 +73,7 @@ ...@@ -72,7 +73,7 @@
<select id="selectById" resultMap="ExtendResultMap"> <select id="selectById" resultMap="ExtendResultMap">
SELECT config_json, SELECT config_json,
<include refid="Rule_Column_List"></include> <include refid="Base_Column_List"></include>
FROM quality_check_rule FROM quality_check_rule
WHERE 1=1 AND id = #{id} WHERE 1=1 AND id = #{id}
</select> </select>
......
datax-modules/data-quality-service-parent/data-quality-service/src/main/resources/mapper/RuleTypeMapper.xml
...@@ -22,7 +22,7 @@ ...@@ -22,7 +22,7 @@
SELECT t.id, t.name, SELECT t.id, t.name,
(SELECT COALESCE(SUM(c.check_error_count), 0) FROM quality_check_rule r (SELECT COALESCE(SUM(c.check_error_count), 0) FROM quality_check_rule r
LEFT JOIN quality_check_report c ON c.check_rule_id = r.id AND c.check_batch = r.last_check_batch LEFT JOIN quality_check_report c ON c.check_rule_id = r.id AND c.check_batch = r.last_check_batch
WHERE r.rule_type_id = t.id AND r.status = 1) AS check_error_count WHERE r.rule_type_id = t.id) AS check_error_count
FROM quality_rule_type t FROM quality_rule_type t
</select> </select>
......
datax-ui/src/views/quality/analysisreport/index.vue
...@@ -32,8 +32,8 @@ ...@@ -32,8 +32,8 @@
<el-table-column prop="ruleName" label="规则名称" align="center" /> <el-table-column prop="ruleName" label="规则名称" align="center" />
<el-table-column prop="checkErrorCount" label="不合规数量" align="center" > <el-table-column prop="checkErrorCount" label="不合规数量" align="center" >
<template scope="scope"> <template scope="scope">
<el-badge :value="scope.row.checkErrorCount" :type="typeFormat(scope.row.ruleLevelName)"> <el-badge :value="scope.row.ruleLevelName" :type="typeFormat(scope.row.ruleLevelName)">
{{scope.row.ruleLevelName}} {{scope.row.checkErrorCount}}
</el-badge> </el-badge>
</template> </template>
</el-table-column> </el-table-column>
......
datax-ui/src/views/quality/checkrule/CheckRuleAdd.vue
...@@ -13,11 +13,11 @@ ...@@ -13,11 +13,11 @@
<el-input v-model="form.ruleName" placeholder="请输入规则名称" /> <el-input v-model="form.ruleName" placeholder="请输入规则名称" />
</el-form-item> </el-form-item>
<el-form-item label="核查类型" prop="ruleItemId"> <el-form-item label="核查类型" prop="ruleItemId">
<el-select v-model="form.ruleItemId" placeholder="请选择核查类型"> <el-select v-model="form.ruleItemId" placeholder="请选择核查类型" @change="ruleItemSelectChanged">
<el-option <el-option
v-for="item in ruleItemOptions" v-for="item in ruleItemOptions"
:key="item.id" :key="item.id"
:label="item.itemCode" :label="item.itemExplain"
:value="item.id" :value="item.id"
/> />
</el-select> </el-select>
...@@ -66,9 +66,77 @@ ...@@ -66,9 +66,77 @@
</el-option> </el-option>
</el-select> </el-select>
</el-form-item> </el-form-item>
<el-form-item label="核查脚本" prop="ruleSql"> <el-divider content-position="left">核查配置</el-divider>
<el-input v-model="form.ruleSql" type="textarea" placeholder="请输入核查脚本" /> <el-row v-if="form.ruleConfig.ruleItemCode === 'timeliness_key'">
</el-form-item> <el-col :span="24">
<el-form-item label="判定阀值">
<el-input-number v-model="form.ruleConfig.timeliness.threshold" :controls="false" :min="1" />
</el-form-item>
</el-col>
</el-row>
<el-row v-if="form.ruleConfig.ruleItemCode === 'consistent_key'">
<el-col :span="12">
<el-form-item label="标准字典类别">
<el-select v-model="form.ruleConfig.consistent.gbTypeId" placeholder="请选择" @change="dictTypeSelectChanged">
<el-option
v-for="item in dictTypeOptions"
:key="item.id"
:label="item.gbTypeName"
:value="item.id"
/>
</el-select>
</el-form-item>
</el-col>
<el-col :span="12">
<el-form-item label="标准字典字段">
<el-select v-model="form.ruleConfig.consistent.bindGbColumn" placeholder="请选择">
<el-option
v-for="item in gbColumnOptions"
:key="item.value"
:label="item.label"
:value="item.value">
</el-option>
</el-select>
</el-form-item>
</el-col>
</el-row>
<el-row v-if="form.ruleConfig.ruleItemCode === 'relevance_key'">
<el-col :span="12">
<el-form-item label="关联表">
<el-select v-model="form.ruleConfig.relevance.relatedTableId" placeholder="请选择" @change="relatedTableSelectChanged">
<el-option
v-for="table in tableOptions"
:key="table.id"
:label="table.tableName"
:value="table.id"
>
<span style="float: left">{{ table.tableName + '(' + table.tableComment + ')' }}</span>
</el-option>
</el-select>
</el-form-item>
</el-col>
<el-col :span="12">
<el-form-item label="关联字段">
<el-select v-model="form.ruleConfig.relevance.relatedColumnId" placeholder="请选择" @change="relatedColumnSelectChanged">
<el-option
v-for="column in relatedColumnOptions"
:key="column.id"
:label="column.columnName"
:value="column.id"
>
<span style="float: left">{{ column.columnName + '(' + column.columnComment + ')' }}</span>
</el-option>
</el-select>
</el-form-item>
</el-col>
</el-row>
<el-row v-if="form.ruleConfig.ruleItemCode === 'accuracy_key_length'">
<el-col :span="24">
<el-form-item label="最大长度">
<el-input-number v-model="form.ruleConfig.accuracy.maxLength" :controls="false" :min="1" />
</el-form-item>
</el-col>
</el-row>
<el-form-item label="状态" prop="status"> <el-form-item label="状态" prop="status">
<el-radio-group v-model="form.status"> <el-radio-group v-model="form.status">
<el-radio <el-radio
...@@ -91,6 +159,7 @@ import { listRuleLevel, listRuleItem, addCheckRule } from '@/api/quality/checkru ...@@ -91,6 +159,7 @@ import { listRuleLevel, listRuleItem, addCheckRule } from '@/api/quality/checkru
import { listDataSource } from '@/api/metadata/datasource' import { listDataSource } from '@/api/metadata/datasource'
import { listDataTable } from '@/api/metadata/datatable' import { listDataTable } from '@/api/metadata/datatable'
import { listDataColumn } from '@/api/metadata/datacolumn' import { listDataColumn } from '@/api/metadata/datacolumn'
import { listDataDictType } from '@/api/standard/datadict'
export default { export default {
name: 'CheckRuleAdd', name: 'CheckRuleAdd',
...@@ -125,13 +194,36 @@ export default { ...@@ -125,13 +194,36 @@ export default {
ruleTypeId: undefined, ruleTypeId: undefined,
ruleItemId: undefined, ruleItemId: undefined,
ruleLevelId: undefined, ruleLevelId: undefined,
ruleDbType: undefined,
ruleSourceId: undefined, ruleSourceId: undefined,
ruleSource: undefined, ruleSource: undefined,
ruleTableId: undefined, ruleTableId: undefined,
ruleTable: undefined, ruleTable: undefined,
ruleTableComment: undefined,
ruleColumnId: undefined, ruleColumnId: undefined,
ruleColumn: undefined, ruleColumn: undefined,
ruleSql: undefined, ruleColumnComment: undefined,
ruleConfig: {
ruleItemCode: undefined,
consistent: {
gbTypeId: undefined,
bindGbColumn: undefined
},
relevance: {
relatedTableId: undefined,
relatedTable: undefined,
relatedTableComment: undefined,
relatedColumnId: undefined,
relatedColumn: undefined,
relatedColumnComment: undefined
},
timeliness: {
threshold: undefined
},
accuracy: {
maxLength: undefined
}
},
status: '1' status: '1'
}, },
// 表单校验 // 表单校验
...@@ -163,7 +255,13 @@ export default { ...@@ -163,7 +255,13 @@ export default {
ruleItemOptions: [], ruleItemOptions: [],
sourceOptions: [], sourceOptions: [],
tableOptions: [], tableOptions: [],
columnOptions: [] columnOptions: [],
dictTypeOptions: [],
gbColumnOptions: [
{ value: 'gb_code', label: '标准编码' },
{ value: 'gb_name', label: '标准名称' }
],
relatedColumnOptions: []
} }
}, },
created() { created() {
...@@ -174,6 +272,11 @@ export default { ...@@ -174,6 +272,11 @@ export default {
this.statusOptions = response.data this.statusOptions = response.data
} }
}) })
listDataDictType().then(response => {
if (response.success) {
this.dictTypeOptions = response.data
}
})
this.getRuleLevelList() this.getRuleLevelList()
this.getRuleItemList() this.getRuleItemList()
this.getDataSourceList() this.getDataSourceList()
...@@ -203,6 +306,12 @@ export default { ...@@ -203,6 +306,12 @@ export default {
} }
}) })
}, },
ruleItemSelectChanged(val) {
const item = this.ruleItemOptions.find(function(item) {
return item.id === val
})
this.form.ruleConfig.ruleItemCode = item.itemCode
},
sourceSelectChanged(val) { sourceSelectChanged(val) {
listDataTable({ sourceId: val }).then(response => { listDataTable({ sourceId: val }).then(response => {
if (response.success) { if (response.success) {
...@@ -212,10 +321,13 @@ export default { ...@@ -212,10 +321,13 @@ export default {
return item.id === val return item.id === val
}) })
this.form.ruleSource = source.sourceName this.form.ruleSource = source.sourceName
this.form.ruleDbType = source.dbType
this.form.ruleTableId = '' this.form.ruleTableId = ''
this.form.ruleTable = '' this.form.ruleTable = ''
this.form.ruleTableComment = ''
this.form.ruleColumnId = '' this.form.ruleColumnId = ''
this.form.ruleColumn = '' this.form.ruleColumn = ''
this.form.ruleColumnComment = ''
} }
}) })
}, },
...@@ -242,6 +354,35 @@ export default { ...@@ -242,6 +354,35 @@ export default {
this.form.ruleColumnComment = column.columnComment this.form.ruleColumnComment = column.columnComment
this.$forceUpdate() this.$forceUpdate()
}, },
dictTypeSelectChanged(val) {
const item = this.dictTypeOptions.find(function(item) {
return item.id === val
})
this.form.ruleConfig.consistent.gbTypeCode = item.gbTypeCode
this.form.ruleConfig.consistent.gbTypeName = item.gbTypeName
},
relatedTableSelectChanged(val) {
listDataColumn({ sourceId: this.form.ruleSourceId, tableId: val }).then(response => {
if (response.success) {
this.relatedColumnOptions = response.data
const table = this.tableOptions.find(function(item) {
return item.id === val
})
this.form.ruleConfig.relevance.relatedTable = table.tableName
this.form.ruleConfig.relevance.relatedTableComment = table.tableComment
this.form.ruleConfig.relevance.relatedColumnId = ''
this.form.ruleConfig.relevance.relatedColumn = ''
this.form.ruleConfig.relevance.relatedColumnComment = ''
}
})
},
relatedColumnSelectChanged(val) {
const column = this.relatedColumnOptions.find(function(item) {
return item.id === val
})
this.form.ruleConfig.relevance.relatedColumn = column.columnName
this.form.ruleConfig.relevance.relatedColumnComment = column.columnComment
},
/** 提交按钮 */ /** 提交按钮 */
submitForm: function() { submitForm: function() {
this.$refs['form'].validate(valid => { this.$refs['form'].validate(valid => {
......
datax-ui/src/views/quality/checkrule/CheckRuleDetail.vue
...@@ -16,7 +16,7 @@ ...@@ -16,7 +16,7 @@
<el-option <el-option
v-for="item in ruleItemOptions" v-for="item in ruleItemOptions"
:key="item.id" :key="item.id"
:label="item.itemCode" :label="item.itemExplain"
:value="item.id" :value="item.id"
/> />
</el-select> </el-select>
...@@ -63,9 +63,77 @@ ...@@ -63,9 +63,77 @@
</el-option> </el-option>
</el-select> </el-select>
</el-form-item> </el-form-item>
<el-form-item label="核查脚本" prop="ruleSql"> <el-divider content-position="left">核查配置</el-divider>
<el-input v-model="form.ruleSql" type="textarea" /> <el-row v-if="form.ruleConfig.ruleItemCode === 'timeliness_key'">
</el-form-item> <el-col :span="24">
<el-form-item label="判定阀值">
<el-input-number v-model="form.ruleConfig.timeliness.threshold" :controls="false" :min="1" />
</el-form-item>
</el-col>
</el-row>
<el-row v-if="form.ruleConfig.ruleItemCode === 'consistent_key'">
<el-col :span="12">
<el-form-item label="标准字典类别">
<el-select v-model="form.ruleConfig.consistent.gbTypeId" placeholder="请选择">
<el-option
v-for="item in dictTypeOptions"
:key="item.id"
:label="item.gbTypeName"
:value="item.id"
/>
</el-select>
</el-form-item>
</el-col>
<el-col :span="12">
<el-form-item label="标准字典字段">
<el-select v-model="form.ruleConfig.consistent.bindGbColumn" placeholder="请选择">
<el-option
v-for="item in gbColumnOptions"
:key="item.value"
:label="item.label"
:value="item.value">
</el-option>
</el-select>
</el-form-item>
</el-col>
</el-row>
<el-row v-if="form.ruleConfig.ruleItemCode === 'relevance_key'">
<el-col :span="12">
<el-form-item label="关联表">
<el-select v-model="form.ruleConfig.relevance.relatedTableId" placeholder="请选择">
<el-option
v-for="table in tableOptions"
:key="table.id"
:label="table.tableName"
:value="table.id"
>
<span style="float: left">{{ table.tableName + '(' + table.tableComment + ')' }}</span>
</el-option>
</el-select>
</el-form-item>
</el-col>
<el-col :span="12">
<el-form-item label="关联字段">
<el-select v-model="form.ruleConfig.relevance.relatedColumnId" placeholder="请选择">
<el-option
v-for="column in relatedColumnOptions"
:key="column.id"
:label="column.columnName"
:value="column.id"
>
<span style="float: left">{{ column.columnName + '(' + column.columnComment + ')' }}</span>
</el-option>
</el-select>
</el-form-item>
</el-col>
</el-row>
<el-row v-if="form.ruleConfig.ruleItemCode === 'accuracy_key_length'">
<el-col :span="24">
<el-form-item label="最大长度">
<el-input-number v-model="form.ruleConfig.accuracy.maxLength" :controls="false" :min="1" />
</el-form-item>
</el-col>
</el-row>
<el-form-item label="状态"> <el-form-item label="状态">
<el-radio-group v-model="form.status"> <el-radio-group v-model="form.status">
<el-radio <el-radio
...@@ -88,6 +156,7 @@ import { listRuleLevel, listRuleItem, getCheckRule } from '@/api/quality/checkru ...@@ -88,6 +156,7 @@ import { listRuleLevel, listRuleItem, getCheckRule } from '@/api/quality/checkru
import { listDataSource } from '@/api/metadata/datasource' import { listDataSource } from '@/api/metadata/datasource'
import { listDataTable } from '@/api/metadata/datatable' import { listDataTable } from '@/api/metadata/datatable'
import { listDataColumn } from '@/api/metadata/datacolumn' import { listDataColumn } from '@/api/metadata/datacolumn'
import { listDataDictType } from '@/api/standard/datadict'
export default { export default {
name: 'CheckRuleDetail', name: 'CheckRuleDetail',
...@@ -120,7 +189,13 @@ export default { ...@@ -120,7 +189,13 @@ export default {
ruleItemOptions: [], ruleItemOptions: [],
sourceOptions: [], sourceOptions: [],
tableOptions: [], tableOptions: [],
columnOptions: [] columnOptions: [],
dictTypeOptions: [],
gbColumnOptions: [
{ value: 'gb_code', label: '标准编码' },
{ value: 'gb_name', label: '标准名称' }
],
relatedColumnOptions: []
} }
}, },
created() { created() {
...@@ -130,6 +205,11 @@ export default { ...@@ -130,6 +205,11 @@ export default {
this.statusOptions = response.data this.statusOptions = response.data
} }
}) })
listDataDictType().then(response => {
if (response.success) {
this.dictTypeOptions = response.data
}
})
this.getRuleLevelList() this.getRuleLevelList()
this.getDataSourceList() this.getDataSourceList()
}, },
...@@ -162,6 +242,13 @@ export default { ...@@ -162,6 +242,13 @@ export default {
return response.data return response.data
} }
}) || [] }) || []
if (this.form.ruleConfig.ruleItemCode === 'relevance_key') {
listDataColumn({ sourceId: this.form.ruleSourceId, tableId: this.form.ruleConfig.relevance.relatedTableId }).then(response => {
if (response.success) {
this.relatedColumnOptions = response.data
}
})
}
}, },
getRuleLevelList() { getRuleLevelList() {
listRuleLevel().then(response => { listRuleLevel().then(response => {
......
datax-ui/src/views/quality/checkrule/CheckRuleEdit.vue
...@@ -13,11 +13,11 @@ ...@@ -13,11 +13,11 @@
<el-input v-model="form.ruleName" placeholder="请输入规则名称" /> <el-input v-model="form.ruleName" placeholder="请输入规则名称" />
</el-form-item> </el-form-item>
<el-form-item label="核查类型" prop="ruleItemId"> <el-form-item label="核查类型" prop="ruleItemId">
<el-select v-model="form.ruleItemId" placeholder="请选择核查类型"> <el-select v-model="form.ruleItemId" placeholder="请选择核查类型" @change="ruleItemSelectChanged">
<el-option <el-option
v-for="item in ruleItemOptions" v-for="item in ruleItemOptions"
:key="item.id" :key="item.id"
:label="item.itemCode" :label="item.itemExplain"
:value="item.id" :value="item.id"
/> />
</el-select> </el-select>
...@@ -64,9 +64,77 @@ ...@@ -64,9 +64,77 @@
</el-option> </el-option>
</el-select> </el-select>
</el-form-item> </el-form-item>
<el-form-item label="核查脚本" prop="ruleSql"> <el-divider content-position="left">核查配置</el-divider>
<el-input v-model="form.ruleSql" type="textarea" placeholder="请输入核查脚本" /> <el-row v-if="form.ruleConfig.ruleItemCode === 'timeliness_key'">
</el-form-item> <el-col :span="24">
<el-form-item label="判定阀值">
<el-input-number v-model="form.ruleConfig.timeliness.threshold" :controls="false" :min="1" />
</el-form-item>
</el-col>
</el-row>
<el-row v-if="form.ruleConfig.ruleItemCode === 'consistent_key'">
<el-col :span="12">
<el-form-item label="标准字典类别">
<el-select v-model="form.ruleConfig.consistent.gbTypeId" placeholder="请选择" @change="dictTypeSelectChanged">
<el-option
v-for="item in dictTypeOptions"
:key="item.id"
:label="item.gbTypeName"
:value="item.id"
/>
</el-select>
</el-form-item>
</el-col>
<el-col :span="12">
<el-form-item label="标准字典字段">
<el-select v-model="form.ruleConfig.consistent.bindGbColumn" placeholder="请选择">
<el-option
v-for="item in gbColumnOptions"
:key="item.value"
:label="item.label"
:value="item.value">
</el-option>
</el-select>
</el-form-item>
</el-col>
</el-row>
<el-row v-if="form.ruleConfig.ruleItemCode === 'relevance_key'">
<el-col :span="12">
<el-form-item label="关联表">
<el-select v-model="form.ruleConfig.relevance.relatedTableId" placeholder="请选择" @change="relatedTableSelectChanged">
<el-option
v-for="table in tableOptions"
:key="table.id"
:label="table.tableName"
:value="table.id"
>
<span style="float: left">{{ table.tableName + '(' + table.tableComment + ')' }}</span>
</el-option>
</el-select>
</el-form-item>
</el-col>
<el-col :span="12">
<el-form-item label="关联字段">
<el-select v-model="form.ruleConfig.relevance.relatedColumnId" placeholder="请选择" @change="relatedColumnSelectChanged">
<el-option
v-for="column in relatedColumnOptions"
:key="column.id"
:label="column.columnName"
:value="column.id"
>
<span style="float: left">{{ column.columnName + '(' + column.columnComment + ')' }}</span>
</el-option>
</el-select>
</el-form-item>
</el-col>
</el-row>
<el-row v-if="form.ruleConfig.ruleItemCode === 'accuracy_key_length'">
<el-col :span="24">
<el-form-item label="最大长度">
<el-input-number v-model="form.ruleConfig.accuracy.maxLength" :controls="false" :min="1" />
</el-form-item>
</el-col>
</el-row>
<el-form-item label="状态" prop="status"> <el-form-item label="状态" prop="status">
<el-radio-group v-model="form.status"> <el-radio-group v-model="form.status">
<el-radio <el-radio
...@@ -89,6 +157,7 @@ import { listRuleLevel, listRuleItem, getCheckRule, updateCheckRule } from '@/ap ...@@ -89,6 +157,7 @@ import { listRuleLevel, listRuleItem, getCheckRule, updateCheckRule } from '@/ap
import { listDataSource } from '@/api/metadata/datasource' import { listDataSource } from '@/api/metadata/datasource'
import { listDataTable } from '@/api/metadata/datatable' import { listDataTable } from '@/api/metadata/datatable'
import { listDataColumn } from '@/api/metadata/datacolumn' import { listDataColumn } from '@/api/metadata/datacolumn'
import { listDataDictType } from '@/api/standard/datadict'
export default { export default {
name: 'CheckRuleEdit', name: 'CheckRuleEdit',
...@@ -148,7 +217,13 @@ export default { ...@@ -148,7 +217,13 @@ export default {
ruleItemOptions: [], ruleItemOptions: [],
sourceOptions: [], sourceOptions: [],
tableOptions: [], tableOptions: [],
columnOptions: [] columnOptions: [],
dictTypeOptions: [],
gbColumnOptions: [
{ value: 'gb_code', label: '标准编码' },
{ value: 'gb_name', label: '标准名称' }
],
relatedColumnOptions: []
} }
}, },
created() { created() {
...@@ -158,6 +233,11 @@ export default { ...@@ -158,6 +233,11 @@ export default {
this.statusOptions = response.data this.statusOptions = response.data
} }
}) })
listDataDictType().then(response => {
if (response.success) {
this.dictTypeOptions = response.data
}
})
this.getRuleLevelList() this.getRuleLevelList()
this.getDataSourceList() this.getDataSourceList()
}, },
...@@ -190,6 +270,13 @@ export default { ...@@ -190,6 +270,13 @@ export default {
return response.data return response.data
} }
}) })
if (this.form.ruleConfig.ruleItemCode === 'relevance_key') {
listDataColumn({ sourceId: this.form.ruleSourceId, tableId: this.form.ruleConfig.relevance.relatedTableId }).then(response => {
if (response.success) {
this.relatedColumnOptions = response.data
}
})
}
}, },
getRuleLevelList() { getRuleLevelList() {
listRuleLevel().then(response => { listRuleLevel().then(response => {
...@@ -205,6 +292,12 @@ export default { ...@@ -205,6 +292,12 @@ export default {
} }
}) })
}, },
ruleItemSelectChanged(val) {
const item = this.ruleItemOptions.find(function(item) {
return item.id === val
})
this.form.ruleConfig.ruleItemCode = item.itemCode
},
sourceSelectChanged(val) { sourceSelectChanged(val) {
listDataTable({ sourceId: val }).then(response => { listDataTable({ sourceId: val }).then(response => {
if (response.success) { if (response.success) {
...@@ -214,10 +307,13 @@ export default { ...@@ -214,10 +307,13 @@ export default {
return item.id === val return item.id === val
}) })
this.form.ruleSource = source.sourceName this.form.ruleSource = source.sourceName
this.form.ruleDbType = source.dbType
this.form.ruleTableId = '' this.form.ruleTableId = ''
this.form.ruleTable = '' this.form.ruleTable = ''
this.form.ruleTableComment = ''
this.form.ruleColumnId = '' this.form.ruleColumnId = ''
this.form.ruleColumn = '' this.form.ruleColumn = ''
this.form.ruleColumnComment = ''
} }
}) })
}, },
...@@ -244,6 +340,35 @@ export default { ...@@ -244,6 +340,35 @@ export default {
this.form.ruleColumnComment = column.columnComment this.form.ruleColumnComment = column.columnComment
this.$forceUpdate() this.$forceUpdate()
}, },
dictTypeSelectChanged(val) {
const item = this.dictTypeOptions.find(function(item) {
return item.id === val
})
this.form.ruleConfig.consistent.gbTypeCode = item.gbTypeCode
this.form.ruleConfig.consistent.gbTypeName = item.gbTypeName
},
relatedTableSelectChanged(val) {
listDataColumn({ sourceId: this.form.ruleSourceId, tableId: val }).then(response => {
if (response.success) {
this.relatedColumnOptions = response.data
const table = this.tableOptions.find(function(item) {
return item.id === val
})
this.form.ruleConfig.relevance.relatedTable = table.tableName
this.form.ruleConfig.relevance.relatedTableComment = table.tableComment
this.form.ruleConfig.relevance.relatedColumnId = ''
this.form.ruleConfig.relevance.relatedColumn = ''
this.form.ruleConfig.relevance.relatedColumnComment = ''
}
})
},
relatedColumnSelectChanged(val) {
const column = this.relatedColumnOptions.find(function(item) {
return item.id === val
})
this.form.ruleConfig.relevance.relatedColumn = column.columnName
this.form.ruleConfig.relevance.relatedColumnComment = column.columnComment
},
/** 提交按钮 */ /** 提交按钮 */
submitForm: function() { submitForm: function() {
this.$refs['form'].validate(valid => { this.$refs['form'].validate(valid => {
......
datax-ui/src/views/standard/contraststatistic/ContrastStatisticList.vue
...@@ -140,7 +140,7 @@ export default { ...@@ -140,7 +140,7 @@ export default {
{ prop: 'columnName', label: '对照字段', show: true }, { prop: 'columnName', label: '对照字段', show: true },
{ prop: 'gbTypeCode', label: '标准类别编码', show: true }, { prop: 'gbTypeCode', label: '标准类别编码', show: true },
{ prop: 'gbTypeName', label: '标准类别名称', show: true }, { prop: 'gbTypeName', label: '标准类别名称', show: true },
{ prop: 'mappingCount', label: '对照数量', show: true }, { prop: 'mappingCount', label: '已对照数量', show: true },
{ prop: 'unMappingCount', label: '未对照数量', show: true }, { prop: 'unMappingCount', label: '未对照数量', show: true },
{ prop: 'mappingPercent', label: '对照比例', show: true, formatter: this.mappingFormatter } { prop: 'mappingPercent', label: '对照比例', show: true, formatter: this.mappingFormatter }
], ],
......
datax-ui/src/views/standard/dictmapping/index.vue
...@@ -228,7 +228,7 @@ export default { ...@@ -228,7 +228,7 @@ export default {
// 双击连接线（删除） // 双击连接线（删除）
_this.jsPlumb.bind('dblclick', function(conn, originalEvent) { _this.jsPlumb.bind('dblclick', function(conn, originalEvent) {
console.log('dblclick', conn) console.log('dblclick', conn)
_this.$confirm('选中数据将被永久删除, 是否继续？', '提示', { _this.$confirm('删除选中连线, 是否继续？', '提示', {
confirmButtonText: '确定', confirmButtonText: '确定',
cancelButtonText: '取消', cancelButtonText: '取消',
type: 'warning' type: 'warning'
...@@ -342,6 +342,7 @@ export default { ...@@ -342,6 +342,7 @@ export default {
tr th { tr th {
color: #909399; color: #909399;
font-weight: bold; font-weight: bold;
background-color: #f5f5f5;
} }
tr th, tr td { tr th, tr td {
font-size: 14px; font-size: 14px;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment