Skip to content

D
datax-cloud
Commit f23aed8a by yuwei
Options

项目初始化

parent 66919cc4
Showing with 783 additions and 44 deletions
datax-common/datax-common-mybatis/src/main/java/cn/datax/common/mybatis/config/DataMetaObjectHandler.java
......@@ -2,7 +2,6 @@ package cn.datax.common.mybatis.config;
import cn.datax.common.core.DataConstant;
import cn.datax.common.utils.SecurityUtil;
import cn.hutool.core.util.StrUtil;
import com.baomidou.mybatisplus.core.handlers.MetaObjectHandler;
import org.apache.ibatis.reflection.MetaObject;
......@@ -15,14 +14,11 @@ public class DataMetaObjectHandler implements MetaObjectHandler {
this.strictInsertFill(metaObject, "createTime", LocalDateTime.class, LocalDateTime.now());
this.strictInsertFill(metaObject, "updateTime", LocalDateTime.class, LocalDateTime.now());
this.strictInsertFill(metaObject, "status", String.class, DataConstant.EnableState.ENABLE.getKey());
String userId = getUserId();
if (StrUtil.isNotBlank(userId)) {
this.strictInsertFill(metaObject, "createBy", String.class, userId);
this.strictInsertFill(metaObject, "updateBy", String.class, userId);
}
this.strictInsertFill(metaObject, "createBy", String.class, SecurityUtil.getUserId());
this.strictInsertFill(metaObject, "updateBy", String.class, SecurityUtil.getUserId());
boolean bolCreateDept = metaObject.hasSetter("createDept");
if (bolCreateDept) {
this.strictInsertFill(metaObject, "createDept", String.class, getUserDeptId());
this.strictInsertFill(metaObject, "createDept", String.class, SecurityUtil.getUserDeptId());
}
boolean bolFlowStatus = metaObject.hasSetter("flowStatus");
if (bolFlowStatus) {
......@@ -33,19 +29,6 @@ public class DataMetaObjectHandler implements MetaObjectHandler {
@Override
public void updateFill(MetaObject metaObject) {
this.strictUpdateFill(metaObject, "updateTime", LocalDateTime.class, LocalDateTime.now());
String userId = getUserId();
if (StrUtil.isNotBlank(userId)) {
this.strictUpdateFill(metaObject, "updateBy", String.class, userId);
}
}
private String getUserId() {
String userId = SecurityUtil.getUserId();
return userId;
}
private String getUserDeptId() {
String deptId = SecurityUtil.getUserDeptId();
return deptId;
this.strictUpdateFill(metaObject, "updateBy", String.class, SecurityUtil.getUserId());
}
}
datax-config/src/main/resources/config/datax-service-codegen-dev.yml
......@@ -28,6 +28,9 @@ mybatis-plus:
global-config:
db-config:
id-type: ASSIGN_ID
select-strategy: not_empty
insert-strategy: not_empty
update-strategy: not_empty
banner: false
configuration:
map-underscore-to-camel-case: true
......
datax-config/src/main/resources/config/datax-service-data-console-dev.yml
......@@ -28,6 +28,9 @@ mybatis-plus:
global-config:
db-config:
id-type: ASSIGN_ID
select-strategy: not_empty
insert-strategy: not_empty
update-strategy: not_empty
banner: false
configuration:
map-underscore-to-camel-case: true
......
datax-config/src/main/resources/config/datax-service-data-integration-dev.yml
......@@ -28,6 +28,9 @@ mybatis-plus:
global-config:
db-config:
id-type: ASSIGN_ID
select-strategy: not_empty
insert-strategy: not_empty
update-strategy: not_empty
banner: false
configuration:
map-underscore-to-camel-case: true
......
datax-config/src/main/resources/config/datax-service-data-mapping-dev.yml
......@@ -38,6 +38,9 @@ mybatis-plus:
global-config:
db-config:
id-type: ASSIGN_ID
select-strategy: not_empty
insert-strategy: not_empty
update-strategy: not_empty
banner: false
configuration:
map-underscore-to-camel-case: true
......
datax-config/src/main/resources/config/datax-service-data-market-dev.yml
......@@ -37,6 +37,9 @@ mybatis-plus:
global-config:
db-config:
id-type: ASSIGN_ID
select-strategy: not_empty
insert-strategy: not_empty
update-strategy: not_empty
banner: false
configuration:
map-underscore-to-camel-case: true
......
datax-config/src/main/resources/config/datax-service-data-masterdata-dev.yml
......@@ -38,6 +38,9 @@ mybatis-plus:
global-config:
db-config:
id-type: ASSIGN_ID
select-strategy: not_empty
insert-strategy: not_empty
update-strategy: not_empty
banner: false
configuration:
map-underscore-to-camel-case: true
......
datax-config/src/main/resources/config/datax-service-data-metadata-dev.yml
......@@ -28,6 +28,9 @@ mybatis-plus:
global-config:
db-config:
id-type: ASSIGN_ID
select-strategy: not_empty
insert-strategy: not_empty
update-strategy: not_empty
banner: false
configuration:
map-underscore-to-camel-case: true
......
datax-config/src/main/resources/config/datax-service-data-quality-dev.yml
......@@ -28,6 +28,9 @@ mybatis-plus:
global-config:
db-config:
id-type: ASSIGN_ID
select-strategy: not_empty
insert-strategy: not_empty
update-strategy: not_empty
banner: false
configuration:
map-underscore-to-camel-case: true
......
datax-config/src/main/resources/config/datax-service-data-standard-dev.yml
......@@ -28,6 +28,9 @@ mybatis-plus:
global-config:
db-config:
id-type: ASSIGN_ID
select-strategy: not_empty
insert-strategy: not_empty
update-strategy: not_empty
banner: false
configuration:
map-underscore-to-camel-case: true
......
datax-config/src/main/resources/config/datax-service-data-visual-dev.yml
......@@ -28,6 +28,9 @@ mybatis-plus:
global-config:
db-config:
id-type: ASSIGN_ID
select-strategy: not_empty
insert-strategy: not_empty
update-strategy: not_empty
banner: false
configuration:
map-underscore-to-camel-case: true
......
datax-config/src/main/resources/config/datax-service-email-dev.yml
......@@ -39,6 +39,9 @@ mybatis-plus:
global-config:
db-config:
id-type: ASSIGN_ID
select-strategy: not_empty
insert-strategy: not_empty
update-strategy: not_empty
banner: false
configuration:
map-underscore-to-camel-case: true
......
datax-config/src/main/resources/config/datax-service-file-dev.yml
......@@ -28,6 +28,9 @@ mybatis-plus:
global-config:
db-config:
id-type: ASSIGN_ID
select-strategy: not_empty
insert-strategy: not_empty
update-strategy: not_empty
banner: false
configuration:
map-underscore-to-camel-case: true
......
datax-config/src/main/resources/config/datax-service-quartz-dev.yml
......@@ -59,6 +59,9 @@ mybatis-plus:
global-config:
db-config:
id-type: ASSIGN_ID
select-strategy: not_empty
insert-strategy: not_empty
update-strategy: not_empty
banner: false
configuration:
map-underscore-to-camel-case: true
......
datax-config/src/main/resources/config/datax-service-system-dev.yml
......@@ -28,6 +28,9 @@ mybatis-plus:
global-config:
db-config:
id-type: ASSIGN_ID
select-strategy: not_empty
insert-strategy: not_empty
update-strategy: not_empty
banner: false
configuration:
map-underscore-to-camel-case: true
......
datax-config/src/main/resources/config/datax-service-websocket-dev.yml
......@@ -28,6 +28,9 @@ mybatis-plus:
global-config:
db-config:
id-type: ASSIGN_ID
select-strategy: not_empty
insert-strategy: not_empty
update-strategy: not_empty
banner: false
configuration:
map-underscore-to-camel-case: true
......
datax-config/src/main/resources/config/datax-service-workflow-dev.yml
......@@ -56,6 +56,9 @@ mybatis-plus:
global-config:
db-config:
id-type: ASSIGN_ID
select-strategy: not_empty
insert-strategy: not_empty
update-strategy: not_empty
banner: false
configuration:
map-underscore-to-camel-case: true
......
datax-modules/codegen-service-parent/codegen-service/src/main/resources/templates/controller.java.vm
......@@ -134,7 +134,7 @@ public class ${table.controllerName} {
* @param ids
* @return
*/
@ApiOperation(value = "批量删除角色", notes = "根据url的ids来批量删除对象")
@ApiOperation(value = "批量删除", notes = "根据url的ids来批量删除对象")
@ApiImplicitParam(name = "ids", value = "ID集合", required = true, dataType = "List", paramType = "path")
@DeleteMapping("/batch/{ids}")
public R delete${className}Batch(@PathVariable List<String> ids) {
......
datax-modules/data-quality-service-parent/data-quality-service-api/src/main/java/cn/datax/service/data/quality/api/dto/ScheduleLogDto.java 0 → 100644
package cn.datax.service.data.quality.api.dto;
import cn.datax.common.validate.ValidationGroups;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import javax.validation.constraints.NotBlank;
import java.io.Serializable;
import java.time.LocalDateTime;
/**
* <p>
* 数据质量监控任务日志信息表 实体DTO
* </p>
*
* @author yuwei
* @since 2020-10-13
*/
@ApiModel(value = "数据质量监控任务日志信息表Model")
@Data
public class ScheduleLogDto implements Serializable {
private static final long serialVersionUID=1L;
@ApiModelProperty(value = "主键ID")
@NotBlank(message = "主键ID不能为空", groups = {ValidationGroups.Update.class})
private String id;
@ApiModelProperty(value = "状态（1成功 0失败）")
private String status;
@ApiModelProperty(value = "执行任务主键")
private String executeJobId;
@ApiModelProperty(value = "执行规则主键")
private String executeRuleId;
@ApiModelProperty(value = "执行时间")
private LocalDateTime executeDate;
@ApiModelProperty(value = "执行结果")
private String executeResult;
@ApiModelProperty(value = "执行批次号")
private String executeBatch;
}
datax-modules/data-quality-service-parent/data-quality-service-api/src/main/java/cn/datax/service/data/quality/api/entity/ScheduleLogEntity.java 0 → 100644
package cn.datax.service.data.quality.api.entity;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableField;
import com.baomidou.mybatisplus.annotation.TableId;
import com.baomidou.mybatisplus.annotation.TableName;
import java.io.Serializable;
import java.time.LocalDateTime;
import lombok.Data;
import lombok.experimental.Accessors;
/**
* <p>
* 数据质量监控任务日志信息表
* </p>
*
* @author yuwei
* @since 2020-10-13
*/
@Data
@Accessors(chain = true)
@TableName(value = "quality_schedule_log")
public class ScheduleLogEntity implements Serializable {
private static final long serialVersionUID=1L;
/**
* 主键
*/
@TableId(value = "id", type = IdType.ASSIGN_ID)
private String id;
/**
* 状态（1成功 0失败）
*/
private String status;
/**
* 执行任务主键
*/
private String executeJobId;
@TableField(exist = false)
private String executeJobName;
/**
* 执行规则主键
*/
private String executeRuleId;
@TableField(exist = false)
private String executeRuleName;
@TableField(exist = false)
private String executeRuleTypeName;
/**
* 执行时间
*/
private LocalDateTime executeDate;
/**
* 执行结果
*/
private String executeResult;
/**
* 执行批次号
*/
private String executeBatch;
}
datax-modules/data-quality-service-parent/data-quality-service-api/src/main/java/cn/datax/service/data/quality/api/query/ScheduleLogQuery.java 0 → 100644
package cn.datax.service.data.quality.api.query;
import cn.datax.common.base.BaseQueryParams;
import lombok.Data;
import lombok.EqualsAndHashCode;
/**
* <p>
* 数据质量监控任务日志信息表 查询实体
* </p>
*
* @author yuwei
* @since 2020-10-13
*/
@Data
@EqualsAndHashCode(callSuper = true)
public class ScheduleLogQuery extends BaseQueryParams {
private static final long serialVersionUID=1L;
private String executeJobId;
private String ruleTypeId;
}
datax-modules/data-quality-service-parent/data-quality-service-api/src/main/java/cn/datax/service/data/quality/api/vo/ScheduleLogVo.java 0 → 100644
package cn.datax.service.data.quality.api.vo;
import com.fasterxml.jackson.annotation.JsonFormat;
import lombok.Data;
import java.io.Serializable;
import java.time.LocalDateTime;
/**
* <p>
* 数据质量监控任务日志信息表 实体VO
* </p>
*
* @author yuwei
* @since 2020-10-13
*/
@Data
public class ScheduleLogVo implements Serializable {
private static final long serialVersionUID=1L;
private String id;
private String status;
private String executeJobId;
private String executeRuleId;
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8")
private LocalDateTime executeDate;
private String executeResult;
private String executeBatch;
private String executeJobName;
private String executeRuleName;
private String executeRuleTypeName;
}
datax-modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/config/StartedUpRunner.java
......@@ -47,7 +47,7 @@ public class StartedUpRunner implements ApplicationRunner {
List<ScheduleJobEntity> list = scheduleJobService.list(Wrappers.<ScheduleJobEntity>lambdaQuery().eq(ScheduleJobEntity::getStatus, DataConstant.TrueOrFalse.TRUE.getKey()));
if (CollUtil.isNotEmpty(list)) {
list.stream().forEach(job -> {
SchedulingRunnable task = new SchedulingRunnable(job.getBeanName(), job.getMethodName(), job.getMethodParams());
SchedulingRunnable task = new SchedulingRunnable(job.getId(), job.getBeanName(), job.getMethodName(), job.getMethodParams());
cronTaskRegistrar.addCronTask(task, job.getCronExpression());
});
}
......
datax-modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/controller/CheckRuleController.java
......@@ -130,7 +130,7 @@ public class CheckRuleController extends BaseController {
* @param ids
* @return
*/
@ApiOperation(value = "批量删除角色", notes = "根据url的ids来批量删除对象")
@ApiOperation(value = "批量删除", notes = "根据url的ids来批量删除对象")
@ApiImplicitParam(name = "ids", value = "ID集合", required = true, dataType = "List", paramType = "path")
@DeleteMapping("/batch/{ids}")
public R deleteCheckRuleBatch(@PathVariable List<String> ids) {
......
datax-modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/controller/ScheduleLogController.java 0 → 100644
package cn.datax.service.data.quality.controller;
import cn.datax.common.core.JsonPage;
import cn.datax.common.core.R;
import cn.datax.service.data.quality.api.entity.ScheduleLogEntity;
import cn.datax.service.data.quality.api.vo.ScheduleLogVo;
import cn.datax.service.data.quality.api.query.ScheduleLogQuery;
import cn.datax.service.data.quality.mapstruct.ScheduleLogMapper;
import cn.datax.service.data.quality.service.ScheduleLogService;
import cn.hutool.core.util.StrUtil;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
import io.swagger.annotations.ApiImplicitParams;
import io.swagger.annotations.ApiOperation;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import cn.datax.common.base.BaseController;
import java.util.List;
import java.util.stream.Collectors;
/**
* <p>
* 数据质量监控任务日志信息表 前端控制器
* </p>
*
* @author yuwei
* @since 2020-10-13
*/
@Api(tags = {"数据质量监控任务日志信息表"})
@RestController
@RequestMapping("/scheduleLogs")
public class ScheduleLogController extends BaseController {
@Autowired
private ScheduleLogService scheduleLogService;
@Autowired
private ScheduleLogMapper scheduleLogMapper;
/**
* 通过ID查询信息
*
* @param id
* @return
*/
@ApiOperation(value = "获取详细信息", notes = "根据url的id来获取详细信息")
@ApiImplicitParam(name = "id", value = "ID", required = true, dataType = "String", paramType = "path")
@GetMapping("/{id}")
public R getScheduleLogById(@PathVariable String id) {
ScheduleLogEntity scheduleLogEntity = scheduleLogService.getScheduleLogById(id);
return R.ok().setData(scheduleLogMapper.toVO(scheduleLogEntity));
}
/**
* 分页查询信息
*
* @param scheduleLogQuery
* @return
*/
@ApiOperation(value = "分页查询", notes = "")
@ApiImplicitParams({
@ApiImplicitParam(name = "scheduleLogQuery", value = "查询实体scheduleLogQuery", required = true, dataTypeClass = ScheduleLogQuery.class)
})
@GetMapping("/page")
public R getScheduleLogPage(ScheduleLogQuery scheduleLogQuery) {
QueryWrapper<ScheduleLogEntity> queryWrapper = new QueryWrapper<>();
queryWrapper.eq(StrUtil.isNotBlank(scheduleLogQuery.getExecuteJobId()), "l.execute_job_id", scheduleLogQuery.getExecuteJobId());
queryWrapper.eq(StrUtil.isNotBlank(scheduleLogQuery.getRuleTypeId()), "t.id", scheduleLogQuery.getRuleTypeId());
IPage<ScheduleLogEntity> page = scheduleLogService.page(new Page<>(scheduleLogQuery.getPageNum(), scheduleLogQuery.getPageSize()), queryWrapper);
List<ScheduleLogVo> collect = page.getRecords().stream().map(scheduleLogMapper::toVO).collect(Collectors.toList());
JsonPage<ScheduleLogVo> jsonPage = new JsonPage<>(page.getCurrent(), page.getSize(), page.getTotal(), collect);
return R.ok().setData(jsonPage);
}
/**
* 删除
* @param id
* @return
*/
@ApiOperation(value = "删除", notes = "根据url的id来指定删除对象")
@ApiImplicitParam(name = "id", value = "ID", required = true, dataType = "String", paramType = "path")
@DeleteMapping("/{id}")
public R deleteScheduleLogById(@PathVariable String id) {
scheduleLogService.deleteScheduleLogById(id);
return R.ok();
}
/**
* 批量删除
* @param ids
* @return
*/
@ApiOperation(value = "批量删除", notes = "根据url的ids来批量删除对象")
@ApiImplicitParam(name = "ids", value = "ID集合", required = true, dataType = "List", paramType = "path")
@DeleteMapping("/batch/{ids}")
public R deleteScheduleLogBatch(@PathVariable List<String> ids) {
scheduleLogService.deleteScheduleLogBatch(ids);
return R.ok();
}
}
datax-modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/dao/ScheduleLogDao.java 0 → 100644
package cn.datax.service.data.quality.dao;
import cn.datax.common.base.BaseDao;
import cn.datax.service.data.quality.api.entity.ScheduleLogEntity;
import com.baomidou.mybatisplus.core.conditions.Wrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.core.toolkit.Constants;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
/**
* <p>
* 数据质量监控任务日志信息表 Mapper 接口
* </p>
*
* @author yuwei
* @since 2020-10-13
*/
@Mapper
public interface ScheduleLogDao extends BaseDao<ScheduleLogEntity> {
@Override
<E extends IPage<ScheduleLogEntity>> E selectPage(E page, @Param(Constants.WRAPPER) Wrapper<ScheduleLogEntity> queryWrapper);
}
datax-modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/mapstruct/ScheduleLogMapper.java 0 → 100644
package cn.datax.service.data.quality.mapstruct;
import cn.datax.service.data.quality.api.entity.ScheduleLogEntity;
import cn.datax.service.data.quality.api.vo.ScheduleLogVo;
import org.mapstruct.Mapper;
import java.util.List;
/**
* <p>
* 数据质量监控任务日志信息表 Mapper 实体映射
* </p>
*
* @author yuwei
* @since 2020-10-13
*/
@Mapper(componentModel = "spring")
public interface ScheduleLogMapper {
/**
* 将源对象转换为VO对象
* @param e
* @return D
*/
ScheduleLogVo toVO(ScheduleLogEntity e);
/**
* 将源对象集合转换为VO对象集合
* @param es
* @return List<D>
*/
List<ScheduleLogVo> toVO(List<ScheduleLogEntity> es);
}
datax-modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/schedule/SchedulingRunnable.java
......@@ -17,17 +17,16 @@ import java.util.Objects;
@Slf4j
public class SchedulingRunnable implements Runnable {
private String id;
private String beanName;
private String methodName;
private String params;
public SchedulingRunnable(String beanName, String methodName) {
this(beanName, methodName, null);
}
public SchedulingRunnable(String beanName, String methodName, String params) {
public SchedulingRunnable(String id, String beanName, String methodName, String params) {
this.id = id;
this.beanName = beanName;
this.methodName = methodName;
this.params = params;
......@@ -35,9 +34,10 @@ public class SchedulingRunnable implements Runnable {
@Override
public void run() {
log.info("定时任务开始执行 - bean：{}，方法：{}，参数：{}", beanName, methodName, params);
log.info("定时任务开始执行 - id：{}，bean：{}，方法：{}，参数：{}", id, beanName, methodName, params);
long startTime = System.currentTimeMillis();
Map<String, Object> map = new HashMap<>();
map.put("id", id);
String batch;
try {
Object target = SpringContextHolder.getBean(beanName);
......@@ -50,10 +50,10 @@ public class SchedulingRunnable implements Runnable {
ReflectionUtils.makeAccessible(method);
method.invoke(target, map);
} catch (Exception ex) {
log.error(String.format("定时任务执行异常 - bean：%s，方法：%s，参数：%s ", beanName, methodName, params), ex);
log.error(String.format("定时任务执行异常 - id：%s，bean：%s，方法：%s，参数：%s ", id, beanName, methodName, params), ex);
}
long times = System.currentTimeMillis() - startTime;
log.info("定时任务执行结束 - bean：{}，方法：{}，参数：{}，耗时：{} 毫秒", beanName, methodName, params, times);
log.info("定时任务执行结束 - id：{}，bean：{}，方法：{}，参数：{}，耗时：{} 毫秒", id, beanName, methodName, params, times);
}
@Override
......@@ -66,11 +66,13 @@ public class SchedulingRunnable implements Runnable {
}
SchedulingRunnable that = (SchedulingRunnable) o;
if (params == null) {
return beanName.equals(that.beanName) &&
return id.equals(that.id) &&
beanName.equals(that.beanName) &&
methodName.equals(that.methodName) &&
that.params == null;
}
return beanName.equals(that.beanName) &&
return id.equals(that.id) &&
beanName.equals(that.beanName) &&
methodName.equals(that.methodName) &&
params.equals(that.params);
}
......@@ -78,8 +80,8 @@ public class SchedulingRunnable implements Runnable {
@Override
public int hashCode() {
if (params == null) {
return Objects.hash(beanName, methodName);
return Objects.hash(id, beanName, methodName);
}
return Objects.hash(beanName, methodName, params);
return Objects.hash(id, beanName, methodName, params);
}
}
datax-modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/schedule/task/QualityTask.java
......@@ -5,10 +5,15 @@ import cn.datax.common.database.DataSourceFactory;
import cn.datax.service.data.metadata.api.feign.MetadataSourceServiceFeign;
import cn.datax.service.data.quality.api.entity.CheckReportEntity;
import cn.datax.service.data.quality.api.entity.CheckRuleEntity;
import cn.datax.service.data.quality.api.entity.ScheduleLogEntity;
import cn.datax.service.data.quality.schedule.exception.ChildThreadException;
import cn.datax.service.data.quality.schedule.thread.MultiThreadHandler;
import cn.datax.service.data.quality.schedule.thread.parallel.ParallelTaskWithThreadPool;
import cn.datax.service.data.quality.service.CheckReportService;
import cn.datax.service.data.quality.service.CheckRuleService;
import cn.datax.service.data.quality.service.ScheduleLogService;
import cn.hutool.core.util.StrUtil;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import com.baomidou.mybatisplus.core.toolkit.Wrappers;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.concurrent.BasicThreadFactory;
......@@ -33,8 +38,14 @@ public class QualityTask {
@Autowired
private MetadataSourceServiceFeign metadataSourceServiceFeign;
@Autowired
private CheckReportService checkReportService;
@Autowired
private ScheduleLogService scheduleLogService;
public void task(Map<String, Object> map) {
System.out.println("执行批次：" + map);
System.out.println("执行参数：" + map);
// 结果集
List<CheckReportEntity> result = new ArrayList<>();
// 获取可执行的核查规则
......@@ -58,5 +69,27 @@ public class QualityTask {
threadPoolExecutor.shutdown();
// 核查报告
System.out.println(result);
result.stream().forEach(s -> {
// 插入核查结果正常的数据
String status = StrUtil.isBlank(s.getCheckResult()) ? DataConstant.TrueOrFalse.TRUE.getKey() : DataConstant.TrueOrFalse.FALSE.getKey();
if (StrUtil.isBlank(s.getCheckResult())) {
s.setCheckBatch((String) map.get("batch"));
checkReportService.save(s);
// 更新最近核查批次号
LambdaUpdateWrapper<CheckRuleEntity> updateWrapper = new LambdaUpdateWrapper<>();
updateWrapper.set(CheckRuleEntity::getLastCheckBatch, (String) map.get("batch"));
updateWrapper.eq(CheckRuleEntity::getId, s.getCheckRuleId());
checkRuleService.update(updateWrapper);
}
// 定时任务日志
ScheduleLogEntity scheduleLogEntity = new ScheduleLogEntity();
scheduleLogEntity.setExecuteJobId((String) map.get("id"));
scheduleLogEntity.setExecuteBatch((String) map.get("batch"));
scheduleLogEntity.setExecuteDate(s.getCheckDate());
scheduleLogEntity.setExecuteRuleId(s.getCheckRuleId());
scheduleLogEntity.setExecuteResult(s.getCheckResult());
scheduleLogEntity.setStatus(status);
scheduleLogService.save(scheduleLogEntity);
});
}
}
datax-modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/service/ScheduleLogService.java 0 → 100644
package cn.datax.service.data.quality.service;
import cn.datax.service.data.quality.api.entity.ScheduleLogEntity;
import cn.datax.common.base.BaseService;
import java.util.List;
/**
* <p>
* 数据质量监控任务日志信息表 服务类
* </p>
*
* @author yuwei
* @since 2020-10-13
*/
public interface ScheduleLogService extends BaseService<ScheduleLogEntity> {
ScheduleLogEntity getScheduleLogById(String id);
void deleteScheduleLogById(String id);
void deleteScheduleLogBatch(List<String> ids);
}
datax-modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/service/impl/ScheduleJobServiceImpl.java
......@@ -43,7 +43,7 @@ public class ScheduleJobServiceImpl extends BaseServiceImpl<ScheduleJobDao, Sche
@Override
public void pauseScheduleJobById(String id) {
ScheduleJobEntity scheduleJobEntity = super.getById(id);
SchedulingRunnable task = new SchedulingRunnable(scheduleJobEntity.getBeanName(), scheduleJobEntity.getMethodName(), scheduleJobEntity.getMethodParams());
SchedulingRunnable task = new SchedulingRunnable(id, scheduleJobEntity.getBeanName(), scheduleJobEntity.getMethodName(), scheduleJobEntity.getMethodParams());
cronTaskRegistrar.removeCronTask(task);
scheduleJobEntity.setStatus(DataConstant.TrueOrFalse.FALSE.getKey());
scheduleJobDao.updateById(scheduleJobEntity);
......@@ -52,7 +52,7 @@ public class ScheduleJobServiceImpl extends BaseServiceImpl<ScheduleJobDao, Sche
@Override
public void resumeScheduleJobById(String id) {
ScheduleJobEntity scheduleJobEntity = super.getById(id);
SchedulingRunnable task = new SchedulingRunnable(scheduleJobEntity.getBeanName(), scheduleJobEntity.getMethodName(), scheduleJobEntity.getMethodParams());
SchedulingRunnable task = new SchedulingRunnable(id, scheduleJobEntity.getBeanName(), scheduleJobEntity.getMethodName(), scheduleJobEntity.getMethodParams());
cronTaskRegistrar.addCronTask(task, scheduleJobEntity.getCronExpression());
scheduleJobEntity.setStatus(DataConstant.TrueOrFalse.TRUE.getKey());
scheduleJobDao.updateById(scheduleJobEntity);
......
datax-modules/data-quality-service-parent/data-quality-service/src/main/java/cn/datax/service/data/quality/service/impl/ScheduleLogServiceImpl.java 0 → 100644
package cn.datax.service.data.quality.service.impl;
import cn.datax.service.data.quality.api.entity.ScheduleLogEntity;
import cn.datax.service.data.quality.mapstruct.ScheduleLogMapper;
import cn.datax.service.data.quality.service.ScheduleLogService;
import cn.datax.service.data.quality.dao.ScheduleLogDao;
import cn.datax.common.base.BaseServiceImpl;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
import java.util.List;
/**
* <p>
* 数据质量监控任务日志信息表 服务实现类
* </p>
*
* @author yuwei
* @since 2020-10-13
*/
@Service
@Transactional(propagation = Propagation.SUPPORTS, readOnly = true, rollbackFor = Exception.class)
public class ScheduleLogServiceImpl extends BaseServiceImpl<ScheduleLogDao, ScheduleLogEntity> implements ScheduleLogService {
@Autowired
private ScheduleLogDao scheduleLogDao;
@Autowired
private ScheduleLogMapper scheduleLogMapper;
@Override
public ScheduleLogEntity getScheduleLogById(String id) {
ScheduleLogEntity scheduleLogEntity = super.getById(id);
return scheduleLogEntity;
}
@Override
@Transactional(rollbackFor = Exception.class)
public void deleteScheduleLogById(String id) {
scheduleLogDao.deleteById(id);
}
@Override
@Transactional(rollbackFor = Exception.class)
public void deleteScheduleLogBatch(List<String> ids) {
scheduleLogDao.deleteBatchIds(ids);
}
}
datax-modules/data-quality-service-parent/data-quality-service/src/main/resources/mapper/CheckRuleMapper.xml
......@@ -14,7 +14,6 @@
<result column="remark" property="remark" />
<result column="rule_name" property="ruleName" />
<result column="rule_type_id" property="ruleTypeId" />
<result column="rule_type" property="ruleType" />
<result column="rule_level" property="ruleLevel" />
<result column="rule_source_id" property="ruleSourceId" />
<result column="rule_source" property="ruleSource" />
......@@ -28,6 +27,10 @@
<result column="last_check_batch" property="lastCheckBatch" />
</resultMap>
<resultMap id="ExtendResultMap" type="cn.datax.service.data.quality.api.entity.CheckRuleEntity" extends="BaseResultMap">
<result column="rule_type" property="ruleType" />
</resultMap>
<!-- 通用查询结果列 -->
<sql id="Base_Column_List">
id,
......@@ -55,7 +58,7 @@
${alias}.rule_table_id, ${alias}.rule_table, ${alias}.rule_table_comment, ${alias}.rule_column_id, ${alias}.rule_column, ${alias}.rule_column_comment, ${alias}.rule_sql, ${alias}.last_check_batch
</sql>
<select id="selectPage" resultMap="BaseResultMap">
<select id="selectPage" resultMap="ExtendResultMap">
SELECT t.name as rule_type,
<include refid="Rule_Column_List"><property name="alias" value="r"/></include>
FROM quality_check_rule r
......
datax-modules/data-quality-service-parent/data-quality-service/src/main/resources/mapper/ScheduleLogMapper.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="cn.datax.service.data.quality.dao.ScheduleLogDao">
<!-- 通用查询映射结果 -->
<resultMap id="BaseResultMap" type="cn.datax.service.data.quality.api.entity.ScheduleLogEntity">
<result column="id" property="id" />
<result column="status" property="status" />
<result column="execute_job_id" property="executeJobId" />
<result column="execute_rule_id" property="executeRuleId" />
<result column="execute_date" property="executeDate" />
<result column="execute_result" property="executeResult" />
<result column="execute_batch" property="executeBatch" />
</resultMap>
<resultMap id="ExtendResultMap" type="cn.datax.service.data.quality.api.entity.ScheduleLogEntity" extends="BaseResultMap">
<result column="execute_job_name" property="executeJobName" />
<result column="execute_rule_name" property="executeRuleName" />
<result column="execute_rule_type_name" property="executeRuleTypeName" />
</resultMap>
<!-- 通用查询结果列 -->
<sql id="Base_Column_List">
id,
status,
execute_job_id, execute_rule_id, execute_date, execute_result, execute_batch
</sql>
<sql id="Log_Column_List">
${alias}.id,
${alias}.status,
${alias}.execute_job_id, ${alias}.execute_rule_id, ${alias}.execute_date, ${alias}.execute_result, ${alias}.execute_batch
</sql>
<select id="selectPage" resultMap="ExtendResultMap">
SELECT r.rule_name as execute_rule_name, t.name as execute_rule_type_name, j.job_name as execute_job_name,
<include refid="Log_Column_List"><property name="alias" value="l"/></include>
FROM quality_schedule_log l
LEFT JOIN quality_schedule_job j ON j.id = l.execute_job_id
LEFT JOIN quality_check_rule r ON r.id = l.execute_rule_id
LEFT JOIN quality_rule_type t ON t.id = r.rule_type_id
${ew.customSqlSegment}
</select>
</mapper>
datax-ui/src/api/quality/checklog.js 0 → 100644
import request from '@/utils/request'
export function pageCheckLog(data) {
return request({
url: '/data/quality/scheduleLogs/page',
method: 'get',
params: data
})
}
datax-ui/src/views/quality/checkjob/CheckJobList.vue
......@@ -45,6 +45,12 @@
icon="el-icon-delete"
@click="resumePause(scope.row)"
>恢复任务</el-button>
<el-button
size="mini"
type="text"
icon="el-icon-delete"
@click="handleLog(scope.row)"
>执行日志</el-button>
<el-button slot="reference">操作</el-button>
</el-popover>
</template>
......@@ -74,7 +80,8 @@ export default {
// 展示切换
showOptions: {
data: {},
showList: true
showList: true,
showLogList: false
},
// 遮罩层
loading: true,
......@@ -156,6 +163,13 @@ export default {
}).catch(() => {
})
},
handleLog(row) {
this.showOptions.data = {}
this.showOptions.data.id = row.id
this.showOptions.showList = false
this.showOptions.showLogList = true
this.$emit('showCard', this.showOptions)
},
handleSizeChange(val) {
console.log(`每页 ${val} 条`)
this.queryParams.pageNum = 1
......
datax-ui/src/views/quality/checkjob/index.vue
......@@ -3,20 +3,25 @@
<transition name="el-zoom-in-center">
<check-job-list v-if="options.showList" @showCard="showCard" />
</transition>
<transition name="el-zoom-in-bottom">
<check-log-list v-if="options.showLogList" :data="options.data" @showCard="showCard"/>
</transition>
</div>
</template>
<script>
import CheckJobList from './CheckJobList'
import CheckLogList from './log/CheckLogList'
export default {
name: 'CheckJob',
components: { CheckJobList },
components: { CheckJobList, CheckLogList },
data() {
return {
options: {
data: {},
showList: true
showList: true,
showLogList: false
}
}
},
......
datax-ui/src/views/quality/checkjob/log/CheckLogList.vue 0 → 100644
<template>
<el-card class="box-card" shadow="always">
<el-form ref="queryForm" :model="queryParams" :inline="true">
<el-form-item label="规则类型" prop="ruleTypeId">
<el-select v-model="queryParams.ruleTypeId" clearable placeholder="请选择规则类型">
<el-option
v-for="item in ruleTypeOptions"
:key="item.id"
:label="item.name"
:value="item.id"
/>
</el-select>
</el-form-item>
<el-form-item>
<el-button type="primary" icon="el-icon-search" size="mini" @click="handleQuery">搜索</el-button>
<el-button icon="el-icon-refresh" size="mini" @click="resetQuery">重置</el-button>
<el-button size="mini" icon="el-icon-back" round @click="showCard">返回主表</el-button>
</el-form-item>
</el-form>
<el-table
v-loading="loading"
:data="tableDataList"
border
tooltip-effect="dark"
:height="tableHeight"
style="width: 100%;margin: 15px 0;"
>
<el-table-column type="selection" width="55" align="center" />
<el-table-column label="序号" width="55" align="center">
<template slot-scope="scope">
<span>{{ scope.$index +1 }}</span>
</template>
</el-table-column>
<template v-for="(item, index) in tableColumns">
<el-table-column
v-if="item.show"
:key="index"
:prop="item.prop"
:label="item.label"
:formatter="item.formatter"
align="center"
show-overflow-tooltip
/>
</template>
</el-table>
<el-pagination
:page-sizes="[10, 20, 50, 100]"
layout="total, sizes, prev, pager, next, jumper"
:current-page.sync="queryParams.pageNum"
:page-size.sync="queryParams.pageSize"
:total="total"
@size-change="handleSizeChange"
@current-change="handleCurrentChange"
/>
</el-card>
</template>
<script>
import { listRuleType } from '@/api/quality/checkrule'
import { pageCheckLog } from '@/api/quality/checklog'
export default {
name: 'CheckLogList',
props: {
data: {
type: Object,
default: function() {
return {}
}
}
},
data() {
return {
tableHeight: document.body.offsetHeight - 310 + 'px',
// 展示切换
showOptions: {
data: {},
showList: true,
showLogList: false
},
// 遮罩层
loading: true,
// 表格头
tableColumns: [
{ prop: 'executeJobName', label: '任务名称', show: true },
{ prop: 'executeRuleTypeName', label: '规则类型', show: true },
{ prop: 'executeRuleName', label: '规则名称', show: true },
{ prop: 'executeBatch', label: '执行批次', show: true },
{
prop: 'status',
label: '状态',
show: true,
formatter: this.statusFormatter
},
{ prop: 'executeDate', label: '执行时间', show: true }
],
// 状态数据字典
statusOptions: [],
// 数据集表格数据
tableDataList: [],
// 总数据条数
total: 0,
// 查询参数
queryParams: {
pageNum: 1,
pageSize: 20,
executeJobId: '',
ruleTypeId: ''
},
ruleTypeOptions: []
}
},
created() {
this.queryParams.executeJobId = this.data.id
this.getDicts('sys_normal_status').then(response => {
if (response.success) {
this.statusOptions = response.data
}
})
this.getRuleTypeList()
this.getList()
},
methods: {
showCard() {
this.$emit('showCard', this.showOptions)
},
getRuleTypeList() {
listRuleType().then(response => {
if (response.success) {
this.ruleTypeOptions = response.data
}
})
},
/** 查询数据集列表 */
getList() {
this.loading = true
pageCheckLog(this.queryParams).then(response => {
this.loading = false
if (response.success) {
const { data } = response
this.tableDataList = data.data
this.total = data.total
}
})
},
/** 搜索按钮操作 */
handleQuery() {
this.queryParams.pageNum = 1
this.getList()
},
/** 重置按钮操作 */
resetQuery() {
this.queryParams = {
pageNum: 1,
pageSize: 20,
executeJobId: this.data.id,
ruleTypeId: ''
}
this.handleQuery()
},
handleSizeChange(val) {
console.log(`每页 ${val} 条`)
this.queryParams.pageNum = 1
this.queryParams.pageSize = val
this.getList()
},
handleCurrentChange(val) {
console.log(`当前页: ${val}`)
this.queryParams.pageNum = val
this.getList()
},
statusFormatter(row, column, cellValue, index) {
const dictLabel = this.selectDictLabel(this.statusOptions, cellValue)
if (cellValue === '1') {
return <el-tag type='success'>{dictLabel}</el-tag>
} else {
return <el-tag type='warning'>{dictLabel}</el-tag>
}
}
}
}
</script>
<style lang="scss" scoped>
.right-toolbar {
float: right;
}
.el-card ::v-deep .el-card__body {
height: calc(100vh - 170px);
}
</style>
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment