Skip to content

D
datax-cloud
Commit cd99b761 by yuwei
Options

2.0.0项目初始化

parent f7b5c412
Showing with 252 additions and 29 deletions
datax-modules/data-market-service-parent/data-market-service-api/src/main/java/cn/datax/service/data/market/api/dto/ExecuteConfig.java
......@@ -22,7 +22,7 @@ public class ExecuteConfig implements Serializable {
@ApiModelProperty(value = "配置方式")
@NotBlank(message = "配置方式不能为空", groups = {ValidationGroups.Insert.class, ValidationGroups.Update.class})
private String configType;
private Integer configType;
@ApiModelProperty(value = "选择表")
private String tableName;
......
datax-modules/data-market-service-parent/data-market-service-api/src/main/java/cn/datax/service/data/market/api/dto/ReqParam.java
......@@ -6,7 +6,6 @@ import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotNull;
import java.io.Serializable;
@ApiModel(value = "请求参数信息Model")
......@@ -19,21 +18,13 @@ public class ReqParam implements Serializable {
@NotBlank(message = "参数名称不能为空", groups = {ValidationGroups.Insert.class, ValidationGroups.Update.class})
private String paramName;
@ApiModelProperty(value = "字段名称")
@NotBlank(message = "字段名称不能为空", groups = {ValidationGroups.Insert.class, ValidationGroups.Update.class})
private String fieldName;
@ApiModelProperty(value = "是否为空")
@NotNull(message = "是否为空不能为空", groups = {ValidationGroups.Insert.class, ValidationGroups.Update.class})
private Integer nullable;
@ApiModelProperty(value = "描述")
@NotBlank(message = "描述不能为空", groups = {ValidationGroups.Insert.class, ValidationGroups.Update.class})
private String remark;
@ApiModelProperty(value = "数据类型")
@NotBlank(message = "数据类型不能为空", groups = {ValidationGroups.Insert.class, ValidationGroups.Update.class})
private String dataType;
private Integer dataType;
@ApiModelProperty(value = "示例值")
@NotBlank(message = "示例值不能为空", groups = {ValidationGroups.Insert.class, ValidationGroups.Update.class})
......
datax-modules/data-market-service-parent/data-market-service-api/src/main/java/cn/datax/service/data/market/api/dto/ResParam.java
......@@ -18,17 +18,13 @@ public class ResParam implements Serializable {
@NotBlank(message = "参数名称不能为空", groups = {ValidationGroups.Insert.class, ValidationGroups.Update.class})
private String paramName;
@ApiModelProperty(value = "字段名称")
@NotBlank(message = "字段名称不能为空", groups = {ValidationGroups.Insert.class, ValidationGroups.Update.class})
private String fieldName;
@ApiModelProperty(value = "描述")
@NotBlank(message = "描述不能为空", groups = {ValidationGroups.Insert.class, ValidationGroups.Update.class})
private String remark;
@ApiModelProperty(value = "数据类型")
@NotBlank(message = "数据类型不能为空", groups = {ValidationGroups.Insert.class, ValidationGroups.Update.class})
private String dataType;
private Integer dataType;
@ApiModelProperty(value = "示例值")
@NotBlank(message = "示例值不能为空", groups = {ValidationGroups.Insert.class, ValidationGroups.Update.class})
......
datax-modules/data-market-service-parent/data-market-service-api/src/main/java/cn/datax/service/data/market/api/dto/SqlParseDto.java 0 → 100644
package cn.datax.service.data.market.api.dto;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import javax.validation.constraints.NotBlank;
import java.io.Serializable;
@Data
public class SqlParseDto implements Serializable {
private static final long serialVersionUID=1L;
@ApiModelProperty(value = "SQL文本")
@NotBlank(message = "SQL不能为空")
private String sqlText;
}
datax-modules/data-market-service-parent/data-market-service-api/src/main/java/cn/datax/service/data/market/api/dto/TryParam.java
package cn.datax.service.data.market.api.dto;
import cn.datax.common.validate.ValidationGroups;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
......@@ -15,7 +14,7 @@ public class TryParam implements Serializable {
private static final long serialVersionUID=1L;
@ApiModelProperty(value = "参数名称")
@NotBlank(message = "参数名称不能为空", groups = {ValidationGroups.Insert.class, ValidationGroups.Update.class})
@NotBlank(message = "参数名称不能为空")
private String paramName;
@ApiModelProperty(value = "参数值")
......
datax-modules/data-market-service-parent/data-market-service-api/src/main/java/cn/datax/service/data/market/api/entity/DataApiEntity.java
......@@ -96,4 +96,9 @@ public class DataApiEntity extends DataScopeBaseEntity {
*/
@TableField(value = "res_json", typeHandler = JacksonTypeHandler.class)
private List<ResParam> resParams;
/**
* 执行sql
*/
private String sqlText;
}
datax-modules/data-market-service-parent/data-market-service-api/src/main/java/cn/datax/service/data/market/api/vo/SqlParseVo.java 0 → 100644
package cn.datax.service.data.market.api.vo;
import cn.datax.service.data.market.api.dto.ReqParam;
import cn.datax.service.data.market.api.dto.ResParam;
import lombok.Data;
import java.io.Serializable;
import java.util.List;
@Data
public class SqlParseVo implements Serializable {
private static final long serialVersionUID=1L;
private String sqlText;
private List<ReqParam> reqParams;
private List<ResParam> resParams;
}
datax-modules/data-market-service-parent/data-market-service/src/main/java/cn/datax/service/data/market/config/ApiInterceptor.java
......@@ -24,6 +24,7 @@ import javax.servlet.http.HttpServletResponse;
import java.time.LocalDateTime;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
@Slf4j
......@@ -59,7 +60,7 @@ public class ApiInterceptor implements HandlerInterceptor {
return false;
}
String deny = dataApiEntity.getDeny();
deny = StrUtil.isBlank(deny) ? "" : deny;
deny = Optional.ofNullable(deny).orElse("");
List<String> denyList = Arrays.asList(deny.split(","));
if (CollUtil.isNotEmpty(denyList)) {
for (String ip : denyList) {
......@@ -72,15 +73,15 @@ public class ApiInterceptor implements HandlerInterceptor {
}
String apiName = dataApiEntity.getApiName();
Integer rateLimit = dataApiEntity.getRateLimit();
rateLimit = rateLimit == null ? 1 : rateLimit;
rateLimit = Optional.ofNullable(rateLimit).orElse(DataConstant.TrueOrFalse.TRUE.getKey());
// 限流
if (rateLimit == 1) {
if (DataConstant.TrueOrFalse.TRUE.getKey() == rateLimit) {
Integer times = dataApiEntity.getTimes();
Integer seconds = dataApiEntity.getSeconds();
// 请求次数
times = times == null ? 5 : times;
times = Optional.ofNullable(times).orElse(5);
// 请求时间范围60秒
seconds = seconds == null ? 60 : seconds;
seconds = Optional.ofNullable(seconds).orElse(60);
// 根据 USER + API 限流
String key = "user:" + userId + "_api:" + apiId;
// 根据key获取已请求次数
......
datax-modules/data-market-service-parent/data-market-service/src/main/java/cn/datax/service/data/market/controller/DataApiController.java
......@@ -4,9 +4,11 @@ import cn.datax.common.core.JsonPage;
import cn.datax.common.core.R;
import cn.datax.common.validate.ValidationGroups;
import cn.datax.service.data.market.api.dto.DataApiDto;
import cn.datax.service.data.market.api.dto.SqlParseDto;
import cn.datax.service.data.market.api.entity.DataApiEntity;
import cn.datax.service.data.market.api.vo.DataApiVo;
import cn.datax.service.data.market.api.query.DataApiQuery;
import cn.datax.service.data.market.api.vo.SqlParseVo;
import cn.datax.service.data.market.mapstruct.DataApiMapper;
import cn.datax.service.data.market.service.DataApiService;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
......@@ -119,4 +121,17 @@ public class DataApiController extends BaseController {
dataApiService.deleteDataApiById(id);
return R.ok();
}
/**
* SQL解析
* @param sqlParseDto
* @return
*/
@ApiOperation(value = "SQL解析")
@ApiImplicitParam(name = "sqlParseDto", value = "SQL解析实体sqlParseDto", required = true, dataType = "SqlParseDto")
@PostMapping("/sql/parse")
public R sqlParse(@RequestBody @Validated SqlParseDto sqlParseDto) {
SqlParseVo sqlParseVo = dataApiService.sqlParse(sqlParseDto);
return R.ok().setData(sqlParseVo);
}
}
datax-modules/data-market-service-parent/data-market-service/src/main/java/cn/datax/service/data/market/enums/ConfigEnum.java 0 → 100644
package cn.datax.service.data.market.enums;
public enum ConfigEnum {
FORM(1, "表引导模式"),
SCRIPT(2, "脚本模式");
private final Integer key;
private final String val;
ConfigEnum(Integer key, String val) {
this.key = key;
this.val = val;
}
public Integer getKey() {
return key;
}
public String getVal() {
return val;
}
}
datax-modules/data-market-service-parent/data-market-service/src/main/java/cn/datax/service/data/market/service/DataApiService.java
package cn.datax.service.data.market.service;
import cn.datax.service.data.market.api.dto.SqlParseDto;
import cn.datax.service.data.market.api.entity.DataApiEntity;
import cn.datax.service.data.market.api.dto.DataApiDto;
import cn.datax.common.base.BaseService;
import cn.datax.service.data.market.api.vo.SqlParseVo;
/**
* <p>
......@@ -19,4 +21,6 @@ public interface DataApiService extends BaseService<DataApiEntity> {
void updateDataApi(DataApiDto dataApi);
void deleteDataApiById(String id);
SqlParseVo sqlParse(SqlParseDto sqlParseDto);
}
datax-modules/data-market-service-parent/data-market-service/src/main/java/cn/datax/service/data/market/service/impl/ApiServiceImpl.java
package cn.datax.service.data.market.service.impl;
import cn.datax.common.exception.DataException;
import cn.datax.common.utils.ThrowableUtil;
import cn.datax.service.data.market.service.ApiService;
import cn.datax.service.data.market.utils.ThreadUtil;
......@@ -17,13 +18,17 @@ public class ApiServiceImpl implements ApiService {
} catch (InterruptedException e) {
e.printStackTrace();
}
System.out.println("time:" + ThreadUtil.get().getTime());
try {
int i = 1/1;
if (ThreadUtil.get().getCallerUrl().contains("/24")) {
int i = 1/0;
} else {
int i = 1/1;
}
} catch (Exception e) {
log.error("全局异常信息ex={}, StackTrace={}", e.getMessage(), ThrowableUtil.getStackTrace(e));
ThreadUtil.get().setStatus(0);
ThreadUtil.get().setMsg(e.getMessage());
throw new DataException("API调用出错");
}
}
}
datax-modules/data-market-service-parent/data-market-service/src/main/java/cn/datax/service/data/market/service/impl/DataApiServiceImpl.java
package cn.datax.service.data.market.service.impl;
import cn.datax.common.exception.DataException;
import cn.datax.common.utils.ThrowableUtil;
import cn.datax.service.data.market.api.dto.*;
import cn.datax.service.data.market.api.entity.DataApiEntity;
import cn.datax.service.data.market.api.dto.DataApiDto;
import cn.datax.service.data.market.api.vo.SqlParseVo;
import cn.datax.service.data.market.enums.ConfigEnum;
import cn.datax.service.data.market.service.DataApiService;
import cn.datax.service.data.market.mapstruct.DataApiMapper;
import cn.datax.service.data.market.dao.DataApiDao;
import cn.datax.common.base.BaseServiceImpl;
import lombok.extern.slf4j.Slf4j;
import net.sf.jsqlparser.JSQLParserException;
import net.sf.jsqlparser.expression.ExpressionVisitorAdapter;
import net.sf.jsqlparser.expression.Function;
import net.sf.jsqlparser.expression.JdbcNamedParameter;
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import net.sf.jsqlparser.parser.SimpleNode;
import net.sf.jsqlparser.schema.Column;
import net.sf.jsqlparser.schema.Table;
import net.sf.jsqlparser.statement.Statement;
import net.sf.jsqlparser.statement.StatementVisitorAdapter;
import net.sf.jsqlparser.statement.select.*;
import net.sf.jsqlparser.util.SelectUtils;
import net.sf.jsqlparser.util.deparser.ExpressionDeParser;
import net.sf.jsqlparser.util.deparser.SelectDeParser;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.CacheConfig;
import org.springframework.cache.annotation.CacheEvict;
......@@ -16,6 +35,9 @@ import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
/**
* <p>
......@@ -25,6 +47,7 @@ import java.io.Serializable;
* @author yuwei
* @since 2020-03-31
*/
@Slf4j
@CacheConfig(cacheNames = {"apis"})
@Service
@Transactional(propagation = Propagation.SUPPORTS, readOnly = true, rollbackFor = Exception.class)
......@@ -39,7 +62,7 @@ public class DataApiServiceImpl extends BaseServiceImpl<DataApiDao, DataApiEntit
@Override
@Transactional(rollbackFor = Exception.class)
public void saveDataApi(DataApiDto dataApiDto) {
DataApiEntity dataApi = dataApiMapper.toEntity(dataApiDto);
DataApiEntity dataApi = shareCode(dataApiDto);
dataApiDao.insert(dataApi);
}
......@@ -47,10 +70,31 @@ public class DataApiServiceImpl extends BaseServiceImpl<DataApiDao, DataApiEntit
@Override
@Transactional(rollbackFor = Exception.class)
public void updateDataApi(DataApiDto dataApiDto) {
DataApiEntity dataApi = dataApiMapper.toEntity(dataApiDto);
DataApiEntity dataApi = shareCode(dataApiDto);
dataApiDao.updateById(dataApi);
}
private DataApiEntity shareCode(DataApiDto dataApiDto) {
DataApiEntity dataApi = dataApiMapper.toEntity(dataApiDto);
Integer configType = dataApi.getExecuteConfig().getConfigType();
if (ConfigEnum.FORM.getKey() == configType) {
try {
dataApi.setSqlText(sqlBuild(dataApi));
} catch (JSQLParserException e) {
log.error("全局异常信息ex={}, StackTrace={}", e.getMessage(), ThrowableUtil.getStackTrace(e));
throw new DataException("SQL语法有问题，解析出错");
}
} else if (ConfigEnum.SCRIPT.getKey() == configType) {
try {
dataApi.setSqlText(sqlJdbcNamedParameterParse(dataApi.getExecuteConfig().getSqlText()));
} catch (JSQLParserException e) {
log.error("全局异常信息ex={}, StackTrace={}", e.getMessage(), ThrowableUtil.getStackTrace(e));
throw new DataException("SQL语法有问题，解析出错");
}
}
return dataApi;
}
@Cacheable(key = "#id", unless = "#result == null")
@Override
public DataApiEntity getById(Serializable id) {
......@@ -63,4 +107,108 @@ public class DataApiServiceImpl extends BaseServiceImpl<DataApiDao, DataApiEntit
public void deleteDataApiById(String id) {
dataApiDao.deleteById(id);
}
@Override
public SqlParseVo sqlParse(SqlParseDto sqlParseDto) {
Statement stmt;
try {
stmt = CCJSqlParserUtil.parse(sqlParseDto.getSqlText());
} catch (JSQLParserException e) {
log.error("全局异常信息ex={}, StackTrace={}", e.getMessage(), ThrowableUtil.getStackTrace(e));
throw new DataException("SQL语法有问题，解析出错");
}
final List<String> variables = new ArrayList<>();
final List<String> cols = new ArrayList<>();
stmt.accept(new StatementVisitorAdapter() {
@Override
public void visit(Select select) {
select.getSelectBody().accept(new SelectVisitorAdapter() {
@Override
public void visit(PlainSelect plainSelect) {
plainSelect.getSelectItems().stream().forEach(selectItem -> {
selectItem.accept(new SelectItemVisitorAdapter() {
@Override
public void visit(SelectExpressionItem item) {
String columnName;
if (item.getAlias() == null) {
SimpleNode node = item.getExpression().getASTNode();
Object value = node.jjtGetValue();
if (value instanceof Column) {
columnName = ((Column) value).getColumnName();
} else if (value instanceof Function) {
columnName = value.toString();
} else {
// 增加对select 'aaa' from table; 的支持
columnName = String.valueOf(value);
columnName = columnName.replace("'", "");
columnName = columnName.replace("\"", "");
columnName = columnName.replace("`", "");
}
} else {
columnName = item.getAlias().getName();
}
columnName = columnName.replace("'", "");
columnName = columnName.replace("\"", "");
columnName = columnName.replace("`", "");
cols.add(columnName);
}
});
});
plainSelect.getWhere().accept(new ExpressionVisitorAdapter() {
@Override
public void visit(JdbcNamedParameter jdbcNamedParameter) {
variables.add(jdbcNamedParameter.getName());
}
});
}
});
}
});
SqlParseVo sqlParseVo = new SqlParseVo();
sqlParseVo.setSqlText(stmt.toString());
List<ReqParam> reqParams = variables.stream().map(s -> {
ReqParam reqParam = new ReqParam();
reqParam.setParamName(s);
return reqParam;
}).collect(Collectors.toList());
sqlParseVo.setReqParams(reqParams);
List<ResParam> resParams = cols.stream().map(s -> {
ResParam resParam = new ResParam();
resParam.setParamName(s);
return resParam;
}).collect(Collectors.toList());
sqlParseVo.setResParams(resParams);
return sqlParseVo;
}
private String sqlBuild(DataApiEntity dataApi) throws JSQLParserException {
Table table = new Table(dataApi.getExecuteConfig().getTableName());
String[] resParams = dataApi.getResParams().stream().map(s -> s.getParamName()).toArray(String[]::new);
Select select = SelectUtils.buildSelectFromTableAndExpressions(table, resParams);
final StringBuilder sb = new StringBuilder(select.toString());
sb.append(" ").append("WHERE").append(" ");
dataApi.getReqParams().stream().forEach(s -> sb.append(s.getParamName()).append(" = ? And "));
return sb.substring(0, sb.length() - 5);
}
private String sqlJdbcNamedParameterParse(String sqlText) throws JSQLParserException {
final StringBuilder buffer = new StringBuilder();
ExpressionDeParser expressionDeParser = new ExpressionDeParser() {
@Override
public void visit(JdbcNamedParameter jdbcNamedParameter) {
this.getBuffer().append("?");
}
};
SelectDeParser deparser = new SelectDeParser(expressionDeParser, buffer);
expressionDeParser.setSelectVisitor(deparser);
expressionDeParser.setBuffer(buffer);
Statement stmt = CCJSqlParserUtil.parse(sqlText);
stmt.accept(new StatementVisitorAdapter() {
@Override
public void visit(Select select) {
select.getSelectBody().accept(deparser);
}
});
return buffer.toString();
}
}
datax-modules/data-market-service-parent/data-market-service/src/main/resources/mapper/DataApiMapper.xml
......@@ -21,6 +21,7 @@
<result column="rate_limit" property="rateLimit" />
<result column="times" property="times" />
<result column="seconds" property="seconds" />
<result column="sql_text" property="sqlText" />
<result column="config_json" property="executeConfig" typeHandler="com.baomidou.mybatisplus.extension.handlers.JacksonTypeHandler"/>
<result column="req_json" property="reqParams" typeHandler="com.baomidou.mybatisplus.extension.handlers.JacksonTypeHandler"/>
<result column="res_json" property="resParams" typeHandler="com.baomidou.mybatisplus.extension.handlers.JacksonTypeHandler"/>
......@@ -35,7 +36,7 @@
create_dept,
update_by,
update_time,
api_name, api_version, api_url, remark, req_method, res_type, deny, rate_limit, times, seconds, config_json, req_json, res_json
api_name, api_version, api_url, remark, req_method, res_type, deny, rate_limit, times, seconds, sql_text, config_json, req_json, res_json
</sql>
</mapper>
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment