Commit d060d162 by y1sa

改造为oracle同步

parent d5f5ba3a
...@@ -40,3 +40,6 @@ build/ ...@@ -40,3 +40,6 @@ build/
.idea/ .idea/
logs/ logs/
dbhistory.dat
offset.dat
\ No newline at end of file
source.host=192.168.0.85 source.host=192.168.0.152
source.port=1433 source.port=1521
source.user=sa source.user=datacenter
source.password=1 source.password=data
source.dbname=dz_his source.dbname=ORCL
sync.tables=dbo.BA_BRDA(ZYH) sync.tables=DATACENTER.TEST_SYNC(ID)
dest.host=192.168.0.85 dest.host=10.99.44.13
dest.port=1433 dest.port=11521
dest.user=sa dest.user=datacenter
dest.password=1 dest.password=TBYF_soft$data
dest.dbname=dz_emr dest.dbname=helowin
\ No newline at end of file \ No newline at end of file
{"source":{"server":"server1"},"position":{"commit_lsn":"000419dc:00000010:007b","snapshot":true,"snapshot_completed":false},"databaseName":"dz_his","schemaName":"dbo","tableChanges":[{"type":"CREATE","id":"\"dz_his\".\"dbo\".\"BA_BRDA\"","table":{"defaultCharsetName":null,"primaryKeyColumnNames":["ZYH"],"columns":[{"name":"ZYH","jdbcType":2,"typeName":"numeric","typeExpression":"numeric","charsetName":null,"length":18,"scale":0,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"BAHM","jdbcType":12,"typeName":"varchar","typeExpression":"varchar","charsetName":null,"length":12,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ZYHM","jdbcType":12,"typeName":"varchar","typeExpression":"varchar","charsetName":null,"length":12,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"MZHM","jdbcType":12,"typeName":"varchar","typeExpression":"varchar","charsetName":null,"length":32,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"FZHM","jdbcType":12,"typeName":"varchar","typeExpression":"varchar","charsetName":null,"length":12,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"BRXM","jdbcType":12,"typeName":"varchar","typeExpression":"varchar","charsetName":null,"length":12,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"BRLY","jdbcType":2,"typeName":"numeric","typeExpression":"numeric","charsetName":null,"length":4,"scale":0,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"RYCS","jdbcType":2,"typeName":"numeric","typeExpression":"numeric","charsetName":null,"length":4,"scale":0,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"defaultValueExpression":"((1))","enumValues":[]},{"name":"BRXB","jdbcType":2,"typeName":"numeric","typeExpression":"numeric","charsetName":null,"length":4,"scale":0,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"BRXZ","jdbcType":2,"typeName":"numeric","typeExpression":"numeric","charsetName":null,"length":4,"scale":0,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"GZDW","jdbcType":12,"typeName":"varchar","typeExpression":"varchar","charsetName":null,"length":40,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"CSNY","jdbcType":93,"typeName":"datetime","typeExpression":"datetime","charsetName":null,"length":23,"scale":3,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"BRNL","jdbcType":2,"typeName":"numeric","typeExpression":"numeric","charsetName":null,"length":6,"scale":0,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"SFZH","jdbcType":12,"typeName":"varchar","typeExpression":"varchar","charsetName":null,"length":20,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"HYZK","jdbcType":2,"typeName":"numeric","typeExpression":"numeric","charsetName":null,"length":4,"scale":0,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ZYDM","jdbcType":2,"typeName":"numeric","typeExpression":"numeric","charsetName":null,"length":4,"scale":0,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"SFDM","jdbcType":2,"typeName":"numeric","typeExpression":"numeric","charsetName":null,"length":4,"scale":0,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"JGDM","jdbcType":2,"typeName":"numeric","typeExpression":"numeric","charsetName":null,"length":4,"scale":0,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"MZDM","jdbcType":2,"typeName":"numeric","typeExpression":"numeric","charsetName":null,"length":4,"scale":0,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"GJDM","jdbcType":2,"typeName":"numeric","typeExpression":"numeric","charsetName":null,"length":4,"scale":0,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"DWDZ","jdbcType":12,"typeName":"varchar","typeExpression":"varchar","charsetName":null,"length":40,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"DWDH","jdbcType":12,"typeName":"varchar","typeExpression":"varchar","charsetName":null,"length":16,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"DWYB","jdbcType":12,"typeName":"varchar","typeExpression":"varchar","charsetName":null,"length":6,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"HKDZ","jdbcType":12,"typeName":"varchar","typeExpression":"varchar","charsetName":null,"length":40,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"HKDH","jdbcType":12,"typeName":"varchar","typeExpression":"varchar","charsetName":null,"length":16,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"HKYB","jdbcType":12,"typeName":"varchar","typeExpression":"varchar","charsetName":null,"length":6,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"LXRM","jdbcType":12,"typeName":"varchar","typeExpression":"varchar","charsetName":null,"length":12,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"LXGX","jdbcType":2,"typeName":"numeric","typeExpression":"numeric","charsetName":null,"length":4,"scale":0,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"LXDZ","jdbcType":12,"typeName":"varchar","typeExpression":"varchar","charsetName":null,"length":40,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"LXDH","jdbcType":12,"typeName":"varchar","typeExpression":"varchar","charsetName":null,"length":16,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"LXYB","jdbcType":12,"typeName":"varchar","typeExpression":"varchar","charsetName":null,"length":6,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ZZTX","jdbcType":2,"typeName":"numeric","typeExpression":"numeric","charsetName":null,"length":4,"scale":0,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"DWBH","jdbcType":2,"typeName":"numeric","typeExpression":"numeric","charsetName":null,"length":6,"scale":0,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"SBHM","jdbcType":12,"typeName":"varchar","typeExpression":"varchar","charsetName":null,"length":10,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"GFZH","jdbcType":12,"typeName":"varchar","typeExpression":"varchar","charsetName":null,"length":20,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"SJDM","jdbcType":2,"typeName":"numeric","typeExpression":"numeric","charsetName":null,"length":4,"scale":0,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"XJDM","jdbcType":2,"typeName":"numeric","typeExpression":"numeric","charsetName":null,"length":4,"scale":0,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"GZDWJDZ","jdbcType":12,"typeName":"varchar","typeExpression":"varchar","charsetName":null,"length":200,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"XZZ","jdbcType":12,"typeName":"varchar","typeExpression":"varchar","charsetName":null,"length":200,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ZZDH","jdbcType":12,"typeName":"varchar","typeExpression":"varchar","charsetName":null,"length":40,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"XZZYB","jdbcType":12,"typeName":"varchar","typeExpression":"varchar","charsetName":null,"length":6,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"YENL","jdbcType":2,"typeName":"numeric","typeExpression":"numeric","charsetName":null,"length":6,"scale":0,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"YETZ1","jdbcType":2,"typeName":"numeric","typeExpression":"numeric","charsetName":null,"length":6,"scale":0,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"YETZ2","jdbcType":2,"typeName":"numeric","typeExpression":"numeric","charsetName":null,"length":6,"scale":0,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"YETZ3","jdbcType":2,"typeName":"numeric","typeExpression":"numeric","charsetName":null,"length":6,"scale":0,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"YETZ4","jdbcType":2,"typeName":"numeric","typeExpression":"numeric","charsetName":null,"length":6,"scale":0,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"YETZ5","jdbcType":2,"typeName":"numeric","typeExpression":"numeric","charsetName":null,"length":6,"scale":0,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"YERYTZ","jdbcType":2,"typeName":"numeric","typeExpression":"numeric","charsetName":null,"length":6,"scale":0,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"CSSF","jdbcType":12,"typeName":"varchar","typeExpression":"varchar","charsetName":null,"length":30,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"CSDS","jdbcType":12,"typeName":"varchar","typeExpression":"varchar","charsetName":null,"length":30,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"CSDX","jdbcType":12,"typeName":"varchar","typeExpression":"varchar","charsetName":null,"length":30,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"YERYTZ1","jdbcType":2,"typeName":"numeric","typeExpression":"numeric","charsetName":null,"length":6,"scale":0,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"YERYTZ2","jdbcType":2,"typeName":"numeric","typeExpression":"numeric","charsetName":null,"length":6,"scale":0,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"YERYTZ3","jdbcType":2,"typeName":"numeric","typeExpression":"numeric","charsetName":null,"length":6,"scale":0,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"YERYTZ4","jdbcType":2,"typeName":"numeric","typeExpression":"numeric","charsetName":null,"length":6,"scale":0,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}]},"comment":null}]}
File deleted
...@@ -20,6 +20,7 @@ ...@@ -20,6 +20,7 @@
<logback.version>1.2.12</logback.version> <logback.version>1.2.12</logback.version>
<lombok.version>1.18.30</lombok.version> <lombok.version>1.18.30</lombok.version>
<druid.version>1.2.22</druid.version> <druid.version>1.2.22</druid.version>
<oracle.version>19.3.0.0</oracle.version>
</properties> </properties>
<dependencies> <dependencies>
...@@ -33,7 +34,7 @@ ...@@ -33,7 +34,7 @@
<artifactId>debezium-embedded</artifactId> <artifactId>debezium-embedded</artifactId>
<version>${debezium.version}</version> <version>${debezium.version}</version>
</dependency> </dependency>
<dependency> <!--<dependency>
<groupId>io.debezium</groupId> <groupId>io.debezium</groupId>
<artifactId>debezium-connector-sqlserver</artifactId> <artifactId>debezium-connector-sqlserver</artifactId>
<version>${debezium.version}</version> <version>${debezium.version}</version>
...@@ -48,7 +49,19 @@ ...@@ -48,7 +49,19 @@
<groupId>com.microsoft.sqlserver</groupId> <groupId>com.microsoft.sqlserver</groupId>
<artifactId>mssql-jdbc</artifactId> <artifactId>mssql-jdbc</artifactId>
<version>${mssql-jdbc.version}</version> <version>${mssql-jdbc.version}</version>
</dependency>-->
<dependency>
<groupId>io.debezium</groupId>
<artifactId>debezium-connector-oracle</artifactId>
<version>${debezium.version}</version>
</dependency>
<dependency>
<groupId>com.oracle.ojdbc</groupId>
<artifactId>ojdbc8</artifactId>
<version>${oracle.version}</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.commons</groupId> <groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId> <artifactId>commons-lang3</artifactId>
......
...@@ -23,6 +23,8 @@ public class Main { ...@@ -23,6 +23,8 @@ public class Main {
return; return;
} }
new AppCleaner().cleanBeforeStart(); new AppCleaner().cleanBeforeStart();
Properties props = DebeziumProperties.get(); Properties props = DebeziumProperties.get();
......
package com.tbyd.data.datasync.core; package com.tbyd.data.datasync.core;
import lombok.extern.slf4j.Slf4j;
import java.sql.Connection; import java.sql.Connection;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
import java.sql.Statement; import java.sql.Statement;
import java.util.HashSet;
import java.util.Set;
@Slf4j /**
* 开启Oracle数据库和表的supplemental logging
*/
public class CdcHelper { public class CdcHelper {
private static final String DB_CDC_ENABLED_QUERY_SQL = private static final String QUERY_DB_SUPPLEMENTAL_LOGGING_SQL = "SELECT SUPPLEMENTAL_LOG_DATA_MIN FROM V$DATABASE";
"SELECT is_cdc_enabled \n" +
"FROM sys.databases \n" +
"WHERE name = '${dbName}'";
private static final String ENABLE_DB_CDC_SQL =
"EXEC sys.sp_cdc_enable_db";
private static final String TABLE_CDC_ENABLED_QUERY_SQL = private static final String ENABLE_DB_SUPPLEMENTAL_LOGGING_SQL = "ALTER DATABASE ADD SUPPLEMENTAL LOG DATA";
"SELECT is_tracked_by_cdc \n" +
"FROM sys.tables \n" +
"WHERE name = '${tableName}' AND schema_id = SCHEMA_ID('dbo')";
private static final String ENABLE_TABLE_CDC_SQL = private static final String QUERY_TABLE_SUPPLEMENTAL_LOGGING_SQL = "SELECT OWNER, TABLE_NAME FROM ALL_LOG_GROUPS WHERE LOG_GROUP_TYPE = 'ALL COLUMN LOGGING'";
"EXEC sys.sp_cdc_enable_table \n" +
"@source_schema = N'dbo', \n" +
"@source_name = N'${tableName}', \n" +
"@role_name = N'NULL', \n" + // TODO role_name
"@supports_net_changes = 0";
private static final String ENABLE_TABLE_SUPPLEMENTAL_LOGGING_SQL = "ALTER TABLE %s.%s ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS";
// 查询数据库是否开启了CDC /**
public static boolean isDBCdcEnabled(Connection conn, String dbName) { * 开启Oracle数据库supplemental logging
*
* @param conn
* @throws SQLException
*/
public static void enableDBSupplementalLogging(Connection conn) throws SQLException {
if (!isDBSupplementalLoggingEnabled(conn)) {
try (Statement stmt = conn.createStatement()) { try (Statement stmt = conn.createStatement()) {
DBUtils.switchDB(stmt, dbName); stmt.executeUpdate(ENABLE_DB_SUPPLEMENTAL_LOGGING_SQL);
String sql = DB_CDC_ENABLED_QUERY_SQL.replace("${dbName}", dbName);
try (ResultSet rs = stmt.executeQuery(sql)) {
if (rs.next()) {
return rs.getInt(1) == 1;
} }
} }
} catch (SQLException e) {
log.error(e.getMessage(), e);
}
throw new IllegalStateException("查询数据库[" + dbName + "]CDC开启状态失败");
} }
public static void enableDBCdcIfNeeded(Connection conn, String dbName) { public static boolean isDBSupplementalLoggingEnabled(Connection conn) throws SQLException {
if (!isDBCdcEnabled(conn, dbName)) { try (Statement stmt = conn.createStatement()) {
enableDBCdc(conn, dbName); try (ResultSet rs = stmt.executeQuery(QUERY_DB_SUPPLEMENTAL_LOGGING_SQL)) {
if (rs.next()) {
String result = rs.getString(1);
return "YES".equals(result) || "IMPLICIT".equals(result);
} else {
throw new SQLException("无法查询数据库的supplemental logging");
} }
} }
public static void enableDBCdc(Connection conn, String dbName) {
try (Statement stmt = conn.createStatement()) {
DBUtils.switchDB(stmt, dbName);
stmt.execute(ENABLE_DB_CDC_SQL);
} catch (SQLException e) {
log.error(e.getMessage(), e);
throw new IllegalStateException("开启数据库[" + dbName + "]CDC失败", e);
} }
} }
// 查询表是否开启了CDC /**
public static boolean isTableCdcEnabled(Connection conn, String dbName, String tableName) { * 开启Oracle schema下指定表的supplemental logging
* @param conn
* @param schema
* @param tables
* @throws SQLException
*/
public static void enableSchemaTablesSupplementalLogging(Connection conn, String schema, Set<String> tables) throws SQLException {
Set<String> needEnableTables = new HashSet<>(tables);
Set<String> enabledTables = getSchemaSupplementalLoggingEnabledTables(conn, schema);
needEnableTables.removeAll(enabledTables);
if (!needEnableTables.isEmpty()) {
try (Statement stmt = conn.createStatement()) { try (Statement stmt = conn.createStatement()) {
DBUtils.switchDB(stmt, dbName); for (String table : needEnableTables) {
String sql = TABLE_CDC_ENABLED_QUERY_SQL.replace("${tableName}", tableName); stmt.executeUpdate(String.format(ENABLE_TABLE_SUPPLEMENTAL_LOGGING_SQL, schema, table));
try (ResultSet rs = stmt.executeQuery(sql)) {
if (rs.next()) {
return rs.getInt(1) == 1;
} }
} }
} catch (SQLException e) {
log.error(e.getMessage(), e);
} }
throw new IllegalStateException("查询表[" + dbName + "." + tableName + "]CDC开启状态失败");
} }
public static void enableTableCdc(Connection conn, String dbName, String tableName) { private static Set<String> getSchemaSupplementalLoggingEnabledTables(Connection conn, String schema) throws SQLException {
try (Statement stmt = conn.createStatement()) { try (Statement stmt = conn.createStatement()) {
DBUtils.switchDB(stmt, dbName); try (ResultSet rs = stmt.executeQuery(QUERY_TABLE_SUPPLEMENTAL_LOGGING_SQL)) {
String sql = ENABLE_TABLE_CDC_SQL.replace("${tableName}", tableName); Set<String> tables = new HashSet<>();
stmt.execute(sql); while (rs.next()) {
} catch (SQLException e) { if (schema.equalsIgnoreCase(rs.getString(1))) {
log.error(e.getMessage(), e); tables.add(rs.getString(2));
throw new IllegalStateException("开启表[" + dbName + "." + tableName + "]CDC失败");
} }
} }
return tables;
public static void enableTableCdcIfNeeded(Connection conn, String dbName, String tableName) { }
if (!isTableCdcEnabled(conn, dbName, tableName)) {
enableTableCdc(conn, dbName, tableName);
} }
} }
} }
\ No newline at end of file
...@@ -14,13 +14,13 @@ import java.util.Map; ...@@ -14,13 +14,13 @@ import java.util.Map;
@Slf4j @Slf4j
public class DBUtils { public class DBUtils {
public static final String SQL_SERVER_DRIVER_CLASS_NAME = "com.microsoft.sqlserver.jdbc.SQLServerDriver"; public static final String ORACLE_DRIVER_CLASS_NAME = "oracle.jdbc.driver.OracleDriver";
static { static {
try { try {
Class.forName(SQL_SERVER_DRIVER_CLASS_NAME); Class.forName(ORACLE_DRIVER_CLASS_NAME);
} catch (ClassNotFoundException e) { } catch (ClassNotFoundException e) {
throw new IllegalStateException("未找到sql server驱动", e); throw new IllegalStateException("未找到Oracle jdbc驱动", e);
} }
} }
...@@ -54,9 +54,8 @@ public class DBUtils { ...@@ -54,9 +54,8 @@ public class DBUtils {
} }
} }
public static String getSqlServerUrl(String host, String port) { public static String buildOracleJdbcUrl(String host, String port, String dbname) {
return SQL_SERVER_URL_PATTERN.replace("<host>", host) return String.format("jdbc:oracle:thin:@%s:%s:%s", host, port, dbname);
.replace("<port>", port);
} }
public static void switchDB(Statement stmt, String dbName) throws SQLException { public static void switchDB(Statement stmt, String dbName) throws SQLException {
...@@ -64,8 +63,8 @@ public class DBUtils { ...@@ -64,8 +63,8 @@ public class DBUtils {
stmt.execute(sql); stmt.execute(sql);
} }
public static Connection getConn(String host, String port, String user, String password) throws SQLException { public static Connection getConn(String host, String port,String dbname, String user, String password) throws SQLException {
return DriverManager.getConnection(getSqlServerUrl(host, port), user, password); return DriverManager.getConnection(buildOracleJdbcUrl(host, port, dbname), user, password);
} }
public static void closeResource(Connection conn) { public static void closeResource(Connection conn) {
......
...@@ -4,7 +4,10 @@ import com.tbyd.data.datasync.config.DataSyncProperties; ...@@ -4,7 +4,10 @@ import com.tbyd.data.datasync.config.DataSyncProperties;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import java.sql.Connection; import java.sql.Connection;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Properties; import java.util.Properties;
import java.util.stream.Collectors;
@Slf4j @Slf4j
public class DataSyncInitializer { public class DataSyncInitializer {
...@@ -19,22 +22,27 @@ public class DataSyncInitializer { ...@@ -19,22 +22,27 @@ public class DataSyncInitializer {
String sourceDbname = DataSyncProperties.getSourceDbname(props); String sourceDbname = DataSyncProperties.getSourceDbname(props);
String[] syncTableNames = DataSyncProperties.getSyncTableNamesArray(props); String[] syncTableNames = DataSyncProperties.getSyncTableNamesArray(props);
try { try {
Connection conn = DBUtils.getConn(sourceHost, sourcePort, sourceUser, sourcePassword); Connection conn = DBUtils.getConn(sourceHost, sourcePort, sourceDbname, sourceUser, sourcePassword);
CdcHelper.enableDBCdcIfNeeded(conn, sourceDbname); CdcHelper.enableDBSupplementalLogging(conn);
DBUtils.closeResource(conn); DBUtils.closeResource(conn);
for (String tableName : syncTableNames) {
conn = DBUtils.getConn(sourceHost, sourcePort, sourceUser, sourcePassword); conn = DBUtils.getConn(sourceHost, sourcePort, sourceDbname, sourceUser, sourcePassword);
CdcHelper.enableTableCdcIfNeeded(conn, sourceDbname, stripSchemaId(tableName)); CdcHelper.enableSchemaTablesSupplementalLogging(conn, sourceUser,
new HashSet<>(Arrays.asList(syncTableNames))
.stream()
.map(DataSyncInitializer::stripSchema)
.collect(Collectors.toSet()));
DBUtils.closeResource(conn); DBUtils.closeResource(conn);
}
} catch (Exception e) { } catch (Exception e) {
log.error(e.getMessage(), e); log.error(e.getMessage(), e);
throw new IllegalStateException("初始化失败", e); throw new IllegalStateException("初始化失败", e);
} }
} }
private static String stripSchemaId(String tableName) { public static String stripSchema(String fullName) {
String schemaIdPrefix = DBO_SCHEMA_ID + "."; return fullName.substring(fullName.indexOf(".") + 1);
return tableName.substring(schemaIdPrefix.length());
} }
} }
...@@ -5,6 +5,7 @@ import com.alibaba.druid.pool.DruidPooledConnection; ...@@ -5,6 +5,7 @@ import com.alibaba.druid.pool.DruidPooledConnection;
import com.tbyd.data.datasync.config.DataSyncProperties; import com.tbyd.data.datasync.config.DataSyncProperties;
import com.tbyd.data.datasync.config.TableAndKeys; import com.tbyd.data.datasync.config.TableAndKeys;
import io.debezium.data.Envelope; import io.debezium.data.Envelope;
import io.debezium.data.VariableScaleDecimal;
import io.debezium.engine.ChangeEvent; import io.debezium.engine.ChangeEvent;
import io.debezium.engine.DebeziumEngine; import io.debezium.engine.DebeziumEngine;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
...@@ -13,10 +14,15 @@ import org.apache.kafka.connect.data.Schema; ...@@ -13,10 +14,15 @@ import org.apache.kafka.connect.data.Schema;
import org.apache.kafka.connect.data.Struct; import org.apache.kafka.connect.data.Struct;
import org.apache.kafka.connect.source.SourceRecord; import org.apache.kafka.connect.source.SourceRecord;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.sql.Connection; import java.sql.Connection;
import java.sql.PreparedStatement; import java.sql.PreparedStatement;
import java.sql.SQLException; import java.sql.SQLException;
import java.sql.Timestamp; import java.sql.Timestamp;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
...@@ -36,7 +42,7 @@ public class DataSyncWriter implements RecordHandler { ...@@ -36,7 +42,7 @@ public class DataSyncWriter implements RecordHandler {
String user = DataSyncProperties.getDestinationUser(props); String user = DataSyncProperties.getDestinationUser(props);
String password = DataSyncProperties.getDestinationPassword(props); String password = DataSyncProperties.getDestinationPassword(props);
String dbname = DataSyncProperties.getDestinationDbname(props); String dbname = DataSyncProperties.getDestinationDbname(props);
String url = buildUrl(host, port, dbname); String url = buildOracleJdbcUrl(host, port, dbname);
dataSource = new DruidDataSource(); dataSource = new DruidDataSource();
dataSource.setUrl(url); dataSource.setUrl(url);
dataSource.setUsername(user); dataSource.setUsername(user);
...@@ -58,10 +64,8 @@ public class DataSyncWriter implements RecordHandler { ...@@ -58,10 +64,8 @@ public class DataSyncWriter implements RecordHandler {
} }
} }
private static String buildUrl(String host, String port, String dbName) { public static String buildOracleJdbcUrl(String host, String port, String dbname) {
return JDBC_PATTERN.replace("<server_name>", host) return String.format("jdbc:oracle:thin:@%s:%s:%s", host, port, dbname);
.replace("<port>", port)
.replace("<database_name>", dbName);
} }
@Override @Override
...@@ -72,7 +76,7 @@ public class DataSyncWriter implements RecordHandler { ...@@ -72,7 +76,7 @@ public class DataSyncWriter implements RecordHandler {
return false; return false;
} }
Schema valueSchema = r.valueSchema(); Schema valueSchema = r.valueSchema();
if (valueSchema.name().equals("io.debezium.connector.sqlserver.SchemaChangeValue")) { if (valueSchema.name().equals("io.debezium.connector.oracle.SchemaChangeValue")) {
return false; return false;
} }
return true; return true;
...@@ -272,25 +276,22 @@ public class DataSyncWriter implements RecordHandler { ...@@ -272,25 +276,22 @@ public class DataSyncWriter implements RecordHandler {
if (value == null) { if (value == null) {
return null; return null;
} }
if (fieldSchema.name() == null) {
if (fieldSchema.type() == Schema.Type.STRING) { if (fieldSchema.type() == Schema.Type.STRUCT) {
return value; Struct structValue = (Struct) value;
if (fieldSchema.name().equals(VariableScaleDecimal.class.getName())) {
value = new BigDecimal(new BigInteger(structValue.getBytes(VariableScaleDecimal.VALUE_FIELD)),
structValue.getInt32(VariableScaleDecimal.SCALE_FIELD));
} }
if (fieldSchema.type() == Schema.Type.INT32) {
return value;
} }
if (fieldSchema.type() == Schema.Type.BYTES) { if (fieldSchema.type() == Schema.Type.INT64) {
return value; Long longValue = (Long) value;
if (fieldSchema.name().equals(io.debezium.time.Timestamp.class.getName())) {
value = LocalDateTime.ofInstant(Instant.ofEpochMilli(longValue), ZoneOffset.UTC);
} }
} }
switch (fieldSchema.name()) {
case "org.apache.kafka.connect.data.Decimal":
return value; return value;
case "io.debezium.time.Timestamp":
return new Timestamp((Long) value);
default:
throw new IllegalStateException("暂不支持这种数据类型的字段:" + fieldSchema.name());
}
} }
private static String buildInsertSql(TableMetadata tableMetadata) { private static String buildInsertSql(TableMetadata tableMetadata) {
......
name=engine name=engine
connector.class=io.debezium.connector.sqlserver.SqlServerConnector connector.class=io.debezium.connector.oracle.OracleConnector
database.history=io.debezium.relational.history.FileDatabaseHistory database.history=io.debezium.relational.history.FileDatabaseHistory
snapshot.mode=schema_only snapshot.mode=schema_only
database.server.name=server1 database.server.name=server1
offset.storage.file.filename=offset.dat offset.storage.file.filename=offset.dat
database.history.file.filename=dbhistory.dat database.history.file.filename=dbhistory.dat
log.mining.strategy=online_catalog
database.history.skip.unparseable.ddl=true
database.history.store.only.captured.tables.ddl=true
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment