From 137923c74002502c0e61d1f65bf813ea70c29848 Mon Sep 17 00:00:00 2001 From: vran Date: Thu, 21 Apr 2022 10:11:07 +0800 Subject: [PATCH] fix: cursor over limit when sync (#114) * fix: cursor over limit when sync * fix: code checkstyle --- .../databasir/api/DatabaseTypeController.java | 2 +- .../main/java/com/databasir/api/Routes.java | 2 + .../impl/jdbc/JdbcColumnMetaRepository.java | 86 ++++++++++--------- .../jdbc/JdbcForeignKeyMetaRepository.java | 6 ++ .../impl/jdbc/JdbcIndexMetaRepository.java | 36 ++++---- .../impl/jdbc/JdbcTableMetaRepository.java | 53 ++++++------ 6 files changed, 103 insertions(+), 82 deletions(-) diff --git a/api/src/main/java/com/databasir/api/DatabaseTypeController.java b/api/src/main/java/com/databasir/api/DatabaseTypeController.java index c6f217a..a68b5b4 100644 --- a/api/src/main/java/com/databasir/api/DatabaseTypeController.java +++ b/api/src/main/java/com/databasir/api/DatabaseTypeController.java @@ -35,7 +35,7 @@ public class DatabaseTypeController { @GetMapping(Routes.DatabaseType.LIST_PAGE) public JsonData> listPage(@PageableDefault(sort = "id", direction = DESC) - Pageable page, + Pageable page, DatabaseTypePageCondition condition) { Page data = databaseTypeService.findByPage(page, condition); return JsonData.ok(data); diff --git a/api/src/main/java/com/databasir/api/Routes.java b/api/src/main/java/com/databasir/api/Routes.java index 52472c9..1d380f3 100644 --- a/api/src/main/java/com/databasir/api/Routes.java +++ b/api/src/main/java/com/databasir/api/Routes.java @@ -169,6 +169,8 @@ public interface Routes { String CREATE = BASE + "/database_types"; String RESOLVE_DRIVER_CLASS_NAME = BASE + "/database_types/driver_class_name"; + + String UPLOAD_DRIVER = BASE + "/database_types/upload_driver"; } interface MockData { diff --git a/plugin/src/main/java/com/databasir/core/meta/repository/impl/jdbc/JdbcColumnMetaRepository.java b/plugin/src/main/java/com/databasir/core/meta/repository/impl/jdbc/JdbcColumnMetaRepository.java index 88f20c8..090709a 100644 --- a/plugin/src/main/java/com/databasir/core/meta/repository/impl/jdbc/JdbcColumnMetaRepository.java +++ b/plugin/src/main/java/com/databasir/core/meta/repository/impl/jdbc/JdbcColumnMetaRepository.java @@ -36,52 +36,56 @@ public class JdbcColumnMetaRepository implements ColumnMetaRepository { columnsResult = connection.getMetaData() .getColumns(databaseName, tableCondition.getSchemaName(), tableName, null); } catch (SQLException e) { - log.warn("warn: ignore columns in " + databaseName + "." + tableName); + log.warn("warn: ignore columns in " + databaseName + "." + tableName + ", error: " + e.getMessage()); return columnDocs; } - while (columnsResult.next()) { - String columnName = columnsResult.getString("COLUMN_NAME"); - if (tableCondition.columnIsIgnored(columnName)) { - if (log.isWarnEnabled()) { - log.warn("ignore column: " + columnName); - } - } else { - String defaultValue = columnsResult.getString("COLUMN_DEF"); - String isNullable = columnsResult.getString("IS_NULLABLE"); - if (isNullable.trim().equals("")) { - isNullable = "UNKNOWN"; - } - String isAutoIncrement = columnsResult.getString("IS_AUTOINCREMENT"); - if (isAutoIncrement.trim().equals("")) { - isAutoIncrement = "UNKNOWN"; - } - if (defaultValue != null && defaultValue.trim().equals("")) { - defaultValue = "'" + defaultValue + "'"; - } - Integer decimalDigits; - if (columnsResult.getObject("DECIMAL_DIGITS") == null) { - decimalDigits = null; + try { + while (columnsResult.next()) { + String columnName = columnsResult.getString("COLUMN_NAME"); + if (tableCondition.columnIsIgnored(columnName)) { + if (log.isWarnEnabled()) { + log.warn("ignore column: " + columnName); + } } else { - decimalDigits = columnsResult.getInt("DECIMAL_DIGITS"); + String defaultValue = columnsResult.getString("COLUMN_DEF"); + String isNullable = columnsResult.getString("IS_NULLABLE"); + if (isNullable.trim().equals("")) { + isNullable = "UNKNOWN"; + } + String isAutoIncrement = columnsResult.getString("IS_AUTOINCREMENT"); + if (isAutoIncrement.trim().equals("")) { + isAutoIncrement = "UNKNOWN"; + } + if (defaultValue != null && defaultValue.trim().equals("")) { + defaultValue = "'" + defaultValue + "'"; + } + Integer decimalDigits; + if (columnsResult.getObject("DECIMAL_DIGITS") == null) { + decimalDigits = null; + } else { + decimalDigits = columnsResult.getInt("DECIMAL_DIGITS"); + } + Integer columnSize = columnsResult.getInt("COLUMN_SIZE"); + String columnType = columnsResult.getString("TYPE_NAME"); + String columnComment = columnsResult.getString("REMARKS"); + int dataType = columnsResult.getInt("DATA_TYPE"); + ColumnMeta columnMeta = ColumnMeta.builder() + .name(columnName) + .dataType(dataType) + .type(columnType) + .size(columnSize) + .decimalDigits(decimalDigits) + .nullable(isNullable) + .autoIncrement(isAutoIncrement) + .comment(columnComment) + .defaultValue(defaultValue) + .isPrimaryKey(primaryKeyColumns.contains(columnName)) + .build(); + columnDocs.add(columnMeta); } - Integer columnSize = columnsResult.getInt("COLUMN_SIZE"); - String columnType = columnsResult.getString("TYPE_NAME"); - String columnComment = columnsResult.getString("REMARKS"); - int dataType = columnsResult.getInt("DATA_TYPE"); - ColumnMeta columnMeta = ColumnMeta.builder() - .name(columnName) - .dataType(dataType) - .type(columnType) - .size(columnSize) - .decimalDigits(decimalDigits) - .nullable(isNullable) - .autoIncrement(isAutoIncrement) - .comment(columnComment) - .defaultValue(defaultValue) - .isPrimaryKey(primaryKeyColumns.contains(columnName)) - .build(); - columnDocs.add(columnMeta); } + } finally { + columnsResult.close(); } return columnDocs; } diff --git a/plugin/src/main/java/com/databasir/core/meta/repository/impl/jdbc/JdbcForeignKeyMetaRepository.java b/plugin/src/main/java/com/databasir/core/meta/repository/impl/jdbc/JdbcForeignKeyMetaRepository.java index 1d0b248..0fef5a8 100644 --- a/plugin/src/main/java/com/databasir/core/meta/repository/impl/jdbc/JdbcForeignKeyMetaRepository.java +++ b/plugin/src/main/java/com/databasir/core/meta/repository/impl/jdbc/JdbcForeignKeyMetaRepository.java @@ -50,6 +50,12 @@ public class JdbcForeignKeyMetaRepository implements ForeignKeyMetaRepository { } } catch (SQLException e) { log.warn("warn: ignore foreign keys in " + databaseName + "." + tableName + ", " + e.getMessage()); + } finally { + try { + keyResult.close(); + } catch (SQLException e) { + log.warn("warn: close key result error " + databaseName + "." + tableName + ", " + e.getMessage()); + } } return foreignKeys; } diff --git a/plugin/src/main/java/com/databasir/core/meta/repository/impl/jdbc/JdbcIndexMetaRepository.java b/plugin/src/main/java/com/databasir/core/meta/repository/impl/jdbc/JdbcIndexMetaRepository.java index f44f3c2..3d91e64 100644 --- a/plugin/src/main/java/com/databasir/core/meta/repository/impl/jdbc/JdbcIndexMetaRepository.java +++ b/plugin/src/main/java/com/databasir/core/meta/repository/impl/jdbc/JdbcIndexMetaRepository.java @@ -31,27 +31,31 @@ public class JdbcIndexMetaRepository implements IndexMetaRepository { indexResults = connection.getMetaData() .getIndexInfo(databaseName, condition.getSchemaName(), tableName, false, false); } catch (SQLException e) { - log.warn("warn: ignore " + databaseName + "." + tableName); + log.warn("warn: ignore " + databaseName + "." + tableName + ", error=" + e.getMessage()); return indexMetas; } Map pojoGroupByName = new HashMap<>(); - while (indexResults.next()) { - Boolean nonUnique = indexResults.getBoolean("NON_UNIQUE"); - String indexName = indexResults.getString("INDEX_NAME"); - String columnName = indexResults.getString("COLUMN_NAME"); - if (pojoGroupByName.containsKey(indexName)) { - pojoGroupByName.get(indexName).getColumnNames().add(columnName); - } else { - List columns = new ArrayList<>(); - columns.add(columnName); - IndexMeta indexMeta = IndexMeta.builder() - .name(indexName) - .columnNames(columns) - .isUniqueKey(Objects.equals(nonUnique, false)) - .build(); - pojoGroupByName.put(indexName, indexMeta); + try { + while (indexResults.next()) { + Boolean nonUnique = indexResults.getBoolean("NON_UNIQUE"); + String indexName = indexResults.getString("INDEX_NAME"); + String columnName = indexResults.getString("COLUMN_NAME"); + if (pojoGroupByName.containsKey(indexName)) { + pojoGroupByName.get(indexName).getColumnNames().add(columnName); + } else { + List columns = new ArrayList<>(); + columns.add(columnName); + IndexMeta indexMeta = IndexMeta.builder() + .name(indexName) + .columnNames(columns) + .isUniqueKey(Objects.equals(nonUnique, false)) + .build(); + pojoGroupByName.put(indexName, indexMeta); + } } + } finally { + indexResults.close(); } return new ArrayList<>(pojoGroupByName.values()); } diff --git a/plugin/src/main/java/com/databasir/core/meta/repository/impl/jdbc/JdbcTableMetaRepository.java b/plugin/src/main/java/com/databasir/core/meta/repository/impl/jdbc/JdbcTableMetaRepository.java index 6b44af2..1d329bf 100644 --- a/plugin/src/main/java/com/databasir/core/meta/repository/impl/jdbc/JdbcTableMetaRepository.java +++ b/plugin/src/main/java/com/databasir/core/meta/repository/impl/jdbc/JdbcTableMetaRepository.java @@ -40,34 +40,39 @@ public class JdbcTableMetaRepository implements TableMetaRepository { String databaseName = condition.getDatabaseName(); ResultSet tablesResult = connection.getMetaData() .getTables(databaseName, condition.getSchemaName(), null, new String[]{"TABLE"}); - while (tablesResult.next()) { - String tableName = tablesResult.getString("TABLE_NAME"); - if (condition.tableIsIgnored(tableName)) { - if (log.isWarnEnabled()) { - log.warn("ignored table: " + databaseName + "." + tableName); - } - } else { - String tableType = tablesResult.getString("TABLE_TYPE"); - String tableComment = tablesResult.getString("REMARKS"); - TableCondition tableCondition = TableCondition.of(condition, tableName); - List columns = columnMetaRepository.selectColumns(connection, tableCondition); - if (columns.isEmpty()) { + try { + while (tablesResult.next()) { + String tableName = tablesResult.getString("TABLE_NAME"); + if (condition.tableIsIgnored(tableName)) { if (log.isWarnEnabled()) { - log.warn("ignored table: " + databaseName + "." + tableName + ", caused by get empty columns"); + log.warn("ignored table: " + databaseName + "." + tableName); + } + } else { + String tableType = tablesResult.getString("TABLE_TYPE"); + String tableComment = tablesResult.getString("REMARKS"); + TableCondition tableCondition = TableCondition.of(condition, tableName); + List columns = columnMetaRepository.selectColumns(connection, tableCondition); + if (columns.isEmpty()) { + if (log.isWarnEnabled()) { + log.warn("ignored table: " + databaseName + "." + tableName + + ", caused by get empty columns"); + } + continue; } - continue; + TableMeta tableMeta = TableMeta.builder() + .name(tableName) + .type(tableType) + .comment(tableComment) + .columns(columns) + .foreignKeys(foreignKeyMetaRepository.selectForeignKeys(connection, tableCondition)) + .indexes(indexMetaRepository.selectIndexes(connection, tableCondition)) + .triggers(triggerMetaRepository.selectTriggers(connection, tableCondition)) + .build(); + tableMetas.add(tableMeta); } - TableMeta tableMeta = TableMeta.builder() - .name(tableName) - .type(tableType) - .comment(tableComment) - .columns(columns) - .foreignKeys(foreignKeyMetaRepository.selectForeignKeys(connection, tableCondition)) - .indexes(indexMetaRepository.selectIndexes(connection, tableCondition)) - .triggers(triggerMetaRepository.selectTriggers(connection, tableCondition)) - .build(); - tableMetas.add(tableMeta); } + } finally { + tablesResult.close(); } return tableMetas; }