Skip to content

Commit a5d009f

Browse files
szetszwoszetszwo
committed
HDDS-12833. Do not use CodecRegistry in OmMetadataManagerImpl.
1 parent 621f018 commit a5d009f

File tree

14 files changed

+202
-308
lines changed

14 files changed

+202
-308
lines changed

‎hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HAUtils.java

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,6 @@
6363
import org.apache.hadoop.hdds.utils.db.DBDefinition;
6464
import org.apache.hadoop.hdds.utils.db.DBStore;
6565
import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
66-
import org.apache.hadoop.hdds.utils.db.RocksDBConfiguration;
6766
import org.apache.hadoop.hdds.utils.db.Table;
6867
import org.apache.hadoop.io.retry.RetryPolicies;
6968
import org.apache.hadoop.io.retry.RetryPolicy;
@@ -309,10 +308,8 @@ public static boolean verifyTransactionInfo(TransactionInfo transactionInfo,
309308

310309
public static DBStore loadDB(OzoneConfiguration configuration, File metaDir,
311310
String dbName, DBDefinition definition) throws IOException {
312-
RocksDBConfiguration rocksDBConfiguration =
313-
configuration.getObject(RocksDBConfiguration.class);
314311
DBStoreBuilder dbStoreBuilder =
315-
DBStoreBuilder.newBuilder(configuration, rocksDBConfiguration)
312+
DBStoreBuilder.newBuilder(configuration)
316313
.setName(dbName)
317314
.setPath(Paths.get(metaDir.getPath()));
318315
// Add column family names and codecs.

‎hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBColumnFamilyDefinition.java

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,16 @@
1717

1818
package org.apache.hadoop.hdds.utils.db;
1919

20+
import static org.apache.ratis.util.JavaUtils.getClassSimpleName;
21+
2022
import java.io.IOException;
2123
import java.util.Arrays;
2224
import java.util.Collections;
2325
import java.util.List;
2426
import java.util.Map;
27+
import java.util.Objects;
2528
import org.apache.hadoop.hdds.utils.CollectionUtils;
29+
import org.apache.hadoop.hdds.utils.db.cache.TableCache.CacheType;
2630
import org.apache.hadoop.hdds.utils.db.managed.ManagedColumnFamilyOptions;
2731

2832
/**
@@ -51,22 +55,27 @@ public class DBColumnFamilyDefinition<KEY, VALUE> {
5155
}
5256

5357
private final String tableName;
54-
5558
private final Codec<KEY> keyCodec;
56-
5759
private final Codec<VALUE> valueCodec;
60+
private final String name;
5861

5962
private volatile ManagedColumnFamilyOptions cfOptions;
6063

6164
public DBColumnFamilyDefinition(String tableName, Codec<KEY> keyCodec, Codec<VALUE> valueCodec) {
62-
this.tableName = tableName;
63-
this.keyCodec = keyCodec;
64-
this.valueCodec = valueCodec;
65+
this.tableName = Objects.requireNonNull(tableName, "tableName == null");
66+
this.keyCodec = Objects.requireNonNull(keyCodec, "keyCodec == null");
67+
this.valueCodec = Objects.requireNonNull(valueCodec, "valueCodec == null");
68+
this.name = tableName + "-def: " + getClassSimpleName(getKeyType())
69+
+ " -> " + getClassSimpleName(getValueType());
6570
this.cfOptions = null;
6671
}
6772

68-
public Table<KEY, VALUE> getTable(DBStore db) throws IOException {
69-
return db.getTable(tableName, getKeyType(), getValueType());
73+
public TypedTable<KEY, VALUE> getTable(DBStore db) throws IOException {
74+
return db.getTable(tableName, keyCodec, valueCodec);
75+
}
76+
77+
public TypedTable<KEY, VALUE> getTable(DBStore db, CacheType cacheType) throws IOException {
78+
return db.getTable(tableName, keyCodec, valueCodec, cacheType);
7079
}
7180

7281
public String getName() {
@@ -96,4 +105,9 @@ public ManagedColumnFamilyOptions getCfOptions() {
96105
public void setCfOptions(ManagedColumnFamilyOptions cfOptions) {
97106
this.cfOptions = cfOptions;
98107
}
108+
109+
@Override
110+
public String toString() {
111+
return name;
112+
}
99113
}

‎hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStore.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import java.util.Map;
2525
import org.apache.hadoop.hdds.annotation.InterfaceStability;
2626
import org.apache.hadoop.hdds.utils.db.cache.TableCache;
27+
import org.apache.hadoop.hdds.utils.db.cache.TableCache.CacheType;
2728
import org.apache.ozone.rocksdiff.RocksDBCheckpointDiffer;
2829

2930
/**
@@ -73,6 +74,12 @@ <KEY, VALUE> Table<KEY, VALUE> getTable(String name,
7374
Class<KEY> keyType, Class<VALUE> valueType,
7475
TableCache.CacheType cacheType) throws IOException;
7576

77+
/** The same as getTable(name, keyCodec, valueCodec, CacheType.PARTIAL_CACHE). */
78+
default <KEY, VALUE> TypedTable<KEY, VALUE> getTable(String name, Codec<KEY> keyCodec, Codec<VALUE> valueCodec)
79+
throws IOException {
80+
return getTable(name, keyCodec, valueCodec, CacheType.PARTIAL_CACHE);
81+
}
82+
7683
/**
7784
* Gets table store with implict key/value conversion.
7885
*

‎hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/DBStoreBuilder.java

Lines changed: 11 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -117,23 +117,18 @@ public static DBStore createDBStore(ConfigurationSource configuration,
117117

118118
public static DBStoreBuilder newBuilder(ConfigurationSource configuration,
119119
DBDefinition definition) {
120+
return newBuilder(configuration, definition, null);
121+
}
120122

121-
DBStoreBuilder builder = newBuilder(configuration);
122-
builder.applyDBDefinition(definition);
123-
124-
return builder;
123+
public static DBStoreBuilder newBuilder(ConfigurationSource conf, DBDefinition definition, File metadataDir) {
124+
return newBuilder(conf).applyDBDefinition(definition, metadataDir);
125125
}
126126

127127
public static DBStoreBuilder newBuilder(ConfigurationSource configuration) {
128-
return newBuilder(configuration,
128+
return new DBStoreBuilder(configuration,
129129
configuration.getObject(RocksDBConfiguration.class));
130130
}
131131

132-
public static DBStoreBuilder newBuilder(ConfigurationSource configuration,
133-
RocksDBConfiguration rocksDBConfiguration) {
134-
return new DBStoreBuilder(configuration, rocksDBConfiguration);
135-
}
136-
137132
private DBStoreBuilder(ConfigurationSource configuration,
138133
RocksDBConfiguration rocksDBConfiguration) {
139134
cfOptions = new HashMap<>();
@@ -173,21 +168,20 @@ public static File getDBDirPath(DBDefinition definition,
173168
return metadataDir;
174169
}
175170

176-
private void applyDBDefinition(DBDefinition definition) {
171+
private DBStoreBuilder applyDBDefinition(DBDefinition definition, File metadataDir) {
177172
// Set metadata dirs.
178-
File metadataDir = getDBDirPath(definition, configuration);
173+
if (metadataDir == null) {
174+
metadataDir = getDBDirPath(definition, configuration);
175+
}
179176

180177
setName(definition.getName());
181178
setPath(Paths.get(metadataDir.getPath()));
182179

183180
// Add column family names and codecs.
184-
for (DBColumnFamilyDefinition columnFamily :
185-
definition.getColumnFamilies()) {
186-
181+
for (DBColumnFamilyDefinition<?, ?> columnFamily : definition.getColumnFamilies()) {
187182
addTable(columnFamily.getName(), columnFamily.getCfOptions());
188-
addCodec(columnFamily.getKeyType(), columnFamily.getKeyCodec());
189-
addCodec(columnFamily.getValueType(), columnFamily.getValueCodec());
190183
}
184+
return this;
191185
}
192186

193187
private void setDBOptionsProps(ManagedDBOptions dbOptions) {

‎hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/RDBStore.java

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_CHECKPOINT_DIR;
2626
import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_DIFF_DIR;
2727
import static org.apache.hadoop.ozone.OzoneConsts.SNAPSHOT_INFO_TABLE;
28+
import static org.apache.ratis.util.JavaUtils.getClassSimpleName;
2829

2930
import com.google.common.base.Preconditions;
3031
import java.io.File;
@@ -59,6 +60,7 @@
5960
public class RDBStore implements DBStore {
6061
private static final Logger LOG =
6162
LoggerFactory.getLogger(RDBStore.class);
63+
private final String dbName;
6264
private final RocksDatabase db;
6365
private final File dbLocation;
6466
private final CodecRegistry codecRegistry;
@@ -89,6 +91,7 @@ public RDBStore(File dbFile, ManagedDBOptions dbOptions, ManagedStatistics stati
8991
Preconditions.checkNotNull(dbFile, "DB file location cannot be null");
9092
Preconditions.checkNotNull(families);
9193
Preconditions.checkArgument(!families.isEmpty());
94+
this.dbName = getClassSimpleName(getClass()) + "[" + dbFile + "]";
9295
this.maxDbUpdatesSizeThreshold = maxDbUpdatesSizeThreshold;
9396
codecRegistry = registry;
9497
dbLocation = dbFile;
@@ -290,10 +293,10 @@ public void commitBatchOperation(BatchOperation operation)
290293
}
291294

292295
@Override
293-
public RDBTable getTable(String name) throws IOException {
296+
public RDBTable getTable(String name) throws RocksDatabaseException {
294297
final ColumnFamily handle = db.getColumnFamily(name);
295298
if (handle == null) {
296-
throw new IOException("No such table in this DB. TableName : " + name);
299+
throw new RocksDatabaseException("Table " + name + " not found in " + this);
297300
}
298301
return new RDBTable(this.db, handle, rdbMetrics);
299302
}
@@ -308,7 +311,9 @@ public <K, V> TypedTable<K, V> getTable(String name,
308311
@Override
309312
public <K, V> TypedTable<K, V> getTable(
310313
String name, Codec<K> keyCodec, Codec<V> valueCodec, TableCache.CacheType cacheType) throws IOException {
311-
return new TypedTable<>(getTable(name), keyCodec, valueCodec, cacheType);
314+
final TypedTable<K, V> table = new TypedTable<>(getTable(name), keyCodec, valueCodec, cacheType);
315+
LOG.info("Created {} from {}", table, this);
316+
return table;
312317
}
313318

314319
@Override
@@ -491,4 +496,9 @@ public String getProperty(ColumnFamily family, String property)
491496
public RDBMetrics getMetrics() {
492497
return rdbMetrics;
493498
}
499+
500+
@Override
501+
public String toString() {
502+
return dbName;
503+
}
494504
}

‎hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/db/Table.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -238,7 +238,7 @@ default void cleanupCache(List<Long> epochs) {
238238
/**
239239
* Create the metrics datasource that emits table cache metrics.
240240
*/
241-
default TableCacheMetrics createCacheMetrics() throws IOException {
241+
default TableCacheMetrics createCacheMetrics() {
242242
throw new NotImplementedException("getCacheValue is not implemented");
243243
}
244244

‎hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestInterSCMGrpcProtocolService.java

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -53,9 +53,10 @@
5353
import org.apache.hadoop.hdds.security.x509.CertificateTestUtils;
5454
import org.apache.hadoop.hdds.security.x509.certificate.client.SCMCertificateClient;
5555
import org.apache.hadoop.hdds.utils.TransactionInfo;
56+
import org.apache.hadoop.hdds.utils.db.Codec;
5657
import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
5758
import org.apache.hadoop.hdds.utils.db.DBStore;
58-
import org.apache.hadoop.hdds.utils.db.Table;
59+
import org.apache.hadoop.hdds.utils.db.TypedTable;
5960
import org.apache.hadoop.ozone.OzoneConfigKeys;
6061
import org.apache.ozone.test.GenericTestUtils.PortAllocator;
6162
import org.junit.jupiter.api.Test;
@@ -172,7 +173,7 @@ private SCMMetadataStore metadataStore() throws IOException {
172173

173174
private DBStore dbStore() throws IOException {
174175
DBStore dbStoreMock = mock(DBStore.class);
175-
doReturn(trInfoTable()).when(dbStoreMock).getTable(any(), any(), any());
176+
doReturn(trInfoTable()).when(dbStoreMock).getTable(any(), (Codec<?>)any(), (Codec<?>)any());
176177
doReturn(checkPoint()).when(dbStoreMock).getCheckpoint(anyBoolean());
177178
return dbStoreMock;
178179
}
@@ -186,9 +187,9 @@ private DBCheckpoint checkPoint() throws IOException {
186187
return checkpoint;
187188
}
188189

189-
private Table<String, TransactionInfo> trInfoTable()
190+
private TypedTable<String, TransactionInfo> trInfoTable()
190191
throws IOException {
191-
Table<String, TransactionInfo> tableMock = mock(Table.class);
192+
final TypedTable<String, TransactionInfo> tableMock = mock(TypedTable.class);
192193
doReturn(mock(TransactionInfo.class)).when(tableMock).get(any());
193194
return tableMock;
194195
}

0 commit comments

Comments
 (0)