Package org.apache.flink.state.rocksdb

Class RocksDBOperationUtils

java.lang.Object

org.apache.flink.state.rocksdb.RocksDBOperationUtils

public class RocksDBOperationUtils
extends Object

Utils for RocksDB Operations.

Constructor Summary

Constructors

Constructor	Description
RocksDBOperationUtils()

Method Summary

Modifier and Type	Method	Description
static void	addColumnFamilyOptionsToCloseLater(List<org.rocksdb.ColumnFamilyOptions> columnFamilyOptions, org.rocksdb.ColumnFamilyHandle columnFamilyHandle)
static org.apache.flink.runtime.memory.OpaqueMemoryResource<org.apache.flink.state.rocksdb.RocksDBSharedResources>	allocateSharedCachesIfConfigured(RocksDBMemoryConfiguration jobMemoryConfig, org.apache.flink.runtime.execution.Environment env, double memoryFraction, org.slf4j.Logger logger, RocksDBMemoryControllerUtils.RocksDBMemoryFactory rocksDBMemoryFactory)
static org.rocksdb.ColumnFamilyHandle	createColumnFamily(org.rocksdb.ColumnFamilyDescriptor columnDescriptor, org.rocksdb.RocksDB db, List<org.rocksdb.ExportImportFilesMetaData> importFilesMetaData, org.apache.flink.core.fs.ICloseableRegistry cancelStreamRegistryForRestore)
static org.rocksdb.ColumnFamilyDescriptor	createColumnFamilyDescriptor(org.apache.flink.runtime.state.RegisteredStateMetaInfoBase metaInfoBase, Function<String,org.rocksdb.ColumnFamilyOptions> columnFamilyOptionsFactory, RocksDbTtlCompactFiltersManager ttlCompactFiltersManager, Long writeBufferManagerCapacity)	Creates a column descriptor for a state column family.
static org.rocksdb.ColumnFamilyOptions	createColumnFamilyOptions(Function<String,org.rocksdb.ColumnFamilyOptions> columnFamilyOptionsFactory, String stateName)
static RocksDBKeyedStateBackend.RocksDbKvStateInfo	createStateInfo(org.apache.flink.runtime.state.RegisteredStateMetaInfoBase metaInfoBase, org.rocksdb.RocksDB db, Function<String,org.rocksdb.ColumnFamilyOptions> columnFamilyOptionsFactory, RocksDbTtlCompactFiltersManager ttlCompactFiltersManager, Long writeBufferManagerCapacity, List<org.rocksdb.ExportImportFilesMetaData> importFilesMetaData, org.apache.flink.core.fs.ICloseableRegistry cancelStreamRegistryForRestore)	Creates a state info from a new meta info to use with a k/v state.
static RocksDBKeyedStateBackend.RocksDbKvStateInfo	createStateInfo(org.apache.flink.runtime.state.RegisteredStateMetaInfoBase metaInfoBase, org.rocksdb.RocksDB db, Function<String,org.rocksdb.ColumnFamilyOptions> columnFamilyOptionsFactory, RocksDbTtlCompactFiltersManager ttlCompactFiltersManager, Long writeBufferManagerCapacity, org.apache.flink.core.fs.ICloseableRegistry cancelStreamRegistryForRestore)	Create RocksDB-backed KV-state, including RocksDB ColumnFamily.
static RocksIteratorWrapper	getRocksIterator(org.rocksdb.RocksDB db, org.rocksdb.ColumnFamilyHandle columnFamilyHandle, org.rocksdb.ReadOptions readOptions)
static org.rocksdb.RocksDB	openDB(String path, List<org.rocksdb.ColumnFamilyDescriptor> stateColumnFamilyDescriptors, List<org.rocksdb.ColumnFamilyHandle> stateColumnFamilyHandles, org.rocksdb.ColumnFamilyOptions columnFamilyOptions, org.rocksdb.DBOptions dbOptions)
static void	registerKvStateInformation(Map<String,RocksDBKeyedStateBackend.RocksDbKvStateInfo> kvStateInformation, RocksDBNativeMetricMonitor nativeMetricMonitor, String columnFamilyName, RocksDBKeyedStateBackend.RocksDbKvStateInfo registeredColumn)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

RocksDBOperationUtils

public RocksDBOperationUtils()

Method Detail

openDB

public static org.rocksdb.RocksDB openDB(String path,
                                         List<org.rocksdb.ColumnFamilyDescriptor> stateColumnFamilyDescriptors,
                                         List<org.rocksdb.ColumnFamilyHandle> stateColumnFamilyHandles,
                                         org.rocksdb.ColumnFamilyOptions columnFamilyOptions,
                                         org.rocksdb.DBOptions dbOptions)
                                  throws IOException

Throws:

IOException

getRocksIterator

public static RocksIteratorWrapper getRocksIterator(org.rocksdb.RocksDB db,
                                                    org.rocksdb.ColumnFamilyHandle columnFamilyHandle,
                                                    org.rocksdb.ReadOptions readOptions)

registerKvStateInformation

public static void registerKvStateInformation(Map<String,RocksDBKeyedStateBackend.RocksDbKvStateInfo> kvStateInformation,
                                              RocksDBNativeMetricMonitor nativeMetricMonitor,
                                              String columnFamilyName,
                                              RocksDBKeyedStateBackend.RocksDbKvStateInfo registeredColumn)

createStateInfo

public static RocksDBKeyedStateBackend.RocksDbKvStateInfo createStateInfo(org.apache.flink.runtime.state.RegisteredStateMetaInfoBase metaInfoBase,
                                                                          org.rocksdb.RocksDB db,
                                                                          Function<String,org.rocksdb.ColumnFamilyOptions> columnFamilyOptionsFactory,
                                                                          @Nullable
                                                                          RocksDbTtlCompactFiltersManager ttlCompactFiltersManager,
                                                                          @Nullable
                                                                          Long writeBufferManagerCapacity,
                                                                          List<org.rocksdb.ExportImportFilesMetaData> importFilesMetaData,
                                                                          org.apache.flink.core.fs.ICloseableRegistry cancelStreamRegistryForRestore)

Creates a state info from a new meta info to use with a k/v state.

Creates the column family for the state. Sets TTL compaction filter if ttlCompactFiltersManager is not null.

Parameters:

importFilesMetaData - if not empty, we import the files specified in the metadata to the column family.

createStateInfo

public static RocksDBKeyedStateBackend.RocksDbKvStateInfo createStateInfo(org.apache.flink.runtime.state.RegisteredStateMetaInfoBase metaInfoBase,
                                                                          org.rocksdb.RocksDB db,
                                                                          Function<String,org.rocksdb.ColumnFamilyOptions> columnFamilyOptionsFactory,
                                                                          @Nullable
                                                                          RocksDbTtlCompactFiltersManager ttlCompactFiltersManager,
                                                                          @Nullable
                                                                          Long writeBufferManagerCapacity,
                                                                          org.apache.flink.core.fs.ICloseableRegistry cancelStreamRegistryForRestore)

Create RocksDB-backed KV-state, including RocksDB ColumnFamily.

Parameters:

cancelStreamRegistryForRestore - closing it interrupts KV state creation

createColumnFamilyDescriptor

public static org.rocksdb.ColumnFamilyDescriptor createColumnFamilyDescriptor(org.apache.flink.runtime.state.RegisteredStateMetaInfoBase metaInfoBase,
                                                                              Function<String,org.rocksdb.ColumnFamilyOptions> columnFamilyOptionsFactory,
                                                                              @Nullable
                                                                              RocksDbTtlCompactFiltersManager ttlCompactFiltersManager,
                                                                              @Nullable
                                                                              Long writeBufferManagerCapacity)

Creates a column descriptor for a state column family.

Sets TTL compaction filter if ttlCompactFiltersManager is not null.

createColumnFamilyOptions

public static org.rocksdb.ColumnFamilyOptions createColumnFamilyOptions(Function<String,org.rocksdb.ColumnFamilyOptions> columnFamilyOptionsFactory,
                                                                        String stateName)

createColumnFamily

public static org.rocksdb.ColumnFamilyHandle createColumnFamily(org.rocksdb.ColumnFamilyDescriptor columnDescriptor,
                                                                org.rocksdb.RocksDB db,
                                                                List<org.rocksdb.ExportImportFilesMetaData> importFilesMetaData,
                                                                org.apache.flink.core.fs.ICloseableRegistry cancelStreamRegistryForRestore)
                                                         throws org.rocksdb.RocksDBException,
                                                                InterruptedException

Throws:

org.rocksdb.RocksDBException

InterruptedException

addColumnFamilyOptionsToCloseLater

public static void addColumnFamilyOptionsToCloseLater(List<org.rocksdb.ColumnFamilyOptions> columnFamilyOptions,
                                                      org.rocksdb.ColumnFamilyHandle columnFamilyHandle)

allocateSharedCachesIfConfigured

@Nullable
public static org.apache.flink.runtime.memory.OpaqueMemoryResource<org.apache.flink.state.rocksdb.RocksDBSharedResources> allocateSharedCachesIfConfigured(RocksDBMemoryConfiguration jobMemoryConfig,
                                                                                                                                                           org.apache.flink.runtime.execution.Environment env,
                                                                                                                                                           double memoryFraction,
                                                                                                                                                           org.slf4j.Logger logger,
                                                                                                                                                           RocksDBMemoryControllerUtils.RocksDBMemoryFactory rocksDBMemoryFactory)
                                                                                                                                                    throws IOException

Throws:

IOException