facebook
segFault while write large data on multiple thread
We are writing without write batch and system got crash
Stack: [0x00007f6fe09de000,0x00007f6fe0adf000], sp=0x00007f6fe0adb750, free space=1013k Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code) C [librocksdbjni8579403097751306729.so+0x5f3eec] rocksdb::WritableFileWriter::Flush(rocksdb::Env::IOPriority)+0xfc C [librocksdbjni8579403097751306729.so+0x58f1ac] rocksdb::EnvLogger::Flush()+0xec C [librocksdbjni8579403097751306729.so+0x5fce80] rocksdb::AutoRollLogger::Flush()+0x60 C [librocksdbjni8579403097751306729.so+0x58656b] C [librocksdbjni8579403097751306729.so+0x5895d4] rocksdb::Log(rocksdb::InfoLogLevel, std::shared_ptrrocksdb::Logger const&, char const*, ...)+0x84 C [librocksdbjni8579403097751306729.so+0x448c78] rocksdb::DBImpl::Open(rocksdb::DBOptions const&, std::string const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocatorrocksdb::ColumnFamilyDescriptor > const&, std::vector<rocksdb::ColumnFamilyHandle*, std::allocatorrocksdb::ColumnFamilyHandle* >, rocksdb::DB**, bool, bool)+0x738 C [librocksdbjni8579403097751306729.so+0x44a7e6] rocksdb::DB::Open(rocksdb::DBOptions const&, std::string const&, std::vector<rocksdb::ColumnFamilyDescriptor, std::allocatorrocksdb::ColumnFamilyDescriptor > const&, std::vector<rocksdb::ColumnFamilyHandle, std::allocatorrocksdb::ColumnFamilyHandle* >, rocksdb::DB**)+0x56 C [librocksdbjni8579403097751306729.so+0x44a996] rocksdb::DB::Open(rocksdb::Options const&, std::string const&, rocksdb::DB**)+0x176 C [librocksdbjni8579403097751306729.so+0x2d4aba] std::_Function_handler<rocksdb::Status (rocksdb::Options const&, std::string const&, rocksdb::DB**), rocksdb::Status ()(rocksdb::Options const&, std::string const&, rocksdb::DB**)>::_M_invoke(std::Any_data const&, rocksdb::Options const&, std::string const&, rocksdb::DB**&&)+0x1a C [librocksdbjni8579403097751306729.so+0x2cf156] rocksdb_open_helper(JNIEnv, long, _jstring, std::function<rocksdb::Status (rocksdb::Options const&, std::string const&, rocksdb::DB**)>)+0x86 C [librocksdbjni8579403097751306729.so+0x2cf25e] Java_org_rocksdb_RocksDB_open__JLjava_lang_String_2+0x3e j org.rocksdb.RocksDB.open(JLjava/lang/String;)J+0 j org.rocksdb.RocksDB.open(Lorg/rocksdb/Options;Ljava/lang/String;)Lorg/rocksdb/RocksDB;+9 j
Hi @ChiragGangwani If you could provide further details of what you are doing, with a reproduction if possible, plus platform and RocksDB version you are using, it would make it more possible for someone to understand what might be going wrong..
Hi @alanpaxton thanks for the response. Please find the further deatils below.
RocksDB version - 8.5.4
So we are doing high amount of write in multiple db & column families in multiple thread with write batch functionality. There is no reproduction code because it happen unexpected anytime but only with large data.
write batch size - 50 mb. Here some configuration -
DB Creation code -
private void initDB() throws StoreInitException {
Logger.info("MapStore write batch size - " + storeProps.getWriteBatchSize());
this.options.setStatistics(new Statistics());
this.initCache();
this.options.setAllowMmapReads(true);
this.options.setInfoLogLevel(InfoLogLevel.WARN_LEVEL);
this.options.setMaxLogFileSize(1000);
this.options.setKeepLogFileNum(1);
if (storeProps.isRead()) {
createDescriptors();
try
{
this.db = RocksDB.openReadOnly(dbOptions, this.storeProps.getDbFolder(), descriptors,
columnFamilyHandles);
}
catch (RocksDBException e)
{
throw new StoreInitException(e);
}
createDescriptorHandleMap();
try {
this.mapStoreMetadata = MapStoreMetadata.deserializeMetadata(storeProps.getDbFolder());
} catch (Exception e) {
Logger.warn("Store metadata is not available, metadata operations might be slow.");
Logger.warn(e);
}
}
else
{
Utility.ensureEmptyFolder(this.storeProps.getDbFolder());
this.options.setCreateIfMissing(true);
this.options.setCreateMissingColumnFamilies(true);
try
{
this.db = RocksDB.open(options, this.storeProps.getDbFolder());
}
catch (RocksDBException e)
{
Logger.error(e);
throw new StoreInitException(e);
}
addSegments(storeProps.getSegmentList());
}
defaultColumnFamily = this.db.getDefaultColumnFamily();
logInitializingStore();
}
Cache -
synchronized private void initCache() {
storeProps.getCacheProperty().ifPresent(cacheProperties -> {
Function<String, Cache> cacheFunc = id -> CacheType.CLOCK.equals(cacheProperties.getCacheType())
? new ClockCache(cacheProperties.getCacheSize())
: new LRUCache(cacheProperties.getCacheSize());
Cache cache = cacheMap.computeIfAbsent(cacheProperties.getId(), cacheFunc);
this.blockBasedTableConfig = new BlockBasedTableConfig().setBlockCache(cache);
this.options.setWriteBufferSize(cacheProperties.getWriteBufferSize());
this.options.setTableFormatConfig(blockBasedTableConfig);
});
}
Put Operation -
public void put(Optional<String> segmentId, K key, V val) throws StoreWriteException {
byte[] keyArr = toByteArr(key);
byte[] valArr = toByteArr(val);
try {
**if(storeProps.getWriteBatchSize()>0){
putInBatch(segmentId, key, val);
}**
else {
this.db.put(getColumnFamilyHandle(segmentId), keyArr, valArr);
}
} catch (RocksDBException e) {
Logger.error(e);
throw new StoreWriteException(e);
}
}
private void putInBatch(Optional<String> segmentId, K key, V val) throws StoreWriteException {
byte[] keyArr = toByteArr(key);
byte[] valArr = toByteArr(val);
try {
writeBatch.put(getColumnFamilyHandle(segmentId), keyArr, valArr);
} catch (RocksDBException e) {
throw new StoreWriteException(e);
}
if (writeBatch.getDataSize() >= storeProps.getWriteBatchSize()) {
putBatchInDb();
}
}
private void putBatchInDb() throws StoreWriteException {
try {
if(writeBatch.count()>0){
db.write(new WriteOptions(), writeBatch);
writeBatch.clear();
}
} catch (RocksDBException e) {
throw new StoreWriteException(e);
}
}
Uploading hs_err_pid3859541.log… Please check the crash logs for further details.