Merge upstream

.travis.yml

@@ -1,7 +1,8 @@
 dist: xenial
 language: go
 go:
-  - 1.11
+  - 1.12.x
+  - 1.13.x
   - tip
 
 before_install:

backup.go

@@ -104,11 +104,12 @@ func (b *BackupEngine) UnsafeGetBackupEngine() unsafe.Pointer {
 	return unsafe.Pointer(b.c)
 }
 
-// CreateNewBackup takes a new backup from db.
-func (b *BackupEngine) CreateNewBackup(db *DB) error {
+// CreateNewBackupFlush takes a new backup from db. If flush is set to true,
+// it flushes the WAL before taking the backup.
+func (b *BackupEngine) CreateNewBackupFlush(db *DB, flush bool) error {
 	var cErr *C.char
 
-	C.rocksdb_backup_engine_create_new_backup(b.c, db.c, &cErr)
+	C.rocksdb_backup_engine_create_new_backup_flush(b.c, db.c, boolToChar(flush), &cErr)
 	if cErr != nil {
 		defer C.rocksdb_free(unsafe.Pointer(cErr))
 		return errors.New(C.GoString(cErr))
@@ -117,6 +118,11 @@ func (b *BackupEngine) CreateNewBackup(db *DB) error {
 	return nil
 }
 
+// CreateNewBackup takes a new backup from db.
+func (b *BackupEngine) CreateNewBackup(db *DB) error {
+	return b.CreateNewBackupFlush(db, false)
+}
+
 // GetInfo gets an object that gives information about
 // the backups that have already been taken
 func (b *BackupEngine) GetInfo() *BackupEngineInfo {
@@ -144,6 +150,17 @@ func (b *BackupEngine) RestoreDBFromLatestBackup(dbDir, walDir string, ro *Resto
 	return nil
 }
 
+// PurgeOldBackups deletes all backups older than the latest 'n' backups
+func (b *BackupEngine) PurgeOldBackups(n uint32) error {
+	var cErr *C.char
+	C.rocksdb_backup_engine_purge_old_backups(b.c, C.uint32_t(n), &cErr)
+	if cErr != nil {
+		defer C.rocksdb_free(unsafe.Pointer(cErr))
+		return errors.New(C.GoString(cErr))
+	}
+	return nil
+}
+
 // Close close the backup engine and cleans up state
 // The backups already taken remain on storage.
 func (b *BackupEngine) Close() {

env.go

@@ -13,6 +13,11 @@ func NewDefaultEnv() *Env {
 	return NewNativeEnv(C.rocksdb_create_default_env())
 }
 
+// NewMemEnv creates MemEnv for in-memory testing.
+func NewMemEnv() *Env {
+	return NewNativeEnv(C.rocksdb_create_mem_env())
+}
+
 // NewNativeEnv creates a Environment object.
 func NewNativeEnv(c *C.rocksdb_env_t) *Env {
 	return &Env{c}

filter_policy.go

@@ -49,6 +49,12 @@ func NewBloomFilter(bitsPerKey int) FilterPolicy {
 	return NewNativeFilterPolicy(C.rocksdb_filterpolicy_create_bloom(C.int(bitsPerKey)))
 }
 
+// NewBloomFilterFull returns a new filter policy created with use_block_based_builder=false
+// (use full or partitioned filter).
+func NewBloomFilterFull(bitsPerKey int) FilterPolicy {
+	return NewNativeFilterPolicy(C.rocksdb_filterpolicy_create_bloom_full(C.int(bitsPerKey)))
+}
+
 // Hold references to filter policies.
 var filterPolicies = NewCOWList()
 

options.go

@@ -1202,6 +1202,8 @@ func (opts *Options) Destroy() {
 	if opts.ccmp != nil {
 		C.rocksdb_comparator_destroy(opts.ccmp)
 	}
+	// don't destroy the opts.cst here, it has already been
+	// associated with a PrefixExtractor and this will segfault
 	if opts.ccf != nil {
 		C.rocksdb_compactionfilter_destroy(opts.ccf)
 	}

options_block_based_table.go

@@ -56,6 +56,14 @@ func (opts *BlockBasedTableOptions) SetCacheIndexAndFilterBlocks(value bool) {
 	C.rocksdb_block_based_options_set_cache_index_and_filter_blocks(opts.c, boolToChar(value))
 }
 
+// SetCacheIndexAndFilterBlocksWithHighPriority sets cache index and filter
+// blocks with high priority (if cache_index_and_filter_blocks is enabled).
+// If set to true, depending on implementation of block cache,
+// index and filter blocks may be less likely to be evicted than data blocks.
+func (opts *BlockBasedTableOptions) SetCacheIndexAndFilterBlocksWithHighPriority(value bool) {
+	C.rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority(opts.c, boolToChar(value))
+}
+
 // SetPinL0FilterAndIndexBlocksInCache sets cache_index_and_filter_blocks.
 // If is true and the below is true (hash_index_allow_collision), then
 // filter and index blocks are stored in the cache, but a reference is
@@ -65,6 +73,15 @@ func (opts *BlockBasedTableOptions) SetPinL0FilterAndIndexBlocksInCache(value bo
 	C.rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache(opts.c, boolToChar(value))
 }
 
+// SetPinTopLevelIndexAndFilter set that if cache_index_and_filter_blocks is true, then
+// the top-level index of partitioned filter and index blocks are stored in
+// the cache, but a reference is held in the "table reader" object so the
+// blocks are pinned and only evicted from cache when the table reader is
+// freed. This is not limited to l0 in LSM tree.
+func (opts *BlockBasedTableOptions) SetPinTopLevelIndexAndFilter(value bool) {
+	C.rocksdb_block_based_options_set_pin_top_level_index_and_filter(opts.c, boolToChar(value))
+}
+
 // SetBlockSize sets the approximate size of user data packed per block.
 // Note that the block size specified here corresponds opts uncompressed data.
 // The actual size of the unit read from disk may be smaller if
@@ -94,6 +111,39 @@ func (opts *BlockBasedTableOptions) SetBlockRestartInterval(blockRestartInterval
 	C.rocksdb_block_based_options_set_block_restart_interval(opts.c, C.int(blockRestartInterval))
 }
 
+// SetIndexBlockRestartInterval is the same as SetBlockRestartInterval but used for the index block.
+// Default: 1
+func (opts *BlockBasedTableOptions) SetIndexBlockRestartInterval(indexBlockRestartInterval int) {
+	C.rocksdb_block_based_options_set_index_block_restart_interval(opts.c, C.int(indexBlockRestartInterval))
+}
+
+// SetMetadataBlockSize sets the block size for partitioned metadata.
+// Currently applied to indexes when
+// kTwoLevelIndexSearch is used and to filters when partition_filters is used.
+// Note: Since in the current implementation the filters and index partitions
+// are aligned, an index/filter block is created when either index or filter
+// block size reaches the specified limit.
+// Note: this limit is currently applied to only index blocks; a filter
+// partition is cut right after an index block is cut
+// Default: 4096
+func (opts *BlockBasedTableOptions) SetMetadataBlockSize(metadataBlockSize uint64) {
+	C.rocksdb_block_based_options_set_metadata_block_size(opts.c, C.uint64_t(metadataBlockSize))
+}
+
+// SetPartitionFilters sets using partitioned full filters for each SST file.
+// This option is incompatible with block-based filters.
+// Note: currently this option requires kTwoLevelIndexSearch to be set as well.
+// Default: false
+func (opts *BlockBasedTableOptions) SetPartitionFilters(value bool) {
+	C.rocksdb_block_based_options_set_partition_filters(opts.c, boolToChar(value))
+}
+
+// SetUseDeltaEncoding sets using delta encoding to compress keys in blocks.
+// ReadOptions::pin_data requires this option to be disabled.
+func (opts *BlockBasedTableOptions) SetUseDeltaEncoding(value bool) {
+	C.rocksdb_block_based_options_set_use_delta_encoding(opts.c, boolToChar(value))
+}
+
 // SetFilterPolicy sets the filter policy opts reduce disk reads.
 // Many applications will benefit from passing the result of
 // NewBloomFilterPolicy() here.
@@ -141,6 +191,35 @@ func (opts *BlockBasedTableOptions) SetWholeKeyFiltering(value bool) {
 	C.rocksdb_block_based_options_set_whole_key_filtering(opts.c, boolToChar(value))
 }
 
+// SetFormatVersion sets the format version.
+// We currently have five versions:
+// 0 -- This version is currently written out by all RocksDB's versions by
+// default.  Can be read by really old RocksDB's. Doesn't support changing
+// checksum (default is CRC32).
+// 1 -- Can be read by RocksDB's versions since 3.0. Supports non-default
+// checksum, like xxHash. It is written by RocksDB when
+// BlockBasedTableOptions::checksum is something other than kCRC32c. (version
+// 0 is silently upconverted)
+// 2 -- Can be read by RocksDB's versions since 3.10. Changes the way we
+// encode compressed blocks with LZ4, BZip2 and Zlib compression. If you
+// don't plan to run RocksDB before version 3.10, you should probably use
+// this.
+// 3 -- Can be read by RocksDB's versions since 5.15. Changes the way we
+// encode the keys in index blocks. If you don't plan to run RocksDB before
+// version 5.15, you should probably use this.
+// This option only affects newly written tables. When reading existing
+// tables, the information about version is read from the footer.
+// 4 -- Can be read by RocksDB's versions since 5.16. Changes the way we
+// encode the values in index blocks. If you don't plan to run RocksDB before
+// version 5.16 and you are using index_block_restart_interval > 1, you should
+// probably use this as it would reduce the index size.
+// This option only affects newly written tables. When reading existing
+// tables, the information about version is read from the footer.
+// Default: 2
+func (opts *BlockBasedTableOptions) SetFormatVersion(version int) {
+	C.rocksdb_block_based_options_set_format_version(opts.c, C.int(version))
+}
+
 // SetIndexType sets the index type used for this table.
 // kBinarySearch:
 // A space efficient index block that is optimized for