Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit b684c8d

Browse filesBrowse files
freemandealerfreemandealer
committed
[enhancement](cloud) file cache evict in advance
evict in advance if current cache size is over threshold to avoid sync evict during query, which may affect query performance. Signed-off-by: zhengyu <zhangzhengyu@selectdb.com>
1 parent b4dcf89 commit b684c8d
Copy full SHA for b684c8d

File tree

5 files changed

+268
-16
lines changed
Filter options

5 files changed

+268
-16
lines changed

‎be/src/common/config.cpp

Copy file name to clipboardExpand all lines: be/src/common/config.cpp
+6Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1048,6 +1048,12 @@ DEFINE_Bool(clear_file_cache, "false");
10481048
DEFINE_Bool(enable_file_cache_query_limit, "false");
10491049
DEFINE_mInt32(file_cache_enter_disk_resource_limit_mode_percent, "88");
10501050
DEFINE_mInt32(file_cache_exit_disk_resource_limit_mode_percent, "80");
1051+
DEFINE_mBool(enable_evict_file_cache_in_advance, "true");
1052+
DEFINE_mInt32(file_cache_enter_need_evict_cache_in_advance_percent, "88");
1053+
DEFINE_mInt32(file_cache_exit_need_evict_cache_in_advance_percent, "80");
1054+
DEFINE_mInt32(file_cache_evict_in_advance_interval_ms, "1000");
1055+
DEFINE_mInt64(file_cache_evict_in_advance_batch_bytes, "31457280"); // 30MB
1056+
10511057
DEFINE_mBool(enable_read_cache_file_directly, "false");
10521058
DEFINE_mBool(file_cache_enable_evict_from_other_queue_by_size, "true");
10531059
// If true, evict the ttl cache using LRU when full.

‎be/src/common/config.h

Copy file name to clipboardExpand all lines: be/src/common/config.h
+5Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1088,6 +1088,11 @@ DECLARE_Bool(clear_file_cache);
10881088
DECLARE_Bool(enable_file_cache_query_limit);
10891089
DECLARE_Int32(file_cache_enter_disk_resource_limit_mode_percent);
10901090
DECLARE_Int32(file_cache_exit_disk_resource_limit_mode_percent);
1091+
DECLARE_mBool(enable_evict_file_cache_in_advance);
1092+
DECLARE_mInt32(file_cache_enter_need_evict_cache_in_advance_percent);
1093+
DECLARE_mInt32(file_cache_exit_need_evict_cache_in_advance_percent);
1094+
DECLARE_mInt32(file_cache_evict_in_advance_interval_ms);
1095+
DECLARE_mInt64(file_cache_evict_in_advance_batch_bytes);
10911096
DECLARE_mBool(enable_read_cache_file_directly);
10921097
DECLARE_Bool(file_cache_enable_evict_from_other_queue_by_size);
10931098
// If true, evict the ttl cache using LRU when full.

‎be/src/io/cache/block_file_cache.cpp

Copy file name to clipboardExpand all lines: be/src/io/cache/block_file_cache.cpp
+113-11Lines changed: 113 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -201,11 +201,15 @@ BlockFileCache::BlockFileCache(const std::string& cache_base_path,
201201
"file_cache_hit_ratio_1h", 0.0);
202202
_disk_limit_mode_metrics = std::make_shared<bvar::Status<size_t>>(
203203
_cache_base_path.c_str(), "file_cache_disk_limit_mode", 0);
204+
_need_evict_cache_in_advance_metrics = std::make_shared<bvar::Status<size_t>>(
205+
_cache_base_path.c_str(), "file_cache_need_evict_cache_in_advance", 0);
204206

205207
_storage_sync_remove_latency = std::make_shared<bvar::LatencyRecorder>(
206208
_cache_base_path.c_str(), "file_cache_storage_sync_remove_latency_ns");
207209
_storage_async_remove_latency = std::make_shared<bvar::LatencyRecorder>(
208210
_cache_base_path.c_str(), "file_cache_storage_async_remove_latency_ns");
211+
_evict_in_advance_latency = std::make_shared<bvar::LatencyRecorder>(
212+
_cache_base_path.c_str(), "file_cache_evict_in_advance_latency_ns");
209213

210214
_disposable_queue = LRUQueue(cache_settings.disposable_queue_size,
211215
cache_settings.disposable_queue_elements, 60 * 60);
@@ -333,6 +337,8 @@ Status BlockFileCache::initialize_unlocked(std::lock_guard<std::mutex>& cache_lo
333337
_cache_background_monitor_thread = std::thread(&BlockFileCache::run_background_monitor, this);
334338
_cache_background_ttl_gc_thread = std::thread(&BlockFileCache::run_background_ttl_gc, this);
335339
_cache_background_gc_thread = std::thread(&BlockFileCache::run_background_gc, this);
340+
_cache_background_evict_in_advance_thread =
341+
std::thread(&BlockFileCache::run_background_evict_in_advance, this);
336342

337343
return Status::OK();
338344
}
@@ -1011,6 +1017,16 @@ bool BlockFileCache::try_reserve(const UInt128Wrapper& hash, const CacheContext&
10111017
return true;
10121018
}
10131019

1020+
void BlockFileCache::try_evict_in_advance(size_t size, std::lock_guard<std::mutex>& cache_lock) {
1021+
UInt128Wrapper hash;
1022+
size_t offset = 0;
1023+
CacheContext context;
1024+
context.cache_type = FileCacheType::NORMAL;
1025+
try_reserve_for_lru(hash, nullptr, context, offset, size, cache_lock, false);
1026+
context.cache_type = FileCacheType::TTL;
1027+
try_reserve_for_lru(hash, nullptr, context, offset, size, cache_lock, false);
1028+
}
1029+
10141030
bool BlockFileCache::remove_if_ttl_file_blocks(const UInt128Wrapper& file_key, bool remove_directly,
10151031
std::lock_guard<std::mutex>& cache_lock, bool sync) {
10161032
auto& ttl_queue = get_queue(FileCacheType::TTL);
@@ -1168,7 +1184,7 @@ void BlockFileCache::reset_range(const UInt128Wrapper& hash, size_t offset, size
11681184

11691185
bool BlockFileCache::try_reserve_from_other_queue_by_time_interval(
11701186
FileCacheType cur_type, std::vector<FileCacheType> other_cache_types, size_t size,
1171-
int64_t cur_time, std::lock_guard<std::mutex>& cache_lock) {
1187+
int64_t cur_time, std::lock_guard<std::mutex>& cache_lock, bool sync_removal) {
11721188
size_t removed_size = 0;
11731189
size_t cur_cache_size = _cur_cache_size;
11741190
std::vector<FileBlockCell*> to_evict;
@@ -1201,7 +1217,7 @@ bool BlockFileCache::try_reserve_from_other_queue_by_time_interval(
12011217
}
12021218
*(_evict_by_time_metrics_matrix[cache_type][cur_type]) << remove_size_per_type;
12031219
}
1204-
remove_file_blocks(to_evict, cache_lock, true);
1220+
remove_file_blocks(to_evict, cache_lock, sync_removal);
12051221

12061222
return !is_overflow(removed_size, size, cur_cache_size);
12071223
}
@@ -1219,7 +1235,7 @@ bool BlockFileCache::is_overflow(size_t removed_size, size_t need_size,
12191235

12201236
bool BlockFileCache::try_reserve_from_other_queue_by_size(
12211237
FileCacheType cur_type, std::vector<FileCacheType> other_cache_types, size_t size,
1222-
std::lock_guard<std::mutex>& cache_lock) {
1238+
std::lock_guard<std::mutex>& cache_lock, bool sync_removal) {
12231239
size_t removed_size = 0;
12241240
size_t cur_cache_size = _cur_cache_size;
12251241
std::vector<FileBlockCell*> to_evict;
@@ -1239,17 +1255,18 @@ bool BlockFileCache::try_reserve_from_other_queue_by_size(
12391255
cur_removed_size);
12401256
*(_evict_by_size_metrics_matrix[cache_type][cur_type]) << cur_removed_size;
12411257
}
1242-
remove_file_blocks(to_evict, cache_lock, true);
1258+
remove_file_blocks(to_evict, cache_lock, sync_removal);
12431259
return !is_overflow(removed_size, size, cur_cache_size);
12441260
}
12451261

12461262
bool BlockFileCache::try_reserve_from_other_queue(FileCacheType cur_cache_type, size_t size,
12471263
int64_t cur_time,
1248-
std::lock_guard<std::mutex>& cache_lock) {
1264+
std::lock_guard<std::mutex>& cache_lock,
1265+
bool sync_removal) {
12491266
// currently, TTL cache is not considered as a candidate
12501267
auto other_cache_types = get_other_cache_type_without_ttl(cur_cache_type);
12511268
bool reserve_success = try_reserve_from_other_queue_by_time_interval(
1252-
cur_cache_type, other_cache_types, size, cur_time, cache_lock);
1269+
cur_cache_type, other_cache_types, size, cur_time, cache_lock, sync_removal);
12531270
if (reserve_success || !config::file_cache_enable_evict_from_other_queue_by_size) {
12541271
return reserve_success;
12551272
}
@@ -1262,14 +1279,15 @@ bool BlockFileCache::try_reserve_from_other_queue(FileCacheType cur_cache_type,
12621279
if (_cur_cache_size + size > _capacity && cur_queue_size + size > cur_queue_max_size) {
12631280
return false;
12641281
}
1265-
return try_reserve_from_other_queue_by_size(cur_cache_type, other_cache_types, size,
1266-
cache_lock);
1282+
return try_reserve_from_other_queue_by_size(cur_cache_type, other_cache_types, size, cache_lock,
1283+
sync_removal);
12671284
}
12681285

12691286
bool BlockFileCache::try_reserve_for_lru(const UInt128Wrapper& hash,
12701287
QueryFileCacheContextPtr query_context,
12711288
const CacheContext& context, size_t offset, size_t size,
1272-
std::lock_guard<std::mutex>& cache_lock) {
1289+
std::lock_guard<std::mutex>& cache_lock,
1290+
bool sync_removal) {
12731291
int64_t cur_time = std::chrono::duration_cast<std::chrono::seconds>(
12741292
std::chrono::steady_clock::now().time_since_epoch())
12751293
.count();
@@ -1282,7 +1300,7 @@ bool BlockFileCache::try_reserve_for_lru(const UInt128Wrapper& hash,
12821300
size_t cur_removed_size = 0;
12831301
find_evict_candidates(queue, size, cur_cache_size, removed_size, to_evict, cache_lock,
12841302
cur_removed_size);
1285-
remove_file_blocks(to_evict, cache_lock, true);
1303+
remove_file_blocks(to_evict, cache_lock, sync_removal);
12861304
*(_evict_by_self_lru_metrics_matrix[context.cache_type]) << cur_removed_size;
12871305

12881306
if (is_overflow(removed_size, size, cur_cache_size)) {
@@ -1627,7 +1645,7 @@ void BlockFileCache::check_disk_resource_limit() {
16271645
LOG_WARNING("config error, set to default value")
16281646
.tag("enter", config::file_cache_enter_disk_resource_limit_mode_percent)
16291647
.tag("exit", config::file_cache_exit_disk_resource_limit_mode_percent);
1630-
config::file_cache_enter_disk_resource_limit_mode_percent = 90;
1648+
config::file_cache_enter_disk_resource_limit_mode_percent = 88;
16311649
config::file_cache_exit_disk_resource_limit_mode_percent = 80;
16321650
}
16331651
if (is_insufficient(space_percentage) || is_insufficient(inode_percentage)) {
@@ -1648,11 +1666,69 @@ void BlockFileCache::check_disk_resource_limit() {
16481666
}
16491667
}
16501668

1669+
void BlockFileCache::check_need_evict_cache_in_advance() {
1670+
if (_storage->get_type() != FileCacheStorageType::DISK) {
1671+
return;
1672+
}
1673+
1674+
std::pair<int, int> percent;
1675+
int ret = disk_used_percentage(_cache_base_path, &percent);
1676+
if (ret != 0) {
1677+
LOG_ERROR("").tag("file cache path", _cache_base_path).tag("error", strerror(errno));
1678+
return;
1679+
}
1680+
auto [space_percentage, inode_percentage] = percent;
1681+
size_t size_percentage = static_cast<size_t>(
1682+
(static_cast<double>(_cur_cache_size) / static_cast<double>(_capacity)) * 100);
1683+
auto is_insufficient = [](const int& percentage) {
1684+
return percentage >= config::file_cache_enter_need_evict_cache_in_advance_percent;
1685+
};
1686+
DCHECK_GE(space_percentage, 0);
1687+
DCHECK_LE(space_percentage, 100);
1688+
DCHECK_GE(inode_percentage, 0);
1689+
DCHECK_LE(inode_percentage, 100);
1690+
// ATTN: due to that can be changed dynamically, set it to default value if it's invalid
1691+
// FIXME: reject with config validator
1692+
if (config::file_cache_enter_need_evict_cache_in_advance_percent <=
1693+
config::file_cache_exit_need_evict_cache_in_advance_percent) {
1694+
LOG_WARNING("config error, set to default value")
1695+
.tag("enter", config::file_cache_enter_need_evict_cache_in_advance_percent)
1696+
.tag("exit", config::file_cache_exit_need_evict_cache_in_advance_percent);
1697+
config::file_cache_enter_need_evict_cache_in_advance_percent = 78;
1698+
config::file_cache_exit_need_evict_cache_in_advance_percent = 75;
1699+
}
1700+
if (is_insufficient(space_percentage) || is_insufficient(inode_percentage) ||
1701+
is_insufficient(size_percentage)) {
1702+
_need_evict_cache_in_advance = true;
1703+
_need_evict_cache_in_advance_metrics->set_value(1);
1704+
} else if (_need_evict_cache_in_advance &&
1705+
(space_percentage < config::file_cache_exit_need_evict_cache_in_advance_percent) &&
1706+
(inode_percentage < config::file_cache_exit_need_evict_cache_in_advance_percent) &&
1707+
(size_percentage < config::file_cache_exit_need_evict_cache_in_advance_percent)) {
1708+
_need_evict_cache_in_advance = false;
1709+
_need_evict_cache_in_advance_metrics->set_value(0);
1710+
}
1711+
if (_need_evict_cache_in_advance) {
1712+
LOG(WARNING) << "file_cache=" << get_base_path() << " space_percent=" << space_percentage
1713+
<< " inode_percent=" << inode_percentage << " size_percent=" << size_percentage
1714+
<< " is_space_insufficient=" << is_insufficient(space_percentage)
1715+
<< " is_inode_insufficient=" << is_insufficient(inode_percentage)
1716+
<< " is_size_insufficient=" << is_insufficient(size_percentage)
1717+
<< " need evict cache in advance";
1718+
}
1719+
}
1720+
16511721
void BlockFileCache::run_background_monitor() {
16521722
int64_t interval_time_seconds = 20;
16531723
while (!_close) {
16541724
TEST_SYNC_POINT_CALLBACK("BlockFileCache::set_sleep_time", &interval_time_seconds);
16551725
check_disk_resource_limit();
1726+
if (config::enable_evict_file_cache_in_advance) {
1727+
check_need_evict_cache_in_advance();
1728+
} else {
1729+
_need_evict_cache_in_advance = false;
1730+
}
1731+
16561732
{
16571733
std::unique_lock close_lock(_close_mtx);
16581734
_close_cv.wait_for(close_lock, std::chrono::seconds(interval_time_seconds));
@@ -1756,6 +1832,32 @@ void BlockFileCache::run_background_gc() {
17561832
}
17571833
}
17581834

1835+
void BlockFileCache::run_background_evict_in_advance() {
1836+
int64_t batch = 0;
1837+
while (!_close) {
1838+
{
1839+
std::unique_lock close_lock(_close_mtx);
1840+
_close_cv.wait_for(
1841+
close_lock,
1842+
std::chrono::milliseconds(config::file_cache_evict_in_advance_interval_ms));
1843+
if (_close) {
1844+
break;
1845+
}
1846+
}
1847+
batch = config::file_cache_evict_in_advance_batch_bytes;
1848+
if (!_need_evict_cache_in_advance || _recycle_keys.size_approx() >= (batch * 10)) {
1849+
continue;
1850+
}
1851+
int64_t duration_ns = 0;
1852+
{
1853+
SCOPED_CACHE_LOCK(_mutex);
1854+
SCOPED_RAW_TIMER(&duration_ns);
1855+
try_evict_in_advance(batch, cache_lock);
1856+
}
1857+
*_evict_in_advance_latency << duration_ns;
1858+
}
1859+
}
1860+
17591861
void BlockFileCache::modify_expiration_time(const UInt128Wrapper& hash,
17601862
uint64_t new_expiration_time) {
17611863
SCOPED_CACHE_LOCK(_mutex);

‎be/src/io/cache/block_file_cache.h

Copy file name to clipboardExpand all lines: be/src/io/cache/block_file_cache.h
+19-4Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,9 @@ class BlockFileCache {
105105
if (_cache_background_gc_thread.joinable()) {
106106
_cache_background_gc_thread.join();
107107
}
108+
if (_cache_background_evict_in_advance_thread.joinable()) {
109+
_cache_background_evict_in_advance_thread.join();
110+
}
108111
}
109112

110113
/// Restore cache from local filesystem.
@@ -186,6 +189,9 @@ class BlockFileCache {
186189
bool try_reserve(const UInt128Wrapper& hash, const CacheContext& context, size_t offset,
187190
size_t size, std::lock_guard<std::mutex>& cache_lock);
188191

192+
// async evict when water mark is reached, unlike try_reserve, removal is done in background
193+
void try_evict_in_advance(size_t size, std::lock_guard<std::mutex>& cache_lock);
194+
189195
void update_ttl_atime(const UInt128Wrapper& hash);
190196

191197
std::map<std::string, double> get_stats();
@@ -391,15 +397,16 @@ class BlockFileCache {
391397

392398
bool try_reserve_for_lru(const UInt128Wrapper& hash, QueryFileCacheContextPtr query_context,
393399
const CacheContext& context, size_t offset, size_t size,
394-
std::lock_guard<std::mutex>& cache_lock);
400+
std::lock_guard<std::mutex>& cache_lock, bool sync_removal = true);
395401

396402
bool try_reserve_during_async_load(size_t size, std::lock_guard<std::mutex>& cache_lock);
397403

398404
std::vector<FileCacheType> get_other_cache_type(FileCacheType cur_cache_type);
399405
std::vector<FileCacheType> get_other_cache_type_without_ttl(FileCacheType cur_cache_type);
400406

401407
bool try_reserve_from_other_queue(FileCacheType cur_cache_type, size_t offset, int64_t cur_time,
402-
std::lock_guard<std::mutex>& cache_lock);
408+
std::lock_guard<std::mutex>& cache_lock,
409+
bool sync_removal = true);
403410

404411
size_t get_available_cache_size(FileCacheType cache_type) const;
405412

@@ -422,6 +429,7 @@ class BlockFileCache {
422429
std::lock_guard<std::mutex>& cache_lock) const;
423430

424431
void check_disk_resource_limit();
432+
void check_need_evict_cache_in_advance();
425433

426434
size_t get_available_cache_size_unlocked(FileCacheType type,
427435
std::lock_guard<std::mutex>& cache_lock) const;
@@ -437,15 +445,18 @@ class BlockFileCache {
437445
void run_background_monitor();
438446
void run_background_ttl_gc();
439447
void run_background_gc();
448+
void run_background_evict_in_advance();
440449

441450
bool try_reserve_from_other_queue_by_time_interval(FileCacheType cur_type,
442451
std::vector<FileCacheType> other_cache_types,
443452
size_t size, int64_t cur_time,
444-
std::lock_guard<std::mutex>& cache_lock);
453+
std::lock_guard<std::mutex>& cache_lock,
454+
bool sync_removal);
445455

446456
bool try_reserve_from_other_queue_by_size(FileCacheType cur_type,
447457
std::vector<FileCacheType> other_cache_types,
448-
size_t size, std::lock_guard<std::mutex>& cache_lock);
458+
size_t size, std::lock_guard<std::mutex>& cache_lock,
459+
bool sync_removal);
449460

450461
bool is_overflow(size_t removed_size, size_t need_size, size_t cur_cache_size) const;
451462

@@ -472,9 +483,11 @@ class BlockFileCache {
472483
std::thread _cache_background_monitor_thread;
473484
std::thread _cache_background_ttl_gc_thread;
474485
std::thread _cache_background_gc_thread;
486+
std::thread _cache_background_evict_in_advance_thread;
475487
std::atomic_bool _async_open_done {false};
476488
// disk space or inode is less than the specified value
477489
bool _disk_resource_limit_mode {false};
490+
bool _need_evict_cache_in_advance {false};
478491
bool _is_initialized {false};
479492

480493
// strategy
@@ -532,9 +545,11 @@ class BlockFileCache {
532545
std::shared_ptr<bvar::Status<double>> _hit_ratio_5m;
533546
std::shared_ptr<bvar::Status<double>> _hit_ratio_1h;
534547
std::shared_ptr<bvar::Status<size_t>> _disk_limit_mode_metrics;
548+
std::shared_ptr<bvar::Status<size_t>> _need_evict_cache_in_advance_metrics;
535549

536550
std::shared_ptr<bvar::LatencyRecorder> _storage_sync_remove_latency;
537551
std::shared_ptr<bvar::LatencyRecorder> _storage_async_remove_latency;
552+
std::shared_ptr<bvar::LatencyRecorder> _evict_in_advance_latency;
538553
};
539554

540555
} // namespace doris::io

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.