Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 10f21ae

Browse filesBrowse files
freemandealerYour Name
freemandealer
authored and
Your Name
committed
[enhancement](cloud) file cache evict in advance (#47473)
evict in advance if current cache size is over threshold to avoid sync evict during query, which may affect query performance.
1 parent 94082a1 commit 10f21ae
Copy full SHA for 10f21ae

File tree

5 files changed

+416
-21
lines changed
Filter options

5 files changed

+416
-21
lines changed

‎be/src/common/config.cpp

Copy file name to clipboardExpand all lines: be/src/common/config.cpp
+6Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1068,6 +1068,12 @@ DEFINE_Bool(clear_file_cache, "false");
10681068
DEFINE_Bool(enable_file_cache_query_limit, "false");
10691069
DEFINE_mInt32(file_cache_enter_disk_resource_limit_mode_percent, "88");
10701070
DEFINE_mInt32(file_cache_exit_disk_resource_limit_mode_percent, "80");
1071+
DEFINE_mBool(enable_evict_file_cache_in_advance, "true");
1072+
DEFINE_mInt32(file_cache_enter_need_evict_cache_in_advance_percent, "78");
1073+
DEFINE_mInt32(file_cache_exit_need_evict_cache_in_advance_percent, "75");
1074+
DEFINE_mInt32(file_cache_evict_in_advance_interval_ms, "1000");
1075+
DEFINE_mInt64(file_cache_evict_in_advance_batch_bytes, "31457280"); // 30MB
1076+
10711077
DEFINE_mBool(enable_read_cache_file_directly, "false");
10721078
DEFINE_mBool(file_cache_enable_evict_from_other_queue_by_size, "true");
10731079
// If true, evict the ttl cache using LRU when full.

‎be/src/common/config.h

Copy file name to clipboardExpand all lines: be/src/common/config.h
+5Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1104,6 +1104,11 @@ DECLARE_Bool(clear_file_cache);
11041104
DECLARE_Bool(enable_file_cache_query_limit);
11051105
DECLARE_Int32(file_cache_enter_disk_resource_limit_mode_percent);
11061106
DECLARE_Int32(file_cache_exit_disk_resource_limit_mode_percent);
1107+
DECLARE_mBool(enable_evict_file_cache_in_advance);
1108+
DECLARE_mInt32(file_cache_enter_need_evict_cache_in_advance_percent);
1109+
DECLARE_mInt32(file_cache_exit_need_evict_cache_in_advance_percent);
1110+
DECLARE_mInt32(file_cache_evict_in_advance_interval_ms);
1111+
DECLARE_mInt64(file_cache_evict_in_advance_batch_bytes);
11071112
DECLARE_mBool(enable_read_cache_file_directly);
11081113
DECLARE_Bool(file_cache_enable_evict_from_other_queue_by_size);
11091114
// If true, evict the ttl cache using LRU when full.

‎be/src/io/cache/block_file_cache.cpp

Copy file name to clipboardExpand all lines: be/src/io/cache/block_file_cache.cpp
+130-16Lines changed: 130 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,8 @@ BlockFileCache::BlockFileCache(const std::string& cache_base_path,
201201
"file_cache_hit_ratio_1h", 0.0);
202202
_disk_limit_mode_metrics = std::make_shared<bvar::Status<size_t>>(
203203
_cache_base_path.c_str(), "file_cache_disk_limit_mode", 0);
204+
_need_evict_cache_in_advance_metrics = std::make_shared<bvar::Status<size_t>>(
205+
_cache_base_path.c_str(), "file_cache_need_evict_cache_in_advance", 0);
204206

205207
_cache_lock_wait_time_us = std::make_shared<bvar::LatencyRecorder>(
206208
_cache_base_path.c_str(), "file_cache_cache_lock_wait_time_us");
@@ -212,6 +214,11 @@ BlockFileCache::BlockFileCache(const std::string& cache_base_path,
212214
_cache_base_path.c_str(), "file_cache_storage_retry_sync_remove_latency_us");
213215
_storage_async_remove_latency_us = std::make_shared<bvar::LatencyRecorder>(
214216
_cache_base_path.c_str(), "file_cache_storage_async_remove_latency_us");
217+
_evict_in_advance_latency_us = std::make_shared<bvar::LatencyRecorder>(
218+
_cache_base_path.c_str(), "file_cache_evict_in_advance_latency_us");
219+
220+
_recycle_keys_length_recorder = std::make_shared<bvar::LatencyRecorder>(
221+
_cache_base_path.c_str(), "file_cache_recycle_keys_length");
215222

216223
_disposable_queue = LRUQueue(cache_settings.disposable_queue_size,
217224
cache_settings.disposable_queue_elements, 60 * 60);
@@ -339,6 +346,8 @@ Status BlockFileCache::initialize_unlocked(std::lock_guard<std::mutex>& cache_lo
339346
_cache_background_monitor_thread = std::thread(&BlockFileCache::run_background_monitor, this);
340347
_cache_background_ttl_gc_thread = std::thread(&BlockFileCache::run_background_ttl_gc, this);
341348
_cache_background_gc_thread = std::thread(&BlockFileCache::run_background_gc, this);
349+
_cache_background_evict_in_advance_thread =
350+
std::thread(&BlockFileCache::run_background_evict_in_advance, this);
342351

343352
return Status::OK();
344353
}
@@ -1021,6 +1030,16 @@ bool BlockFileCache::try_reserve(const UInt128Wrapper& hash, const CacheContext&
10211030
return true;
10221031
}
10231032

1033+
void BlockFileCache::try_evict_in_advance(size_t size, std::lock_guard<std::mutex>& cache_lock) {
1034+
UInt128Wrapper hash = UInt128Wrapper();
1035+
size_t offset = 0;
1036+
CacheContext context;
1037+
context.cache_type = FileCacheType::NORMAL;
1038+
try_reserve_for_lru(hash, nullptr, context, offset, size, cache_lock, false);
1039+
context.cache_type = FileCacheType::TTL;
1040+
try_reserve_for_lru(hash, nullptr, context, offset, size, cache_lock, false);
1041+
}
1042+
10241043
bool BlockFileCache::remove_if_ttl_file_blocks(const UInt128Wrapper& file_key, bool remove_directly,
10251044
std::lock_guard<std::mutex>& cache_lock, bool sync) {
10261045
auto& ttl_queue = get_queue(FileCacheType::TTL);
@@ -1178,7 +1197,7 @@ void BlockFileCache::reset_range(const UInt128Wrapper& hash, size_t offset, size
11781197

11791198
bool BlockFileCache::try_reserve_from_other_queue_by_time_interval(
11801199
FileCacheType cur_type, std::vector<FileCacheType> other_cache_types, size_t size,
1181-
int64_t cur_time, std::lock_guard<std::mutex>& cache_lock) {
1200+
int64_t cur_time, std::lock_guard<std::mutex>& cache_lock, bool sync_removal) {
11821201
size_t removed_size = 0;
11831202
size_t cur_cache_size = _cur_cache_size;
11841203
std::vector<FileBlockCell*> to_evict;
@@ -1211,7 +1230,7 @@ bool BlockFileCache::try_reserve_from_other_queue_by_time_interval(
12111230
}
12121231
*(_evict_by_time_metrics_matrix[cache_type][cur_type]) << remove_size_per_type;
12131232
}
1214-
remove_file_blocks(to_evict, cache_lock, true);
1233+
remove_file_blocks(to_evict, cache_lock, sync_removal);
12151234

12161235
return !is_overflow(removed_size, size, cur_cache_size);
12171236
}
@@ -1229,7 +1248,7 @@ bool BlockFileCache::is_overflow(size_t removed_size, size_t need_size,
12291248

12301249
bool BlockFileCache::try_reserve_from_other_queue_by_size(
12311250
FileCacheType cur_type, std::vector<FileCacheType> other_cache_types, size_t size,
1232-
std::lock_guard<std::mutex>& cache_lock) {
1251+
std::lock_guard<std::mutex>& cache_lock, bool sync_removal) {
12331252
size_t removed_size = 0;
12341253
size_t cur_cache_size = _cur_cache_size;
12351254
std::vector<FileBlockCell*> to_evict;
@@ -1249,17 +1268,18 @@ bool BlockFileCache::try_reserve_from_other_queue_by_size(
12491268
cur_removed_size);
12501269
*(_evict_by_size_metrics_matrix[cache_type][cur_type]) << cur_removed_size;
12511270
}
1252-
remove_file_blocks(to_evict, cache_lock, true);
1271+
remove_file_blocks(to_evict, cache_lock, sync_removal);
12531272
return !is_overflow(removed_size, size, cur_cache_size);
12541273
}
12551274

12561275
bool BlockFileCache::try_reserve_from_other_queue(FileCacheType cur_cache_type, size_t size,
12571276
int64_t cur_time,
1258-
std::lock_guard<std::mutex>& cache_lock) {
1277+
std::lock_guard<std::mutex>& cache_lock,
1278+
bool sync_removal) {
12591279
// currently, TTL cache is not considered as a candidate
12601280
auto other_cache_types = get_other_cache_type_without_ttl(cur_cache_type);
12611281
bool reserve_success = try_reserve_from_other_queue_by_time_interval(
1262-
cur_cache_type, other_cache_types, size, cur_time, cache_lock);
1282+
cur_cache_type, other_cache_types, size, cur_time, cache_lock, sync_removal);
12631283
if (reserve_success || !config::file_cache_enable_evict_from_other_queue_by_size) {
12641284
return reserve_success;
12651285
}
@@ -1272,14 +1292,15 @@ bool BlockFileCache::try_reserve_from_other_queue(FileCacheType cur_cache_type,
12721292
if (_cur_cache_size + size > _capacity && cur_queue_size + size > cur_queue_max_size) {
12731293
return false;
12741294
}
1275-
return try_reserve_from_other_queue_by_size(cur_cache_type, other_cache_types, size,
1276-
cache_lock);
1295+
return try_reserve_from_other_queue_by_size(cur_cache_type, other_cache_types, size, cache_lock,
1296+
sync_removal);
12771297
}
12781298

12791299
bool BlockFileCache::try_reserve_for_lru(const UInt128Wrapper& hash,
12801300
QueryFileCacheContextPtr query_context,
12811301
const CacheContext& context, size_t offset, size_t size,
1282-
std::lock_guard<std::mutex>& cache_lock) {
1302+
std::lock_guard<std::mutex>& cache_lock,
1303+
bool sync_removal) {
12831304
int64_t cur_time = std::chrono::duration_cast<std::chrono::seconds>(
12841305
std::chrono::steady_clock::now().time_since_epoch())
12851306
.count();
@@ -1292,7 +1313,7 @@ bool BlockFileCache::try_reserve_for_lru(const UInt128Wrapper& hash,
12921313
size_t cur_removed_size = 0;
12931314
find_evict_candidates(queue, size, cur_cache_size, removed_size, to_evict, cache_lock,
12941315
cur_removed_size);
1295-
remove_file_blocks(to_evict, cache_lock, true);
1316+
remove_file_blocks(to_evict, cache_lock, sync_removal);
12961317
*(_evict_by_self_lru_metrics_matrix[context.cache_type]) << cur_removed_size;
12971318

12981319
if (is_overflow(removed_size, size, cur_cache_size)) {
@@ -1345,7 +1366,9 @@ void BlockFileCache::remove(FileBlockSPtr file_block, T& cache_lock, U& block_lo
13451366
// so there will be a window that the file is not in the cache but still in the storage
13461367
// but it's ok, because the rowset is stale already
13471368
bool ret = _recycle_keys.enqueue(key);
1348-
if (!ret) {
1369+
if (ret) [[likely]] {
1370+
*_recycle_keys_length_recorder << _recycle_keys.size_approx();
1371+
} else {
13491372
LOG_WARNING("Failed to push recycle key to queue, do it synchronously");
13501373
int64_t duration_ns = 0;
13511374
Status st;
@@ -1551,6 +1574,10 @@ int disk_used_percentage(const std::string& path, std::pair<int, int>* percent)
15511574
int inode_percentage = int(inode_free * 1.0 / inode_total * 100);
15521575
percent->first = capacity_percentage;
15531576
percent->second = 100 - inode_percentage;
1577+
1578+
// Add sync point for testing
1579+
TEST_SYNC_POINT_CALLBACK("BlockFileCache::disk_used_percentage:1", percent);
1580+
15541581
return 0;
15551582
}
15561583

@@ -1643,7 +1670,7 @@ void BlockFileCache::check_disk_resource_limit() {
16431670
LOG_WARNING("config error, set to default value")
16441671
.tag("enter", config::file_cache_enter_disk_resource_limit_mode_percent)
16451672
.tag("exit", config::file_cache_exit_disk_resource_limit_mode_percent);
1646-
config::file_cache_enter_disk_resource_limit_mode_percent = 90;
1673+
config::file_cache_enter_disk_resource_limit_mode_percent = 88;
16471674
config::file_cache_exit_disk_resource_limit_mode_percent = 80;
16481675
}
16491676
if (is_insufficient(space_percentage) || is_insufficient(inode_percentage)) {
@@ -1664,11 +1691,69 @@ void BlockFileCache::check_disk_resource_limit() {
16641691
}
16651692
}
16661693

1694+
void BlockFileCache::check_need_evict_cache_in_advance() {
1695+
if (_storage->get_type() != FileCacheStorageType::DISK) {
1696+
return;
1697+
}
1698+
1699+
std::pair<int, int> percent;
1700+
int ret = disk_used_percentage(_cache_base_path, &percent);
1701+
if (ret != 0) {
1702+
LOG_ERROR("").tag("file cache path", _cache_base_path).tag("error", strerror(errno));
1703+
return;
1704+
}
1705+
auto [space_percentage, inode_percentage] = percent;
1706+
size_t size_percentage = static_cast<size_t>(
1707+
(static_cast<double>(_cur_cache_size) / static_cast<double>(_capacity)) * 100);
1708+
auto is_insufficient = [](const int& percentage) {
1709+
return percentage >= config::file_cache_enter_need_evict_cache_in_advance_percent;
1710+
};
1711+
DCHECK_GE(space_percentage, 0);
1712+
DCHECK_LE(space_percentage, 100);
1713+
DCHECK_GE(inode_percentage, 0);
1714+
DCHECK_LE(inode_percentage, 100);
1715+
// ATTN: due to that can be changed dynamically, set it to default value if it's invalid
1716+
// FIXME: reject with config validator
1717+
if (config::file_cache_enter_need_evict_cache_in_advance_percent <=
1718+
config::file_cache_exit_need_evict_cache_in_advance_percent) {
1719+
LOG_WARNING("config error, set to default value")
1720+
.tag("enter", config::file_cache_enter_need_evict_cache_in_advance_percent)
1721+
.tag("exit", config::file_cache_exit_need_evict_cache_in_advance_percent);
1722+
config::file_cache_enter_need_evict_cache_in_advance_percent = 78;
1723+
config::file_cache_exit_need_evict_cache_in_advance_percent = 75;
1724+
}
1725+
if (is_insufficient(space_percentage) || is_insufficient(inode_percentage) ||
1726+
is_insufficient(size_percentage)) {
1727+
_need_evict_cache_in_advance = true;
1728+
_need_evict_cache_in_advance_metrics->set_value(1);
1729+
} else if (_need_evict_cache_in_advance &&
1730+
(space_percentage < config::file_cache_exit_need_evict_cache_in_advance_percent) &&
1731+
(inode_percentage < config::file_cache_exit_need_evict_cache_in_advance_percent) &&
1732+
(size_percentage < config::file_cache_exit_need_evict_cache_in_advance_percent)) {
1733+
_need_evict_cache_in_advance = false;
1734+
_need_evict_cache_in_advance_metrics->set_value(0);
1735+
}
1736+
if (_need_evict_cache_in_advance) {
1737+
LOG(WARNING) << "file_cache=" << get_base_path() << " space_percent=" << space_percentage
1738+
<< " inode_percent=" << inode_percentage << " size_percent=" << size_percentage
1739+
<< " is_space_insufficient=" << is_insufficient(space_percentage)
1740+
<< " is_inode_insufficient=" << is_insufficient(inode_percentage)
1741+
<< " is_size_insufficient=" << is_insufficient(size_percentage)
1742+
<< " need evict cache in advance";
1743+
}
1744+
}
1745+
16671746
void BlockFileCache::run_background_monitor() {
16681747
int64_t interval_time_seconds = 20;
16691748
while (!_close) {
16701749
TEST_SYNC_POINT_CALLBACK("BlockFileCache::set_sleep_time", &interval_time_seconds);
16711750
check_disk_resource_limit();
1751+
if (config::enable_evict_file_cache_in_advance) {
1752+
check_need_evict_cache_in_advance();
1753+
} else {
1754+
_need_evict_cache_in_advance = false;
1755+
}
1756+
16721757
{
16731758
std::unique_lock close_lock(_close_mtx);
16741759
_close_cv.wait_for(close_lock, std::chrono::seconds(interval_time_seconds));
@@ -1753,11 +1838,8 @@ void BlockFileCache::run_background_gc() {
17531838
break;
17541839
}
17551840
}
1756-
while (_recycle_keys.try_dequeue(key)) {
1757-
if (batch_count >= batch_limit) {
1758-
break;
1759-
}
17601841

1842+
while (batch_count < batch_limit && _recycle_keys.try_dequeue(key)) {
17611843
int64_t duration_ns = 0;
17621844
Status st;
17631845
{
@@ -1771,10 +1853,42 @@ void BlockFileCache::run_background_gc() {
17711853
}
17721854
batch_count++;
17731855
}
1856+
*_recycle_keys_length_recorder << _recycle_keys.size_approx();
17741857
batch_count = 0;
17751858
}
17761859
}
17771860

1861+
void BlockFileCache::run_background_evict_in_advance() {
1862+
LOG(INFO) << "Starting background evict in advance thread";
1863+
int64_t batch = 0;
1864+
while (!_close) {
1865+
{
1866+
std::unique_lock close_lock(_close_mtx);
1867+
_close_cv.wait_for(
1868+
close_lock,
1869+
std::chrono::milliseconds(config::file_cache_evict_in_advance_interval_ms));
1870+
if (_close) {
1871+
LOG(INFO) << "Background evict in advance thread exiting due to cache closing";
1872+
break;
1873+
}
1874+
}
1875+
batch = config::file_cache_evict_in_advance_batch_bytes;
1876+
1877+
// Skip if eviction not needed or too many pending recycles
1878+
if (!_need_evict_cache_in_advance || _recycle_keys.size_approx() >= (batch * 10)) {
1879+
continue;
1880+
}
1881+
1882+
int64_t duration_ns = 0;
1883+
{
1884+
SCOPED_CACHE_LOCK(_mutex, this);
1885+
SCOPED_RAW_TIMER(&duration_ns);
1886+
try_evict_in_advance(batch, cache_lock);
1887+
}
1888+
*_evict_in_advance_latency_us << (duration_ns / 1000);
1889+
}
1890+
}
1891+
17781892
void BlockFileCache::modify_expiration_time(const UInt128Wrapper& hash,
17791893
uint64_t new_expiration_time) {
17801894
SCOPED_CACHE_LOCK(_mutex, this);

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.