From e6dc4fe49d9e6ce59d4a888b82460a9ec89e3449 Mon Sep 17 00:00:00 2001 From: "Thiago Marcos P. Santos" Date: Mon, 21 Dec 2015 17:53:34 +0200 Subject: [PATCH] [tests] Add unit tests for SQLiteCache size limit --- test/fixtures/database/cache.db | Bin 4096 -> 0 bytes test/fixtures/database/invalid.db | Bin 4096 -> 0 bytes test/storage/cache_size.cpp | 232 ++++++++++++++++++++++++++++++ test/test.gypi | 1 + 4 files changed, 233 insertions(+) delete mode 100644 test/fixtures/database/cache.db delete mode 100644 test/fixtures/database/invalid.db create mode 100644 test/storage/cache_size.cpp diff --git a/test/fixtures/database/cache.db b/test/fixtures/database/cache.db deleted file mode 100644 index 04bbdbb34b8381d7f2c3d12681135ca96ee828f3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4096 zcmeHG(N4lJ6zw)5#)Kyyc)fd*L=z1^;E+OybLcDyJgt=#Sp)~HB)p>E<5&1K{DF^l z8?a?jKR|kuu08kOp5C7IK0eri;4lgoQzoE}N=VbtH2^|Lm!l-d&Wc#u1^$?Ibg|h~ zj4x#9Eu?=KpK?<6?-bxhxmq{1wGq7fd^Hwg<_+0!%)J-s@B%;QD&4r)A{aVO6Ze7s zkV~QKL(hYMR>nT(GAJ&-P|3eM?WQhK&m>AdM~;6AA?P3EaD#JIfuw zEgpc=CE)aI`#i~s1QW}M${Go7W491ZYWXEL4gFvgaDTryCFFuVr>Ac5Y8EVbc0%g) znP3zecDIq@VK|-1tSCpfF9n*oW%XZ>jqKApW(wq*9=$fa?FEA_^-74o87+2nLf3 diff --git a/test/fixtures/database/invalid.db b/test/fixtures/database/invalid.db deleted file mode 100644 index b068b15465aee5d167748150c4eb516d65dd507f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4096 zcmeH{!A=4(5J1~4(Zz(sL=zHE!%0arXt;S2q!8l@ic5m0ZDosYWL;oO!d3kqzrtVf z;>oKA4;B_!AnFH@PCDsKr=8BcZ1?)g@}z)v(D%87JW3#p(Fp)Th$u}cor*dnvn4i?{BD^Pm7(vg>&g;W0-C@I64=4X-K1%z%x%->-P>SzoXbfO%98Md s4b?9upGXZYnt&#-+5|SqZgSxt5WW9zM7~$MaowFJpb0DzAjC}m06E8`=Kufz diff --git a/test/storage/cache_size.cpp b/test/storage/cache_size.cpp new file mode 100644 index 000000000..87f033e71 --- /dev/null +++ b/test/storage/cache_size.cpp @@ -0,0 +1,232 @@ +#include "storage.hpp" + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +bool tileIsCached(mbgl::SQLiteCache* cache, unsigned id) { + using namespace mbgl; + + auto url = std::string("http://tile") + mbgl::util::toString(id); + bool replied = false; + + std::unique_ptr response; + auto callback = [&] (std::unique_ptr res) { + replied = true; + response = std::move(res); + }; + + Resource resource{ Resource::Kind::Tile, url }; + auto req = cache->get(resource, callback); + + while (!replied) { + util::RunLoop::Get()->runOnce(); + } + + return response != nullptr; +} + +void insertTile(mbgl::SQLiteCache* cache, unsigned id, uint64_t size) { + using namespace mbgl; + + auto url = std::string("http://tile") + mbgl::util::toString(id); + + auto response = std::make_shared(); + response->modified = toSeconds(SystemClock::now()); + response->expires = toSeconds(SystemClock::now()) + Seconds(5000); + response->etag = url; + + auto data = std::make_shared(size, 0); + + // Fill data with garbage so SQLite won't try to + // optimize allocation by reusing pages. + static std::mt19937 generator; + std::generate_n(data->begin(), size, generator); + + response->data = data; + + Resource resource{ Resource::Kind::Tile, url }; + cache->put(resource, response, SQLiteCache::Hint::Full); +} + +void refreshTile(mbgl::SQLiteCache* cache, unsigned id) { + using namespace mbgl; + + auto url = std::string("http://tile") + mbgl::util::toString(id); + + auto response = std::make_shared(); + response->modified = toSeconds(SystemClock::now()); + response->expires = toSeconds(SystemClock::now()) + Seconds(5000); + + Resource resource{ Resource::Kind::Tile, url }; + cache->put(resource, response, SQLiteCache::Hint::Refresh); +} + +uint64_t cacheSize(mbgl::SQLiteCache* cache, unsigned entryCount, uint64_t entrySize) { + uint64_t total = 0; + + for (unsigned i = 0; i < entryCount; ++i) { + if (tileIsCached(cache, i)) { + total += entrySize; + } + } + + return total; +} + +TEST_F(Storage, CacheEntrySizeLimit) { + using namespace mbgl; + + util::RunLoop loop; + SQLiteCache cache(":memory:"); + + const uint64_t entrySize = 5 * 1024 * 1024; // 5 MB + + insertTile(&cache, 0, entrySize); + EXPECT_TRUE(tileIsCached(&cache, 0)); + + insertTile(&cache, 1, entrySize + 1); + EXPECT_FALSE(tileIsCached(&cache, 1)); + + insertTile(&cache, 2, entrySize - 1); + EXPECT_TRUE(tileIsCached(&cache, 2)); + + // Setting a new size should not delete existing entries. + cache.setMaximumCacheEntrySize(entrySize / 2); + EXPECT_TRUE(tileIsCached(&cache, 2)); + + insertTile(&cache, 3, entrySize / 2 - 1); + EXPECT_TRUE(tileIsCached(&cache, 3)); + + insertTile(&cache, 4, entrySize); + EXPECT_FALSE(tileIsCached(&cache, 4)); + + cache.setMaximumCacheEntrySize(entrySize * 2); + insertTile(&cache, 5, entrySize); + EXPECT_TRUE(tileIsCached(&cache, 5)); +} + +TEST_F(Storage, CacheSizeSetNewLimit) { + using namespace mbgl; + + util::RunLoop loop; + SQLiteCache cache(":memory:"); + + const unsigned entryCount = 800; + const uint64_t entrySize = 10 * 1024; // 10 KB + + cache.setMaximumCacheEntrySize(entrySize + 1); + + // Cache size defaults to unlimited, all these + // inserts should work. + for (unsigned i = 0; i < entryCount; ++i) { + insertTile(&cache, i, entrySize); + } + + for (unsigned i = 0; i < entryCount; ++i) { + EXPECT_TRUE(tileIsCached(&cache, i)); + } + + uint64_t expectedCacheSize = entryCount * entrySize; + EXPECT_EQ(cacheSize(&cache, entryCount, entrySize), expectedCacheSize); + + // Setting a new size should remove records until the new + // size limit is satisfied. + cache.setMaximumCacheSize(expectedCacheSize / 2); + EXPECT_LT(cacheSize(&cache, entryCount, entrySize), expectedCacheSize / 2); + + // Cache size 1 should practically clean the cache and + // prevent adding any record, although it makes no sense + // to use such size limit IRL. + cache.setMaximumCacheSize(1); + EXPECT_EQ(cacheSize(&cache, entryCount, entrySize), 0); + + insertTile(&cache, 1000, entrySize); + EXPECT_FALSE(tileIsCached(&cache, 1000)); + + // Zero should be treated as unlimited. + cache.setMaximumCacheSize(0); + + for (unsigned i = 0; i < entryCount; ++i) { + insertTile(&cache, i, entrySize); + } + + EXPECT_EQ(cacheSize(&cache, entryCount, entrySize), expectedCacheSize); +} + +TEST_F(Storage, CacheSizePruneLeastAccessed) { + using namespace mbgl; + + util::RunLoop loop; + SQLiteCache cache(":memory:"); + + const unsigned entryCount = 400; + const uint64_t entrySize = 10 * 1024; // 10 KB + + cache.setMaximumCacheEntrySize(entrySize + 1); + cache.setMaximumCacheSize(entrySize * 350); + + for (unsigned i = 0; i < entryCount; ++i) { + insertTile(&cache, i, entrySize); + + if (i == entryCount / 2) { + // We need to sleep for 1s here because + // that is the time resolution for the + // `accessed` time. Then we 'ping' the + // entry, that should update the + // `accessed` time, so it won't get + // pruned when we need more space. + sleep(1); + + EXPECT_TRUE(tileIsCached(&cache, 7)); + + // Refresh should also update the `accessed` + // time of a tile. + refreshTile(&cache, 9); + } + } + + EXPECT_FALSE(tileIsCached(&cache, 6)); + EXPECT_FALSE(tileIsCached(&cache, 8)); + EXPECT_FALSE(tileIsCached(&cache, 10)); + + EXPECT_TRUE(tileIsCached(&cache, 7)); + EXPECT_TRUE(tileIsCached(&cache, 9)); +} + +TEST_F(Storage, CacheSizeStress) { + using namespace mbgl; + + util::RunLoop loop; + SQLiteCache cache(":memory:"); + + const unsigned entryCount = 2000; + const uint64_t entrySize = 10 * 1024; // 10 KB + + cache.setMaximumCacheEntrySize(entrySize + 1); + cache.setMaximumCacheSize(entrySize * 300); + + for (unsigned i = 0; i < entryCount; ++i) { + insertTile(&cache, i, entrySize); + } + + // Should not be in the cache as they were + // first inserted. + EXPECT_FALSE(tileIsCached(&cache, 0)); + EXPECT_FALSE(tileIsCached(&cache, 99)); + EXPECT_FALSE(tileIsCached(&cache, 199)); + EXPECT_FALSE(tileIsCached(&cache, 299)); + EXPECT_FALSE(tileIsCached(&cache, 399)); + + EXPECT_TRUE(tileIsCached(&cache, entryCount - 1)); + + EXPECT_LT(cacheSize(&cache, entryCount, entrySize), entrySize * 300); +} diff --git a/test/test.gypi b/test/test.gypi index 4a570932a..6a6ba507c 100644 --- a/test/test.gypi +++ b/test/test.gypi @@ -71,6 +71,7 @@ 'storage/storage.cpp', 'storage/cache_response.cpp', 'storage/cache_revalidate.cpp', + 'storage/cache_size.cpp', 'storage/database.cpp', 'storage/directory_reading.cpp', 'storage/file_reading.cpp',