tei/MapLibreParticleLab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

core: better handle tile compression in PMTiles sources (#4159)

Browse Source 
Co-authored-by: Bart Louwers <bart@emeel.net>
This commit is contained in:
Tim Sylvester
2026-03-18 16:08:46 -07:00
committed by GitHub
parent 6d8e321be0
commit 9a4fd44c3a
6 changed files with 231 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
platform/default/src/mbgl/storage/pmtiles_file_source.cpp
View File

@@ -166,7 +166,18 @@ public:
response.etag = tileResponse.etag;
if (header.tile_compression == pmtiles::COMPRESSION_GZIP) {
response.data = std::make_shared<std::string>(util::decompress(*tileResponse.data));
// Support uncompressed tiles
if (util::is_compressed(*tileResponse.data)) {
try {
response.data = std::make_shared<std::string>(util::decompress(*tileResponse.data));
} catch (const std::exception& e) {
response.error = std::make_unique<Response::Error>(
Response::Error::Reason::Other,
std::string("Error decompressing PMTiles tile: ") + e.what());
}
} else {
response.data = tileResponse.data;
}
}
ref.invoke(&FileSourceRequest::setResponse, response);

BIN
test/fixtures/storage/pmtiles/corrupt-gzip-tile.pmtiles vendored Normal file
View File

Binary file not shown.

88
test/fixtures/storage/pmtiles/corrupt-gzip-tile.py vendored Normal file
View File

@@ -0,0 +1,88 @@
#!/usr/bin/env python3
"""
Generate corrupt-gzip-tile.pmtiles fixture.
This fixture is a minimal PMTiles v3 archive with:
- tile_compression = COMPRESSION_GZIP (0x2)
- Tile data starts with gzip magic bytes (0x1f 0x8b) but is otherwise corrupt
Used by test PMTilesFileSource.CorruptGzipTile to verify that decompression
failures are converted to error responses instead of propagating as exceptions.
"""
import struct
def write_varint(value):
result = bytearray()
while value >= 0x80:
result.append((value & 0x7f) | 0x80)
value >>= 7
result.append(value)
return bytes(result)
# Tile data: starts with gzip magic bytes (0x1f 0x8b) so is_compressed() returns True,
# but the rest is garbage so util::decompress() throws.
TILE_DATA = b'\x1f\x8b' + b'\xde\xad\xbe\xef' * 4 # 18 bytes of corrupt "gzip"
tile_length = len(TILE_DATA)
n_entries = 1
# Serialize directory: 1 entry, tile_id=0 (z=0,x=0,y=0 → hilbert id 0)
dir_data = write_varint(n_entries)
dir_data += write_varint(0) # tile_id delta
dir_data += write_varint(1) # run_length = 1
dir_data += write_varint(tile_length) # length
dir_data += write_varint(0 + 1) # offset encoded as offset+1
root_dir_offset = 127
root_dir_bytes = len(dir_data)
json_meta_offset = root_dir_offset + root_dir_bytes
json_meta_bytes = 0
leaf_dirs_offset = json_meta_offset
leaf_dirs_bytes = 0
tile_data_offset = json_meta_offset
tile_data_bytes = tile_length
hdr = b'PMTiles'
hdr += struct.pack('<B', 3)
hdr += struct.pack('<Q', root_dir_offset)
hdr += struct.pack('<Q', root_dir_bytes)
hdr += struct.pack('<Q', json_meta_offset) # json_metadata_offset
hdr += struct.pack('<Q', json_meta_bytes) # json_metadata_bytes = 0
hdr += struct.pack('<Q', leaf_dirs_offset) # leaf_dirs_offset
hdr += struct.pack('<Q', leaf_dirs_bytes) # leaf_dirs_bytes = 0
hdr += struct.pack('<Q', tile_data_offset)
hdr += struct.pack('<Q', tile_data_bytes)
hdr += struct.pack('<Q', 1) # addressed_tiles_count
hdr += struct.pack('<Q', 1) # tile_entries_count
hdr += struct.pack('<Q', 1) # tile_contents_count
hdr += struct.pack('<B', 0) # clustered = false
hdr += struct.pack('<B', 1) # internal_compression = NONE
hdr += struct.pack('<B', 2) # tile_compression = GZIP
hdr += struct.pack('<B', 2) # tile_type = PNG
hdr += struct.pack('<B', 0) # min_zoom
hdr += struct.pack('<B', 0) # max_zoom
hdr += struct.pack('<i', -1800000000)
hdr += struct.pack('<i', -900000000)
hdr += struct.pack('<i', 1800000000)
hdr += struct.pack('<i', 900000000)
hdr += struct.pack('<B', 0)
hdr += struct.pack('<i', 0)
hdr += struct.pack('<i', 0)
assert len(hdr) == 127, f"Header is {len(hdr)} bytes"
out = hdr + dir_data + TILE_DATA
# Write in the same directory as this script
import os
script_dir = os.path.dirname(os.path.abspath(__file__))
path = os.path.join(script_dir, 'corrupt-gzip-tile.pmtiles')
with open(path, 'wb') as f:
f.write(out)
print(f"Written {len(out)} bytes to {path}")
print(f" tile_compression = GZIP (0x02) — tile data starts with gzip magic but is corrupt")

BIN
test/fixtures/storage/pmtiles/uncompressed-tiles.pmtiles vendored Normal file
View File

Binary file not shown.

91
test/fixtures/storage/pmtiles/uncompressed-tiles.py vendored Normal file
View File

@@ -0,0 +1,91 @@
#!/usr/bin/env python3
"""
Generate uncompressed-tiles.pmtiles fixture.
This fixture is a minimal PMTiles v3 archive with:
- tile_compression = COMPRESSION_GZIP (0x2) — header claims gzip-compressed tiles
- Actual tile data: raw PNG signature bytes (0x89 0x50...), NOT gzip-wrapped
Used by test PMTilesFileSource.UncompressedTile to verify that the implementation
correctly handles uncompressed tiles within an archive that claims compression.
"""
import struct
def write_varint(value):
result = bytearray()
while value >= 0x80:
result.append((value & 0x7f) | 0x80)
value >>= 7
result.append(value)
return bytes(result)
# Tile data: PNG signature bytes (0x89 0x50...) so is_compressed() returns false.
TILE_DATA = b'\x89PNG\r\n\x1a\n' + b'\x00' * 20 # 28 bytes
tile_length = len(TILE_DATA)
n_entries = 1
# Serialize directory: 1 entry, tile_id=0 (z=0,x=0,y=0 → hilbert id 0)
dir_data = write_varint(n_entries)
dir_data += write_varint(0) # tile_id delta (0 - 0 = 0)
dir_data += write_varint(1) # run_length = 1
dir_data += write_varint(tile_length) # length
dir_data += write_varint(0 + 1) # offset = 0, encoded as offset+1 = 1 (first entry)
root_dir_offset = 127
root_dir_bytes = len(dir_data)
json_meta_offset = root_dir_offset + root_dir_bytes
json_meta_bytes = 0
leaf_dirs_offset = json_meta_offset
leaf_dirs_bytes = 0
tile_data_offset = json_meta_offset # tile data immediately after directory
tile_data_bytes = tile_length
hdr = b'PMTiles'
hdr += struct.pack('<B', 3) # version 3 (offset 7)
hdr += struct.pack('<Q', root_dir_offset) # offset 8
hdr += struct.pack('<Q', root_dir_bytes) # offset 16
hdr += struct.pack('<Q', json_meta_offset) # offset 24
hdr += struct.pack('<Q', json_meta_bytes) # offset 32
hdr += struct.pack('<Q', leaf_dirs_offset) # offset 40
hdr += struct.pack('<Q', leaf_dirs_bytes) # offset 48
hdr += struct.pack('<Q', tile_data_offset) # offset 56
hdr += struct.pack('<Q', tile_data_bytes) # offset 64
hdr += struct.pack('<Q', 1) # addressed_tiles_count (offset 72)
hdr += struct.pack('<Q', 1) # tile_entries_count (offset 80)
hdr += struct.pack('<Q', 1) # tile_contents_count (offset 88)
hdr += struct.pack('<B', 0) # clustered = false (offset 96)
hdr += struct.pack('<B', 1) # internal_compression = NONE (offset 97)
hdr += struct.pack('<B', 2) # tile_compression = GZIP (offset 98) ← KEY
hdr += struct.pack('<B', 2) # tile_type = PNG (offset 99)
hdr += struct.pack('<B', 0) # min_zoom (offset 100)
hdr += struct.pack('<B', 0) # max_zoom (offset 101)
hdr += struct.pack('<i', -1800000000) # min_lon_e7 (offset 102)
hdr += struct.pack('<i', -900000000) # min_lat_e7 (offset 106)
hdr += struct.pack('<i', 1800000000) # max_lon_e7 (offset 110)
hdr += struct.pack('<i', 900000000) # max_lat_e7 (offset 114)
hdr += struct.pack('<B', 0) # center_zoom (offset 118)
hdr += struct.pack('<i', 0) # center_lon_e7 (offset 119)
hdr += struct.pack('<i', 0) # center_lat_e7 (offset 123)
# total = 127 (offset 127)
assert len(hdr) == 127, f"Header is {len(hdr)} bytes, expected 127"
out = hdr + dir_data + TILE_DATA
# Write in the same directory as this script
import os
script_dir = os.path.dirname(os.path.abspath(__file__))
path = os.path.join(script_dir, 'uncompressed-tiles.pmtiles')
with open(path, 'wb') as f:
f.write(out)
print(f"Written {len(out)} bytes to {path}")
print(f" Header: 127 bytes (offset 0)")
print(f" Root dir: {len(dir_data)} bytes (offset {root_dir_offset})")
print(f" Tile data: {tile_length} bytes (offset {tile_data_offset})")
print(f" tile_compression field = 0x02 (GZIP) — but tiles are raw PNG bytes (not gzip)")

40
test/storage/pmtiles_file_source.test.cpp
View File

@@ -103,3 +103,43 @@ TEST(PMTilesFileSource, NonExistentTile) {
loop.run();
}
// A tile whose bytes start with gzip magic but are otherwise corrupt must yield an error
// response — the decompression failure must not propagate as an exception.
TEST(PMTilesFileSource, CorruptGzipTile) {
util::RunLoop loop;
PMTilesFileSource pmtiles(ResourceOptions::Default(), ClientOptions());
std::unique_ptr<AsyncRequest> req = pmtiles.request(
Resource::tile(toAbsoluteURL("corrupt-gzip-tile.pmtiles"), 1.0, 0, 0, 0, Tileset::Scheme::XYZ),
[&](Response res) {
req.reset();
ASSERT_NE(nullptr, res.error);
EXPECT_EQ(Response::Error::Reason::Other, res.error->reason);
EXPECT_NE(res.error->message.find("Error decompressing PMTiles tile:"), std::string::npos);
loop.stop();
});
loop.run();
}
// An archive whose header flags tile_compression=GZIP but whose individual tiles are stored
// without compression must be served without error (uncompressed bytes passed through as-is).
TEST(PMTilesFileSource, UncompressedTile) {
util::RunLoop loop;
PMTilesFileSource pmtiles(ResourceOptions::Default(), ClientOptions());
std::unique_ptr<AsyncRequest> req = pmtiles.request(
Resource::tile(toAbsoluteURL("uncompressed-tiles.pmtiles"), 1.0, 0, 0, 0, Tileset::Scheme::XYZ),
[&](Response res) {
req.reset();
EXPECT_EQ(nullptr, res.error);
ASSERT_TRUE(res.data.get());
ASSERT_EQ(res.noContent, false);
loop.stop();
});
loop.run();
}