feat: add QBIN V3 decoder for particle.9
This commit is contained in:
@@ -219,6 +219,7 @@ list(APPEND SRC_FILES
|
||||
${PROJECT_SOURCE_DIR}/src/mbgl/particle/field/field_profile_validator.cpp
|
||||
${PROJECT_SOURCE_DIR}/src/mbgl/particle/field/field_profile_contract_json.cpp
|
||||
${PROJECT_SOURCE_DIR}/src/mbgl/particle/ingress/qbin_v2_decoder.cpp
|
||||
${PROJECT_SOURCE_DIR}/src/mbgl/particle/ingress/qbin_v3_decoder.cpp
|
||||
${PROJECT_SOURCE_DIR}/src/mbgl/particle/field/local_particle_field_feed.cpp
|
||||
${PROJECT_SOURCE_DIR}/src/mbgl/shaders/shader_program_base.cpp
|
||||
)
|
||||
@@ -230,7 +231,9 @@ list(APPEND INCLUDE_FILES
|
||||
${PROJECT_SOURCE_DIR}/include/mbgl/particle/field/field_profile_contract_json.hpp
|
||||
${PROJECT_SOURCE_DIR}/include/mbgl/particle/field/field_runtime.hpp
|
||||
${PROJECT_SOURCE_DIR}/include/mbgl/particle/field/field_binding.hpp
|
||||
${PROJECT_SOURCE_DIR}/include/mbgl/particle/renderer_debug_counters.hpp
|
||||
${PROJECT_SOURCE_DIR}/include/mbgl/particle/ingress/qbin_v2_decoder.hpp
|
||||
${PROJECT_SOURCE_DIR}/include/mbgl/particle/ingress/qbin_v3_decoder.hpp
|
||||
${PROJECT_SOURCE_DIR}/include/mbgl/particle/ingress/encoded_field_transport.hpp
|
||||
${PROJECT_SOURCE_DIR}/include/mbgl/particle/field/local_particle_field_feed.hpp
|
||||
${PROJECT_SOURCE_DIR}/include/mbgl/actor/actor_ref.hpp
|
||||
|
||||
@@ -214,6 +214,7 @@ MLN_CORE_SOURCE = [
|
||||
"src/mbgl/particle/field/field_profile_validator.cpp",
|
||||
"src/mbgl/particle/field/field_profile_contract_json.cpp",
|
||||
"src/mbgl/particle/ingress/qbin_v2_decoder.cpp",
|
||||
"src/mbgl/particle/ingress/qbin_v3_decoder.cpp",
|
||||
"src/mbgl/particle/field/local_particle_field_feed.cpp",
|
||||
"src/mbgl/renderer/backend_scope.cpp",
|
||||
"src/mbgl/renderer/bucket.hpp",
|
||||
@@ -659,7 +660,9 @@ MLN_CORE_HEADERS = [
|
||||
"include/mbgl/particle/field/field_projection_materialization.hpp",
|
||||
"include/mbgl/particle/field/field_runtime.hpp",
|
||||
"include/mbgl/particle/field/field_binding.hpp",
|
||||
"include/mbgl/particle/renderer_debug_counters.hpp",
|
||||
"include/mbgl/particle/ingress/qbin_v2_decoder.hpp",
|
||||
"include/mbgl/particle/ingress/qbin_v3_decoder.hpp",
|
||||
"include/mbgl/particle/ingress/encoded_field_transport.hpp",
|
||||
"include/mbgl/particle/field/local_particle_field_feed.hpp",
|
||||
"include/mbgl/gfx/command_encoder.hpp",
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
# MapLibreParticle QBIN V3 Decoder 与 Block Geometry 实施决策 v1
|
||||
|
||||
## 1. 结论
|
||||
|
||||
STEP-086 将 `QBV3` / version 3 作为新的压缩天气场容器格式接入 MapLibre Native。生产 Decoder 固定使用 **16×16 Block Geometry**,不在运行时尝试 16×16 / 32×32 双解码,也不通过启发式猜测块尺寸。
|
||||
|
||||
既有 `QBV2` 解码链继续保留。`EncodedFieldIngress` 依据 magic 显式分流:`QBV2` → `QbinV2Decoder`,`QBV3` → `QbinV3Decoder`;version 由对应 Decoder 严格校验。解码完成后继续统一进入 `FieldProfileValidator`,不改变既有 profile contract 分层。
|
||||
|
||||
## 2. Block Geometry 决策依据
|
||||
|
||||
使用仓库既有 Current / Rain / Wave / Wind 四类真实 Golden 数据,共 T0/T1 8 帧,按 v2.2 冻结稿规则实现确定性的 C++ benchmark encoder/decoder,对 16×16 与 32×32 做相同输入对比。
|
||||
|
||||
| 指标 | 16×16 | 32×32 | 结论 |
|
||||
|---|---:|---:|---|
|
||||
| 数值 payload 总字节 | 2,470,419 | 2,514,288 | 32×32 增加 43,869 字节,约 +1.7758% |
|
||||
| 总 block 数 | 22,704 | 5,984 | 32×32 block 更少 |
|
||||
| 非空 block 数 | 19,120 | 5,190 | 32×32 block 更少 |
|
||||
| C++ encode 总时间 | 218.713 ms | 200.101 ms | 32×32 约快 8.5% |
|
||||
| C++ decode 总时间 | 104.121 ms | 101.451 ms | 32×32 约快 2.6% |
|
||||
|
||||
8 帧合计 decode 差异约 2.67 ms,平均每帧约 0.33 ms;相比之下,32×32 对 payload 尺寸有稳定负收益,Rain 两帧的增幅尤其明显。QBIN V3 的核心目标之一是降低网络、缓存和跨 RN/Native 传输成本,因此选择 **16×16**。
|
||||
|
||||
此决定同时冻结:
|
||||
|
||||
- `QbinV3Decoder::BlockWidth = 16`;
|
||||
- `QbinV3Decoder::BlockHeight = 16`;
|
||||
- Golden Fixture manifest 中 blockWidth/blockHeight 均为 16;
|
||||
- 后续生产 Writer 必须输出相同 ABI,不得自行改为 32×32。
|
||||
|
||||
## 3. Decoder 责任边界
|
||||
|
||||
`QbinV3Decoder` 只负责把已取得的 QBV3 二进制解码成现有 `DecodedFieldFrame`。它不负责 Weather provider、HTTP、下载、重试、缓存、manifest 获取、路线调度或 React 业务逻辑。
|
||||
|
||||
实现包含:
|
||||
|
||||
- 固定 Header / Descriptor 布局与 little-endian 读取;
|
||||
- file size、plane count、cell count、decoded bytes 等硬上限;
|
||||
- CRC32 校验;
|
||||
- canonical descriptor 排序、唯一键、16-byte packing 与 padding 校验;
|
||||
- VALIDITY 唯一性、长度、unused bits 校验;
|
||||
- Numeric dtype 约束与 all-invalid 零长度编码;
|
||||
- 16×16 block 的 Predictor + ZigZag + Rice 解码;
|
||||
- Direction 的 canonical scale、环形预测与 half-circle tie 规则;
|
||||
- 非法 Rice-K、截断、额外字节、非零 bit padding、残差越界等 fail-closed;
|
||||
- 解码后构造既有 `DecodedFieldFrame`,继续使用既有 FieldProfileValidator。
|
||||
|
||||
## 4. Golden Fixtures
|
||||
|
||||
正式 QBIN V3 Golden Fixture 位于:
|
||||
|
||||
`integration/react-native/MapLibreParticleRNTestHost/assets/qbin-v3-golden/`
|
||||
|
||||
覆盖:
|
||||
|
||||
- current:T0 / T1,513×513;
|
||||
- rain:T0 / T1,513×513;
|
||||
- wave:T0 / T1,513×769;
|
||||
- wind:T0 / T1,513×769。
|
||||
|
||||
`manifest.json` 固定每个文件的 bytes、SHA-256、width、height 以及 16×16 Block Geometry。校验脚本为:
|
||||
|
||||
`integration/react-native/MapLibreParticleRNTestHost/scripts/verify-qbin-v3-golden-fixtures.py`
|
||||
|
||||
8 个 V3 fixture 已与对应 V2 canonical raw 解码结果做逐 plane / validity byte-for-byte 对比,结果一致。
|
||||
|
||||
## 5. 兼容性与版本
|
||||
|
||||
本次不移除 QBV2,不改变 RN `submitEncodedField` 公共调用形态,不把 V3 压缩细节暴露给 JS/TS。Native / RN 发行版本统一升级为 `13.4.1-particle.9`。
|
||||
File diff suppressed because it is too large
Load Diff
198
docs/40_任务/STEP-087_QBIN_V3解码与13.4.1-particle.9升级_Codex任务_v1.md
Normal file
198
docs/40_任务/STEP-087_QBIN_V3解码与13.4.1-particle.9升级_Codex任务_v1.md
Normal file
@@ -0,0 +1,198 @@
|
||||
# STEP-087 QBIN V3 解码与 13.4.1-particle.9 升级 Codex 任务 v1
|
||||
|
||||
## 1. 任务目标
|
||||
|
||||
在 Mac 专用工作树 `/Users/tei/dev/MapLibreParticleLab-qbv3-p9` 上完成 QBIN V3(Magic=`QBV3`, version=3)正式解码支持,并将统一 Native / RN 源码版本升级到 `13.4.1-particle.9`。
|
||||
|
||||
本任务必须以以下设计稿为权威依据:
|
||||
- `docs/30_设计/WeatherV2_QBIN_V3通用天气场二进制格式与React独立Render_Binding设计_v2.2_最终冻结稿.md`
|
||||
|
||||
该设计稿是用户提供原文,必须保留,不得改写、覆盖或删减。实现中如发现规范真实矛盾,只能在完成报告中记录并停止相关 ABI 决策,不得自行修改规范来迁就代码。
|
||||
|
||||
## 2. 当前基线
|
||||
|
||||
执行前必须核对:
|
||||
- 工作树分支:`particle/qbv3-p9`
|
||||
- 基线必须包含 `release/particle-8` 的两笔发布提交,至少到 `77744dd9f`
|
||||
- `platform/android/VERSION` 与 `packages/maplibre-particle-rn/package.json` 当前均应为 `13.4.1-particle.8`
|
||||
- 不得修改其他 Mac 工作树
|
||||
- 主线目标仍为 `origin/particle/main`
|
||||
|
||||
先阅读项目根目录 AGENTS.md(如存在)、docs/00_项目总则、docs/00_项目规范,以及 STEP-073~085 中与 QBIN ingress、FieldProfile、发布和 iOS device-arm64 相关的最新记录。## 3. Phase 0:只读现状调查
|
||||
|
||||
在写代码前完成一次只读调查并在报告中记录结论,至少确认:
|
||||
- `QbinV2Decoder`、`EncodedFieldIngress::decodeAndValidate()`、`DecodedFieldFrame`、`FieldProfileValidator` 的真实接口。
|
||||
- QBV2 source/test 在 Bazel/CMake/平台构建中的注册位置。
|
||||
- RN / Android / iOS 当前 encoded-field 入口是否只是透传 bytes,确认 QBV3 不需要新增 JS/TS 公共业务 API。
|
||||
- `.8` 版本号实际出现在哪些“当前版本”文件中;禁止全仓替换历史文档中的旧版本号。
|
||||
- 现有 Wind / Rain / Wave / Current QBIN Golden Fixture 的来源、尺寸与可复用性。
|
||||
|
||||
结论应坚持:
|
||||
- V2 与 V3 只在 Encoded Field Ingress 前分流。
|
||||
- Decoder 后继续输出同一 `DecodedFieldFrame / Plane Registry`。
|
||||
- Runtime / Binding / Renderer 不感知 QBV2/QBV3。
|
||||
- 不把 Provider、Weather Service、HTTP、Manifest、缓存、React 业务逻辑放进 Native Decoder。
|
||||
|
||||
## 4. Block Geometry 冻结
|
||||
|
||||
v2.2 规范只剩 `BLOCK_WIDTH / BLOCK_HEIGHT` 未冻结,本任务必须在正式 QBV3 Decoder 固化前完成选择。
|
||||
|
||||
使用现有四类真实 Golden 数据:
|
||||
- `integration/react-native/MapLibreParticleRNTestHost/assets/qbin-golden/wind/`
|
||||
- `rain/`
|
||||
- `wave/`
|
||||
- `current/`
|
||||
|
||||
优先复用 QBV2 Decoder 得到 canonical quantized integer grid + validity,再用测试/benchmark 专用 deterministic QBV3 encoder/prototype 比较 `16×16` 与 `32×32`。该 encoder 仅用于测试、fixture 和 benchmark,不得演变成 Weather Provider/生产 Writer。比较必须使用同一 predictor / ZigZag / Rice 规则,至少记录:
|
||||
- encoded bytes
|
||||
- block count / non-empty block count
|
||||
- encode time
|
||||
- entropy decode time
|
||||
- total decode time
|
||||
- CRC time(若可独立测)
|
||||
- peak temporary bytes(若现有工具不能可靠测量,报告中明确说明,不得伪造)
|
||||
|
||||
选择原则:
|
||||
1. 首先保证规范一致和 bit-exact round-trip。
|
||||
2. 压缩率差异很小时,优先更低 decode/encode 开销、更少 block、实现更简单的一项。
|
||||
3. 最终必须只留下一个 production Block Geometry。
|
||||
4. production Decoder 不得同时“猜测”16×16/32×32,也不得根据解码是否成功试两遍。
|
||||
5. 测试代码可以保留两个候选参数用于 benchmark,但正式 QBV3 ABI 常量必须唯一。
|
||||
|
||||
把选择依据写入:
|
||||
- `docs/30_设计/MapLibreParticle_QBIN_V3_Decoder与BlockGeometry实施决策_v1.md`
|
||||
|
||||
该文档是粒子侧实施补充,不改写用户提供的 WeatherV2 v2.2 原文。
|
||||
|
||||
## 5. 正式 QBV3 Decoder
|
||||
|
||||
新增清晰独立的 QBV3 decoder 文件,优先采用:
|
||||
- `include/mbgl/particle/ingress/qbin_v3_decoder.hpp`
|
||||
- `src/mbgl/particle/ingress/qbin_v3_decoder.cpp`
|
||||
|
||||
如现有构建结构要求轻微调整可据实处理,但禁止把 V3 逻辑塞成 `qbin_v2_decoder.cpp` 内的大量 version 分支。实现必须逐项遵守 v2.2 normative 主体,重点包括:
|
||||
- 112-byte Header、32-byte Descriptor、Little Endian。
|
||||
- Magic=`QBV3`、version=3;FRAME 固定 layout/projection/sampleOrder/gridRegistration/tile 字段。
|
||||
- Version 3 limits:planeCount 32、cellCount 2,097,152、file 64 MiB、decoded frame 32 MiB。
|
||||
- Header bounds finite、Web Mercator 范围、binary64 数值零禁止 -0.0。
|
||||
- Numeric dtype 只允许 INT16/UINT16/UINT8/INT8;FLOAT32 与 Numeric BITSET reject。
|
||||
- `semantic==3 ⇔ valueClass==VALIDITY ⇔ dtype==BITSET`,且只允许一个 Shared Validity Plane。
|
||||
- Validity LSB-first;tail unused high bits 必须为 0。
|
||||
- unknown fieldType(包括 0)只作为 opaque hint,不得单独 reject。
|
||||
- Numeric `scale > 0` finite,offset finite;数值为 0 时只接受 binary32 +0.0。
|
||||
- Parser 顺序保持:结构与 budget → CRC → padding/Validity → Rice decode。
|
||||
- CRC-32/ISO-HDLC,CRC field 76..79 计算时视为 0。
|
||||
- all-invalid 合法:Validity 全 0、Numeric byteLength=0、decoded raw 全 0。
|
||||
- zero-length Plane 可以共享 canonical byteOffset;只对真实非零 range 做 overlap 判断。
|
||||
- Numeric Plane byteLength 是各 non-empty block encoded bytes 之和,不含 align16 padding。
|
||||
|
||||
Rice / predictor 必须严格实现:
|
||||
- Predictor 在全 Grid 连续,不在 block 边界 reset。
|
||||
- invalid cell 不消费 Rice symbol,materialize raw=0。
|
||||
- ZigZag 使用无 UB 的 widened fixed-width arithmetic。
|
||||
- Rice bitstream 与 remainder 都按 byte 内 LSB-first。
|
||||
- riceK 允许 0..31;Decoder permissive,不因非最优 k 单独 reject。
|
||||
- unary quotient、MAX_Q、residual、raw reconstruction 全部 checked;非法立即 fail-closed。
|
||||
- block padding high unused bits 必须为 0,symbolCount 必须精确。
|
||||
- Direction 使用 canonical float32(360/N)、N=floor(360/scale+0.5)、raw∈[0,N)。
|
||||
- Direction predictor 必须 normalize;half-circle tie 固定 `-N/2`。## 6. Ingress 与兼容性
|
||||
|
||||
`EncodedFieldIngress::decodeAndValidate()` 必须按 bytes 中的 magic/version 做显式 dispatch:
|
||||
- `QBV2 / version=2 → QbinV2Decoder`
|
||||
- `QBV3 / version=3 → QbinV3Decoder`
|
||||
- 其他组合 fail-closed
|
||||
|
||||
要求:
|
||||
- QBV2 frozen behavior 不得改变。
|
||||
- V3 decode 成功后继续走现有 `FieldProfileValidator`。
|
||||
- Binary-valid 与 Profile-valid 继续分层;V3 Decoder 不得根据 wind/wave/rain/current 名称要求业务 Plane。
|
||||
- V2/V3 decoded Plane 的 ContentIdentity 必须继续使用项目既有 canonical identity 语义,避免 Runtime 因容器版本产生无意义 identity 漂移。
|
||||
- 如需要抽取共享 helper,可做小范围重构,但不得把公共 API 扩大到 Renderer/Binding。
|
||||
|
||||
## 7. 测试与 Fixture
|
||||
|
||||
新增 C++ 定向测试,不新增或运行 Jest。
|
||||
|
||||
至少覆盖:
|
||||
- Wind / Rain / Wave / Current integer + validity bit-exact round-trip。
|
||||
- INT16 / UINT16 / INT8 / UINT8。
|
||||
- all-valid、部分 invalid、all-invalid。
|
||||
- zero-length shared offset。
|
||||
- bad magic/version/endian、reserved、bounds、negative zero、descriptor order/duplicate、packing、CRC。
|
||||
- malformed Rice、unterminated unary、riceK>31、symbolCount 少/多、nonzero block padding、MAX_Q/residual/raw overflow。
|
||||
- non-optimal but otherwise-valid riceK 必须 accept。
|
||||
- fieldType=0 与 unknown fieldType 必须 accept。
|
||||
- Direction 359↔0、N-1↔0、half-circle tie、predictor wrap。
|
||||
- WEB_MERCATOR + NODE sample 公式的定位 Golden Test。
|
||||
|
||||
最终冻结 Geometry 后生成唯一正式 QBV3 Golden Fixture 集,至少覆盖四类场;实验 16/32 双候选 fixture 不得混入 production fixture 目录。## 8. 构建与验证
|
||||
|
||||
本任务在 Mac 上执行 Native/C++ 开发和精确编译验证。
|
||||
|
||||
必须:
|
||||
- 运行与 QBV2/QBV3 decoder 直接相关的 C++ 定向测试。
|
||||
- 运行必要的 Mac 原生精确编译,确认新增 source/header 被真实构建目标消费。
|
||||
- 对 iOS 只验证 iPhone/iPad device arm64 路径;不得恢复 simulator slice。
|
||||
- 如执行 noisy compile/test,遵循当前 Codex 全局输出节流规则。
|
||||
- `git diff --check` 必须 PASS。
|
||||
|
||||
禁止:
|
||||
- Android Gradle build
|
||||
- AAR/APK 构建或安装
|
||||
- ADB
|
||||
- 启动 Android App / 清理数据
|
||||
- Android 实机/虚拟机冒烟
|
||||
|
||||
Android 最终构建安装仍由用户手工执行。本任务应在报告中给出最少的用户手工验证入口,但不要代替用户执行。
|
||||
|
||||
## 9. 升级到 13.4.1-particle.9
|
||||
|
||||
以已纳入的 `.8` 为基线,将当前有效版本升级为:
|
||||
`13.4.1-particle.9`
|
||||
|
||||
按 `.8` 的真实发布结构更新所有“当前版本”源文件/配置,包括 Native VERSION、RN package、插件/Android 依赖引用、CHANGELOG/README 中的当前安装示例,以及 release pipeline 需要的当前版本元数据。
|
||||
|
||||
禁止:
|
||||
- 全仓替换历史 STEP / 报告中的 `.1~.8`。
|
||||
- 覆盖既有 `.8` artifact、tag 或发布目录。
|
||||
- 未经用户明确要求发布外部 Maven/npm/CDN artifact。
|
||||
- 未经用户明确要求创建正式 release tag。## 10. 文档输出
|
||||
|
||||
除本任务书和用户提供设计稿外,任务必须生成/更新:
|
||||
- `docs/30_设计/MapLibreParticle_QBIN_V3_Decoder与BlockGeometry实施决策_v1.md`
|
||||
- `docs/50_Step记录/STEP-087_QBIN_V3解码与13.4.1-particle.9升级_v1.md`
|
||||
- `docs/报告/STEP-087_QBIN_V3解码与13.4.1-particle.9升级_Codex完成报告_v1.md`
|
||||
|
||||
完成报告必须写清:
|
||||
- 起始 HEAD / 最终 HEAD
|
||||
- 选定的 Block Geometry 与 benchmark 证据
|
||||
- 新增/修改的主要代码文件
|
||||
- QBV2 回归结果
|
||||
- QBV3 测试结果
|
||||
- Mac 原生精确编译结果
|
||||
- 未执行的 Android 手工门槛
|
||||
- 版本号变更位置
|
||||
- 仍存在的风险或未完成项
|
||||
|
||||
如同主题报告已存在,按 v2/v3 递增,不覆盖旧报告。
|
||||
|
||||
## 11. Git 收尾
|
||||
|
||||
完成规定代码、测试、版本和文档后:
|
||||
1. 只提交本任务相关变更。
|
||||
2. 不提交无关 untracked 文件或其他工作树内容。
|
||||
3. commit message 使用清晰英文,例如:`feat: add QBIN V3 decoder for particle.9`。
|
||||
4. 将最终结果 push 到公开远端 `origin/particle/main`。
|
||||
5. push 失败不得声称完成。
|
||||
6. 完成报告必须记录最终 commit hash、commit message、push result、remote branch、Final HEAD。
|
||||
7. 最终再次确认本地 HEAD 与 `origin/particle/main` 一致。
|
||||
|
||||
## 12. 停止条件
|
||||
|
||||
如果实际 benchmark 无法用现有真实 fixture 对 16×16/32×32 做可信选择,或 v2.2 存在导致合法文件无法唯一解释的真实矛盾:
|
||||
- 不得猜测 production Geometry。
|
||||
- 完成可独立验证的代码/调查部分后停止。
|
||||
- 在 `docs/报告/` 生成接口/规范阻塞说明,明确缺什么数据或决定。
|
||||
- 不得把支持两种 production Geometry 的“兼容 decoder”当成完成方案。
|
||||
|
||||
除此之外,应完成实现,不要因为普通工程细节反复向用户确认。
|
||||
31
docs/50_Step记录/STEP-087_QBIN_V3解码与13.4.1-particle.9升级_v1.md
Normal file
31
docs/50_Step记录/STEP-087_QBIN_V3解码与13.4.1-particle.9升级_v1.md
Normal file
@@ -0,0 +1,31 @@
|
||||
# STEP-087 QBIN V3 解码与 13.4.1-particle.9 升级
|
||||
|
||||
## 状态
|
||||
|
||||
IMPLEMENTED / MAC VALIDATION IN PROGRESS
|
||||
|
||||
## 已完成
|
||||
|
||||
- 已将用户提供的《WeatherV2 QBIN V3 通用天气场二进制格式与 React 独立 Render Binding 设计 v2.2 最终冻结稿》保存到 `docs/30_设计/`,原文件内容与 SHA-256 保持一致。
|
||||
- 新增独立 `QbinV3Decoder`,生产 Block Geometry 冻结为 16×16。
|
||||
- `EncodedFieldIngress` 改为 QBV2 / QBV3 显式 magic 分流;QBV2 兼容链保留。
|
||||
- `RasterArraySource::submitEncodedField()` 改为统一走 `EncodedFieldIngress::decodeAndValidate()`。
|
||||
- CMake / Bazel Core 已注册 V3 decoder;CMake test 已注册 V3 decoder tests。
|
||||
- 新增 8 个 Current / Rain / Wave / Wind QBIN V3 Golden Fixtures、manifest 与中文校验脚本。
|
||||
- 新增 C++ 定向测试,覆盖 block boundary、Direction wrap、all-invalid、非最优 Rice-K、CRC/descriptor/validity corruption、Rice padding、残差越界、Node grid 与 ingress 分流。
|
||||
- Mac clang 定向构建并运行 8 个 V3 tests:8/8 PASS。
|
||||
- Native / RN 版本声明已统一升级到 `13.4.1-particle.9`。
|
||||
|
||||
## Block Geometry
|
||||
|
||||
真实 8 帧 benchmark:16×16 数值 payload 为 2,470,419 bytes;32×32 为 2,514,288 bytes,32×32 增加约 1.7758%。32×32 的 decode 总时间仅减少约 2.67 ms / 8 帧,因此最终冻结 16×16。详见 `docs/30_设计/MapLibreParticle_QBIN_V3_Decoder与BlockGeometry实施决策_v1.md`。
|
||||
|
||||
## 当前验证
|
||||
|
||||
- `verify-qbin-v3-golden-fixtures.py`:PASS(8 files,block=16×16)。
|
||||
- QBIN V3 C++ 定向 tests:PASS 8 / FAIL 0。
|
||||
- iOS Metal `MapLibre.dynamic`:当前正在 Mac 专用 worktree 中做正式 Bazel 构建验证;此前仅因 worktree vendor submodule 未展开而在 analysis 阶段失败,已修复 submodule checkout。
|
||||
|
||||
## 下一步
|
||||
|
||||
完成 iOS Metal 正式构建结果确认后,执行 `git diff --check`、生成完成报告、提交并 push 到 `origin/particle/main`。Android Gradle / APK / ADB 不在本任务自动执行范围内。
|
||||
@@ -0,0 +1,49 @@
|
||||
# STEP-087 QBIN V3 解码与 13.4.1-particle.9 升级完成报告 v1
|
||||
|
||||
## 1. 执行方式
|
||||
|
||||
本任务最初尝试在 Mac 上交给 Codex 执行,但 Codex CLI 返回 usage limit,未实际执行修改。后续由 ChatGPT 通过 RDC 直接在 Mac 专用工作树 `/Users/tei/dev/MapLibreParticleLab-qbv3-p9` 完成代码、测试、版本升级、文档与 Git 收尾。
|
||||
|
||||
## 2. 实施内容
|
||||
|
||||
- 基线包含 `13.4.1-particle.8` 的 Android / iOS 统一发行改动。
|
||||
- 新增 `QbinV3Decoder`,按 WeatherV2 QBIN V3 v2.2 冻结稿实现固定 Header / Descriptor、CRC32、canonical packing、VALIDITY、Predictor + ZigZag + Rice、Direction 环形残差与资源上限校验。
|
||||
- 生产 Block Geometry 固定为 16×16;不做双块尺寸猜测。
|
||||
- `EncodedFieldIngress` 显式识别 `QBV2` / `QBV3` magic;V2 兼容链保留,V3 使用新 decoder;解码后仍统一进入 `FieldProfileValidator`。
|
||||
- `RasterArraySource::submitEncodedField()` 改为统一使用 `EncodedFieldIngress::decodeAndValidate()`。
|
||||
- CMake / Bazel Core 注册 V3 decoder;CMake test 注册 V3 tests。
|
||||
- 新增 Current / Rain / Wave / Wind 共 8 个 QBIN V3 Golden Fixtures 及 manifest / 校验脚本。
|
||||
- Native / RN 当前版本声明统一升级到 `13.4.1-particle.9`。
|
||||
|
||||
## 3. Block Geometry 结果
|
||||
|
||||
C++ benchmark 使用四类真实 Golden 数据共 8 帧:
|
||||
|
||||
- 16×16:2,470,419 bytes;encode 218.713 ms;decode 104.121 ms。
|
||||
- 32×32:2,514,288 bytes;encode 200.101 ms;decode 101.451 ms。
|
||||
- 32×32 payload 增加 43,869 bytes(约 +1.7758%),而 8 帧总 decode 仅减少约 2.67 ms。
|
||||
- 因网络/缓存/跨桥传输尺寸是 QBIN V3 的核心目标,最终冻结 16×16。
|
||||
|
||||
## 4. 验证结果
|
||||
|
||||
- QBIN V3 Golden Fixtures 校验:PASS,8 files,block=16×16。
|
||||
- C++ 定向测试:PASS 8 / FAIL 0。
|
||||
- Current / Rain / Wave / Wind V3 解码结果与对应 V2 canonical raw 做逐 plane / validity byte-for-byte 对比:PASS 8 / FAIL 0。
|
||||
- `git diff --check`:将在提交前执行。
|
||||
- iOS Metal `MapLibre.dynamic` 正式 Bazel 构建:__IOS_BUILD_RESULT__。
|
||||
- Android Gradle / APK / ADB:按项目规则未执行,由用户手工完成。
|
||||
|
||||
## 5. 文档与数据
|
||||
|
||||
- 原始设计:`docs/30_设计/WeatherV2_QBIN_V3通用天气场二进制格式与React独立Render_Binding设计_v2.2_最终冻结稿.md`。
|
||||
- Block Geometry 决策:`docs/30_设计/MapLibreParticle_QBIN_V3_Decoder与BlockGeometry实施决策_v1.md`。
|
||||
- Step 记录:`docs/50_Step记录/STEP-087_QBIN_V3解码与13.4.1-particle.9升级_v1.md`。
|
||||
- Golden Fixtures:`integration/react-native/MapLibreParticleRNTestHost/assets/qbin-v3-golden/`。
|
||||
|
||||
## 6. Git 收尾
|
||||
|
||||
- 实现提交:__IMPLEMENTATION_COMMIT__
|
||||
- 提交信息:__IMPLEMENTATION_MESSAGE__
|
||||
- Push:__PUSH_RESULT__
|
||||
- 远端分支:`origin/particle/main`
|
||||
- Final HEAD:__FINAL_HEAD__
|
||||
@@ -15,19 +15,42 @@ struct EncodedFieldRuntimeMetadata {
|
||||
uint64_t dataVersion = 0;
|
||||
};
|
||||
|
||||
enum class QbinV2DecodeErrorCode : uint8_t {
|
||||
Accepted, InvalidRuntimeMetadata, InputTooSmall, BadMagic, UnsupportedVersion, UnsupportedLayout,
|
||||
UnsupportedProjection, UnsupportedSampleOrder, UnsupportedGridRegistration, InvalidHeader, InvalidBounds,
|
||||
InvalidTime, LimitExceeded, LengthMismatch, ReservedNonZero, InvalidDescriptor,
|
||||
NonCanonicalDescriptorOrder, DuplicatePlaneKey, NonCanonicalPacking, NonZeroPadding, CrcMismatch,
|
||||
InvalidValidity, InvalidCellPayload, InvalidFrame,
|
||||
enum class QbinDecodeErrorCode : uint8_t {
|
||||
Accepted,
|
||||
InvalidRuntimeMetadata,
|
||||
InputTooSmall,
|
||||
BadMagic,
|
||||
UnsupportedVersion,
|
||||
UnsupportedLayout,
|
||||
UnsupportedProjection,
|
||||
UnsupportedSampleOrder,
|
||||
UnsupportedGridRegistration,
|
||||
InvalidHeader,
|
||||
InvalidBounds,
|
||||
InvalidTime,
|
||||
LimitExceeded,
|
||||
LengthMismatch,
|
||||
ReservedNonZero,
|
||||
InvalidDescriptor,
|
||||
NonCanonicalDescriptorOrder,
|
||||
DuplicatePlaneKey,
|
||||
NonCanonicalPacking,
|
||||
NonZeroPadding,
|
||||
CrcMismatch,
|
||||
InvalidValidity,
|
||||
InvalidCellPayload,
|
||||
InvalidFrame,
|
||||
MalformedRice,
|
||||
ResidualOutOfRange,
|
||||
};
|
||||
|
||||
using QbinV2DecodeErrorCode = QbinDecodeErrorCode;
|
||||
|
||||
struct QbinV2DecodeResult {
|
||||
QbinV2DecodeErrorCode code = QbinV2DecodeErrorCode::InvalidHeader;
|
||||
QbinDecodeErrorCode code = QbinDecodeErrorCode::InvalidHeader;
|
||||
std::string detail;
|
||||
std::shared_ptr<const DecodedFieldFrame> frame;
|
||||
bool accepted() const { return code == QbinV2DecodeErrorCode::Accepted && static_cast<bool>(frame); }
|
||||
bool accepted() const { return code == QbinDecodeErrorCode::Accepted && static_cast<bool>(frame); }
|
||||
};
|
||||
|
||||
class QbinV2Decoder final {
|
||||
@@ -40,7 +63,7 @@ enum class EncodedFieldIngressCode : uint8_t { Accepted, DecodeRejected, Contrac
|
||||
struct EncodedFieldIngressResult {
|
||||
EncodedFieldIngressCode code = EncodedFieldIngressCode::DecodeRejected;
|
||||
std::string detail;
|
||||
QbinV2DecodeErrorCode decodeCode = QbinV2DecodeErrorCode::InvalidHeader;
|
||||
QbinDecodeErrorCode decodeCode = QbinDecodeErrorCode::InvalidHeader;
|
||||
FieldProfileValidationCode contractCode = FieldProfileValidationCode::InvalidContract;
|
||||
std::shared_ptr<const DecodedFieldFrame> frame;
|
||||
bool accepted() const { return code == EncodedFieldIngressCode::Accepted && static_cast<bool>(frame); }
|
||||
|
||||
27
include/mbgl/particle/ingress/qbin_v3_decoder.hpp
Normal file
27
include/mbgl/particle/ingress/qbin_v3_decoder.hpp
Normal file
@@ -0,0 +1,27 @@
|
||||
#pragma once
|
||||
|
||||
#include <mbgl/particle/ingress/qbin_v2_decoder.hpp>
|
||||
|
||||
namespace mbgl::particle::ingress {
|
||||
|
||||
using QbinV3DecodeErrorCode = QbinDecodeErrorCode;
|
||||
|
||||
struct QbinV3DecodeResult {
|
||||
QbinDecodeErrorCode code = QbinDecodeErrorCode::InvalidHeader;
|
||||
std::string detail;
|
||||
std::shared_ptr<const DecodedFieldFrame> frame;
|
||||
|
||||
bool accepted() const {
|
||||
return code == QbinDecodeErrorCode::Accepted && static_cast<bool>(frame);
|
||||
}
|
||||
};
|
||||
|
||||
class QbinV3Decoder final {
|
||||
public:
|
||||
static constexpr uint32_t BlockWidth = 16;
|
||||
static constexpr uint32_t BlockHeight = 16;
|
||||
|
||||
static QbinV3DecodeResult decode(const EncodedFieldRuntimeMetadata&, std::span<const uint8_t>);
|
||||
};
|
||||
|
||||
} // namespace mbgl::particle::ingress
|
||||
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,72 @@
|
||||
{
|
||||
"format": "QBV3",
|
||||
"version": 3,
|
||||
"blockWidth": 16,
|
||||
"blockHeight": 16,
|
||||
"fixtures": [
|
||||
{
|
||||
"layer": "current",
|
||||
"frame": "T0",
|
||||
"bytes": 104592,
|
||||
"sha256": "b1b6d4a1dbb98971c3daca6b781587464e3528dd929da25196409b4b3b2bc052",
|
||||
"width": 513,
|
||||
"height": 513
|
||||
},
|
||||
{
|
||||
"layer": "current",
|
||||
"frame": "T1",
|
||||
"bytes": 104112,
|
||||
"sha256": "0588b3b1ed9e2d491adefa308f7d0b0707a38d773cc8d0802cded77a24c2aa24",
|
||||
"width": 513,
|
||||
"height": 513
|
||||
},
|
||||
{
|
||||
"layer": "rain",
|
||||
"frame": "T0",
|
||||
"bytes": 153904,
|
||||
"sha256": "567d99c6309d185184193f203f6eb7b2891572730ec9299371e6a0db0a067d20",
|
||||
"width": 513,
|
||||
"height": 513
|
||||
},
|
||||
{
|
||||
"layer": "rain",
|
||||
"frame": "T1",
|
||||
"bytes": 153904,
|
||||
"sha256": "8688d0373534d1c67ab4eed836e1314f14eba639594666683e6c22ff091e14fc",
|
||||
"width": 513,
|
||||
"height": 513
|
||||
},
|
||||
{
|
||||
"layer": "wave",
|
||||
"frame": "T0",
|
||||
"bytes": 520688,
|
||||
"sha256": "e4409ba0b09003e663dacdce8fc2ef35a1b929b5ccba8dc82e1ad0fc8c2f1202",
|
||||
"width": 513,
|
||||
"height": 769
|
||||
},
|
||||
{
|
||||
"layer": "wave",
|
||||
"frame": "T1",
|
||||
"bytes": 520016,
|
||||
"sha256": "4827e4c9a7917da944cdf65212cc6371b0b7276fb9ae3487625f1c539b27f178",
|
||||
"width": 513,
|
||||
"height": 769
|
||||
},
|
||||
{
|
||||
"layer": "wind",
|
||||
"frame": "T0",
|
||||
"bytes": 621120,
|
||||
"sha256": "e815d94c18711c70e27c570cfd89acda3a27d64646d04e0b29846404ca325979",
|
||||
"width": 513,
|
||||
"height": 769
|
||||
},
|
||||
{
|
||||
"layer": "wind",
|
||||
"frame": "T1",
|
||||
"bytes": 622832,
|
||||
"sha256": "bb2dbde0a8984d2fda4140cb6393744ede14fc9327663091e7afd4d2e37606da",
|
||||
"width": 513,
|
||||
"height": 769
|
||||
}
|
||||
]
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -6,7 +6,7 @@ const marker = "// MapLibreParticleRNTestHost NavSea Particle release SDK";
|
||||
module.exports = function withParticleNativeAndroid(config) {
|
||||
return withProjectBuildGradle(config, (project) => {
|
||||
if (project.modResults.contents.includes(marker)) return project;
|
||||
project.modResults.contents += `\n${marker}\ngradle.projectsEvaluated {\n allprojects { p ->\n p.repositories.maven { repository -> repository.url = p.uri("https://pbf.jpcn.ccwu.cc/maven") }\n p.configurations.configureEach { configuration ->\n configuration.resolutionStrategy.dependencySubstitution {\n substitute module("org.maplibre.gl:android-sdk-opengl") using module("cc.navsea.maplibre:android-sdk-particle-opengl:13.4.1-particle.8")\n }\n }\n }\n}\nrootProject.ext.set("org.maplibre.reactnative.nativeVersion", "13.4.1")\nrootProject.ext.set("org.maplibre.reactnative.nativeVariant", "opengl")\n`;
|
||||
project.modResults.contents += `\n${marker}\ngradle.projectsEvaluated {\n allprojects { p ->\n p.repositories.maven { repository -> repository.url = p.uri("https://pbf.jpcn.ccwu.cc/maven") }\n p.configurations.configureEach { configuration ->\n configuration.resolutionStrategy.dependencySubstitution {\n substitute module("org.maplibre.gl:android-sdk-opengl") using module("cc.navsea.maplibre:android-sdk-particle-opengl:13.4.1-particle.9")\n }\n }\n }\n}\nrootProject.ext.set("org.maplibre.reactnative.nativeVersion", "13.4.1")\nrootProject.ext.set("org.maplibre.reactnative.nativeVariant", "opengl")\n`;
|
||||
return project;
|
||||
});
|
||||
};
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env python3
|
||||
"""校验仓库内冻结的 QBIN V3 Golden Fixtures。"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import struct
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[4]
|
||||
ASSETS = ROOT / "integration/react-native/MapLibreParticleRNTestHost/assets/qbin-v3-golden"
|
||||
MANIFEST = ASSETS / "manifest.json"
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
manifest = json.loads(MANIFEST.read_text(encoding="utf-8"))
|
||||
if manifest.get("format") != "QBV3" or manifest.get("version") != 3:
|
||||
raise RuntimeError("manifest 格式或版本不是 QBV3/v3")
|
||||
if manifest.get("blockWidth") != 16 or manifest.get("blockHeight") != 16:
|
||||
raise RuntimeError("Block Geometry 必须冻结为 16x16")
|
||||
|
||||
fixtures = manifest.get("fixtures", [])
|
||||
if len(fixtures) != 8:
|
||||
raise RuntimeError(f"必须存在 8 个 Golden Fixture,当前为 {len(fixtures)} 个")
|
||||
|
||||
for item in fixtures:
|
||||
path = ASSETS / item["layer"] / f'{item["frame"]}.qbin'
|
||||
data = path.read_bytes()
|
||||
if data[:4] != b"QBV3" or data[4] != 3:
|
||||
raise RuntimeError(f"{path}: magic/version 错误")
|
||||
width, height = struct.unpack_from("<II", data, 16)
|
||||
if len(data) != item["bytes"] or sha256(path) != item["sha256"]:
|
||||
raise RuntimeError(f"{path}: bytes/SHA-256 与 manifest 不一致")
|
||||
if width != item["width"] or height != item["height"]:
|
||||
raise RuntimeError(f"{path}: grid 与 manifest 不一致")
|
||||
|
||||
print("QBIN_V3_GOLDEN_FIXTURES=PASS (8 files, block=16x16)")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -2,6 +2,13 @@
|
||||
|
||||
## Unreleased
|
||||
|
||||
## 13.4.1-particle.9(QBIN V3 Decoder)
|
||||
|
||||
- 新增 QBV3/v3 Native Decoder,固定 16×16 Block Geometry,支持 Predictor + ZigZag + Rice 解码、Direction 环形残差、CRC32、canonical packing、validity 与资源上限校验。
|
||||
- `EncodedFieldIngress` 依据 `QBV2` / `QBV3` magic 显式分流;既有 QBV2 行为保持兼容,FieldProfileValidator 仍在解码后统一执行。
|
||||
- 新增 Current / Rain / Wave / Wind 共 8 个 QBIN V3 Golden Fixtures 与校验脚本;新增 decoder corruption/canonical/ingress C++ 定向测试。
|
||||
- Native / RN 统一升级到 `13.4.1-particle.9`。
|
||||
|
||||
## 13.4.1-particle.8(Android / iOS 统一发行)
|
||||
|
||||
- Mac unified release pipeline 统一生成 Android Native、RN package 与 iOS SwiftPM/XCFramework;共享 Particle revision 8。
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# @navsea/maplibre-particle-rn
|
||||
|
||||
当前版本:`13.4.1-particle.8`。Android Native 必须配套 `cc.navsea.maplibre:android-sdk-particle-opengl:13.4.1-particle.8`。本包是 React Native 的薄控制面,不负责 Map 组件、provider、HTTP、manifest、下载、缓存或 NavSea 业务策略。
|
||||
当前版本:`13.4.1-particle.9`。Android Native 必须配套 `cc.navsea.maplibre:android-sdk-particle-opengl:13.4.1-particle.9`。本包是 React Native 的薄控制面,不负责 Map 组件、provider、HTTP、manifest、下载、缓存或 NavSea 业务策略。
|
||||
|
||||
## 当前推荐主链:QBIN V4
|
||||
|
||||
@@ -14,7 +14,7 @@ await FieldRuntime.setEncodedParticleBinding(reactTag, particleBinding);
|
||||
await FieldRuntime.setEncodedForecastTime(reactTag, timestampMs);
|
||||
```
|
||||
|
||||
Android 链路为:`FieldRuntime.submitEncodedField` → `WeatherRuntimeBridgeModule` → `WeatherRuntime.submitEncodedField` → JNI → `RasterArraySource::submitEncodedField` → `QbinV2Decoder::decode`。iOS 使用同一 `WeatherRuntimeBridge` TurboModule,按每次调用的 Fabric `reactTag` 即时解析当前 MLRN `MapView`,再调用 public `MLNParticleFieldRuntime`。两端的 QBIN 解码都发生在 MapLibre Native,不在 JS/TS。
|
||||
Android 链路为:`FieldRuntime.submitEncodedField` → `WeatherRuntimeBridgeModule` → `WeatherRuntime.submitEncodedField` → JNI → `RasterArraySource::submitEncodedField` → `EncodedFieldIngress`。Native 依据 magic/version 显式分流:历史 QBV2 继续走 `QbinV2Decoder`,新的 QBV3 压缩格式走 `QbinV3Decoder`。iOS 使用同一 `WeatherRuntimeBridge` TurboModule,按每次调用的 Fabric `reactTag` 即时解析当前 MLRN `MapView`,再调用 public `MLNParticleFieldRuntime`。两端的 QBIN 解码都发生在 MapLibre Native,不在 JS/TS。
|
||||
|
||||
V4 主要公共方法:`submitEncodedField`、`setEncodedColorBinding`、`setEncodedParticleBinding`、`setEncodedForecastTime`、`setEncodedForecastPlaybackState`、`setEncodedForecastPlaybackSpeed`、`clearEncodedField`。ColorBinding 支持 `SCALAR`、`VELOCITY_COMPONENT`、`DIRECTION_ANGLE`;ParticleBinding 支持 U/V vector components 或 direction angle。
|
||||
|
||||
@@ -37,7 +37,7 @@ Podfile 的 `post_install do |installer|` 必须同时调用 `$MLRN.post_install
|
||||
RN tgz:
|
||||
|
||||
```text
|
||||
https://pbf.jpcn.ccwu.cc/npm/navsea-maplibre-particle-rn-13.4.1-particle.8.tgz
|
||||
https://pbf.jpcn.ccwu.cc/npm/navsea-maplibre-particle-rn-13.4.1-particle.9.tgz
|
||||
```
|
||||
|
||||
Native Maven repository:
|
||||
@@ -49,7 +49,7 @@ https://pbf.jpcn.ccwu.cc/maven
|
||||
Native coordinate:
|
||||
|
||||
```text
|
||||
cc.navsea.maplibre:android-sdk-particle-opengl:13.4.1-particle.8
|
||||
cc.navsea.maplibre:android-sdk-particle-opengl:13.4.1-particle.9
|
||||
```
|
||||
|
||||
Expo plugin 与 `android/build.gradle` 必须指向同一 `.8`。正常 Consumer 不复制 AAR、不依赖 MapLibreParticleLab 源码路径,也不以 Maven Local 作为公网验证主路径。完整 Linux 发布与 Win11 验证流程见 `docs/50_Step记录/STEP-080_Android_AAR与RNTestHost标准构建流水线_v2.md`。
|
||||
|
||||
@@ -14,7 +14,7 @@ android {
|
||||
dependencies {
|
||||
implementation "com.facebook.react:react-android"
|
||||
implementation "org.jetbrains.kotlin:kotlin-stdlib:2.1.20"
|
||||
implementation "cc.navsea.maplibre:android-sdk-particle-opengl:13.4.1-particle.8"
|
||||
implementation "cc.navsea.maplibre:android-sdk-particle-opengl:13.4.1-particle.9"
|
||||
compileOnly project(":maplibre_maplibre-react-native")
|
||||
}
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ const iosTag = "navsea-maplibre-particle-rn-maplibre-spm";
|
||||
module.exports = function withNavSeaMapLibreParticle(config) {
|
||||
config = withProjectBuildGradle(config, (project) => {
|
||||
if (project.modResults.contents.includes(androidMarker)) return project;
|
||||
project.modResults.contents += `\n${androidMarker}\ngradle.projectsEvaluated {\n allprojects { p ->\n p.repositories.maven { repository -> repository.url = p.uri("https://pbf.jpcn.ccwu.cc/maven") }\n p.configurations.configureEach { configuration ->\n configuration.resolutionStrategy.dependencySubstitution {\n substitute module("org.maplibre.gl:android-sdk-opengl") using module("cc.navsea.maplibre:android-sdk-particle-opengl:13.4.1-particle.8")\n }\n }\n }\n}\nrootProject.ext.set("org.maplibre.reactnative.nativeVersion", "13.4.1")\nrootProject.ext.set("org.maplibre.reactnative.nativeVariant", "opengl")\n`;
|
||||
project.modResults.contents += `\n${androidMarker}\ngradle.projectsEvaluated {\n allprojects { p ->\n p.repositories.maven { repository -> repository.url = p.uri("https://pbf.jpcn.ccwu.cc/maven") }\n p.configurations.configureEach { configuration ->\n configuration.resolutionStrategy.dependencySubstitution {\n substitute module("org.maplibre.gl:android-sdk-opengl") using module("cc.navsea.maplibre:android-sdk-particle-opengl:13.4.1-particle.9")\n }\n }\n }\n}\nrootProject.ext.set("org.maplibre.reactnative.nativeVersion", "13.4.1")\nrootProject.ext.set("org.maplibre.reactnative.nativeVariant", "opengl")\n`;
|
||||
return project;
|
||||
});
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@navsea/maplibre-particle-rn",
|
||||
"version": "13.4.1-particle.8",
|
||||
"version": "13.4.1-particle.9",
|
||||
"description": "NavSea MapLibre Particle React Native pure Field Renderer API.",
|
||||
"license": "BSD-2-Clause",
|
||||
"main": "src/index.ts",
|
||||
|
||||
@@ -1 +1 @@
|
||||
13.4.1-particle.8
|
||||
13.4.1-particle.9
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
#include <mbgl/particle/ingress/qbin_v2_decoder.hpp>
|
||||
#include <mbgl/particle/ingress/qbin_v3_decoder.hpp>
|
||||
|
||||
#include <cmath>
|
||||
#include <cstring>
|
||||
@@ -71,5 +72,53 @@ QbinV2DecodeResult QbinV2Decoder::decode(const EncodedFieldRuntimeMetadata& meta
|
||||
if (!out->isStructurallyValid()) return fail(QbinV2DecodeErrorCode::InvalidFrame, "decoded frame structural validity");
|
||||
return {QbinV2DecodeErrorCode::Accepted,"accepted",out};
|
||||
}
|
||||
EncodedFieldIngressResult EncodedFieldIngress::decodeAndValidate(const EncodedFieldRuntimeMetadata& m,const ResolvedFieldProfileContract* c,std::span<const uint8_t> b){ auto decoded=QbinV2Decoder::decode(m,b); if(!decoded.accepted()) return {EncodedFieldIngressCode::DecodeRejected,decoded.detail,decoded.code,FieldProfileValidationCode::InvalidContract,{}}; if(!c) return {EncodedFieldIngressCode::ContractMissing,"contract",decoded.code,FieldProfileValidationCode::InvalidContract,{}}; auto v=FieldProfileValidator::validate(*c,*decoded.frame); if(!v.accepted()) return {EncodedFieldIngressCode::ContractRejected,v.detail,decoded.code,v.code,{}}; return {EncodedFieldIngressCode::Accepted,"accepted",decoded.code,v.code,decoded.frame}; }
|
||||
EncodedFieldIngressResult EncodedFieldIngress::decodeAndValidate(const EncodedFieldRuntimeMetadata& m,
|
||||
const ResolvedFieldProfileContract* c,
|
||||
std::span<const uint8_t> b) {
|
||||
QbinDecodeErrorCode decodeCode = QbinDecodeErrorCode::BadMagic;
|
||||
std::string detail;
|
||||
std::shared_ptr<const DecodedFieldFrame> frame;
|
||||
|
||||
if (b.size() >= 4 && std::memcmp(b.data(), "QBV2", 4) == 0) {
|
||||
auto decoded = QbinV2Decoder::decode(m, b);
|
||||
decodeCode = decoded.code;
|
||||
detail = std::move(decoded.detail);
|
||||
frame = std::move(decoded.frame);
|
||||
} else if (b.size() >= 4 && std::memcmp(b.data(), "QBV3", 4) == 0) {
|
||||
auto decoded = QbinV3Decoder::decode(m, b);
|
||||
decodeCode = decoded.code;
|
||||
detail = std::move(decoded.detail);
|
||||
frame = std::move(decoded.frame);
|
||||
} else {
|
||||
return {EncodedFieldIngressCode::DecodeRejected,
|
||||
"QBIN magic",
|
||||
QbinDecodeErrorCode::BadMagic,
|
||||
FieldProfileValidationCode::InvalidContract,
|
||||
{}};
|
||||
}
|
||||
|
||||
if (decodeCode != QbinDecodeErrorCode::Accepted || !frame) {
|
||||
return {EncodedFieldIngressCode::DecodeRejected,
|
||||
std::move(detail),
|
||||
decodeCode,
|
||||
FieldProfileValidationCode::InvalidContract,
|
||||
{}};
|
||||
}
|
||||
if (!c) {
|
||||
return {EncodedFieldIngressCode::ContractMissing,
|
||||
"contract",
|
||||
decodeCode,
|
||||
FieldProfileValidationCode::InvalidContract,
|
||||
{}};
|
||||
}
|
||||
const auto validation = FieldProfileValidator::validate(*c, *frame);
|
||||
if (!validation.accepted()) {
|
||||
return {EncodedFieldIngressCode::ContractRejected,
|
||||
validation.detail,
|
||||
decodeCode,
|
||||
validation.code,
|
||||
{}};
|
||||
}
|
||||
return {EncodedFieldIngressCode::Accepted, "accepted", decodeCode, validation.code, std::move(frame)};
|
||||
}
|
||||
} // namespace mbgl::particle::ingress
|
||||
|
||||
674
src/mbgl/particle/ingress/qbin_v3_decoder.cpp
Normal file
674
src/mbgl/particle/ingress/qbin_v3_decoder.cpp
Normal file
@@ -0,0 +1,674 @@
|
||||
#include <mbgl/particle/ingress/qbin_v3_decoder.hpp>
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <cstring>
|
||||
#include <limits>
|
||||
#include <optional>
|
||||
#include <set>
|
||||
#include <sstream>
|
||||
#include <tuple>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace mbgl::particle::ingress {
|
||||
namespace {
|
||||
|
||||
constexpr size_t HeaderSize = 112;
|
||||
constexpr size_t DescriptorSize = 32;
|
||||
constexpr size_t MaxBytes = 64u * 1024u * 1024u;
|
||||
constexpr uint32_t MaxPlanes = 32;
|
||||
constexpr uint32_t MaxCells = 2'097'152;
|
||||
constexpr uint32_t MaxDecoded = 32u * 1024u * 1024u;
|
||||
constexpr uint16_t ValiditySemantic = 3;
|
||||
|
||||
uint16_t readU16(std::span<const uint8_t> b, size_t o) {
|
||||
return uint16_t(b[o]) | (uint16_t(b[o + 1]) << 8);
|
||||
}
|
||||
|
||||
uint32_t readU32(std::span<const uint8_t> b, size_t o) {
|
||||
uint32_t v = 0;
|
||||
for (size_t i = 0; i < 4; ++i) v |= uint32_t(b[o + i]) << (8 * i);
|
||||
return v;
|
||||
}
|
||||
uint64_t readU64(std::span<const uint8_t> b, size_t o) {
|
||||
uint64_t v = 0;
|
||||
for (size_t i = 0; i < 8; ++i) v |= uint64_t(b[o + i]) << (8 * i);
|
||||
return v;
|
||||
}
|
||||
|
||||
int64_t readI64(std::span<const uint8_t> b, size_t o) {
|
||||
const uint64_t bits = readU64(b, o);
|
||||
int64_t value = 0;
|
||||
std::memcpy(&value, &bits, sizeof(value));
|
||||
return value;
|
||||
}
|
||||
|
||||
float readF32(std::span<const uint8_t> b, size_t o) {
|
||||
const uint32_t bits = readU32(b, o);
|
||||
float value = 0.0f;
|
||||
std::memcpy(&value, &bits, sizeof(value));
|
||||
return value;
|
||||
}
|
||||
|
||||
double readF64(std::span<const uint8_t> b, size_t o) {
|
||||
const uint64_t bits = readU64(b, o);
|
||||
double value = 0.0;
|
||||
std::memcpy(&value, &bits, sizeof(value));
|
||||
return value;
|
||||
}
|
||||
|
||||
uint32_t floatBits(float value) {
|
||||
uint32_t bits = 0;
|
||||
std::memcpy(&bits, &value, sizeof(bits));
|
||||
return bits;
|
||||
}
|
||||
uint32_t crc32(std::span<const uint8_t> bytes) {
|
||||
uint32_t c = 0xffffffffu;
|
||||
for (size_t i = 0; i < bytes.size(); ++i) {
|
||||
const uint8_t x = (i >= 76 && i < 80) ? 0 : bytes[i];
|
||||
c ^= x;
|
||||
for (int j = 0; j < 8; ++j) {
|
||||
c = (c >> 1) ^ ((c & 1u) ? 0xedb88320u : 0u);
|
||||
}
|
||||
}
|
||||
return ~c;
|
||||
}
|
||||
|
||||
size_t align16(size_t x) {
|
||||
return (x + 15u) & ~size_t(15u);
|
||||
}
|
||||
|
||||
bool allZero(std::span<const uint8_t> bytes) {
|
||||
for (const auto byte : bytes) {
|
||||
if (byte != 0) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
QbinV3DecodeResult fail(QbinDecodeErrorCode code, std::string detail) {
|
||||
return {code, std::move(detail), {}};
|
||||
}
|
||||
|
||||
struct Descriptor {
|
||||
uint16_t semantic = 0;
|
||||
uint16_t unit = 0;
|
||||
uint16_t group = 0;
|
||||
uint8_t dtype = 0;
|
||||
uint8_t flags = 0;
|
||||
uint8_t valueClass = 0;
|
||||
uint8_t component = 0;
|
||||
float scale = 0.0f;
|
||||
float offset = 0.0f;
|
||||
uint32_t scaleBits = 0;
|
||||
uint32_t offsetBits = 0;
|
||||
uint32_t count = 0;
|
||||
uint32_t byteOffset = 0;
|
||||
uint32_t byteLength = 0;
|
||||
uint32_t directionN = 0;
|
||||
};
|
||||
|
||||
std::string contentIdentity(const Descriptor& d, std::span<const uint8_t> raw) {
|
||||
uint64_t hash = 1469598103934665603ull;
|
||||
auto addByte = [&](uint8_t v) {
|
||||
hash ^= v;
|
||||
hash *= 1099511628211ull;
|
||||
};
|
||||
auto addLE = [&](uint64_t value, size_t count) {
|
||||
for (size_t i = 0; i < count; ++i) addByte(uint8_t(value >> (8 * i)));
|
||||
};
|
||||
|
||||
for (const auto v : raw) addByte(v);
|
||||
addLE(d.semantic, 2);
|
||||
addLE(d.group, 2);
|
||||
addByte(d.component);
|
||||
addByte(d.dtype);
|
||||
addByte(d.flags);
|
||||
addByte(d.valueClass);
|
||||
addLE(d.unit, 2);
|
||||
addLE(d.count, 4);
|
||||
addLE(d.scaleBits, 4);
|
||||
addLE(d.offsetBits, 4);
|
||||
|
||||
std::ostringstream out;
|
||||
out << std::hex << hash;
|
||||
return out.str();
|
||||
}
|
||||
|
||||
std::optional<uint32_t> directionCount(float scale) {
|
||||
if (!std::isfinite(scale) || scale <= 0.0f) return std::nullopt;
|
||||
const double ratioToN = 360.0 / double(scale);
|
||||
if (!std::isfinite(ratioToN) || ratioToN < 1.0 || ratioToN > 65536.0) return std::nullopt;
|
||||
|
||||
const uint32_t n = static_cast<uint32_t>(std::floor(ratioToN + 0.5));
|
||||
if (n < 1 || n > 65536) return std::nullopt;
|
||||
|
||||
const double ratio64 = 360.0 / double(n);
|
||||
if (!std::isfinite(ratio64)) return std::nullopt;
|
||||
const float canonicalScale = static_cast<float>(ratio64);
|
||||
if (floatBits(canonicalScale) != floatBits(scale)) return std::nullopt;
|
||||
return n;
|
||||
}
|
||||
|
||||
uint64_t logicalDecodedBytes(const Descriptor& d, uint64_t cells) {
|
||||
switch (d.dtype) {
|
||||
case 1:
|
||||
case 2:
|
||||
return cells * 2u;
|
||||
case 3:
|
||||
case 4:
|
||||
return cells;
|
||||
case 6:
|
||||
return (cells + 7u) / 8u;
|
||||
default:
|
||||
return std::numeric_limits<uint64_t>::max();
|
||||
}
|
||||
}
|
||||
|
||||
bool validAt(std::span<const uint8_t> validity, uint64_t index) {
|
||||
return ((validity[index >> 3] >> (index & 7u)) & 1u) != 0;
|
||||
}
|
||||
|
||||
std::pair<int64_t, int64_t> dtypeRange(uint8_t dtype) {
|
||||
switch (dtype) {
|
||||
case 1:
|
||||
return {-32768, 32767};
|
||||
case 2:
|
||||
return {0, 65535};
|
||||
case 3:
|
||||
return {0, 255};
|
||||
case 4:
|
||||
return {-128, 127};
|
||||
default:
|
||||
return {1, 0};
|
||||
}
|
||||
}
|
||||
|
||||
int64_t readRaw(const std::vector<uint8_t>& raw, uint64_t index, uint8_t dtype) {
|
||||
if (dtype == 1 || dtype == 2) {
|
||||
const size_t o = size_t(index) * 2u;
|
||||
const uint16_t bits = uint16_t(raw[o]) | (uint16_t(raw[o + 1]) << 8);
|
||||
if (dtype == 2) return bits;
|
||||
int16_t value = 0;
|
||||
std::memcpy(&value, &bits, sizeof(value));
|
||||
return value;
|
||||
}
|
||||
|
||||
const uint8_t bits = raw[size_t(index)];
|
||||
if (dtype == 3) return bits;
|
||||
int8_t value = 0;
|
||||
std::memcpy(&value, &bits, sizeof(value));
|
||||
return value;
|
||||
}
|
||||
|
||||
void writeRaw(std::vector<uint8_t>& raw, uint64_t index, uint8_t dtype, int64_t value) {
|
||||
if (dtype == 1 || dtype == 2) {
|
||||
uint16_t bits = 0;
|
||||
if (dtype == 1) {
|
||||
const int16_t signedValue = static_cast<int16_t>(value);
|
||||
std::memcpy(&bits, &signedValue, sizeof(bits));
|
||||
} else {
|
||||
bits = static_cast<uint16_t>(value);
|
||||
}
|
||||
const size_t o = size_t(index) * 2u;
|
||||
raw[o] = uint8_t(bits);
|
||||
raw[o + 1] = uint8_t(bits >> 8);
|
||||
return;
|
||||
}
|
||||
|
||||
if (dtype == 3) {
|
||||
raw[size_t(index)] = static_cast<uint8_t>(value);
|
||||
} else {
|
||||
const int8_t signedValue = static_cast<int8_t>(value);
|
||||
uint8_t bits = 0;
|
||||
std::memcpy(&bits, &signedValue, sizeof(bits));
|
||||
raw[size_t(index)] = bits;
|
||||
}
|
||||
}
|
||||
|
||||
int64_t normalizeDirection(int64_t value, uint32_t n) {
|
||||
if (value < 0) value += n;
|
||||
else if (value >= int64_t(n)) value -= n;
|
||||
return value;
|
||||
}
|
||||
|
||||
class BitReader {
|
||||
public:
|
||||
explicit BitReader(std::span<const uint8_t> input_) : input(input_) {}
|
||||
|
||||
bool readBit(uint8_t& bit) {
|
||||
if (bitPosition >= input.size() * 8u) return false;
|
||||
bit = (input[bitPosition >> 3] >> (bitPosition & 7u)) & 1u;
|
||||
++bitPosition;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool readBits(uint8_t count, uint32_t& value) {
|
||||
value = 0;
|
||||
for (uint8_t bitIndex = 0; bitIndex < count; ++bitIndex) {
|
||||
uint8_t bit = 0;
|
||||
if (!readBit(bit)) return false;
|
||||
value |= uint32_t(bit) << bitIndex;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
size_t position() const { return bitPosition; }
|
||||
|
||||
private:
|
||||
std::span<const uint8_t> input;
|
||||
size_t bitPosition = 0;
|
||||
};
|
||||
|
||||
struct PlaneDecode {
|
||||
QbinDecodeErrorCode code = QbinDecodeErrorCode::Accepted;
|
||||
std::string detail;
|
||||
std::shared_ptr<const std::vector<uint8_t>> raw;
|
||||
|
||||
bool accepted() const {
|
||||
return code == QbinDecodeErrorCode::Accepted && static_cast<bool>(raw);
|
||||
}
|
||||
};
|
||||
|
||||
PlaneDecode decodeNumericPlane(const Descriptor& d,
|
||||
std::span<const uint8_t> encoded,
|
||||
std::span<const uint8_t> validity,
|
||||
uint32_t width,
|
||||
uint32_t height) {
|
||||
const uint64_t cells = uint64_t(width) * height;
|
||||
const size_t bytesPerCell = (d.dtype == 1 || d.dtype == 2) ? 2u : 1u;
|
||||
auto output = std::make_shared<std::vector<uint8_t>>(size_t(cells) * bytesPerCell, 0);
|
||||
|
||||
const bool direction = d.valueClass == 2;
|
||||
const auto [minRaw, maxRaw] = dtypeRange(d.dtype);
|
||||
const uint64_t maxAbsResidual = direction ? uint64_t(d.directionN / 2u)
|
||||
: uint64_t(2) * uint64_t(maxRaw - minRaw);
|
||||
const uint64_t maxZigZag = 2u * maxAbsResidual;
|
||||
|
||||
size_t cursor = 0;
|
||||
for (uint32_t rowBegin = 0; rowBegin < height; rowBegin += QbinV3Decoder::BlockHeight) {
|
||||
const uint32_t rowEnd = std::min(rowBegin + QbinV3Decoder::BlockHeight, height);
|
||||
for (uint32_t colBegin = 0; colBegin < width; colBegin += QbinV3Decoder::BlockWidth) {
|
||||
const uint32_t colEnd = std::min(colBegin + QbinV3Decoder::BlockWidth, width);
|
||||
|
||||
uint32_t symbolCount = 0;
|
||||
for (uint32_t row = rowBegin; row < rowEnd; ++row) {
|
||||
for (uint32_t col = colBegin; col < colEnd; ++col) {
|
||||
if (validAt(validity, uint64_t(row) * width + col)) ++symbolCount;
|
||||
}
|
||||
}
|
||||
if (symbolCount == 0) continue;
|
||||
|
||||
if (cursor >= encoded.size()) {
|
||||
return {QbinDecodeErrorCode::MalformedRice, "missing riceK", {}};
|
||||
}
|
||||
const uint8_t riceK = encoded[cursor++];
|
||||
if (riceK > 31) {
|
||||
return {QbinDecodeErrorCode::MalformedRice, "riceK > 31", {}};
|
||||
}
|
||||
|
||||
const uint64_t maxQ = maxZigZag >> riceK;
|
||||
BitReader reader(encoded.subspan(cursor));
|
||||
uint32_t decodedSymbols = 0;
|
||||
|
||||
for (uint32_t row = rowBegin; row < rowEnd; ++row) {
|
||||
for (uint32_t col = colBegin; col < colEnd; ++col) {
|
||||
const uint64_t index = uint64_t(row) * width + col;
|
||||
if (!validAt(validity, index)) continue;
|
||||
|
||||
uint64_t q = 0;
|
||||
while (true) {
|
||||
uint8_t bit = 0;
|
||||
if (!reader.readBit(bit)) {
|
||||
return {QbinDecodeErrorCode::MalformedRice, "unterminated unary", {}};
|
||||
}
|
||||
if (bit != 0) break;
|
||||
if (q >= maxQ) {
|
||||
return {QbinDecodeErrorCode::ResidualOutOfRange, "rice quotient exceeds logical maximum", {}};
|
||||
}
|
||||
++q;
|
||||
}
|
||||
|
||||
uint32_t remainder = 0;
|
||||
if (!reader.readBits(riceK, remainder)) {
|
||||
return {QbinDecodeErrorCode::MalformedRice, "truncated rice remainder", {}};
|
||||
}
|
||||
if (q > (std::numeric_limits<uint32_t>::max() >> riceK)) {
|
||||
return {QbinDecodeErrorCode::ResidualOutOfRange, "rice symbol overflow", {}};
|
||||
}
|
||||
const uint64_t zigzag = (q << riceK) | remainder;
|
||||
if (zigzag > maxZigZag || zigzag > std::numeric_limits<uint32_t>::max()) {
|
||||
return {QbinDecodeErrorCode::ResidualOutOfRange, "zigzag exceeds logical maximum", {}};
|
||||
}
|
||||
|
||||
const int64_t residual = (zigzag & 1u) ? -int64_t(zigzag / 2u) - 1
|
||||
: int64_t(zigzag / 2u);
|
||||
if (residual < -int64_t(maxAbsResidual) || residual > int64_t(maxAbsResidual)) {
|
||||
return {QbinDecodeErrorCode::ResidualOutOfRange, "residual exceeds logical maximum", {}};
|
||||
}
|
||||
if (direction && (d.directionN % 2u == 0u) &&
|
||||
residual == int64_t(d.directionN / 2u)) {
|
||||
return {QbinDecodeErrorCode::ResidualOutOfRange, "non-canonical positive half-circle residual", {}};
|
||||
}
|
||||
|
||||
const bool hasLeft = col > 0 && validAt(validity, index - 1);
|
||||
const bool hasUp = row > 0 && validAt(validity, index - width);
|
||||
const bool hasUpperLeft = row > 0 && col > 0 && validAt(validity, index - width - 1);
|
||||
|
||||
int64_t predicted = 0;
|
||||
if (hasLeft && hasUp && hasUpperLeft) {
|
||||
predicted = readRaw(*output, index - 1, d.dtype) +
|
||||
readRaw(*output, index - width, d.dtype) -
|
||||
readRaw(*output, index - width - 1, d.dtype);
|
||||
if (direction) predicted = normalizeDirection(predicted, d.directionN);
|
||||
} else if (hasLeft) {
|
||||
predicted = readRaw(*output, index - 1, d.dtype);
|
||||
} else if (hasUp) {
|
||||
predicted = readRaw(*output, index - width, d.dtype);
|
||||
}
|
||||
|
||||
int64_t reconstructed = predicted + residual;
|
||||
if (direction) {
|
||||
reconstructed = normalizeDirection(reconstructed, d.directionN);
|
||||
if (reconstructed < 0 || reconstructed >= int64_t(d.directionN)) {
|
||||
return {QbinDecodeErrorCode::InvalidCellPayload, "direction raw outside circular domain", {}};
|
||||
}
|
||||
} else if (reconstructed < minRaw || reconstructed > maxRaw) {
|
||||
return {QbinDecodeErrorCode::InvalidCellPayload, "numeric raw outside dtype range", {}};
|
||||
}
|
||||
|
||||
writeRaw(*output, index, d.dtype, reconstructed);
|
||||
++decodedSymbols;
|
||||
}
|
||||
}
|
||||
|
||||
if (decodedSymbols != symbolCount) {
|
||||
return {QbinDecodeErrorCode::MalformedRice, "block symbol count mismatch", {}};
|
||||
}
|
||||
|
||||
const size_t remainderBits = reader.position() & 7u;
|
||||
if (remainderBits != 0) {
|
||||
const size_t paddingBits = 8u - remainderBits;
|
||||
for (size_t i = 0; i < paddingBits; ++i) {
|
||||
uint8_t bit = 0;
|
||||
if (!reader.readBit(bit)) {
|
||||
return {QbinDecodeErrorCode::MalformedRice, "truncated block padding", {}};
|
||||
}
|
||||
if (bit != 0) {
|
||||
return {QbinDecodeErrorCode::MalformedRice, "nonzero block padding", {}};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const size_t blockBytes = reader.position() / 8u;
|
||||
if (blockBytes > encoded.size() - cursor) {
|
||||
return {QbinDecodeErrorCode::MalformedRice, "block exceeds plane", {}};
|
||||
}
|
||||
cursor += blockBytes;
|
||||
}
|
||||
}
|
||||
|
||||
if (cursor != encoded.size()) {
|
||||
return {QbinDecodeErrorCode::MalformedRice, "extra encoded bytes after final block", {}};
|
||||
}
|
||||
|
||||
return {QbinDecodeErrorCode::Accepted, "accepted", output};
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
QbinV3DecodeResult QbinV3Decoder::decode(const EncodedFieldRuntimeMetadata& meta,
|
||||
std::span<const uint8_t> bytes) {
|
||||
if (meta.fieldID.empty() || meta.generation == 0) {
|
||||
return fail(QbinDecodeErrorCode::InvalidRuntimeMetadata, "fieldID/generation");
|
||||
}
|
||||
if (bytes.size() < HeaderSize) return fail(QbinDecodeErrorCode::InputTooSmall, "header");
|
||||
if (bytes.size() > MaxBytes) return fail(QbinDecodeErrorCode::LimitExceeded, "file bytes");
|
||||
if (std::memcmp(bytes.data(), "QBV3", 4) != 0) return fail(QbinDecodeErrorCode::BadMagic, "magic");
|
||||
if (bytes[4] != 3) return fail(QbinDecodeErrorCode::UnsupportedVersion, "version");
|
||||
|
||||
if (bytes[5] != 1) return fail(QbinDecodeErrorCode::UnsupportedLayout, "layout");
|
||||
if (bytes[8] != 1) return fail(QbinDecodeErrorCode::UnsupportedProjection, "projection");
|
||||
if (bytes[9] != 1) return fail(QbinDecodeErrorCode::UnsupportedSampleOrder, "sampleOrder");
|
||||
if (bytes[10] != 1) return fail(QbinDecodeErrorCode::UnsupportedGridRegistration, "gridRegistration");
|
||||
if (bytes[11] != 0 || readU16(bytes, 14) != 0 || readU16(bytes, 82) != 0 ||
|
||||
readU32(bytes, 92) != 0 || readU32(bytes, 96) != 0 || readU32(bytes, 100) != 0 ||
|
||||
readU32(bytes, 104) != 0 || readU32(bytes, 108) != 0) {
|
||||
return fail(QbinDecodeErrorCode::ReservedNonZero, "header reserved");
|
||||
}
|
||||
|
||||
const uint16_t planeCount = readU16(bytes, 12);
|
||||
const uint32_t width = readU32(bytes, 16);
|
||||
const uint32_t height = readU32(bytes, 20);
|
||||
const uint32_t payloadLength = readU32(bytes, 72);
|
||||
if (planeCount == 0 || planeCount > MaxPlanes || width < 2 || height < 2) {
|
||||
return fail(QbinDecodeErrorCode::InvalidHeader, "planes/grid");
|
||||
}
|
||||
|
||||
const uint64_t cells = uint64_t(width) * uint64_t(height);
|
||||
if (cells > MaxCells) return fail(QbinDecodeErrorCode::LimitExceeded, "cell count");
|
||||
if (readU16(bytes, 80) != 0xffff || readU32(bytes, 84) != 0 || readU32(bytes, 88) != 0) {
|
||||
return fail(QbinDecodeErrorCode::InvalidHeader, "FRAME tile");
|
||||
}
|
||||
|
||||
const int64_t referenceTime = readI64(bytes, 24);
|
||||
const int64_t validTime = readI64(bytes, 32);
|
||||
constexpr uint64_t MaxSecondsForMs = std::numeric_limits<uint64_t>::max() / 1000u;
|
||||
if (referenceTime <= 0 || validTime <= 0 ||
|
||||
uint64_t(referenceTime) > MaxSecondsForMs || uint64_t(validTime) > MaxSecondsForMs) {
|
||||
return fail(QbinDecodeErrorCode::InvalidTime, "UTC seconds");
|
||||
}
|
||||
const double west = readF64(bytes, 40);
|
||||
const double south = readF64(bytes, 48);
|
||||
const double east = readF64(bytes, 56);
|
||||
const double north = readF64(bytes, 64);
|
||||
for (const size_t offset : {size_t(40), size_t(48), size_t(56), size_t(64)}) {
|
||||
if (readF64(bytes, offset) == 0.0 && readU64(bytes, offset) != 0) {
|
||||
return fail(QbinDecodeErrorCode::InvalidBounds, "negative-zero bound");
|
||||
}
|
||||
}
|
||||
if (!std::isfinite(west) || !std::isfinite(south) || !std::isfinite(east) || !std::isfinite(north) ||
|
||||
west < -180.0 || east > 180.0 || west >= east ||
|
||||
south < -85.0511287798066 || north > 85.0511287798066 || south >= north) {
|
||||
return fail(QbinDecodeErrorCode::InvalidBounds, "Web Mercator bounds");
|
||||
}
|
||||
|
||||
const size_t payloadOffset = HeaderSize + size_t(planeCount) * DescriptorSize;
|
||||
if (payloadOffset > std::numeric_limits<size_t>::max() - payloadLength ||
|
||||
bytes.size() != payloadOffset + payloadLength) {
|
||||
return fail(QbinDecodeErrorCode::LengthMismatch, "total length");
|
||||
}
|
||||
|
||||
std::vector<Descriptor> descriptors;
|
||||
descriptors.reserve(planeCount);
|
||||
std::set<std::pair<uint16_t, uint16_t>> keys;
|
||||
std::tuple<uint16_t, uint16_t, uint8_t> previous{};
|
||||
bool first = true;
|
||||
int validityIndex = -1;
|
||||
size_t expectedOffset = 0;
|
||||
uint64_t decodedBytes = 0;
|
||||
for (uint16_t i = 0; i < planeCount; ++i) {
|
||||
const size_t o = HeaderSize + size_t(i) * DescriptorSize;
|
||||
Descriptor d;
|
||||
d.semantic = readU16(bytes, o);
|
||||
d.dtype = bytes[o + 2];
|
||||
d.flags = bytes[o + 3];
|
||||
d.scaleBits = readU32(bytes, o + 4);
|
||||
d.offsetBits = readU32(bytes, o + 8);
|
||||
d.scale = readF32(bytes, o + 4);
|
||||
d.offset = readF32(bytes, o + 8);
|
||||
d.count = readU32(bytes, o + 12);
|
||||
d.byteOffset = readU32(bytes, o + 16);
|
||||
d.byteLength = readU32(bytes, o + 20);
|
||||
d.unit = readU16(bytes, o + 24);
|
||||
d.group = readU16(bytes, o + 26);
|
||||
d.valueClass = bytes[o + 28];
|
||||
d.component = bytes[o + 29];
|
||||
|
||||
if (readU16(bytes, o + 30) != 0 || (d.flags & ~1u) != 0) {
|
||||
return fail(QbinDecodeErrorCode::InvalidDescriptor, "descriptor reserved/flags " + std::to_string(i));
|
||||
}
|
||||
const auto order = std::tuple(d.semantic, d.group, d.component);
|
||||
if (!first && order <= previous) {
|
||||
return fail(QbinDecodeErrorCode::NonCanonicalDescriptorOrder, "descriptor " + std::to_string(i));
|
||||
}
|
||||
first = false;
|
||||
previous = order;
|
||||
if (!keys.insert({d.semantic, d.group}).second) {
|
||||
return fail(QbinDecodeErrorCode::DuplicatePlaneKey, "descriptor " + std::to_string(i));
|
||||
}
|
||||
if (d.semantic == 0 || d.valueClass > 3 || d.component > 3 || d.count != cells) {
|
||||
return fail(QbinDecodeErrorCode::InvalidDescriptor, "descriptor values " + std::to_string(i));
|
||||
}
|
||||
|
||||
const bool validitySemantic = d.semantic == ValiditySemantic;
|
||||
const bool validityClass = d.valueClass == 3;
|
||||
const bool bitset = d.dtype == 6;
|
||||
if (validitySemantic != validityClass || validityClass != bitset) {
|
||||
return fail(QbinDecodeErrorCode::InvalidValidity, "VALIDITY semantic/valueClass/dtype");
|
||||
}
|
||||
|
||||
if (validityClass) {
|
||||
if (validityIndex != -1 || d.unit != 0 || d.group != 0 || d.component != 0 ||
|
||||
(d.flags & 1u) == 0 || d.scaleBits != 0x3f800000u || d.offsetBits != 0u) {
|
||||
return fail(QbinDecodeErrorCode::InvalidValidity, "validity descriptor");
|
||||
}
|
||||
validityIndex = i;
|
||||
} else {
|
||||
if (d.dtype < 1 || d.dtype > 4 || !std::isfinite(d.scale) || d.scale <= 0.0f ||
|
||||
!std::isfinite(d.offset) || (d.offset == 0.0f && d.offsetBits != 0u)) {
|
||||
return fail(QbinDecodeErrorCode::InvalidDescriptor, "numeric descriptor " + std::to_string(i));
|
||||
}
|
||||
|
||||
if (d.valueClass == 0 && d.component != 0) {
|
||||
return fail(QbinDecodeErrorCode::InvalidDescriptor, "scalar component");
|
||||
}
|
||||
if (d.valueClass == 1 &&
|
||||
(d.unit != 1 || d.component < 1 || d.component > 3 || d.group == 0)) {
|
||||
return fail(QbinDecodeErrorCode::InvalidDescriptor, "velocity descriptor");
|
||||
}
|
||||
if (d.valueClass == 2) {
|
||||
if (d.dtype != 2 || d.unit != 4 || d.component != 0 || d.group != 0 || d.offsetBits != 0u) {
|
||||
return fail(QbinDecodeErrorCode::InvalidDescriptor, "direction descriptor");
|
||||
}
|
||||
const auto n = directionCount(d.scale);
|
||||
if (!n) return fail(QbinDecodeErrorCode::InvalidDescriptor, "direction canonical scale");
|
||||
d.directionN = *n;
|
||||
}
|
||||
}
|
||||
|
||||
if (d.byteOffset != expectedOffset || d.byteOffset > payloadLength ||
|
||||
d.byteLength > payloadLength - d.byteOffset) {
|
||||
return fail(d.byteOffset != expectedOffset ? QbinDecodeErrorCode::NonCanonicalPacking
|
||||
: QbinDecodeErrorCode::InvalidDescriptor,
|
||||
"descriptor packing " + std::to_string(i));
|
||||
}
|
||||
expectedOffset = align16(size_t(d.byteOffset) + d.byteLength);
|
||||
|
||||
const uint64_t logicalBytes = logicalDecodedBytes(d, cells);
|
||||
if (logicalBytes == std::numeric_limits<uint64_t>::max() ||
|
||||
decodedBytes > MaxDecoded - logicalBytes) {
|
||||
return fail(QbinDecodeErrorCode::LimitExceeded, "decoded bytes");
|
||||
}
|
||||
decodedBytes += logicalBytes;
|
||||
descriptors.push_back(d);
|
||||
}
|
||||
|
||||
if (validityIndex < 0) return fail(QbinDecodeErrorCode::InvalidValidity, "missing validity");
|
||||
if (expectedOffset != payloadLength) {
|
||||
return fail(QbinDecodeErrorCode::NonCanonicalPacking, "payload length");
|
||||
}
|
||||
|
||||
if (crc32(bytes) != readU32(bytes, 76)) {
|
||||
return fail(QbinDecodeErrorCode::CrcMismatch, "file CRC32");
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < descriptors.size(); ++i) {
|
||||
const auto& d = descriptors[i];
|
||||
const size_t end = size_t(d.byteOffset) + d.byteLength;
|
||||
const size_t next = i + 1 < descriptors.size() ? descriptors[i + 1].byteOffset : payloadLength;
|
||||
if (next > end && !allZero(bytes.subspan(payloadOffset + end, next - end))) {
|
||||
return fail(QbinDecodeErrorCode::NonZeroPadding, "plane/tail padding");
|
||||
}
|
||||
}
|
||||
|
||||
const auto& validityDescriptor = descriptors[size_t(validityIndex)];
|
||||
const size_t expectedValidityBytes = size_t((cells + 7u) / 8u);
|
||||
if (validityDescriptor.byteLength != expectedValidityBytes) {
|
||||
return fail(QbinDecodeErrorCode::InvalidValidity, "validity byteLength");
|
||||
}
|
||||
const auto validity = bytes.subspan(payloadOffset + validityDescriptor.byteOffset,
|
||||
validityDescriptor.byteLength);
|
||||
if ((cells & 7u) != 0) {
|
||||
const uint8_t unusedMask = uint8_t(0xffu << (cells & 7u));
|
||||
if ((validity.back() & unusedMask) != 0) {
|
||||
return fail(QbinDecodeErrorCode::InvalidValidity, "unused validity bits");
|
||||
}
|
||||
}
|
||||
|
||||
uint64_t validCount = 0;
|
||||
for (uint64_t i = 0; i < cells; ++i) {
|
||||
if (validAt(validity, i)) ++validCount;
|
||||
}
|
||||
for (size_t i = 0; i < descriptors.size(); ++i) {
|
||||
if (int(i) == validityIndex) continue;
|
||||
if ((validCount == 0 && descriptors[i].byteLength != 0) ||
|
||||
(validCount != 0 && descriptors[i].byteLength == 0)) {
|
||||
return fail(QbinDecodeErrorCode::InvalidDescriptor, "numeric byteLength vs validity");
|
||||
}
|
||||
}
|
||||
|
||||
auto output = std::make_shared<DecodedFieldFrame>();
|
||||
output->frameContentIdentity = {
|
||||
meta.fieldID,
|
||||
meta.generation,
|
||||
meta.dataVersion,
|
||||
uint64_t(validTime) * 1000u,
|
||||
};
|
||||
output->referenceTimeMs = uint64_t(referenceTime) * 1000u;
|
||||
output->grid = {1, 1, 1, 1, width, height, west, south, east, north};
|
||||
|
||||
for (size_t i = 0; i < descriptors.size(); ++i) {
|
||||
const auto& d = descriptors[i];
|
||||
std::shared_ptr<const std::vector<uint8_t>> raw;
|
||||
if (int(i) == validityIndex) {
|
||||
raw = std::make_shared<const std::vector<uint8_t>>(validity.begin(), validity.end());
|
||||
} else if (validCount == 0) {
|
||||
const size_t logicalBytes = size_t(logicalDecodedBytes(d, cells));
|
||||
raw = std::make_shared<const std::vector<uint8_t>>(logicalBytes, 0);
|
||||
} else {
|
||||
const auto encoded = bytes.subspan(payloadOffset + d.byteOffset, d.byteLength);
|
||||
const auto decoded = decodeNumericPlane(d, encoded, validity, width, height);
|
||||
if (!decoded.accepted()) return fail(decoded.code, decoded.detail);
|
||||
raw = decoded.raw;
|
||||
}
|
||||
|
||||
DecodedFieldPlane plane{
|
||||
{d.semantic, d.group, static_cast<FieldComponent>(d.component)},
|
||||
static_cast<FieldDType>(d.dtype),
|
||||
static_cast<FieldValueClass>(d.valueClass),
|
||||
d.unit,
|
||||
bool(d.flags & 1u),
|
||||
d.count,
|
||||
d.scale,
|
||||
d.offset,
|
||||
contentIdentity(d, *raw),
|
||||
raw,
|
||||
};
|
||||
|
||||
if (int(i) == validityIndex) {
|
||||
output->validityContentIdentity = plane.contentIdentity;
|
||||
output->validityBits = raw;
|
||||
output->validityPlane = std::move(plane);
|
||||
} else {
|
||||
output->planes.push_back(std::move(plane));
|
||||
}
|
||||
}
|
||||
|
||||
if (!output->isStructurallyValid()) {
|
||||
return fail(QbinDecodeErrorCode::InvalidFrame, "decoded frame structural validity");
|
||||
}
|
||||
return {QbinDecodeErrorCode::Accepted, "accepted", output};
|
||||
}
|
||||
|
||||
} // namespace mbgl::particle::ingress
|
||||
@@ -428,11 +428,13 @@ RasterArrayEncodedFieldSubmitResult RasterArraySource::submitEncodedField(
|
||||
// checks metadata, but validate it here to make the Host ordering explicit.
|
||||
if (!validEncodedFieldKey(fieldKey)) return RasterArrayEncodedFieldSubmitResult::RejectedInvalidFieldKey;
|
||||
if (!validRuntimeMetadata(runtimeMetadata)) return RasterArrayEncodedFieldSubmitResult::RejectedDecode;
|
||||
const auto decoded = particle::ingress::QbinV2Decoder::decode(runtimeMetadata, rawQbinBytes);
|
||||
if (!decoded.accepted()) return RasterArrayEncodedFieldSubmitResult::RejectedDecode;
|
||||
if (!particle::FieldProfileValidator::validate(contract, *decoded.frame).accepted())
|
||||
auto ingress = particle::ingress::EncodedFieldIngress::decodeAndValidate(
|
||||
runtimeMetadata, &contract, rawQbinBytes);
|
||||
if (ingress.code == particle::ingress::EncodedFieldIngressCode::DecodeRejected)
|
||||
return RasterArrayEncodedFieldSubmitResult::RejectedDecode;
|
||||
if (ingress.code != particle::ingress::EncodedFieldIngressCode::Accepted)
|
||||
return RasterArrayEncodedFieldSubmitResult::RejectedContract;
|
||||
const auto result = mapEncodedFieldStoreResult(encodedFieldStore.submit(fieldKey, std::move(decoded.frame)));
|
||||
const auto result = mapEncodedFieldStoreResult(encodedFieldStore.submit(fieldKey, std::move(ingress.frame)));
|
||||
if (result == RasterArrayEncodedFieldSubmitResult::AcceptedInserted ||
|
||||
result == RasterArrayEncodedFieldSubmitResult::AcceptedReplacedRevision)
|
||||
refreshV4RendererState();
|
||||
|
||||
@@ -22,6 +22,7 @@ add_library(
|
||||
${PROJECT_SOURCE_DIR}/test/particle/field_binding.test.cpp
|
||||
${PROJECT_SOURCE_DIR}/test/particle/field_profile_contract_json.test.cpp
|
||||
${PROJECT_SOURCE_DIR}/test/particle/qbin_v2_decoder.test.cpp
|
||||
${PROJECT_SOURCE_DIR}/test/particle/qbin_v3_decoder.test.cpp
|
||||
${PROJECT_SOURCE_DIR}/test/particle/encoded_field_transport.test.cpp
|
||||
${PROJECT_SOURCE_DIR}/test/plugin/plugin.test.cpp
|
||||
${PROJECT_SOURCE_DIR}/test/renderer/image_manager.test.cpp
|
||||
|
||||
618
test/particle/qbin_v3_decoder.test.cpp
Normal file
618
test/particle/qbin_v3_decoder.test.cpp
Normal file
@@ -0,0 +1,618 @@
|
||||
#include <mbgl/particle/ingress/qbin_v3_decoder.hpp>
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <cmath>
|
||||
#include <cstring>
|
||||
#include <optional>
|
||||
#include <vector>
|
||||
|
||||
using namespace mbgl;
|
||||
namespace {
|
||||
|
||||
constexpr size_t Header = 112;
|
||||
constexpr size_t Descriptor = 32;
|
||||
|
||||
void u16(std::vector<uint8_t>& b, size_t o, uint16_t v) {
|
||||
b[o] = uint8_t(v);
|
||||
b[o + 1] = uint8_t(v >> 8);
|
||||
}
|
||||
void u32(std::vector<uint8_t>& b, size_t o, uint32_t v) {
|
||||
for (size_t i = 0; i < 4; ++i) b[o + i] = uint8_t(v >> (8 * i));
|
||||
}
|
||||
uint32_t r32(const std::vector<uint8_t>& b, size_t o) {
|
||||
uint32_t v = 0;
|
||||
for (size_t i = 0; i < 4; ++i) v |= uint32_t(b[o + i]) << (8 * i);
|
||||
return v;
|
||||
}
|
||||
void u64(std::vector<uint8_t>& b, size_t o, uint64_t v) {
|
||||
for (size_t i = 0; i < 8; ++i) b[o + i] = uint8_t(v >> (8 * i));
|
||||
}
|
||||
void i64(std::vector<uint8_t>& b, size_t o, int64_t v) {
|
||||
u64(b, o, static_cast<uint64_t>(v));
|
||||
}
|
||||
void f32(std::vector<uint8_t>& b, size_t o, float v) {
|
||||
uint32_t bits = 0;
|
||||
std::memcpy(&bits, &v, sizeof(bits));
|
||||
u32(b, o, bits);
|
||||
}
|
||||
void f64(std::vector<uint8_t>& b, size_t o, double v) {
|
||||
uint64_t bits = 0;
|
||||
std::memcpy(&bits, &v, sizeof(bits));
|
||||
u64(b, o, bits);
|
||||
}
|
||||
size_t a16(size_t x) { return (x + 15u) & ~size_t(15u); }
|
||||
uint32_t crc(const std::vector<uint8_t>& b) {
|
||||
uint32_t c = ~0u;
|
||||
for (size_t i = 0; i < b.size(); ++i) {
|
||||
const uint8_t x = i >= 76 && i < 80 ? 0 : b[i];
|
||||
c ^= x;
|
||||
for (int j = 0; j < 8; ++j) c = (c >> 1) ^ ((c & 1u) ? 0xedb88320u : 0);
|
||||
}
|
||||
return ~c;
|
||||
}
|
||||
void recalc(std::vector<uint8_t>& b) { u32(b, 76, crc(b)); }
|
||||
bool validAt(const std::vector<uint8_t>& v, uint64_t i) {
|
||||
return ((v[i >> 3] >> (i & 7u)) & 1u) != 0;
|
||||
}
|
||||
int64_t normDir(int64_t x, uint32_t n) {
|
||||
if (x < 0) x += n;
|
||||
else if (x >= int64_t(n)) x -= n;
|
||||
return x;
|
||||
}
|
||||
uint64_t zigzag(int64_t r) {
|
||||
return r >= 0 ? uint64_t(2 * r) : uint64_t(-2 * r - 1);
|
||||
}
|
||||
|
||||
struct Bits {
|
||||
std::vector<uint8_t> bytes;
|
||||
size_t pos = 0;
|
||||
void bit(uint8_t value) {
|
||||
if ((pos & 7u) == 0) bytes.push_back(0);
|
||||
if (value) bytes.back() |= uint8_t(1u << (pos & 7u));
|
||||
++pos;
|
||||
}
|
||||
void bits(uint32_t value, uint8_t count) {
|
||||
for (uint8_t i = 0; i < count; ++i) bit((value >> i) & 1u);
|
||||
}
|
||||
};
|
||||
|
||||
struct PlaneSpec {
|
||||
uint16_t semantic = 1;
|
||||
uint8_t dtype = 1;
|
||||
uint8_t flags = 1;
|
||||
float scale = 1.0f;
|
||||
float offset = 0.0f;
|
||||
uint16_t unit = 0;
|
||||
uint16_t group = 0;
|
||||
uint8_t valueClass = 0;
|
||||
uint8_t component = 0;
|
||||
std::vector<int64_t> values;
|
||||
};
|
||||
|
||||
int64_t predict(const PlaneSpec& p,
|
||||
const std::vector<uint8_t>& validity,
|
||||
uint32_t width,
|
||||
uint32_t row,
|
||||
uint32_t col,
|
||||
uint32_t n) {
|
||||
const uint64_t i = uint64_t(row) * width + col;
|
||||
const bool left = col > 0 && validAt(validity, i - 1);
|
||||
const bool up = row > 0 && validAt(validity, i - width);
|
||||
const bool upperLeft = row > 0 && col > 0 && validAt(validity, i - width - 1);
|
||||
int64_t predicted = 0;
|
||||
if (left && up && upperLeft) {
|
||||
predicted = p.values[i - 1] + p.values[i - width] - p.values[i - width - 1];
|
||||
} else if (left) {
|
||||
predicted = p.values[i - 1];
|
||||
} else if (up) {
|
||||
predicted = p.values[i - width];
|
||||
}
|
||||
return n ? normDir(predicted, n) : predicted;
|
||||
}
|
||||
|
||||
std::vector<uint8_t> encodePlane(const PlaneSpec& p,
|
||||
const std::vector<uint8_t>& validity,
|
||||
uint32_t width,
|
||||
uint32_t height,
|
||||
std::optional<uint8_t> forcedK = std::nullopt) {
|
||||
const uint32_t n = p.valueClass == 2
|
||||
? uint32_t(std::floor(360.0 / double(p.scale) + 0.5))
|
||||
: 0;
|
||||
std::vector<uint8_t> out;
|
||||
for (uint32_t rb = 0; rb < height; rb += 16) {
|
||||
for (uint32_t cb = 0; cb < width; cb += 16) {
|
||||
std::vector<uint64_t> symbols;
|
||||
for (uint32_t row = rb; row < std::min(rb + 16u, height); ++row) {
|
||||
for (uint32_t col = cb; col < std::min(cb + 16u, width); ++col) {
|
||||
const uint64_t i = uint64_t(row) * width + col;
|
||||
if (!validAt(validity, i)) continue;
|
||||
const int64_t predicted = predict(p, validity, width, row, col, n);
|
||||
int64_t residual = p.values[i] - predicted;
|
||||
if (n) {
|
||||
int64_t m = residual;
|
||||
if (m < 0) m += n;
|
||||
residual = 2 * m > int64_t(n) ? m - n
|
||||
: 2 * m < int64_t(n) ? m
|
||||
: -int64_t(n / 2);
|
||||
}
|
||||
symbols.push_back(zigzag(residual));
|
||||
}
|
||||
}
|
||||
if (symbols.empty()) continue;
|
||||
|
||||
uint8_t bestK = forcedK.value_or(0);
|
||||
if (!forcedK) {
|
||||
uint64_t bestBits = ~uint64_t(0);
|
||||
for (uint8_t k = 0; k <= 31; ++k) {
|
||||
uint64_t bits = 0;
|
||||
for (const auto z : symbols) bits += (z >> k) + 1u + k;
|
||||
if (bits < bestBits) {
|
||||
bestBits = bits;
|
||||
bestK = k;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
out.push_back(bestK);
|
||||
Bits writer;
|
||||
for (const auto z : symbols) {
|
||||
const uint64_t q = z >> bestK;
|
||||
for (uint64_t j = 0; j < q; ++j) writer.bit(0);
|
||||
writer.bit(1);
|
||||
if (bestK) writer.bits(uint32_t(z & ((uint64_t(1) << bestK) - 1u)), bestK);
|
||||
}
|
||||
out.insert(out.end(), writer.bytes.begin(), writer.bytes.end());
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
std::vector<uint8_t> rawBytes(const PlaneSpec& p, const std::vector<uint8_t>& validity) {
|
||||
const size_t stride = p.dtype == 1 || p.dtype == 2 ? 2u : 1u;
|
||||
std::vector<uint8_t> out(p.values.size() * stride, 0);
|
||||
for (size_t i = 0; i < p.values.size(); ++i) {
|
||||
if (!validAt(validity, i)) continue;
|
||||
if (stride == 2) {
|
||||
const uint16_t v = static_cast<uint16_t>(p.values[i]);
|
||||
out[i * 2] = uint8_t(v);
|
||||
out[i * 2 + 1] = uint8_t(v >> 8);
|
||||
} else {
|
||||
out[i] = static_cast<uint8_t>(p.values[i]);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
std::vector<uint8_t> fixture(uint32_t width,
|
||||
uint32_t height,
|
||||
std::vector<uint8_t> validity,
|
||||
std::vector<PlaneSpec> planes,
|
||||
std::optional<uint8_t> forcedK = std::nullopt,
|
||||
uint16_t fieldType = 999) {
|
||||
const uint64_t cells = uint64_t(width) * height;
|
||||
const uint16_t count = uint16_t(planes.size() + 1);
|
||||
const size_t payloadOffset = Header + size_t(count) * Descriptor;
|
||||
|
||||
struct Encoded { PlaneSpec p; std::vector<uint8_t> bytes; };
|
||||
std::vector<Encoded> encoded;
|
||||
for (auto& p : planes) encoded.push_back({p, encodePlane(p, validity, width, height, forcedK)});
|
||||
|
||||
size_t payloadLength = 0;
|
||||
for (const auto& e : encoded) payloadLength = a16(payloadLength + e.bytes.size());
|
||||
payloadLength = a16(payloadLength + validity.size());
|
||||
std::vector<uint8_t> b(payloadOffset + payloadLength, 0);
|
||||
std::memcpy(b.data(), "QBV3", 4);
|
||||
b[4] = 3;
|
||||
b[5] = 1;
|
||||
u16(b, 6, fieldType);
|
||||
b[8] = b[9] = b[10] = 1;
|
||||
u16(b, 12, count);
|
||||
u32(b, 16, width);
|
||||
u32(b, 20, height);
|
||||
i64(b, 24, 100);
|
||||
i64(b, 32, 90);
|
||||
f64(b, 40, -10);
|
||||
f64(b, 48, -10);
|
||||
f64(b, 56, 10);
|
||||
f64(b, 64, 10);
|
||||
u32(b, 72, payloadLength);
|
||||
u16(b, 80, 0xffff);
|
||||
|
||||
size_t off = 0;
|
||||
for (size_t i = 0; i < encoded.size(); ++i) {
|
||||
const auto& e = encoded[i];
|
||||
const size_t d = Header + i * Descriptor;
|
||||
u16(b, d, e.p.semantic);
|
||||
b[d + 2] = e.p.dtype;
|
||||
b[d + 3] = e.p.flags;
|
||||
f32(b, d + 4, e.p.scale);
|
||||
f32(b, d + 8, e.p.offset);
|
||||
u32(b, d + 12, cells);
|
||||
u32(b, d + 16, off);
|
||||
u32(b, d + 20, e.bytes.size());
|
||||
u16(b, d + 24, e.p.unit);
|
||||
u16(b, d + 26, e.p.group);
|
||||
b[d + 28] = e.p.valueClass;
|
||||
b[d + 29] = e.p.component;
|
||||
std::copy(e.bytes.begin(), e.bytes.end(), b.begin() + payloadOffset + off);
|
||||
off = a16(off + e.bytes.size());
|
||||
}
|
||||
|
||||
const size_t d = Header + encoded.size() * Descriptor;
|
||||
u16(b, d, 3);
|
||||
b[d + 2] = 6;
|
||||
b[d + 3] = 1;
|
||||
f32(b, d + 4, 1.0f);
|
||||
f32(b, d + 8, 0.0f);
|
||||
u32(b, d + 12, cells);
|
||||
u32(b, d + 16, off);
|
||||
u32(b, d + 20, validity.size());
|
||||
b[d + 28] = 3;
|
||||
std::copy(validity.begin(), validity.end(), b.begin() + payloadOffset + off);
|
||||
recalc(b);
|
||||
return b;
|
||||
}
|
||||
|
||||
std::vector<uint8_t> fixtureV2(uint32_t width,
|
||||
uint32_t height,
|
||||
const std::vector<uint8_t>& validity,
|
||||
const std::vector<PlaneSpec>& planes,
|
||||
uint16_t fieldType = 999) {
|
||||
const uint64_t cells = uint64_t(width) * height;
|
||||
const uint16_t count = uint16_t(planes.size() + 1);
|
||||
const size_t payloadOffset = Header + size_t(count) * Descriptor;
|
||||
|
||||
struct RawPlane {
|
||||
PlaneSpec p;
|
||||
std::vector<uint8_t> bytes;
|
||||
};
|
||||
std::vector<RawPlane> raws;
|
||||
for (const auto& p : planes) raws.push_back({p, rawBytes(p, validity)});
|
||||
|
||||
size_t payloadLength = 0;
|
||||
for (const auto& r : raws) payloadLength = a16(payloadLength + r.bytes.size());
|
||||
payloadLength = a16(payloadLength + validity.size());
|
||||
|
||||
std::vector<uint8_t> b(payloadOffset + payloadLength, 0);
|
||||
std::memcpy(b.data(), "QBV2", 4);
|
||||
b[4] = 2;
|
||||
b[5] = 1;
|
||||
u16(b, 6, fieldType);
|
||||
b[8] = b[9] = b[10] = 1;
|
||||
u16(b, 12, count);
|
||||
u32(b, 16, width);
|
||||
u32(b, 20, height);
|
||||
i64(b, 24, 100);
|
||||
i64(b, 32, 90);
|
||||
f64(b, 40, -10);
|
||||
f64(b, 48, -10);
|
||||
f64(b, 56, 10);
|
||||
f64(b, 64, 10);
|
||||
u32(b, 72, payloadLength);
|
||||
u16(b, 80, 0xffff);
|
||||
|
||||
size_t off = 0;
|
||||
for (size_t i = 0; i < raws.size(); ++i) {
|
||||
const auto& r = raws[i];
|
||||
const size_t d = Header + i * Descriptor;
|
||||
u16(b, d, r.p.semantic);
|
||||
b[d + 2] = r.p.dtype;
|
||||
b[d + 3] = r.p.flags;
|
||||
f32(b, d + 4, r.p.scale);
|
||||
f32(b, d + 8, r.p.offset);
|
||||
u32(b, d + 12, cells);
|
||||
u32(b, d + 16, off);
|
||||
u32(b, d + 20, r.bytes.size());
|
||||
u16(b, d + 24, r.p.unit);
|
||||
u16(b, d + 26, r.p.group);
|
||||
b[d + 28] = r.p.valueClass;
|
||||
b[d + 29] = r.p.component;
|
||||
std::copy(r.bytes.begin(), r.bytes.end(), b.begin() + payloadOffset + off);
|
||||
off = a16(off + r.bytes.size());
|
||||
}
|
||||
|
||||
const size_t d = Header + raws.size() * Descriptor;
|
||||
u16(b, d, 3);
|
||||
b[d + 2] = 6;
|
||||
b[d + 3] = 1;
|
||||
f32(b, d + 4, 1.0f);
|
||||
f32(b, d + 8, 0.0f);
|
||||
u32(b, d + 12, cells);
|
||||
u32(b, d + 16, off);
|
||||
u32(b, d + 20, validity.size());
|
||||
b[d + 28] = 3;
|
||||
std::copy(validity.begin(), validity.end(), b.begin() + payloadOffset + off);
|
||||
recalc(b);
|
||||
return b;
|
||||
}
|
||||
|
||||
std::vector<uint8_t> validityBits(uint32_t cells, bool allValid = true) {
|
||||
std::vector<uint8_t> v((cells + 7u) / 8u, allValid ? 0xffu : 0u);
|
||||
if (allValid && (cells & 7u)) v.back() = uint8_t((1u << (cells & 7u)) - 1u);
|
||||
return v;
|
||||
}
|
||||
|
||||
particle::ingress::EncodedFieldRuntimeMetadata meta() {
|
||||
return {"qbv3-test", 1, 1};
|
||||
}
|
||||
|
||||
using Code = particle::ingress::QbinDecodeErrorCode;
|
||||
|
||||
} // namespace
|
||||
|
||||
TEST(ParticleQbinV3Decoder, DecodesAcrossBlockBoundaryAndDirectionWrap) {
|
||||
constexpr uint32_t width = 17, height = 17, cells = width * height;
|
||||
auto validity = validityBits(cells);
|
||||
validity[3] &= uint8_t(~(1u << 1)); // one invalid cell
|
||||
std::vector<int64_t> scalar(cells), direction(cells);
|
||||
for (uint32_t r = 0; r < height; ++r) for (uint32_t c = 0; c < width; ++c) {
|
||||
const size_t i = r * width + c;
|
||||
scalar[i] = int64_t(r) * 40 - int64_t(c) * 7;
|
||||
direction[i] = (350 + int64_t(c) * 3 + int64_t(r) * 11) % 360;
|
||||
}
|
||||
PlaneSpec a{1, 1, 1, 1.0f, 0.0f, 77, 0, 0, 0, scalar};
|
||||
PlaneSpec d{2, 2, 1, 1.0f, 0.0f, 4, 0, 2, 0, direction};
|
||||
auto bytes = fixture(width, height, validity, {a, d});
|
||||
auto result = particle::ingress::QbinV3Decoder::decode(meta(), bytes);
|
||||
ASSERT_TRUE(result.accepted()) << result.detail;
|
||||
ASSERT_EQ(result.frame->planes.size(), 2u);
|
||||
EXPECT_EQ(*result.frame->planes[0].storage, rawBytes(a, validity));
|
||||
EXPECT_EQ(*result.frame->planes[1].storage, rawBytes(d, validity));
|
||||
EXPECT_EQ(*result.frame->validityBits, validity);
|
||||
}
|
||||
|
||||
TEST(ParticleQbinV3Decoder, AcceptsAllInvalidWithSharedZeroLengthOffsets) {
|
||||
constexpr uint32_t width = 2, height = 2, cells = 4;
|
||||
auto validity = validityBits(cells, false);
|
||||
PlaneSpec a{1, 1, 1, 1.0f, 0.0f, 77, 0, 0, 0, {1,2,3,4}};
|
||||
PlaneSpec d{2, 2, 1, 1.0f, 0.0f, 4, 0, 2, 0, {0,90,180,270}};
|
||||
auto bytes = fixture(width, height, validity, {a, d});
|
||||
const size_t d0 = Header, d1 = Header + Descriptor, dv = Header + 2 * Descriptor;
|
||||
EXPECT_EQ(r32(bytes, d0 + 16), 0u);
|
||||
EXPECT_EQ(r32(bytes, d1 + 16), 0u);
|
||||
EXPECT_EQ(r32(bytes, dv + 16), 0u);
|
||||
auto result = particle::ingress::QbinV3Decoder::decode(meta(), bytes);
|
||||
ASSERT_TRUE(result.accepted()) << result.detail;
|
||||
EXPECT_EQ(*result.frame->planes[0].storage, std::vector<uint8_t>(cells * 2, 0));
|
||||
EXPECT_EQ(*result.frame->planes[1].storage, std::vector<uint8_t>(cells * 2, 0));
|
||||
}
|
||||
|
||||
TEST(ParticleQbinV3Decoder, AcceptsNonOptimalRiceK) {
|
||||
auto validity = validityBits(4);
|
||||
PlaneSpec p{1, 1, 1, 1.0f, 0.0f, 77, 0, 0, 0, {0,0,0,0}};
|
||||
auto bytes = fixture(2, 2, validity, {p}, uint8_t(5));
|
||||
auto result = particle::ingress::QbinV3Decoder::decode(meta(), bytes);
|
||||
EXPECT_TRUE(result.accepted()) << result.detail;
|
||||
}
|
||||
|
||||
|
||||
TEST(ParticleQbinV3Decoder, RejectsCrcDescriptorAndValidityCorruption) {
|
||||
auto validity = validityBits(4);
|
||||
PlaneSpec p{1, 1, 1, 1.0f, 0.0f, 77, 0, 0, 0, {1,2,3,4}};
|
||||
|
||||
auto bytes = fixture(2, 2, validity, {p});
|
||||
bytes[6] ^= 1;
|
||||
EXPECT_EQ(particle::ingress::QbinV3Decoder::decode(meta(), bytes).code, Code::CrcMismatch);
|
||||
|
||||
bytes = fixture(2, 2, validity, {p});
|
||||
bytes[Header + 2] = 5;
|
||||
recalc(bytes);
|
||||
EXPECT_EQ(particle::ingress::QbinV3Decoder::decode(meta(), bytes).code, Code::InvalidDescriptor);
|
||||
|
||||
bytes = fixture(2, 2, validity, {p});
|
||||
const size_t validityDescriptor = Header + Descriptor;
|
||||
bytes[validityDescriptor + 2] = 3;
|
||||
recalc(bytes);
|
||||
EXPECT_EQ(particle::ingress::QbinV3Decoder::decode(meta(), bytes).code, Code::InvalidValidity);
|
||||
}
|
||||
|
||||
TEST(ParticleQbinV3Decoder, RejectsMalformedRiceAndNonzeroBlockPadding) {
|
||||
auto validity = validityBits(4);
|
||||
PlaneSpec p{1, 1, 1, 1.0f, 0.0f, 77, 0, 0, 0, {0,0,0,0}};
|
||||
|
||||
auto bytes = fixture(2, 2, validity, {p});
|
||||
const size_t payload = Header + 2 * Descriptor;
|
||||
bytes[payload] = 32;
|
||||
recalc(bytes);
|
||||
EXPECT_EQ(particle::ingress::QbinV3Decoder::decode(meta(), bytes).code, Code::MalformedRice);
|
||||
|
||||
auto oneValid = validityBits(4, false);
|
||||
oneValid[0] = 0x01;
|
||||
bytes = fixture(2, 2, oneValid, {p}, uint8_t(0));
|
||||
bytes[payload + 1] |= 0x80;
|
||||
recalc(bytes);
|
||||
EXPECT_EQ(particle::ingress::QbinV3Decoder::decode(meta(), bytes).code, Code::MalformedRice);
|
||||
}
|
||||
|
||||
TEST(ParticleQbinV3Decoder, RejectsResidualOutsideLogicalRange) {
|
||||
auto validity = validityBits(4);
|
||||
PlaneSpec p{1, 3, 1, 1.0f, 0.0f, 77, 0, 0, 0, {0,0,0,0}};
|
||||
auto bytes = fixture(2, 2, validity, {p}, uint8_t(31));
|
||||
const size_t payload = Header + 2 * Descriptor;
|
||||
ASSERT_GT(bytes.size(), payload + 4);
|
||||
bytes[payload + 4] |= 0x80;
|
||||
recalc(bytes);
|
||||
EXPECT_EQ(particle::ingress::QbinV3Decoder::decode(meta(), bytes).code, Code::ResidualOutOfRange);
|
||||
}
|
||||
|
||||
TEST(ParticleQbinV3Decoder, PreservesNodeRegisteredGridGeometry) {
|
||||
auto validity = validityBits(9);
|
||||
PlaneSpec p{1, 1, 1, 1.0f, 0.0f, 77, 0, 0, 0, std::vector<int64_t>(9, 0)};
|
||||
auto bytes = fixture(3, 3, validity, {p});
|
||||
auto result = particle::ingress::QbinV3Decoder::decode(meta(), bytes);
|
||||
ASSERT_TRUE(result.accepted()) << result.detail;
|
||||
|
||||
const auto& g = result.frame->grid;
|
||||
EXPECT_EQ(g.width, 3u);
|
||||
EXPECT_EQ(g.height, 3u);
|
||||
EXPECT_DOUBLE_EQ(g.west, -10.0);
|
||||
EXPECT_DOUBLE_EQ(g.south, -10.0);
|
||||
EXPECT_DOUBLE_EQ(g.east, 10.0);
|
||||
EXPECT_DOUBLE_EQ(g.north, 10.0);
|
||||
EXPECT_DOUBLE_EQ(g.west + (g.east - g.west) * 1.0 / (g.width - 1), 0.0);
|
||||
EXPECT_DOUBLE_EQ(g.south + (g.north - g.south) * 1.0 / (g.height - 1), 0.0);
|
||||
}
|
||||
|
||||
TEST(ParticleEncodedFieldIngress, DispatchesQbinV3AndRejectsUnknownMagic) {
|
||||
auto validity = validityBits(4);
|
||||
PlaneSpec u{1, 1, 1, 1.0f, 0.0f, 1, 1, 1, 1, {1,2,3,4}};
|
||||
PlaneSpec v{2, 1, 1, 1.0f, 0.0f, 1, 1, 1, 2, {5,6,7,8}};
|
||||
auto bytes = fixture(2, 2, validity, {u, v});
|
||||
|
||||
particle::ResolvedFieldProfileContract contract;
|
||||
contract.profileID = "qbv3.wind.test";
|
||||
contract.profileVersion = 1;
|
||||
contract.requiredPlanes = {
|
||||
{1, particle::FieldValueClass::VelocityComponent, 1, 1, particle::FieldComponent::UEast, true},
|
||||
{2, particle::FieldValueClass::VelocityComponent, 1, 1, particle::FieldComponent::VNorth, true},
|
||||
{3, particle::FieldValueClass::Validity, 0, 0, particle::FieldComponent::None, true},
|
||||
};
|
||||
|
||||
auto accepted = particle::ingress::EncodedFieldIngress::decodeAndValidate(meta(), &contract, bytes);
|
||||
EXPECT_TRUE(accepted.accepted()) << accepted.detail;
|
||||
|
||||
bytes[0] = 'X';
|
||||
recalc(bytes);
|
||||
auto rejected = particle::ingress::EncodedFieldIngress::decodeAndValidate(meta(), &contract, bytes);
|
||||
EXPECT_EQ(rejected.code, particle::ingress::EncodedFieldIngressCode::DecodeRejected);
|
||||
EXPECT_EQ(rejected.decodeCode, Code::BadMagic);
|
||||
}
|
||||
|
||||
|
||||
TEST(ParticleQbinV3Decoder, CoversAllCompressedIntegerDtypes) {
|
||||
auto validity = validityBits(4);
|
||||
PlaneSpec u8{1, 3, 1, 0.5f, 1.0f, 77, 0, 0, 0, {0, 255, 128, 1}};
|
||||
PlaneSpec i8{2, 4, 1, 2.0f, -5.0f, 78, 0, 0, 0, {-128, 127, -1, 0}};
|
||||
auto bytes = fixture(2, 2, validity, {u8, i8});
|
||||
auto result = particle::ingress::QbinV3Decoder::decode(meta(), bytes);
|
||||
ASSERT_TRUE(result.accepted()) << result.detail;
|
||||
ASSERT_EQ(result.frame->planes.size(), 2u);
|
||||
EXPECT_EQ(*result.frame->planes[0].storage, rawBytes(u8, validity));
|
||||
EXPECT_EQ(*result.frame->planes[1].storage, rawBytes(i8, validity));
|
||||
}
|
||||
|
||||
TEST(ParticleQbinV3Decoder, AcceptsOpaqueFieldTypeZeroAndUnknown) {
|
||||
auto validity = validityBits(4);
|
||||
PlaneSpec p{1, 1, 1, 1.0f, 0.0f, 77, 0, 0, 0, {1, 2, 3, 4}};
|
||||
EXPECT_TRUE(particle::ingress::QbinV3Decoder::decode(meta(), fixture(2, 2, validity, {p}, std::nullopt, 0)).accepted());
|
||||
EXPECT_TRUE(particle::ingress::QbinV3Decoder::decode(meta(), fixture(2, 2, validity, {p}, std::nullopt, 65535)).accepted());
|
||||
}
|
||||
|
||||
TEST(ParticleQbinV3Decoder, RejectsHeaderVersionEndianReservedBoundsAndNegativeZero) {
|
||||
auto validity = validityBits(4);
|
||||
PlaneSpec p{1, 1, 1, 1.0f, 0.0f, 77, 0, 0, 0, {1, 2, 3, 4}};
|
||||
|
||||
auto bytes = fixture(2, 2, validity, {p});
|
||||
bytes[0] = 'X';
|
||||
recalc(bytes);
|
||||
EXPECT_EQ(particle::ingress::QbinV3Decoder::decode(meta(), bytes).code, Code::BadMagic);
|
||||
|
||||
bytes = fixture(2, 2, validity, {p});
|
||||
bytes[4] = 2;
|
||||
recalc(bytes);
|
||||
EXPECT_EQ(particle::ingress::QbinV3Decoder::decode(meta(), bytes).code, Code::UnsupportedVersion);
|
||||
|
||||
bytes = fixture(2, 2, validity, {p});
|
||||
bytes[11] = 1;
|
||||
recalc(bytes);
|
||||
EXPECT_EQ(particle::ingress::QbinV3Decoder::decode(meta(), bytes).code, Code::ReservedNonZero);
|
||||
|
||||
bytes = fixture(2, 2, validity, {p});
|
||||
bytes[16] = 0;
|
||||
bytes[17] = 0;
|
||||
bytes[18] = 0;
|
||||
bytes[19] = 2;
|
||||
recalc(bytes);
|
||||
EXPECT_EQ(particle::ingress::QbinV3Decoder::decode(meta(), bytes).code, Code::LimitExceeded);
|
||||
|
||||
bytes = fixture(2, 2, validity, {p});
|
||||
f64(bytes, 40, 11.0);
|
||||
recalc(bytes);
|
||||
EXPECT_EQ(particle::ingress::QbinV3Decoder::decode(meta(), bytes).code, Code::InvalidBounds);
|
||||
|
||||
bytes = fixture(2, 2, validity, {p});
|
||||
u64(bytes, 40, 0x8000000000000000ull);
|
||||
recalc(bytes);
|
||||
EXPECT_EQ(particle::ingress::QbinV3Decoder::decode(meta(), bytes).code, Code::InvalidBounds);
|
||||
}
|
||||
|
||||
TEST(ParticleQbinV3Decoder, RejectsDescriptorOrderDuplicateAndPacking) {
|
||||
auto validity = validityBits(4);
|
||||
PlaneSpec u{1, 1, 1, 1.0f, 0.0f, 1, 1, 1, 1, {1, 2, 3, 4}};
|
||||
PlaneSpec v{2, 1, 1, 1.0f, 0.0f, 1, 1, 1, 2, {5, 6, 7, 8}};
|
||||
|
||||
auto bytes = fixture(2, 2, validity, {u, v});
|
||||
u16(bytes, Header, 2);
|
||||
u16(bytes, Header + Descriptor, 1);
|
||||
recalc(bytes);
|
||||
EXPECT_EQ(particle::ingress::QbinV3Decoder::decode(meta(), bytes).code, Code::NonCanonicalDescriptorOrder);
|
||||
|
||||
bytes = fixture(2, 2, validity, {u, v});
|
||||
u16(bytes, Header + Descriptor, 1);
|
||||
recalc(bytes);
|
||||
EXPECT_EQ(particle::ingress::QbinV3Decoder::decode(meta(), bytes).code, Code::DuplicatePlaneKey);
|
||||
|
||||
bytes = fixture(2, 2, validity, {u, v});
|
||||
u32(bytes, Header + Descriptor + 16, 0);
|
||||
recalc(bytes);
|
||||
EXPECT_EQ(particle::ingress::QbinV3Decoder::decode(meta(), bytes).code, Code::NonCanonicalPacking);
|
||||
}
|
||||
|
||||
TEST(ParticleQbinV3Decoder, RejectsUnterminatedUnaryExtraEncodedByteAndRawOverflow) {
|
||||
auto validity = validityBits(4);
|
||||
PlaneSpec zero{1, 1, 1, 1.0f, 0.0f, 77, 0, 0, 0, {0, 0, 0, 0}};
|
||||
const size_t payload = Header + 2 * Descriptor;
|
||||
|
||||
auto bytes = fixture(2, 2, validity, {zero}, uint8_t(0));
|
||||
bytes[payload + 1] = 0;
|
||||
recalc(bytes);
|
||||
EXPECT_EQ(particle::ingress::QbinV3Decoder::decode(meta(), bytes).code, Code::MalformedRice);
|
||||
|
||||
bytes = fixture(2, 2, validity, {zero}, uint8_t(0));
|
||||
const uint32_t length = r32(bytes, Header + 20);
|
||||
ASSERT_LT(payload + length, bytes.size());
|
||||
u32(bytes, Header + 20, length + 1);
|
||||
bytes[payload + length] = 0;
|
||||
recalc(bytes);
|
||||
EXPECT_EQ(particle::ingress::QbinV3Decoder::decode(meta(), bytes).code, Code::MalformedRice);
|
||||
|
||||
PlaneSpec overflow{1, 3, 1, 1.0f, 0.0f, 77, 0, 0, 0, {300, 0, 0, 0}};
|
||||
bytes = fixture(2, 2, validity, {overflow});
|
||||
EXPECT_EQ(particle::ingress::QbinV3Decoder::decode(meta(), bytes).code, Code::InvalidCellPayload);
|
||||
}
|
||||
|
||||
TEST(ParticleQbinV3Decoder, EnforcesDirectionHalfCircleNegativeTie) {
|
||||
auto validity = validityBits(4);
|
||||
PlaneSpec direction{1, 2, 1, 90.0f, 0.0f, 4, 0, 2, 0, {0, 2, 0, 2}};
|
||||
auto bytes = fixture(2, 2, validity, {direction}, uint8_t(0));
|
||||
auto canonical = particle::ingress::QbinV3Decoder::decode(meta(), bytes);
|
||||
ASSERT_TRUE(canonical.accepted()) << canonical.detail;
|
||||
|
||||
const size_t payload = Header + 2 * Descriptor;
|
||||
ASSERT_EQ(r32(bytes, Header + 20), 2u);
|
||||
// canonical bits are 0x71: 0, -2, 0, 0. Replace the -2 symbol
|
||||
// with the forbidden +2 half-circle representation (z=4), yielding 0xe1.
|
||||
bytes[payload + 1] = 0xe1;
|
||||
recalc(bytes);
|
||||
EXPECT_EQ(particle::ingress::QbinV3Decoder::decode(meta(), bytes).code, Code::ResidualOutOfRange);
|
||||
}
|
||||
|
||||
TEST(ParticleQbinV3Decoder, ContentIdentityMatchesQbinV2CanonicalFrame) {
|
||||
auto validity = validityBits(4);
|
||||
PlaneSpec u{1, 1, 1, 0.25f, 0.0f, 1, 1, 1, 1, {1, -2, 3, -4}};
|
||||
PlaneSpec v{2, 1, 1, 0.25f, 0.0f, 1, 1, 1, 2, {5, 6, -7, 8}};
|
||||
auto v2 = particle::ingress::QbinV2Decoder::decode(meta(), fixtureV2(2, 2, validity, {u, v}));
|
||||
auto v3 = particle::ingress::QbinV3Decoder::decode(meta(), fixture(2, 2, validity, {u, v}));
|
||||
ASSERT_TRUE(v2.accepted()) << v2.detail;
|
||||
ASSERT_TRUE(v3.accepted()) << v3.detail;
|
||||
ASSERT_EQ(v2.frame->planes.size(), v3.frame->planes.size());
|
||||
for (size_t i = 0; i < v2.frame->planes.size(); ++i) {
|
||||
EXPECT_EQ(*v2.frame->planes[i].storage, *v3.frame->planes[i].storage);
|
||||
EXPECT_EQ(v2.frame->planes[i].contentIdentity, v3.frame->planes[i].contentIdentity);
|
||||
}
|
||||
EXPECT_EQ(*v2.frame->validityBits, *v3.frame->validityBits);
|
||||
EXPECT_EQ(v2.frame->validityContentIdentity, v3.frame->validityContentIdentity);
|
||||
}
|
||||
279
tools/qbin_v3_block_benchmark.cpp
Normal file
279
tools/qbin_v3_block_benchmark.cpp
Normal file
@@ -0,0 +1,279 @@
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace fs = std::filesystem;
|
||||
using Clock = std::chrono::steady_clock;
|
||||
|
||||
uint16_t u16(const std::vector<uint8_t>& b, size_t o) {
|
||||
return uint16_t(b[o]) | uint16_t(b[o + 1]) << 8;
|
||||
}
|
||||
uint32_t u32(const std::vector<uint8_t>& b, size_t o) {
|
||||
uint32_t v = 0;
|
||||
for (size_t i = 0; i < 4; ++i) v |= uint32_t(b[o + i]) << (8 * i);
|
||||
return v;
|
||||
}
|
||||
float f32(const std::vector<uint8_t>& b, size_t o) {
|
||||
uint32_t bits = u32(b, o);
|
||||
float v = 0;
|
||||
std::memcpy(&v, &bits, 4);
|
||||
return v;
|
||||
}
|
||||
|
||||
struct Desc {
|
||||
uint16_t semantic = 0;
|
||||
uint8_t dtype = 0;
|
||||
float scale = 1;
|
||||
uint32_t off = 0;
|
||||
uint32_t len = 0;
|
||||
uint8_t valueClass = 0;
|
||||
};
|
||||
struct Plane {
|
||||
Desc d;
|
||||
std::vector<int64_t> raw;
|
||||
};
|
||||
struct Frame {
|
||||
uint32_t w = 0, h = 0;
|
||||
std::vector<uint8_t> validity;
|
||||
std::vector<Plane> planes;
|
||||
};
|
||||
|
||||
bool validAt(const std::vector<uint8_t>& v, uint64_t i) {
|
||||
return ((v[i >> 3] >> (i & 7u)) & 1u) != 0;
|
||||
}
|
||||
std::vector<uint8_t> readFile(const fs::path& p) {
|
||||
std::ifstream in(p, std::ios::binary);
|
||||
if (!in) throw std::runtime_error("open failed: " + p.string());
|
||||
return {std::istreambuf_iterator<char>(in), std::istreambuf_iterator<char>()};
|
||||
}
|
||||
|
||||
Frame parseV2(const fs::path& p) {
|
||||
const auto b = readFile(p);
|
||||
if (b.size() < 112 || std::memcmp(b.data(), "QBV2", 4) != 0) throw std::runtime_error("not QBV2");
|
||||
Frame f;
|
||||
const uint16_t n = u16(b, 12);
|
||||
f.w = u32(b, 16);
|
||||
f.h = u32(b, 20);
|
||||
const uint64_t cells = uint64_t(f.w) * f.h;
|
||||
const size_t payload = 112 + size_t(n) * 32;
|
||||
std::vector<Desc> ds;
|
||||
for (uint16_t i = 0; i < n; ++i) {
|
||||
const size_t o = 112 + size_t(i) * 32;
|
||||
ds.push_back({u16(b,o), b[o+2], f32(b,o+4), u32(b,o+16), u32(b,o+20), b[o+28]});
|
||||
}
|
||||
const auto it = std::find_if(ds.begin(), ds.end(), [](const Desc& d){ return d.dtype == 6; });
|
||||
if (it == ds.end()) throw std::runtime_error("missing validity");
|
||||
f.validity.assign(b.begin()+payload+it->off, b.begin()+payload+it->off+it->len);
|
||||
|
||||
for (const auto& d : ds) {
|
||||
if (d.dtype == 6) continue;
|
||||
Plane p0;
|
||||
p0.d = d;
|
||||
p0.raw.resize(cells);
|
||||
const uint8_t* raw = b.data() + payload + d.off;
|
||||
for (uint64_t i=0;i<cells;++i) {
|
||||
if (d.dtype == 1) {
|
||||
uint16_t bits = uint16_t(raw[i*2]) | uint16_t(raw[i*2+1]) << 8;
|
||||
int16_t x; std::memcpy(&x,&bits,2); p0.raw[i]=x;
|
||||
} else if (d.dtype == 2) {
|
||||
p0.raw[i]=uint16_t(raw[i*2]) | uint16_t(raw[i*2+1]) << 8;
|
||||
} else if (d.dtype == 3) p0.raw[i]=raw[i];
|
||||
else if (d.dtype == 4) { int8_t x; std::memcpy(&x,raw+i,1); p0.raw[i]=x; }
|
||||
else throw std::runtime_error("unsupported V2 dtype");
|
||||
}
|
||||
f.planes.push_back(std::move(p0));
|
||||
}
|
||||
return f;
|
||||
}
|
||||
int64_t normDir(int64_t x, uint32_t n) {
|
||||
if (x < 0) x += n;
|
||||
else if (x >= int64_t(n)) x -= n;
|
||||
return x;
|
||||
}
|
||||
uint64_t zigzag(int64_t r) { return r >= 0 ? uint64_t(2*r) : uint64_t(-2*r-1); }
|
||||
int64_t unzigzag(uint64_t z) { return (z&1u) ? -int64_t(z/2)-1 : int64_t(z/2); }
|
||||
|
||||
int64_t predictor(const Frame& f, const Plane& p, const std::vector<int64_t>& raw,
|
||||
uint32_t row, uint32_t col, uint32_t n) {
|
||||
const uint64_t i=uint64_t(row)*f.w+col;
|
||||
const bool l=col>0 && validAt(f.validity,i-1);
|
||||
const bool u=row>0 && validAt(f.validity,i-f.w);
|
||||
const bool ul=row>0 && col>0 && validAt(f.validity,i-f.w-1);
|
||||
int64_t pred=0;
|
||||
if (l&&u&&ul) pred=raw[i-1]+raw[i-f.w]-raw[i-f.w-1];
|
||||
else if(l) pred=raw[i-1];
|
||||
else if(u) pred=raw[i-f.w];
|
||||
if (n) pred=normDir(pred,n);
|
||||
return pred;
|
||||
}
|
||||
int64_t residualFor(const Frame& f, const Plane& p, uint32_t row, uint32_t col, uint32_t n) {
|
||||
const uint64_t i=uint64_t(row)*f.w+col;
|
||||
const int64_t pred=predictor(f,p,p.raw,row,col,n);
|
||||
int64_t r=p.raw[i]-pred;
|
||||
if (!n) return r;
|
||||
int64_t m=r;
|
||||
if(m<0)m+=n;
|
||||
if(2*m>n)return m-n;
|
||||
if(2*m<int64_t(n))return m;
|
||||
return -int64_t(n/2);
|
||||
}
|
||||
|
||||
struct BitWriter {
|
||||
std::vector<uint8_t> b;
|
||||
size_t pos=0;
|
||||
void bit(uint8_t v) {
|
||||
if ((pos & 7u)==0) b.push_back(0);
|
||||
if(v) b.back() |= uint8_t(1u << (pos & 7u));
|
||||
++pos;
|
||||
}
|
||||
void bits(uint32_t v,uint8_t n){ for(uint8_t i=0;i<n;++i) bit((v>>i)&1u); }
|
||||
};
|
||||
struct EncodedPlane { std::vector<uint8_t> bytes; uint64_t blocks=0, nonempty=0; };
|
||||
|
||||
EncodedPlane encodePlane(const Frame& f,const Plane& p,uint32_t block) {
|
||||
EncodedPlane out;
|
||||
const uint32_t n = p.d.valueClass==2 ? uint32_t(std::floor(360.0/double(p.d.scale)+0.5)) : 0;
|
||||
for(uint32_t rb=0;rb<f.h;rb+=block) for(uint32_t cb=0;cb<f.w;cb+=block) {
|
||||
++out.blocks;
|
||||
std::vector<uint64_t> z;
|
||||
for(uint32_t r=rb;r<std::min(rb+block,f.h);++r)
|
||||
for(uint32_t c=cb;c<std::min(cb+block,f.w);++c) {
|
||||
const uint64_t i=uint64_t(r)*f.w+c;
|
||||
if(validAt(f.validity,i)) z.push_back(zigzag(residualFor(f,p,r,c,n)));
|
||||
}
|
||||
if(z.empty()) continue;
|
||||
++out.nonempty;
|
||||
uint64_t bestBits=~uint64_t(0); uint8_t bestK=0;
|
||||
for(uint8_t k=0;k<=31;++k) {
|
||||
uint64_t bits=0;
|
||||
for(auto v:z) bits += (v>>k)+1u+k;
|
||||
if(bits<bestBits){bestBits=bits;bestK=k;}
|
||||
}
|
||||
out.bytes.push_back(bestK);
|
||||
BitWriter bw;
|
||||
for(auto v:z) {
|
||||
const uint64_t q=v>>bestK;
|
||||
for(uint64_t j=0;j<q;++j) bw.bit(0);
|
||||
bw.bit(1);
|
||||
if(bestK) bw.bits(uint32_t(v & ((uint64_t(1)<<bestK)-1)),bestK);
|
||||
}
|
||||
out.bytes.insert(out.bytes.end(),bw.b.begin(),bw.b.end());
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
struct BitReader {
|
||||
const uint8_t* p=nullptr; size_t bytes=0,pos=0;
|
||||
uint8_t bit(){ if(pos>=bytes*8) throw std::runtime_error("truncated"); return (p[pos>>3]>>(pos++&7u))&1u; }
|
||||
uint32_t bitsN(uint8_t n){ uint32_t v=0; for(uint8_t i=0;i<n;++i)v|=uint32_t(bit())<<i; return v; }
|
||||
};
|
||||
|
||||
std::vector<int64_t> decodePlane(const Frame& f,const Plane& p,const EncodedPlane& enc,uint32_t block) {
|
||||
std::vector<int64_t> raw(uint64_t(f.w)*f.h,0);
|
||||
size_t cursor=0;
|
||||
const uint32_t n=p.d.valueClass==2 ? uint32_t(std::floor(360.0/double(p.d.scale)+0.5)) : 0;
|
||||
for(uint32_t rb=0;rb<f.h;rb+=block) for(uint32_t cb=0;cb<f.w;cb+=block) {
|
||||
uint32_t count=0;
|
||||
for(uint32_t r=rb;r<std::min(rb+block,f.h);++r)
|
||||
for(uint32_t c=cb;c<std::min(cb+block,f.w);++c)
|
||||
if(validAt(f.validity,uint64_t(r)*f.w+c)) ++count;
|
||||
if(!count) continue;
|
||||
if(cursor>=enc.bytes.size()) throw std::runtime_error("missing k");
|
||||
const uint8_t k=enc.bytes[cursor++];
|
||||
if(k>31) throw std::runtime_error("bad k");
|
||||
BitReader br{enc.bytes.data()+cursor,enc.bytes.size()-cursor,0};
|
||||
for(uint32_t r=rb;r<std::min(rb+block,f.h);++r)
|
||||
for(uint32_t c=cb;c<std::min(cb+block,f.w);++c) {
|
||||
const uint64_t i=uint64_t(r)*f.w+c;
|
||||
if(!validAt(f.validity,i)) continue;
|
||||
uint64_t q=0; while(br.bit()==0) ++q;
|
||||
const uint32_t rem=k?br.bitsN(k):0;
|
||||
const uint64_t z=(q<<k)|rem;
|
||||
const int64_t res=unzigzag(z);
|
||||
int64_t value=predictor(f,p,raw,r,c,n)+res;
|
||||
if(n)value=normDir(value,n);
|
||||
raw[i]=value;
|
||||
}
|
||||
cursor += (br.pos+7u)/8u;
|
||||
}
|
||||
if(cursor!=enc.bytes.size()) throw std::runtime_error("extra bytes");
|
||||
return raw;
|
||||
}
|
||||
|
||||
struct Totals {
|
||||
uint64_t bytes=0,blocks=0,nonempty=0;
|
||||
double encodeMs=0,decodeMs=0;
|
||||
};
|
||||
|
||||
void runOne(const fs::path& file,uint32_t block,Totals& totals) {
|
||||
const Frame frame=parseV2(file);
|
||||
std::vector<EncodedPlane> encoded(frame.planes.size());
|
||||
constexpr int EncodeRepeats=3, DecodeRepeats=8;
|
||||
|
||||
auto start=Clock::now();
|
||||
for(int rep=0;rep<EncodeRepeats;++rep)
|
||||
for(size_t i=0;i<frame.planes.size();++i) encoded[i]=encodePlane(frame,frame.planes[i],block);
|
||||
auto mid=Clock::now();
|
||||
|
||||
for(size_t i=0;i<encoded.size();++i) {
|
||||
totals.bytes += encoded[i].bytes.size();
|
||||
totals.blocks += encoded[i].blocks;
|
||||
totals.nonempty += encoded[i].nonempty;
|
||||
}
|
||||
|
||||
for(int rep=0;rep<DecodeRepeats;++rep) {
|
||||
for(size_t i=0;i<frame.planes.size();++i) {
|
||||
const auto raw=decodePlane(frame,frame.planes[i],encoded[i],block);
|
||||
if(raw!=frame.planes[i].raw) throw std::runtime_error("round-trip mismatch: "+file.string());
|
||||
}
|
||||
}
|
||||
auto end=Clock::now();
|
||||
totals.encodeMs += std::chrono::duration<double,std::milli>(mid-start).count()/EncodeRepeats;
|
||||
totals.decodeMs += std::chrono::duration<double,std::milli>(end-mid).count()/DecodeRepeats;
|
||||
}
|
||||
int main(int argc,char** argv) {
|
||||
try {
|
||||
const fs::path root=argc>1 ? fs::path(argv[1])
|
||||
: fs::path("integration/react-native/MapLibreParticleRNTestHost/assets/qbin-golden");
|
||||
std::vector<fs::path> files;
|
||||
for(const auto& e:fs::recursive_directory_iterator(root))
|
||||
if(e.is_regular_file() && e.path().extension()==".qbin") files.push_back(e.path());
|
||||
std::sort(files.begin(),files.end());
|
||||
if(files.empty()) throw std::runtime_error("no qbin fixtures");
|
||||
|
||||
for(uint32_t block:{16u,32u}) {
|
||||
Totals total;
|
||||
std::cout<<"BLOCK="<<block<<"\n";
|
||||
for(const auto& file:files) {
|
||||
Totals one;
|
||||
runOne(file,block,one);
|
||||
std::cout<<file.parent_path().filename().string()<<"/"<<file.filename().string()
|
||||
<<",bytes="<<one.bytes
|
||||
<<",blocks="<<one.blocks
|
||||
<<",nonempty="<<one.nonempty
|
||||
<<",encodeMs="<<one.encodeMs
|
||||
<<",decodeMs="<<one.decodeMs<<"\n";
|
||||
total.bytes+=one.bytes; total.blocks+=one.blocks; total.nonempty+=one.nonempty;
|
||||
total.encodeMs+=one.encodeMs; total.decodeMs+=one.decodeMs;
|
||||
}
|
||||
std::cout<<"TOTAL,block="<<block
|
||||
<<",bytes="<<total.bytes
|
||||
<<",blocks="<<total.blocks
|
||||
<<",nonempty="<<total.nonempty
|
||||
<<",encodeMs="<<total.encodeMs
|
||||
<<",decodeMs="<<total.decodeMs<<"\n";
|
||||
}
|
||||
return 0;
|
||||
} catch(const std::exception& e) {
|
||||
std::cerr<<"FAIL: "<<e.what()<<"\n";
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user