From daeae05181e14b07a8b80eb45fb4fa4e2121b7c7 Mon Sep 17 00:00:00 2001 From: mwilsnd <53413200+mwilsnd@users.noreply.github.com> Date: Tue, 4 Apr 2023 13:09:57 -0500 Subject: [PATCH] Add shader code generator (#978) --- package-lock.json | 13 ++ package.json | 1 + shaders/generate_shader_code.js | 204 ++++++++++++++++++++++++++++++++ shaders/manifest.json | 164 +++++++++++++++++++++++++ 4 files changed, 382 insertions(+) create mode 100644 shaders/generate_shader_code.js create mode 100644 shaders/manifest.json diff --git a/package-lock.json b/package-lock.json index f7b51fc7e..e43bb5e3a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,6 +20,7 @@ "@mapbox/mvt-fixtures": "3.10.0", "@octokit/plugin-retry": "^4.1.2", "@octokit/rest": "^19.0.7", + "argparse": "^2.0.1", "aws-sdk": "^2.1318.0", "csscolorparser": "~1.0.2", "d3-queue": "3.0.7", @@ -636,6 +637,12 @@ "node": ">= 6" } }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true + }, "node_modules/array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -4859,6 +4866,12 @@ } } }, + "argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true + }, "array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", diff --git a/package.json b/package.json index 754231aac..096ff723b 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "@mapbox/mvt-fixtures": "3.10.0", "@octokit/plugin-retry": "^4.1.2", "@octokit/rest": "^19.0.7", + "argparse": "^2.0.1", "aws-sdk": "^2.1318.0", "csscolorparser": "~1.0.2", "d3-queue": "3.0.7", diff --git a/shaders/generate_shader_code.js b/shaders/generate_shader_code.js new file mode 100644 index 000000000..4ab2602e3 --- /dev/null +++ b/shaders/generate_shader_code.js @@ -0,0 +1,204 @@ +#!/usr/bin/env node +'use strict'; + +const { ArgumentParser } = require("argparse"); +const path = require("node:path"); +const fs = require("node:fs") +const os = require("node:os"); + +const generatedHeader = `// Generated code, do not modify this file! +// Generated on ${new Date().toISOString()} by ${os.userInfo().username} using shaders/generate_shader_code.js +`; + +const pragmaMapConvert = (source, pragmaMap, pipelineStage) => { + const re = /#pragma mapbox: ([\w]+) ([\w]+) ([\w]+) ([\w]+)/g; + + if (pipelineStage == "fragment") { + return source.replace(re, (match, operation, precision, type, name) => { + pragmaMap[name] = true; + if (operation === 'define') { + return `#ifndef HAS_UNIFORM_u_${name} +varying ${precision} ${type} ${name}; +#else +uniform ${precision} ${type} u_${name}; +#endif`; + } else /* if (operation === 'initialize') */ { + return `#ifdef HAS_UNIFORM_u_${name} +${precision} ${type} ${name} = u_${name}; +#endif`; + } + }); + } + + // else pipelineStage == "vertex" + + return source.replace(re, (match, operation, precision, type, name) => { + const attrType = type === 'float' ? 'vec2' : 'vec4'; + const unpackType = name.match(/color/) ? 'color' : attrType; + + if (pragmaMap[name]) { + if (operation === 'define') { + return `#ifndef HAS_UNIFORM_u_${name} +uniform lowp float u_${name}_t; +attribute ${precision} ${attrType} a_${name}; +varying ${precision} ${type} ${name}; +#else +uniform ${precision} ${type} u_${name}; +#endif`; + } else /* if (operation === 'initialize') */ { + if (unpackType === 'vec4') { + // vec4 attributes are only used for cross-faded properties, and are not packed + return `#ifndef HAS_UNIFORM_u_${name} +${name} = a_${name}; +#else +${precision} ${type} ${name} = u_${name}; +#endif`; + } else { + return `#ifndef HAS_UNIFORM_u_${name} +${name} = unpack_mix_${unpackType}(a_${name}, u_${name}_t); +#else +${precision} ${type} ${name} = u_${name}; +#endif`; + } + } + } else { + if (operation === 'define') { + return `#ifndef HAS_UNIFORM_u_${name} +uniform lowp float u_${name}_t; +attribute ${precision} ${attrType} a_${name}; +#else +uniform ${precision} ${type} u_${name}; +#endif`; + } else /* if (operation === 'initialize') */ { + if (unpackType === 'vec4') { + // vec4 attributes are only used for cross-faded properties, and are not packed + return `#ifndef HAS_UNIFORM_u_${name} +${precision} ${type} ${name} = a_${name}; +#else +${precision} ${type} ${name} = u_${name}; +#endif`; + } else /* */{ + return `#ifndef HAS_UNIFORM_u_${name} +${precision} ${type} ${name} = unpack_mix_${unpackType}(a_${name}, u_${name}_t); +#else +${precision} ${type} ${name} = u_${name}; +#endif`; + } + } + } + }); +}; + +const strip = (source) => { + return source + .replace(/^\s+/gm, "\n") // indentation, leading whitespace + .replace(/\s*\/\/[^\n]*\n/g, "\n") // Single line comments + .replace(/\n+\n/g, "\n") // extra new lines + .replace(/\s?([+-\/<>*\?:=,])\s?/g, "$1") // whitespace around operators + .replace(/([;\(\),\{\}])\n(?=[^#])/g, "$1"); // line breaks +}; + +// Parse command line +const args = (() => { + const parser = new ArgumentParser({ + description: "MapLibre Shader Tools" + }); + parser.add_argument("--input", "--i", { + help: "Input folder location containing shaders and the manifest JSON (Must be named 'manifest.json')", + required: true + + }); + parser.add_argument("--output", "--o", { + help: "Output folder location", + required: true + }); + parser.add_argument("--compress", "--c", { + help: "Compress shader text with zlib and output byte arrays instead of strings", + required: false + }); + parser.add_argument("--strip", "--s", { + help: "Strip comments, new lines and whitespace", + required: false, + action: "store_true" + }); + return parser.parse_args(); +})(); + + +// Generate shader source headers +let generatedHeaders = []; +let shaderNames = []; +JSON.parse(fs.readFileSync(path.join(args.input, "manifest.json"))) + .filter(it => typeof it == "object") + .forEach((elem) => { + let pragmaMap = []; + const fragmentSource = fs.readFileSync(path.join(args.input, elem.glsl_frag), {encoding: "utf8"}); + const vertexSource = fs.readFileSync(path.join(args.input, elem.glsl_vert), {encoding: "utf8"}); + const frag = pragmaMapConvert(fragmentSource, pragmaMap, "fragment"); + const vert = pragmaMapConvert(vertexSource, pragmaMap, "vertex"); + + fs.writeFileSync( + path.join(args.output, elem.header + ".hpp"), + `${generatedHeader} +#pragma once +#include + +namespace mbgl { +namespace shaders { + +template <> struct ShaderSource { + static constexpr const char* vertex = R"(${args.strip ? strip(vert) : vert})"; + static constexpr const char* fragment = R"(${args.strip ? strip(frag) : frag})"; +}; + +} // namespace shaders +} // namespace mbgl +`); + generatedHeaders.push("#include "); + shaderNames.push(elem.name); + } +); + +// Generate the manifest +fs.writeFileSync(path.join(args.output, "shader_manifest.hpp"), +`${generatedHeader} +#pragma once +#include + +#ifdef MBGL_RENDER_BACKEND_OPENGL +${generatedHeaders.join('\n')} +#endif +`); + +// Generate shader_source.hpp +fs.writeFileSync(path.join(args.output, "shader_source.hpp"), +`${generatedHeader} +#pragma once +#include + +namespace mbgl { +namespace shaders { + +/// @brief This enum is used with the ShaderSource template to select +/// source code for the desired program and graphics back-end. +enum class BuiltIn { + None, + ${shaderNames.join(',\n ')} +}; + +/// @brief Select shader source based on a program type and a desired +/// graphics API. +/// @tparam T One of the built-in shader types available in the BuiltIn enum +/// @tparam The desired graphics API to request shader code for. One of +/// gfx::Backend::Type enums. +template struct ShaderSource; + +/// @brief A specialization of the ShaderSource template for no shader code. +template <> struct ShaderSource { + static constexpr const char* vertex = ""; + static constexpr const char* fragment = ""; +}; + +} // namespace shaders +} // namespace mbgl +`); diff --git a/shaders/manifest.json b/shaders/manifest.json new file mode 100644 index 000000000..87b00a181 --- /dev/null +++ b/shaders/manifest.json @@ -0,0 +1,164 @@ +[ + { + "name": "Prelude", + "header": "prelude", + "glsl_vert": "_prelude.vertex.glsl", + "glsl_frag": "_prelude.fragment.glsl" + }, + { + "name": "BackgroundProgram", + "header": "background", + "glsl_vert": "background.vertex.glsl", + "glsl_frag": "background.fragment.glsl" + }, + { + "name": "BackgroundPatternProgram", + "header": "background_pattern", + "glsl_vert": "background_pattern.vertex.glsl", + "glsl_frag": "background_pattern.fragment.glsl" + }, + { + "name": "CircleProgram", + "header": "circle", + "glsl_vert": "circle.vertex.glsl", + "glsl_frag": "circle.fragment.glsl" + }, + { + "name": "ClippingMaskProgram", + "header": "clipping_mask", + "glsl_vert": "clipping_mask.vertex.glsl", + "glsl_frag": "clipping_mask.fragment.glsl" + }, + { + "name": "CollisionBoxProgram", + "header": "collision_box", + "glsl_vert": "collision_box.vertex.glsl", + "glsl_frag": "collision_box.fragment.glsl" + }, + { + "name": "CollisionCircleProgram", + "header": "collision_circle", + "glsl_vert": "collision_circle.vertex.glsl", + "glsl_frag": "collision_circle.fragment.glsl" + }, + { + "name": "DebugProgram", + "header": "debug", + "glsl_vert": "debug.vertex.glsl", + "glsl_frag": "debug.fragment.glsl" + }, + { + "name": "FillExtrusionPatternProgram", + "header": "fill_extrusion_pattern", + "glsl_vert": "fill_extrusion_pattern.vertex.glsl", + "glsl_frag": "fill_extrusion_pattern.fragment.glsl" + }, + { + "name": "FillExtrusionProgram", + "header": "fill_extrusion", + "glsl_vert": "fill_extrusion.vertex.glsl", + "glsl_frag": "fill_extrusion.fragment.glsl" + }, + { + "name": "FillOutlinePatternProgram", + "header": "fill_outline_pattern", + "glsl_vert": "fill_outline_pattern.vertex.glsl", + "glsl_frag": "fill_outline_pattern.fragment.glsl" + }, + { + "name": "FillOutlineProgram", + "header": "fill_outline", + "glsl_vert": "fill_outline.vertex.glsl", + "glsl_frag": "fill_outline.fragment.glsl" + }, + { + "name": "FillPatternProgram", + "header": "fill_pattern", + "glsl_vert": "fill_pattern.vertex.glsl", + "glsl_frag": "fill_pattern.fragment.glsl" + }, + { + "name": "FillProgram", + "header": "fill", + "glsl_vert": "fill.vertex.glsl", + "glsl_frag": "fill.fragment.glsl" + }, + { + "name": "HeatmapTextureProgram", + "header": "heatmap_texture", + "glsl_vert": "heatmap_texture.vertex.glsl", + "glsl_frag": "heatmap_texture.fragment.glsl" + }, + { + "name": "HeatmapProgram", + "header": "heatmap", + "glsl_vert": "heatmap.vertex.glsl", + "glsl_frag": "heatmap.fragment.glsl" + }, + { + "name": "HillshadePrepareProgram", + "header": "hillshade_prepare", + "glsl_vert": "hillshade_prepare.vertex.glsl", + "glsl_frag": "hillshade_prepare.fragment.glsl" + }, + { + "name": "HillshadeProgram", + "header": "hillshade", + "glsl_vert": "hillshade.vertex.glsl", + "glsl_frag": "hillshade.fragment.glsl" + }, + { + "name": "LineGradientProgram", + "header": "line_gradient", + "glsl_vert": "line_gradient.vertex.glsl", + "glsl_frag": "line_gradient.fragment.glsl" + }, + { + "name": "LinePatternProgram", + "header": "line_pattern", + "glsl_vert": "line_pattern.vertex.glsl", + "glsl_frag": "line_pattern.fragment.glsl" + }, + { + "name": "LineSDFProgram", + "header": "line_sdf", + "glsl_vert": "line_sdf.vertex.glsl", + "glsl_frag": "line_sdf.fragment.glsl" + }, + { + "name": "LineProgram", + "header": "line", + "glsl_vert": "line.vertex.glsl", + "glsl_frag": "line.fragment.glsl" + }, + { + "name": "RasterProgram", + "header": "raster", + "glsl_vert": "raster.vertex.glsl", + "glsl_frag": "raster.fragment.glsl" + }, + { + "name": "SymbolIconProgram", + "header": "symbol_icon", + "glsl_vert": "symbol_icon.vertex.glsl", + "glsl_frag": "symbol_icon.fragment.glsl" + }, + { + "name": "SymbolSDFTextProgram", + "header": "symbol_sdf_text", + "glsl_vert": "symbol_sdf.vertex.glsl", + "glsl_frag": "symbol_sdf.fragment.glsl" + }, + { + "name": "SymbolSDFIconProgram", + "header": "symbol_sdf_icon", + "glsl_vert": "symbol_sdf.vertex.glsl", + "glsl_frag": "symbol_sdf.fragment.glsl" + }, + { + "name": "SymbolTextAndIconProgram", + "header": "symbol_text_and_icon", + "glsl_vert": "symbol_text_and_icon.vertex.glsl", + "glsl_frag": "symbol_text_and_icon.fragment.glsl" + } +] \ No newline at end of file