opengl-es-3.0 - 如何使用 16 位数据在 WebGL2 中创建直方图？
问题描述
var gl, utils, pseudoImg, vertices;
var img = null;
document.addEventListener('DOMContentLoaded', () => {
utils = new WebGLUtils();
vertices = utils.prepareVec2({x1 : -1.0, y1 : -1.0, x2 : 1.0, y2 : 1.0});
gl = utils.getGLContext(document.getElementById('canvas'));
var program = utils.getProgram(gl, 'render-vs', 'render16bit-fs');
var histogramProgram = utils.getProgram(gl, 'histogram-vs', 'histogram-fs');
var sortProgram = utils.getProgram(gl, 'sorting-vs', 'sorting-fs');
var showProgram = utils.getProgram(gl, 'showhistogram-vs', 'showhistogram-fs');
utils.activateTextureByIndex(gl, showProgram, 'histTex', 3);
utils.activateTextureByIndex(gl, showProgram, 'maxTex', 4);
utils.activateTextureByIndex(gl, sortProgram, 'tex3', 2);
utils.activateTextureByIndex(gl, histogramProgram, 'tex2', 1);
utils.activateTextureByIndex(gl, program, 'u_texture', 0);
var vertexBuffer = utils.createAndBindBuffer(gl, vertices);
var imageTexture;
computeHistogram = (AR, myFB) => {
gl.useProgram(histogramProgram);
var width = AR.width;
var height = AR.height;
var numOfPixels = width * height;
var pixelIds = new Float32Array(numOfPixels);
for (var i = 0; i < numOfPixels; i++) {
pixelIds[i] = i;
}
var histogramFbObj = utils.createTextureAndFramebuffer(gl, {
format : gl.RED,
internalFormat : gl.R32F,
filter : gl.NEAREST,
dataType : gl.FLOAT,
mipMapST : gl.CLAMP_TO_EDGE,
width : 256,
height : 256
});
gl.bindFramebuffer(gl.FRAMEBUFFER, histogramFbObj.fb);
gl.viewport(0, 0, 256, 256);
var pixelBuffer = utils.createAndBindBuffer(gl, pixelIds, true);
gl.blendFunc(gl.ONE, gl.ONE);
gl.enable(gl.BLEND);
utils.linkAndSendDataToGPU(gl, histogramProgram, 'pixelIds', pixelBuffer, 1);
gl.uniform2fv(gl.getUniformLocation(histogramProgram, 'imageDimensions'), [width, height]);
utils.sendTextureToGPU(gl, myFB.tex, 1);
gl.drawArrays(gl.POINTS, 0, numOfPixels);
gl.blendFunc(gl.ONE, gl.ZERO);
gl.disable(gl.BLEND);
return histogramFbObj;
};
sortHistogram = (histogramFbObj) => {
gl.useProgram(sortProgram);
utils.linkAndSendDataToGPU(gl, sortProgram, 'vertices', vertexBuffer, 2);
var sortFbObj = utils.createTextureAndFramebuffer(gl, {
format : gl.RED,
internalFormat : gl.R32F,
filter : gl.NEAREST,
dataType : gl.FLOAT,
mipMapST : gl.CLAMP_TO_EDGE,
width : 1,
height : 1
});
gl.bindFramebuffer(gl.FRAMEBUFFER, sortFbObj.fb);
gl.viewport(0, 0, 1, 1);
utils.sendTextureToGPU(gl, histogramFbObj.tex, 2);
gl.drawArrays(gl.TRIANGLES, 0, 6);
return sortFbObj;
};
showHistogram = (histFb, sortFb) => {
gl.useProgram(showProgram);
utils.linkAndSendDataToGPU(gl, showProgram, 'vertices', vertexBuffer, 2);
utils.sendTextureToGPU(gl, histFb.tex, 3);
utils.sendTextureToGPU(gl, sortFb.tex, 4);
gl.uniform2fv(gl.getUniformLocation(showProgram, 'imageDimensions'), [gl.canvas.width, gl.canvas.height]);
gl.drawArrays(gl.TRIANGLES, 0, 6);
};
showTexture = (AR) => {
imageTexture = utils.createAndBindTexture(gl, {
filter : gl.NEAREST,
mipMapST : gl.CLAMP_TO_EDGE,
dataType : gl.UNSIGNED_SHORT,
format : gl.RGBA_INTEGER,
internalFormat : gl.RGBA16UI,
img : AR.img,
width : AR.width,
height : AR.height
});
gl.useProgram(program);
var myFB = utils.createTextureAndFramebuffer(gl, {
filter : gl.NEAREST,
mipMapST : gl.CLAMP_TO_EDGE,
dataType : gl.UNSIGNED_BYTE,
format : gl.RGBA,
internalFormat : gl.RGBA,
width : AR.width,
height : AR.height,
});
gl.bindFramebuffer(gl.FRAMEBUFFER, myFB.fb);
gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
utils.linkAndSendDataToGPU(gl, program, 'vertices', vertexBuffer, 2);
gl.uniform1f(gl.getUniformLocation(program, 'flipY'), 1.0);
utils.sendTextureToGPU(gl, imageTexture, 0);
gl.drawArrays(gl.TRIANGLES, 0, 6);
var fb1 = computeHistogram(AR, myFB);
var fb2 = sortHistogram(fb1);
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
showHistogram(fb1, fb2);
};
var w = 128;
var h = 128;
var size = w * h * 4;
var img = new Uint16Array(size); // need Uint16Array
for (var i = 0; i < img.length; i += 4) {
img[i + 0] = 65535; // r
img[i + 1] = i/64 * 256; // g
img[i + 2] = 0; // b
img[i + 3] = 65535; // a
}
showTexture({
img : img,
width : w,
height : h
});
});<script id="render16bit-fs" type="not-js">
#version 300 es
precision highp float;
uniform highp usampler2D tex;
in vec2 texcoord; // receive pixel position from vertex shader
out vec4 fooColor;
void main() {
uvec4 unsignedIntValues = texture(tex, texcoord);
vec4 floatValues0To65535 = vec4(unsignedIntValues);
vec4 colorValues0To1 = floatValues0To65535;
fooColor = colorValues0To1;
}
</script>
<script type="not-js" id="render-vs">
#version 300 es
in vec2 vertices;
out vec2 texcoord;
uniform float flipY;
void main() {
texcoord = vertices.xy * 0.5 + 0.5;
gl_Position = vec4(vertices.x, vertices.y * flipY, 0.0, 1.0);
}
</script>
<script type="not-js" id="histogram-vs">
#version 300 es
in float pixelIds; //0,1,2,3,4,.......width*height
uniform sampler2D tex2;
uniform vec2 imageDimensions;
void main () {
vec2 pixel = vec2(mod(pixelIds, imageDimensions.x), floor(pixelIds / imageDimensions.x));
vec2 xy = pixel/imageDimensions;
float pixelValue = texture(tex2, xy).r;//Pick Pixel value from GPU texture ranges from 0-65535
float xDim = mod(pixelValue, 255.0)/256.0;
float yDim = floor(pixelValue / 255.0)/256.0;
float xVertex = (xDim*2.0) - 1.0;//convert 0.0 to 1.0 -> -1.0 -> 1.0, it will increment because we have used gl.blendFunc
float yVertex = 1.0 - (yDim*2.0);
gl_Position = vec4(xVertex, yVertex, 0.0, 1.0);
gl_PointSize = 1.0;
}
</script>
<script type="not-js" id="histogram-fs">
#version 300 es
precision mediump float;
out vec4 fcolor;
void main() {
fcolor = vec4(1.0, 1.0, 1.0, 1.0);
}
</script>
<script type="not-js" id="sorting-vs">
#version 300 es
in vec2 vertices;
void main () {
gl_Position = vec4(vertices, 0.0, 1.0);
}
</script>
<script type="not-js" id="sorting-fs">
#version 300 es
precision mediump float;
out vec4 fcolor;
uniform sampler2D tex3;
const int MAX_WIDTH = 65536;
void main() {
vec4 maxColor = vec4(0.0);
for (int i = 0; i < MAX_WIDTH; i++) {
float xDim = mod(float(i), 256.0)/256.0;
float yDim = floor(float(i) / 256.0)/256.0;
vec2 xy = vec2(xDim, yDim);
vec4 currPixel = texture(tex3, xy).rrra;
maxColor = max(maxColor, currPixel);
}
fcolor = vec4(maxColor);
}
</script>
<script type="not-js" id="showhistogram-vs">
#version 300 es
in vec2 vertices;
void main () {
gl_Position = vec4(vertices, 0.0, 1.0);
}
</script>
<script type="not-js" id="showhistogram-fs">
#version 300 es
precision mediump float;
uniform sampler2D histTex, maxTex;
uniform vec2 imageDimensions;
out vec4 fcolor;
void main () {
// get the max color constants
vec4 maxColor = texture(maxTex, vec2(0));
// compute our current UV position
vec2 uv = gl_FragCoord.xy / imageDimensions;
vec2 uv2 = gl_FragCoord.xy / vec2(256.0, 256.0);
// Get the history for this color
vec4 hist = texture(histTex, uv2);
// scale by maxColor so scaled goes from 0 to 1 with 1 = maxColor
vec4 scaled = hist / maxColor;
// 1 > maxColor, 0 otherwise
vec4 color = step(uv2.yyyy, scaled);
fcolor = vec4(color.rgb, 1);
}
</script>
<canvas id="canvas"></canvas>
<script type="text/javascript">
class WebGLUtils {
getGLContext = (canvas, version) => {
canvas.width = window.innerWidth * 0.99;
canvas.height = window.innerHeight * 0.85;
var gl = canvas.getContext(version ? 'webgl' : 'webgl2');
const ext = gl.getExtension("EXT_color_buffer_float");
if (!ext) {
console.log("sorry, can't render to floating point textures");
}
gl.clearColor(0, 0, 0, 1.0);
gl.clear(gl.COLOR_BUFFER_BIT|gl.DEPTH_BUFFER_BIT);
gl.lineWidth(0.5);
return gl;
};
clear = (gl) => {
gl.clear(gl.COLOR_BUFFER_BIT|gl.DEPTH_BUFFER_BIT);
};
getShader = (gl, type, shaderText) => {
var vfs = gl.createShader(type);
gl.shaderSource(vfs, shaderText);
gl.compileShader(vfs);
if (!gl.getShaderParameter(vfs, gl.COMPILE_STATUS)) {
console.error(gl.getShaderInfoLog(vfs));
}
return vfs;
};
getProgram = (gl, vertexShader, fragmentShader) => {
var program = gl.createProgram();
gl.attachShader(program, this.getShader(gl, gl.VERTEX_SHADER, document.getElementById(vertexShader).text.trim()));
gl.attachShader(program, this.getShader(gl, gl.FRAGMENT_SHADER, document.getElementById(fragmentShader).text.trim()));
gl.linkProgram(program);
if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
console.error(gl.getProgramInfoLog(program));
}
return program;
};
createAndBindBuffer = (gl, relatedVertices, isNotJSArray) => {
var buffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, buffer);
gl.bufferData(gl.ARRAY_BUFFER, isNotJSArray ? relatedVertices : new Float32Array(relatedVertices), gl.STATIC_DRAW);
gl.bindBuffer(gl.ARRAY_BUFFER, null);
return buffer;
};
createAndBindTexture = (gl, _) => {
var texBuffer = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, texBuffer);
if (_.img.width) {
gl.texImage2D(gl.TEXTURE_2D, 0, _.internalFormat, _.format, _.dataType, _.img);
} else {
gl.texImage2D(gl.TEXTURE_2D, 0, _.internalFormat, _.width, _.height, 0, _.format, _.dataType, _.img);
}
// set the filtering so we don't need mips
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, _.filter);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, _.filter);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, _.mipMapST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, _.mipMapST);
gl.bindTexture(gl.TEXTURE_2D, null);
return texBuffer;
};
createTextureAndFramebuffer = (gl, _) => {
const tex = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, tex);
gl.texImage2D(gl.TEXTURE_2D, 0, _.internalFormat, _.width, _.height, 0, _.format, _.dataType, null);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, _.filter);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, _.filter);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, _.mipMapST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, _.mipMapST);
const fb = gl.createFramebuffer();
gl.bindFramebuffer(gl.FRAMEBUFFER, fb);
gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, tex, 0);
const status = gl.checkFramebufferStatus(gl.FRAMEBUFFER);
console.log(`can ${status === gl.FRAMEBUFFER_COMPLETE ? "" : "NOT "}render to R32`);
return {tex: tex, fb: fb};
};
linkAndSendDataToGPU = (gl, program, linkedVariable, buffer, dimensions) => {
var vertices = gl.getAttribLocation(program, linkedVariable);
gl.enableVertexAttribArray(vertices);
gl.bindBuffer(gl.ARRAY_BUFFER, buffer);
gl.vertexAttribPointer(vertices, dimensions, gl.FLOAT, gl.FALSE, 0, 0);
return vertices;
};
sendDataToGPU = (gl, buffer, vertices, dimensions) => {
gl.bindBuffer(gl.ARRAY_BUFFER, buffer);
gl.vertexAttribPointer(vertices, dimensions, gl.FLOAT, gl.FALSE, 0, 0);
};
sendTextureToGPU = (gl, tex, index) => {
gl.activeTexture(gl.TEXTURE0 + index);
gl.bindTexture(gl.TEXTURE_2D, tex);
};
calculateAspectRatio = (img, gl) => {
var cols = img.width;
var rows = img.height;
var imageAspectRatio = cols / rows;
var ele = gl.canvas;
var windowWidth = ele.width;
var windowHeight = ele.height;
var canvasAspectRatio = windowWidth / windowHeight;
var renderableHeight, renderableWidth;
var xStart, yStart;
/// If image's aspect ratio is less than canvas's we fit on height
/// and place the image centrally along width
if(imageAspectRatio < canvasAspectRatio) {
renderableHeight = windowHeight;
renderableWidth = cols * (renderableHeight / rows);
xStart = (windowWidth - renderableWidth) / 2;
yStart = 0;
}
/// If image's aspect ratio is greater than canvas's we fit on width
/// and place the image centrally along height
else if(imageAspectRatio > canvasAspectRatio) {
renderableWidth = windowWidth;
renderableHeight = rows * (renderableWidth / cols);
xStart = 0;
yStart = ( windowHeight - renderableHeight) / 2;
}
///keep aspect ratio
else {
renderableHeight = windowHeight ;
renderableWidth = windowWidth;
xStart = 0;
yStart = 0;
}
return {
y2 : yStart + renderableHeight,
x2 : xStart + renderableWidth,
x1 : xStart,
y1 : yStart
};
};
convertCanvasCoordsToGPUCoords = (canvas, AR) => {
//GPU -> -1, -1, 1, 1
//convert to 0 -> 1
var _0To1 = {
y2 : AR.y2/canvas.height,
x2 : AR.x2/canvas.width,
x1 : AR.x1/canvas.width,
y1 : AR.y1/canvas.height
};
//Convert -1 -> 1
return {
y2 : -1 + _0To1.y2 * 2.0,
x2 : -1 + _0To1.x2 * 2.0,
x1 : -1 + _0To1.x1 * 2.0,
y1 : -1 + _0To1.y1 * 2.0
};
};
//convert -1->+1 to 0.0->1.0
convertVertexToTexCoords = (x1, y1, x2, y2) => {
return {
y2 : (y2 + 1.0)/2.0,
x2 : (x2 + 1.0)/2.0,
x1 : (x1 + 1.0)/2.0,
y1 : (y1 + 1.0)/2.0
};
};
activateTextureByIndex = (gl, program, gpuRef, gpuTextureIndex) => {
gl.useProgram(program);
gl.uniform1i(gl.getUniformLocation(program, gpuRef), gpuTextureIndex);
};
prepareVec4 = (_) => {
return [_.x1, _.y1, 0.0, 1.0,
_.x2, _.y1, 0.0, 1.0,
_.x1, _.y2, 0.0, 1.0,
_.x2, _.y1, 0.0, 1.0,
_.x1, _.y2, 0.0, 1.0,
_.x2, _.y2, 0.0, 1.0];
};
prepareVec2 = (_) => {
return [_.x1, _.y1,
_.x2, _.y1,
_.x1, _.y2,
_.x2, _.y1,
_.x1, _.y2,
_.x2, _.y2];
};
};
</script>我可以使用此代码在 WebGL1 和 WebGL2 中呈现 8 位直方图。但我需要使用 16 位纹理生成 16 位直方图。
以下是将纹理发送到 GPU 的方式：
var tex = gl.createTexture(); // create empty texture
gl.bindTexture(gl.TEXTURE_2D, tex);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
gl.texImage2D(
gl.TEXTURE_2D, // target
0, // mip level
gl.RGBA16UI, // internal format -> gl.RGBA16UI
w, h, // width and height
0, // border
gl.RGBA_INTEGER, //format -> gl.RGBA_INTEGER
gl.UNSIGNED_SHORT, // type -> gl.UNSIGNED_SHORT
img // texture data
);
因此，请记住工作示例，我遇到了一些事情：
1）如何创建一个 65536 X 1 帧缓冲区/纹理以保持 16 位直方图，如 WebGL 清楚地说：WebGL: INVALID_VALUE: texImage2D: width or height out of range。我们可以试试 256 x 256 帧缓冲区吗？我试过但坚持不。2 下面。
2) 在 16 位的情况下如何读取顶点着色器内的像素，下面的代码是针对 8 位数据的，它也适用于 16 位吗？由于我无法调试，所以不能说它是否有效：
<script id="hist-vs" type="not-js">
attribute float pixelId;
uniform vec2 u_resolution;
uniform sampler2D u_texture;
uniform vec4 u_colorMult;
void main() {
// based on an id (0, 1, 2, 3 ...) compute the pixel x, y for the source image
vec2 pixel = vec2(mod(pixelId, u_resolution.x), floor(pixelId / u_resolution.x));
// compute corresponding uv center of that pixel
vec2 uv = (pixel + 0.5) / u_resolution;
// get the pixels but 0 out channels we don't want
vec4 color = texture2D(u_texture, uv) * u_colorMult;
// add up all the channels. Since 3 are zeroed out we'll get just one channel
float colorSum = color.r + color.g + color.b + color.a;
// set the position to be over a single pixel in the 256x1 destination texture
gl_Position = vec4((colorSum * 255.0 + 0.5) / 256.0 * 2.0 - 1.0, 0.5, 0, 1);
gl_PointSize = 1.0;
}
</script>
解决方案
如果您只想回答您的 2 个问题，那么
1）如何创建一个 65536 X 1 帧缓冲区/纹理以保持 16 位直方图，因为 WebGL 清楚地说：WebGL：INVALID_VALUE：texImage2D：宽度或高度超出范围。我们可以试试 256 x 256 帧缓冲区吗？
是的，如果你想知道每个 65536 个可能值的总数，你会制作 256x256 纹理
2) 在 16 位的情况下如何读取顶点着色器内的像素，下面的代码是针对 8 位数据的，它也适用于 16 位吗？由于我无法调试，所以不能说它是否有效：
当然可以调试。你试试看结果是否正确。如果不是，请查看您的代码和/或错误消息并尝试找出原因。这叫做调试。制作一个 1x1 纹理，调用您的函数，通过调用gl.readPixels. 然后尝试 2x1 或 2x2。
在任何情况下，您都无法gl.RGBA16UI使用 GLSL 1.0 es 读取纹理。您必须使用版本 300 es，所以如果您真的想为所有 65536 值创建一个单独的存储桶，那么
这是一些 WebGL2 GLSL 3.00 ES 着色器，它将填充 256x256 纹理中从 0 到 65535 的值的总数
#version 300 es
uniform usampler2D u_texture;
uniform uvec4 u_colorMult;
void main() {
const int mipLevel = 0;
ivec2 size = textureSize(u_texture, mipLevel);
// based on an id (0, 1, 2, 3 ...) compute the pixel x, y for the source image
vec2 pixel = vec2(
gl_VertexID % size.x,
gl_VertexID / size.x);
// get the pixels but 0 out channels we don't want
uvec4 color = texelFetch(u_texture, pixel, mipLevel) * u_colorMult;
// add up all the channels. Since 3 are zeroed out we'll get just one channel
uint colorSum = color.r + color.g + color.b + color.a;
vec2 outputPixel = vec2(
colorSum % 256u,
colorSum / 256u);
// set the position to be over a single pixel in the 256x256 destination texture
gl_Position = vec4((outputPixel + 0.5) / 256.0 * 2.0 - 1.0, 0, 1);
gl_PointSize = 1.0;
}
笔记：
在 WebGL2 中，您不需要 pixelID，您可以使用
gl_VertexID，因此无需设置任何缓冲区或属性。打电话const numPixels = img.width * img.height; gl.drawArrays(gl.POINTS, 0, numPixels);您可以使用
textureSize来获取纹理的大小，因此无需传递它。您可以使用
texelFetch从纹理中获取单个纹素（像素）。它采用整数像素坐标。要读取像 RGBA16UI 这样的无符号整数纹理格式，您必须使用 a
usampler2D否则在绘制时绘制使用RGBA16UI纹理时会出错sampler2D（这就是我知道您实际上没有使用RGBA16UI纹理的方式，因为您会得到JavaScript 控制台中的错误告诉您并引导您更改着色器。您仍然需要使用浮点纹理作为目标，因为使用的技术需要混合，但混合不适用于整数纹理（以防万一您想到尝试在帧缓冲区中使用基于整数的纹理）
推荐阅读
- javascript - Javascript - 如何找到两个更有价值的元素？
- python - 如何使用 Python ElementTree 从 XML 文件中的不同子级元素中提取相关属性
- sql - 刷新物化视图时间随时间增加
- python - Django 管理员不允许我为一个应用程序创建一个组
- apache-kafka - 什么是 kafka.consumergroup.threads 属性？
- php - PHP ForEach 数组 - 显示变量
- reactjs - 提供者更新的测试效果
- javascript - 如何使已经可点击以实现不同功能的行的表格单元格可点击（并添加单独的功能）？
- java - 控制台功能出现 NullPointerException
- reactjs - 带有测试和 CI/CD 的 Reac Native 课程