时间戳

首页 > 解决方案 > 如何提供着色器功能来执行视频的 Alpha 遮罩

android - 如何提供着色器功能来执行视频的 Alpha 遮罩

问题描述

在此处输入图像描述在此处输入图像描述我正在使用 android 中的 virocore 库开发 Vr 应用程序。我必须在球体上显示视频。我必须实现的视频实际上不是视频，但提供的两个帧是颜色帧（左）和 alpha 蒙版（右）帧。我没有使用过 openGl，但似乎我需要提供一个着色器函数来执行 alpha 遮罩。

我已将其用于着色器从黑白（和灰色）Alpha 信息视频图像中为视频添加透明度

但是我如何在绘制方法中将它与 OpenGL 一起使用？或者如果在 virocore 中有任何方法可以使用它来进行 alpha 遮罩。我在 virocore 中尝试过色度过滤方法，但这会使整个视频透明。

public class VideoSurfaceView   extends GLSurfaceView {

VideoRender mRenderer;
private MediaPlayer mMediaPlayer = null;

public VideoSurfaceView(Context context, MediaPlayer mp) {
    super(context);

    setEGLContextClientVersion(2);
    mMediaPlayer = mp;
    mRenderer = new VideoRender(context);

    this.getHolder().setFormat(PixelFormat.RGB_565);
    this.getHolder().setFormat(PixelFormat.TRANSPARENT);
    setEGLConfigChooser(8,8,8,8,16,0);
    setEGLContextClientVersion(2);

    setRenderer(mRenderer);

}

@Override
public void onResume() {
    Log.e("onResume ", "onResume");
    queueEvent(new Runnable(){
        public void run() {
            Log.e("runnable ", "runnable");
            mRenderer.setMediaPlayer(mMediaPlayer);
        }});

    super.onResume();
}

private static class VideoRender
        implements Renderer, SurfaceTexture.OnFrameAvailableListener, MediaPlayer.OnPreparedListener {
    private static String TAG = "VideoRender";

    private static final int FLOAT_SIZE_BYTES = 4;
    private static final int TRIANGLE_VERTICES_DATA_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES;
    private static final int TRIANGLE_VERTICES_DATA_POS_OFFSET = 0;
    private static final int TRIANGLE_VERTICES_DATA_UV_OFFSET = 3;
    private final float[] mTriangleVerticesData = {
            // X, Y, Z, U, V
            -1.0f, -1.0f, 0, 0.f, 0.f,
            1.0f, -1.0f, 0, 1.f, 0.f,
            -1.0f,  1.0f, 0, 0.f, 1.f,
            1.0f,  1.0f, 0, 1.f, 1.f,
    };

    private FloatBuffer mTriangleVertices;


    private static final String mVertexShader =
            "uniform mat4 uMVPMatrix;\n" +
                    "uniform mat4 uSTMatrix;\n" +
            "attribute vec4 position;\n" +
            "attribute vec4 inputTextureCoordinate;\n" +
            " \n" +
            "varying vec2 textureCoordinate;\n" +
            "varying vec2 textureCoordinate2;\n" +
            " \n" +
            "void main()\n" +
            "{\n" +
            "    gl_Position = uMVPMatrix * position;\n" +
            "    vec4 texCoord = uSTMatrix * inputTextureCoordinate;\n"+

                    "textureCoordinate  = vec2(inputTextureCoordinate.x * 0.5, inputTextureCoordinate.y);\n" +
  "   textureCoordinate2 = vec2(inputTextureCoordinate.x * 0.5 + 0.5, inputTextureCoordinate.y);\n" +
            "}";


    public static final String mFragmentShader = "#extension GL_OES_EGL_image_external : require\n"+
            "varying highp vec2 textureCoordinate;\n"+
            "varying highp vec2 textureCoordinate2;\n"+
            "uniform samplerExternalOES inputImageTexture;\n" +
            "void main() {\n"+
            "    lowp vec4 rgbcolor = texture2D(inputImageTexture, textureCoordinate);\n"+
            "    lowp vec4 alphaValue = texture2D(inputImageTexture, textureCoordinate2);\n"+
            "    if (alphaValue.g < 0.5)\n"+
            "    discard;\n"+
            "    gl_FragColor = vec4(rgbcolor.rgb, 1.0);\n"+
            "}";

    private float[] mMVPMatrix = new float[16];
    private float[] mSTMatrix = new float[16];

    private int mProgram;
    private int mTextureID;
    private int muMVPMatrixHandle;
    private int muSTMatrixHandle;
    private int maPositionHandle;
    private int maTextureHandle;

    private SurfaceTexture mSurface;
    private boolean updateSurface = false;

    private static int GL_TEXTURE_EXTERNAL_OES = 0x8D65;

    private MediaPlayer mMediaPlayer;

    public VideoRender(Context context) {
        mTriangleVertices = ByteBuffer.allocateDirect(
                mTriangleVerticesData.length * FLOAT_SIZE_BYTES)
                .order(ByteOrder.nativeOrder()).asFloatBuffer();
        mTriangleVertices.put(mTriangleVerticesData).position(0);

        Matrix.setIdentityM(mSTMatrix, 0);

    }

    public void setMediaPlayer(MediaPlayer player) {
        mMediaPlayer = player;
    }

    @Override
    public void onDrawFrame(GL10 glUnused) {
        synchronized(this) {
            if (updateSurface) {
                mSurface.updateTexImage();
                mSurface.getTransformMatrix(mSTMatrix);
                updateSurface = false;
            }
        }

        GLES20.glClearColor(0.0f, 0.0f, 0.0f, .0f);
        GLES20.glClear(GLES20.GL_DEPTH_BUFFER_BIT|GLES20.GL_COLOR_BUFFER_BIT);
        GLES20.glEnable(GLES20.GL_BLEND);
        GLES20.glBlendFunc(GLES20.GL_SRC_ALPHA, GLES20.GL_ONE_MINUS_SRC_ALPHA);

        GLES20.glUseProgram(mProgram);
        checkGlError("glUseProgram");

        GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
        GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, mTextureID);

        mTriangleVertices.position(TRIANGLE_VERTICES_DATA_POS_OFFSET);
        GLES20.glVertexAttribPointer(maPositionHandle, 3, GLES20.GL_FLOAT, false,
                TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
        checkGlError("glVertexAttribPointer maPosition");
        GLES20.glEnableVertexAttribArray(maPositionHandle);
        checkGlError("glEnableVertexAttribArray maPositionHandle");

        mTriangleVertices.position(TRIANGLE_VERTICES_DATA_UV_OFFSET);
        GLES20.glVertexAttribPointer(maTextureHandle, 3, GLES20.GL_FLOAT, false,
                TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
        checkGlError("glVertexAttribPointer maTextureHandle");
        GLES20.glEnableVertexAttribArray(maTextureHandle);
        checkGlError("glEnableVertexAttribArray maTextureHandle");

        Matrix.setIdentityM(mMVPMatrix, 0);
        GLES20.glUniformMatrix4fv(muMVPMatrixHandle, 1, false, mMVPMatrix, 0);
        GLES20.glUniformMatrix4fv(muSTMatrixHandle, 1, false, mSTMatrix, 0);

        GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
        checkGlError("glDrawArrays");
        GLES20.glFinish();

    }

    @Override
    public void onSurfaceChanged(GL10 glUnused, int width, int height) {

    }

    @Override
    public void onSurfaceCreated(GL10 glUnused, EGLConfig config) {
        mProgram = createProgram(mVertexShader, mFragmentShader);
        if (mProgram == 0) {
            return;
        }
        maPositionHandle = GLES20.glGetAttribLocation(mProgram, "position");
        checkGlError("glGetAttribLocation aPosition");
        if (maPositionHandle == -1) {
            throw new RuntimeException("Could not get attrib location for aPosition");
        }
        maTextureHandle = GLES20.glGetAttribLocation(mProgram, "inputTextureCoordinate");
        checkGlError("glGetAttribLocation aTextureCoord");
        if (maTextureHandle == -1) {
            throw new RuntimeException("Could not get attrib location for aTextureCoord");
        }

        muMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix");
        checkGlError("glGetUniformLocation uMVPMatrix");
        if (muMVPMatrixHandle == -1) {
            throw new RuntimeException("Could not get attrib location for uMVPMatrix");
        }

        muSTMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uSTMatrix");
        checkGlError("glGetUniformLocation uSTMatrix");
        if (muSTMatrixHandle == -1) {
            throw new RuntimeException("Could not get attrib location for uSTMatrix");
        }


        int[] textures = new int[1];
        GLES20.glGenTextures(1, textures, 0);

        mTextureID = textures[0];
        GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, mTextureID);
        checkGlError("glBindTexture mTextureID");

        GLES20.glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER,
                GLES20.GL_NEAREST);
        GLES20.glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER,
                GLES20.GL_LINEAR);

        /*
         * Create the SurfaceTexture that will feed this textureID,
         * and pass it to the MediaPlayer
         */
        mSurface = new SurfaceTexture(mTextureID);
        mSurface.setOnFrameAvailableListener(this);
        Log.e("surface ", "surface");
        Surface surface = new Surface(mSurface);
        mMediaPlayer.setSurface(surface);
        mMediaPlayer.setScreenOnWhilePlaying(true);
        surface.release();
        mMediaPlayer.setOnPreparedListener(this);
        mMediaPlayer.prepareAsync();

        synchronized(this) {
            updateSurface = false;
        }
    }

    synchronized public void onFrameAvailable(SurfaceTexture surface) {
        updateSurface = true;
    }

    private int loadShader(int shaderType, String source) {
        int shader = GLES20.glCreateShader(shaderType);
        if (shader != 0) {
            GLES20.glShaderSource(shader, source);
            GLES20.glCompileShader(shader);
            int[] compiled = new int[1];
            GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0);
            if (compiled[0] == 0) {
                Log.e(TAG, "Could not compile shader " + shaderType + ":");
                Log.e(TAG, GLES20.glGetShaderInfoLog(shader));
                GLES20.glDeleteShader(shader);
                shader = 0;
            }
        }
        return shader;
    }

    private int createProgram(String vertexSource, String fragmentSource) {
        int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource);
        if (vertexShader == 0) {
            return 0;
        }
        int pixelShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource);
        if (pixelShader == 0) {
            return 0;
        }

        int program = GLES20.glCreateProgram();
        if (program != 0) {
            GLES20.glAttachShader(program, vertexShader);
            checkGlError("glAttachShader");
            GLES20.glAttachShader(program, pixelShader);
            checkGlError("glAttachShader");
            GLES20.glLinkProgram(program);
            int[] linkStatus = new int[1];
            GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0);
            if (linkStatus[0] != GLES20.GL_TRUE) {
                Log.e(TAG, "Could not link program: ");
                Log.e(TAG, GLES20.glGetProgramInfoLog(program));
                GLES20.glDeleteProgram(program);
                program = 0;
            }
        }
        return program;
    }

    private void checkGlError(String op) {
        int error;
        while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) {
            Log.e(TAG, op + ": glError " + error);
            throw new RuntimeException(op + ": glError " + error);
        }
    }

    @Override
    public void onPrepared(MediaPlayer mediaPlayer) {
        mediaPlayer.start();
    }
}  // End of class VideoRender.

} // VideoSurfaceView 类结束。

使用此代码，视频看起来倒置

标签： androidopengl-esglslvertex-shaderglsurfaceview

解决方案

如果要丢弃片段，则可以使用片段着色器discard中的关键字。

例如丢弃所有 alpha 值小于 0.5 的片段：

void main()
{
    lowp vec4 rgbcolor   = texture2D(inputImageTexture, textureCoordinate);
    lowp vec4 alphaValue = texture2D(inputImageTexture, textureCoordinate2);

    if (alphaValue.g < 0.5)
        discard;   

    gl_FragColor = vec4(rgbcolor.rgb, 1.0);
}

另请参阅OpenGL ES 着色语言 1.00 规范；6.4 跳跃；第 58 页：

丢弃关键字只允许在片段着色器中使用。可以在片段着色器中使用它来放弃对当前片段的操作。此关键字导致片段被丢弃，并且不会发生对任何缓冲区的更新。它通常在条件语句中使用，例如：

if (intensity < 0.0)
    discard;

参考评论

由于没有矩阵，所以我将使用GLES20.glGetUniformLocation什么？

当然，您可以将矩阵添加到顶点着色器：

attribute vec4 inputTextureCoordinate;

varying vec2 textureCoordinate;
varying vec2 textureCoordinate2;

uniform mat4 matMVP;
uniform mat4 matST;

void main()
{
    gl_Position = matMVP * position;

    vec4 texCoord = matST * inputTextureCoordinate;

    textureCoordinate  = vec2(texCoord.x * 0.5,       1.0 - texCoord.y);
    textureCoordinate2 = vec2(texCoord.x * 0.5 + 0.5, 1.0 - texCoord.y);
}

muMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "matMVP");
muSTMatrixHandle  = GLES20.glGetUniformLocation(mProgram, "matST");

GLES20.glUniformMatrix4fv(muMVPMatrixHandle, 1, false, mMVPMatrix, 0);
GLES20.glUniformMatrix4fv(muSTMatrixHandle, 1, false, mSTMatrix, 0);

推荐阅读