android - 我正在尝试让代码工作以解码并显示原始 H264 tcp/ip 视频流。有人可以诊断出什么问题吗？
问题描述
我的目标是使用 MediaCodec 类播放通过 Android 设备（三星 S10）上的 tcp/ip 端口馈送的原始 H264 流。它不起作用，我只是看到一个空白屏幕。
我对这个问题有一些猜测：1）我需要将 NAL 单元组合在一起吗？现在我将每个单元单独提供给 MediaCodec。2）我需要对服务器进行更改吗？是否存在 MediaCodec 类无法处理的 H264 变体？
我能够将 FFmpeg 移植到 Android Studio 并让这个解决方案发挥作用。但是，由于它使用软件编解码器，因此速度很慢。我决定使用 MediaCodec 来尝试使用硬件编解码器。下面的代码显示了我的努力。编解码器以异步模式初始化。我有一个单独的线程来读取和排队来自 tcp 套接字的 NAL 帧。帧存储在缓冲区中，如果缓冲区溢出，则某些帧将被丢弃。onInputBufferAvailable 编解码器一次将一个 NAL 单元提供给 MediaCodec 类。
public void initializePlaybackCodec()
{
mWidth = 1536;
mHeight = 864;
MediaFormat decoderFormat = MediaFormat.createVideoFormat(MIME_TYPE, mWidth, mHeight);
try {
codec = MediaCodec.createDecoderByType(MIME_TYPE);
} catch (IOException e) {
Log.e(TAG, "CODEC INIT: Failed to initialize media codec", e);
Toast.makeText(this, "Failed to initialize media codec",
Toast.LENGTH_LONG).show();
finish();
return;
}
Log.i(TAG,"HERE CODEC INITIALIZED");
final int videoQueueSize = 10;
final Semaphore mutex = new Semaphore(1);
final Semaphore queueData = new Semaphore(0);
final ArrayBlockingQueue<ByteBuffer> queue = new ArrayBlockingQueue<ByteBuffer>(videoQueueSize);
codec.setCallback(new MediaCodec.Callback() {
long reference_epoch = System.currentTimeMillis();
long current_epoch = reference_epoch;
byte[] buffer = new byte[blockSize];
int nextStart = 0;
@Override
public void onInputBufferAvailable(MediaCodec mc, int inputBufferId) {
current_epoch = System.currentTimeMillis();
ByteBuffer inputBuffer = codec.getInputBuffer(inputBufferId);
android.media.MediaCodecInfo info = codec.getCodecInfo();
//Log.i(TAG,"CODEC CALLBACK: info "+info.getName()+" Encoder: "+info.isEncoder()+" ");
//String[] types = info.getSupportedTypes();
//for (int j = 0; j < types.length; j++) {
// Log.i(TAG,"CODEC CALLBACK: supportedType "+types[j]);
//}
// Read data from the Queue
ByteBuffer b = null;
Log.i(TAG,"CODEC CALLBACK: input");
try {
queueData.acquire();
} catch (InterruptedException e) {
Log.e(TAG, "CODEC CALLBACK: queueData acquire interrupted");
codec.stop();
finish();
return;
}
try {
mutex.acquire();
} catch (InterruptedException e) {
Log.e(TAG, "CODEC CALLBACK: mutex acquire interrupted");
codec.stop();
finish();
return;
}
try {
b = queue.take();
} catch (InterruptedException e) {
Log.e(TAG, "CODEC CALLBACK: take interrupted");
codec.stop();
finish();
return;
}
byte[] bb = b.array();
//Log.i(TAG,"CODEC CALLBACK: Contents being sent "+bb[4]/32+" "+bb[4]%32+" "+bb.length);
Log.i(TAG,"CODEC CALLBACK: Contents being sent "+Integer.toHexString(bb[0])+" "+Integer.toHexString(bb[1])+" "+Integer.toHexString(bb[2])+" "+Integer.toHexString(bb[3])+" "+Integer.toHexString(bb[4])+" ");
int ref_idc = bb[4]/32;
int unit_type = bb[4]%32;
//for (int i = 0; i < bb.length && i < 5; ++i) {
// Log.i(TAG, "CODEC CALLBACK: bb["+i+"]="+bb[i]);
//}
mutex.release();
// fill inputBuffer with valid data
//Log.i(TAG,"CODEC CALLBACK: put "+b.remaining()+" "+b.capacity());
inputBuffer.clear();
//Log.i(TAG,"CODEC CALLBACK: before put "+inputBuffer.remaining()+" "+b.position());
b.position(0);
inputBuffer.put(b);
//Log.i(TAG,"CODEC CALLBACK: after put "+inputBuffer.remaining());
//Log.i(TAG,"CODEC CALLBACK: queue "+(current_epoch-reference_epoch)*1000+" "+inputBuffer.capacity()+" "+inputBuffer.remaining());
codec.queueInputBuffer(inputBufferId,0, b.remaining(), (current_epoch-reference_epoch)*1000, 0);
}
@Override
public void onOutputBufferAvailable(MediaCodec mc, int outputBufferId,
MediaCodec.BufferInfo info) {
ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferId);
MediaFormat bufferFormat = codec.getOutputFormat(outputBufferId); // option A
// bufferFormat is equivalent to mOutputFormat
// outputBuffer is ready to be processed or rendered.
Log.i(TAG,"CODEC CALLBACK: output");
codec.releaseOutputBuffer(outputBufferId, true);
Log.i(TAG,"CODEC CALLBACK: output done");
}
@Override
public void onOutputFormatChanged(MediaCodec mc, MediaFormat format) {
// Subsequent data will conform to new format.
// Can ignore if using getOutputFormat(outputBufferId)
//mOutputFormat = format; // option B
Log.i(TAG,"CODEC CALLBACK: output format changed");
}
@Override
public void onError(MediaCodec codec, MediaCodec.CodecException e) {
Log.e(TAG,"CODEC CALLBACK: Media Codec Error");
}
});
codec.configure(decoderFormat, m_surface.getHolder().getSurface(), null, 0);
Thread thread = new Thread(){
public void run(){
Socket socket;
InputStream input;
try {
socket = new Socket(mServerAddr, Integer.parseInt(mServerPort));
input = socket.getInputStream();
} catch (IOException e) {
Log.e(TAG, "RLOOP: Failed to open video socket", e);
Toast.makeText(ARActivity.this, "Failed to open video socket",
Toast.LENGTH_LONG).show();
finish();
return;
}
Log.i(TAG,"RLOOP: HERE SOCKET OPENED");
System.out.println("Socket opened");
byte[] buffer = new byte[blockSize];
java.nio.ByteBuffer byteBuffer = java.nio.ByteBuffer.allocate(blockSize);
int nextStart = 0;
while (true) {
int size = 1;
try {
size = input.read(buffer,nextStart,blockSize-nextStart);
Log.i(TAG,"RLOOP: Read from video stream "+size+" bytes start="+nextStart);
Log.i(TAG, "RLOOP: First bytes "+buffer[nextStart]+" "+buffer[nextStart+1]+" "+
buffer[nextStart+2]+" "+buffer[nextStart+3]+" "+buffer[nextStart+4]);
if (size==0) {
Log.e(TAG, "RLOOP: Video stream finished");
Toast.makeText(ARActivity.this, "Video stream finished",
Toast.LENGTH_LONG).show();
codec.stop();
finish();
return;
}
int endPos = 2;
while (endPos > 0) {
endPos = -1;
int zeroCount = 0;
for (int i = nextStart; (i < size+nextStart && endPos < 1); ++i) {
//Log.i(TAG,"Zero count pos "+i+" "+zeroCount);
if (buffer[i]==0) {
++zeroCount;
} else if (buffer[i]==1 && zeroCount > 1) {
if (zeroCount > 3) {
zeroCount = 3;
}
endPos = i-zeroCount;
Log.i(TAG,"RLOOP: Found marker at pos "+(i-zeroCount));
zeroCount = 0;
} else {
zeroCount = 0;
}
}
Log.i(TAG,"RLOOP: State nextStart="+nextStart+" endPos="+endPos+" size="+size);
if (endPos < 0) {
if (size + nextStart == blockSize) {
Log.e(TAG, "RLOOP: Error reading video stream2");
//Toast.makeText(ARActivity.this, "Error reading video stream2",
// Toast.LENGTH_LONG).show();
//finish();
endPos = blockSize;
nextStart = 0;
Log.i(TAG, "RLOOP: BLOCK OVERFLOW " + endPos);
} else {
nextStart = size + nextStart;
}
} else if (endPos==0) {
Log.i(TAG, "RLOOP: BLOCK NOT COMPLETE " + endPos);
//nextStart = size+nextStart;
} else {
Log.i(TAG, "RLOOP: PROCESSING BLOCK " + endPos);
//Log.i(TAG,"BUFFER REMAINING "+byteBuffer.remaining());
//Log.i(TAG,"BUFFER POSITION "+byteBuffer.position());
//System.arraycopy(buffer, 4, buffer, 0, size + nextStart - 4);
//nextStart = nextStart - 4;
//if (nextStart < 0) {
// size = size + nextStart;
// nextStart = 0;
//}
//endPos = endPos-4;
byteBuffer = java.nio.ByteBuffer.allocate(endPos+3);
byteBuffer.put(buffer, 0, endPos);
//byteBuffer = java.nio.ByteBuffer.wrap(buffer, 0, endPos);
//byteBuffer.put(buffer,0, endPos);
Log.i(TAG, "RLOOP: BUFFER REMAINING2 " + byteBuffer.remaining());
Log.i(TAG, "RLOOP: BUFFER POSITION2 " + byteBuffer.position());
Log.i(TAG, "RLOOP: First send bytes " + buffer[0] + " " + buffer[1] + " " +
buffer[2] + " " + buffer[3] + " " + buffer[4]);
//byte[] bb = byteBuffer.array();
Log.i(TAG,"RLOOP: Contents being sent");
//for (int i = 0; i < bb.length && i < 10; ++i) {
// Log.i(TAG, "RLOOP: bb["+i+"]="+bb[i]);
//}
try {
mutex.acquire();
} catch (InterruptedException e) {
Log.e(TAG, "RLOOP: Mutex interrupted");
codec.stop();
finish();
return;
}
Log.i(TAG,"RLOOP: HERE1");
if (queue.size() == videoQueueSize) {
try {
queue.take();
} catch (InterruptedException e) {
Log.e(TAG, "RLOOP: queue.take interrupted 2");
codec.stop();
finish();
return;
}
Log.i(TAG,"RLOOP: HERE2");
try {
queueData.acquire();
} catch (InterruptedException e) {
Log.e(TAG, "RLOOP: queueData.acquire() interrupted 2");
codec.stop();
finish();
return;
}
}
Log.i(TAG,"RLOOP: HERE3");
try {
queue.put(byteBuffer);
} catch (InterruptedException e) {
Log.e(TAG, "RLOOP: queue put interrupted");
codec.stop();
finish();
return;
}
queueData.release();
mutex.release();
if (endPos < size+nextStart) {
System.arraycopy(buffer, endPos, buffer, 0, size + nextStart - endPos);
nextStart = nextStart - endPos;
if (nextStart < 0) {
size = size + nextStart;
nextStart = 0;
}
}
}
}
nextStart = nextStart + size;
} catch (IOException e) {
Log.e(TAG, "RLOOP: Error reading from video stream");
Toast.makeText(ARActivity.this, "Error reading from video stream",
Toast.LENGTH_LONG).show();
codec.stop();
finish();
return;
}
}
}
};
thread.start();
codec.start();
return;
}
我的预期结果是在 android 设备上看到视频。我的实际结果是永远不会调用 onOutputBufferAvailable 函数。
我包括一个示例调试输出，以显示一些正在发送到 MediaCodec 类的 NAL 单元。
2019-06-19 12:22:38.229 3325-3325/com.example.unrealar I/ARActivity: CODEC CALLBACK: input
2019-06-19 12:22:38.249 3325-3325/com.example.unrealar I/ARActivity: CODEC CALLBACK: Contents being sent 0 0 0 1 61
2019-06-19 12:22:38.251 3325-3325/com.example.unrealar I/ARActivity: CODEC CALLBACK: input
2019-06-19 12:22:38.266 3325-3325/com.example.unrealar I/ARActivity: CODEC CALLBACK: Contents being sent 0 0 0 1 61
2019-06-19 12:22:38.268 3325-3325/com.example.unrealar I/ARActivity: CODEC CALLBACK: input
2019-06-19 12:22:38.281 3325-3325/com.example.unrealar I/ARActivity: CODEC CALLBACK: Contents being sent 0 0 0 1 61
2019-06-19 12:22:38.282 3325-3539/com.example.unrealar I/MediaCodec: setCodecState state : 0
2019-06-19 12:22:38.282 3325-3325/com.example.unrealar I/ARActivity: CODEC CALLBACK: input
解决方案
我没有看到您配置编解码器。我的意思是发送SPS和PPS带有标志BUFFER_FLAG_CODEC_CONFIG。
此类数据 [CSD] 必须
BUFFER_FLAG_CODEC_CONFIG在调用queueInputBuffer
H264有多种传输方式。最常见的（至少在我看来）是：
在流的开头和每次编码参数更改时。
每个 NALU。每个 NALU 都有自己的一套 CSD。如果值发生变化，您只需要重新配置。
SPSPPS在每个关键帧之前和其他PPS切片之前。它被称为附件B
由于 FFMPEG 能够解码流，我猜这些值是流的一部分。所以我猜你需要解析你的 H264 流来确定 SPS 和 PPS 并将带有这些值的缓冲区发送BUFFER_FLAG_CODEC_CONFIG到解码器。或者，如果您决定在开始时缓冲一些帧，在开始解码之前，您也可以将这些值放入您的MediaFormatas"csd-0" (SPS)和"csd-1" (PPS)
SPS从 NALU 序列开始0x00 0x00 0x00 0x01 0x67。PPS从 NALU 序列开始0x00 0x00 0x00 0x01 0x68。
推荐阅读
- javascript - 如何在具有 Angular 8 的 FullCalendar 中使用 gotoDate()
- javascript - JS中没有任何其他字符的十进制验证
- java - 找不到表
- javascript - Bootstrap + popover - Skype href 链接不起作用
- c++ - 我可以在声明后将成员变量添加到结构中吗？
- botframework - 如何使用 Luis 更好地匹配城市名称？
- javascript - 我想在最大化模式下单击按钮在浏览器中打开新窗口
- c# - 在 Xamarin.Forms 中将 C# 文件设置为启动文件？
- powershell - Powershell 中的过滤器版本
- jquery - 信用卡屏蔽 - jquery