在上一篇中简单设计了一个视频播放器，但是没有声音，于是本篇的代码就是在上一篇的基础上加上声音。
在ffmpeg给的官方例程中，给视频添加声音是有问题的，这一块可以参考下面的这篇博客：
http://www.cnblogs.com/ansersion/p/5265033.html
其中播放声音比较ffmpeg以前版本增加了重采样的概念，而我添加声音这块代码也是参考这篇博客得来。
代码是在上一篇的基础上添加，对于之前的代码几乎没有删改，程序最终运行效果为视频跑的很快，但是声音则正常播放。
开发环境：
操作系统：ubuntu14
ffmpeg版本：3.2.2
sdl版本：2
编译与运行：
gcc -g main.c -o test -I /usr/local/ffmpeg/include -L /usr/local/ffmpeg/lib -lavutil -lavformat -lavcodec -lz -lavutil -lswscale -L /usr/lib/x86_64-linux-gnu -lSDL2 -lSDL2main（其中路径按照自己的安装路径来 -g 是加GDB调试）
./test
1 #ifdef _cplusplus 2 extern "C" 3 { 4 #endif 5 6 #include<stdio.h> 7 #include<assert.h> 8 #include<libavcodec/avcodec.h> 9 #include<libavformat/avformat.h> 10 #include<libavutil/avutil.h> 11 #include<libswscale/swscale.h> 12 #include <libswresample/swresample.h> 13 #include<libavutil/avutil.h> 14 #include<libavutil/imgutils.h> 15 #include<SDL2/SDL.h> 16 #include<SDL2/SDL_thread.h> 17 18 //是否将YUV420P内容输出到文件 19 #define OUTPUT_YUV420P 0 20 //要播放的文件路径 21 #define filename "/home/sns/test.flv" 22 //要输出YUV420P内容的文件路径 23 #define outfilename "/home/sns/output.yuv" 24 25 #define SDL_AUDIO_BUFFER_SIZE 1024 26 #define MAX_AUDIO_FRAME_SIZE 192000 27 //结构体定义 28 typedef struct PacketQueue{ 29 AVPacketList *first_pkt,*last_pkt;//队首、队尾 30 int nb_packets; //包的个数 31 int size;//队列的字节数 32 SDL_mutex *mutex;//互斥量 33 SDL_cond *cond;//条件变量 34 }PacketQueue; 35 36 typedef struct AudioParams { 37 int freq; 38 int channels; 39 int64_t channel_layout; 40 enum AVSampleFormat fmt; 41 int frame_size; 42 int bytes_per_sec; 43 } AudioParams; 44 //函数定义 45 static void audio_callback(void *userdata, Uint8 * stream, int len);// 46 static int packet_queue_get(PacketQueue *q,AVPacket *pkt,int block); 47 int packet_queue_put(PacketQueue *q,AVPacket *pkt); 48 void packet_queue_init(PacketQueue *q); 49 static int audio_decode_frame(AVCodecContext *aCodecCtx,uint8_t *audio_buf,int buf_size); 50 int resample(AVFrame *af,uint8_t *audio_buf,int *audio_buf_size); 51 52 //全局变量定义 53 int quit =0; 54 PacketQueue audioQ; 55 int sample_rate, nb_channels; 56 int64_t channel_layout; 57 struct SwrContext * swr_ctx = NULL; 58 AudioParams audio_hw_params_tgt; 59 AudioParams audio_hw_params_src; 60 61 //主函数 62 int main(int argc, char **argv) 63 { 64 //变量定义********************************************************************* 65 AVFormatContext *pFormatCtx; 66 int i=0; 67 int videoStream; 68 int audioStream; 69 AVCodecContext *pCodecCtx; 70 AVCodecContext *aCodecCtxOrig; 71 AVCodecContext *aCodecCtx; 72 AVCodec *pCodec; 73 AVCodec *aCodec; 74 AVFrame *pFrame; 75 AVFrame *pFrameYUV; 76 uint8_t *buffer; 77 int numBytes; 78 79 SDL_Window *screen; 80 SDL_Renderer *sdlRender; 81 SDL_Texture *sdlTexture; 82 SDL_Rect sdlRect; 83 int frameFinished; 84 AVPacket packet; 85 struct SwsContext *img_convert_ctx; 86 int err_code; 87 char buf[1024]; 88 FILE *fp_yuv; 89 int y_size; 90 SDL_AudioSpec audioSpec; 91 SDL_AudioSpec spec; 92 SDL_Event event; 93 //******************************************************************************* 94 av_register_all(); 95 //1、打开视频文件************************************************* 96 pFormatCtx = avformat_alloc_context(); 97 err_code = avformat_open_input(&pFormatCtx, filename, NULL, NULL); 98 if (err_code != 0) 99 {//打开文件失败100 av_strerror(err_code, buf, 1024);101 printf("coundn't open the file!,error code = %d(%s)\n", err_code, buf);102 return -1;103 }104 if (avformat_find_stream_info(pFormatCtx, NULL) < 0)105 {106 printf("Couldn't find stream information.\n");107 return -1;108 }109 // 打印信息110 av_dump_format(pFormatCtx, 0, filename, 0);111 //2、找到第一个视频流和第一个音频流****************************112 videoStream = -1;113 audioStream = -1;114 for (i = 0; i < pFormatCtx->nb_streams; i++)115 {116 if (pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO && videoStream<0)117 {118 videoStream = i;//得到视频流的索引119 }120 if(pFormatCtx->streams[i]->codec->codec_type == AVMEDIA_TYPE_AUDIO &&audioStream < 0)121 {122 audioStream = i;//得到音频流的索引123 }124 }125 if (videoStream == -1)126 {127 printf("Didn't find a video stream.\n");128 return -1;129 }130 if(audioStream == -1)131 {132 printf("coundn't find a audio stream!\n");133 return -1;134 }135 /* 3、从视频流中得到一个音频和视频编解码上下文,里面包含了编解码器的所有信息和一个136 指向真正的编解码器 ,然后我们找到音频和视频编解码器*/137 pCodecCtx = pFormatCtx->streams[videoStream]->codec;138 aCodecCtxOrig = pFormatCtx->streams[audioStream]->codec;139 pCodec = avcodec_find_decoder(pCodecCtx->codec_id);140 aCodec = avcodec_find_decoder(aCodecCtxOrig->codec_id);141 if (pCodec == NULL)142 {143 fprintf(stderr, "Unsupported codec !\n");144 return -1;145 }146 if(aCodec == NULL)147 {148 fprintf(stderr,"Unsupported codec!\n");149 return -1;150 }151 //拷贝上下文152 aCodecCtx = avcodec_alloc_context3(aCodec);153 if(avcodec_copy_context(aCodecCtx,aCodecCtxOrig) != 0)154 {155 fprintf(stderr,"couldn't copy codec context!\n");156 return -1;157 }158 //4、打开音频和视频编解码器159 if (avcodec_open2(pCodecCtx, pCodec, NULL) < 0)160 {161 printf("cann't open the codec!\n");162 return -1;163 }164 if(avcodec_open2(aCodecCtx,aCodec,NULL) < 0 )165 {166 printf("cann't open the audio codec!\n");167 return -1;168 }169 //设置声音参数170 sample_rate = aCodecCtx->sample_rate;171 nb_channels = aCodecCtx->channels;172 channel_layout = aCodecCtx->channel_layout;173 174 // printf("channel_layout=%" PRId64 "\n", channel_layout);175 printf("nb_channels=%d\n", nb_channels);176 printf("freq=%d\n", sample_rate);177 178 if (!channel_layout|| nb_channels != av_get_channel_layout_nb_channels(channel_layout))179 {180 channel_layout = av_get_default_channel_layout(nb_channels);181 channel_layout &= ~AV_CH_LAYOUT_STEREO_DOWNMIX;182 printf("correction\n");183 }184 /*通故编解码上下文中的所有信息来建立音频的信息*/185 audioSpec.freq = aCodecCtx->sample_rate;186 audioSpec.format = AUDIO_S16SYS;187 audioSpec.channels = aCodecCtx->channels;188 audioSpec.silence = 0;189 audioSpec.samples = SDL_AUDIO_BUFFER_SIZE;190 audioSpec.callback = audio_callback;191 audioSpec.userdata = aCodecCtx;192 //打开音频设备和初始化193 if(SDL_OpenAudio(&audioSpec,&spec) < 0)194 //其中回调函数在需要更多音频数据的时候被调用（即播放完后需要从回调取数据播放）195 {196 fprintf(stderr,"SDL_OpenAudio: %s\n",SDL_GetError());197 return -1;198 }199 printf("freq: %d\tchannels: %d\n", spec.freq, spec.channels);200 //5、分配两个视频帧，一个保存得到的原始视频帧，一个保存为指定格式的视频帧（该帧通过原始帧转换得来）201 pFrame = av_frame_alloc();202 if (pFrame == NULL)203 {204 printf("pFrame alloc fail!\n");205 return -1;206 }207 pFrameYUV = av_frame_alloc();208 if (pFrameYUV == NULL)209 {210 printf("pFrameYUV alloc fail!\n");211 return -1;212 }213 //6、得到一帧视频截图的内存大小并分配内存，并将YUV数据填充进去214 numBytes = av_image_get_buffer_size(AV_PIX_FMT_YUV420P, pCodecCtx->width,215 pCodecCtx->height,1);216 buffer = (uint8_t*) av_mallocz(numBytes * sizeof(uint8_t));217 if (!buffer)218 {219 printf("numBytes :%d , buffer malloc 's mem \n", numBytes);220 return -1;221 }222 //打印信息223 printf("--------------- File Information ----------------\n");224 av_dump_format(pFormatCtx, 0, filename, 0);225 printf("-------------------------------------------------\n");226 av_image_fill_arrays(pFrameYUV->data, pFrameYUV->linesize,buffer,227 AV_PIX_FMT_YUV420P,pCodecCtx->width, pCodecCtx->height,1);228 //7、得到指定转换格式的上下文**********************************229 img_convert_ctx = sws_getContext(pCodecCtx->width, pCodecCtx->height,230 pCodecCtx->pix_fmt, pCodecCtx->width, pCodecCtx->height,231 AV_PIX_FMT_YUV420P,232 SWS_BICUBIC,233 NULL, NULL, NULL);234 if (img_convert_ctx == NULL)235 {236 fprintf(stderr, "Cannot initialize the conversion context!\n");237 return -1;238 }239 //***********************************************************240 #if OUTPUT_YUV420P241 fp_yuv = fopen(outfilename, "wb+");242 #endif243 //8、SDL初始化和创建多重windows等准备工作244 if (SDL_Init(SDL_INIT_AUDIO | SDL_INIT_TIMER | SDL_INIT_VIDEO))245 {246 fprintf(stderr, "Could not initialize SDL - %s\n", SDL_GetError());247 return -1;248 }249 //使用SDL_CreateWindow代替SDL_SetVideoMode250 //创建一个给定高度和宽度、位置和标示的windows。251 screen = SDL_CreateWindow("Simplest ffmpeg player's Window",252 SDL_WINDOWPOS_UNDEFINED, SDL_WINDOWPOS_UNDEFINED, pCodecCtx->width,253 pCodecCtx->height, SDL_WINDOW_OPENGL);254 if (!screen)255 {256 fprintf(stderr, "SDL: could not create window - exiting - %s\n",SDL_GetError());257 return -1;258 }259 //对该window创建一个2D渲染上下文260 sdlRender = SDL_CreateRenderer(screen, -1, 0);261 if (!sdlRender)262 {263 fprintf(stderr, "SDL:cound not create render : %s\n", SDL_GetError());264 return -1;265 }266 //Create a texture for a rendering context.267 //为一个渲染上下文创建一个纹理268 //IYUV: Y + U + V (3 planes)269 //YV12: Y + V + U (3 planes)270 sdlTexture = SDL_CreateTexture(sdlRender, SDL_PIXELFORMAT_IYUV,271 SDL_TEXTUREACCESS_STREAMING, pCodecCtx->width, pCodecCtx->height);272 if (!sdlTexture)273 {274 fprintf(stderr, "SDL:cound not create Texture : %s\n", SDL_GetError());275 return -1;276 }277 //建立一个矩形变量，提供后面使用278 sdlRect.x = 0;279 sdlRect.y = 0;280 sdlRect.w = pCodecCtx->width;281 sdlRect.h = pCodecCtx->height;282 //*****************************************************283 //声音部分代码284 audio_hw_params_tgt.fmt = AV_SAMPLE_FMT_S16;285 audio_hw_params_tgt.freq = spec.freq;286 audio_hw_params_tgt.channel_layout = channel_layout;287 audio_hw_params_tgt.channels = spec.channels;288 audio_hw_params_tgt.frame_size = av_samples_get_buffer_size(NULL,289 audio_hw_params_tgt.channels, 1, audio_hw_params_tgt.fmt, 1);290 audio_hw_params_tgt.bytes_per_sec = av_samples_get_buffer_size(NULL,291 audio_hw_params_tgt.channels, audio_hw_params_tgt.freq,292 audio_hw_params_tgt.fmt, 1);293 if (audio_hw_params_tgt.bytes_per_sec <= 0|| audio_hw_params_tgt.frame_size <= 0)294 {295 printf("size error\n");296 return -1;297 }298 audio_hw_params_src = audio_hw_params_tgt;299 //*****************************************************300 packet_queue_init(&audioQ);301 SDL_PauseAudio(0);302 //9、正式开始读取数据*****************************************303 while (av_read_frame(pFormatCtx, &packet) >= 0)304 {305 //如果读取的包来自视频流306 if (packet.stream_index == videoStream)307 {308 //从包中得到解码后的帧309 if (avcodec_decode_video2(pCodecCtx, pFrame, &frameFinished,&packet) < 0)310 {311 printf("Decode Error!\n");312 return -1;313 }314 //如果确定完成得到该视频帧315 if (frameFinished)316 {317 //转换帧数据格式318 sws_scale(img_convert_ctx, pFrame->data, pFrame->linesize, 0,319 pCodecCtx->height,320 pFrameYUV->data,321 pFrameYUV->linesize);322 #if OUTPUT_YUV420P323 y_size = pCodecCtx->width * pCodecCtx->height;324 fwrite(pFrameYUV->data[0], 1, y_size, fp_yuv); //Y325 fwrite(pFrameYUV->data[1], 1, y_size / 4, fp_yuv); //U326 fwrite(pFrameYUV->data[2], 1, y_size / 4, fp_yuv); //V327 #endif328 //SDL显示~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~329 #if 0330 SDL_UpdateTexture(sdlTexture,NULL,pFrameYUV->data[0],pFrameYUV->linesize[0]);331 #else332 SDL_UpdateYUVTexture(sdlTexture, &sdlRect, pFrameYUV->data[0],333 pFrameYUV->linesize[0], pFrameYUV->data[1],334 pFrameYUV->linesize[1], pFrameYUV->data[2],335 pFrameYUV->linesize[2]);336 #endif337 SDL_RenderClear(sdlRender);338 SDL_RenderCopy(sdlRender, sdlTexture, NULL, &sdlRect);339 SDL_RenderPresent(sdlRender);340 //结束SDL~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~341 }342 }343 else if(packet.stream_index == audioStream)344 {345 packet_queue_put(&audioQ,&packet);346 }else{347 av_free_packet(&packet);//释放读出来的包348 }349 SDL_PollEvent(&event);350 switch (event.type)351 {352 case SDL_QUIT:353 quit = 1;354 SDL_Quit();355 exit(0);356 break;357 default:358 break;359 }360 }361 while(1) SDL_Delay(1000);362 //**************************************************************************************363 //10、释放分配的内存或关闭文件等操作364 #if OUTPUT_YUV420P365 fclose(fp_yuv);366 #endif367 sws_freeContext(img_convert_ctx);368 SDL_Quit();369 av_free(buffer);370 av_free(pFrame);371 av_free(pFrameYUV);372 avcodec_close(pCodecCtx);373 avcodec_close(aCodecCtxOrig);374 avcodec_close(aCodecCtx);375 avformat_close_input(&pFormatCtx);376 return EXIT_SUCCESS;377 }378 379 380 static void audio_callback(void *userdata, Uint8 * stream, int len)381 {382 AVCodecContext *aCodecCtx = (AVCodecContext*)userdata;383 int len1,audio_size;384 static uint8_t audio_buf[(MAX_AUDIO_FRAME_SIZE*3)/2];385 static unsigned int audio_buf_size = 0;386 static unsigned int audio_buf_index =0;387 while(len>0){388 if(audio_buf_index >= audio_buf_size){389 audio_size = audio_decode_frame(aCodecCtx,audio_buf,sizeof(audio_buf));390 if(audio_size < 0){391 audio_buf_size = 1024;392 memset(audio_buf,0,audio_buf_size);393 }else{394 audio_buf_size = audio_size;395 }396 audio_buf_index = 0;397 }398 len1 = audio_buf_size - audio_buf_index;399 if(len1 > len)400 len1 = len;401 memcpy(stream, (uint8_t *)audio_buf + audio_buf_index, len1);402 len -= len1;403 stream += len1;404 audio_buf_index += len1;405 }406 }407 408 void packet_queue_init(PacketQueue *q)409 {410 memset(q,0,sizeof(PacketQueue));411 q->mutex = SDL_CreateMutex();412 q->cond = SDL_CreateCond();413 }414 415 int packet_queue_put(PacketQueue *q,AVPacket *pkt)416 {417 AVPacketList *pkt1;418 if(av_dup_packet(pkt)<0)419 {420 printf("dup packet error!\n");421 return -1;422 }423 pkt1 = av_malloc(sizeof(AVPacketList));424 if(!pkt1)425 {426 printf("malloc AVPacketList error!\n");427 return -1;428 }429 pkt1->pkt = *pkt;430 pkt1->next = NULL;431 SDL_LockMutex(q->mutex);432 if(!q->last_pkt){433 q->first_pkt = pkt1;434 }else{435 q->last_pkt->next = pkt1;436 }437 q->last_pkt = pkt1;438 q->nb_packets ++;439 q->size +=pkt1->pkt.size;440 SDL_CondSignal(q->cond);441 SDL_UnlockMutex(q->mutex);442 return 0;443 }444 static int packet_queue_get(PacketQueue *q,AVPacket *pkt,int block){445 int ret;446 AVPacketList *pkt1;447 SDL_LockMutex(q->mutex);448 for(;;){449 if(quit){450 printf("packet_queue has quit!\n");451 ret =-1;452 break;453 }454 pkt1 = q->first_pkt;455 if(pkt1){456 q->first_pkt = pkt1->next;457 if(!q->first_pkt){458 q->last_pkt = NULL;459 }460 q->nb_packets--;461 q->size -= pkt1->pkt.size;462 *pkt = pkt1->pkt;463 av_free(pkt1);464 ret =1;465 break;466 }else if(!block){467 ret =0;468 break;469 }else{470 SDL_CondWait(q->cond,q->mutex);//做了解锁互斥量的动作471 }472 }473 SDL_UnlockMutex(q->mutex);474 return ret;475 }476 static int audio_decode_frame(AVCodecContext *aCodecCtx,uint8_t *audio_buf,int buf_size)477 {478 static AVPacket pkt;479 static uint8_t *audio_pkt_data = NULL;480 static int audio_pkt_size = 0;481 int len1,data_size=0;482 static AVFrame frame;483 int got_frame=0;484 485 for(;;){486 while(audio_pkt_size >0 ){487 len1 = avcodec_decode_audio4(aCodecCtx, &frame, &got_frame, &pkt);488 if (len1 < 0)489 {490 /* if error, skip frame */491 audio_pkt_size = 0;492 break;493 }494 audio_pkt_data += len1;495 audio_pkt_size -= len1;496 data_size =0;497 if(got_frame){498 data_size = resample(&frame, audio_buf, &buf_size);499 assert(data_size <= buf_size);500 }501 if (data_size <= 0)502 {503 /* No data yet, get more frames */504 continue;505 }506 /* We have data, return it and come back for more later */507 return data_size;508 }509 if (pkt.data)510 av_free_packet(&pkt);511 if (quit)512 {513 return -1;514 }515 if (packet_queue_get(&audioQ, &pkt, 1) < 0)516 {517 printf("packet_queue_get error!\n");518 return -1;519 }520 audio_pkt_data = pkt.data;521 audio_pkt_size = pkt.size;522 }523 return 1;524 }525 526 //重采样527 int resample(AVFrame *af, uint8_t *audio_buf, int *audio_buf_size)528 {529 int data_size = 0;530 int resampled_data_size = 0;531 int64_t dec_channel_layout;532 data_size = av_samples_get_buffer_size(NULL,533 av_frame_get_channels(af),534 af->nb_samples,535 af->format, 1);536 dec_channel_layout =(af->channel_layout&&537 av_frame_get_channels(af)== av_get_channel_layout_nb_channels(538 af->channel_layout)) ?539 af->channel_layout :540 av_get_default_channel_layout(av_frame_get_channels(af));541 if (af->format != audio_hw_params_src.fmt542 || af->sample_rate != audio_hw_params_src.freq543 || dec_channel_layout != audio_hw_params_src.channel_layout544 || !swr_ctx)545 {546 swr_free(&swr_ctx);547 swr_ctx = swr_alloc_set_opts(NULL, audio_hw_params_tgt.channel_layout,548 audio_hw_params_tgt.fmt, audio_hw_params_tgt.freq,549 dec_channel_layout, af->format, af->sample_rate, 0, NULL);550 if (!swr_ctx || swr_init(swr_ctx) < 0)551 {552 av_log(NULL, AV_LOG_ERROR,553 "Cannot create sample rate converter for conversion of %d Hz %s %d channels to %d Hz %s %d channels!\n",554 af->sample_rate, av_get_sample_fmt_name(af->format),555 av_frame_get_channels(af), audio_hw_params_tgt.freq,556 av_get_sample_fmt_name(audio_hw_params_tgt.fmt),557 audio_hw_params_tgt.channels);558 swr_free(&swr_ctx);559 return -1;560 }561 printf("swr_init\n");562 audio_hw_params_src.channels = av_frame_get_channels(af);563 audio_hw_params_src.fmt = af->format;564 audio_hw_params_src.freq = af->sample_rate;565 }566 567 if (swr_ctx)568 {569 const uint8_t **in = (const uint8_t **) af->extended_data;570 uint8_t **out = &audio_buf;571 int out_count = (int64_t) af->nb_samples * audio_hw_params_tgt.freq572 / af->sample_rate + 256;573 int out_size = av_samples_get_buffer_size(NULL,574 audio_hw_params_tgt.channels, out_count,575 audio_hw_params_tgt.fmt, 0);576 int len2;577 if (out_size < 0)578 {579 av_log(NULL, AV_LOG_ERROR, "av_samples_get_buffer_size() failed\n");580 return -1;581 }582 av_fast_malloc(&audio_buf, audio_buf_size, out_size);583 if (!audio_buf)584 return AVERROR(ENOMEM);585 len2 = swr_convert(swr_ctx, out, out_count, in, af->nb_samples);586 if (len2 < 0)587 {588 av_log(NULL, AV_LOG_ERROR, "swr_convert() failed\n");589 return -1;590 }591 if (len2 == out_count)592 {593 av_log(NULL, AV_LOG_WARNING,594 "audio buffer is probably too small\n");595 if (swr_init(swr_ctx) < 0)596 swr_free(&swr_ctx);597 }598 resampled_data_size = len2 * audio_hw_params_tgt.channels599 * av_get_bytes_per_sample(audio_hw_params_tgt.fmt);600 }601 else602 {603 audio_buf = af->data[0];604 resampled_data_size = data_size;605 }606 607 return resampled_data_size;608 }609 610 #ifdef _cplusplus611 }612 #endif
代码那么多，不好理解呀～～～～～
所以程序还是最好模块化和分文件来做才行，或者封装成一个类。
会得到最大的满足，因为它填补了你的空虚。
