scottk/UltraGrid
1
0
Fork 0
mirror of https://github.com/outbackdingo/UltraGrid.git synced 2026-04-05 10:04:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make audio more robust

Browse Source 
Use three-times multiplication by default.
This commit is contained in:
Martin Pulec
2012-05-25 11:27:27 +02:00
parent f35c1fd70d
commit 7f318f7c79
10 changed files with 363 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
ultragrid/Makefile.in
View File

@@ -73,6 +73,7 @@ OBJS = @OBJS@ \
src/crypto/random.o \
src/ihdtv/ihdtv.o \
src/utils/fs_lock.o \
src/utils/packet_counter.o \
src/utils/ring_buffer.o \
src/video.o \
src/video_codec.o \
@@ -289,6 +290,9 @@ src/mac_gl_common.o: src/mac_gl_common.m src/mac_gl_common.h
src/utils/autorelease_pool.o: src/utils/autorelease_pool.m src/utils/autorelease_pool.h
$(CC) $(CFLAGS) $(INC) -x objective-c -c $< -o $@
src/utils/packet_counter.o: src/utils/packet_counter.cpp src/utils/packet_counter.h
$(CXX) $(CXXFLAGS) $(INC) -c $< -o $@
src/video_capture/DeckLinkAPIDispatch.o: $(DECKLINK_PATH)/DeckLinkAPIDispatch.cpp
$(CXX) $(CXXFLAGS) -c $(INC) -o src/video_capture/DeckLinkAPIDispatch.o $(DECKLINK_PATH)/DeckLinkAPIDispatch.cpp

7
ultragrid/src/audio/audio.c
View File

@@ -119,7 +119,7 @@ static struct rtp *initialize_audio_network(char *addr, int recv_port, int send_
/**
* take care that addrs can also be comma-separated list of addresses !
*/
struct state_audio * audio_cfg_init(char *addrs, int recv_port, int send_port, char *send_cfg, char *recv_cfg, char *jack_cfg)
struct state_audio * audio_cfg_init(char *addrs, int recv_port, int send_port, char *send_cfg, char *recv_cfg, char *jack_cfg, char *fec_cfg)
{
struct state_audio *s = NULL;
char *tmp, *unused = NULL;
@@ -145,7 +145,8 @@ struct state_audio * audio_cfg_init(char *addrs, int recv_port, int send_port, c
s = calloc(1, sizeof(struct state_audio));
s->audio_participants = NULL;
s->tx_session = tx_init(1500, NULL);
printf("Using audio FEC: %s\n", fec_cfg);
s->tx_session = tx_init(1500, fec_cfg);
gettimeofday(&s->start_time, NULL);
tmp = strdup(addrs);
@@ -315,7 +316,7 @@ static void *audio_receiver_thread(void *arg)
while (cp != NULL) {
if(pbuf_data.buffer != NULL) {
if (pbuf_decode(cp->playout_buffer, curr_time, decode_audio_frame, &pbuf_data, FALSE)) {
if (audio_pbuf_decode(cp->playout_buffer, curr_time, decode_audio_frame, &pbuf_data)) {
audio_playback_put_frame(s->audio_playback_device, pbuf_data.buffer);
pbuf_data.buffer = audio_playback_get_frame(s->audio_playback_device);
}

2
ultragrid/src/audio/audio.h
View File

@@ -68,7 +68,7 @@ typedef struct audio_frame
}
audio_frame;
struct state_audio * audio_cfg_init(char *addrs, int recv_port, int send_port, char *send_cfg, char *recv_cfg, char *jack_cfg);
struct state_audio * audio_cfg_init(char *addrs, int recv_port, int send_port, char *send_cfg, char *recv_cfg, char *jack_cfg, char *fec_cfg);
void audio_finish(struct state_audio *s);
void audio_done(struct state_audio *s);
void audio_join(struct state_audio *s);

11
ultragrid/src/main.c
View File

@@ -90,6 +90,10 @@
#define PORT_BASE 5004
#define PORT_AUDIO 5006
/* please see comments before transmit.c:audio_tx_send() */
/* also note that this actually differs from video */
#define DEFAULT_AUDIO_FEC "mult:3"
struct state_uv {
int recv_port_number;
int send_port_number;
@@ -510,7 +514,7 @@ static void *receiver_thread(void *arg)
/* Decode and render video... */
if (pbuf_decode
(cp->playout_buffer, uv->curr_time, decode_frame, &pbuf_data, TRUE)) {
(cp->playout_buffer, uv->curr_time, decode_frame, &pbuf_data)) {
tiles_post++;
/* we have data from all connections we need */
if(tiles_post == uv->connections_count)
@@ -935,7 +939,10 @@ int main(int argc, char *argv[])
}
}
uv->audio = audio_cfg_init (network_device, uv->recv_port_number + 2, uv->send_port_number + 2, audio_send, audio_recv, jack_cfg);
char *tmp_requested_fec = strdup(DEFAULT_AUDIO_FEC);
uv->audio = audio_cfg_init (network_device, uv->recv_port_number + 2, uv->send_port_number + 2, audio_send, audio_recv, jack_cfg,
tmp_requested_fec);
free(tmp_requested_fec);
if(!uv->audio)
goto cleanup;

41
ultragrid/src/rtp/audio_decoders.c
View File

@@ -67,9 +67,17 @@
#include "audio/audio.h"
#include "audio/utils.h"
#include "utils/packet_counter.h"
#include <time.h>
#define AUDIO_DECODER_MAGIC 0x12ab332bu
struct state_audio_decoder {
uint32_t magic;
struct timeval t0;
struct packet_counter *packet_counter;
};
void *audio_decoder_init(void)
@@ -79,6 +87,9 @@ void *audio_decoder_init(void)
s = (struct state_audio_decoder *) malloc(sizeof(struct state_audio_decoder));
s->magic = AUDIO_DECODER_MAGIC;
gettimeofday(&s->t0, NULL);
s->packet_counter = NULL;
return s;
}
@@ -89,6 +100,8 @@ void audio_decoder_destroy(void *state)
assert(s != NULL);
assert(s->magic == AUDIO_DECODER_MAGIC);
packet_counter_destroy(s->packet_counter);
free(s);
}
@@ -96,6 +109,7 @@ int decode_audio_frame(struct coded_data *cdata, void *data)
{
struct pbuf_audio_data *s = (struct pbuf_audio_data *) data;
struct audio_frame *buffer = s->buffer;
struct state_audio_decoder *decoder = s->decoder;
int total_channels = 0;
int bps, sample_rate, channel;
@@ -114,6 +128,7 @@ int decode_audio_frame(struct coded_data *cdata, void *data)
assert(total_channels > 0);
channel = (ntohl(hdr->substream_bufnum) >> 22) & 0x3ff;
int bufnum = ntohl(hdr->substream_bufnum) & 0x3fffff;
sample_rate = ntohl(hdr->quant_sample_rate) & 0x3fffff;
bps = (ntohl(hdr->quant_sample_rate) >> 26) / 8;
@@ -122,14 +137,18 @@ int decode_audio_frame(struct coded_data *cdata, void *data)
s->saved_sample_rate != sample_rate) {
if(audio_reconfigure(s->audio_state, bps * 8, total_channels,
sample_rate) != TRUE) {
fprintf(stderr, "Audio reconfiguration failed!\n");
fprintf(stderr, "Audio reconfiguration failed!");
return FALSE;
}
else fprintf(stderr, "Audio reconfiguration succeeded.\n");
else fprintf(stderr, "Audio reconfiguration succeeded.");
fprintf(stderr, " (%d channels, %d bps, %d Hz)\n", total_channels,
bps, sample_rate);
s->saved_channels = total_channels;
s->saved_bps = bps;
s->saved_sample_rate = sample_rate;
buffer = audio_get_frame(s->audio_state);
packet_counter_destroy(decoder->packet_counter);
decoder->packet_counter = packet_counter_init(total_channels);
}
data = cdata->data->data + sizeof(audio_payload_hdr_t);
@@ -137,6 +156,8 @@ int decode_audio_frame(struct coded_data *cdata, void *data)
int length = cdata->data->data_len - sizeof(audio_payload_hdr_t);
int offset = ntohl(hdr->offset);
//fprintf(stderr, "%d-%d-%d ", length, bufnum, channel);
packet_counter_register_packet(decoder->packet_counter, channel, bufnum, offset, length);
if(length * total_channels <= ((int) buffer->max_size) - offset) {
mux_channel(buffer->data + offset * total_channels, data, bps, length, total_channels, channel);
//memcpy(buffer->data + ntohl(hdr->offset), data, ntohs(hdr->length));
@@ -162,6 +183,22 @@ int decode_audio_frame(struct coded_data *cdata, void *data)
cdata = cdata->nxt;
}
double seconds;
struct timeval t;
gettimeofday(&t, 0);
seconds = tv_diff(t, decoder->t0);
if(seconds > 5.0) {
int bytes_received = packet_counter_get_total_bytes(decoder->packet_counter);
fprintf(stderr, "[Audio decoder] Received and decoded %u bytes (%d channels, %d samples) in last %f seconds (expected %d).\n",
bytes_received, total_channels,
bytes_received / (bps * total_channels),
seconds,
packet_counter_get_all_bytes(decoder->packet_counter));
decoder->t0 = t;
packet_counter_clear(decoder->packet_counter);
}
return TRUE;
}

46
ultragrid/src/rtp/pbuf.c
View File

@@ -335,13 +335,9 @@ static int frame_complete(struct pbuf_node *frame)
return (frame->mbit == 1);
}
/*
* wait_for_playout parameter specifies if we want to wait for playout time or not
* (audio case). If not, we play the frame immediatelly after it is complete.
*/
int
pbuf_decode(struct pbuf *playout_buf, struct timeval curr_time,
decode_frame_t decode_func, void *data, int wait_for_playout)
decode_frame_t decode_func, void *data)
{
/* Find the first complete frame that has reached it's playout */
/* time, and decode it into the framebuffer. Mark the frame as */
@@ -352,16 +348,46 @@ pbuf_decode(struct pbuf *playout_buf, struct timeval curr_time,
curr = playout_buf->frst;
while (curr != NULL) {
if (!curr->decoded && (!wait_for_playout || tv_gt(curr_time, curr->playout_time))) {
if (!curr->decoded && tv_gt(curr_time, curr->playout_time)) {
if (frame_complete(curr)) {
int ret = decode_func(curr->cdata, data);
curr->decoded = 1;
return ret;
} else {
if(wait_for_playout)
debug_msg
("Unable to decode frame due to missing data (RTP TS=%u)\n",
curr->rtp_timestamp);
debug_msg
("Unable to decode frame due to missing data (RTP TS=%u)\n",
curr->rtp_timestamp);
}
}
curr = curr->nxt;
}
return 0;
}
int
audio_pbuf_decode(struct pbuf *playout_buf, struct timeval curr_time,
decode_frame_t decode_func, void *data)
{
/* Find the first complete frame that has reached it's playout */
/* time, and decode it into the framebuffer. Mark the frame as */
/* decoded, but otherwise leave it in the playout buffer. */
struct pbuf_node *curr;
pbuf_validate(playout_buf);
curr = playout_buf->frst;
while (curr != NULL) {
/* WARNING: this one differs from video - we need to push audio immediately, because we do
* _not_ know the granularity of audio (typically 256 B for ALSA) which is only small fractal
* of frame time. The current RTP library isn't currently able to keep concurrently more frames.
*/
UNUSED(curr_time);
if (!curr->decoded // && tv_gt(curr_time, curr->playout_time)
) {
if (frame_complete(curr)) {
int ret = decode_func(curr->cdata, data);
curr->decoded = 1;
return ret;
}
}
curr = curr->nxt;

4
ultragrid/src/rtp/pbuf.h
View File

@@ -100,8 +100,10 @@ typedef int decode_frame_t(struct coded_data *cdata, void *decode_data);
*/
struct pbuf *pbuf_init(void);
void pbuf_insert(struct pbuf *playout_buf, rtp_packet *r);
int audio_pbuf_decode(struct pbuf *playout_buf, struct timeval curr_time,
decode_frame_t decode_func, void *data);
int pbuf_decode(struct pbuf *playout_buf, struct timeval curr_time,
decode_frame_t decode_func, void *data, int wait_for_playout_time);
decode_frame_t decode_func, void *data);
//struct video_frame *framebuffer, int i, struct state_decoder *decoder);
void pbuf_remove(struct pbuf *playout_buf, struct timeval curr_time);

46
ultragrid/src/transmit.c
View File

@@ -401,6 +401,10 @@ tx_send_base(struct tx *tx, struct tile *tile, struct rtp *rtp_session,
return packets;
}
/*
* This multiplication scheme relies upon the fact, that our RTP/pbuf implementation is
* not sensitive to packet duplication. Otherwise, we can get into serious problems.
*/
void audio_tx_send(struct tx* tx, struct rtp *rtp_session, audio_frame * buffer)
{
const int pt = 21; /* PT set for audio in our packet format */
@@ -418,7 +422,11 @@ void audio_tx_send(struct tx* tx, struct rtp *rtp_session, audio_frame * buffer)
struct timespec start, stop;
#endif /* HAVE_MACOSX */
long delta;
int mult_pos[FEC_MAX_MULT];
int mult_index = 0;
int mult_first_sent = 0;
timestamp = get_local_mediatime();
perf_record(UVP_SEND, timestamp);
@@ -427,6 +435,14 @@ void audio_tx_send(struct tx* tx, struct rtp *rtp_session, audio_frame * buffer)
demux_channel(chan_data, buffer->data, buffer->bps, buffer->data_len, buffer->ch_count, channel);
pos = 0u;
if(tx->fec_scheme == FEC_MULT) {
int i;
for (i = 0; i < tx->mult_count; ++i) {
mult_pos[i] = 0;
}
mult_index = 0;
}
uint32_t tmp;
tmp = channel << 22; /* bits 0-9 */
tmp |= tx->buffer; /* bits 10-31 */
@@ -443,6 +459,10 @@ void audio_tx_send(struct tx* tx, struct rtp *rtp_session, audio_frame * buffer)
payload_hdr.audio_tag = htonl(1); /* PCM */
do {
if(tx->fec_scheme == FEC_MULT) {
pos = mult_pos[mult_index];
}
data = chan_data + pos;
data_len = tx->mtu - 40 - sizeof(audio_payload_hdr_t);
if(pos + data_len >= (unsigned int) buffer->data_len / buffer->ch_count) {
@@ -455,17 +475,33 @@ void audio_tx_send(struct tx* tx, struct rtp *rtp_session, audio_frame * buffer)
GET_STARTTIME;
rtp_send_data_hdr(rtp_session, timestamp, pt, m, 0, /* contributing sources */
0, /* contributing sources length */
(char *) &payload_hdr, sizeof(payload_hdr),
data, data_len,
0, 0, 0);
if(data_len) { /* check needed for FEC_MULT */
rtp_send_data_hdr(rtp_session, timestamp, pt, m, 0, /* contributing sources */
0, /* contributing sources length */
(char *) &payload_hdr, sizeof(payload_hdr),
data, data_len,
0, 0, 0);
}
if(tx->fec_scheme == FEC_MULT) {
mult_pos[mult_index] = pos;
mult_first_sent ++;
if(mult_index != 0 || mult_first_sent >= (tx->mult_count - 1))
mult_index = (mult_index + 1) % tx->mult_count;
}
do {
GET_STOPTIME;
GET_DELTA;
if (delta < 0)
delta += 1000000000L;
} while (packet_rate - delta > 0);
/* when trippling, we need all streams goes to end */
if(tx->fec_scheme == FEC_MULT) {
pos = mult_pos[tx->mult_count - 1];
}
} while (pos < (unsigned int) buffer->data_len / buffer->ch_count);
}

156
ultragrid/src/utils/packet_counter.cpp Normal file
View File

@@ -0,0 +1,156 @@
/*
* FILE: utils/packet_counter.c
* AUTHORS: Martin Benes <martinbenesh@gmail.com>
* Lukas Hejtmanek <xhejtman@ics.muni.cz>
* Petr Holub <hopet@ics.muni.cz>
* Milos Liska <xliska@fi.muni.cz>
* Jiri Matela <matela@ics.muni.cz>
* Dalibor Matura <255899@mail.muni.cz>
* Ian Wesley-Smith <iwsmith@cct.lsu.edu>
*
* Copyright (c) 2005-2010 CESNET z.s.p.o.
*
* Redistribution and use in source and binary forms, with or without
* modification, is permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* 3. All advertising materials mentioning features or use of this software
* must display the following acknowledgement:
*
* This product includes software developed by CESNET z.s.p.o.
*
* 4. Neither the name of the CESNET nor the names of its contributors may be
* used to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING,
* BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
* AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
* EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*/
#include "utils/packet_counter.h"
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <iostream>
#include <map>
#include <vector>
using namespace std;
struct packet_counter {
packet_counter(int num_substreams) {
this->num_substreams = num_substreams;
substream_data.reserve(num_substreams);
for(int i = 0; i < num_substreams; ++i) {
substream_data.push_back(map<int, map<int, int> > ());
}
}
~packet_counter() {
}
void register_packet(int substream_id, int bufnum, int offset, int len) {
assert(substream_id < num_substreams);
substream_data[substream_id][bufnum][offset] = len;
}
int get_total_bytes() {
int ret = 0;
for(int i = 0; i < num_substreams; ++i) {
for(map<int, map<int, int> >::const_iterator it = substream_data[i].begin();
it != substream_data[i].end();
++it) {
for(map<int, int>::const_iterator it2 = it->second.begin();
it2 != it->second.end();
++it2) {
ret += it2->second;
}
}
}
return ret;
}
int get_all_bytes() {
int ret = 0;
for(int i = 0; i < num_substreams; ++i) {
for(map<int, map<int, int> >::const_iterator it = substream_data[i].begin();
it != substream_data[i].end();
++it) {
if(!it->second.empty()) {
ret += (--it->second.end())->first + (--it->second.end())->second;
}
}
}
return ret;
}
void clear() {
for(int i = 0; i < num_substreams; ++i) {
substream_data[i].clear();
}
}
vector<map<int, map<int, int> > > substream_data;
int num_substreams;
};
struct packet_counter *packet_counter_init(int num_substreams) {
struct packet_counter *state;
state = new packet_counter(num_substreams);
return state;
}
void packet_counter_destroy(struct packet_counter *state) {
if(state) {
delete state;
}
}
void packet_counter_register_packet(struct packet_counter *state, unsigned int substream_id, unsigned int bufnum,
unsigned int offset, unsigned int len)
{
state->register_packet(substream_id, bufnum, offset, len);
}
int packet_counter_get_total_bytes(struct packet_counter *state)
{
return state->get_total_bytes();
}
int packet_counter_get_all_bytes(struct packet_counter *state)
{
return state->get_all_bytes();
}
void packet_counter_clear(struct packet_counter *state)
{
state->clear();
}

70
ultragrid/src/utils/packet_counter.h Normal file
View File

@@ -0,0 +1,70 @@
/*
* FILE: utils/packet_counter.h
* AUTHORS: Martin Benes <mrtinbenesh@gmail.com>
* Lukas Hejtmanek <xhejtman@ics.muni.cz>
* Petr Holub <hopet@ics.muni.cz>
* Milos Liska <xliska@fi.muni.cz>
* Jiri Matela <matela@ics.muni.cz>
* Dalibor Matura <255899@mail.muni.cz>
* Ian Wesley-Smith <iwsmith@cct.lsu.edu>
*
* Copyright (c) 2005-2010 CESNET z.s.p.o.
*
* Redistribution and use in source and binary forms, with or without
* modification, is permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* 3. All advertising materials mentioning features or use of this software
* must display the following acknowledgement:
*
* This product includes software developed by CESNET z.s.p.o.
*
* 4. Neither the name of the CESNET nor the names of its contributors may be
* used to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING,
* BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
* AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
* EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
* OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
* EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*/
#ifndef __PACKET_COUNTER_H
#define __PACKET_COUNTER_H
struct packet_counter;
#ifdef __cplusplus
extern "C" {
#endif /* __cplusplus */
struct packet_counter *packet_counter_init(int num_substreams);
void packet_counter_destroy(struct packet_counter *state);
void packet_counter_register_packet(struct packet_counter *state, unsigned int substream_id,
unsigned int bufnum, unsigned int offset, unsigned int len);
int packet_counter_get_total_bytes(struct packet_counter *state);
int packet_counter_get_all_bytes(struct packet_counter *state);
void packet_counter_clear(struct packet_counter *state);
#ifdef __cplusplus
}
#endif /* __cplusplus */
#endif /* __PACKET_COUNTER_H */