Audio mixing and sound effects

Video SDK makes it simple for you to publish audio captured through the microphone to subscribers in a channel. In some real-time audio and video scenarios, such as games or karaoke, you need to play sound effects or mix in music files to enhance the atmosphere and add interest. Video SDK enables you to add sound effects and mix in pre-recorded audio.

This page shows you how to implement audio mixing and playing sound effects in your app.

Video SDK provides APIs that enable you to implement:

• Audio mixing

  Mix in music file such as background music with microphone audio. Using this feature, you can play only one file at a time.

• Sound effects

  Play audios with a short duration. For example, applause, cheers, or gunshots. You can play multiple sound effects at the same time.

Ensure that you have:

• Implemented the SDK quickstart in your project.

• Audio files in one of the supported formats.

This section shows you how to implement playing sound effects and add audio mixing in your app.

To manage audio mixing and voice effects, Video SDK provides the following APIs:

Function	Sound effect	Audio mixing
Play or stop playing a specific audio file	preloadEffect unloadEffect playEffect stopEffect stopAllEffects	startAudioMixing stopAudioMixing
Pause or resume playing an audio file	pauseEffect pauseAllEffects resumeEffect resumeAllEffects	pauseAudioMixing resumeAudioMixing
Get and adjust playback position and volume	setEffectPosition getEffectCurrentPosition getEffectsVolume setEffectsVolume setVolumeOfEffect	getAudioMixingCurrentPosition setAudioMixingPosition getAudioMixingPublishVolume adjustAudioMixingPublishVolume getAudioMixingPlayoutVolume adjustAudioMixingPlayoutVolume
Report playback status of audio files	onAudioEffectFinished	onAudioMixingStateChanged

Before joining a channel, call preloadEffect to preload the sound effect file. After joining the channel, call playEffect to play the specified sound effect file. To play multiple sound effect files simultaneously, set multiple sound effect IDs and call playEffect multiple times. After a sound effect is played, the Video SDK triggers the onAudioEffectFinished callback.

To implement this logic, refer to the following sample code:

void CAgoraEffectDlg::OnBnClickedButtonPreload()

{

if (m_cmbEffect.GetCurSel() < 0)

{

return;

}

CString strEffect;

m_cmbEffect.GetWindowText(strEffect);

std::string strPath = cs2utf8(strEffect);

// Preload the sound effect file

m_rtcEngine->preloadEffect(m_mapEffect[strEffect], strPath.c_str());

CString strInfo;

strInfo.Format(_T("preload effect: path:%s"), strEffect);

m_lstInfo.InsertString(m_lstInfo.GetCount(), strInfo);

}

void CAgoraEffectDlg::OnBnClickedButtonUnloadEffect()

{

if (m_cmbEffect.GetCurSel() < 0)

{

return;

}

CString strEffect;

m_cmbEffect.GetWindowText(strEffect);

// Unload the preloaded sound effect file

m_rtcEngine->unloadEffect(m_mapEffect[strEffect]);

CString strInfo;

strInfo.Format(_T("unload effect: path:%s"), strEffect);

m_lstInfo.InsertString(m_lstInfo.GetCount(), strInfo);

}

void CAgoraEffectDlg::OnBnClickedButtonPauseEffect()

{

if (m_cmbEffect.GetCurSel() < 0)

{

return;

}

CString strEffect;

m_cmbEffect.GetWindowText(strEffect);

// Pause playing the specified sound effect file

m_rtcEngine->pauseEffect(m_mapEffect[strEffect]);

CString strInfo;

strInfo.Format(_T("pause effect: path:%s"), strEffect);

m_lstInfo.InsertString(m_lstInfo.GetCount(), strInfo);

}

void CAgoraEffectDlg::OnBnClickedButtonResumeEffect()

{

if (m_cmbEffect.GetCurSel() < 0)

{

return;

}

CString strEffect;

m_cmbEffect.GetWindowText(strEffect);

// Resume playing the specified sound effect file

int ret = m_rtcEngine->resumeEffect(m_mapEffect[strEffect]);

CString strInfo;

strInfo.Format(_T("resume effect ret:%d : path:%s"), ret, strEffect);

m_lstInfo.InsertString(m_lstInfo.GetCount(), strInfo);

}

void CAgoraEffectDlg::OnBnClickedButtonPlayEffect()

{

if (m_cmbEffect.GetCurSel() < 0)

{

return;

}

CString strEffect;

m_cmbEffect.GetWindowText(strEffect);

std::string strFile = cs2utf8(strEffect);

CString strLoops;

m_edtLoops.GetWindowText(strLoops);

int loops = _ttol(strLoops);

if (loops == 0) {

m_edtLoops.SetWindowText(_T("1"));

loops = 1;

}

CString strPitch;

m_edtPitch.GetWindowText(strPitch);

double pitch = _ttof(strPitch);

CString strGain;

m_edtGain.GetWindowText(strGain);

int gain = _ttol(strGain);

CString strPan;

m_cmbPan.GetWindowText(strPan);

double pan = _ttof(strPan);

BOOL publish = m_chkPublish.GetCheck();

// Play the sound effect file

int ret = m_rtcEngine->playEffect(m_mapEffect[strEffect], strFile.c_str(), loops, pitch, pan, gain, publish);

CString strInfo;

strInfo.Format(_T("play effect: path:%s, ret:%d"), strEffect, ret);

m_lstInfo.InsertString(m_lstInfo.GetCount(), strInfo);

strInfo.Format(_T("loops:%d, pitch:%.1f, pan:%.0f, gain:%d, publish:%d"),

loops, pitch, pan, gain, publish);

m_lstInfo.InsertString(m_lstInfo.GetCount(), strInfo);

}

void CAgoraEffectDlg::OnBnClickedButtonStopEffect()

{

if (m_cmbEffect.GetCurSel() < 0)

{

return;

}

CString strEffect;

m_cmbEffect.GetWindowText(strEffect);

// Stop playing the specified sound effect file

m_rtcEngine->stopEffect(m_mapEffect[strEffect]);

CString strInfo;

strInfo.Format(_T("stop effect: path:%s"), strEffect);

m_lstInfo.InsertString(m_lstInfo.GetCount(), strInfo);

}

void CAgoraEffectDlg::OnBnClickedButtonPauseAllEffect()

{

if (!m_pauseAll)

{

// Pause playing all sound effect files

m_rtcEngine->pauseAllEffects();

CString strInfo;

strInfo.Format(_T("pause All Effects"));

m_lstInfo.InsertString(m_lstInfo.GetCount(), strInfo);

m_btnPauseAll.SetWindowText(AudioEffectCtrlResumeEffect);

}

else {

// Resume playing all sound effect files

m_rtcEngine->resumeAllEffects();

CString strInfo;

strInfo.Format(_T("resume All Effects"));

m_lstInfo.InsertString(m_lstInfo.GetCount(), strInfo);

m_btnPauseAll.SetWindowText(AudioEffectCtrlPauseAllEffect);

}

m_pauseAll = !m_pauseAll;

}

void CAgoraEffectDlg::OnBnClickedButtonStopAllEffect2()

{

// Stop playing all sound effect files

m_rtcEngine->stopAllEffects();

CString strInfo;

strInfo.Format(_T("stop All Effects"));

m_lstInfo.InsertString(m_lstInfo.GetCount(), strInfo);

}

To play a music file, call startAudioMixing before or after joining a channel. After you successfully call this method, Video SDK triggers the onAudioMixingStateChanged callback when the mixing status changes. This callback also reports the reason for the state change.

To implement this logic, refer to the following code:

void CAgoraAudioMixingDlg::OnBnClickedButtonMixingStart()

{

CString audioUrl = GetExePath() + _T("\\ID_MUSIC_01.m4a");

// Start playing the music file

int ret = m_rtcEngine->startAudioMixing(cs2utf8(audioUrl).c_str(), false, -1);

CString strInfo;

strInfo.Format(_T("startAudioMixing path:%s, ret:%d"), audioUrl.AllocSysString(), ret);

m_lstInfo.InsertString(m_lstInfo.GetCount(), strInfo);

}

// Listen for the onAudioMixingStateChanged callback

void CAudioMixingEventHandler::onAudioMixingStateChanged(AUDIO_MIXING_STATE_TYPE state, AUDIO_MIXING_REASON_TYPE reason)

{

if (m_hMsgHanlder) {

PAudioMixingState stateChanged = new AudioMixingState;

stateChanged->error = reason;

stateChanged->state = state;

::PostMessage(m_hMsgHanlder, WM_MSGID(EID_REMOTE_AUDIO_MIXING_STATE_CHANED), (WPARAM)stateChanged, 0);

}

}

void CAgoraAudioMixingDlg::OnBnClickedButtonMixingPause()

{ // Pause playback

int ret = m_rtcEngine->pauseAudioMixing();

CString strInfo;

strInfo.Format(_T("pauseAudioMixing ret:%d"), ret);

m_lstInfo.InsertString(m_lstInfo.GetCount(), strInfo);

}

void CAgoraAudioMixingDlg::OnBnClickedButtonMixingResume()

{ // Resume playback

int ret = m_rtcEngine->resumeAudioMixing();

CString strInfo;

strInfo.Format(_T("resumeAudioMixing ret:%d"), ret);

m_lstInfo.InsertString(m_lstInfo.GetCount(), strInfo);

}

void CAgoraAudioMixingDlg::OnBnClickedButtonMixingStop()

{ // Stop playback

int ret = m_rtcEngine->stopAudioMixing();

CString strInfo;

strInfo.Format(_T("stopAudioMixing ret:%d"), ret);

m_lstInfo.InsertString(m_lstInfo.GetCount(), strInfo);

}

void CAgoraAudioMixingDlg::OnNMCustomdrawSliderMixingPlayoutVolume(NMHDR* pNMHDR, LRESULT* pResult)

{

LPNMCUSTOMDRAW pNMCD = reinterpret_cast<LPNMCUSTOMDRAW>(pNMHDR);

int pos = m_sldMixingPlayoutVolume.GetPos();

// Adjust the playback volume of the current music file locally

int ret = m_rtcEngine->adjustAudioMixingPlayoutVolume(pos);

*pResult = 0;

}

void CAgoraAudioMixingDlg::OnNMCustomdrawSliderMixingPublishVolume(NMHDR* pNMHDR, LRESULT* pResult)

{

LPNMCUSTOMDRAW pNMCD = reinterpret_cast<LPNMCUSTOMDRAW>(pNMHDR);

int pos = m_sldMixingPublishVolume.GetPos();

// Adjust the playback volume of the current music file on the remote end

int ret = m_rtcEngine->adjustAudioMixingPublishVolume(pos);

*pResult = 0;

}

This section contains content that completes the information on this page, or points you to documentation that explains other aspects to this product.

Agora provides the following open source sample projects on GitHub for your reference.

• AudioEffect
• AudioMixing

Download or view the source code for more detailed examples.

• playEffect
• onAudioEffectFinished
• startAudioMixing
• onAudioMixingStateChanged