Portaudio is only needed for LIRC_DEVICES=audio, so not checking for Portaudio when the device is disabled will prevent automagic dependencies.
--- configure.ac.orig	2009-03-13 10:02:50.000000000 +0100
+++ configure.ac	2009-03-13 10:04:00.000000000 +0100
@@ -336,14 +336,6 @@
   AC_DEFINE(HAVE_LIBIRMAN_SW)
   possible_drivers="${possible_drivers} (irman_sw)"
 )
-dnl audio driver requires PortAudio library installed and some linker flags
-AC_CHECK_HEADERS(portaudio.h,[
-  AC_CHECK_LIB(portaudio, Pa_Initialize,[
-    AC_DEFINE(HAVE_LIBPORTAUDIO)
-    possible_drivers="${possible_drivers} (audio)"
-    ],,${portaudio_lib_other}
-  )]
-)
 dnl audio_alsa driver requires ALSA library installed and some linker flags
 have_alsa=no
 AC_CHECK_HEADERS(alsa/asoundlib.h,[