diff --git a/configure.in b/configure.in
index 08f4e3763..7a650322a 100644
--- a/configure.in
+++ b/configure.in
@@ -391,38 +391,50 @@ AM_CONDITIONAL([HAVE_ALSA], [test "$with_alsa" != "no"])
 AC_ARG_WITH(portaudio,[  --without-portaudio     build without PortAudio audio provider.
                           (default: auto)], portaudio_disabled="(disabled)")
 
-if test "x$with_portaudio" != xno; then
+if test -z "$PORTAUDIO_LDFLAGS"; then
+  PORTAUDIO_LDFLAGS="$LDFLAGS -lportaudio"
+fi
+
+if test -z "$PORTAUDIO_CFLAGS"; then
+  PORTAUDIO_CFLAGS="$PTHREAD_CFLAGS $PTHREAD_LIBS"
+fi
+
+
+if test "$with_portaudio" != "no"; then
   aegisub_save_CPPFLAGS="$CPPFLAGS"
-  CPPFLAGS="$CPPFLAGS $PTHREAD_CFLAGS $PTHREAD_LIBS"
-  AC_CHECK_LIB([portaudio], [Pa_Initialize], [PORTAUDIO_LDFLAGS="-lportaudio"; found_audio=yes; with_portaudio=yes], [with_portaudio=no])
+  aegisub_save_LDFLAGS="$LDFLAGS"
+  CPPFLAGS="$PORTAUDIO_CFLAGS"
+  LDFLAGS="$LDFLAGS $PORTAUDIO_LDFLAGS"
+  AC_CHECK_LIB([portaudio], [Pa_Initialize], [with_portaudio="yes"], [with_portaudio="no"])
+  AC_CHECK_LIB([portaudio], [Pa_GetStreamTime], [
+    AC_DEFINE(HAVE_PA_GETSTREAMTIME, 1, [Define to 1 if Pa_GetStreamTime is available in PortAudio])
+  ], [])
+  CPPFLAGS="$aegisub_save_CPPFLAGS"
+  LDFLAGS="$aegisub_save_LDFLAGS"
 fi
 
-if test "x$with_portaudio" != xno; then
-  AC_CHECK_LIB([portaudio], [Pa_GetStreamTime], [LIBS="-lportaudio"], [with_portaudio_getstreamtime=no])
-    if test "x$with_portaudio_getstreamtime" != xno; then
-     AC_DEFINE(HAVE_PA_GETSTREAMTIME, 1, [Define to 1 if Pa_GetStreamTime is available in PortAudio])
-  fi
-CPPFLAGS="$aegisub_save_CPPFLAGS"
-fi
-
-AC_AGI_COMPILE([PortAudio], [portaudio], [$CFLAGS $PTHREAD_CFLAGS], [$LDFLAGS $PORTAUDIO_LDFLAGS],[
+if test "$with_portaudio" != "no"; then
+  AC_AGI_COMPILE([PortAudio], [portaudio], [$PORTAUDIO_CFLAGS], [$PORTAUDIO_LDFLAGS],[
 #include <portaudio.h>
 int main(void) {
   PaError err = Pa_Initialize();
   if (err != paNoError) return 1;
   return 0;
-}])
+} ])
+fi
 
 if test "$agi_cv_with_portaudio" = "no" && test "$with_portaudio" = "yes"; then
   AC_MSG_WARN([PortAudio detected, but it doesn't work...])
   with_portaudio="no"
 fi
 
-if test "$with_portaudio" = "yes"; then
+if test "$with_portaudio" = "yes" && test "$agi_cv_with_portaudio" = "yes"; then
+  found_audio="yes"
+  AC_SUBST(PORTAUDIO_LDFLAGS)
+  AC_SUBST(PORTAUDIO_CFLAGS)
   AC_DEFINE(WITH_PORTAUDIO, 1, [Enable PortAudio Audio Provider])
 fi
 
-AC_SUBST(PORTAUDIO_LDFLAGS)
 AM_CONDITIONAL([HAVE_PORTAUDIO], [test "$with_portaudio" != "no"])