diff --git a/mautrix_telegram/matrix.py b/mautrix_telegram/matrix.py
index e4c147dd..f96fb0db 100644
--- a/mautrix_telegram/matrix.py
+++ b/mautrix_telegram/matrix.py
@@ -90,10 +90,16 @@ class MatrixHandler:
     async def handle_invite(self, room, user, inviter):
         self.log.debug(f"{inviter} invited {user} to {room}")
         inviter = await User.get_by_mxid(inviter).ensure_started()
-        if not inviter.whitelisted:
-            return
-        elif user == self.az.bot_mxid:
+        if user == self.az.bot_mxid:
             await self.az.intent.join_room(room)
+            if not inviter.whitelisted:
+                await self.az.intent.send_notice(room, text=None,
+                                                 html="You are not whitelisted to use this bridge.<br/><br/>"
+                                                      "If you are the owner of this bridge, see the "
+                                                      "<code>bridge.permissions</code> section in your config file.")
+                await self.az.intent.leave_room(room)
+            return
+        elif not inviter.whitelisted:
             return
 
         puppet = Puppet.get_by_mxid(user)