diff --git a/mautrix_telegram/abstract_user.py b/mautrix_telegram/abstract_user.py
index 053bc89f..dba567ec 100644
--- a/mautrix_telegram/abstract_user.py
+++ b/mautrix_telegram/abstract_user.py
@@ -390,13 +390,13 @@ class AbstractUser(ABC):
             update = update.message
             if isinstance(update, MessageEmpty):
                 return update, None, None
-            if isinstance(update.to_id, PeerUser) and not update.out:
-                portal = po.Portal.get_by_tgid(update.from_id, peer_type="user",
-                                               tg_receiver=self.tgid)
+            portal = po.Portal.get_by_entity(update.peer_id, receiver_id=self.tgid)
+            if update.out:
+                sender = pu.Puppet.get(self.tgid)
+            elif isinstance(update.from_id, PeerUser):
+                sender = pu.Puppet.get(TelegramID(update.from_id.user_id))
             else:
-                portal = po.Portal.get_by_entity(update.to_id, receiver_id=self.tgid)
-            sender = (pu.Puppet.get(TelegramID(update.from_id.user_id))
-                      if isinstance(update.from_id, PeerUser) else None)
+                sender = None
         else:
             self.log.warning("Unexpected message type in User#get_message_details: "
                              f"{type(update)}")
diff --git a/mautrix_telegram/user.py b/mautrix_telegram/user.py
index 00435068..ab44a432 100644
--- a/mautrix_telegram/user.py
+++ b/mautrix_telegram/user.py
@@ -258,12 +258,7 @@ class User(AbstractUser, BaseUser):
             return False
 
         if isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage)):
-            message = update.message
-            if isinstance(message.to_id, PeerUser) and not message.out:
-                portal = po.Portal.get_by_tgid(message.from_id, peer_type="user",
-                                               tg_receiver=self.tgid)
-            else:
-                portal = po.Portal.get_by_entity(message.to_id, receiver_id=self.tgid)
+            portal = po.Portal.get_by_entity(update.message.peer_id, receiver_id=self.tgid)
         elif isinstance(update, UpdateShortChatMessage):
             portal = po.Portal.get_by_tgid(TelegramID(update.chat_id))
         elif isinstance(update, UpdateShortMessage):