diff --git a/mautrix_telegram/puppet.py b/mautrix_telegram/puppet.py
index f2299046..2933d8a2 100644
--- a/mautrix_telegram/puppet.py
+++ b/mautrix_telegram/puppet.py
@@ -87,7 +87,7 @@ class Puppet(BasePuppet):
                  username: Optional[str] = None,
                  displayname: Optional[str] = None,
                  displayname_source: Optional[TelegramID] = None,
-                 displayname_contact: bool = False,
+                 displayname_contact: bool = True,
                  photo_id: Optional[str] = None,
                  is_bot: bool = False,
                  is_registered: bool = False,
@@ -282,7 +282,10 @@ class Puppet(BasePuppet):
         if not info.contact:
             self.displayname_contact = False
         elif not self.displayname_contact:
-            return False
+            if not self.displayname:
+                self.displayname_contact = True
+            else:
+                return False
 
         displayname = self.get_displayname(info)
         if displayname != self.displayname: