diff --git a/mautrix_telegram/config.py b/mautrix_telegram/config.py
index 978990af..7c07f678 100644
--- a/mautrix_telegram/config.py
+++ b/mautrix_telegram/config.py
@@ -83,6 +83,7 @@ class Config(BaseBridgeConfig):
 
         copy("bridge.displayname_preference")
         copy("bridge.displayname_max_length")
+        copy("bridge.allow_avatar_remove")
 
         copy("bridge.max_initial_member_sync")
         copy("bridge.sync_channel_members")
diff --git a/mautrix_telegram/example-config.yaml b/mautrix_telegram/example-config.yaml
index 4981dee6..219f69d9 100644
--- a/mautrix_telegram/example-config.yaml
+++ b/mautrix_telegram/example-config.yaml
@@ -121,6 +121,10 @@ bridge:
     - phone number
     # Maximum length of displayname
     displayname_max_length: 100
+    # Remove avatars from Telegram ghost users when removed on Telegram. This is disabled by default
+    # as there's no way to determine whether an avatar is removed or just hidden from some users. If
+    # you're on a single-user instance, this should be safe to enable.
+    allow_avatar_remove: false
 
     # Maximum number of members to sync per portal when starting up. Other members will be
     # synced when they send messages. The maximum is 10000, after which the Telegram server
diff --git a/mautrix_telegram/portal/metadata.py b/mautrix_telegram/portal/metadata.py
index 69e4fcfe..0e60d9bb 100644
--- a/mautrix_telegram/portal/metadata.py
+++ b/mautrix_telegram/portal/metadata.py
@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2020 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -695,6 +695,8 @@ class PortalMetadata(BasePortal, ABC):
             loc = None
         else:
             raise ValueError(f"Unknown photo type {type(photo)}")
+        if self.peer_type == "user" and not photo_id and not config["bridge.allow_avatar_remove"]:
+            return False
         if self.photo_id != photo_id:
             if not photo_id:
                 await self._try_set_state(sender, EventType.ROOM_AVATAR,
diff --git a/mautrix_telegram/puppet.py b/mautrix_telegram/puppet.py
index 9f8249f8..c0507500 100644
--- a/mautrix_telegram/puppet.py
+++ b/mautrix_telegram/puppet.py
@@ -1,5 +1,5 @@
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2019 Tulir Asokan
+# Copyright (C) 2020 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -300,6 +300,8 @@ class Puppet(CustomPuppetMixin):
         else:
             self.log.warning(f"Unknown user profile photo type: {type(photo)}")
             return False
+        if not photo_id and not config["bridge.allow_avatar_remove"]:
+            return False
         if self.photo_id != photo_id:
             if not photo_id:
                 self.photo_id = ""