diff --git a/telegram/ext/dispatcher.py b/telegram/ext/dispatcher.py
index 95f0e87c0..3be75d470 100644
--- a/telegram/ext/dispatcher.py
+++ b/telegram/ext/dispatcher.py
@@ -238,6 +238,9 @@ class Dispatcher(object):
         """
         if handler in self.handlers[group]:
             self.handlers[group].remove(handler)
+            if not self.handlers[group]:
+                del self.handlers[group]
+                self.groups.remove(group)
 
     def addErrorHandler(self, callback):
         """