diff --git a/roles/borgbackup-client/tasks/main.yml b/roles/borgbackup-client/tasks/main.yml
index 0e195fc01bedcb1ad301700714a3999ba07523ca..25b8ea0d05f4b2515365488d13cd914751c70530 100644
--- a/roles/borgbackup-client/tasks/main.yml
+++ b/roles/borgbackup-client/tasks/main.yml
@@ -11,6 +11,17 @@
     - ansible_distribution_major_version | int <= 10
     - ansible_distribution_release != "bullseye"
 
+- name: Don't pin borgmatic if we are on bullseye
+  file:
+    path: "/etc/apt/{{ item }}"
+    state: absent
+  loop:
+    - sources.list.d/bullseye.list
+    - preferences.d/borgmatic-bullseye
+  when:
+    - ansible_distribution == "Debian"
+    - ansible_distribution_release == "bullseye"
+
 - name: Install borgbackup
   apt:
     update_cache: true