diff --git a/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/StoryViewerFragment.kt b/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/StoryViewerFragment.kt
index 70cb4ee733..e1dd89c2fa 100644
--- a/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/StoryViewerFragment.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/StoryViewerFragment.kt
@@ -5,6 +5,7 @@ import android.graphics.Shader
 import android.os.Build
 import android.os.Bundle
 import android.view.View
+import androidx.activity.OnBackPressedCallback
 import androidx.core.app.ActivityCompat
 import androidx.core.view.ViewCompat
 import androidx.fragment.app.Fragment
@@ -89,7 +90,7 @@ class StoryViewerFragment :
     lifecycleDisposable.bindTo(viewLifecycleOwner)
     lifecycleDisposable += viewModel.state.observeOn(AndroidSchedulers.mainThread()).subscribe { state ->
       if (state.noPosts) {
-        ActivityCompat.finishAfterTransition(requireActivity())
+        onBackPressed(adapter.getRecipientId(storyPager.currentItem))
       }
 
       adapter.setPages(state.pages)
@@ -100,7 +101,7 @@ class StoryViewerFragment :
         pagerOnPageSelectedLock = false
 
         if (state.page >= state.pages.size) {
-          ActivityCompat.finishAfterTransition(requireActivity())
+          onBackPressed(adapter.getRecipientId(storyPager.currentItem))
           lifecycleDisposable.clear()
         }
       }
@@ -148,6 +149,23 @@ class StoryViewerFragment :
         }
       }
     }
+
+    requireActivity().onBackPressedDispatcher.addCallback(
+      viewLifecycleOwner,
+      object : OnBackPressedCallback(true) {
+        override fun handleOnBackPressed() {
+          onBackPressed(adapter.getRecipientId(storyPager.currentItem))
+        }
+      }
+    )
+  }
+
+  private fun onBackPressed(currentId: RecipientId) {
+    if (viewModel.getInitialRecipientId() != currentId) {
+      requireActivity().finish()
+    } else {
+      ActivityCompat.finishAfterTransition(requireActivity())
+    }
   }
 
   override fun onSaveInstanceState(outState: Bundle) {
diff --git a/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/StoryViewerPagerAdapter.kt b/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/StoryViewerPagerAdapter.kt
index c841889c7e..4cfd7ec10a 100644
--- a/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/StoryViewerPagerAdapter.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/StoryViewerPagerAdapter.kt
@@ -24,6 +24,10 @@ class StoryViewerPagerAdapter(
     DiffUtil.calculateDiff(callback).dispatchUpdatesTo(this)
   }
 
+  fun getRecipientId(position: Int): RecipientId {
+    return pages[position]
+  }
+
   override fun onAttachedToRecyclerView(recyclerView: RecyclerView) {
     super.onAttachedToRecyclerView(recyclerView)
     recyclerView.overScrollMode = RecyclerView.OVER_SCROLL_NEVER
diff --git a/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/StoryViewerViewModel.kt b/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/StoryViewerViewModel.kt
index 67f179b28d..196aa77578 100644
--- a/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/StoryViewerViewModel.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/StoryViewerViewModel.kt
@@ -91,6 +91,10 @@ class StoryViewerViewModel(
     refresh()
   }
 
+  fun getInitialRecipientId(): RecipientId {
+    return storyViewerArgs.recipientId
+  }
+
   fun setIsDisplayingFirstTimeNavigation(isDisplayingFirstTimeNavigation: Boolean) {
     firstTimeNavigationPublisher.onNext(isDisplayingFirstTimeNavigation)
   }
diff --git a/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/page/StoryViewerPageFragment.kt b/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/page/StoryViewerPageFragment.kt
index ecb2c6a06e..a770775396 100644
--- a/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/page/StoryViewerPageFragment.kt
+++ b/app/src/main/java/org/thoughtcrime/securesms/stories/viewer/page/StoryViewerPageFragment.kt
@@ -24,6 +24,7 @@ import android.widget.FrameLayout
 import android.widget.TextView
 import androidx.constraintlayout.widget.ConstraintLayout
 import androidx.constraintlayout.widget.ConstraintSet
+import androidx.core.app.ActivityCompat
 import androidx.core.content.ContextCompat
 import androidx.core.os.bundleOf
 import androidx.core.view.GestureDetectorCompat
@@ -230,7 +231,7 @@ class StoryViewerPageFragment :
     )
 
     closeView.setOnClickListener {
-      requireActivity().onBackPressed()
+      onBackPressed()
     }
 
     val addToGroupStoryDelegate = AddToGroupStoryDelegate(this)
@@ -288,7 +289,7 @@ class StoryViewerPageFragment :
         val canCloseFromHorizontalSlide = requireView().translationX > DimensionUnit.DP.toPixels(56f)
         val canCloseFromVerticalSlide = requireView().translationY > DimensionUnit.DP.toPixels(56f) || requireView().translationY < -DimensionUnit.DP.toPixels(56f)
         if ((canCloseFromHorizontalSlide || canCloseFromVerticalSlide) && event.actionMasked == MotionEvent.ACTION_UP) {
-          requireActivity().onBackPressed()
+          onBackPressed()
         } else {
           sharedViewModel.setIsChildScrolling(false)
           requireView().animate()
@@ -516,6 +517,14 @@ class StoryViewerPageFragment :
     }
   }
 
+  private fun onBackPressed() {
+    if (sharedViewModel.getInitialRecipientId() != storyViewerPageArgs.recipientId) {
+      requireActivity().finish()
+    } else {
+      ActivityCompat.finishAfterTransition(requireActivity())
+    }
+  }
+
   override fun onResume() {
     super.onResume()
     viewModel.setIsFragmentResumed(true)