diff --git a/internal/terraform/context_apply.go b/internal/terraform/context_apply.go
index 8631f04c30a0..8a5ee3f7482b 100644
--- a/internal/terraform/context_apply.go
+++ b/internal/terraform/context_apply.go
@@ -25,10 +25,6 @@ func (c *Context) Apply(plan *plans.Plan, config *configs.Config) (*states.State
 
 	log.Printf("[DEBUG] Building and walking apply graph for %s plan", plan.UIMode)
 
-	// FIXME: refresh plans still store outputs as changes, so we can't use
-	// Empty()
-	possibleRefresh := len(plan.Changes.Resources) == 0
-
 	graph, operation, diags := c.applyGraph(plan, config, true)
 	if diags.HasErrors() {
 		return nil, diags
@@ -81,7 +77,10 @@ Note that the -target option is not suitable for routine use, and is provided on
 	// cleanup is going to be needed to make the plan state match what apply
 	// would do. For now we can copy the checks over which were overwritten
 	// during the apply walk.
-	if possibleRefresh {
+	// Despite the intent of UIMode, it must still be used for apply-time
+	// differences in destroy plans too, so we can make use of that here as
+	// well.
+	if plan.UIMode == plans.RefreshOnlyMode {
 		newState.CheckResults = plan.Checks.DeepCopy()
 	}