diff --git a/src/backends/x11/meta-monitor-manager-xrandr.c b/src/backends/x11/meta-monitor-manager-xrandr.c
index 0290475a7..98eff49a3 100644
--- a/src/backends/x11/meta-monitor-manager-xrandr.c
+++ b/src/backends/x11/meta-monitor-manager-xrandr.c
@@ -550,7 +550,7 @@ apply_crtc_assignments (MetaMonitorManager *manager,
   MetaX11ScaleMode scale_mode = meta_settings_get_x11_scale_mode (settings);
   unsigned i, valid_crtcs;
   GList *l;
-  int width, height;
+  int width, height, scaled_width, scaled_height;
   float max_scale;
   float avg_screen_scale;
   gboolean have_scaling;
@@ -563,6 +563,7 @@ apply_crtc_assignments (MetaMonitorManager *manager,
   /* Compute the new size of the screen (framebuffer) */
   max_scale = get_maximum_crtc_info_scale (crtcs, n_crtcs);
   width = 0; height = 0;
+  scaled_width = 0; scaled_height = 0;
   avg_screen_scale = 0;
   valid_crtcs = 0;
   for (i = 0; i < n_crtcs; i++)
@@ -578,6 +579,13 @@ apply_crtc_assignments (MetaMonitorManager *manager,
 
       if (have_scaling && scale_mode == META_X11_SCALE_MODE_UI_DOWN)
         scale = (ceilf (max_scale) / crtc_info->scale) * crtc_info->scale;
+      else
+        {
+          scaled_width = MAX (scaled_width, crtc_info->layout.origin.x +
+                              crtc_info->layout.size.width * crtc_info->scale);
+          scaled_height = MAX (scaled_height, crtc_info->layout.origin.y +
+                               crtc_info->layout.size.height * crtc_info->scale);
+        }
 
       width = MAX (width, (int) roundf (crtc_info->layout.origin.x +
                                         crtc_info->layout.size.width * scale));
@@ -609,7 +617,7 @@ apply_crtc_assignments (MetaMonitorManager *manager,
       y2 = (int) roundf (crtc_config->layout.origin.y +
                          crtc_config->layout.size.height);
 
-      if (!crtc_info->mode || x2 > width || y2 > height)
+      if (!crtc_info->mode || width < scaled_width || height < scaled_height || x2 > width || y2 > height)
         {
           xrandr_set_crtc_config (manager_xrandr,
                                   crtc,
@@ -661,7 +669,7 @@ apply_crtc_assignments (MetaMonitorManager *manager,
   if (!n_crtcs)
     goto out;
 
-  if (width > manager->screen_width || height > manager->screen_height)
+  if (width > 0 && height > 0)
     {
       meta_monitor_manager_xrandr_update_screen_size (manager_xrandr,
                                                       width, height,
@@ -783,13 +791,6 @@ apply_crtc_assignments (MetaMonitorManager *manager,
       output->is_primary = FALSE;
     }
 
-  if (width > 0 && height > 0)
-    {
-      meta_monitor_manager_xrandr_update_screen_size (manager_xrandr,
-                                                      width, height,
-                                                      avg_screen_scale);
-    }
-
 out:
   XUngrabServer (manager_xrandr->xdisplay);
   XFlush (manager_xrandr->xdisplay);
@@ -846,7 +847,7 @@ meta_monitor_manager_xrandr_update_screen_size_derived (MetaMonitorManager *mana
       if (!crtc || !crtc->config)
         continue;
 
-      if (!have_scaling || scale_mode != META_X11_SCALE_MODE_UI_DOWN)
+      if (!have_scaling)
         {
           /* When scaling up we should not reduce the screen size, or X will
            * fail miserably, while we must do it when scaling down, in order to