config/output: Pick among all 8-bit and 10-bit formats

The usual 8-bit and 10-bit formats have a lot of variations:

        XRGB, XBGR, RGBX, BGRX, ARGB, ABGR, RGBA, BGRA

A particular display is likely to support multiple of these, and as we
need to allocate a buffer of a particular type to test it, we'd like to
avoid doing too many unnecessary tests.

If we go off on the assumption that a display device generally supports
a subset of format variations but otherwise support each variation
equally well - that is, if both RGB and BGR is supported, they will
either both succeed or both fail at any given time - then instead of
trying all formats, we can simply pick one supported format within each
format group to try. For example, XBGR2101010 and XRGB8888.

By being conservative in how many formats we try, we can aggressively
expand the list of candidate formats to handle odd hardware.

We can get rid of the the conservative aspect if KMS ever grows the
ability to commit-test a buffer format without allocating the buffer
first, allowing us to quickly test every format under the sun.

sway/config/output.c

@@ -402,10 +402,6 @@ static bool render_format_is_10bit(uint32_t render_format) {
 		render_format == DRM_FORMAT_XBGR2101010;
 }
 
-static bool render_format_is_bgr(uint32_t fmt) {
-	return fmt == DRM_FORMAT_XBGR2101010 || fmt == DRM_FORMAT_XBGR8888;
-}
-
 static bool output_config_is_disabling(struct output_config *oc) {
 	return oc && (!oc->enabled || oc->power == 0);
 }
@@ -773,36 +769,64 @@ static bool search_mode(struct search_context *ctx, size_t output_idx) {
 	return false;
 }
 
+static const uint32_t fmts10bit[] = {
+	DRM_FORMAT_XRGB2101010,
+	DRM_FORMAT_XBGR2101010,
+	DRM_FORMAT_RGBX1010102,
+	DRM_FORMAT_BGRX1010102,
+	DRM_FORMAT_ARGB2101010,
+	DRM_FORMAT_ABGR2101010,
+	DRM_FORMAT_RGBA1010102,
+	DRM_FORMAT_BGRA1010102,
+	DRM_FORMAT_INVALID,
+};
+
+static const uint32_t fmts8bit[] = {
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_XBGR8888,
+	DRM_FORMAT_RGBX8888,
+	DRM_FORMAT_BGRX8888,
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_ABGR8888,
+	DRM_FORMAT_RGBA8888,
+	DRM_FORMAT_BGRA8888,
+	DRM_FORMAT_INVALID,
+};
+
 static bool search_render_format(struct search_context *ctx, size_t output_idx) {
 	struct matched_output_config *cfg = &ctx->configs[output_idx];
 	struct wlr_backend_output_state *backend_state = &ctx->states[output_idx];
 	struct wlr_output_state *state = &backend_state->base;
 	struct wlr_output *wlr_output = backend_state->output;
 
-	uint32_t fmts[] = {
-		DRM_FORMAT_XRGB2101010,
-		DRM_FORMAT_XBGR2101010,
-		DRM_FORMAT_XRGB8888,
-		DRM_FORMAT_ARGB8888,
-		DRM_FORMAT_INVALID,
-	};
-	if (render_format_is_bgr(wlr_output->render_format)) {
-		// Start with BGR in the unlikely event that we previously required it.
-		fmts[0] = DRM_FORMAT_XBGR2101010;
-		fmts[1] = DRM_FORMAT_XRGB2101010;
-	}
-
 	const struct wlr_drm_format_set *primary_formats =
 		wlr_output_get_primary_formats(wlr_output, WLR_BUFFER_CAP_DMABUF);
-	bool need_10bit = cfg->config && cfg->config->render_bit_depth == RENDER_BIT_DEPTH_10;
-	for (size_t idx = 0; fmts[idx] != DRM_FORMAT_INVALID; idx++) {
-		if (!need_10bit && render_format_is_10bit(fmts[idx])) {
-			continue;
+
+	// Must be big enough to hold all format candidates
+	uint32_t fmts[8] = { 0 };
+	size_t fmtidx = 0;
+
+	if (cfg->config && cfg->config->render_bit_depth == RENDER_BIT_DEPTH_10) {
+		// Pick 10-bit format to try
+		for (size_t idx = 0; fmts10bit[idx] != DRM_FORMAT_INVALID; idx++) {
+			if (wlr_drm_format_set_get(primary_formats, fmts10bit[idx])) {
+				fmts[fmtidx++] = fmts10bit[idx];
+				break;
+			}
 		}
-		if (!wlr_drm_format_set_get(primary_formats, fmts[idx])) {
-			// This is not a supported format for this output
-			continue;
+	}
+
+	// Pick 8-bit format to try
+	for (size_t idx = 0; fmts8bit[idx] != DRM_FORMAT_INVALID; idx++) {
+		if (wlr_drm_format_set_get(primary_formats, fmts8bit[idx])) {
+			fmts[fmtidx++] = fmts8bit[idx];
+			break;
 		}
+	}
+
+	assert(fmtidx < sizeof(fmts));
+
+	for (size_t idx = 0; fmts[idx] != DRM_FORMAT_INVALID; idx++) {
 		wlr_output_state_set_render_format(state, fmts[idx]);
 		if (search_mode(ctx, output_idx)) {
 			return true;