fix: registry mirror fallback handling

Fixes #9613

This has two changes:

* adjust Talos registry resolver to match containerd (CRI) resolver: use
  by default upstream as a fallback
* add a machine config option to skip upstream as a fallback, and adjust
  CRI configuration accordingly

See https://github.com/containerd/containerd/blob/main/docs/hosts.md#registry-configuration---examples
for details on CRI's `hosts.toml`.

Signed-off-by: Andrey Smirnov <andrey.smirnov@siderolabs.com>

hack/release.toml

@@ -69,6 +69,18 @@ This command allows you to view the cgroup resource consumption and limits for a
         title = "udevd"
         description = """\
 Talos previously used `eudev` to provide `udevd`, now it uses `systemd-udevd` instead.
+"""
+
+    [notes.registry-mirrors]
+        title = "Registry Mirrors"
+        description = """\
+In versions before Talos 1.9, there was a discrepancy between the way Talos itself and CRI plugin resolves registry mirrors:
+Talos will never fall back to the default registry if endpoints are configured, while CRI plugin will.
+
+> Note: Talos Linux pulls images for the `installer`, `kubelet`, `etcd`, while all workload images are pulled by the CRI plugin.
+
+In Talos 1.9 this was fixed, so that by default an upstream registry is used as a fallback in all cases, while new registry mirror
+configuration option `.skipFallback` can be used to disable this behavior both for Talos and CRI plugin.
 """
 
 [make_deps]

internal/pkg/containers/cri/containerd/hosts.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/containerd/containerd/v2/core/remotes/docker"
 	"github.com/pelletier/go-toml/v2"
+	"github.com/siderolabs/gen/optional"
 
 	"github.com/siderolabs/talos/pkg/machinery/config/config"
 )
@@ -42,7 +43,7 @@ type HostsFile struct {
 
 // GenerateHosts generates a structure describing contents of the containerd hosts configuration.
 //
-//nolint:gocyclo,cyclop
+//nolint:gocyclo
 func GenerateHosts(cfg config.Registries, basePath string) (*HostsConfig, error) {
 	config := &HostsConfig{
 		Directories: map[string]*HostsDirectory{},
@@ -106,65 +107,41 @@ func GenerateHosts(cfg config.Registries, basePath string) (*HostsConfig, error)
 
 		directory := &HostsDirectory{}
 
-		// toml marshaling doesn't guarantee proper order of map keys, so instead we should marshal
+		var hostsConfig HostsConfiguration
-		// each time and append to the output
-
-		var buf bytes.Buffer
-
-		for i, endpoint := range endpoints.Endpoints() {
-			hostsToml := HostsToml{
-				HostConfigs: map[string]*HostToml{},
-			}
 
+		for _, endpoint := range endpoints.Endpoints() {
 			u, err := url.Parse(endpoint)
 			if err != nil {
 				return nil, fmt.Errorf("error parsing endpoint %q for host %q: %w", endpoint, registryName, err)
 			}
 
-			hostsToml.HostConfigs[endpoint] = &HostToml{
+			hostEntry := HostEntry{
-				Capabilities: []string{"pull", "resolve"}, // TODO: we should make it configurable eventually
+				Host: endpoint,
-				OverridePath: endpoints.OverridePath(),
+				HostToml: HostToml{
+					Capabilities: []string{"pull", "resolve"}, // TODO: we should make it configurable eventually
+					OverridePath: endpoints.OverridePath(),
+				},
 			}
 
-			configureEndpoint(u.Host, directoryName, hostsToml.HostConfigs[endpoint], directory)
+			configureEndpoint(u.Host, directoryName, &hostEntry.HostToml, directory)
 
-			var tomlBuf bytes.Buffer
+			hostsConfig.HostEntries = append(hostsConfig.HostEntries, hostEntry)
+		}
 
-			if err := toml.NewEncoder(&tomlBuf).SetIndentTables(true).Encode(hostsToml); err != nil {
+		if endpoints.SkipFallback() {
-				return nil, err
+			hostsConfig.DisableFallback()
-			}
+		}
 
-			tomlBytes := tomlBuf.Bytes()
+		cfgOut, err := hostsConfig.RenderTOML()
-
+		if err != nil {
-			// this is an ugly hack, and neither TOML format nor go-toml library make it easier
+			return nil, err
-			//
-			// we need to marshal each endpoint in the order they are specified in the config, but go-toml defines
-			// the tree as map[string]interface{} and doesn't guarantee the order of keys
-			//
-			// so we marshal each entry separately and combine the output, which results in something like:
-			//
-			//   [host]
-			//     [host."foo.bar"]
-			//	 [host]
-			//     [host."bar.foo"]
-			//
-			// but this is invalid TOML, as `[host]' is repeated, so we do an ugly hack and remove it below
-			const hostPrefix = "[host]\n"
-
-			if i > 0 {
-				if bytes.HasPrefix(tomlBytes, []byte(hostPrefix)) {
-					tomlBytes = tomlBytes[len(hostPrefix):]
-				}
-			}
-
-			buf.Write(tomlBytes)
 		}
 
 		directory.Files = append(directory.Files,
 			&HostsFile{
 				Name:     "hosts.toml",
 				Mode:     0o600,
-				Contents: buf.Bytes(),
+				Contents: cfgOut,
 			},
 		)
 
@@ -199,17 +176,18 @@ func GenerateHosts(cfg config.Registries, basePath string) (*HostsConfig, error)
 
 		defaultHost = "https://" + defaultHost
 
-		hostsToml := HostsToml{
+		rootEntry := HostEntry{
-			HostConfigs: map[string]*HostToml{
+			Host: defaultHost,
-				defaultHost: {},
-			},
 		}
 
-		configureEndpoint(hostname, directoryName, hostsToml.HostConfigs[defaultHost], directory)
+		configureEndpoint(hostname, directoryName, &rootEntry.HostToml, directory)
 
-		var tomlBuf bytes.Buffer
+		hostsToml := HostsConfiguration{
+			RootEntry: optional.Some(rootEntry),
+		}
 
-		if err = toml.NewEncoder(&tomlBuf).SetIndentTables(true).Encode(hostsToml); err != nil {
+		cfgOut, err := hostsToml.RenderTOML()
+		if err != nil {
 			return nil, err
 		}
 
@@ -217,7 +195,7 @@ func GenerateHosts(cfg config.Registries, basePath string) (*HostsConfig, error)
 			&HostsFile{
 				Name:     "hosts.toml",
 				Mode:     0o600,
-				Contents: tomlBuf.Bytes(),
+				Contents: cfgOut,
 			},
 		)
 
@@ -241,10 +219,106 @@ func hostDirectory(host string) string {
 	return host
 }
 
-// HostsToml describes the contents of the `hosts.toml` file.
+// HostEntry describes the configuration for a single host.
-type HostsToml struct {
+type HostEntry struct {
-	Server      string               `toml:"server,omitempty"`
+	Host string
-	HostConfigs map[string]*HostToml `toml:"host"`
+	HostToml
+}
+
+// HostsConfiguration describes the configuration of `hosts.toml` file in the format not compatible with TOML.
+//
+// The hosts entries should come in order, and go-toml only supports map[string]any, so we need to do some tricks.
+type HostsConfiguration struct {
+	RootEntry optional.Optional[HostEntry] // might be missing
+
+	HostEntries []HostEntry
+}
+
+// DisableFallback disables the fallback to the default host.
+func (hc *HostsConfiguration) DisableFallback() {
+	if len(hc.HostEntries) == 0 {
+		return
+	}
+
+	// push the last entry as the root entry
+	hc.RootEntry = optional.Some(hc.HostEntries[len(hc.HostEntries)-1])
+
+	hc.HostEntries = hc.HostEntries[:len(hc.HostEntries)-1]
+}
+
+// RenderTOML renders the configuration to TOML format.
+func (hc *HostsConfiguration) RenderTOML() ([]byte, error) {
+	var out bytes.Buffer
+
+	// toml marshaling doesn't guarantee proper order of map keys, so instead we should marshal
+	// each time and append to the output
+
+	if rootEntry, ok := hc.RootEntry.Get(); ok {
+		server := HostsTomlServer{
+			Server:   rootEntry.Host,
+			HostToml: rootEntry.HostToml,
+		}
+
+		if err := toml.NewEncoder(&out).SetIndentTables(true).Encode(server); err != nil {
+			return nil, err
+		}
+	}
+
+	for i, entry := range hc.HostEntries {
+		hostEntry := HostsTomlHost{
+			HostConfigs: map[string]HostToml{
+				entry.Host: entry.HostToml,
+			},
+		}
+
+		var tomlBuf bytes.Buffer
+
+		if err := toml.NewEncoder(&tomlBuf).SetIndentTables(true).Encode(hostEntry); err != nil {
+			return nil, err
+		}
+
+		tomlBytes := tomlBuf.Bytes()
+
+		// this is an ugly hack, and neither TOML format nor go-toml library make it easier
+		//
+		// we need to marshal each endpoint in the order they are specified in the config, but go-toml defines
+		// the tree as map[string]interface{} and doesn't guarantee the order of keys
+		//
+		// so we marshal each entry separately and combine the output, which results in something like:
+		//
+		//   [host]
+		//     [host."foo.bar"]
+		//	 [host]
+		//     [host."bar.foo"]
+		//
+		// but this is invalid TOML, as `[host]' is repeated, so we do an ugly hack and remove it below
+		const hostPrefix = "[host]\n"
+
+		if i > 0 {
+			if bytes.HasPrefix(tomlBytes, []byte(hostPrefix)) {
+				tomlBytes = tomlBytes[len(hostPrefix):]
+			}
+		}
+
+		out.Write(tomlBytes)
+	}
+
+	return out.Bytes(), nil
+}
+
+// HostsTomlServer describes only 'server' part of the `hosts.toml` file.
+type HostsTomlServer struct {
+	// top-level entry is used as the last one in the fallback chain.
+	Server   string `toml:"server,omitempty"`
+	HostToml        // embedded, matches the server
+}
+
+// HostsTomlHost describes the `hosts.toml` file entry for hosts.
+//
+// It is supposed to be marshaled as a single-entry map to keep the order correct.
+type HostsTomlHost struct {
+	// Note: this doesn't match the TOML format, but allows use to keep endpoints ordered properly.
+	HostConfigs map[string]HostToml `toml:"host"`
 }
 
 // HostToml is a single entry in `hosts.toml`.

internal/pkg/containers/cri/containerd/hosts_test.go

@@ -83,7 +83,7 @@ func TestGenerateHostsWithTLS(t *testing.T) {
 					{
 						Name:     "hosts.toml",
 						Mode:     0o600,
-						Contents: []byte("[host]\n  [host.'https://some.host:123']\n    ca = '/etc/cri/conf.d/hosts/some.host_123_/some.host:123-ca.crt'\n    client = [['/etc/cri/conf.d/hosts/some.host_123_/some.host:123-client.crt', '/etc/cri/conf.d/hosts/some.host_123_/some.host:123-client.key']]\n    skip_verify = true\n"), //nolint:lll
+						Contents: []byte("server = 'https://some.host:123'\nca = '/etc/cri/conf.d/hosts/some.host_123_/some.host:123-ca.crt'\nclient = [['/etc/cri/conf.d/hosts/some.host_123_/some.host:123-client.crt', '/etc/cri/conf.d/hosts/some.host_123_/some.host:123-client.key']]\nskip_verify = true\n"), //nolint:lll
 					},
 				},
 			},
@@ -92,7 +92,7 @@ func TestGenerateHostsWithTLS(t *testing.T) {
 					{
 						Name:     "hosts.toml",
 						Mode:     0o600,
-						Contents: []byte("[host]\n  [host.'https://registry-2.docker.io']\n    skip_verify = true\n"),
+						Contents: []byte("server = 'https://registry-2.docker.io'\nskip_verify = true\n"),
 					},
 				},
 			},
@@ -210,7 +210,7 @@ func TestGenerateHostsTLSWildcard(t *testing.T) {
 					{
 						Name:     "hosts.toml",
 						Mode:     0o600,
-						Contents: []byte("[host]\n  [host.'https://my-registry1']\n    ca = '/etc/cri/conf.d/hosts/my-registry1/my-registry1-ca.crt'\n"),
+						Contents: []byte("server = 'https://my-registry1'\nca = '/etc/cri/conf.d/hosts/my-registry1/my-registry1-ca.crt'\n"),
 					},
 				},
 			},
@@ -278,7 +278,58 @@ func TestGenerateHostsWithHarbor(t *testing.T) {
 					{
 						Name:     "hosts.toml",
 						Mode:     0o600,
-						Contents: []byte("[host]\n  [host.'https://harbor']\n    skip_verify = true\n"),
+						Contents: []byte("server = 'https://harbor'\nskip_verify = true\n"),
+					},
+				},
+			},
+		},
+	}, result)
+}
+
+func TestGenerateHostsSkipFallback(t *testing.T) {
+	cfg := &mockConfig{
+		mirrors: map[string]*v1alpha1.RegistryMirrorConfig{
+			"docker.io": {
+				MirrorEndpoints:    []string{"https://harbor/v2/mirrors/proxy.docker.io", "http://127.0.0.1:5001/v2/"},
+				MirrorOverridePath: pointer.To(true),
+				MirrorSkipFallback: pointer.To(true),
+			},
+			"ghcr.io": {
+				MirrorEndpoints:    []string{"http://127.0.0.1:5002"},
+				MirrorSkipFallback: pointer.To(true),
+			},
+		},
+	}
+
+	result, err := containerd.GenerateHosts(cfg, "/etc/cri/conf.d/hosts")
+	require.NoError(t, err)
+
+	t.Logf(
+		"config docker.io %q",
+		string(result.Directories["docker.io"].Files[0].Contents),
+	)
+	t.Logf(
+		"config ghcr.io %q",
+		string(result.Directories["ghcr.io"].Files[0].Contents),
+	)
+
+	assert.Equal(t, &containerd.HostsConfig{
+		Directories: map[string]*containerd.HostsDirectory{
+			"docker.io": {
+				Files: []*containerd.HostsFile{
+					{
+						Name:     "hosts.toml",
+						Mode:     0o600,
+						Contents: []byte("server = 'http://127.0.0.1:5001/v2/'\ncapabilities = ['pull', 'resolve']\noverride_path = true\n[host]\n  [host.'https://harbor/v2/mirrors/proxy.docker.io']\n    capabilities = ['pull', 'resolve']\n    override_path = true\n"), //nolint:lll
+					},
+				},
+			},
+			"ghcr.io": {
+				Files: []*containerd.HostsFile{
+					{
+						Name:     "hosts.toml",
+						Mode:     0o600,
+						Contents: []byte("server = 'http://127.0.0.1:5002'\ncapabilities = ['pull', 'resolve']\n"),
 					},
 				},
 			},

internal/pkg/containers/image/resolver.go

@@ -15,6 +15,7 @@ import (
 	"github.com/containerd/containerd/v2/core/remotes"
 	"github.com/containerd/containerd/v2/core/remotes/docker"
 	"github.com/hashicorp/go-cleanhttp"
+	"github.com/siderolabs/gen/xslices"
 
 	"github.com/siderolabs/talos/pkg/httpdefaults"
 	"github.com/siderolabs/talos/pkg/machinery/config/config"
@@ -34,15 +35,15 @@ func RegistryHosts(reg config.Registries) docker.RegistryHosts {
 	return func(host string) ([]docker.RegistryHost, error) {
 		var registries []docker.RegistryHost
 
-		endpoints, overridePath, err := RegistryEndpoints(reg, host)
+		endpoints, err := RegistryEndpoints(reg, host)
 		if err != nil {
 			return nil, err
 		}
 
 		for _, endpoint := range endpoints {
-			u, err := url.Parse(endpoint)
+			u, err := url.Parse(endpoint.Endpoint)
 			if err != nil {
-				return nil, fmt.Errorf("error parsing endpoint %q for host %q: %w", endpoint, host, err)
+				return nil, fmt.Errorf("error parsing endpoint %q for host %q: %w", endpoint.Endpoint, host, err)
 			}
 
 			transport := newTransport()
@@ -62,13 +63,13 @@ func RegistryHosts(reg config.Registries) docker.RegistryHosts {
 			}
 
 			if u.Path == "" {
-				if !overridePath {
+				if !endpoint.OverridePath {
 					u.Path = "/v2"
 				}
 			} else {
 				u.Path = path.Clean(u.Path)
 
-				if !strings.HasSuffix(u.Path, "/v2") && !overridePath {
+				if !strings.HasSuffix(u.Path, "/v2") && !endpoint.OverridePath {
 					u.Path += "/v2"
 				}
 			}
@@ -97,25 +98,56 @@ func RegistryHosts(reg config.Registries) docker.RegistryHosts {
 	}
 }
 
+// EndpointEntry represents a registry endpoint.
+type EndpointEntry struct {
+	Endpoint     string
+	OverridePath bool
+}
+
+// RegistryEndpointEntriesFromConfig returns registry endpoints per host.
+func RegistryEndpointEntriesFromConfig(host string, reg config.RegistryMirrorConfig) ([]EndpointEntry, error) {
+	entries := xslices.Map(reg.Endpoints(), func(endpoint string) EndpointEntry {
+		return EndpointEntry{Endpoint: endpoint, OverridePath: reg.OverridePath()}
+	})
+
+	if reg.SkipFallback() {
+		return entries, nil
+	}
+
+	defaultHost, err := docker.DefaultHost(host)
+	if err != nil {
+		return nil, fmt.Errorf("error getting default host for %q: %w", host, err)
+	}
+
+	entries = append(entries, EndpointEntry{Endpoint: "https://" + defaultHost, OverridePath: false})
+
+	return entries, nil
+}
+
 // RegistryEndpoints returns registry endpoints per host using reg.
-func RegistryEndpoints(reg config.Registries, host string) (endpoints []string, overridePath bool, err error) {
+func RegistryEndpoints(reg config.Registries, host string) (endpoints []EndpointEntry, err error) {
 	// direct hit by host
 	if hostConfig, ok := reg.Mirrors()[host]; ok {
-		return hostConfig.Endpoints(), hostConfig.OverridePath(), nil
+		return RegistryEndpointEntriesFromConfig(host, hostConfig)
 	}
 
 	// '*'
 	if catchAllConfig, ok := reg.Mirrors()["*"]; ok {
-		return catchAllConfig.Endpoints(), catchAllConfig.OverridePath(), nil
+		return RegistryEndpointEntriesFromConfig(host, catchAllConfig)
 	}
 
 	// still no endpoints, use default
 	defaultHost, err := docker.DefaultHost(host)
 	if err != nil {
-		return nil, false, fmt.Errorf("error getting default host for %q: %w", host, err)
+		return nil, fmt.Errorf("error getting default host for %q: %w", host, err)
 	}
 
-	return []string{"https://" + defaultHost}, false, nil
+	return []EndpointEntry{
+		{
+			Endpoint:     "https://" + defaultHost,
+			OverridePath: false,
+		},
+	}, nil
 }
 
 // PrepareAuth returns authentication info in the format expected by containerd.

internal/pkg/containers/image/resolver_test.go

@@ -55,8 +55,7 @@ func (suite *ResolverSuite) TestRegistryEndpoints() {
 	type request struct {
 		host string
 
-		expectedEndpoints    []string
+		expectedEndpoints []image.EndpointEntry
-		expectedOverridePath bool
 	}
 
 	for _, tt := range []struct {
@@ -70,20 +69,61 @@ func (suite *ResolverSuite) TestRegistryEndpoints() {
 			config: &mockConfig{},
 			requests: []request{
 				{
-					host:              "docker.io",
+					host: "docker.io",
-					expectedEndpoints: []string{"https://registry-1.docker.io"},
+					expectedEndpoints: []image.EndpointEntry{
+						{
+							Endpoint: "https://registry-1.docker.io",
+						},
+					},
 				},
 				{
-					host:              "quay.io",
+					host: "quay.io",
-					expectedEndpoints: []string{"https://quay.io"},
+					expectedEndpoints: []image.EndpointEntry{
+						{
+							Endpoint: "https://quay.io",
+						},
+					},
 				},
 			},
 		},
 		{
-			name: "config with mirror",
+			name: "config with mirror and no fallback",
 			config: &mockConfig{
 				mirrors: map[string]*v1alpha1.RegistryMirrorConfig{
 					"docker.io": {
+						MirrorEndpoints:    []string{"http://127.0.0.1:5000", "https://some.host"},
+						MirrorSkipFallback: pointer.To(true),
+					},
+				},
+			},
+
+			requests: []request{
+				{
+					host: "docker.io",
+					expectedEndpoints: []image.EndpointEntry{
+						{
+							Endpoint: "http://127.0.0.1:5000",
+						},
+						{
+							Endpoint: "https://some.host",
+						},
+					},
+				},
+				{
+					host: "quay.io",
+					expectedEndpoints: []image.EndpointEntry{
+						{
+							Endpoint: "https://quay.io",
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "config with mirror and fallback",
+			config: &mockConfig{
+				mirrors: map[string]*v1alpha1.RegistryMirrorConfig{
+					"ghcr.io": {
 						MirrorEndpoints: []string{"http://127.0.0.1:5000", "https://some.host"},
 					},
 				},
@@ -91,22 +131,70 @@ func (suite *ResolverSuite) TestRegistryEndpoints() {
 
 			requests: []request{
 				{
-					host:              "docker.io",
+					host: "ghcr.io",
-					expectedEndpoints: []string{"http://127.0.0.1:5000", "https://some.host"},
+					expectedEndpoints: []image.EndpointEntry{
+						{
+							Endpoint: "http://127.0.0.1:5000",
+						},
+						{
+							Endpoint: "https://some.host",
+						},
+						{
+							Endpoint: "https://ghcr.io",
+						},
+					},
 				},
 				{
-					host:              "quay.io",
+					host: "docker.io",
-					expectedEndpoints: []string{"https://quay.io"},
+					expectedEndpoints: []image.EndpointEntry{
+						{
+							Endpoint: "https://registry-1.docker.io",
+						},
+					},
 				},
 			},
 		},
 		{
-			name: "config with catch-all",
+			name: "config with catch-all and no fallback",
 			config: &mockConfig{
 				mirrors: map[string]*v1alpha1.RegistryMirrorConfig{
 					"docker.io": {
-						MirrorEndpoints: []string{"http://127.0.0.1:5000", "https://some.host"},
+						MirrorEndpoints:    []string{"http://127.0.0.1:5000", "https://some.host"},
+						MirrorSkipFallback: pointer.To(true),
 					},
+					"*": {
+						MirrorEndpoints:    []string{"http://127.0.0.1:5001"},
+						MirrorSkipFallback: pointer.To(true),
+					},
+				},
+			},
+
+			requests: []request{
+				{
+					host: "docker.io",
+					expectedEndpoints: []image.EndpointEntry{
+						{
+							Endpoint: "http://127.0.0.1:5000",
+						},
+						{
+							Endpoint: "https://some.host",
+						},
+					},
+				},
+				{
+					host: "quay.io",
+					expectedEndpoints: []image.EndpointEntry{
+						{
+							Endpoint: "http://127.0.0.1:5001",
+						},
+					},
+				},
+			},
+		},
+		{
+			name: "config with catch-all and fallback",
+			config: &mockConfig{
+				mirrors: map[string]*v1alpha1.RegistryMirrorConfig{
 					"*": {
 						MirrorEndpoints: []string{"http://127.0.0.1:5001"},
 					},
@@ -115,12 +203,26 @@ func (suite *ResolverSuite) TestRegistryEndpoints() {
 
 			requests: []request{
 				{
-					host:              "docker.io",
+					host: "docker.io",
-					expectedEndpoints: []string{"http://127.0.0.1:5000", "https://some.host"},
+					expectedEndpoints: []image.EndpointEntry{
+						{
+							Endpoint: "http://127.0.0.1:5001",
+						},
+						{
+							Endpoint: "https://registry-1.docker.io",
+						},
+					},
 				},
 				{
-					host:              "quay.io",
+					host: "quay.io",
-					expectedEndpoints: []string{"http://127.0.0.1:5001"},
+					expectedEndpoints: []image.EndpointEntry{
+						{
+							Endpoint: "http://127.0.0.1:5001",
+						},
+						{
+							Endpoint: "https://quay.io",
+						},
+					},
 				},
 			},
 		},
@@ -131,6 +233,7 @@ func (suite *ResolverSuite) TestRegistryEndpoints() {
 					"docker.io": {
 						MirrorEndpoints:    []string{"https://harbor/v2/registry.docker.io"},
 						MirrorOverridePath: pointer.To(true),
+						MirrorSkipFallback: pointer.To(true),
 					},
 					"ghcr.io": {
 						MirrorEndpoints:    []string{"https://harbor/v2/registry.ghcr.io"},
@@ -141,18 +244,33 @@ func (suite *ResolverSuite) TestRegistryEndpoints() {
 
 			requests: []request{
 				{
-					host:                 "docker.io",
+					host: "docker.io",
-					expectedEndpoints:    []string{"https://harbor/v2/registry.docker.io"},
+					expectedEndpoints: []image.EndpointEntry{
-					expectedOverridePath: true,
+						{
+							Endpoint:     "https://harbor/v2/registry.docker.io",
+							OverridePath: true,
+						},
+					},
 				},
 				{
-					host:                 "ghcr.io",
+					host: "ghcr.io",
-					expectedEndpoints:    []string{"https://harbor/v2/registry.ghcr.io"},
+					expectedEndpoints: []image.EndpointEntry{
-					expectedOverridePath: true,
+						{
+							Endpoint:     "https://harbor/v2/registry.ghcr.io",
+							OverridePath: true,
+						},
+						{
+							Endpoint: "https://ghcr.io",
+						},
+					},
 				},
 				{
-					host:              "quay.io",
+					host: "quay.io",
-					expectedEndpoints: []string{"https://quay.io"},
+					expectedEndpoints: []image.EndpointEntry{
+						{
+							Endpoint: "https://quay.io",
+						},
+					},
 				},
 			},
 		},
@@ -160,11 +278,10 @@ func (suite *ResolverSuite) TestRegistryEndpoints() {
 		suite.Run(tt.name, func() {
 			for _, req := range tt.requests {
 				suite.Run(req.host, func() {
-					endpoints, overridePath, err := image.RegistryEndpoints(tt.config, req.host)
+					endpoints, err := image.RegistryEndpoints(tt.config, req.host)
 
 					suite.Assert().NoError(err)
 					suite.Assert().Equal(req.expectedEndpoints, endpoints)
-					suite.Assert().Equal(req.expectedOverridePath, overridePath)
 				})
 			}
 		})
@@ -223,11 +340,13 @@ func (suite *ResolverSuite) TestRegistryHosts() {
 	cfg := &mockConfig{
 		mirrors: map[string]*v1alpha1.RegistryMirrorConfig{
 			"docker.io": {
-				MirrorEndpoints: []string{"http://127.0.0.1:5000/docker.io", "https://some.host"},
+				MirrorEndpoints:    []string{"http://127.0.0.1:5000/docker.io", "https://some.host"},
+				MirrorSkipFallback: pointer.To(true),
 			},
 			"ghcr.io": {
 				MirrorEndpoints:    []string{"https://harbor/v2/registry.ghcr.io"},
 				MirrorOverridePath: pointer.To(true),
+				MirrorSkipFallback: pointer.To(true),
 			},
 		},
 	}
@@ -254,7 +373,8 @@ func (suite *ResolverSuite) TestRegistryHosts() {
 	cfg = &mockConfig{
 		mirrors: map[string]*v1alpha1.RegistryMirrorConfig{
 			"docker.io": {
-				MirrorEndpoints: []string{"https://some.host:123"},
+				MirrorEndpoints:    []string{"https://some.host:123"},
+				MirrorSkipFallback: pointer.To(true),
 			},
 		},
 		config: map[string]*v1alpha1.RegistryConfig{

pkg/machinery/config/config/machine.go

@@ -366,6 +366,7 @@ type Registries interface {
 type RegistryMirrorConfig interface {
 	Endpoints() []string
 	OverridePath() bool
+	SkipFallback() bool
 }
 
 // RegistryConfig specifies auth & TLS config per registry.

pkg/machinery/config/schemas/config.schema.json

@@ -3239,6 +3239,13 @@
           "description": "Use the exact path specified for the endpoint (don’t append /v2/).\nThis setting is often required for setting up multiple mirrors\non a single instance of a registry.\n",
           "markdownDescription": "Use the exact path specified for the endpoint (don't append /v2/).\nThis setting is often required for setting up multiple mirrors\non a single instance of a registry.",
           "x-intellij-html-description": "\u003cp\u003eUse the exact path specified for the endpoint (don\u0026rsquo;t append /v2/).\nThis setting is often required for setting up multiple mirrors\non a single instance of a registry.\u003c/p\u003e\n"
+        },
+        "skipFallback": {
+          "type": "boolean",
+          "title": "skipFallback",
+          "description": "Skip fallback to the upstream endpoint, for example the mirror configuration\nfor docker.io will not fallback to registry-1.docker.io.\n",
+          "markdownDescription": "Skip fallback to the upstream endpoint, for example the mirror configuration\nfor `docker.io` will not fallback to `registry-1.docker.io`.",
+          "x-intellij-html-description": "\u003cp\u003eSkip fallback to the upstream endpoint, for example the mirror configuration\nfor \u003ccode\u003edocker.io\u003c/code\u003e will not fallback to \u003ccode\u003eregistry-1.docker.io\u003c/code\u003e.\u003c/p\u003e\n"
         }
       },
       "additionalProperties": false,

pkg/machinery/config/types/v1alpha1/v1alpha1_provider.go

@@ -1464,6 +1464,11 @@ func (r *RegistryMirrorConfig) OverridePath() bool {
 	return pointer.SafeDeref(r.MirrorOverridePath)
 }
 
+// SkipFallback implements the Registries interface.
+func (r *RegistryMirrorConfig) SkipFallback() bool {
+	return pointer.SafeDeref(r.MirrorSkipFallback)
+}
+
 // Content implements the config.Provider interface.
 func (f *MachineFile) Content() string {
 	return f.FileContent

pkg/machinery/config/types/v1alpha1/v1alpha1_types.go

@@ -2057,6 +2057,10 @@ type RegistryMirrorConfig struct {
 	//     This setting is often required for setting up multiple mirrors
 	//     on a single instance of a registry.
 	MirrorOverridePath *bool `yaml:"overridePath,omitempty"`
+	//   description: |
+	//     Skip fallback to the upstream endpoint, for example the mirror configuration
+	//     for `docker.io` will not fallback to `registry-1.docker.io`.
+	MirrorSkipFallback *bool `yaml:"skipFallback,omitempty"`
 }
 
 // RegistryConfig specifies auth & TLS config per registry.

pkg/machinery/config/types/v1alpha1/v1alpha1_types_doc.go

@@ -3224,6 +3224,13 @@ func (RegistryMirrorConfig) Doc() *encoder.Doc {
 				Description: "Use the exact path specified for the endpoint (don't append /v2/).\nThis setting is often required for setting up multiple mirrors\non a single instance of a registry.",
 				Comments:    [3]string{"" /* encoder.HeadComment */, "Use the exact path specified for the endpoint (don't append /v2/)." /* encoder.LineComment */, "" /* encoder.FootComment */},
 			},
+			{
+				Name:        "skipFallback",
+				Type:        "bool",
+				Note:        "",
+				Description: "Skip fallback to the upstream endpoint, for example the mirror configuration\nfor `docker.io` will not fallback to `registry-1.docker.io`.",
+				Comments:    [3]string{"" /* encoder.HeadComment */, "Skip fallback to the upstream endpoint, for example the mirror configuration" /* encoder.LineComment */, "" /* encoder.FootComment */},
+			},
 		},
 	}
 

website/content/v1.9/reference/configuration/v1alpha1/config.md

@@ -2152,6 +2152,7 @@ machine:
 |-------|------|-------------|----------|
 |`endpoints` |[]string |<details><summary>List of endpoints (URLs) for registry mirrors to use.</summary>Endpoint configures HTTP/HTTPS access mode, host name,<br />port and path (if path is not set, it defaults to `/v2`).</details>  | |
 |`overridePath` |bool |<details><summary>Use the exact path specified for the endpoint (don't append /v2/).</summary>This setting is often required for setting up multiple mirrors<br />on a single instance of a registry.</details>  | |
+|`skipFallback` |bool |<details><summary>Skip fallback to the upstream endpoint, for example the mirror configuration</summary>for `docker.io` will not fallback to `registry-1.docker.io`.</details>  | |
 
 
 

website/content/v1.9/schemas/config.schema.json

@@ -3239,6 +3239,13 @@
           "description": "Use the exact path specified for the endpoint (don’t append /v2/).\nThis setting is often required for setting up multiple mirrors\non a single instance of a registry.\n",
           "markdownDescription": "Use the exact path specified for the endpoint (don't append /v2/).\nThis setting is often required for setting up multiple mirrors\non a single instance of a registry.",
           "x-intellij-html-description": "\u003cp\u003eUse the exact path specified for the endpoint (don\u0026rsquo;t append /v2/).\nThis setting is often required for setting up multiple mirrors\non a single instance of a registry.\u003c/p\u003e\n"
+        },
+        "skipFallback": {
+          "type": "boolean",
+          "title": "skipFallback",
+          "description": "Skip fallback to the upstream endpoint, for example the mirror configuration\nfor docker.io will not fallback to registry-1.docker.io.\n",
+          "markdownDescription": "Skip fallback to the upstream endpoint, for example the mirror configuration\nfor `docker.io` will not fallback to `registry-1.docker.io`.",
+          "x-intellij-html-description": "\u003cp\u003eSkip fallback to the upstream endpoint, for example the mirror configuration\nfor \u003ccode\u003edocker.io\u003c/code\u003e will not fallback to \u003ccode\u003eregistry-1.docker.io\u003c/code\u003e.\u003c/p\u003e\n"
         }
       },
       "additionalProperties": false,