Optimize image loading for Podman machines

Add support for loading images directly from machine paths to avoid
unnecessary file transfers when the image archive is already accessible
on the running machine through mounted directories.

Changes include:
- New /libpod/local/images/load API endpoint for direct machine loading
- Machine detection and path mapping functionality
- Fallback in tunnel mode to try optimized loading first

This optimization significantly speeds up image loading operations
when working with remote Podman machines by eliminating redundant
file transfers for already-accessible image archives.

Fixes: https://issues.redhat.com/browse/RUN-3249
Fixes: #26321

Signed-off-by: Jan Rodák <[email protected]>

Author: Honny1

cmd/podman/compose_machine.go

@@ -3,57 +3,20 @@
 package main
 
 import (
-	"fmt"
 	"net/url"
-	"strconv"
 
-	"github.com/containers/podman/v5/pkg/machine/define"
-	"github.com/containers/podman/v5/pkg/machine/env"
-	"github.com/containers/podman/v5/pkg/machine/provider"
-	"github.com/containers/podman/v5/pkg/machine/vmconfigs"
+	"github.com/containers/podman/v5/internal/local_utils"
 )
 
 func getMachineConn(connectionURI string, parsedConnection *url.URL) (string, error) {
-	machineProvider, err := provider.Get()
-	if err != nil {
-		return "", fmt.Errorf("getting machine provider: %w", err)
-	}
-	dirs, err := env.GetMachineDirs(machineProvider.VMType())
+	mc, machineProvider, err := local_utils.FindMachineByPort(connectionURI, parsedConnection)
 	if err != nil {
 		return "", err
 	}
 
-	machineList, err := vmconfigs.LoadMachinesInDir(dirs)
-	if err != nil {
-		return "", fmt.Errorf("listing machines: %w", err)
-	}
-
-	// Now we know that the connection points to a machine and we
-	// can find the machine by looking for the one with the
-	// matching port.
-	connectionPort, err := strconv.Atoi(parsedConnection.Port())
+	podmanSocket, podmanPipe, err := mc.ConnectionInfo(machineProvider.VMType())
 	if err != nil {
-		return "", fmt.Errorf("parsing connection port: %w", err)
-	}
-	for _, mc := range machineList {
-		if connectionPort != mc.SSH.Port {
-			continue
-		}
-
-		state, err := machineProvider.State(mc, false)
-		if err != nil {
-			return "", err
-		}
-
-		if state != define.Running {
-			return "", fmt.Errorf("machine %s is not running but in state %s", mc.Name, state)
-		}
-
-		podmanSocket, podmanPipe, err := mc.ConnectionInfo(machineProvider.VMType())
-		if err != nil {
-			return "", err
-		}
-		return extractConnectionString(podmanSocket, podmanPipe)
+		return "", err
 	}
-	return "", fmt.Errorf("could not find a matching machine for connection %q", connectionURI)
+	return extractConnectionString(podmanSocket, podmanPipe)
 }

internal/local_utils/local_types.go

@@ -0,0 +1,7 @@
+package local_utils
+
+// LocalAPIMap is a map of local paths to their target paths in the VM
+type LocalAPIMap struct {
+	ClientPath string `json:"ClientPath,omitempty"`
+	RemotePath string `json:"RemotePath,omitempty"`
+}

internal/local_utils/local_utils.go

@@ -0,0 +1,167 @@
+//go:build amd64 || arm64
+
+package local_utils
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io/fs"
+	"net/url"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/containers/podman/v5/pkg/bindings"
+	"github.com/containers/podman/v5/pkg/machine/define"
+	"github.com/containers/podman/v5/pkg/machine/env"
+	"github.com/containers/podman/v5/pkg/machine/provider"
+	"github.com/containers/podman/v5/pkg/machine/vmconfigs"
+	"github.com/containers/podman/v5/pkg/specgen"
+	"github.com/containers/storage/pkg/fileutils"
+	"github.com/sirupsen/logrus"
+)
+
+// FindMachineByPort finds a running machine that matches the given connection port.
+// It returns the machine configuration and provider, or an error if not found.
+func FindMachineByPort(connectionURI string, parsedConnection *url.URL) (*vmconfigs.MachineConfig, vmconfigs.VMProvider, error) {
+	machineProvider, err := provider.Get()
+	if err != nil {
+		return nil, nil, fmt.Errorf("getting machine provider: %w", err)
+	}
+
+	dirs, err := env.GetMachineDirs(machineProvider.VMType())
+	if err != nil {
+		return nil, nil, err
+	}
+
+	machineList, err := vmconfigs.LoadMachinesInDir(dirs)
+	if err != nil {
+		return nil, nil, fmt.Errorf("listing machines: %w", err)
+	}
+
+	// Now we know that the connection points to a machine and we
+	// can find the machine by looking for the one with the
+	// matching port.
+	connectionPort, err := strconv.Atoi(parsedConnection.Port())
+	if err != nil {
+		return nil, nil, fmt.Errorf("parsing connection port: %w", err)
+	}
+
+	for _, mc := range machineList {
+		if connectionPort != mc.SSH.Port {
+			continue
+		}
+
+		state, err := machineProvider.State(mc, false)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		if state != define.Running {
+			return nil, nil, fmt.Errorf("machine %s is not running but in state %s", mc.Name, state)
+		}
+
+		return mc, machineProvider, nil
+	}
+
+	return nil, nil, fmt.Errorf("could not find a matching machine for connection %q", connectionURI)
+}
+
+// getMachineMounts retrieves the mounts of a machine based on the connection URI and parsed URL.
+// It returns a slice of mounts or an error if the machine cannot be found or is not running.
+func getMachineMounts(connectionURI string, parsedConnection *url.URL, machineProvider vmconfigs.VMProvider) ([]*vmconfigs.Mount, error) {
+	mc, _, err := FindMachineByPort(connectionURI, parsedConnection)
+	if err != nil {
+		return nil, err
+	}
+	return mc.Mounts, nil
+}
+
+// isPathAvailableOnMachine checks if a local path is available on the machine through mounted directories.
+// If the path is available, it returns a LocalAPIMap with the corresponding remote path.
+func isPathAvailableOnMachine(mounts []*vmconfigs.Mount, vmType define.VMType, path string) (*LocalAPIMap, bool) {
+	pathABS, err := filepath.Abs(path)
+	if err != nil {
+		logrus.Debugf("Failed to get absolute path for %s: %v", path, err)
+		return nil, false
+	}
+
+	// WSLVirt is a special case where there is no real concept of doing a mount in WSL,
+	// WSL by default mounts the drives to /mnt/c, /mnt/d, etc...
+	if vmType == define.WSLVirt {
+		converted_path, err := specgen.ConvertWinMountPath(pathABS)
+		if err != nil {
+			logrus.Debugf("Failed to convert Windows mount path: %v", err)
+			return nil, false
+		}
+
+		return &LocalAPIMap{
+			ClientPath: pathABS,
+			RemotePath: converted_path,
+		}, true
+	}
+
+	for _, mount := range mounts {
+		mountSource := filepath.Clean(mount.Source)
+		if strings.HasPrefix(pathABS, mountSource) {
+			// Ensure we're matching directory boundaries, not just prefixes
+			// e.g., /home/user should not match /home/username
+			if len(pathABS) > len(mountSource) && pathABS[len(mountSource)] != filepath.Separator {
+				continue
+			}
+
+			relPath, err := filepath.Rel(mountSource, pathABS)
+			if err != nil {
+				logrus.Debugf("Failed to get relative path: %v", err)
+				continue
+			}
+			target := filepath.Join(mount.Target, relPath)
+
+			converted_path, err := specgen.ConvertWinMountPath(target)
+			if err != nil {
+				logrus.Debugf("Failed to convert Windows mount path: %v", err)
+				return nil, false
+			}
+			logrus.Debugf("Converted client path: %q", converted_path)
+			return &LocalAPIMap{
+				ClientPath: pathABS,
+				RemotePath: converted_path,
+			}, true
+		}
+	}
+	return nil, false
+}
+
+// CheckPathOnRunningMachine is a convenience function that checks if a path is available
+// on any currently running machine. It combines machine inspection and path checking.
+func CheckPathOnRunningMachine(ctx context.Context, path string) (*LocalAPIMap, bool) {
+	if err := fileutils.Exists(path); errors.Is(err, fs.ErrNotExist) {
+		logrus.Debugf("Path %s does not exist locally, skipping machine check", path)
+		return nil, false
+	}
+
+	if machineMode := bindings.GetMachineMode(ctx); !machineMode {
+		logrus.Debug("Machine mode is not enabled, skipping machine check")
+		return nil, false
+	}
+
+	conn, err := bindings.GetClient(ctx)
+	if err != nil {
+		logrus.Debugf("Failed to get client connection: %v", err)
+		return nil, false
+	}
+
+	machineProvider, err := provider.Get()
+	if err != nil {
+		return nil, false
+	}
+
+	mounts, err := getMachineMounts(conn.URI.String(), conn.URI, machineProvider)
+	if err != nil {
+		logrus.Debugf("Failed to get machine mounts: %v", err)
+		return nil, false
+	}
+
+	return isPathAvailableOnMachine(mounts, machineProvider.VMType(), path)
+}

internal/local_utils/local_utils_unsupported.go

@@ -0,0 +1,21 @@
+//go:build !amd64 && !arm64
+
+package local_utils
+
+import (
+	"context"
+	"net/url"
+
+	"github.com/containers/podman/v5/pkg/machine/vmconfigs"
+	"github.com/sirupsen/logrus"
+)
+
+func CheckPathOnRunningMachine(ctx context.Context, path string) (*LocalAPIMap, bool) {
+	logrus.Debug("CheckPathOnRunningMachine is not supported")
+	return nil, false
+}
+
+func FindMachineByPort(connectionURI string, parsedConnection *url.URL) (*vmconfigs.MachineConfig, vmconfigs.VMProvider, error) {
+	logrus.Debug("FindMachineByPort is not supported")
+	return nil, nil, nil
+}

pkg/api/handlers/libpod/images.go

@@ -8,8 +8,10 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"net/http"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 
@@ -36,6 +38,7 @@ import (
 	"github.com/containers/storage"
 	"github.com/containers/storage/pkg/archive"
 	"github.com/containers/storage/pkg/chrootarchive"
+	"github.com/containers/storage/pkg/fileutils"
 	"github.com/containers/storage/pkg/idtools"
 	"github.com/docker/docker/pkg/jsonmessage"
 	"github.com/gorilla/schema"
@@ -374,6 +377,44 @@ func ImagesLoad(w http.ResponseWriter, r *http.Request) {
 	utils.WriteResponse(w, http.StatusOK, loadReport)
 }
 
+func ImagesLocalLoad(w http.ResponseWriter, r *http.Request) {
+	runtime := r.Context().Value(api.RuntimeKey).(*libpod.Runtime)
+	decoder := r.Context().Value(api.DecoderKey).(*schema.Decoder)
+	query := struct {
+		Path string `schema:"path"`
+	}{}
+
+	if err := decoder.Decode(&query, r.URL.Query()); err != nil {
+		utils.Error(w, http.StatusBadRequest, fmt.Errorf("failed to parse parameters for %s: %w", r.URL.String(), err))
+		return
+	}
+
+	if query.Path == "" {
+		utils.Error(w, http.StatusBadRequest, fmt.Errorf("path query parameter is required"))
+		return
+	}
+
+	cleanPath := filepath.Clean(query.Path)
+	switch err := fileutils.Exists(cleanPath); {
+	case err == nil:
+	case errors.Is(err, fs.ErrNotExist):
+		utils.Error(w, http.StatusNotFound, fmt.Errorf("file does not exist: %q", cleanPath))
+		return
+	default:
+		utils.Error(w, http.StatusInternalServerError, fmt.Errorf("failed to access file: %w", err))
+		return
+	}
+
+	imageEngine := abi.ImageEngine{Libpod: runtime}
+	loadOptions := entities.ImageLoadOptions{Input: cleanPath}
+	loadReport, err := imageEngine.Load(r.Context(), loadOptions)
+	if err != nil {
+		utils.Error(w, http.StatusInternalServerError, fmt.Errorf("unable to load image: %w", err))
+		return
+	}
+	utils.WriteResponse(w, http.StatusOK, loadReport)
+}
+
 func ImagesImport(w http.ResponseWriter, r *http.Request) {
 	runtime := r.Context().Value(api.RuntimeKey).(*libpod.Runtime)
 	decoder := r.Context().Value(api.DecoderKey).(*schema.Decoder)

pkg/api/server/register_images.go

@@ -941,6 +941,30 @@ func (s *APIServer) registerImagesHandlers(r *mux.Router) error {
 	//   500:
 	//     $ref: '#/responses/internalError'
 	r.Handle(VersionedPath("/libpod/images/load"), s.APIHandler(libpod.ImagesLoad)).Methods(http.MethodPost)
+	// swagger:operation POST /libpod/local/images/load libpod LocalImagesLibpod
+	// ---
+	// tags:
+	//  - images
+	// summary: Load image from local path
+	// description: Load an image (oci-archive or docker-archive) from a file path accessible on the server.
+	// parameters:
+	//   - in: query
+	//     name: path
+	//     type: string
+	//     required: true
+	//     description: Path to the image archive file on the server filesystem
+	// produces:
+	// - application/json
+	// responses:
+	//   200:
+	//     $ref: "#/responses/imagesLoadResponseLibpod"
+	//   400:
+	//     $ref: "#/responses/badParamError"
+	//   404:
+	//     $ref: "#/responses/imageNotFound"
+	//   500:
+	//     $ref: '#/responses/internalError'
+	r.Handle(VersionedPath("/libpod/local/images/load"), s.APIHandler(libpod.ImagesLocalLoad)).Methods(http.MethodPost)
 	// swagger:operation POST /libpod/images/import libpod ImageImportLibpod
 	// ---
 	// tags:

pkg/bindings/connection.go

@@ -38,8 +38,9 @@ type Connection struct {
 type valueKey string
 
 const (
-	clientKey  = valueKey("Client")
-	versionKey = valueKey("ServiceVersion")
+	clientKey      = valueKey("Client")
+	versionKey     = valueKey("ServiceVersion")
+	machineModeKey = valueKey("MachineMode")
 )
 
 type ConnectError struct {
@@ -66,6 +67,13 @@ func GetClient(ctx context.Context) (*Connection, error) {
 	return nil, fmt.Errorf("%s not set in context", clientKey)
 }
 
+func GetMachineMode(ctx context.Context) bool {
+	if v, ok := ctx.Value(machineModeKey).(bool); ok {
+		return v
+	}
+	return false
+}
+
 // ServiceVersion from context build by NewConnection()
 func ServiceVersion(ctx context.Context) *semver.Version {
 	if v, ok := ctx.Value(versionKey).(*semver.Version); ok {
@@ -142,6 +150,8 @@ func NewConnectionWithIdentity(ctx context.Context, uri string, identity string,
 		return nil, newConnectError(err)
 	}
 	ctx = context.WithValue(ctx, versionKey, serviceVersion)
+
+	ctx = context.WithValue(ctx, machineModeKey, machine)
 	return ctx, nil
 }