Merge pull request #196 from stgraber/main

Add image customizer
This commit is contained in:
Stéphane Graber
2025-07-11 20:52:07 -04:00
committed by GitHub
18 changed files with 523 additions and 74 deletions

1
.gitignore vendored
View File

@@ -11,6 +11,7 @@ mkosi.images/base/mkosi.extra/boot/EFI/
mkosi.images/base/mkosi.extra/usr/local/bin/incus-osd
incus-osd/flasher-tool
incus-osd/image-customizer
incus-osd/image-publisher
incus-osd/incus-osd
mkosi.packages/initrd-tmpfs-root*

View File

@@ -0,0 +1,13 @@
package seed
// Applications represents the applications seed file.
type Applications struct {
Version string `json:"version" yaml:"version"`
Applications []Application `json:"applications" yaml:"applications"`
}
// Application represents a single application with the applications seed.
type Application struct {
Name string `json:"name" yaml:"name"`
}

View File

@@ -0,0 +1,2 @@
// Package seed contains the API files used for image seed files.
package seed

View File

@@ -0,0 +1,13 @@
package seed
import (
incusapi "github.com/lxc/incus/v6/shared/api"
)
// Incus represents the Incus seed file.
type Incus struct {
Version string `json:"version" yaml:"version"`
ApplyDefaults bool `json:"apply_defaults" yaml:"apply_defaults"`
Preseed *incusapi.InitPreseed `json:"preseed" yaml:"preseed"`
}

View File

@@ -0,0 +1,15 @@
package seed
// Install represents the install seed.
type Install struct {
Version string `json:"version" yaml:"version"`
ForceInstall bool `json:"force_install" yaml:"force_install"` // If true, ignore any existing data on target install disk.
ForceReboot bool `json:"force_reboot" yaml:"force_reboot"` // If true, reboot the system automatically upon completion rather than waiting for the install media to be removed.
Target *InstallTarget `json:"target" yaml:"target"` // Optional selector for the target install disk; if not set, expect a single drive to be present.
}
// InstallTarget defines options used to select the target install disk.
type InstallTarget struct {
ID string `json:"id" yaml:"id"` // Name as listed in /dev/disk/by-id/, glob supported.
}

View File

@@ -0,0 +1,12 @@
package seed
import (
"github.com/lxc/incus-os/incus-osd/api"
)
// Network represents the network seed.
type Network struct {
api.SystemNetworkConfig `yaml:",inline"`
Version string `json:"version" yaml:"version"`
}

View File

@@ -0,0 +1,12 @@
package seed
import (
"github.com/lxc/incus-os/incus-osd/api"
)
// Provider represents the provider seed.
type Provider struct {
api.SystemProviderConfig `yaml:",inline"`
Version string `json:"version" yaml:"version"`
}

View File

@@ -23,17 +23,18 @@ import (
"github.com/lxc/incus/v6/shared/revert"
"gopkg.in/yaml.v3"
apiseed "github.com/lxc/incus-os/incus-osd/api/seed"
"github.com/lxc/incus-os/incus-osd/internal/seed"
"github.com/lxc/incus-os/incus-osd/internal/systemd"
)
var applicationsSeed *seed.Applications
var applicationsSeed *apiseed.Applications
var incusSeed *seed.IncusConfig
var incusSeed *apiseed.Incus
var installSeed *seed.InstallSeed
var installSeed *apiseed.Install
var networkSeed *seed.NetworkSeed
var networkSeed *apiseed.Network
func main() {
var err error
@@ -130,7 +131,7 @@ func mainMenu(asker ask.Asker, imageFilename string) error {
menuOptions = append(menuOptions, "Configure network seed")
menuSelectionOptions = append(menuSelectionOptions, strconv.Itoa(len(menuOptions)))
if applicationsSeed != nil && slices.ContainsFunc(applicationsSeed.Applications, func(a seed.Application) bool {
if applicationsSeed != nil && slices.ContainsFunc(applicationsSeed.Applications, func(a apiseed.Application) bool {
return a.Name == "incus"
}) {
menuOptions = append(menuOptions, "Configure Incus seed")
@@ -225,14 +226,14 @@ func toggleInstallRunningMode(asker ask.Asker, imageFilename string) error {
return err
}
var target *seed.InstallSeedTarget
var target *apiseed.InstallTarget
if targetID != "" {
target = &seed.InstallSeedTarget{
target = &apiseed.InstallTarget{
ID: targetID,
}
}
installSeed = &seed.InstallSeed{
installSeed = &apiseed.Install{
ForceInstall: forceInstall,
ForceReboot: forceReboot,
Target: target,
@@ -247,10 +248,10 @@ func selectApplications(asker ask.Asker) error {
return err
}
applicationsSeed = &seed.Applications{}
applicationsSeed = &apiseed.Applications{}
if installIncus {
applicationsSeed.Applications = append(applicationsSeed.Applications, seed.Application{
applicationsSeed.Applications = append(applicationsSeed.Applications, apiseed.Application{
Name: "incus",
})
}
@@ -278,7 +279,7 @@ func configureNetworkSeed() error {
return nil
}
var newSeed seed.NetworkSeed
var newSeed apiseed.Network
err = yaml.Unmarshal(newContents, &newSeed)
if err != nil {
@@ -327,7 +328,7 @@ func configureIncusSeed() error {
return nil
}
var newSeed seed.IncusConfig
var newSeed apiseed.Incus
err = yaml.Unmarshal(newContents, &newSeed)
if err != nil {

View File

@@ -0,0 +1,416 @@
// Package main is used for the image customizer.
package main
import (
"archive/tar"
"compress/gzip"
"context"
"errors"
"fmt"
"io"
"net"
"net/http"
"os"
"path/filepath"
"slices"
"strings"
"sync"
"time"
"github.com/google/uuid"
"github.com/timpalpant/gzran"
"gopkg.in/yaml.v3"
apiseed "github.com/lxc/incus-os/incus-osd/api/seed"
"github.com/lxc/incus-os/incus-osd/internal/rest/response"
)
const (
imageTypeISO = "iso"
imageTypeRaw = "raw"
)
var (
images map[string]apiImagesPost
imagesMu sync.Mutex
)
type apiImagesPost struct {
Type string `json:"type" yaml:"type"`
Seeds apiImagesPostSeeds `json:"seeds" yaml:"seeds"`
}
type apiImagesPostSeeds struct {
Applications *apiseed.Applications `json:"applications" yaml:"applications"`
Incus *apiseed.Incus `json:"incus" yaml:"incus"`
Install *apiseed.Install `json:"install" yaml:"install"`
Network *apiseed.Network `json:"network" yaml:"network"`
Provider *apiseed.Provider `json:"provider" yaml:"provider"`
}
func main() {
images = map[string]apiImagesPost{}
err := do(context.TODO())
if err != nil {
_, _ = fmt.Fprintf(os.Stderr, "Error: %v\n", err)
os.Exit(1)
}
}
func do(_ context.Context) error {
// Arguments.
if len(os.Args) != 2 {
return errors.New("missing image path")
}
// Check that image files exist.
imagePath := os.Args[1]
_, err := os.Stat(filepath.Join(imagePath, "image.iso.gz"))
if err != nil {
return fmt.Errorf("couldn't find 'image.iso.gz': %w", err)
}
_, err = os.Stat(filepath.Join(imagePath, "image.img.gz"))
if err != nil {
return fmt.Errorf("couldn't find 'image.img.gz': %w", err)
}
// Start REST server.
listener, err := net.Listen("tcp", ":8080") //nolint:gosec
if err != nil {
return err
}
// Setup routing.
router := http.NewServeMux()
router.HandleFunc("/", apiRoot)
router.HandleFunc("/1.0", apiRoot10)
router.HandleFunc("/1.0/images", apiImages)
router.HandleFunc("/1.0/images/{uuid}", apiImage)
// Setup server.
server := &http.Server{
Handler: router,
ReadTimeout: 10 * time.Second,
WriteTimeout: 0,
}
return server.Serve(listener)
}
func apiRoot(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if r.Method != http.MethodGet {
_ = response.NotImplemented(nil).Render(w)
return
}
if r.URL.Path != "/" {
_ = response.NotFound(nil).Render(w)
return
}
_ = response.SyncResponse(true, []string{"/1.0"}).Render(w)
}
func apiRoot10(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if r.Method != http.MethodGet {
_ = response.NotImplemented(nil).Render(w)
return
}
_ = response.SyncResponse(true, map[string]any{}).Render(w)
}
func apiImages(w http.ResponseWriter, r *http.Request) {
// Confirm HTTP method.
if r.Method != http.MethodPost {
w.Header().Set("Content-Type", "application/json")
_ = response.NotImplemented(nil).Render(w)
return
}
// Parse the request.
var req apiImagesPost
err := yaml.NewDecoder(http.MaxBytesReader(w, r.Body, 1024*1024)).Decode(&req)
if err != nil {
w.Header().Set("Content-Type", "application/json")
_ = response.BadRequest(err).Render(w)
return
}
// Validate input.
if !slices.Contains([]string{imageTypeISO, imageTypeRaw}, req.Type) {
w.Header().Set("Content-Type", "application/json")
_ = response.BadRequest(errors.New("invalid image type")).Render(w)
return
}
// Store the request.
imagesMu.Lock()
defer imagesMu.Unlock()
imageUUID := uuid.New().String()
images[imageUUID] = req
// Return image details to the user.
w.Header().Set("Content-Type", "application/json")
err = response.SyncResponse(true, map[string]any{"image": "/1.0/images/" + imageUUID}).Render(w)
if err != nil {
_ = response.BadRequest(err).Render(w)
return
}
}
func apiImage(w http.ResponseWriter, r *http.Request) {
// Confirm HTTP method.
if r.Method != http.MethodGet {
w.Header().Set("Content-Type", "application/json")
_ = response.NotImplemented(nil).Render(w)
return
}
// Image UUID.
imageUUID := r.PathValue("uuid")
imagesMu.Lock()
req, ok := images[imageUUID]
if ok {
delete(images, imageUUID)
}
imagesMu.Unlock()
if !ok {
w.Header().Set("Content-Type", "application/json")
_ = response.NotFound(nil).Render(w)
return
}
// Determine source image name.
var fileName string
switch req.Type {
case imageTypeISO:
fileName = "image.iso.gz"
case imageTypeRaw:
fileName = "image.img.gz"
}
// Check if we have compression in-transit.
compress := strings.Contains(r.Header.Get("Accept-Encoding"), "gzip")
// Open the image file.
imageFile, err := os.Open(filepath.Join(os.Args[1], fileName)) //nolint:gosec
if err != nil {
w.Header().Set("Content-Type", "application/json")
_ = response.InternalError(err).Render(w)
return
}
defer func() { _ = imageFile.Close() }()
// Setup gzip seeking decompressor.
rc, err := gzran.NewReader(imageFile)
if err != nil {
w.Header().Set("Content-Type", "application/json")
_ = response.InternalError(err).Render(w)
return
}
// Track down image file.
fileTarget, err := os.Readlink(filepath.Join(os.Args[1], fileName))
if err != nil {
w.Header().Set("Content-Type", "application/json")
_ = response.InternalError(err).Render(w)
return
}
fileName = filepath.Base(fileTarget)
// Serve the image.
if compress {
w.Header().Set("Content-Encoding", "gzip")
w.Header().Set("Content-Type", "application/octet-stream")
fileName = strings.TrimSuffix(fileName, ".gz")
} else {
w.Header().Set("Content-Type", "application/gzip")
}
w.Header().Set("Content-Disposition", "attachment; filename=\""+fileName+"\"")
w.Header().Set("Access-Control-Allow-Origin", "*")
w.WriteHeader(http.StatusOK)
// Setup compressor.
writer := gzip.NewWriter(w)
defer writer.Close()
// Write leading part.
remainder := int64(2148532224)
for {
chunk := int64(4 * 1024 * 1024)
if remainder < chunk {
chunk = remainder
}
if chunk == 0 {
break
}
n, err := io.CopyN(writer, rc, chunk)
if err != nil {
if errors.Is(err, io.EOF) {
break
}
return
}
remainder -= n
}
// Write seed file.
seedSize, err := writeSeed(writer, req.Seeds)
if err != nil {
return
}
// Write trailing part.
_, err = rc.Seek(int64(seedSize), 1)
if err != nil {
return
}
for {
_, err = io.CopyN(writer, rc, 4*1024*1024)
if err != nil {
if errors.Is(err, io.EOF) {
break
}
return
}
}
}
func writeSeed(writer io.Writer, seeds apiImagesPostSeeds) (int, error) {
archiveContents := [][]string{}
// Create applications yaml contents.
if seeds.Applications != nil {
yamlContents, err := yaml.Marshal(seeds.Applications)
if err != nil {
return -1, err
}
archiveContents = append(archiveContents, []string{"applications.yaml", string(yamlContents)})
}
// Create incus yaml contents.
if seeds.Incus != nil {
yamlContents, err := yaml.Marshal(seeds.Incus)
if err != nil {
return -1, err
}
archiveContents = append(archiveContents, []string{"incus.yaml", string(yamlContents)})
}
// Create install yaml contents.
if seeds.Install != nil {
yamlContents, err := yaml.Marshal(seeds.Install)
if err != nil {
return -1, err
}
archiveContents = append(archiveContents, []string{"install.yaml", string(yamlContents)})
}
// Create network yaml contents.
if seeds.Network != nil {
yamlContents, err := yaml.Marshal(seeds.Network)
if err != nil {
return -1, err
}
archiveContents = append(archiveContents, []string{"network.yaml", string(yamlContents)})
}
// Create provider yaml contents.
if seeds.Provider != nil {
yamlContents, err := yaml.Marshal(seeds.Provider)
if err != nil {
return -1, err
}
archiveContents = append(archiveContents, []string{"provider.yaml", string(yamlContents)})
}
// Put a size counter in place.
wc := &writeCounter{}
// Create the tar archive.
tw := tar.NewWriter(io.MultiWriter(wc, writer))
for _, file := range archiveContents {
hdr := &tar.Header{
Name: file[0],
Mode: 0o600,
Size: int64(len(file[1])),
}
err := tw.WriteHeader(hdr)
if err != nil {
return -1, err
}
_, err = tw.Write([]byte(file[1]))
if err != nil {
return -1, err
}
}
err := tw.Close()
if err != nil {
return -1, err
}
return wc.size, nil
}
type writeCounter struct {
size int
}
func (wc *writeCounter) Write(buf []byte) (int, error) {
size := len(buf)
wc.size += size
return size, nil
}

View File

@@ -50,6 +50,7 @@ require (
github.com/rootless-containers/proto/go-proto v0.0.0-20230421021042-4cd87ebadd67 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/timpalpant/gzran v0.0.0-20201127163450-7b631e56f57b // indirect
github.com/urfave/cli v1.22.17 // indirect
github.com/vbatts/go-mtree v0.5.4 // indirect
github.com/zitadel/logging v0.6.2 // indirect

View File

@@ -156,6 +156,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/timpalpant/gzran v0.0.0-20201127163450-7b631e56f57b h1:BbST6DwxZOOs2SlOn0T4ueIEOzrFfs/0gZZLjbWrIoY=
github.com/timpalpant/gzran v0.0.0-20201127163450-7b631e56f57b/go.mod h1:yTxMuBKYLrj6gYYtK3gK0ifBhjiBYtD3URZiNK7vBt0=
github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0=
github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=
github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk=

View File

@@ -7,6 +7,7 @@ import (
incusclient "github.com/lxc/incus/v6/client"
incusapi "github.com/lxc/incus/v6/shared/api"
apiseed "github.com/lxc/incus-os/incus-osd/api/seed"
"github.com/lxc/incus-os/incus-osd/internal/seed"
"github.com/lxc/incus-os/incus-osd/internal/systemd"
)
@@ -64,7 +65,7 @@ func (a *incus) Initialize(ctx context.Context) error {
// If no seed, build one for auto-configuration.
if incusSeed == nil {
incusSeed = &seed.IncusConfig{
incusSeed = &apiseed.Incus{
ApplyDefaults: true,
}
}

View File

@@ -17,6 +17,7 @@ import (
"github.com/lxc/incus/v6/shared/subprocess"
"golang.org/x/sys/unix"
apiseed "github.com/lxc/incus-os/incus-osd/api/seed"
"github.com/lxc/incus-os/incus-osd/internal/seed"
"github.com/lxc/incus-os/incus-osd/internal/systemd"
"github.com/lxc/incus-os/incus-osd/internal/tui"
@@ -24,7 +25,7 @@ import (
// Install holds information necessary to perform an installation.
type Install struct {
config *seed.InstallSeed
config *apiseed.Install
tui *tui.TUI
}
@@ -346,7 +347,7 @@ func getAllTargets(ctx context.Context, sourceDevice string) ([]blockdevices, er
}
// getTargetDevice determines the underlying device to install incus-osd on.
func getTargetDevice(potentialTargets []blockdevices, seedTarget *seed.InstallSeedTarget) (string, error) {
func getTargetDevice(potentialTargets []blockdevices, seedTarget *apiseed.InstallTarget) (string, error) {
// Ensure we found at least one potential install device. If no Target configuration was found,
// only proceed if exactly one device was found.
if len(potentialTargets) == 0 {

View File

@@ -2,23 +2,14 @@ package seed
import (
"context"
apiseed "github.com/lxc/incus-os/incus-osd/api/seed"
)
// Application represents an application.
type Application struct {
Name string `json:"name" yaml:"name"`
}
// Applications represents a list of application.
type Applications struct {
Applications []Application `json:"applications" yaml:"applications"`
Version string `json:"version" yaml:"version"`
}
// GetApplications extracts the list of applications from the seed data.
func GetApplications(_ context.Context, partition string) (*Applications, error) {
func GetApplications(_ context.Context, partition string) (*apiseed.Applications, error) {
// Get applications list
var apps Applications
var apps apiseed.Applications
err := parseFileContents(partition, "applications", &apps)
if err != nil {

View File

@@ -3,22 +3,13 @@ package seed
import (
"context"
incusapi "github.com/lxc/incus/v6/shared/api"
apiseed "github.com/lxc/incus-os/incus-osd/api/seed"
)
// IncusConfig is a wrapper around the Incus preseed.
type IncusConfig struct {
Version string `json:"version" yaml:"version"`
ApplyDefaults bool `json:"apply_defaults" yaml:"apply_defaults"`
Preseed *incusapi.InitPreseed `json:"preseed" yaml:"preseed"`
}
// GetIncus extracts the Incus preseed from the seed data.
func GetIncus(_ context.Context, partition string) (*IncusConfig, error) {
func GetIncus(_ context.Context, partition string) (*apiseed.Incus, error) {
// Get the preseed.
var preseed IncusConfig
var preseed apiseed.Incus
err := parseFileContents(partition, "incus", &preseed)
if err != nil {

View File

@@ -1,27 +1,17 @@
package seed
// InstallSeed defines a struct to hold install configuration.
type InstallSeed struct {
Version string `json:"version" yaml:"version"`
ForceInstall bool `json:"force_install" yaml:"force_install"` // If true, ignore any existing data on target install disk.
ForceReboot bool `json:"force_reboot" yaml:"force_reboot"` // If true, reboot the system automatically upon completion rather than waiting for the install media to be removed.
Target *InstallSeedTarget `json:"target" yaml:"target"` // Optional selector for the target install disk; if not set, expect a single drive to be present.
}
// InstallSeedTarget defines options used to select the target install disk.
type InstallSeedTarget struct {
ID string `json:"id" yaml:"id"` // Name as listed in /dev/disk/by-id/, glob supported.
}
import (
apiseed "github.com/lxc/incus-os/incus-osd/api/seed"
)
// GetInstall extracts the installation config from the seed data.
func GetInstall(partition string) (*InstallSeed, error) {
func GetInstall(partition string) (*apiseed.Install, error) {
// Get the install configuration.
var config InstallSeed
var config apiseed.Install
err := parseFileContents(partition, "install", &config)
if err != nil {
return &InstallSeed{}, err
return nil, err
}
return &config, nil

View File

@@ -10,20 +10,14 @@ import (
"github.com/lxc/incus/v6/shared/subprocess"
"github.com/lxc/incus-os/incus-osd/api"
apiseed "github.com/lxc/incus-os/incus-osd/api/seed"
)
// NetworkSeed defines a struct to hold network configuration.
type NetworkSeed struct {
api.SystemNetworkConfig `yaml:",inline"`
Version string `json:"version" yaml:"version"`
}
// GetNetwork extracts the network configuration from the seed data.
// If no seed network found, a default minimal network config will be returned.
func GetNetwork(ctx context.Context, partition string) (*api.SystemNetworkConfig, error) {
// Get the network configuration.
var config NetworkSeed
var config apiseed.Network
err := parseFileContents(partition, "network", &config)
if err != nil {

View File

@@ -3,20 +3,13 @@ package seed
import (
"context"
"github.com/lxc/incus-os/incus-osd/api"
apiseed "github.com/lxc/incus-os/incus-osd/api/seed"
)
// ProviderSeed defines a struct to hold provider configuration.
type ProviderSeed struct {
api.SystemProviderConfig `yaml:",inline"`
Version string `json:"version" yaml:"version"`
}
// GetProvider extracts the provider configuration from the seed data.
func GetProvider(_ context.Context, partition string) (*ProviderSeed, error) {
func GetProvider(_ context.Context, partition string) (*apiseed.Provider, error) {
// Get the install configuration.
var config ProviderSeed
var config apiseed.Provider
err := parseFileContents(partition, "provider", &config)
if err != nil {