build: use buildx to build multi-platform docker images (#30530)
This commit is contained in:
Martin HS
GitHub
parent
0a21cb4d21
commit
cfe25c7a3b
22
.travis.yml
22
.travis.yml
|
|
@ -9,8 +9,7 @@ jobs:
|
|||
- azure-osx
|
||||
|
||||
include:
|
||||
# These builders create the Docker sub-images for multi-arch push and each
|
||||
# will attempt to push the multi-arch image if they are the last builder
|
||||
# This builder create and push the Docker images for all architectures
|
||||
- stage: build
|
||||
if: type = push
|
||||
os: linux
|
||||
|
|
@ -26,24 +25,7 @@ jobs:
|
|||
before_install:
|
||||
- export DOCKER_CLI_EXPERIMENTAL=enabled
|
||||
script:
|
||||
- go run build/ci.go docker -image -manifest amd64,arm64 -upload ethereum/client-go
|
||||
|
||||
- stage: build
|
||||
if: type = push
|
||||
os: linux
|
||||
arch: arm64
|
||||
dist: focal
|
||||
go: 1.23.x
|
||||
env:
|
||||
- docker
|
||||
services:
|
||||
- docker
|
||||
git:
|
||||
submodules: false # avoid cloning ethereum/tests
|
||||
before_install:
|
||||
- export DOCKER_CLI_EXPERIMENTAL=enabled
|
||||
script:
|
||||
- go run build/ci.go docker -image -manifest amd64,arm64 -upload ethereum/client-go
|
||||
- go run build/ci.go dockerx -platform "linux/amd64,linux/arm64" -upload ethereum/client-go
|
||||
|
||||
# This builder does the Linux Azure uploads
|
||||
- stage: build
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ ARG VERSION=""
|
|||
ARG BUILDNUM=""
|
||||
|
||||
# Build Geth in a stock Go builder container
|
||||
FROM golang:1.23-alpine as builder
|
||||
FROM golang:1.23-alpine AS builder
|
||||
|
||||
RUN apk add --no-cache gcc musl-dev linux-headers git
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ ARG VERSION=""
|
|||
ARG BUILDNUM=""
|
||||
|
||||
# Build Geth in a stock Go builder container
|
||||
FROM golang:1.23-alpine as builder
|
||||
FROM golang:1.23-alpine AS builder
|
||||
|
||||
RUN apk add --no-cache gcc musl-dev linux-headers git
|
||||
|
||||
|
|
@ -14,6 +14,13 @@ COPY go.sum /go-ethereum/
|
|||
RUN cd /go-ethereum && go mod download
|
||||
|
||||
ADD . /go-ethereum
|
||||
|
||||
# This is not strictly necessary, but it matches the "Dockerfile" steps, thus
|
||||
# makes it so that under certain circumstances, the docker layer can be cached,
|
||||
# and the builder can jump to the next (build all) command, with the go cache fully loaded.
|
||||
#
|
||||
RUN cd /go-ethereum && go run build/ci.go install -static ./cmd/geth
|
||||
|
||||
RUN cd /go-ethereum && go run build/ci.go install -static
|
||||
|
||||
# Pull all binaries into a second stage deploy alpine container
|
||||
|
|
|
|||
151
build/ci.go
151
build/ci.go
|
|
@ -50,7 +50,6 @@ import (
|
|||
"path"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
|
|
@ -159,8 +158,8 @@ func main() {
|
|||
doLint(os.Args[2:])
|
||||
case "archive":
|
||||
doArchive(os.Args[2:])
|
||||
case "docker":
|
||||
doDocker(os.Args[2:])
|
||||
case "dockerx":
|
||||
doDockerBuildx(os.Args[2:])
|
||||
case "debsrc":
|
||||
doDebianSource(os.Args[2:])
|
||||
case "nsis":
|
||||
|
|
@ -723,10 +722,9 @@ func maybeSkipArchive(env build.Environment) {
|
|||
}
|
||||
|
||||
// Builds the docker images and optionally uploads them to Docker Hub.
|
||||
func doDocker(cmdline []string) {
|
||||
func doDockerBuildx(cmdline []string) {
|
||||
var (
|
||||
image = flag.Bool("image", false, `Whether to build and push an arch specific docker image`)
|
||||
manifest = flag.String("manifest", "", `Push a multi-arch docker image for the specified architectures (usually "amd64,arm64")`)
|
||||
platform = flag.String("platform", "", `Push a multi-arch docker image for the specified architectures (usually "linux/amd64,linux/arm64")`)
|
||||
upload = flag.String("upload", "", `Where to upload the docker image (usually "ethereum/client-go")`)
|
||||
)
|
||||
flag.CommandLine.Parse(cmdline)
|
||||
|
|
@ -761,129 +759,26 @@ func doDocker(cmdline []string) {
|
|||
case strings.HasPrefix(env.Tag, "v1."):
|
||||
tags = []string{"stable", fmt.Sprintf("release-1.%d", params.VersionMinor), "v" + params.Version}
|
||||
}
|
||||
// If architecture specific image builds are requested, build and push them
|
||||
if *image {
|
||||
build.MustRunCommand("docker", "build", "--build-arg", "COMMIT="+env.Commit, "--build-arg", "VERSION="+params.VersionWithMeta, "--build-arg", "BUILDNUM="+env.Buildnum, "--tag", fmt.Sprintf("%s:TAG", *upload), ".")
|
||||
build.MustRunCommand("docker", "build", "--build-arg", "COMMIT="+env.Commit, "--build-arg", "VERSION="+params.VersionWithMeta, "--build-arg", "BUILDNUM="+env.Buildnum, "--tag", fmt.Sprintf("%s:alltools-TAG", *upload), "-f", "Dockerfile.alltools", ".")
|
||||
// Need to create a mult-arch builder
|
||||
build.MustRunCommand("docker", "buildx", "create", "--use", "--name", "multi-arch-builder", "--platform", *platform)
|
||||
|
||||
// Tag and upload the images to Docker Hub
|
||||
for _, tag := range tags {
|
||||
gethImage := fmt.Sprintf("%s:%s-%s", *upload, tag, runtime.GOARCH)
|
||||
toolImage := fmt.Sprintf("%s:alltools-%s-%s", *upload, tag, runtime.GOARCH)
|
||||
|
||||
// If the image already exists (non version tag), check the build
|
||||
// number to prevent overwriting a newer commit if concurrent builds
|
||||
// are running. This is still a tiny bit racey if two published are
|
||||
// done at the same time, but that's extremely unlikely even on the
|
||||
// master branch.
|
||||
for _, img := range []string{gethImage, toolImage} {
|
||||
if exec.Command("docker", "pull", img).Run() != nil {
|
||||
continue // Generally the only failure is a missing image, which is good
|
||||
}
|
||||
buildnum, err := exec.Command("docker", "inspect", "--format", "{{index .Config.Labels \"buildnum\"}}", img).CombinedOutput()
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to inspect container: %v\nOutput: %s", err, string(buildnum))
|
||||
}
|
||||
buildnum = bytes.TrimSpace(buildnum)
|
||||
|
||||
if len(buildnum) > 0 && len(env.Buildnum) > 0 {
|
||||
oldnum, err := strconv.Atoi(string(buildnum))
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to parse old image build number: %v", err)
|
||||
}
|
||||
newnum, err := strconv.Atoi(env.Buildnum)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to parse current build number: %v", err)
|
||||
}
|
||||
if oldnum > newnum {
|
||||
log.Fatalf("Current build number %d not newer than existing %d", newnum, oldnum)
|
||||
} else {
|
||||
log.Printf("Updating %s from build %d to %d", img, oldnum, newnum)
|
||||
}
|
||||
}
|
||||
}
|
||||
build.MustRunCommand("docker", "image", "tag", fmt.Sprintf("%s:TAG", *upload), gethImage)
|
||||
build.MustRunCommand("docker", "image", "tag", fmt.Sprintf("%s:alltools-TAG", *upload), toolImage)
|
||||
build.MustRunCommand("docker", "push", gethImage)
|
||||
build.MustRunCommand("docker", "push", toolImage)
|
||||
}
|
||||
}
|
||||
// If multi-arch image manifest push is requested, assemble it
|
||||
if len(*manifest) != 0 {
|
||||
// Since different architectures are pushed by different builders, wait
|
||||
// until all required images are updated.
|
||||
var mismatch bool
|
||||
for i := 0; i < 2; i++ { // 2 attempts, second is race check
|
||||
mismatch = false // hope there's no mismatch now
|
||||
|
||||
for _, tag := range tags {
|
||||
for _, arch := range strings.Split(*manifest, ",") {
|
||||
gethImage := fmt.Sprintf("%s:%s-%s", *upload, tag, arch)
|
||||
toolImage := fmt.Sprintf("%s:alltools-%s-%s", *upload, tag, arch)
|
||||
|
||||
for _, img := range []string{gethImage, toolImage} {
|
||||
if out, err := exec.Command("docker", "pull", img).CombinedOutput(); err != nil {
|
||||
log.Printf("Required image %s unavailable: %v\nOutput: %s", img, err, out)
|
||||
mismatch = true
|
||||
break
|
||||
}
|
||||
buildnum, err := exec.Command("docker", "inspect", "--format", "{{index .Config.Labels \"buildnum\"}}", img).CombinedOutput()
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to inspect container: %v\nOutput: %s", err, string(buildnum))
|
||||
}
|
||||
buildnum = bytes.TrimSpace(buildnum)
|
||||
|
||||
if string(buildnum) != env.Buildnum {
|
||||
log.Printf("Build number mismatch on %s: want %s, have %s", img, env.Buildnum, buildnum)
|
||||
mismatch = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if mismatch {
|
||||
break
|
||||
}
|
||||
}
|
||||
if mismatch {
|
||||
break
|
||||
}
|
||||
}
|
||||
if mismatch {
|
||||
// Build numbers mismatching, retry in a short time to
|
||||
// avoid concurrent fails in both publisher images. If
|
||||
// however the retry failed too, it means the concurrent
|
||||
// builder is still crunching, let that do the publish.
|
||||
if i == 0 {
|
||||
time.Sleep(30 * time.Second)
|
||||
}
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
if mismatch {
|
||||
log.Println("Relinquishing publish to other builder")
|
||||
return
|
||||
}
|
||||
// Assemble and push the Geth manifest image
|
||||
for _, tag := range tags {
|
||||
gethImage := fmt.Sprintf("%s:%s", *upload, tag)
|
||||
|
||||
var gethSubImages []string
|
||||
for _, arch := range strings.Split(*manifest, ",") {
|
||||
gethSubImages = append(gethSubImages, gethImage+"-"+arch)
|
||||
}
|
||||
build.MustRunCommand("docker", append([]string{"manifest", "create", gethImage}, gethSubImages...)...)
|
||||
build.MustRunCommand("docker", "manifest", "push", gethImage)
|
||||
}
|
||||
// Assemble and push the alltools manifest image
|
||||
for _, tag := range tags {
|
||||
toolImage := fmt.Sprintf("%s:alltools-%s", *upload, tag)
|
||||
|
||||
var toolSubImages []string
|
||||
for _, arch := range strings.Split(*manifest, ",") {
|
||||
toolSubImages = append(toolSubImages, toolImage+"-"+arch)
|
||||
}
|
||||
build.MustRunCommand("docker", append([]string{"manifest", "create", toolImage}, toolSubImages...)...)
|
||||
build.MustRunCommand("docker", "manifest", "push", toolImage)
|
||||
for _, spec := range []struct {
|
||||
file string
|
||||
base string
|
||||
}{
|
||||
{file: "Dockerfile", base: fmt.Sprintf("%s:", *upload)},
|
||||
{file: "Dockerfile.alltools", base: fmt.Sprintf("%s:alltools-", *upload)},
|
||||
} {
|
||||
for _, tag := range tags { // latest, stable etc
|
||||
gethImage := fmt.Sprintf("%s%s", spec.base, tag)
|
||||
build.MustRunCommand("docker", "buildx", "build",
|
||||
"--build-arg", "COMMIT="+env.Commit,
|
||||
"--build-arg", "VERSION="+params.VersionWithMeta,
|
||||
"--build-arg", "BUILDNUM="+env.Buildnum,
|
||||
"--tag", gethImage,
|
||||
"--platform", *platform,
|
||||
"--push",
|
||||
"--file", spec.file, ".")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue