diff --git a/__tests__/docker/install.test.itg.ts b/__tests__/docker/install.test.itg.ts index 679fa85..822da61 100644 --- a/__tests__/docker/install.test.itg.ts +++ b/__tests__/docker/install.test.itg.ts @@ -19,7 +19,7 @@ import fs from 'fs'; import os from 'os'; import path from 'path'; -import {Install} from '../../src/docker/install'; +import {Install, InstallSourceArchive, InstallSourceImage} from '../../src/docker/install'; import {Docker} from '../../src/docker/docker'; import {Exec} from '../../src/exec'; @@ -40,8 +40,12 @@ aarch64:https://cloud.debian.org/images/cloud/bookworm/20231013-1532/debian-12-g process.env = originalEnv; }); // prettier-ignore - test.each(['v26.1.4'])( - 'install docker %s', async (version) => { + test.each([ + {type: 'image', tag: '27.3.1'} as InstallSourceImage, + {type: 'image', tag: 'master'} as InstallSourceImage, + {type: 'archive', version: 'v26.1.4', channel: 'stable'} as InstallSourceArchive, + ])( + 'install docker %s', async (source) => { if (process.env.ImageOS && process.env.ImageOS.startsWith('ubuntu')) { // Remove containerd first on ubuntu runners to make sure it takes // ones packaged with docker @@ -53,18 +57,19 @@ aarch64:https://cloud.debian.org/images/cloud/bookworm/20231013-1532/debian-12-g } }); } + const install = new Install({ + source: source, + runDir: tmpDir, + contextName: 'foo', + daemonConfig: `{"debug":true,"features":{"containerd-snapshotter":true}}` + }); await expect((async () => { - const install = new Install({ - version: version, - runDir: tmpDir, - contextName: 'foo', - daemonConfig: `{"debug":true,"features":{"containerd-snapshotter":true}}` - }); await install.download(); await install.install(); await Docker.printVersion(); await Docker.printInfo(); + })().finally(async () => { await install.tearDown(); - })()).resolves.not.toThrow(); - }, 1200000); + })).resolves.not.toThrow(); + }, 30 * 60 * 1000); }); diff --git a/__tests__/docker/install.test.ts b/__tests__/docker/install.test.ts index 80ecc29..7016c56 100644 --- a/__tests__/docker/install.test.ts +++ b/__tests__/docker/install.test.ts @@ -21,7 +21,7 @@ import path from 'path'; import * as rimraf from 'rimraf'; import osm = require('os'); -import {Install} from '../../src/docker/install'; +import {Install, InstallSourceArchive, InstallSourceImage} from '../../src/docker/install'; const tmpDir = fs.mkdtempSync(path.join(process.env.TEMP || os.tmpdir(), 'docker-install-')); @@ -29,18 +29,39 @@ afterEach(function () { rimraf.sync(tmpDir); }); +const archive = (version: string, channel: string): InstallSourceArchive => { + return { + type: 'archive', + version: version, + channel: channel + }; +}; + +const image = (tag: string): InstallSourceImage => { + return { + type: 'image', + tag: tag + }; +}; + describe('download', () => { // prettier-ignore test.each([ - ['v19.03.14', 'linux'], - ['v20.10.22', 'linux'], - ['v20.10.22', 'darwin'], - ['v20.10.22', 'win32'], + [archive('v19.03.14', 'stable'), 'linux'], + [archive('v20.10.22', 'stable'), 'linux'], + [archive('v20.10.22', 'stable'), 'darwin'], + [archive('v20.10.22', 'stable'), 'win32'], + + [image('master'), 'linux'], + [image('master'), 'win32'], + + [image('27.3.1'), 'linux'], + [image('27.3.1'), 'win32'], ])( - 'acquires %p of docker (%s)', async (version, platformOS) => { + 'acquires %p of docker (%s)', async (source, platformOS) => { jest.spyOn(osm, 'platform').mockImplementation(() => platformOS as NodeJS.Platform); const install = new Install({ - version: version, + source: source, runDir: tmpDir, }); const toolPath = await install.download(); diff --git a/src/docker/assets.ts b/src/docker/assets.ts index 15e39c2..00f7332 100644 --- a/src/docker/assets.ts +++ b/src/docker/assets.ts @@ -221,16 +221,49 @@ provision: EOF fi export DEBIAN_FRONTEND=noninteractive - curl -fsSL https://get.docker.com | sh -s -- --channel {{dockerBinChannel}} --version {{dockerBinVersion}} + if [ "{{srcType}}" == "archive" ]; then + curl -fsSL https://get.docker.com | sh -s -- --channel {{srcArchiveChannel}} --version {{srcArchiveVersion}} + elif [ "{{srcType}}" == "image" ]; then + arch=$(uname -m) + case $arch in + x86_64) arch=amd64;; + aarch64) arch=arm64;; + esac + url="https://github.com/crazy-max/undock/releases/download/v0.8.0/undock_0.8.0_linux_$arch.tar.gz" + + wget "$url" -O /tmp/undock.tar.gz + tar -C /usr/local/bin -xvf /tmp/undock.tar.gz + undock --version + + HOME=/tmp undock moby/moby-bin:{{srcImageTag}} /usr/local/bin + + wget https://raw.githubusercontent.com/moby/moby/{{srcImageTag}}/contrib/init/systemd/docker.service \ + https://raw.githubusercontent.com/moby/moby/v{{srcImageTag}}/contrib/init/systemd/docker.service \ + -O /etc/systemd/system/docker.service || true + wget https://raw.githubusercontent.com/moby/moby/{{srcImageTag}}/contrib/init/systemd/docker.socket \ + https://raw.githubusercontent.com/moby/moby/v{{srcImageTag}}/contrib/init/systemd/docker.socket \ + -O /etc/systemd/system/docker.socket || true + + sed -i 's|^ExecStart=.*|ExecStart=/usr/local/bin/dockerd -H fd://|' /etc/systemd/system/docker.service + sed -i 's|containerd.service||' /etc/systemd/system/docker.service + if ! getent group docker; then + groupadd --system docker + fi + systemctl daemon-reload + fail=0 + if ! systemctl enable --now docker; then + fail=1 + fi + systemctl status docker.socket || true + systemctl status docker.service || true + exit $fail + fi probes: - script: | #!/bin/bash set -eux -o pipefail - if ! timeout 30s bash -c "until command -v docker >/dev/null 2>&1; do sleep 3; done"; then - echo >&2 "docker is not installed yet" - exit 1 - fi + # Don't check for docker CLI as it's not installed in the VM (only on the host) if ! timeout 30s bash -c "until pgrep dockerd; do sleep 3; done"; then echo >&2 "dockerd is not running" exit 1 diff --git a/src/docker/install.ts b/src/docker/install.ts index c681d97..d2c89a3 100644 --- a/src/docker/install.ts +++ b/src/docker/install.ts @@ -33,10 +33,25 @@ import {Exec} from '../exec'; import {Util} from '../util'; import {limaYamlData, dockerServiceLogsPs1, setupDockerWinPs1} from './assets'; import {GitHubRelease} from '../types/github'; +import {HubRepository} from '../hubRepository'; + +export interface InstallSourceImage { + type: 'image'; + tag: string; +} + +export interface InstallSourceArchive { + type: 'archive'; + version: string; + channel: string; +} + +export type InstallSource = InstallSourceImage | InstallSourceArchive; export interface InstallOpts { - version?: string; - channel?: string; + source?: InstallSource; + + // ... runDir: string; contextName?: string; daemonConfig?: string; @@ -50,8 +65,7 @@ interface LimaImage { export class Install { private readonly runDir: string; - private readonly version: string; - private readonly channel: string; + private readonly source: InstallSource; private readonly contextName: string; private readonly daemonConfig?: string; private _version: string | undefined; @@ -61,8 +75,11 @@ export class Install { constructor(opts: InstallOpts) { this.runDir = opts.runDir; - this.version = opts.version || 'latest'; - this.channel = opts.channel || 'stable'; + this.source = opts.source || { + type: 'archive', + version: 'latest', + channel: 'stable' + }; this.contextName = opts.contextName || 'setup-docker-action'; this.daemonConfig = opts.daemonConfig; } @@ -71,12 +88,12 @@ export class Install { return this._toolDir || Context.tmpDir(); } - public async download(): Promise { - const release: GitHubRelease = await Install.getRelease(this.version); + async downloadStaticArchive(src: InstallSourceArchive): Promise { + const release: GitHubRelease = await Install.getRelease(src.version); this._version = release.tag_name.replace(/^v+|v+$/g, ''); core.debug(`docker.Install.download version: ${this._version}`); - const downloadURL = this.downloadURL(this._version, this.channel); + const downloadURL = this.downloadURL(this._version, src.channel); core.info(`Downloading ${downloadURL}`); const downloadPath = await tc.downloadTool(downloadURL); @@ -92,6 +109,46 @@ export class Install { extractFolder = path.join(extractFolder, 'docker'); } core.debug(`docker.Install.download extractFolder: ${extractFolder}`); + return extractFolder; + } + + public async download(): Promise { + let extractFolder: string; + let cacheKey: string; + const platform = os.platform(); + + switch (this.source.type) { + case 'image': { + const tag = this.source.tag; + this._version = tag; + cacheKey = `docker-image`; + + core.info(`Downloading docker cli from dockereng/cli-bin:${tag}`); + const cli = await HubRepository.build('dockereng/cli-bin'); + extractFolder = await cli.extractImage(tag); + + if (['win32', 'linux'].includes(platform)) { + core.info(`Downloading dockerd from moby/moby-bin:${tag}`); + const moby = await HubRepository.build('moby/moby-bin'); + await moby.extractImage(tag, extractFolder); + } else if (platform == 'darwin') { + // On macOS, the docker daemon binary will be downloaded inside the lima VM + } else { + core.warning(`dockerd not supported on ${platform}, only the Docker cli will be available`); + } + break; + } + case 'archive': { + const version = this.source.version; + const channel = this.source.channel; + cacheKey = `docker-archive-${channel}`; + this._version = version; + + core.info(`Downloading Docker ${version} from ${this.source.channel} at download.docker.com`); + extractFolder = await this.downloadStaticArchive(this.source); + break; + } + } core.info('Fixing perms'); fs.readdir(path.join(extractFolder), function (err, files) { @@ -104,7 +161,7 @@ export class Install { }); }); - const tooldir = await tc.cacheDir(extractFolder, `docker-${this.channel}`, this._version.replace(/(0+)([1-9]+)/, '$2')); + const tooldir = await tc.cacheDir(extractFolder, cacheKey, this._version.replace(/(0+)([1-9]+)/, '$2')); core.addPath(tooldir); core.info('Added Docker to PATH'); @@ -136,6 +193,7 @@ export class Install { } private async installDarwin(): Promise { + const src = this.source; const limaDir = path.join(os.homedir(), '.lima', this.limaInstanceName); await io.mkdirP(limaDir); const dockerHost = `unix://${limaDir}/docker.sock`; @@ -166,12 +224,15 @@ export class Install { handlebars.registerHelper('stringify', function (obj) { return new handlebars.SafeString(JSON.stringify(obj)); }); + const srcArchive = src as InstallSourceArchive; const limaCfg = handlebars.compile(limaYamlData)({ customImages: Install.limaCustomImages(), daemonConfig: limaDaemonConfig, dockerSock: `${limaDir}/docker.sock`, - dockerBinVersion: this._version, - dockerBinChannel: this.channel + srcType: src.type, + srcArchiveVersion: srcArchive.version?.replace(/^v/, ''), + srcArchiveChannel: srcArchive.channel, + srcImageTag: (src as InstallSourceImage).tag }); core.info(`Writing lima config to ${path.join(limaDir, 'lima.yaml')}`); fs.writeFileSync(path.join(limaDir, 'lima.yaml'), limaCfg); @@ -527,7 +588,10 @@ EOF`, } const releases = >JSON.parse(body); if (!releases[version]) { - throw new Error(`Cannot find Docker release ${version} in ${url}`); + if (!releases['v' + version]) { + throw new Error(`Cannot find Docker release ${version} in ${url}`); + } + return releases['v' + version]; } return releases[version]; } diff --git a/src/dockerhub.ts b/src/dockerhub.ts index 62bea8d..3bf1d59 100644 --- a/src/dockerhub.ts +++ b/src/dockerhub.ts @@ -111,17 +111,21 @@ export class DockerHub { const body = await resp.readBody(); resp.message.statusCode = resp.message.statusCode || HttpCodes.InternalServerError; if (resp.message.statusCode < 200 || resp.message.statusCode >= 300) { - if (resp.message.statusCode == HttpCodes.Unauthorized) { - throw new Error(`Docker Hub API: operation not permitted`); - } - const errResp = >JSON.parse(body); - for (const k of ['message', 'detail', 'error']) { - if (errResp[k]) { - throw new Error(`Docker Hub API: bad status code ${resp.message.statusCode}: ${errResp[k]}`); - } - } - throw new Error(`Docker Hub API: bad status code ${resp.message.statusCode}`); + throw DockerHub.parseError(resp, body); } return body; } + + public static parseError(resp: httpm.HttpClientResponse, body: string): Error { + if (resp.message.statusCode == HttpCodes.Unauthorized) { + throw new Error(`Docker Hub API: operation not permitted`); + } + const errResp = >JSON.parse(body); + for (const k of ['message', 'detail', 'error']) { + if (errResp[k]) { + throw new Error(`Docker Hub API: bad status code ${resp.message.statusCode}: ${errResp[k]}`); + } + } + throw new Error(`Docker Hub API: bad status code ${resp.message.statusCode}`); + } } diff --git a/src/hubRepository.ts b/src/hubRepository.ts new file mode 100644 index 0000000..e2a8884 --- /dev/null +++ b/src/hubRepository.ts @@ -0,0 +1,157 @@ +/** + * Copyright 2023 actions-toolkit authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as httpm from '@actions/http-client'; +import {Index} from './types/oci'; +import os from 'os'; +import * as core from '@actions/core'; +import {Manifest} from './types/oci/manifest'; +import * as tc from '@actions/tool-cache'; +import fs from 'fs'; +import {MEDIATYPE_IMAGE_INDEX_V1, MEDIATYPE_IMAGE_MANIFEST_V1} from './types/oci/mediatype'; +import {MEDIATYPE_IMAGE_MANIFEST_V2, MEDIATYPE_IMAGE_MANIFEST_LIST_V2} from './types/docker/mediatype'; +import {DockerHub} from './dockerhub'; + +export class HubRepository { + private repo: string; + private token: string; + private static readonly http: httpm.HttpClient = new httpm.HttpClient('setup-docker-action'); + + private constructor(repository: string, token: string) { + this.repo = repository; + this.token = token; + } + + public static async build(repository: string): Promise { + const token = await this.getToken(repository); + return new HubRepository(repository, token); + } + + // Unpacks the image layers and returns the path to the extracted image. + // Only OCI indexes/manifest list are supported for now. + public async extractImage(tag: string, destDir?: string): Promise { + const index = await this.getManifest(tag); + if (index.mediaType != MEDIATYPE_IMAGE_INDEX_V1 && index.mediaType != MEDIATYPE_IMAGE_MANIFEST_LIST_V2) { + throw new Error(`Unsupported image media type: ${index.mediaType}`); + } + const digest = HubRepository.getPlatformManifestDigest(index); + const manifest = await this.getManifest(digest); + + const paths = manifest.layers.map(async layer => { + const url = this.blobUrl(layer.digest); + + return await tc.downloadTool(url, undefined, undefined, { + authorization: `Bearer ${this.token}` + }); + }); + + let files = await Promise.all(paths); + let extractFolder: string; + if (!destDir) { + extractFolder = await tc.extractTar(files[0]); + files = files.slice(1); + } else { + extractFolder = destDir; + } + + await Promise.all( + files.map(async file => { + return await tc.extractTar(file, extractFolder); + }) + ); + + fs.readdirSync(extractFolder).forEach(file => { + core.info(`extractImage(${this.repo}:${tag}) file: ${file}`); + }); + + return extractFolder; + } + + private static async getToken(repo: string): Promise { + const url = `https://auth.docker.io/token?service=registry.docker.io&scope=repository:${repo}:pull`; + + const resp = await this.http.get(url); + const body = await resp.readBody(); + const statusCode = resp.message.statusCode || 500; + if (statusCode != 200) { + throw DockerHub.parseError(resp, body); + } + + const json = JSON.parse(body); + return json.token; + } + + private blobUrl(digest: string): string { + return `https://registry-1.docker.io/v2/${this.repo}/blobs/${digest}`; + } + + public async getManifest(tagOrDigest: string): Promise { + const url = `https://registry-1.docker.io/v2/${this.repo}/manifests/${tagOrDigest}`; + + const headers = { + Authorization: `Bearer ${this.token}`, + Accept: [MEDIATYPE_IMAGE_INDEX_V1, MEDIATYPE_IMAGE_MANIFEST_LIST_V2, MEDIATYPE_IMAGE_MANIFEST_V1, MEDIATYPE_IMAGE_MANIFEST_V2].join(', ') + }; + const resp = await HubRepository.http.get(url, headers); + const body = await resp.readBody(); + const statusCode = resp.message.statusCode || 500; + if (statusCode != 200) { + throw DockerHub.parseError(resp, body); + } + + return JSON.parse(body); + } + + private static getPlatformManifestDigest(index: Index): string { + // This doesn't handle all possible platforms normalizations, but it's good enough for now. + let pos: string = os.platform(); + if (pos == 'win32') { + pos = 'windows'; + } + let arch = os.arch(); + if (arch == 'x64') { + arch = 'amd64'; + } + let variant = ''; + if (arch == 'arm') { + variant = 'v7'; + } + + const manifest = index.manifests.find(m => { + if (!m.platform) { + return false; + } + if (m.platform.os != pos) { + core.debug(`Skipping manifest ${m.digest} because of os: ${m.platform.os} != ${pos}`); + return false; + } + if (m.platform.architecture != arch) { + core.debug(`Skipping manifest ${m.digest} because of arch: ${m.platform.architecture} != ${arch}`); + return false; + } + if ((m.platform.variant || '') != variant) { + core.debug(`Skipping manifest ${m.digest} because of variant: ${m.platform.variant} != ${variant}`); + return false; + } + + return true; + }); + if (!manifest) { + throw new Error(`Cannot find manifest for ${pos}/${arch}/${variant}`); + } + return manifest.digest; + } +} diff --git a/src/types/docker/mediatype.ts b/src/types/docker/mediatype.ts new file mode 100644 index 0000000..d06d1e9 --- /dev/null +++ b/src/types/docker/mediatype.ts @@ -0,0 +1,19 @@ +/** + * Copyright 2023 actions-toolkit authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export const MEDIATYPE_IMAGE_MANIFEST_LIST_V2 = 'application/vnd.docker.distribution.manifest.list.v2+json'; + +export const MEDIATYPE_IMAGE_MANIFEST_V2 = 'application/vnd.docker.distribution.manifest.v2+json';