Merge pull request #307 from crazy-max/build-export

buildx: build history export

.github/workflows/test.yml

@@ -131,6 +131,14 @@ jobs:
         with:
           version: ${{ env.BUILDX_VERSION }}
           driver: docker
+      -
+        name: Set up container builder
+        if: startsWith(matrix.os, 'ubuntu')
+        id: builder
+        uses: docker/setup-buildx-action@v3
+        with:
+          version: ${{ env.BUILDX_VERSION }}
+          use: false
       -
         name: Install
         run: yarn install
@@ -140,6 +148,7 @@ jobs:
           yarn test:itg-coverage --runTestsByPath __tests__/${{ matrix.test }} --coverageDirectory=./coverage
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          CTN_BUILDER_NAME: ${{ steps.builder.outputs.name }}
       -
         name: Check coverage
         run: |

__tests__/buildx/history.test.itg.ts

@@ -0,0 +1,151 @@
+/**
+ * Copyright 2024 actions-toolkit authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {beforeEach, describe, expect, jest, test} from '@jest/globals';
+import * as fs from 'fs';
+import * as path from 'path';
+
+import {Buildx} from '../../src/buildx/buildx';
+import {Bake} from '../../src/buildx/bake';
+import {Build} from '../../src/buildx/build';
+import {History} from '../../src/buildx/history';
+import {Exec} from '../../src/exec';
+
+const fixturesDir = path.join(__dirname, '..', 'fixtures');
+
+// prettier-ignore
+const tmpDir = path.join(process.env.TEMP || '/tmp', 'buildx-history-jest');
+
+const maybe = !process.env.GITHUB_ACTIONS || (process.env.GITHUB_ACTIONS === 'true' && process.env.ImageOS && process.env.ImageOS.startsWith('ubuntu')) ? describe : describe.skip;
+
+beforeEach(() => {
+  jest.clearAllMocks();
+});
+
+maybe('exportBuild', () => {
+  // prettier-ignore
+  test.each([
+    [
+      'single',
+      [
+        'build',
+        '-f', path.join(fixturesDir, 'hello.Dockerfile'),
+        fixturesDir
+      ],
+    ],
+    [
+      'multi-platform',
+      [
+        'build',
+        '-f', path.join(fixturesDir, 'hello.Dockerfile'),
+        '--platform', 'linux/amd64,linux/arm64',
+        fixturesDir
+      ],
+    ]
+  ])('export build %p', async (_, bargs) => {
+    const buildx = new Buildx();
+    const build = new Build({buildx: buildx});
+
+    fs.mkdirSync(tmpDir, {recursive: true});
+    await expect(
+      (async () => {
+        // prettier-ignore
+        const buildCmd = await buildx.getCommand([
+          '--builder', process.env.CTN_BUILDER_NAME ?? 'default',
+          ...bargs,
+          '--metadata-file', build.getMetadataFilePath()
+        ]);
+        await Exec.exec(buildCmd.command, buildCmd.args);
+      })()
+    ).resolves.not.toThrow();
+
+    const metadata = build.resolveMetadata();
+    expect(metadata).toBeDefined();
+    const buildRef = build.resolveRef(metadata);
+    expect(buildRef).toBeDefined();
+
+    const history = new History({buildx: buildx});
+    const exportRes = await history.export({
+      refs: [buildRef ?? '']
+    });
+
+    expect(exportRes).toBeDefined();
+    expect(exportRes?.dockerbuildFilename).toBeDefined();
+    expect(exportRes?.dockerbuildSize).toBeDefined();
+    expect(fs.existsSync(exportRes?.dockerbuildFilename)).toBe(true);
+  });
+
+  // prettier-ignore
+  test.each([
+    [
+      'single',
+      [
+        'bake',
+        '-f', path.join(fixturesDir, 'hello-bake.hcl'),
+        'hello'
+      ],
+    ],
+    [
+      'group',
+      [
+        'bake',
+        '-f', path.join(fixturesDir, 'hello-bake.hcl'),
+        'hello-all'
+      ],
+    ],
+    [
+      'matrix',
+      [
+        'bake',
+        '-f', path.join(fixturesDir, 'hello-bake.hcl'),
+        'hello-matrix'
+      ],
+    ]
+  ])('export bake build %p', async (_, bargs) => {
+    const buildx = new Buildx();
+    const bake = new Bake({buildx: buildx});
+
+    fs.mkdirSync(tmpDir, {recursive: true});
+    await expect(
+      (async () => {
+        // prettier-ignore
+        const buildCmd = await buildx.getCommand([
+          '--builder', process.env.CTN_BUILDER_NAME ?? 'default',
+          ...bargs,
+          '--metadata-file', bake.getMetadataFilePath()
+        ]);
+        await Exec.exec(buildCmd.command, buildCmd.args, {
+          cwd: fixturesDir
+        });
+      })()
+    ).resolves.not.toThrow();
+
+    const metadata = bake.resolveMetadata();
+    expect(metadata).toBeDefined();
+    const buildRefs = bake.resolveRefs(metadata);
+    expect(buildRefs).toBeDefined();
+
+    const history = new History({buildx: buildx});
+    const exportRes = await history.export({
+      refs: buildRefs ?? []
+    });
+
+    expect(exportRes).toBeDefined();
+    expect(exportRes?.dockerbuildFilename).toBeDefined();
+    expect(exportRes?.dockerbuildSize).toBeDefined();
+    expect(fs.existsSync(exportRes?.dockerbuildFilename)).toBe(true);
+  });
+});

__tests__/fixtures/hello-bake.hcl

@@ -0,0 +1,39 @@
+// Copyright 2024 actions-toolkit authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+target "hello" {
+  dockerfile = "hello.Dockerfile"
+}
+
+target "hello-bar" {
+  dockerfile = "hello.Dockerfile"
+  args = {
+    NAME = "bar"
+  }
+}
+
+group "hello-all" {
+  targets = ["hello", "hello-bar"]
+}
+
+target "hello-matrix" {
+  name = "matrix-${name}"
+  matrix = {
+    name = ["bar", "baz", "boo", "far", "faz", "foo", "aaa", "bbb", "ccc", "ddd", "eee", "fff", "ggg", "hhh", "iii", "jjj"]
+  }
+  dockerfile = "hello.Dockerfile"
+  args = {
+    NAME = name
+  }
+}

__tests__/fixtures/hello.Dockerfile

@@ -0,0 +1,20 @@
+# syntax=docker/dockerfile:1
+
+# Copyright 2024 actions-toolkit authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM busybox
+ARG NAME=foo
+ARG TARGETPLATFORM
+RUN echo "Hello $NAME from $TARGETPLATFORM"

jest.config.itg.ts

@@ -14,6 +14,21 @@
  * limitations under the License.
  */
 
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+
+const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'docker-actions-toolkit-'));
+
+process.env = Object.assign({}, process.env, {
+  TEMP: tmpDir,
+  GITHUB_REPOSITORY: 'docker/actions-toolkit',
+  RUNNER_TEMP: path.join(tmpDir, 'runner-temp'),
+  RUNNER_TOOL_CACHE: path.join(tmpDir, 'runner-tool-cache')
+}) as {
+  [key: string]: string;
+};
+
 module.exports = {
   clearMocks: true,
   testEnvironment: 'node',

src/buildx/history.ts

@@ -0,0 +1,163 @@
+/**
+ * Copyright 2024 actions-toolkit authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {ChildProcessByStdio, spawn} from 'child_process';
+import fs from 'fs';
+import {Readable, Writable} from 'node:stream';
+import os from 'os';
+import path from 'path';
+import * as core from '@actions/core';
+
+import {Buildx} from './buildx';
+import {Context} from '../context';
+import {Docker} from '../docker/docker';
+import {Exec} from '../exec';
+import {GitHub} from '../github';
+
+import {ExportRecordOpts, ExportRecordResponse} from '../types/history';
+
+export interface HistoryOpts {
+  buildx?: Buildx;
+}
+
+export class History {
+  private readonly buildx: Buildx;
+
+  private static readonly EXPORT_TOOL_IMAGE: string = 'docker.io/dockereng/export-build:latest';
+
+  constructor(opts?: HistoryOpts) {
+    this.buildx = opts?.buildx || new Buildx();
+  }
+
+  public async export(opts: ExportRecordOpts): Promise<ExportRecordResponse> {
+    if (os.platform() === 'win32') {
+      throw new Error('Exporting a build record is currently not supported on Windows');
+    }
+    if (!(await Docker.isAvailable())) {
+      throw new Error('Docker is required to export a build record');
+    }
+
+    let builderName: string = '';
+    let nodeName: string = '';
+    const refs: Array<string> = [];
+    for (const ref of opts.refs) {
+      const refParts = ref.split('/');
+      if (refParts.length != 3) {
+        throw new Error(`Invalid build ref: ${ref}`);
+      }
+      refs.push(refParts[2]);
+
+      // Set builder name and node name from the first ref if not already set.
+      // We assume all refs are from the same builder and node.
+      if (!builderName) {
+        builderName = refParts[0];
+      }
+      if (!nodeName) {
+        nodeName = refParts[1];
+      }
+    }
+    if (refs.length === 0) {
+      throw new Error('No build refs provided');
+    }
+
+    const outDir = path.join(Context.tmpDir(), 'export');
+    core.info(`exporting build record to ${outDir}`);
+    fs.mkdirSync(outDir, {recursive: true});
+
+    const buildxInFifoPath = Context.tmpName({
+      template: 'buildx-in-XXXXXX.fifo',
+      tmpdir: Context.tmpDir()
+    });
+    await Exec.exec('mkfifo', [buildxInFifoPath]);
+
+    const buildxOutFifoPath = Context.tmpName({
+      template: 'buildx-out-XXXXXX.fifo',
+      tmpdir: Context.tmpDir()
+    });
+    await Exec.exec('mkfifo', [buildxOutFifoPath]);
+
+    const buildxCmd = await this.buildx.getCommand(['--builder', builderName, 'dial-stdio']);
+    const buildxDialStdioProc = History.spawn(buildxCmd.command, buildxCmd.args);
+    fs.createReadStream(buildxInFifoPath).pipe(buildxDialStdioProc.stdin);
+    buildxDialStdioProc.stdout.pipe(fs.createWriteStream(buildxOutFifoPath));
+
+    const tmpDockerbuildFilename = path.join(outDir, 'rec.dockerbuild');
+
+    await new Promise<void>((resolve, reject) => {
+      const ebargs: Array<string> = ['--ref-state-dir=/buildx-refs', `--node=${builderName}/${nodeName}`];
+      for (const ref of refs) {
+        ebargs.push(`--ref=${ref}`);
+      }
+      if (typeof process.getuid === 'function') {
+        ebargs.push(`--uid=${process.getuid()}`);
+      }
+      if (typeof process.getgid === 'function') {
+        ebargs.push(`--gid=${process.getgid()}`);
+      }
+      // prettier-ignore
+      const dockerRunProc = History.spawn('docker', [
+        'run', '--rm', '-i',
+        '-v', `${Buildx.refsDir}:/buildx-refs`,
+        '-v', `${outDir}:/out`,
+        opts.image || History.EXPORT_TOOL_IMAGE,
+        ...ebargs
+      ]);
+      fs.createReadStream(buildxOutFifoPath).pipe(dockerRunProc.stdin);
+      dockerRunProc.stdout.pipe(fs.createWriteStream(buildxInFifoPath));
+      dockerRunProc.on('close', code => {
+        if (code === 0) {
+          if (!fs.existsSync(tmpDockerbuildFilename)) {
+            reject(new Error(`Failed to export build record: ${tmpDockerbuildFilename} not found`));
+          } else {
+            resolve();
+          }
+        } else {
+          reject(new Error(`Process "docker run" exited with code ${code}`));
+        }
+      });
+      dockerRunProc.on('error', err => {
+        core.error(`Error executing buildx dial-stdio: ${err}`);
+        reject(err);
+      });
+    }).catch(err => {
+      throw err;
+    });
+
+    let dockerbuildFilename = `${GitHub.context.repo.owner}~${GitHub.context.repo.repo}~${refs[0].substring(0, 6).toUpperCase()}`;
+    if (refs.length > 1) {
+      dockerbuildFilename += `+${refs.length - 1}`;
+    }
+
+    const dockerbuildPath = path.join(outDir, `${dockerbuildFilename}.dockerbuild`);
+    fs.renameSync(tmpDockerbuildFilename, dockerbuildPath);
+    const dockerbuildStats = fs.statSync(dockerbuildPath);
+
+    return {
+      dockerbuildFilename: dockerbuildPath,
+      dockerbuildSize: dockerbuildStats.size,
+      builderName: builderName,
+      nodeName: nodeName,
+      refs: refs
+    };
+  }
+
+  private static spawn(command: string, args?: ReadonlyArray<string>): ChildProcessByStdio<Writable, Readable, null> {
+    core.info(`[command]${command}${args ? ` ${args.join(' ')}` : ''}`);
+    return spawn(command, args || [], {
+      stdio: ['pipe', 'pipe', 'inherit']
+    });
+  }
+}

src/types/history.ts

@@ -0,0 +1,28 @@
+/**
+ * Copyright 2024 actions-toolkit authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export interface ExportRecordOpts {
+  refs: Array<string>;
+  image?: string;
+}
+
+export interface ExportRecordResponse {
+  dockerbuildFilename: string;
+  dockerbuildSize: number;
+  builderName: string;
+  nodeName: string;
+  refs: Array<string>;
+}