Merge pull request #78 from crazy-max/bake-remote-def

bake: support remote definition when parsing

.github/workflows/e2e.yml

@@ -10,15 +10,54 @@ on:
       - '.github/*-releases.json'
 
 jobs:
+  prepare:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.tests.outputs.matrix }}
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v3
+      -
+        name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16
+          cache: 'yarn'
+      -
+        name: Install
+        run: yarn install
+      -
+        name: Create matrix
+        id: tests
+        run: |
+          declare -a tests
+          for test in $(yarn run test:e2e-list); do
+            tests+=("${test#$(pwd)/__tests__/}")
+          done
+          echo "matrix=$(echo ${tests[@]} | jq -cR 'split(" ")')" >>${GITHUB_OUTPUT}
+      -
+        name: Show matrix
+        run: |
+          echo ${{ steps.tests.outputs.matrix }}
+
   test:
     runs-on: ${{ matrix.os }}
+    needs:
+      - prepare
     strategy:
       fail-fast: false
       matrix:
+        test: ${{ fromJson(needs.prepare.outputs.matrix) }}
         os:
           - ubuntu-latest
           - macos-latest
           - windows-latest
+        exclude:
+          - os: macos-latest
+            test: buildx/bake.test.e2e.ts
+          - os: windows-latest
+            test: buildx/bake.test.e2e.ts
     steps:
       -
         name: Checkout
@@ -34,7 +73,7 @@ jobs:
         run: yarn install
       -
         name: Test
-        run: yarn test-coverage:e2e --coverageDirectory=./coverage
+        run: yarn test-coverage:e2e --runTestsByPath __tests__/${{ matrix.test }} --coverageDirectory=./coverage
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       -

__tests__/buildx/bake.test.e2e.ts

@@ -0,0 +1,43 @@
+/**
+ * Copyright 2023 actions-toolkit authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {beforeEach, describe, expect, jest, test} from '@jest/globals';
+import * as fs from 'fs';
+import * as path from 'path';
+
+import {Bake} from '../../src/buildx/bake';
+import {BakeDefinition} from '../../src/types/bake';
+
+const fixturesDir = path.join(__dirname, '..', 'fixtures');
+
+beforeEach(() => {
+  jest.clearAllMocks();
+});
+
+describe('parseDefinitions', () => {
+  // prettier-ignore
+  test.each([
+    [
+      ['https://github.com/docker/buildx.git#v0.10.4'],
+      ['binaries-cross'],
+      path.join(fixturesDir, 'bake-buildx-0.10.4-binaries-cross.json')
+    ]
+  ])('given %p', async (sources: string[], targets: string[], out: string) => {
+    const bake = new Bake();
+    const expectedDef = <BakeDefinition>JSON.parse(fs.readFileSync(out, {encoding: 'utf-8'}).trim())
+    expect(await bake.parseDefinitions(sources, targets)).toEqual(expectedDef);
+  });
+});

__tests__/buildx/bake.test.ts

@@ -15,6 +15,7 @@
  */
 
 import {beforeEach, describe, expect, jest, test} from '@jest/globals';
+import * as fs from 'fs';
 import * as path from 'path';
 
 import {Bake} from '../../src/buildx/bake';
@@ -32,65 +33,12 @@ describe('parseDefinitions', () => {
     [
       [path.join(fixturesDir, 'bake.hcl')],
       ['validate'],
-      {
-        "group": {
-          "default": {
-            "targets": [
-              "validate"
-            ]
-          },
-          "validate": {
-            "targets": [
-              "lint",
-              "validate-vendor",
-              "validate-docs"
-            ]
-          }
-        },
-        "target": {
-          "lint": {
-            "context": ".",
-            "dockerfile": "./hack/dockerfiles/lint.Dockerfile",
-            "args": {
-              "BUILDKIT_CONTEXT_KEEP_GIT_DIR": "1",
-              "GO_VERSION": "1.20"
-            },
-            "output": [
-              "type=cacheonly"
-            ]
-          },
-          "validate-docs": {
-            "context": ".",
-            "dockerfile": "./hack/dockerfiles/docs.Dockerfile",
-            "args": {
-              "BUILDKIT_CONTEXT_KEEP_GIT_DIR": "1",
-              "BUILDX_EXPERIMENTAL": "1",
-              "FORMATS": "md",
-              "GO_VERSION": "1.20"
-            },
-            "target": "validate",
-            "output": [
-              "type=cacheonly"
-            ]
-          },
-          "validate-vendor": {
-            "context": ".",
-            "dockerfile": "./hack/dockerfiles/vendor.Dockerfile",
-            "args": {
-              "BUILDKIT_CONTEXT_KEEP_GIT_DIR": "1",
-              "GO_VERSION": "1.20"
-            },
-            "target": "validate",
-            "output": [
-              "type=cacheonly"
-            ]
-          }
-        }
-      }
+      path.join(fixturesDir, 'bake-validate.json')
     ]
-  ])('given %p', async (files, targets, expected: BakeDefinition) => {
+  ])('given %p', async (sources: string[], targets: string[], out: string) => {
     const bake = new Bake();
-    expect(await bake.parseDefinitions(files, targets)).toEqual(expected);
+    const expectedDef = <BakeDefinition>JSON.parse(fs.readFileSync(out, {encoding: 'utf-8'}).trim())
+    expect(await bake.parseDefinitions(sources, targets)).toEqual(expectedDef);
   });
 });
 

__tests__/fixtures/bake-buildx-0.10.4-binaries-cross.json

@@ -0,0 +1,36 @@
+{
+  "group": {
+    "default": {
+      "targets": [
+        "binaries-cross"
+      ]
+    }
+  },
+  "target": {
+    "binaries-cross": {
+      "context": "https://github.com/docker/buildx.git#v0.10.4",
+      "dockerfile": "Dockerfile",
+      "args": {
+        "BUILDKIT_CONTEXT_KEEP_GIT_DIR": "1",
+        "GO_VERSION": "1.19"
+      },
+      "target": "binaries",
+      "platforms": [
+        "darwin/amd64",
+        "darwin/arm64",
+        "linux/amd64",
+        "linux/arm/v6",
+        "linux/arm/v7",
+        "linux/arm64",
+        "linux/ppc64le",
+        "linux/riscv64",
+        "linux/s390x",
+        "windows/amd64",
+        "windows/arm64"
+      ],
+      "output": [
+        "./bin/build"
+      ]
+    }
+  }
+}

__tests__/fixtures/bake-validate.json

@@ -0,0 +1,55 @@
+{
+  "group": {
+    "default": {
+      "targets": [
+        "validate"
+      ]
+    },
+    "validate": {
+      "targets": [
+        "lint",
+        "validate-vendor",
+        "validate-docs"
+      ]
+    }
+  },
+  "target": {
+    "lint": {
+      "context": ".",
+      "dockerfile": "./hack/dockerfiles/lint.Dockerfile",
+      "args": {
+        "BUILDKIT_CONTEXT_KEEP_GIT_DIR": "1",
+        "GO_VERSION": "1.20"
+      },
+      "output": [
+        "type=cacheonly"
+      ]
+    },
+    "validate-docs": {
+      "context": ".",
+      "dockerfile": "./hack/dockerfiles/docs.Dockerfile",
+      "args": {
+        "BUILDKIT_CONTEXT_KEEP_GIT_DIR": "1",
+        "BUILDX_EXPERIMENTAL": "1",
+        "FORMATS": "md",
+        "GO_VERSION": "1.20"
+      },
+      "target": "validate",
+      "output": [
+        "type=cacheonly"
+      ]
+    },
+    "validate-vendor": {
+      "context": ".",
+      "dockerfile": "./hack/dockerfiles/vendor.Dockerfile",
+      "args": {
+        "BUILDKIT_CONTEXT_KEEP_GIT_DIR": "1",
+        "GO_VERSION": "1.20"
+      },
+      "target": "validate",
+      "output": [
+        "type=cacheonly"
+      ]
+    }
+  }
+}

__tests__/util.test.ts

@@ -194,13 +194,29 @@ describe('asyncForEach', () => {
   });
 });
 
-describe('isValidUrl', () => {
+describe('isValidURL', () => {
   test.each([
     ['https://github.com/docker/buildx.git', true],
     ['https://github.com/docker/buildx.git#refs/pull/648/head', true],
+    ['git@github.com:moby/buildkit.git', false],
+    ['git://github.com/user/repo.git', false],
+    ['github.com/moby/buildkit.git#main', false],
     ['v0.4.1', false]
   ])('given %p', async (url, expected) => {
-    expect(Util.isValidUrl(url)).toEqual(expected);
+    expect(Util.isValidURL(url)).toEqual(expected);
+  });
+});
+
+describe('isValidRef', () => {
+  test.each([
+    ['https://github.com/docker/buildx.git', true],
+    ['https://github.com/docker/buildx.git#refs/pull/648/head', true],
+    ['git@github.com:moby/buildkit.git', true],
+    ['git://github.com/user/repo.git', true],
+    ['github.com/moby/buildkit.git#main', true],
+    ['v0.4.1', false]
+  ])('given %p', async (url, expected) => {
+    expect(Util.isValidRef(url)).toEqual(expected);
   });
 });
 

package.json

@@ -12,6 +12,7 @@
     "prettier:fix": "prettier --write \"./**/*.ts\"",
     "test": "jest",
     "test:e2e": "jest -c jest.config.e2e.ts --runInBand --detectOpenHandles",
+    "test:e2e-list": "jest -c jest.config.e2e.ts --listTests",
     "test-coverage": "jest --coverage",
     "test-coverage:e2e": "jest --coverage -c jest.config.e2e.ts --runInBand --detectOpenHandles"
   },

src/buildx/bake.ts

@@ -17,6 +17,7 @@
 import {Buildx} from './buildx';
 import {Exec} from '../exec';
 import {Inputs} from './inputs';
+import {Util} from '../util';
 
 import {BakeDefinition} from '../types/bake';
 
@@ -31,13 +32,29 @@ export class Bake {
     this.buildx = opts?.buildx || new Buildx();
   }
 
-  public async parseDefinitions(files: Array<string>, targets: Array<string>): Promise<BakeDefinition> {
+  public async parseDefinitions(sources: Array<string>, targets: Array<string>): Promise<BakeDefinition> {
     const args = ['bake'];
-    if (files) {
-      for (const file of files) {
-        args.push('--file', file);
+
+    let remoteDef;
+    const files: Array<string> = [];
+    if (sources) {
+      for (const source of sources) {
+        if (!Util.isValidRef(source)) {
+          files.push(source);
+          continue;
+        }
+        if (remoteDef) {
+          throw new Error(`Only one remote bake definition is allowed`);
+        }
+        remoteDef = source;
       }
     }
+    if (remoteDef) {
+      args.push(remoteDef);
+    }
+    for (const file of files) {
+      args.push('--file', file);
+    }
 
     const printCmd = await this.buildx.getCommand([...args, '--print', ...targets]);
     return await Exec.getExecOutput(printCmd.command, printCmd.args, {

src/util.ts

@@ -65,13 +65,26 @@ export class Util {
     }
   }
 
-  public static isValidUrl(url: string): boolean {
+  public static isValidURL(urlStr: string): boolean {
+    let url;
     try {
-      new URL(url);
+      url = new URL(urlStr);
     } catch (e) {
       return false;
     }
-    return true;
+    return url.protocol === 'http:' || url.protocol === 'https:';
+  }
+
+  public static isValidRef(refStr: string): boolean {
+    if (Util.isValidURL(refStr)) {
+      return true;
+    }
+    for (const prefix of ['git://', 'github.com/', 'git@']) {
+      if (refStr.startsWith(prefix)) {
+        return true;
+      }
+    }
+    return false;
   }
 
   public static async powershellCommand(script: string, params?: Record<string, string>) {