feat: separate test pipeline

dcda17fd · Marco Wettstein · d48bdd75 · dcda17fd · dcda17fd · dcda17fd
Commit dcda17fd authored Nov 08, 2021 by Marco Wettstein
--- a/dockerfiles/pipeline/package.json
+++ b/dockerfiles/pipeline/package.json
@@ -3,11 +3,13 @@
  "version": "1.0.0",
  "devDependencies": {
    "@types/lodash": "^4.14.172",
-    "@types/node": "^16.6.1"
+    "@types/node": "^16.6.1",
+    "@types/yargs": "^17.0.5"
  },
  "dependencies": {
-    "lodash": "^4.17.21",
    "@gitbeaker/node": "^32.1.2",
-    "yaml": "^1.10.2"
+    "lodash": "^4.17.21",
+    "yaml": "^1.10.2",
+    "yargs": "^17.2.1"
  }
 }
--- a/dockerfiles/pipeline/scripts/monorepoGenerate.ts
+++ b/dockerfiles/pipeline/scripts/monorepoGenerate.ts
@@ -3,7 +3,11 @@
 import { readdirSync, readFileSync, lstatSync, existsSync } from "fs";
 import { join } from "path";
 import { parse } from "yaml";
+import yargs from "yargs";

+import { hideBin } from "yargs/helpers";
+type Type = "test" | "deploy";
+const args = yargs(hideBin(process.argv)).argv as Record<string, unknown>;
 const thisGitlabCi = parse(
  readFileSync(".gitlab-ci.yml", { encoding: "utf-8" })
 );
@@ -16,42 +20,51 @@ const subcis = readdirSync(".")
  }))
  .filter(({ ci }) => existsSync(ci));

-const subpipeline = subcis.reduce((acc, { dirname, ci }) => {
-  const changes = [dirname + "/**/*"];
-
-  return {
-    ...acc,
-    ["deploy " + dirname]: {
-      variables: {
-        // merge
-        ...thisGitlabCi.variables,
-        APP_DIR: dirname,
-        VALUES_DIR: dirname,
-        COMPONENT_NAME: dirname,
+const generateSubPipelein = (type: Type) => {
+  return subcis.reduce((acc, { dirname, ci }) => {
+    const changes = [dirname + "/**/*"];
+    const rules = [
+      // same as rules "always", but with `changes` to only trigger changed branches
+      { if: "$CI_COMMIT_TAG" },
+      {
+        if: "$CI_COMMIT_MESSAGE =~ /^chore(release).*/",
+        when: "never",
+      },
+      { if: "$CI_COMMIT_BRANCH =~ /^[0-9]+.([0-9]+|x).x$/" },
+      {
+        if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH",
+        changes,
+      },
+      // add it manually when no changes
+      {
+        if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH",
+        when: "manual",
      },
-      stage: "deploy",
-      rules: [
-        // same as rules "always"
-        { if: "$CI_COMMIT_TAG" },
-        {
-          if: "$CI_COMMIT_MESSAGE =~ /^chore(release).*/",
-          when: "never",
+      { if: "$CI_MERGE_REQUEST_ID", changes },
+      // add it manually when no changes
+      { if: "$CI_MERGE_REQUEST_ID", when: "manual" },
+    ];
+
+    return {
+      ...acc,
+      [type + " " + dirname]: {
+        variables: {
+          // merge
+          ...thisGitlabCi.variables,
+          APP_DIR: dirname,
+          VALUES_DIR: dirname,
+          COMPONENT_NAME: dirname,
+          [type.toUpperCase() + "_ONLY"]: "1",
        },
-        { if: "$CI_COMMIT_BRANCH =~ /^[0-9]+.([0-9]+|x).x$/" },
-        {
-          if: "$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH",
+        stage: type,
+        rules: rules,
+        trigger: {
+          include: ci,
+          strategy: "depend",
        },
-
-        { if: "$CI_MERGE_REQUEST_ID", changes },
-        // if no changes, schedule it as manual
-        { if: "$CI_MERGE_REQUEST_ID", when: "manual" },
-      ],
-      trigger: {
-        include: ci,
-        strategy: "depend",
      },
-    },
-  };
-}, {});
+    };
+  }, {});
+};

-console.log(JSON.stringify(subpipeline, null, 2));
+console.log(JSON.stringify(generateSubPipelein(args.t as Type), null, 2));
--- a/includes/docker-build.yml
+++ b/includes/docker-build.yml
@@ -57,7 +57,7 @@ variables:
 .docker-build:
  stage: docker-build
  extends:
-    - .rules-always
+    - .rules-build-and-deploy
    - .docker-build-app-base
  needs: ["app-build"]


--- a/includes/env.yml
+++ b/includes/env.yml
@@ -20,7 +20,6 @@
    MONGODB_REPLICAS: 1
    MONGODB_STORAGE_CLASS: "standard"
    MONGODB_BACKUP_ENABLED: "false"
-    DEPLOY_SSL_ISSUER: letsencrypt-staging
    KUBE_APP_NAME: ${COMPONENT_NAME}-${CI_COMMIT_REF_SLUG}

  environment:
@@ -35,14 +34,12 @@
    MONGODB_REPLICAS: 1
    MONGODB_BACKUP_ENABLED: "false"
    MONGODB_STORAGE_CLASS: "standard"
-    DEPLOY_SSL_ISSUER: letsencrypt-staging
    ENV_SHORT: dev

 .env-stage:
  extends: .env-base
  variables:
    DEFAULT_VALUE_application_replicas: 2
-    DEPLOY_SSL_ISSUER: letsencrypt-staging
    ENV_SHORT: stage

 .env-prod:

--- a/includes/rules.yml
+++ b/includes/rules.yml
@@ -7,8 +7,21 @@
    - if: $CI_COMMIT_BRANCH =~ /^[0-9]+\.([0-9]+|x)\.x$/ # hotfix branches
    - if: $CI_MERGE_REQUEST_ID

-.rules-always-but-not-on-tags:
+.rules-build-and-deploy:
  rules:
+    - if: $TEST_ONLY
+      when: never
+    - if: $CI_COMMIT_TAG
+    - if: $CI_COMMIT_MESSAGE =~ /^chore\(release\).*/ # not after semantic release commit
+      when: never
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: $CI_COMMIT_BRANCH =~ /^[0-9]+\.([0-9]+|x)\.x$/ # hotfix branches
+    - if: $CI_MERGE_REQUEST_ID
+
+.rules-tests:
+  rules:
+    - if: $DEPLOY_ONLY
+      when: never
    - if: $CI_COMMIT_MESSAGE =~ /^chore\(release\).*/ # not after semantic release commit
      when: never
    - if: $CI_COMMIT_TAG
@@ -17,8 +30,16 @@
    - if: $CI_COMMIT_BRANCH =~ /^[0-9]+\.([0-9]+|x)\.x$/ # hotfix branches
    - if: $CI_MERGE_REQUEST_ID

+.rules-review-deploy:
+  rules:
+    - if: $TEST_ONLY
+      when: never
+    - if: $CI_MERGE_REQUEST_ID
+
 .rules-dev-deploy:
  rules:
+    - if: $TEST_ONLY
+      when: never
    - if: $CI_COMMIT_TAG # after releases
      when: on_success
    - if: $CI_COMMIT_MESSAGE =~ /^chore\(release\).*/ # not after semaitc release commit
@@ -30,6 +51,8 @@

 .rules-stage-deploy:
  rules:
+    - if: $TEST_ONLY
+      when: never
    - if: $CI_COMMIT_TAG && $CI_COMMIT_MESSAGE =~ /^chore\(maintenance\).*/ # d
      when: manual # do not do automatic release after a hotfix / maintenance release
    - if: $CI_COMMIT_TAG && $STAGING_ENABLED == "true"
@@ -37,6 +60,8 @@

 .rules-prod-deploy:
  rules:
+    - if: $TEST_ONLY
+      when: never
    - if: $CI_COMMIT_TAG && $CI_COMMIT_MESSAGE =~ /^chore\(maintenance\).*/ # d
      when: manual # do not do automatic release after a hotfix / maintenance release
    - if: $CI_COMMIT_TAG && $STAGING_ENABLED != "true" # do only automatic release when staging is not enabled
@@ -46,6 +71,8 @@

 .rules-stoybook:
  rules:
+    - if: $DEPLOY_ONLY
+      when: never
    - if: $STORYBOOK != "true"
      when: never
    - if: $CI_COMMIT_TAG

--- a/includes/setup.yml
+++ b/includes/setup.yml
@@ -3,7 +3,7 @@
  interruptible: true
  extends:
    - .retry-default
-    - .rules-always
+    - .rules-build-and-deploy
  script:
    - echo "triggered by $CI_PIPELINE_SOURCE"
    - BUILD_TAG="$(git describe --tags || git rev-parse HEAD)"

--- a/includes/verify.yml
+++ b/includes/verify.yml
@@ -47,6 +47,8 @@
    - .verify-base
    - .env-review
  rules:
+    - if: $TEST_ONLY
+      when: never
    - if: $CI_MERGE_REQUEST_ID
  needs: ["review-deploy", "setup-build-info"]

@@ -57,6 +59,8 @@
    - .env-dev
  needs: ["dev-deploy", "setup-build-info"]
  rules:
+    - if: $TEST_ONLY
+      when: never
    - if: $CI_COMMIT_TAG # after releases
    - if: $CI_COMMIT_MESSAGE =~ /^chore\(release\).*/ # not after semantic release commit
      when: never
@@ -69,6 +73,8 @@
    - .env-stage
  needs: ["stage-deploy", "setup-build-info"]
  rules:
+    - if: $TEST_ONLY
+      when: never
    - if: $STAGING_ENABLED == "true" && $CI_COMMIT_TAG

 .verify-prod:
@@ -78,4 +84,6 @@
    - .env-prod
  needs: ["prod-deploy", "setup-build-info"]
  rules:
+    - if: $TEST_ONLY
+      when: never
    - if: $CI_COMMIT_TAG
--- a/monorepo.yml
+++ b/monorepo.yml
@@ -17,31 +17,47 @@ variables:
 image: git.panter.ch:5001/catladder/gitlab-ci/pipeline:$PIPELINE_IMAGE_TAG

 stages:
-  - build
-  - childpipelines
+  - setup
+  - test
+  - deploy
  - verify
  - create-release

-build:
-  stage: build
+setup:
+  stage: setup
  extends:
-    - .rules-always
+    - .rules-build-and-deploy
  script:
-    - monorepoGenerate.ts > generated-config.yml
+    - monorepoGenerate.ts -t=test
+    - monorepoGenerate.ts -t=test > generated-test-config.yml
+    - monorepoGenerate.ts -t=deploy
+    - monorepoGenerate.ts -t=deploy > generated-deploy-config.yml
  artifacts:
    paths:
-      - generated-config.yml
+      - generated-test-config.yml
+      - generated-deploy-config.yml

-child-pipeline:
-  stage: childpipelines
+test:
+  stage: test
+  needs: [setup]
  extends:
    - .rules-always
+  trigger:
+    strategy: depend
+    include:
+      - artifact: generated-test-config.yml
+        job: setup

+deploy:
+  stage: deploy
+  needs: [setup]
+  extends:
+    - .rules-always
  trigger:
    strategy: depend
    include:
-      - artifact: generated-config.yml
-        job: build
+      - artifact: generated-deploy-config.yml
+        job: setup

 create-release:
  extends: .create-release
--- a/panter-kubernetes-base.yml
+++ b/panter-kubernetes-base.yml
@@ -84,7 +84,7 @@ image: git.panter.ch:5001/catladder/gitlab-ci/pipeline:$PIPELINE_IMAGE_TAG
  needs: []
  extends:
    - .retry-default
-    - .rules-always-but-not-on-tags
+    - .rules-tests

 .test:
  extends: .test-base
@@ -101,8 +101,7 @@ image: git.panter.ch:5001/catladder/gitlab-ci/pipeline:$PIPELINE_IMAGE_TAG
  stage: test
  needs: []
  extends:
-    - .retry-default
-    - .rules-always # while lint and test is useless after tagging, running an audit on a tag might be useful
+    - .test-base

 .lint:
  extends: .lint-base
@@ -119,7 +118,7 @@ image: git.panter.ch:5001/catladder/gitlab-ci/pipeline:$PIPELINE_IMAGE_TAG
  interruptible: true
  extends:
    - .retry-default
-    - .rules-always
+    - .rules-build-and-deploy
  needs: ["setup-build-info"]

 .app-build:
@@ -133,10 +132,10 @@ image: git.panter.ch:5001/catladder/gitlab-ci/pipeline:$PIPELINE_IMAGE_TAG

 .review-deploy:
  stage: deploy
-  rules:
-    - if: $CI_MERGE_REQUEST_ID
+
  extends:
    - .retry-default
+    - .rules-review-deploy
    - .deploy-to-kubernetes-review

 .review-stop:
@@ -146,6 +145,8 @@ image: git.panter.ch:5001/catladder/gitlab-ci/pipeline:$PIPELINE_IMAGE_TAG
    - .retry-default
    - .kubernetes-review-stop
  rules:
+    - if: $TEST_ONLY
+      when: never
    - if: $CI_COMMIT_BRANCH =~ /^[0-9]+\.([0-9]+|x)\.x$/ # automatic on hotfix branches
      when: on_success
      allow_failure: true

--- a/static-js-kubernetes.yml
+++ b/static-js-kubernetes.yml
@@ -17,6 +17,8 @@ docker-build:
 app-build-prod:
  extends: .app-build-node
  rules:
+    - if: $TEST_ONLY
+      when: never
    - if: $PROD_BUILD_COMMAND && $CI_COMMIT_TAG
  script:
    - $PROD_BUILD_COMMAND
@@ -24,6 +26,8 @@ app-build-prod:
 docker-build-prod:
  extends: docker-build
  rules:
+    - if: $TEST_ONLY
+      when: never
    - if: $PROD_BUILD_COMMAND && $CI_COMMIT_TAG
  needs:
    - job: app-build-prod

--- a/yarn.lock
+++ b/yarn.lock
@@ -678,6 +678,13 @@
  dependencies:
    "@types/yargs-parser" "*"

+"@types/yargs@^17.0.5":
+  version "17.0.5"
+  resolved "https://registry.yarnpkg.com/@types/yargs/-/yargs-17.0.5.tgz#1e7e59a88420872875842352b73618f5e77e835f"
+  integrity sha512-4HNq144yhaVjJs+ON6A07NEoi9Hh0Rhl/jI9Nt/l/YRjt+T6St/QK3meFARWZ8IgkzoD1LC0PdTdJenlQQi2WQ==
+  dependencies:
+    "@types/yargs-parser" "*"
+
 abab@^2.0.3, abab@^2.0.5:
  version "2.0.5"
  resolved "https://registry.yarnpkg.com/abab/-/abab-2.0.5.tgz#c0b678fb32d60fc1219c784d6a826fe385aeb79a"
@@ -2825,3 +2832,16 @@ yargs@^16.0.3:
    string-width "^4.2.0"
    y18n "^5.0.5"
    yargs-parser "^20.2.2"
+
+yargs@^17.2.1:
+  version "17.2.1"
+  resolved "https://registry.yarnpkg.com/yargs/-/yargs-17.2.1.tgz#e2c95b9796a0e1f7f3bf4427863b42e0418191ea"
+  integrity sha512-XfR8du6ua4K6uLGm5S6fA+FIJom/MdJcFNVY8geLlp2v8GYbOXD4EB1tPNZsRn4vBzKGMgb5DRZMeWuFc2GO8Q==
+  dependencies:
+    cliui "^7.0.2"
+    escalade "^3.1.1"
+    get-caller-file "^2.0.5"
+    require-directory "^2.1.1"
+    string-width "^4.2.0"
+    y18n "^5.0.5"
+    yargs-parser "^20.2.2"