Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • catladder/catladder
1 result
Show changes
Commits on Source (5)
Hide whitespace changes
Inline Side-by-side
Showing
with 420 additions and 331 deletions
CHANGELOG.md
View file @ 186923b2
# [1.13.0](https://git.panter.ch/catladder/catladder/compare/v1.12.0...v1.13.0) (2022-03-15)
### Bug Fixes
* higher limit for cpu ([7a0c02c](https://git.panter.ch/catladder/catladder/commit/7a0c02cdfd8eb6efafb586e6f6c6912a4a24e6ec))
### Features
* yaml for gitlab pipeline instead of js ([64b628f](https://git.panter.ch/catladder/catladder/commit/64b628f854b772844e22b56471281b67436161a6))
# [1.12.0](https://git.panter.ch/catladder/catladder/compare/v1.11.0...v1.12.0) (2022-03-15)
......
pipeline/src/build/base/createBuildJob.ts
View file @ 186923b2
import { merge } from "lodash";
import { Context, getRunnerImage, GitlabJob, GitlabJobDef } from "../..";
import { BASE_RETRY } from "../../defaults";
import { Context, getRunnerImage } from "../..";
import { CatladderJob } from "../../types/jobs";
import { ensureArray } from "../../utils";
import {
APP_BUILD_JOB_NAME,
......@@ -10,36 +10,32 @@ import { getBuildInfo } from "./getBuildInfo";
export const createBuildJob = (
context: Context,
{ script, variables, ...def }: Partial<GitlabJobDef>
): GitlabJob => {
return {
name: APP_BUILD_JOB_NAME,
envMode: "jobPerEnv",
job: merge(
{
stage: "build",
image: getRunnerImage("jobs-default"),
needs: [],
cache: [],
variables: {
...RUNNER_BUILD_RESOURCE_VARIABLES,
...(variables ?? {}),
...context.environment.envVars,
...(context.componentConfig.build.extraVars ?? {}),
},
retry: BASE_RETRY,
interruptible: true,
{ script, variables, ...def }: Partial<CatladderJob>
): CatladderJob => {
return merge(
{
name: APP_BUILD_JOB_NAME,
envMode: "jobPerEnv",
stage: "build",
image: getRunnerImage("jobs-default"),
needs: [],
cache: [],
variables: {
...RUNNER_BUILD_RESOURCE_VARIABLES,
...(variables ?? {}),
...context.environment.envVars,
...(context.componentConfig.build.extraVars ?? {}),
},
script: [
...getBuildInfo(context),
`cd ${context.componentConfig.dir}`,
...(ensureArray(script) ?? []),
],
artifacts: {
paths: [context.componentConfig.dir + "/__build_info.json"],
},
script: [
...getBuildInfo(context),
`cd ${context.componentConfig.dir}`,
...(ensureArray(script) ?? []),
],
artifacts: {
paths: [context.componentConfig.dir + "/__build_info.json"],
},
def
),
};
},
def
);
};
pipeline/src/build/docker.ts
View file @ 186923b2
import { getRunnerImage } from "../runner";
import { GitlabJobDef, Context, GitlabJob } from "../types";
import { merge } from "lodash";
import { getRunnerImage } from "../runner";
import { Context } from "../types";
import { CatladderJob } from "../types/jobs";
const DOCKER_RUNNER_BUILD_VARIABLES = {
KUBERNETES_CPU_REQUEST: "0.5",
......@@ -14,55 +14,53 @@ export const DOCKER_BUILD_JOB_NAME = "🔨 docker";
export const createDockerBuildJob = (
context: Context,
{ script, variables, ...def }: Partial<GitlabJobDef>
): GitlabJob => {
return {
name: DOCKER_BUILD_JOB_NAME,
envMode: "jobPerEnv",
job: merge(
{
stage: "build",
image: getRunnerImage("docker-build"),
interruptible: true,
services: [
{
name: "docker:20-dind", // see see https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27300#note_466755332
command: ["--tls=false"],
},
],
variables: {
...DOCKER_RUNNER_BUILD_VARIABLES,
DOCKER_BUILDKIT: "1", // see https://docs.docker.com/develop/develop-images/build_enhancements/
DOCKERFILE_ADDITIONS:
context.componentConfig.build.docker?.additionsBegin?.join("\n"),
DOCKERFILE_ADDITIONS_END:
context.componentConfig.build.docker?.additionsEnd?.join("\n"),
APP_DIR: context.componentConfig.dir,
DOCKER_HOST: "tcp://0.0.0.0:2375",
DOCKER_TLS_CERTDIR: "",
DOCKER_DIR: ".", // relative to componentdir
IMAGE_TAG: "$CI_COMMIT_SHA",
DOCKER_DRIVER: "overlay2",
{ script, variables, ...def }: Partial<CatladderJob>
): CatladderJob => {
return merge(
{
name: DOCKER_BUILD_JOB_NAME,
envMode: "jobPerEnv",
stage: "build",
image: getRunnerImage("docker-build"),
IMAGE_NAME:
"$CI_REGISTRY_IMAGE/" +
context.environment.shortName +
"/" +
context.componentName,
CACHE_IMAGE: "$CI_REGISTRY_IMAGE/caches/" + context.componentName,
...(variables ?? {}),
services: [
{
name: "docker:20-dind", // see see https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27300#note_466755332
command: ["--tls=false"],
},
script: [
...(script || []),
"docker login --username gitlab-ci-token --password $CI_JOB_TOKEN $CI_REGISTRY",
"docker build --network host --cache-from $CACHE_IMAGE --tag $IMAGE_NAME:$IMAGE_TAG -f $APP_DIR/Dockerfile . --build-arg BUILDKIT_INLINE_CACHE=1", //BUILDKIT_INLINE_CACHE, see https://testdriven.io/blog/faster-ci-builds-with-docker-cache/
"docker push $IMAGE_NAME:$IMAGE_TAG",
"docker tag $IMAGE_NAME:$IMAGE_TAG $CACHE_IMAGE",
"docker push $CACHE_IMAGE",
],
],
variables: {
...DOCKER_RUNNER_BUILD_VARIABLES,
DOCKER_BUILDKIT: "1", // see https://docs.docker.com/develop/develop-images/build_enhancements/
DOCKERFILE_ADDITIONS:
context.componentConfig.build.docker?.additionsBegin?.join("\n"),
DOCKERFILE_ADDITIONS_END:
context.componentConfig.build.docker?.additionsEnd?.join("\n"),
APP_DIR: context.componentConfig.dir,
DOCKER_HOST: "tcp://0.0.0.0:2375",
DOCKER_TLS_CERTDIR: "",
DOCKER_DIR: ".", // relative to componentdir
IMAGE_TAG: "$CI_COMMIT_SHA",
DOCKER_DRIVER: "overlay2",
IMAGE_NAME:
"$CI_REGISTRY_IMAGE/" +
context.environment.shortName +
"/" +
context.componentName,
CACHE_IMAGE: "$CI_REGISTRY_IMAGE/caches/" + context.componentName,
...(variables ?? {}),
},
def
),
};
script: [
...(script || []),
"docker login --username gitlab-ci-token --password $CI_JOB_TOKEN $CI_REGISTRY",
"docker build --network host --cache-from $CACHE_IMAGE --tag $IMAGE_NAME:$IMAGE_TAG -f $APP_DIR/Dockerfile . --build-arg BUILDKIT_INLINE_CACHE=1", //BUILDKIT_INLINE_CACHE, see https://testdriven.io/blog/faster-ci-builds-with-docker-cache/
"docker push $IMAGE_NAME:$IMAGE_TAG",
"docker tag $IMAGE_NAME:$IMAGE_TAG $CACHE_IMAGE",
"docker push $CACHE_IMAGE",
],
},
def
);
};
pipeline/src/build/index.ts
View file @ 186923b2
import { Context } from "../types/context";
import { GitlabJobs } from "../types/gitlab-types";
import { CatladderJob } from "../types/jobs";
import { createNodeJobs, createStorybookJobs, createMeteorJobs } from "./node";
import { BuildConfig } from "./types";
......@@ -8,7 +9,7 @@ export * from "./node";
export type BuildTypes = {
[type in BuildConfig["type"]]: {
jobs: (context: Context) => GitlabJobs;
jobs: (context: Context) => CatladderJob[];
defaults: () => Partial<Extract<BuildConfig, { type: type }>>;
};
};
......
pipeline/src/build/node/buildJob.ts
View file @ 186923b2
import { Context } from "../../types/context";
import { GitlabJob, GitlabJobs } from "../../types/gitlab-types";
import { ensureArray } from "../../utils";
import { APP_BUILD_JOB_NAME } from "../base/constants";
import { createBuildJob } from "../base/createBuildJob";
......@@ -9,8 +8,9 @@ import { isOfBuildType } from "../types";
import { getNextCache, getNodeCache, getYarnCache } from "./cache";
import { NODE_RUNNER_BUILD_VARIABLES } from "./constants";
import { getYarnInstall, getYarnInstallCommand } from "./yarn";
import { CatladderJob } from "../../types/jobs";
export const createNodeBuildJobs = (context: Context): GitlabJobs => {
export const createNodeBuildJobs = (context: Context): CatladderJob[] => {
const buildConfig = context.componentConfig.build;
if (!isOfBuildType(buildConfig, "node", "node-static", "storybook")) {
......@@ -18,7 +18,7 @@ export const createNodeBuildJobs = (context: Context): GitlabJobs => {
}
const yarnInstall = getYarnInstall(context);
const appBuildJob: GitlabJob | null =
const appBuildJob: CatladderJob | null =
buildConfig.buildCommand !== null
? createBuildJob(context, {
variables: {
......
pipeline/src/build/node/index.ts
View file @ 186923b2
import type { Context } from "../../types/context";
import type { GitlabJobs } from "../../types/gitlab-types";
import { CatladderJob } from "../../types/jobs";
import { createNodeBuildJobs } from "./buildJob";
import { createMeteorBuildJobs } from "./meteor";
import { createNodeTestJobs } from "./testJob";
export const createNodeJobs = (context: Context): GitlabJobs => {
export const createNodeJobs = (context: Context): CatladderJob[] => {
return [...createNodeTestJobs(context), ...createNodeBuildJobs(context)];
};
export const createStorybookJobs = (context: Context): GitlabJobs => {
export const createStorybookJobs = (context: Context): CatladderJob[] => {
return [...createNodeBuildJobs(context)];
};
export const createMeteorJobs = (context: Context): GitlabJobs => {
export const createMeteorJobs = (context: Context): CatladderJob[] => {
return [...createNodeTestJobs(context), ...createMeteorBuildJobs(context)];
};
pipeline/src/build/node/meteor.ts
View file @ 186923b2
import { join } from "path";
import { getRunnerImage } from "../../runner";
import { GitlabJobCache } from "../../types";
import type { Context } from "../../types/context";
import type {
GitlabJob,
GitlabJobCache,
GitlabJobs,
} from "../../types/gitlab-types";
import { CatladderJob } from "../../types/jobs";
import { APP_BUILD_JOB_NAME } from "../base/constants";
import { createBuildJob } from "../base/createBuildJob";
......@@ -29,7 +27,7 @@ const getMeteorCache = (context: Context): GitlabJobCache[] => [
],
},
];
export const createMeteorBuildJobs = (context: Context): GitlabJobs => {
export const createMeteorBuildJobs = (context: Context): CatladderJob[] => {
const buildConfig = context.componentConfig.build;
if (!isOfBuildType(buildConfig, "meteor")) {
......@@ -37,7 +35,7 @@ export const createMeteorBuildJobs = (context: Context): GitlabJobs => {
}
const yarnInstall = getYarnInstall(context);
const appBuildJob: GitlabJob | null =
const appBuildJob: CatladderJob | null =
buildConfig.buildCommand !== null
? createBuildJob(context, {
cache: [...getNodeCache(context), ...getMeteorCache(context)],
......
pipeline/src/build/node/testJob.ts
View file @ 186923b2
import { BASE_RETRY } from "../../defaults";
import { Context } from "../../types/context";
import { GitlabJob, GitlabJobDef, GitlabJobs } from "../../types/gitlab-types";
import { CatladderJob } from "../../types/jobs";
import { ensureArray, notNil } from "../../utils";
import { getNodeCache } from "./cache";
import { NODE_RUNNER_BUILD_VARIABLES } from "./constants";
import { getYarnInstall } from "./yarn";
export const createNodeTestJobs = (context: Context): GitlabJobs => {
export const createNodeTestJobs = (context: Context): CatladderJob[] => {
// don't run tests after release
if (context.commitInfo?.trigger === "taggedRelease") {
return [];
......@@ -14,66 +13,58 @@ export const createNodeTestJobs = (context: Context): GitlabJobs => {
const buildConfig = context.componentConfig.build;
const base: Omit<GitlabJobDef, "script"> = {
const base: Omit<CatladderJob, "script" | "name"> = {
variables: {
APP_PATH: context.componentConfig.dir,
...NODE_RUNNER_BUILD_VARIABLES,
...(buildConfig.extraVars ?? {}),
},
stage: "test",
interruptible: true,
needs: [],
retry: BASE_RETRY,
envMode: "none",
};
const yarnInstall = getYarnInstall(context);
const auditJob: GitlabJob | null =
const auditJob: CatladderJob | null =
buildConfig.audit !== false
? {
name: "🛡 audit",
envMode: "none",
job: {
...base,
cache: undefined, // audit does not need yarn install and no cache
script: [
`cd ${context.componentConfig.dir}`,
...(ensureArray(buildConfig.audit?.command) ?? ["yarn audit"]),
],
allow_failure: true,
},
...base,
cache: undefined, // audit does not need yarn install and no cache
script: [
`cd ${context.componentConfig.dir}`,
...(ensureArray(buildConfig.audit?.command) ?? ["yarn audit"]),
],
allow_failure: true,
}
: null;
const lintJob: GitlabJob | null =
const lintJob: CatladderJob | null =
buildConfig.lint !== false
? {
name: "👮 lint",
envMode: "none",
job: {
...base,
cache: getNodeCache(context),
script: [
`cd ${context.componentConfig.dir}`,
...yarnInstall,
...(ensureArray(buildConfig.lint?.command) ?? ["yarn lint"]),
],
},
...base,
cache: getNodeCache(context),
script: [
`cd ${context.componentConfig.dir}`,
...yarnInstall,
...(ensureArray(buildConfig.lint?.command) ?? ["yarn lint"]),
],
}
: null;
const testJob: GitlabJob | null =
const testJob: CatladderJob | null =
buildConfig.test !== false
? {
name: "🧪 test",
envMode: "none",
job: {
...base,
cache: getNodeCache(context),
script: [
`cd ${context.componentConfig.dir}`,
...yarnInstall,
...(ensureArray(buildConfig.test?.command) ?? ["yarn test"]),
],
},
...base,
cache: getNodeCache(context),
script: [
`cd ${context.componentConfig.dir}`,
...yarnInstall,
...(ensureArray(buildConfig.test?.command) ?? ["yarn test"]),
],
}
: null;
return [auditJob, lintJob, testJob].filter(notNil);
......
pipeline/src/build/tests/storybook.test.ts
View file @ 186923b2
......@@ -26,7 +26,8 @@ describe("storybook build", () => {
};
it("creates a pipeline for storybook that does not contain test stages", async () => {
const { image, stages, workflow, ...jobs } = await createChildPipeline(
const { jobs } = await createChildPipeline(
"gitlab",
"mainBranch",
config
);
......@@ -40,7 +41,8 @@ describe("storybook build", () => {
});
it("runs build and renames folder", async () => {
const { image, stages, workflow, ...jobs } = await createChildPipeline(
const { jobs } = await createChildPipeline(
"gitlab",
"mainBranch",
config
);
......
pipeline/src/catladder-gitlab.ts
View file @ 186923b2
import { writeFileSync } from "fs";
import { dump } from "js-yaml";
import { readConfigSync } from "./config";
import { PIPELINE_IMAGE_TAG } from "./constants";
import { createChildPipeline } from "./pipeline";
......@@ -32,11 +33,14 @@ if (trigger) {
if (!config) {
throw new Error("no catladder config found");
}
createChildPipeline(trigger, config).then((mainPipeline) => {
writeFileSync(`__pipeline.yml`, JSON.stringify(mainPipeline, null, 2), {
encoding: "utf-8",
});
});
createChildPipeline("gitlab", trigger, config).then(
({ jobs, ...mainPipeline }) => {
// need to spread out the jobs
writeFileSync(`__pipeline.yml`, dump({ ...jobs, ...mainPipeline }), {
encoding: "utf-8",
});
}
);
} else {
throw new Error(
"no matching trigger: " +
......
pipeline/src/deploy/base.ts
View file @ 186923b2
import { GitlabJob, Context, GitlabJobDef } from "../types";
import { Context } from "../types";
import { CatladderJob } from "../types/jobs";
export const DEPLOY_JOB_NAME = "🚀 Deploy";
export const STOP_JOB_NAME = "🛑 Stop ⚠️";
......@@ -9,9 +10,7 @@ const DEPLOY_RUNNER_VARIABLES = {
KUBERNETES_MEMORY_REQUEST: "200Mi",
KUBERNETES_MEMORY_LIMIT: "500Mi",
};
type JobWithoutScript = Omit<GitlabJob, "job"> & {
job: Omit<GitlabJobDef, "script">;
};
type JobWithoutScript = Omit<CatladderJob, "script">;
export const getBaseDeploymentJob = (context: Context): JobWithoutScript => {
const environment = {
name: context.environment.fullName,
......@@ -45,26 +44,24 @@ export const getBaseDeploymentJob = (context: Context): JobWithoutScript => {
artifacts: false,
},
], // workaround for https://gitlab.com/gitlab-org/gitlab/-/issues/220758
job: {
rules: [
autoDeploy
? {
when: "on_success",
}
: {
when: "manual",
},
],
stage: "deploy",
dependencies: [],
variables: {
...DEPLOY_RUNNER_VARIABLES,
},
environment: {
...environment,
on_stop: STOP_JOB_NAME,
auto_stop_in: autoStop,
},
rules: [
autoDeploy
? {
when: "on_success",
}
: {
when: "manual",
},
],
stage: "deploy",
variables: {
...DEPLOY_RUNNER_VARIABLES,
},
environment: {
...environment,
on_stop: STOP_JOB_NAME,
auto_stop_in: autoStop,
},
};
};
......@@ -79,29 +76,27 @@ export const getBaseDeploymentStopJob = (
return {
name: STOP_JOB_NAME,
envMode: "stagePerEnv", // makes it easier to run manual tasks er env
job: {
needs: [DEPLOY_JOB_NAME],
rules: [
{
if: "$CI_COMMIT_BRANCH =~ /^[0-9]+\\.([0-9]+|x)\\.x$/", // automatic on hotfix branches
when: "on_success",
allow_failure: true,
},
{
when: "manual",
allow_failure: true,
},
],
variables: {
...DEPLOY_RUNNER_VARIABLES,
GIT_STRATEGY: "none",
needs: [DEPLOY_JOB_NAME],
rules: [
{
if: "$CI_COMMIT_BRANCH =~ /^[0-9]+\\.([0-9]+|x)\\.x$/", // automatic on hotfix branches
when: "on_success",
allow_failure: true,
},
stage: "stop",
dependencies: [],
environment: {
...environment,
action: "stop",
{
when: "manual",
allow_failure: true,
},
],
variables: {
...DEPLOY_RUNNER_VARIABLES,
GIT_STRATEGY: "none",
},
stage: "stop",
environment: {
...environment,
action: "stop",
},
};
};
pipeline/src/deploy/index.ts
View file @ 186923b2
import { GitlabJobs } from "../types/gitlab-types";
import { Context } from "../types/context";
import { CatladderJob } from "../types/jobs";
import { createKubernetesDeployJobs } from "./kubernetes";
import { DeployConfig } from "./types";
export * from "./kubernetes";
......@@ -7,7 +7,7 @@ export * from "./types";
export * from "./utils";
export type DeployTypes = {
[type in DeployConfig["type"]]: {
jobs: (context: Context) => GitlabJobs;
jobs: (context: Context) => CatladderJob[];
defaults: () => Partial<Extract<DeployConfig, { type: type }>>;
};
};
......
pipeline/src/deploy/kubernetes/deployJob.ts
View file @ 186923b2
import { GitlabJobs } from "../../types/gitlab-types";
import { Context } from "../../types/context";
import { isOfDeployType } from "../types";
import { getRunnerImage } from "../../runner";
import { getBaseDeploymentJob, getBaseDeploymentStopJob } from "../base";
import { merge } from "lodash";
import { dump } from "js-yaml";
import { createMongodbBaseConfig } from "./mongodb";
import { createCloudsqlBaseConfig } from "./cloudsql";
import { merge } from "lodash";
import { DeployConfigKubernetesValues } from "..";
import { getSecretVarNameForContext } from "../..";
import { getRunnerImage } from "../../runner";
import { Context } from "../../types/context";
import { CatladderJob } from "../../types/jobs";
import { mergeWithMergingArrays } from "../../utils";
import { DeployConfigKubernetesValues } from "..";
import { PartialDeep } from "type-fest";
import { getBaseDeploymentJob, getBaseDeploymentStopJob } from "../base";
import { isOfDeployType } from "../types";
import { createCloudsqlBaseConfig } from "./cloudsql";
import { createMongodbBaseConfig } from "./mongodb";
export const createKubernetesDeployJobs = (context: Context): GitlabJobs => {
export const createKubernetesDeployJobs = (
context: Context
): CatladderJob[] => {
const deployConfig = context.componentConfig.deploy;
if (deployConfig === false) {
return [];
......@@ -91,28 +92,26 @@ export const createKubernetesDeployJobs = (context: Context): GitlabJobs => {
namespace: context.environment.envVars.KUBE_NAMESPACE,
};
const shared = {
job: {
image: getRunnerImage("kubernetes"),
variables: {
...context.environment.envVars,
HELM_EXPERIMENTAL_OCI: "1",
IMAGE_PULL_SECRET: `gitlab-registry-${context.componentName}`,
KUBE_VALUES: dump(kubeValues, {
lineWidth: -1,
quotingType: "'",
forceQuotes: true,
}),
HELM_GITLAB_CHART_NAME: "the-panter-chart",
HELM_ARGS: [
...(deployConfig.debug ? ["--debug"] : []),
...(deployConfig.additionalHelmArgs ?? []),
].join(" "),
COMPONENT_NAME: context.componentName,
BUILD_ID: context.commitInfo?.buildId,
// TODO: unify with docker build stage
IMAGE_NAME: context.environment.shortName + "/" + context.componentName,
IMAGE_TAG: "$CI_COMMIT_SHA",
},
image: getRunnerImage("kubernetes"),
variables: {
...context.environment.envVars,
HELM_EXPERIMENTAL_OCI: "1",
IMAGE_PULL_SECRET: `gitlab-registry-${context.componentName}`,
KUBE_VALUES: dump(kubeValues, {
lineWidth: -1,
quotingType: "'",
forceQuotes: true,
}),
HELM_GITLAB_CHART_NAME: "the-panter-chart",
HELM_ARGS: [
...(deployConfig.debug ? ["--debug"] : []),
...(deployConfig.additionalHelmArgs ?? []),
].join(" "),
COMPONENT_NAME: context.componentName,
BUILD_ID: context.commitInfo?.buildId,
// TODO: unify with docker build stage
IMAGE_NAME: context.environment.shortName + "/" + context.componentName,
IMAGE_TAG: "$CI_COMMIT_SHA",
},
};
......@@ -136,24 +135,20 @@ export const createKubernetesDeployJobs = (context: Context): GitlabJobs => {
];
return [
merge({}, baseDeploymentJob, shared, {
job: {
script: [
...connectContext,
"kubernetesCreateSecret",
"kubernetesDeploy",
"echo deployment successful 😻",
],
environment: {
kubernetes: kubernetesEnvironment,
},
script: [
...connectContext,
"kubernetesCreateSecret",
"kubernetesDeploy",
"echo deployment successful 😻",
],
environment: {
kubernetes: kubernetesEnvironment,
},
}),
merge({}, baseStopJob, shared, {
job: {
script: [...connectContext, "kubernetesDelete"],
environment: {
kubernetes: kubernetesEnvironment,
},
script: [...connectContext, "kubernetesDelete"],
environment: {
kubernetes: kubernetesEnvironment,
},
}),
];
......
pipeline/src/pipeline/createChildPipeline.test.ts
View file @ 186923b2
......@@ -26,7 +26,8 @@ describe("createChildPipeline", () => {
};
it("creates a pipeline for a single app on the main branch", async () => {
const { image, stages, workflow, ...jobs } = await createChildPipeline(
const { image, jobs } = await createChildPipeline(
"gitlab",
"mainBranch",
config
);
......
pipeline/src/pipeline/createChildPipeline.ts
View file @ 186923b2
import { getAllEnvsByTrigger, getAllEnvsInAllComponents } from "../config";
import { createJobs } from "./createJobs";
import { RULES_ALWAYS } from "../rules";
import { getRunnerImage } from "../runner";
import { GitlabPipeline, Pipeline, PipelineJob, PipelineType } from "../types";
import { Config, PipelineTrigger } from "../types/config";
import { GitlabJobDef } from "../types/gitlab-types";
import { createJobs } from "./createJobs";
const baseStages = ["setup", "test", "build", "deploy", "verify", "stop"];
export const createChildPipeline = async (
export const createChildPipeline = async <T extends PipelineType>(
type: T,
trigger: PipelineTrigger,
config: Config
) => {
): Promise<Pipeline<T>> => {
const components = Object.keys(config.components);
// 2. write the triggering pipeline
const jobs = await components.reduce<Promise<Record<string, GitlabJobDef>>>(
const jobs = await components.reduce<Promise<Record<string, PipelineJob<T>>>>(
async (acc, componentName) => {
const envs = getAllEnvsByTrigger(config, componentName, trigger);
return {
...(await acc),
...(await createJobs(envs, config, componentName, trigger)),
...(await createJobs(type, envs, config, componentName, trigger)),
};
},
Promise.resolve({})
......@@ -37,18 +38,19 @@ export const createChildPipeline = async (
],
[]
);
const childPipeline = {
image: getRunnerImage("jobs-default"), // default image
variables: {
FF_USE_FASTZIP: "true",
},
workflow: {
rules: RULES_ALWAYS,
},
stages,
...jobs,
};
return childPipeline as typeof childPipeline & Record<string, GitlabJobDef>;
if (type === "gitlab") {
const pipeline: GitlabPipeline = {
image: getRunnerImage("jobs-default"), // default image
variables: {
FF_USE_FASTZIP: "true",
},
workflow: {
rules: RULES_ALWAYS,
},
stages,
jobs,
};
return pipeline as Pipeline<T>;
}
throw new Error(`${type} is not supported`);
};
pipeline/src/pipeline/createJobs.ts
View file @ 186923b2
......@@ -3,13 +3,15 @@ import { isObject } from "lodash";
import { BUILD_TYPES } from "../build";
import { createContext } from "../context";
import { DEPLOY_TYPES } from "../deploy";
import { PipelineJob, PipelineType } from "../types";
import { Config, PipelineTrigger } from "../types/config";
import { Context, CommitInfo } from "../types/context";
import { GitlabJob, GitlabJobDef, GitlabJobs } from "../types/gitlab-types";
import { CommitInfo, Context } from "../types/context";
import { CatladderJob } from "../types/jobs";
import { notNil } from "../utils";
import { makeGitlabJob } from "./gitlab/makeGitlabJob";
import { getPackageManagerInfo } from "./packageManager";
const createRawJobs = (context: Context): GitlabJobs => {
const createRawJobs = (context: Context): CatladderJob[] => {
if (context.componentConfig.deploy === false) {
return [];
}
......@@ -35,7 +37,7 @@ const getFullReferencedJobName = (
referencedJobName: string,
componentName: string,
env: string,
allRawJobs: GitlabJobs
allRawJobs: CatladderJob[]
) => {
const referencedJob = allRawJobs.find((j) => j.name === referencedJobName);
if (!referencedJob) {
......@@ -47,18 +49,17 @@ const getFullReferencedJobName = (
// replaces references to other jobs with the full name
// the full name contains the componentname and the env name (if any)
const replaceReferences = (
job: GitlabJob,
job: CatladderJob,
componentName: string,
env: string,
allRawJobs: GitlabJobs
): GitlabJobDef => {
const def = job.job;
allRawJobs: CatladderJob[]
): CatladderJob<string> => {
const stage =
job.envMode === "stagePerEnv" ? `${def.stage} ${env}` : def.stage;
job.envMode === "stagePerEnv" ? `${job.stage} ${env}` : job.stage;
return {
...def,
...job,
stage,
needs: def.needs?.map((n) =>
needs: job.needs?.map((n) =>
isObject(n)
? {
job: getFullReferencedJobName(
......@@ -71,25 +72,22 @@ const replaceReferences = (
}
: getFullReferencedJobName(n, componentName, env, allRawJobs)
),
environment: def.environment?.on_stop
environment: job.environment?.on_stop
? {
...def.environment,
...job.environment,
on_stop: getFullReferencedJobName(
def.environment.on_stop,
job.environment.on_stop,
componentName,
env,
allRawJobs
),
}
: def.environment,
dependencies: def.dependencies?.map((n) =>
getFullReferencedJobName(n, componentName, env, allRawJobs)
),
: job.environment,
};
};
// this can be removed once https://gitlab.com/gitlab-org/gitlab/-/issues/220758 is resolved
const addStageNeeds = (jobs: GitlabJobs): GitlabJobs => {
const addStageNeeds = (jobs: CatladderJob[]): CatladderJob[] => {
// when a job defines needsStages, we add these as needs
return jobs.map((job) => {
if (!job.needsStages || job.needsStages.length === 0) {
......@@ -98,9 +96,7 @@ const addStageNeeds = (jobs: GitlabJobs): GitlabJobs => {
const neededJobs = jobs
.map((j) => {
const neededStage = job.needsStages?.find(
(s) => j.job.stage === s.stage
);
const neededStage = job.needsStages?.find((s) => j.stage === s.stage);
if (neededStage) {
return {
job: j.name,
......@@ -111,19 +107,19 @@ const addStageNeeds = (jobs: GitlabJobs): GitlabJobs => {
.filter(notNil);
return {
...job,
job: {
...job.job,
needs: [...(job.job.needs ?? []), ...neededJobs],
},
needs: [...(job.needs ?? []), ...neededJobs],
};
});
};
export const createJobs = async (
export const createJobs = async <T extends PipelineType>(
type: T,
envs: string[],
config: Config,
componentName: string,
trigger: PipelineTrigger
): Promise<Record<string, GitlabJobDef>> => {
): Promise<Record<string, PipelineJob<T>>> => {
const commitInfo: CommitInfo = {
refName: process.env.CI_COMMIT_REF_NAME ?? "unknown",
refSlug: process.env.CI_COMMIT_REF_SLUG ?? "unknown",
......@@ -146,18 +142,31 @@ export const createJobs = async (
);
const jobs = addStageNeeds(createRawJobs(context));
return {
const result = {
...acc,
...jobs.reduce((acc, job) => {
return {
...acc,
[getFullJobName(
job.name,
componentName,
job.envMode !== "none" ? env : undefined
)]: replaceReferences(job, componentName, env, jobs),
};
...jobs.reduce<Record<string, PipelineJob<T>>>((acc, job) => {
const jobWithResolvedReferences = replaceReferences(
job,
componentName,
env,
jobs
);
const jobName = getFullJobName(
job.name,
componentName,
job.envMode !== "none" ? env : undefined
);
if (type === "gitlab") {
return {
...acc,
[jobName]: makeGitlabJob(
jobWithResolvedReferences
) as PipelineJob<T>,
};
}
throw new Error("not supported");
}, {}),
};
return result;
}, {});
};
pipeline/src/pipeline/gitlab/makeGitlabJob.ts 0 → 100644
View file @ 186923b2
import { BASE_RETRY } from "../../defaults";
import { GitlabJobDef } from "../../types";
import { CatladderJob } from "../../types/jobs";
export const makeGitlabJob = ({
envMode,
needsStages,
name,
...rest
}: CatladderJob<string>): GitlabJobDef => {
return {
...rest,
retry: BASE_RETRY,
interruptible: true,
};
};
pipeline/src/types/gitlab-types.ts
View file @ 186923b2
......@@ -56,39 +56,19 @@ export type GitlabJobDef = {
};
};
export const GITLAB_BASE_STAGES = [
"setup",
"test",
"build",
"deploy",
"verify",
"stop",
] as const;
export type GitlabBaseStage = typeof GITLAB_BASE_STAGES[number];
export type GitlabVariables = Record<string, string | undefined>;
export type GitlabJob = {
/**
* the name of the job (without any env or app prefix and suffix)
*/
name: string;
/**
* envMode sets the behavior of the job regarding multiple envs:
* - none: the job does not run per env, but once for all envs
* - jobPerEnv: the job runs once per env
* - stagePerEnv: the job runs once per env and is organized in its own stage. This mproves usability in gitlab, but works the same as `jobPerEnv`
*/
envMode: "jobPerEnv" | "stagePerEnv" | "none";
job: GitlabJobDef;
needsStages?: {
stage: GitlabBaseStage;
artifacts?: boolean;
}[];
};
export type GitlabJobs = GitlabJob[];
/**
* this is not precicily the type of a gitlab-pipeline
* the jobs nee to be merge into the object.
* Problem is, that this type cannot be represented properly with typescript, see https://github.com/microsoft/TypeScript/issues/17867
*/
export type GitlabPipeline = {
image: string;
variables: GitlabVariables;
workflow?: {
rules: GitlabRule[];
};
stages: string[];
jobs: GitlabJobs;
jobs: Record<string, GitlabJobDef>;
};
pipeline/src/types/index.ts
View file @ 186923b2
export * from "./config";
export * from "./gitlab-types";
export * from "./context";
export * from "./pipeline";
pipeline/src/types/jobs.ts 0 → 100644
View file @ 186923b2
import {
Artifacts,
GitlabEnvironment,
GitlabJobCache,
GitlabRule,
GitlabVariables,
Service,
} from "./gitlab-types";
export const BASE_STAGES = [
"setup",
"test",
"build",
"deploy",
"verify",
"stop",
] as const;
export type BaseStage = typeof BASE_STAGES[number];
export type CatladderJob<S = BaseStage> = {
/**
* the name of the job (without any env or app prefix and suffix)
*/
name: string;
/**
* envMode sets the behavior of the job regarding multiple envs:
* - none: the job does not run per env, but once for all envs
* - jobPerEnv: the job runs once per env
* - stagePerEnv: the job runs once per env and is organized in its own stage. This mproves usability in gitlab, but works the same as `jobPerEnv`
*/
envMode: "jobPerEnv" | "stagePerEnv" | "none";
/**
* the stage of the job
*/
stage: S;
/**
* does this require another stage?
*/
/**
* script to run
*/
script: (string | undefined)[];
needsStages?: {
stage: S;
artifacts?: boolean;
}[];
/**
* does this require another job?
*/
needs?: Array<string | { job: string; artifacts: boolean }>;
/**
* cache config, we use here the same shape as gitlab itself
*/
cache?: GitlabJobCache | GitlabJobCache[];
/**
* job artifacts, we also use gitlab shape here
*/
artifacts?: Artifacts;
/**
* additional services, mainly used for docker
*/
services?: Service[];
/**
* image to use
*/
image?: string;
/**
* variables to pass
*/
variables: GitlabVariables;
/**
* whether failures are allowed
*/
allow_failure?: boolean;
environment?: GitlabEnvironment;
rules?: GitlabRule[];
};