icecheng/freeleaps-ops
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor(SourceFetcher, ServiceLanguage, DependenciesResolver): improve workspace handling, standardize language naming, and streamline dependency resolution

Browse Source 
Signed-off-by: 孙振宇 <>
This commit is contained in:
孙振宇 2025-02-04 10:23:25 +08:00
parent 2d925213e3
commit adfc71ea22
8 changed files with 283 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
first-class-pipeline/resources/com/freeleaps/devops/builtins/commitlint/default.js Normal file
View File

@ -0,0 +1,30 @@
module.exports = {
extends: ["@commitlint/config-angular"], // Extends with the Angular commitlint configuration
rules: {
"type-enum": [
2,
"always",
[
"feat",
"fix",
"docs",
"style",
"refactor",
"perf",
"test",
"build",
"ci",
"chore",
"revert", // Add or remove types as needed
],
],
"type-case": [2, "always", "lower-case"], // Type must be in lower case
"type-empty": [2, "never"], // Type must not be empty
"scope-empty": [2, "never"], // Scope must not be empty
"scope-case": [2, "always", "lower-case"], // Scope must be in lower case
"subject-empty": [2, "never"], // Subject must not be empty
"subject-case": [2, "never", []], // Subject must be in sentence case
"subject-full-stop": [2, "never", "."], // Subject must not end with a period
"header-max-length": [2, "always", 100], // Header must not exceed 100 characters
},
};

31
first-class-pipeline/src/com/freeleaps/devops/ChangedComponentsDetector.groovy Normal file
View File

@ -0,0 +1,31 @@
package com.freeleaps.devops
class ChangedComponentsDetector {
def steps
def workspace
ChangedComponentsDetector(steps) {
this.steps = steps
}
def detect(workspace, components) {
def changedComponents = [] as Set
dir(workspace) {
// using git command to get changed files list
def changedFiles = sh(script: 'git diff --name-only HEAD~1 HEAD', returnStdout: true)
.trim()
.split('\n')
changedFiles.each { file ->
components.each { component ->
if (file.startsWith("${component}/")) {
changedComponents.add(component)
}
}
}
}
return changedComponents.toList()
}
}

28
first-class-pipeline/src/com/freeleaps/devops/CommitMessageLinter.groovy Normal file
View File

@ -0,0 +1,28 @@
package com.freeleaps.devops
class CommitMessageLinter {
def steps
private defaultRule = 'com/freeleaps/devops/builtins/commitlint/default.js'
CommitMessageLinter(steps) {
this.steps = steps
}
def lint(configurations) {
def rules = steps.libraryResource 'com/freeleaps/devops/builtins/commitlint/default.js'
steps.log.info "Check if there has custom commit lint rules specified..."
if (configurations.commitLintRules != null && !configurations.commitLintRules.isEmpty()) {
steps.log.info "Custom commit lint rules found, using custom rules files: ${configurations.commitLintRules}"
rules = configurations.commitLintRules
} else {
steps.log.info "No custom commit lint rules found, using built-in rules at: ${defaultRules}"
steps.sh "echo ${rules} > .commitlintrc.js"
rules = '.commitlintrc.js'
}
steps.log.info "Linting commit messages from HEAD..."
steps.sh "commitlint --verbose -g ${rules} -f HEAD^"
}
}

19
first-class-pipeline/src/com/freeleaps/devops/DependenciesResolver.groovy
View File

@ -43,19 +43,8 @@ class DependenciesResolver {
switch (mgr) {
case DependenciesManager.PIP:
if (configurations.pipRequirementsFile == null || configurations.pipRequirementsFile.isEmpty()) {
steps.error("pipRequirementsFile is required when using PIP as dependencies manager")
}
def requirementsFile = configurations.pipRequirementsFile
if (cachingEnabled) {
steps.cache(maxCacheSize: 512, caches: [[$class: 'ArbitraryFileCache', excludes: '', includes: '**/*', path: '.pip-cache']]) {
steps.sh "pip install -r ${requirementsFile} --cache-dir .pip-cache"
}
} else {
steps.sh "pip install -r ${requirementsFile}"
}
steps.log.warn "Python project no need to resolving dependencies, skipping..."
break
case DependenciesManager.NPM:
if (configurations.npmPackageJsonFile == null || configurations.npmPackageJsonFile.isEmpty()) {
steps.error("npmPackageJsonFile is required when using NPM as dependencies manager")
@ -70,6 +59,8 @@ class DependenciesResolver {
} else {
steps.sh "npm install"
}
break
case DependenciesManager.YARN:
if (configurations.yarnPackageJsonFile == null || configurations.yarnPackageJsonFile.isEmpty()) {
steps.error("yarnPackageJsonFile is required when using YARN as dependencies manager")
@ -84,6 +75,8 @@ class DependenciesResolver {
} else {
steps.sh "yarn install"
}
break
default:
steps.error("Unsupported dependencies manager")
}

4
first-class-pipeline/src/com/freeleaps/devops/SourceFetcher.groovy
View File

@ -16,6 +16,10 @@ class SourceFetcher {
steps.error("serviceGitBranch is required")
}
steps.env.workspace = "workspace"
dir(steps.env.workspace) {
steps.git branch: configurations.serviceGitBranch, credentialsId: 'git-bot-credentials', url: configurations.serviceGitRepo
}
}
}

10
first-class-pipeline/src/com/freeleaps/devops/enums/ServiceLanguage.groovy
View File

@ -2,8 +2,8 @@ package com.freeleaps.devops.enums
enum ServiceLanguage {
PYTHON('Python'),
NODE('Node (JS,TS)'),
PYTHON('python'),
JS('javascript'),
UNKNOWN('Unknown')
final String language
@ -14,10 +14,10 @@ enum ServiceLanguage {
static ServiceLanguage parse(String language) {
switch (language) {
case 'Python':
case 'python':
return ServiceLanguage.PYTHON
case 'Node (JS,TS)':
return ServiceLanguage.NODE
case 'javascript':
return ServiceLanguage.JS
default:
return ServiceLanguage.UNKNOWN
}

53
first-class-pipeline/tests/Jenkinsfile vendored
View File

@ -2,11 +2,56 @@ library 'first-class-pipeline'
executeFreeleapsPipeline {
serviceName = 'magicleaps'
serviceLang = 'Python'
environmentSlug = 'alpha'
serviceGitBranch = 'master'
serviceGitRepo = "https://freeleaps@dev.azure.com/freeleaps/magicleaps/_git/magicleaps"
environmentSlug = 'alpha'
dependenciesManager = 'pip'
pipRequirementsFile = 'requirements.txt'
serviceGitRepoType = 'monorepo'
executeMode = 'on-demand' // on-demand, full
commitMessageLintEnabled = true
components {
frontend {
root = 'frontend'
language = 'javascript'
dependenciesManager = 'npm'
buildAgentImage = 'node:lts-alpine'
buildCacheEnabled = true
buildCommand = 'npm run build'
lintEnabled = true
linter = 'eslint'
sastEnabled = true
sastProvider = 'NodeJsScan'
imageRegistry = 'docker.io'
imageRepository = 'sunzhenyucn'
imageName = 'magicleaps-frontend'
imageBuilder = 'dind'
dockerfilePath = 'Dockerfile'
imageBuildRoot = '.'
imageReleaseArchitectures = ['amd64', 'arm64']
registryCredentialName = 'first-class-pipeline-dev-secret'
semanticReleaseEnabled = true
semanticReleaseBranch = 'master'
}
backend {
root = 'backend'
language = 'python'
dependenciesManager = 'pip'
buildAgentImage = 'python:3.10-slim-buster'
buildCacheEnabled = true
lintEnabled = true
linter = 'PyLint'
sastEnabled = true
sastProvider = 'Bandit'
imageRegistry = 'docker.io'
imageRepository = 'sunzhenyucn'
imageName = 'magicleaps-backend'
imageBuilder = 'dind'
dockerfilePath = 'Dockerfile'
imageBuildRoot = '.'
imageReleaseArchitectures = ['amd64', 'arm64']
registryCredentialName = 'first-class-pipeline-dev-secret'
semanticReleaseEnabled = true
semanticReleaseBranch = 'master'
}
}
}

116
first-class-pipeline/vars/executeFreeleapsPipeline.groovy
View File

@ -4,6 +4,8 @@ import com.freeleaps.devops.SourceFetcher
import com.freeleaps.devops.DependenciesResolver
import com.freeleaps.devops.enums.DependenciesManager
import com.freeleaps.devops.enums.ServiceLanguage
import com.freeleaps.devops.CommitMessageLinter
import com.freeleaps.devops.ChangedComponentsDetector
def call(body) {
def configurations = [:]
@ -11,6 +13,8 @@ def call(body) {
body.delegate = configurations
body()
def sourceFetcher = new SourceFetcher(this)
pipeline {
agent any
options {
@ -19,55 +23,127 @@ def call(body) {
}
stages {
stage("Source Codes Checkout") {
steps {
script {
def sourceFetcher = new SourceFetcher(this)
sourceFetcher.fetch(configurations)
}
}
}
stage("Commit Linting If Enabled") {
agent {
kubernetes {
defaultContainer 'commit-message-linter'
yaml """
apiVersion: v1
kind: Pod
metadata:
labels:
freeleaps-devops-system/milestone: commit-message-linting
spec:
containers:
- name: commit-message-linter
image: docker.io/commitlint/commitlint:master
command:
- cat
tty: true
volumeMounts:
- name: workspace
mountPath: /workspace
volumes:
- name: workspace
emptyDir: {}
"""
}
}
steps {
script {
def enabled = configurations.commitMessageLintEnabled
if (enabled == null || !enabled) {
log.warn "Commit message linting is disabled"
} else if (enabled) {
log.info "Commit message linting is enabled"
sourceFetcher.fetch(configurations)
def linter = new CommitMessageLinter(this)
linter.lint(configurations)
}
}
}
}
stage("Build Agent Setup") {
stage("Execute Mode Detection") {
steps {
script {
def buildAgentImage = configurations.buildAgentImage
def executeMode = configurations.executeMode
if (executeMode == null || executeMode.isEmpty()) {
log.warn "Not set executeMode, using fully as default execute mode"
env.executeMode = "fully"
} else if (executeMode == 'on-demand' && serviceGitRepoType != 'monorepo') {
log.warn "serviceGirRepoType is not monorepo, on-demand mode is not supported, using fully mode"
env.executeMode = "fully"
} else {
log.info "Using ${executeMode} as execute mode"
env.executeMode = executeMode
}
}
}
}
stage("Code Changes Detection") {
steps {
when {
expression {
return env.executeMode == "on-demand"
}
}
script {
sourceFetcher.fetch(configurations)
def changedComponentsDetector = new ChangedComponentsDetector(this)
def changedComponents = changedComponentsDetector.detect(env.workspace, configurations.components)
log.info "Changed components: ${changedComponents}"
env.changedComponents = changedComponents.join(' ')
}
}
}
configurations.components.each { component ->
stage("${component} :: Build Agent Setup") {
when {
expression {
return env.executeMode == "fully" || env.changedComponents.contains(component)
}
}
steps {
script {
def buildAgentImage = component.buildAgentImage
if (buildAgentImage == null || buildAgentImage.isEmpty()) {
log.warn "Not set buildAgentImage, using default build agent image"
log.warn "Not set buildAgentImage for ${component}, using default build agent image"
def language = ServiceLanguage.parse(configurations.serviceLang)
switch(language) {
case ServiceLanguage.PYTHON:
buildAgentImage = "python:3.10-slim-buster"
break
case ServiceLanguage.NODE:
case ServiceLanguage.JS:
buildAgentImage = "node:lts-alpine"
break
default:
error("Unknown service language")
}
log.info "Using ${buildAgentImage} as build agent image"
log.info "Using ${buildAgentImage} as build agent image for ${component}"
env.buildAgentImage = buildAgentImage
}
}
}
}
stage("Dependencies Resolving") {
stage("${component} :: Dependencies Resolving") {
when {
expression {
return env.executeMode == "fully" || env.changedComponents.contains(component)
}
}
agent {
kubernetes {
defaultContainer 'dep-resolver'
@ -93,16 +169,17 @@ spec:
"""
}
}
steps {
script {
def language = ServiceLanguage.parse(configurations.serviceLang)
def language = ServiceLanguage.parse(component.language)
def depManager = DependenciesManager.parse(configurations.dependenciesManager)
def depManager = DependenciesManager.parse(component.dependenciesManager)
def dependenciesResolver = new DependenciesResolver(this, language)
dependenciesResolver.useManager(depManager)
if (configurations.buildCacheEnabled) {
if (component.buildCacheEnabled) {
dependenciesResolver.enableCachingSupport()
} else {
dependenciesResolver.disableCachingSupport()
@ -112,6 +189,7 @@ spec:
}
}
}
}
}
}