Skip to content

Commit

Permalink
Merge pull request #77 from CybercentreCanada/feature/updated_python_…
Browse files Browse the repository at this point in the history 
…debian

Feature/updated python debian
  • Loading branch information
@cccs-sgaron
cccs-sgaron authored Jul 28, 2021
2 parents 8448eca + f552a2f commit f49b569
Show file tree
Hide file tree
Showing 2 changed files with 144 additions and 159 deletions.
249 changes: 117 additions & 132 deletions pipelines/azure-build.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,140 +6,125 @@ trigger:
pr: none

pool:
vmImage: 'ubuntu-18.04'

#resources:
# containers:
# - container: redis
# image: redis
# ports:
# - 6379:6379
# - container: elasticsearch
# image: cccs/elasticsearch:7.8.0
# env:
# ES_JAVA_OPTS: "-Xms256m -Xmx512m"
# DISCOVERY_TYPE: 'single-node'
# ports:
# - 9200:9200
vmImage: "ubuntu-20.04"

stages:
- stage: build
jobs:
- job: build_package
displayName: Build Package
steps:
- task: UsePythonVersion@0
displayName: Set python version
inputs: {versionSpec: 3.8}
- script: |
export VERSION=${BUILD_SOURCEBRANCH#"refs/tags/v"}
export COMMIT=`git rev-parse --verify ${BUILD_SOURCEBRANCH}`
export BRANCH=`git ls-remote --heads origin | grep $COMMIT | sed "s/.*\///"`
echo "Building $VERSION On branch $BRANCH"
if [[ "$VERSION" == *stable* ]] && [[ $BRANCH != "master" ]]; then exit 1; fi
if [[ "$VERSION" == *dev* ]] && [[ $BRANCH != "dev" ]]; then exit 1; fi
export VERSION=${VERSION/stable}
export VERSION=${VERSION/beta/b}
echo $VERSION > assemblyline_v4_service/VERSION
sudo env "PATH=$PATH" python -m pip install --no-cache-dir -U wheel cython pip
python setup.py bdist_wheel
- publish: $(System.DefaultWorkingDirectory)/dist/
artifact: dist
- publish: $(System.DefaultWorkingDirectory)/pipelines/
artifact: pipelines
- publish: $(System.DefaultWorkingDirectory)/test/
artifact: test
#- stage: test
# jobs:
# - job: run_test
# strategy:
# matrix:
# python3_7:
# python.version: '3.7'
# Python3_8:
# python.version: '3.8'
# timeoutInMinutes: 10
# services:
# elasticsearch: elasticsearch
# redis: redis
# steps:
# - checkout: none
# - task: UsePythonVersion@0
# displayName: Set python version
# inputs:
# versionSpec: '$(python.version)'
# - download: current
# artifact: dist
# - download: current
# artifact: pipelines
# - download: current
# artifact: test
# - script: |
# set -x # Echo commands before they are run
# sudo apt-get update
# sudo apt-get install -y build-essential libffi-dev libfuzzy-dev python3-dev
# export VERSION=${BUILD_SOURCEBRANCH#"refs/tags/v"}
# sudo -E env "PATH=$PATH" python -m pip install --no-cache-dir -f $(Pipeline.Workspace)/dist/ "assemblyline[test]==${VERSION}"
# sudo mkdir -p /etc/assemblyline/
# sudo mkdir -p /var/cache/assemblyline/
# sudo chmod a+rw /var/cache/assemblyline/
# sudo cp $(Pipeline.Workspace)/pipelines/config.yml /etc/assemblyline
# displayName: Install package
# - script: pytest -rsx -vv
# workingDirectory: $(Pipeline.Workspace)/test
# displayName: Test
- stage: deploy
jobs:
- job: deploy
displayName: Deploy packages and containers
variables:
- group: deployment-information
steps:
- task: UsePythonVersion@0
displayName: Set python version
inputs: {versionSpec: 3.8}
- download: current
artifact: dist
- script: |
set -xv # Echo commands before they are run
sudo env "PATH=$PATH" python -m pip install --no-cache-dir twine
ls dist
twine upload --skip-existing --repository-url $TEST_REPOSITORY_URL dist/*
workingDirectory: $(Pipeline.Workspace)
displayName: Deploy to Test PyPI
env:
TWINE_USERNAME: $(twineUsername)
TWINE_PASSWORD: $(twinePassword)
- script: |
set -xv # Echo commands before they are run
sudo env "PATH=$PATH" python -m pip install --no-cache-dir twine
ls dist
twine upload --skip-existing dist/*
workingDirectory: $(Pipeline.Workspace)
displayName: Deploy to PyPI
env:
TWINE_USERNAME: $(twineUsername)
TWINE_PASSWORD: $(twinePassword)
- task: Docker@2
displayName: Login to docker hub
inputs:
command: login
containerRegistry: dockerhub
- script: |
set -xv # Echo commands before they are run
export TAG=${BUILD_SOURCEBRANCH#"refs/tags/v"}
if [[ "$TAG" == *stable* ]]; then export BUILD_TYPE=stable; else export BUILD_TYPE=latest; fi
export VERSION=${TAG/stable}
export VERSION=${VERSION/beta/b}
export SERIES="`expr $TAG : '\([0-9]\+\.[0-9]\+\.\)'`${BUILD_TYPE}"
until sudo env "PATH=$PATH" python -m pip download "assemblyline-v4-service==$VERSION" --pre --no-deps &> /dev/null; do sleep 2; done
- stage: build
jobs:
- job: build_package
displayName: Build Package
steps:
- task: UsePythonVersion@0
displayName: Set python version
inputs: { versionSpec: 3.9 }
- script: |
export VERSION=${BUILD_SOURCEBRANCH#"refs/tags/v"}
export COMMIT=`git rev-parse --verify ${BUILD_SOURCEBRANCH}`
export BRANCH=`git ls-remote --heads origin | grep $COMMIT | sed "s/.*\///"`
echo "Building $VERSION On branch $BRANCH"
if [[ "$VERSION" == *stable* ]] && [[ $BRANCH != "master" ]]; then exit 1; fi
if [[ "$VERSION" == *dev* ]] && [[ $BRANCH != "dev" ]]; then exit 1; fi
export VERSION=${VERSION/stable}
export VERSION=${VERSION/beta/b}
echo $VERSION > assemblyline_v4_service/VERSION
sudo env "PATH=$PATH" python -m pip install --no-cache-dir -U wheel cython pip
python setup.py bdist_wheel
- publish: $(System.DefaultWorkingDirectory)/dist/
artifact: dist
- publish: $(System.DefaultWorkingDirectory)/pipelines/
artifact: pipelines
- publish: $(System.DefaultWorkingDirectory)/test/
artifact: test
- stage: test
jobs:
- job: run_test
timeoutInMinutes: 10
steps:
- task: UsePythonVersion@0
displayName: Set python version
inputs: { versionSpec: 3.9 }
- download: current
artifact: dist
- download: current
artifact: pipelines
- download: current
artifact: test
- script: |
set -x # Echo commands before they are run
sudo apt-get update
sudo apt-get install -y build-essential libffi-dev libfuzzy-dev python3-dev
export VERSION=${BUILD_SOURCEBRANCH#"refs/tags/v"}
if [[ "$VERSION" == *stable* ]]; then
sudo -E env "PATH=$PATH" python -m pip install "assemblyline" "assemblyline-core"
else
sudo -E env "PATH=$PATH" python -m pip install --pre "assemblyline" "assemblyline-core"
fi
export VERSION=${VERSION/stable}
export VERSION=${VERSION/beta/b}
sudo -E env "PATH=$PATH" python -m pip install --no-cache-dir -f $(Pipeline.Workspace)/dist/ "assemblyline-v4-service==${VERSION}"
sudo -E env "PATH=$PATH" python -m pip install --no-cache-dir -r $(Pipeline.Workspace)/test/requirements.txt
sudo mkdir -p /etc/assemblyline/
sudo mkdir -p /var/cache/assemblyline/
sudo mkdir -p /var/lib/assemblyline/
sudo chmod a+rw /var/cache/assemblyline/
sudo chmod a+rw /var/lib/assemblyline/
sudo cp $(Pipeline.Workspace)/pipelines/config.yml /etc/assemblyline
displayName: Setup environment
- script: python -m pytest -rsx -vv
workingDirectory: $(Pipeline.Workspace)/test
displayName: Test
- stage: deploy
jobs:
- job: deploy
displayName: Deploy packages and containers
variables:
- group: deployment-information
steps:
- task: UsePythonVersion@0
displayName: Set python version
inputs: { versionSpec: 3.9 }
- download: current
artifact: dist
- script: |
set -xv # Echo commands before they are run
sudo env "PATH=$PATH" python -m pip install --no-cache-dir twine
ls dist
twine upload --skip-existing --repository-url $TEST_REPOSITORY_URL dist/*
workingDirectory: $(Pipeline.Workspace)
displayName: Deploy to Test PyPI
env:
TWINE_USERNAME: $(twineUsername)
TWINE_PASSWORD: $(twinePassword)
- script: |
set -xv # Echo commands before they are run
sudo env "PATH=$PATH" python -m pip install --no-cache-dir twine
ls dist
twine upload --skip-existing dist/*
workingDirectory: $(Pipeline.Workspace)
displayName: Deploy to PyPI
env:
TWINE_USERNAME: $(twineUsername)
TWINE_PASSWORD: $(twinePassword)
- task: Docker@2
displayName: Login to docker hub
inputs:
command: login
containerRegistry: dockerhub
- script: |
set -xv # Echo commands before they are run
export TAG=${BUILD_SOURCEBRANCH#"refs/tags/v"}
if [[ "$TAG" == *stable* ]]; then export BUILD_TYPE=stable; else export BUILD_TYPE=latest; fi
export VERSION=${TAG/stable}
export VERSION=${VERSION/beta/b}
export SERIES="`expr $TAG : '\([0-9]\+\.[0-9]\+\.\)'`${BUILD_TYPE}"
until sudo env "PATH=$PATH" python -m pip download "assemblyline-v4-service==$VERSION" --pre --no-deps &> /dev/null; do sleep 2; done
export IMAGE="cccs/assemblyline-v4-service-base"
docker build --build-arg version=$VERSION --build-arg branch=$BUILD_TYPE -t $IMAGE:$TAG -t $IMAGE:$SERIES -t $IMAGE:$BUILD_TYPE docker
docker push $IMAGE --all-tags
export IMAGE="cccs/assemblyline-v4-service-base"
docker build --build-arg version=$VERSION --build-arg branch=$BUILD_TYPE -t $IMAGE:$TAG -t $IMAGE:$SERIES -t $IMAGE:$BUILD_TYPE docker
docker push $IMAGE --all-tags
export IMAGE="cccs/assemblyline-service-resultsample"
docker build --build-arg version=$VERSION --build-arg branch=$BUILD_TYPE -f assemblyline_result_sample_service/Dockerfile -t $IMAGE:$TAG -t $IMAGE:$SERIES -t $IMAGE:$BUILD_TYPE .
docker push $IMAGE --all-tags
export IMAGE="cccs/assemblyline-service-resultsample"
docker build --build-arg version=$VERSION --build-arg branch=$BUILD_TYPE -f assemblyline_result_sample_service/Dockerfile -t $IMAGE:$TAG -t $IMAGE:$SERIES -t $IMAGE:$BUILD_TYPE .
docker push $IMAGE --all-tags
displayName: Deploy to Docker Hub
displayName: Deploy to Docker Hub
54 changes: 27 additions & 27 deletions pipelines/azure-tests.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,34 +4,34 @@ trigger: ["*"]
pr: ["*"]

pool:
vmImage: 'ubuntu-18.04'
vmImage: "ubuntu-20.04"

jobs:
- job: run_test
strategy:
matrix:
python3_7:
python.version: '3.7'
Python3_8:
python.version: '3.8'
- job: run_test
strategy:
matrix:
python3_9:
python.version: "3.9"
# Python3_10:
# python.version: "3.10"

timeoutInMinutes: 10
timeoutInMinutes: 10

steps:
- task: UsePythonVersion@0
displayName: Set python version
inputs:
versionSpec: '$(python.version)'
- script: |
sudo mkdir -p /etc/assemblyline/
sudo mkdir -p /var/cache/assemblyline/
sudo cp pipelines/config.yml /etc/assemblyline
sudo chmod a+rw /var/cache/assemblyline/
sudo apt-get update
sudo apt-get install -y build-essential libffi-dev libfuzzy-dev python3-dev
sudo env "PATH=$PATH" python -m pip install --no-cache-dir -U pip cython setuptools
sudo env "PATH=$PATH" python -m pip install --no-cache-dir -e .
sudo env "PATH=$PATH" python -m pip install --no-cache-dir -r test/requirements.txt
displayName: Setup environment
- script: python -m pytest -rsx -vv
displayName: Test
steps:
- task: UsePythonVersion@0
displayName: Set python version
inputs:
versionSpec: "$(python.version)"
- script: |
sudo mkdir -p /etc/assemblyline/
sudo mkdir -p /var/cache/assemblyline/
sudo cp pipelines/config.yml /etc/assemblyline
sudo chmod a+rw /var/cache/assemblyline/
sudo apt-get update
sudo apt-get install -y build-essential libffi-dev libfuzzy-dev python3-dev
sudo env "PATH=$PATH" python -m pip install --no-cache-dir -U pip cython setuptools
sudo env "PATH=$PATH" python -m pip install --no-cache-dir -e .
sudo env "PATH=$PATH" python -m pip install --no-cache-dir -r test/requirements.txt
displayName: Setup environment
- script: python -m pytest -rsx -vv
displayName: Test

0 comments on commit f49b569

Please sign in to comment.