feat(hos_client_create, hos_client_destory): 多次调用destory不会导致重复释放

This commit is contained in:
彭宣正
2020-12-14 17:24:58 +08:00
parent 505d529c32
commit 10b370e486
55976 changed files with 8544395 additions and 2 deletions

View File

@@ -0,0 +1,45 @@
import os
import json
import boto3
import argparse
UPDATE_STATUS_LAMBDA_FUNCTION_NAME = os.environ['UPDATE_STATUS_LAMBDA_FUNCTION_NAME']
lambdaClient = boto3.client('lambda', region_name = os.environ['AWS_REGION'])
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-s', '--stage_name', default = 'Unknown')
parser.add_argument('-e', '--internal_message', default = '')
parser.add_argument('-i', '--release_id', default = '')
parser.add_argument('-m', '--status_message', default = '')
parser.add_argument('-b', '--build_succeeding', default = '0')
parser.add_argument('-o', '--internal_only', action = 'store_true')
parser.add_argument('-c', '--release_complete', action = 'store_true')
args = parser.parse_args()
updateStatus({
'stageName': args.stage_name,
'internalMessage': args.internal_message,
'internalOnly': args.internal_only,
'messageToTrebuchet': {
'releaseId' : args.release_id,
'language' : 'CPP',
'releaseState' : 'Success' if args.release_complete else ('InProgress' if args.build_succeeding == '1' else 'Blocked'),
'statusMessage' : args.status_message
}
})
def updateStatus(updateStatusMessage):
print('[Lambda] Triggering Lambda function to update status:', end = ' ')
print(updateStatusMessage)
response = lambdaClient.invoke(
FunctionName = UPDATE_STATUS_LAMBDA_FUNCTION_NAME,
InvocationType = 'RequestResponse',
Payload = json.dumps(updateStatusMessage)
)
print('Response:', end = ' ')
print(response)
if response['ResponseMetadata']['HTTPStatusCode'] != 200:
quit(1)
main()

View File

@@ -0,0 +1,26 @@
version: 0.2
phases:
build:
commands:
- echo $CODEBUILD_SOURCE_VERSION
- export RELEASE_ID=$(cat $RELEASE_ID_FILENAME)
- python3 aws-sdk-cpp/CI/trebuchet-release-pipeline/UpdateStatus.py -s Build -i "$RELEASE_ID" -m "Step 2 of 4. Verifying Build." -b $CODEBUILD_BUILD_SUCCEEDING
- mv * /tmp && mkdir -p /tmp/build
- cd /tmp/aws-sdk-cpp
- python ./scripts/endpoints_checker.py
- cd ../build
- cmake ../aws-sdk-cpp -DCMAKE_BUILD_TYPE=Debug -DENABLE_ADDRESS_SANITIZER=ON -DMINIMIZE_SIZE=ON
- make -j 3
post_build:
commands:
- cd /tmp
- export BUILD_JOB_NAME=$(echo "${CODEBUILD_BUILD_ID}" | cut -f1 -d ":")
- export BUILD_URL="https://${AWS_REGION}.console.aws.amazon.com/codesuite/codebuild/projects/${BUILD_JOB_NAME}/build/${CODEBUILD_BUILD_ID}"
- |
if [ "${CODEBUILD_BUILD_SUCCEEDING}" = "0" ]; then
python3 aws-sdk-cpp/CI/trebuchet-release-pipeline/UpdateStatus.py -s Build -e "[BUILD FAILURE](${BUILD_URL}) (${CODEBUILD_BUILD_ID})" -i $RELEASE_ID -m "Step 2 of 4. Verification of Build Failed. A technician has already been notified." -b $CODEBUILD_BUILD_SUCCEEDING;
fi
artifacts:
files:
- "**/*"
base-directory: /tmp

View File

@@ -0,0 +1,22 @@
version: 0.2
phases:
pre_build:
commands:
- export RELEASE_ID=$(cat $RELEASE_ID_FILENAME)
- python3 aws-sdk-cpp/CI/trebuchet-release-pipeline/UpdateStatus.py -s IntegrationTests -i "$RELEASE_ID" -m "Step 3 of 4. Running Integration Tests." -b $CODEBUILD_BUILD_SUCCEEDING
build:
commands:
- echo $CODEBUILD_SOURCE_VERSION
- mv aws-sdk-cpp build /tmp
- cd /tmp/build
- python ../aws-sdk-cpp/scripts/run_integration_tests.py --testDir .
post_build:
commands:
- cd /tmp
- aws s3 cp ./build s3://${S3_BUCKET_NAME}/log/${CODEBUILD_BUILD_ID}/ --recursive --exclude "*" --include "aws*.log"
- export BUILD_JOB_NAME=$(echo "${CODEBUILD_BUILD_ID}" | cut -f1 -d ":")
- export BUILD_URL="https://${AWS_REGION}.console.aws.amazon.com/codesuite/codebuild/projects/${BUILD_JOB_NAME}/build/${CODEBUILD_BUILD_ID}"
- |
if [ "${CODEBUILD_BUILD_SUCCEEDING}" = "0" ]; then
python3 aws-sdk-cpp/CI/trebuchet-release-pipeline/UpdateStatus.py -s IntegrationTests -e "[BUILD FAILURE](${BUILD_URL}) (${CODEBUILD_BUILD_ID})" -i $RELEASE_ID -m "Step 3 of 4. Integration Tests Failed. A technician has already been notified." -b $CODEBUILD_BUILD_SUCCEEDING;
fi

View File

@@ -0,0 +1,29 @@
version: 0.2
phases:
build:
commands:
- echo $CODEBUILD_SOURCE_VERSION
- rm -rf aws-sdk-cpp
- git clone https://github.com/${GITHUB_PUBLIC_REPOSITORY}.git
- cd aws-sdk-cpp
- export VERSION_NUM=$(grep AWS_SDK_VERSION_STRING ./aws-cpp-sdk-core/include/aws/core/VersionConfig.h | cut -d '"' -f2)
- sed -i "s/PROJECT_NUMBER .*/PROJECT_NUMBER = $VERSION_NUM/" ./doxygen/doxygen.config
- doxygen ./doxygen/doxygen.config
- python doc_crosslinks/generate_cross_link_data.py --apiDefinitionsPath ./code-generation/api-descriptions/ --templatePath ./doc_crosslinks/crosslink_redirect.html --outputPath ./crosslink_redirect.html
- aws s3 cp ./doxygen/html s3://${DOCS_S3_BUCKET_NAME}/cpp/api/$VERSION_NUM --recursive
- aws s3 cp s3://${DOCS_S3_BUCKET_NAME}/cpp/api/$VERSION_NUM s3://${DOCS_S3_BUCKET_NAME}/cpp/api/LATEST --recursive
- aws s3 cp ./crosslink_redirect.html s3://${DOCS_S3_BUCKET_NAME}/cpp/api/crosslink_redirect.html
- mkdir aws_sdk_cpp
- cp -r ./doxygen/html aws_sdk_cpp
- cp -r ./crosslink_redirect.html aws_sdk_cpp
- zip -r documentation.zip ./aws_sdk_cpp
- aws s3 cp documentation.zip s3://${BINARY_S3_BUCKET_NAME}/cpp/builds/$VERSION_NUM/documentation.zip
post_build:
commands:
- cd $CODEBUILD_SRC_DIR
- export BUILD_JOB_NAME=$(echo "${CODEBUILD_BUILD_ID}" | cut -f1 -d ":")
- export BUILD_URL="https://${AWS_REGION}.console.aws.amazon.com/codesuite/codebuild/projects/${BUILD_JOB_NAME}/build/${CODEBUILD_BUILD_ID}"
- |
if [ "${CODEBUILD_BUILD_SUCCEEDING}" = "0" ]; then
python3 aws-sdk-cpp/CI/trebuchet-release-pipeline/UpdateStatus.py -s PublishAPIDocs -e "[BUILD FAILURE](${BUILD_URL}) (${CODEBUILD_BUILD_ID})" -i $RELEASE_ID -m "Publish API Docs Failed." -b $CODEBUILD_BUILD_SUCCEEDING -o;
fi

View File

@@ -0,0 +1,71 @@
version: 0.2
phases:
build:
commands:
- echo $CODEBUILD_SOURCE_VERSION
- export RELEASE_ID=$(cat $RELEASE_ID_FILENAME)
- if [ -s $RELEASE_NOTES_FILENAME ]; then export COMMIT_MSG="$(cat $RELEASE_NOTES_FILENAME)"; fi;
- python3 aws-sdk-cpp/CI/trebuchet-release-pipeline/UpdateStatus.py -s PushToGithub -i "$RELEASE_ID" -m "Step 4 of 4. Pushing Code to Public Github." -b $CODEBUILD_BUILD_SUCCEEDING
- cd aws-sdk-cpp
# Verify the candidate commit, in case there is new merge without testing during release.
- if [ "$(git rev-parse --abbrev-ref HEAD)" != "master" ]; then exit 1; fi;
- git fetch --all
- if [ -n "$(git diff master origin/master)" ]; then exit 1; fi;
# Get highest tag number
- export VERSION=$(git describe --abbrev=0 --tags)
# Calculate new version
- export VERSION_MAJOR=$(echo $VERSION | cut -d '.' -f1)
- export VERSION_MINOR=$(echo $VERSION | cut -d '.' -f2)
- export VERSION_PATCH=$(echo $VERSION | cut -d '.' -f3)
- export VERSION_PATCH=$((VERSION_PATCH+1))
- export VERSION_BUMP=$VERSION_MAJOR.$VERSION_MINOR.$VERSION_PATCH
- echo "Updating $VERSION to $VERSION_BUMP"
# Write new version to VersionConfig.h
- sed -i "s/AWS_SDK_VERSION_STRING.*/AWS_SDK_VERSION_STRING \"$VERSION_BUMP\"/" aws-cpp-sdk-core/include/aws/core/VersionConfig.h
# git add
- git add --all
- git status
# Generate release notes
- if [ -z "$COMMIT_MSG" ]; then export COMMIT_MSG="Auto commit from CI."; fi;
# Commit to release candidate branch
- git config --global user.name "$GIT_COMMIT_AUTHOR_NAME"
- git config --global user.email "$GIT_COMMIT_AUTHOR_EMAIL"
- git commit -m "$COMMIT_MSG"
- git checkout release-candidate
- git merge master
- git push origin release-candidate
# Get current hash and see if it already has a tag
- export GIT_COMMIT=$(git rev-parse HEAD)
- export NEEDS_TAG=$(git describe --contains $GIT_COMMIT)
# Only tag if no tag already (would be better if the git describe command above could have a silent option)
- |
if [ -z "$NEEDS_TAG" ]; then
echo "Tagged with $VERSION_BUMP (Ignoring fatal:cannot describe - this means commit is untagged) "
git tag $VERSION_BUMP
git push --tags
else
echo "Already a tag on this commit"
fi
# Push code to Github
# - git fetch --tags
# - git fetch --all
# - git reset --hard HEAD
# - git checkout release-candidate
# - git pull
- git checkout master
- git pull
- git merge release-candidate
- git push https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/${GITHUB_PRIVATE_REPOSITORY}.git master
- git push https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/${GITHUB_PUBLIC_REPOSITORY}.git master
- git push --tags https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/${GITHUB_PUBLIC_REPOSITORY}.git
post_build:
commands:
- cd $CODEBUILD_SRC_DIR
- export BUILD_JOB_NAME=$(echo "${CODEBUILD_BUILD_ID}" | cut -f1 -d ":")
- export BUILD_URL="https://${AWS_REGION}.console.aws.amazon.com/codesuite/codebuild/projects/${BUILD_JOB_NAME}/build/${CODEBUILD_BUILD_ID}"
- |
if [ "${CODEBUILD_BUILD_SUCCEEDING}" = "1" ]; then
python3 aws-sdk-cpp/CI/trebuchet-release-pipeline/UpdateStatus.py -s PushToGithub -e "[BUILD SUCCESS](${BUILD_URL}) (${CODEBUILD_BUILD_ID})" -i $RELEASE_ID -m "Step 4 of 4. Code Pushed to Public Github." -b $CODEBUILD_BUILD_SUCCEEDING -c;
else
python3 aws-sdk-cpp/CI/trebuchet-release-pipeline/UpdateStatus.py -s PushToGithub -e "[BUILD FAILURE](${BUILD_URL}) (${CODEBUILD_BUILD_ID})" -i $RELEASE_ID -m "Step 4 of 4. Push to Public Github Failed. A technician has already been notified." -b $CODEBUILD_BUILD_SUCCEEDING;
fi

View File

@@ -0,0 +1,30 @@
# This buildspec is source controlled, whenever you make any change in the AWS console, you should update it to Github.
version: 0.2
phases:
build:
commands:
- echo $CODEBUILD_SOURCE_VERSION
- git clone https://${GIT_USERNAME}:${GIT_PASSWORD}@github.com/${GITHUB_PRIVATE_REPOSITORY}.git aws-sdk-cpp
- export RELEASE_ID=$(cat $RELEASE_ID_FILENAME)
- python3 aws-sdk-cpp/CI/trebuchet-release-pipeline/UpdateStatus.py -s RegenerateCode -i "$RELEASE_ID" -m "Step 1 of 4. Regenerating Code with New Models." -b $CODEBUILD_BUILD_SUCCEEDING
- cp models/*.normal.json aws-sdk-cpp/code-generation/api-descriptions/
- cd aws-sdk-cpp
- mkdir build
- cd build
- cmake .. -DREGENERATE_CLIENTS=ON
- cd ..
- rm -rf build
post_build:
commands:
- cd $CODEBUILD_SRC_DIR
- export BUILD_JOB_NAME=$(echo "${CODEBUILD_BUILD_ID}" | cut -f1 -d ":")
- export BUILD_URL="https://${AWS_REGION}.console.aws.amazon.com/codesuite/codebuild/projects/${BUILD_JOB_NAME}/build/${CODEBUILD_BUILD_ID}"
- |
if [ "${CODEBUILD_BUILD_SUCCEEDING}" = "1" ]; then
python3 aws-sdk-cpp/CI/trebuchet-release-pipeline/UpdateStatus.py -s RegenerateCode -e "[BUILD SUCCESS](${BUILD_URL}) (${CODEBUILD_BUILD_ID})" -i $RELEASE_ID -m "Step 1 of 4. Regenerated Code with New Models." -b $CODEBUILD_BUILD_SUCCEEDING;
else
python3 aws-sdk-cpp/CI/trebuchet-release-pipeline/UpdateStatus.py -s RegenerateCode -e "[BUILD FAILURE](${BUILD_URL}) (${CODEBUILD_BUILD_ID})" -i $RELEASE_ID -m "Step 1 of 4. Code Generation with New Models Failed. A technician has already been notified." -b $CODEBUILD_BUILD_SUCCEEDING;
fi
artifacts:
files:
- "**/*"

View File

@@ -0,0 +1,28 @@
version: 0.2
phases:
build:
commands:
- echo ${Env:CODEBUILD_SOURCE_VERSION}
- $RELEASE_ID=$(cat ${Env:RELEASE_ID_FILENAME})
- mkdir C:\tmp
- mkdir C:\tmp\build
- mv * C:\tmp
- cd C:\tmp\build
- cmake.exe -G "Visual Studio 14 2015 Win64" -DCMAKE_BUILD_TYPE=Debug -DMINIMIZE_SIZE=ON ../aws-sdk-cpp
- msbuild.exe ALL_BUILD.vcxproj -p:Configuration=Debug -m
- cd ..
- Get-ChildItem aws-sdk-cpp -Exclude *tests | Where-Object Name -Like 'aws-cpp-sdk-*' | Remove-Item -Recurse -Force
- Get-ChildItem build -Exclude bin | Remove-Item -Recurse -Force
post_build:
commands:
- cd C:\tmp
- $BUILD_JOB_NAME=${Env:CODEBUILD_BUILD_ID}.Substring(0, ${Env:CODEBUILD_BUILD_ID}.IndexOf(":"))
- $BUILD_URL="https://${Env:AWS_REGION}.console.aws.amazon.com/codesuite/codebuild/projects/$BUILD_JOB_NAME/build/${Env:CODEBUILD_BUILD_ID}"
- |
if (${Env:CODEBUILD_BUILD_SUCCEEDING} -eq 0) {
python aws-sdk-cpp/CI/trebuchet-release-pipeline/UpdateStatus.py -s Build -e "[BUILD FAILURE](${BUILD_URL}) (${Env:CODEBUILD_BUILD_ID})" -i $RELEASE_ID -m "Step 2 of 4. Verification of Build Failed. A technician has already been notified." -b $CODEBUILD_BUILD_SUCCEEDING;
}
artifacts:
files:
- "**/*"
base-directory: C:\tmp

View File

@@ -0,0 +1,28 @@
version: 0.2
phases:
build:
commands:
- echo ${Env:CODEBUILD_SOURCE_VERSION}
- $RELEASE_ID=$(cat ${Env:RELEASE_ID_FILENAME})
- mkdir C:\tmp
- mkdir C:\tmp\build
- mv * C:\tmp
- cd C:\tmp\build
- cmake.exe -G "Visual Studio 15 2017 Win64" -DCMAKE_BUILD_TYPE=Debug -DMINIMIZE_SIZE=ON ../aws-sdk-cpp
- msbuild.exe ALL_BUILD.vcxproj -p:Configuration=Debug -m
- cd ..
- Get-ChildItem aws-sdk-cpp -Exclude *tests | Where-Object Name -Like 'aws-cpp-sdk-*' | Remove-Item -Recurse -Force
- Get-ChildItem build -Exclude bin | Remove-Item -Recurse -Force
post_build:
commands:
- cd C:\tmp
- $BUILD_JOB_NAME=${Env:CODEBUILD_BUILD_ID}.Substring(0, ${Env:CODEBUILD_BUILD_ID}.IndexOf(":"))
- $BUILD_URL="https://${Env:AWS_REGION}.console.aws.amazon.com/codesuite/codebuild/projects/$BUILD_JOB_NAME/build/${Env:CODEBUILD_BUILD_ID}"
- |
if (${Env:CODEBUILD_BUILD_SUCCEEDING} -eq 0) {
python aws-sdk-cpp/CI/trebuchet-release-pipeline/UpdateStatus.py -s Build -e "[BUILD FAILURE](${BUILD_URL}) (${Env:CODEBUILD_BUILD_ID})" -i $RELEASE_ID -m "Step 2 of 4. Verification of Build Failed. A technician has already been notified." -b $CODEBUILD_BUILD_SUCCEEDING;
}
artifacts:
files:
- "**/*"
base-directory: C:\tmp

View File

@@ -0,0 +1,21 @@
version: 0.2
phases:
build:
commands:
- echo ${Env:CODEBUILD_SOURCE_VERSION}
- $RELEASE_ID=$(cat ${Env:RELEASE_ID_FILENAME})
- mkdir C:\tmp
- mv aws-sdk-cpp C:\tmp
- mv build C:\tmp
- cd C:\tmp\build
- python ../aws-sdk-cpp/scripts/run_integration_tests.py --testDir ./bin/Debug
post_build:
commands:
- cd C:\tmp
- aws s3 cp ./build s3://${Env:S3_BUCKET_NAME}/log/${Env:CODEBUILD_BUILD_ID}/ --recursive --exclude "*" --include "aws*.log"
- $BUILD_JOB_NAME=${Env:CODEBUILD_BUILD_ID}.Substring(0, ${Env:CODEBUILD_BUILD_ID}.IndexOf(":"))
- $BUILD_URL="https://${Env:AWS_REGION}.console.aws.amazon.com/codesuite/codebuild/projects/$BUILD_JOB_NAME/build/${Env:CODEBUILD_BUILD_ID}"
- |
if (${Env:CODEBUILD_BUILD_SUCCEEDING} -eq 0) {
python aws-sdk-cpp/CI/trebuchet-release-pipeline/UpdateStatus.py -s IntegrationTests -e "[BUILD FAILURE](${BUILD_URL}) (${Env:CODEBUILD_BUILD_ID})" -i $RELEASE_ID -m "Step 3 of 4. Integration Tests Failed. A technician has already been notified." -b $CODEBUILD_BUILD_SUCCEEDING;
}

View File

@@ -0,0 +1,127 @@
# Whenever you make any change here, you should update it in Amazon S3.
# This function serves as glue between SNS and S3.
# 1- Receives SNS message when Trebuchet release starts
# 2- Extracts the message (which should be JSON)
# 3- Writes the JSON to a file on disk
# 4- Downloads models with the presigned URL
# 5- Writes release notes to a file
# 6- Writes release id to a file
# 7- Upload all these files as a zip file to S3
import os
import shutil
import re
import json
import zipfile
import traceback
import boto3
from botocore.vendored import requests
S3_BUCKET_NAME = os.environ['S3_BUCKET_NAME']
RELEASE_MESSAGE_FILENAME = os.environ['RELEASE_MESSAGE_FILENAME']
RELEASE_ID_FILENAME = os.environ['RELEASE_ID_FILENAME']
RELEASE_NOTES_FILENAME = os.environ['RELEASE_NOTES_FILENAME']
PIPELINE_SOURCE = os.environ['PIPELINE_SOURCE']
UPDATE_STATUS_LAMBDA_FUNCTION_NAME = os.environ['UPDATE_STATUS_LAMBDA_FUNCTION_NAME']
OUTPUT_PATH = os.path.join('/tmp', 'output')
MODELS_OUTPUT_PATH = os.path.join(OUTPUT_PATH, 'models')
s3Resource = boto3.resource('s3', region_name = os.environ['AWS_REGION'])
lambdaClient = boto3.client('lambda', region_name = os.environ['AWS_REGION'])
updateStatusMessage = {
'stageName': 'HandleTrebuchetReleaseNotification',
'internalMessage': '',
'internalOnly': False,
'messageToTrebuchet': {
'releaseId' : '',
'language' : 'CPP',
'releaseState' : '',
'statusMessage' : ''
}
}
def lambda_handler(event, context):
try:
releaseMessage = json.loads(event['Records'][0]['Sns']['Message'])
# For local testing:
# with open(RELEASE_MESSAGE_FILENAME, 'r') as releaseMessageFile:
# releaseMessage = json.loads(releaseMessageFile.read())
print('[SNS] Receiving message from Trebuchet:', end = ' ')
print(releaseMessage)
if os.path.isdir(OUTPUT_PATH):
shutil.rmtree(OUTPUT_PATH)
os.mkdir(OUTPUT_PATH)
os.mkdir(MODELS_OUTPUT_PATH)
with open(os.path.join(OUTPUT_PATH, RELEASE_MESSAGE_FILENAME), 'w') as releaseMessageFile:
releaseMessageFile.write(json.dumps(releaseMessage))
releaseMessageFile.close()
with open(os.path.join(OUTPUT_PATH, RELEASE_ID_FILENAME), 'w') as releaseIdFile:
releaseIdFile.write(releaseMessage['release']['id'])
with open(os.path.join(OUTPUT_PATH, RELEASE_NOTES_FILENAME), 'w') as releaseNotesFile:
releaseNotesFile.write('')
updateStatusMessage['messageToTrebuchet'] = {
'releaseId' : releaseMessage['release']['id'],
'language' : 'CPP',
'releaseState' : 'InProgress',
'statusMessage' : 'Step 0 of 4. Handling release notification from Trebuchet.'
}
updateStatus(updateStatusMessage)
for feature in releaseMessage['release']['features']:
print('Downloading c2j model files for ' + feature['serviceId'])
response = requests.get(feature['c2jModels'])
if response.status_code != 200:
raise Exception('Error downloading c2j model with feature: ' + feature['featureArn'])
with open(os.path.join('/tmp', 'models.tmp.zip'), 'wb') as c2jModelsZipFile:
c2jModelsZipFile.write(response.content)
archive = zipfile.ZipFile(os.path.join('/tmp', 'models.tmp.zip'), 'r')
archive.debug = 3
for info in archive.infolist():
print(' ' + info.filename)
if re.match(r'output/.*\.normal\.json', info.filename):
outputPath = os.path.join(MODELS_OUTPUT_PATH, os.path.basename(info.filename))
print('* copying {0} to {1}'.format(info.filename, outputPath))
fileHandle = archive.open(info.filename, 'r')
fileOutput = fileHandle.read()
with open(outputPath, 'wb') as destination:
destination.write(fileOutput)
fileHandle.close()
releaseNotes = feature['releaseNotes']
print('Append release notes for ' + feature['serviceId'])
with open(os.path.join(OUTPUT_PATH, RELEASE_NOTES_FILENAME), 'a') as releaseNotesFile:
releaseNotesFile.write(releaseNotes + '\n\n')
updateStatusMessage['messageToTrebuchet']['statusMessage'] = 'Step 0 of 4. Handled release notification from Trebuchet.'
updateStatus(updateStatusMessage)
print('Archiving release-message, release-id, release-notes, and models directory into a zip file.')
shutil.make_archive('/tmp/models', 'zip', OUTPUT_PATH)
print('[S3] Sending zip file including json file to S3://{0}/{1}.'.format(S3_BUCKET_NAME, PIPELINE_SOURCE))
response = s3Resource.meta.client.upload_file('/tmp/models.zip', S3_BUCKET_NAME, PIPELINE_SOURCE)
print('Response:', end = ' ')
print(response)
except Exception:
traceback.print_exc()
updateStatusMessage['internalMessage'] = traceback.format_exc()
updateStatusMessage['messageToTrebuchet']['releaseState'] = 'Blocked'
updateStatusMessage['messageToTrebuchet']['statusMessage'] = 'Step 0 of 4. Failed to handle release notification from Trebuchet.'
updateStatus(updateStatusMessage)
def updateStatus(releaseStatus):
print('[Lambda] Triggering Lambda function to update status.')
response = lambdaClient.invoke(
FunctionName = UPDATE_STATUS_LAMBDA_FUNCTION_NAME,
InvocationType = 'RequestResponse',
Payload = json.dumps(releaseStatus)
)
print('Response:', end = ' ')
print(response)
# lambda_handler('', '')

View File

@@ -0,0 +1,29 @@
import os
import json
from botocore.vendored import requests
CHIME_BOT_URL = os.environ['CHIME_BOT_URL']
TREBUCHET_RELEASE_PIPELINE_NAME = os.environ['TREBUCHET_RELEASE_PIPELINE_NAME']
SOURCE_STAGE_NAME = os.environ['SOURCE_STAGE_NAME']
PROD_STAGE_NAME = os.environ['PROD_STAGE_NAME']
def lambda_handler(event, context):
print('Received Event: ' + json.dumps(event))
message = json.loads(event['Records'][0]['Sns']['Message'])
pipeline = message['detail']['pipeline']
stage = message['detail']['stage']
state = message['detail']['state']
if (state == 'SUCCEEDED' and pipeline == TREBUCHET_RELEASE_PIPELINE_NAME and (stage == SOURCE_STAGE_NAME or stage == PROD_STAGE_NAME)) or (state == 'FAILED'):
headers = {'Content-Type': 'application/json'}
data = {}
data['Content'] = '/md {mentionAll}\nPipeline: {pipeline}\nStage: {stage}\nState: {state}'.format(
mentionAll = '@All' if state == 'FAILED' else '',
pipeline = pipeline,
stage = stage,
state = state)
print('[Chime] Sending message to Chime Bot: ' + json.dumps(data['Content']))
respone = requests.post(CHIME_BOT_URL, headers = headers, data = json.dumps(data))
print('Response:', end=' ')
print(respone)

View File

@@ -0,0 +1,122 @@
# Whenever you make any change here, you should update it in Amazon S3.
# This Lambda function will make notifications to:
# 1. SQS queue to update status with Trebuchet
# 2. ChimeBot to notify engineers in the Chime room
# 3. CloudWatch metrics to trigger alarms and cut tickets
# Expected inputs of this Lambda function:
# {
# "stageName": "HandleTrebuchetReleaseNotification|RegenerateCode|Build|IntegrationTests|PublishToGithub",
# "internalMessage": "",
# "internalOnly": True|False
# "messageToTrebuchet": {
# "releaseId" : "",
# "language" : "CPP",
# "releaseState" : "InProgress|Success|Blocked|Failed",
# "statusMessage" : "",
# "additionalDetails" : {
# "generatedCodePresignedUrl":"",
# "logPresignedUrl":""
# }
# }
# }
import os
import json
import boto3
import traceback
from botocore.vendored import requests
CHIME_BOT_URL = os.environ['CHIME_BOT_URL']
TREBUCHET_QUEUE_URL = os.environ['TREBUCHET_QUEUE_URL']
sqsClient = boto3.client('sqs')
cloudwatchClient = boto3.client('cloudwatch')
def lambda_handler(event, context):
print('Received Event: ' + json.dumps(event))
if 'stageName' not in event or event['stageName'] == "":
event['stageName'] = 'Unknown'
if 'internalMessage' not in event:
event['internalMessage'] = ''
if 'internalOnly' not in event:
event['internalOnly'] = False
try:
failure = 0.0
sendMessageToChimeBot = False
mentionAll = False
if 'messageToTrebuchet' not in event or 'releaseId' not in event['messageToTrebuchet'] or event['messageToTrebuchet']['releaseId'] == "":
raise Exception('Missing releaseId in the received release message.')
messageToTrebuchet = event['messageToTrebuchet']
if messageToTrebuchet['releaseState'] == 'InProgress' or messageToTrebuchet['releaseState'] == 'Success':
pass
elif messageToTrebuchet['releaseState'] == 'Blocked' or messageToTrebuchet['releaseState'] == 'Failed':
failure = 1.0
sendMessageToChimeBot = True
mentionAll = True
else:
failure = 1.0
sendMessageToChimeBot = True
mentionAll = True
event['internalMessage'] = ('{originalInternalMessage} releaseState ({releaseState}) should be one of these: InProgress|Success|Blocked|Failed, this build will be marked as Blocked.'.format(
originalInternalMessage = event['internalMessage'],
releaseState = messageToTrebuchet['releaseState']
)).strip()
if not event['internalOnly']:
notifyTrebuchetSQS(messageToTrebuchet)
notifyCloudWatch(failure)
if sendMessageToChimeBot:
notifyChimeBot(event['stageName'], event['internalMessage'], mentionAll)
except Exception:
traceback.print_exc()
notifyChimeBot(
stageName = event['stageName'],
message = '\n'.join([event['internalMessage'], traceback.format_exc()]).strip(),
mentionAll = True)
if 'messageToTrebuchet' in event and 'releaseId' in event['messageToTrebuchet'] and not event['messageToTrebuchet']['releaseId'] == "":
notifyTrebuchetSQS({
"releaseId" : event['messageToTrebuchet']['releaseId'],
"language" : "CPP",
"releaseState" : "Blocked",
"statusMessage" : "Encountered internal errors."
})
def notifyChimeBot(stageName, message, mentionAll = False):
headers = {'Content-Type': 'application/json'}
data = {}
data['Content'] = '/md {mentionAll}\nStage: {stageName}\nMessage: {message}'.format(
mentionAll = '@All' if mentionAll else '',
stageName = stageName,
message = message)
print('[Chime] Sending message to Chime Bot: ' + json.dumps(data['Content']))
respone = requests.post(CHIME_BOT_URL, headers = headers, data = json.dumps(data))
print('Response:', end=' ')
print(respone)
def notifyCloudWatch(value):
print('[CloudWatch] Puting data to Metric: BuildFailure with value: ' + str(value))
response = cloudwatchClient.put_metric_data(
Namespace='BuildPipeline',
MetricData=[{
'MetricName' : "BuildFailure",
'Value' : value,
'Unit' : 'Count',
'StorageResolution' : 60
}]
)
print('Response:', end=' ')
print(response)
def notifyTrebuchetSQS(message):
print('[SQS] Sending message to Trebuchet queue:', end=' ')
print(message)
response = sqsClient.send_message(
QueueUrl = TREBUCHET_QUEUE_URL,
MessageBody = json.dumps(message),
MessageGroupId = 'CppSdkRelease'
)
print('Response:', end=' ')
print(response)

File diff suppressed because it is too large Load Diff