133 lines
4.4 KiB
Python
133 lines
4.4 KiB
Python
|
import os
|
||
|
|
import sys
|
||
|
|
import json
|
||
|
|
import time
|
||
|
|
import logging
|
||
|
|
|
||
|
|
try:
|
||
|
|
from functools import reduce
|
||
|
|
except ImportError:
|
||
|
|
pass
|
||
|
|
|
||
|
|
try:
|
||
|
|
import urllib.request as urllib2
|
||
|
|
except ImportError:
|
||
|
|
import urllib2
|
||
|
|
|
||
|
|
log = logging.getLogger("travis.leader")
|
||
|
|
log.addHandler(logging.StreamHandler())
|
||
|
|
log.setLevel(logging.INFO)
|
||
|
|
|
||
|
|
TRAVIS_JOB_NUMBER = 'TRAVIS_JOB_NUMBER'
|
||
|
|
TRAVIS_BUILD_ID = 'TRAVIS_BUILD_ID'
|
||
|
|
POLLING_INTERVAL = 'LEADER_POLLING_INTERVAL'
|
||
|
|
GITHUB_TOKEN = 'GITHUB_TOKEN'
|
||
|
|
|
||
|
|
|
||
|
|
# Travis API entry point, there are at least https://api.travis-ci.com and https://api.travis-ci.org
|
||
|
|
travis_entry = sys.argv[1] if len(sys.argv) > 1 else 'https://api.travis-ci.org'
|
||
|
|
|
||
|
|
build_id = os.getenv(TRAVIS_BUILD_ID)
|
||
|
|
polling_interval = int(os.getenv(POLLING_INTERVAL, '5'))
|
||
|
|
gh_token = os.getenv(GITHUB_TOKEN)
|
||
|
|
|
||
|
|
# assume, first job is the leader
|
||
|
|
|
||
|
|
|
||
|
|
def is_leader(job_number):
|
||
|
|
return job_number.endswith('.1')
|
||
|
|
|
||
|
|
|
||
|
|
job_number = os.getenv(TRAVIS_JOB_NUMBER)
|
||
|
|
|
||
|
|
if not job_number:
|
||
|
|
# seems even for builds with only one job, this won't get here
|
||
|
|
log.fatal("Don't use defining leader for build without matrix")
|
||
|
|
exit(1)
|
||
|
|
elif is_leader(job_number):
|
||
|
|
log.info("This is a leader")
|
||
|
|
else:
|
||
|
|
# since python is subprocess, env variables are exported back via file
|
||
|
|
with open(".to_export_back", "w") as export_var:
|
||
|
|
export_var.write("BUILD_MINION=YES")
|
||
|
|
log.info("This is a minion")
|
||
|
|
exit(0)
|
||
|
|
|
||
|
|
|
||
|
|
class MatrixElement(object):
|
||
|
|
|
||
|
|
def __init__(self, json_raw):
|
||
|
|
self.is_finished = json_raw['finished_at'] is not None
|
||
|
|
self.is_succeeded = json_raw['result'] == 0
|
||
|
|
self.number = json_raw['number']
|
||
|
|
self.is_leader = is_leader(self.number)
|
||
|
|
|
||
|
|
|
||
|
|
def matrix_snapshot(travis_token):
|
||
|
|
"""
|
||
|
|
:return: Matrix List
|
||
|
|
"""
|
||
|
|
headers = {'content-type': 'application/json', 'Authorization': 'token {}'.format(travis_token)}
|
||
|
|
req = urllib2.Request("{0}/builds/{1}".format(travis_entry, build_id), headers=headers)
|
||
|
|
response = urllib2.urlopen(req).read()
|
||
|
|
raw_json = json.loads(response.decode('utf-8'))
|
||
|
|
matrix_without_leader = [MatrixElement(job) for job in raw_json["matrix"] if not is_leader(job['number'])]
|
||
|
|
return matrix_without_leader
|
||
|
|
|
||
|
|
|
||
|
|
def wait_others_to_finish(travis_token):
|
||
|
|
def others_finished():
|
||
|
|
"""
|
||
|
|
Dumps others to finish
|
||
|
|
Leader cannot finish, it is working now
|
||
|
|
:return: tuple(True or False, List of not finished jobs)
|
||
|
|
"""
|
||
|
|
snapshot = matrix_snapshot(travis_token)
|
||
|
|
finished = [job.is_finished for job in snapshot if not job.is_leader]
|
||
|
|
return reduce(lambda a, b: a and b, finished), [job.number for job in snapshot if
|
||
|
|
not job.is_leader and not job.is_finished]
|
||
|
|
|
||
|
|
while True:
|
||
|
|
finished, waiting_list = others_finished()
|
||
|
|
if finished:
|
||
|
|
break
|
||
|
|
log.info("Leader waits for minions {0}...".format(waiting_list)) # just in case do not get "silence timeout"
|
||
|
|
time.sleep(polling_interval)
|
||
|
|
|
||
|
|
|
||
|
|
def get_token():
|
||
|
|
assert gh_token, 'GITHUB_TOKEN is not set'
|
||
|
|
data = {"github_token": gh_token}
|
||
|
|
headers = {'content-type': 'application/json', 'User-Agent': 'Travis/1.0'}
|
||
|
|
|
||
|
|
req = urllib2.Request("{0}/auth/github".format(travis_entry), json.dumps(data).encode('utf-8'), headers)
|
||
|
|
response = urllib2.urlopen(req).read()
|
||
|
|
travis_token = json.loads(response.decode('utf-8')).get('access_token')
|
||
|
|
|
||
|
|
return travis_token
|
||
|
|
|
||
|
|
|
||
|
|
try:
|
||
|
|
token = get_token()
|
||
|
|
wait_others_to_finish(token)
|
||
|
|
|
||
|
|
final_snapshot = matrix_snapshot(token)
|
||
|
|
log.info("Final Results: {0}".format([(e.number, e.is_succeeded) for e in final_snapshot]))
|
||
|
|
|
||
|
|
BUILD_AGGREGATE_STATUS = 'BUILD_AGGREGATE_STATUS'
|
||
|
|
others_snapshot = [el for el in final_snapshot if not el.is_leader]
|
||
|
|
if reduce(lambda a, b: a and b, [e.is_succeeded for e in others_snapshot]):
|
||
|
|
os.environ[BUILD_AGGREGATE_STATUS] = "others_succeeded"
|
||
|
|
elif reduce(lambda a, b: a and b, [not e.is_succeeded for e in others_snapshot]):
|
||
|
|
log.error("Others Failed")
|
||
|
|
os.environ[BUILD_AGGREGATE_STATUS] = "others_failed"
|
||
|
|
else:
|
||
|
|
log.warn("Others Unknown")
|
||
|
|
os.environ[BUILD_AGGREGATE_STATUS] = "unknown"
|
||
|
|
# since python is subprocess, env variables are exported back via file
|
||
|
|
with open(".to_export_back", "w") as export_var:
|
||
|
|
export_var.write("BUILD_LEADER=YES {0}={1}".format(BUILD_AGGREGATE_STATUS, os.environ[BUILD_AGGREGATE_STATUS]))
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
log.fatal(e)
|