From ffbc25c1d2f67bbbe80909629f02b0eeb19d5ef5 Mon Sep 17 00:00:00 2001 From: Jeffrey Zhang Date: Fri, 10 Feb 2017 11:39:14 +0800 Subject: Get OpenStack service versions from openstack/releases repo In old implementation, version-check always get the latest versions rather than versions for certain release. it is useless for stable branch. And it parses from tarballs.openstack.org site, which is very slow. In this new implementation, version-check gets versions from openstack/releases repo, which is faster and accurate. It also support get versions for certain OpenStack release. For example, get the Ocata latest release version. ./verion-check.py --openstack-release ocata Closes-Bug: #1554337 Change-Id: Ie8951625817d82f38cf820cf9bd44ae447e2eb75 --- tools/version-check.py | 233 ++++++++++++++++++++++++++----------------------- 1 file changed, 124 insertions(+), 109 deletions(-) (limited to 'tools/version-check.py') diff --git a/tools/version-check.py b/tools/version-check.py index 58aa03e6e..a4d7cbcd0 100755 --- a/tools/version-check.py +++ b/tools/version-check.py @@ -12,17 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -import collections +import argparse import logging import os import re +import subprocess import sys -import bs4 -from oslo_config import cfg -import pkg_resources -import prettytable -import requests +import yaml + # NOTE(SamYaple): Update the search path to prefer PROJECT_ROOT as the source # of packages to import if we are using local tools instead of @@ -32,129 +30,146 @@ PROJECT_ROOT = os.path.abspath(os.path.join( if PROJECT_ROOT not in sys.path: sys.path.insert(0, PROJECT_ROOT) -from kolla.common import config as common_config - -logging.basicConfig(format="%(message)s") -LOG = logging.getLogger('version-check') - -# Filter list for non-projects -NOT_PROJECTS = [ - 'nova-novncproxy', - 'nova-spicehtml5proxy', - 'openstack-base', - 'profiles' -] -TARBALLS_BASE_URL = 'http://tarballs.openstack.org' -VERSIONS = {'local': dict()} - - -def retrieve_upstream_versions(): - upstream_versions = dict() - for project in VERSIONS['local']: - winner = None - series = VERSIONS['local'][project].split('.')[0] - base = '{}/{}'.format(TARBALLS_BASE_URL, project) - LOG.debug("Getting latest version for project %s from %s", - project, base) - r = requests.get(base) - s = bs4.BeautifulSoup(r.text, 'html.parser') - - for link in s.find_all('a'): - version = link.get('href') - if (version.endswith('.tar.gz') and - version.startswith('{}-{}'.format(project, series))): - split = '{}-|.tar.gz'.format(project) - candidate = re.split(split, version)[1] - # Ignore 2014, 2015 versions as they are older - if candidate.startswith('201'): - continue - if not winner or more_recent(candidate, winner): - winner = candidate - - if not winner: - LOG.warning("Could not find a version for %s", project) - continue - if '-' in winner: - winner = winner.split('-')[1] - upstream_versions[project] = winner - LOG.debug("Found latest version %s for project %s", winner, project) +from kolla.common import config - VERSIONS['upstream'] = collections.OrderedDict( - sorted(upstream_versions.items())) +logging.basicConfig(level=logging.INFO) +LOG = logging.getLogger(__name__) -def retrieve_local_versions(conf): - for section in common_config.SOURCES: - if section in NOT_PROJECTS: - continue +RELEASE_REPO = 'https://github.com/openstack/releases' +TARGET = '/tmp/releases' - project = section.split('-')[0] +SKIP_PROJECTS = { + 'rally': 'Rally is not managed by openstack/releases project', + 'nova-novncproxy': ('nova-novncproxy is not managed by' + ' openstack/releases project'), + 'nova-spicehtml5proxy': ('nova-spicehtml5proxy is not managed' + ' by openstack/releases project'), + 'openstack-base': 'There is no tag for requirements project', + 'tempest': 'tempest is not managed by openstack/releases project' +} - if section not in conf.list_all_sections(): - LOG.debug("Project %s not found in configuration file, using " - "default from kolla.common.config", project) - raw_version = common_config.SOURCES[section]['location'] - else: - raw_version = getattr(conf, section).location +RE_DEFAULT_BRANCH = re.compile('^defaultbranch=stable/(.*)') +RE_FILENAME = re.compile('(?P.*)-(?P[^-]*).tar.gz') - version = raw_version.split('/')[-1].split('.tar.gz')[0] - if '-' in version: - version = version.split('-')[1] - LOG.debug("Use local version %s for project %s", version, project) - VERSIONS['local'][project] = version +def update_releases_repo(): + if not os.path.exists(TARGET): + cmd = ['git', 'clone', RELEASE_REPO, TARGET] + else: + cmd = ['git', '--git-dir', os.path.join(TARGET, '.git'), '--work-tree', + TARGET, 'pull'] + subprocess.call(cmd) -def more_recent(candidate, reference): - return pkg_resources.parse_version(candidate) > \ - pkg_resources.parse_version(reference) +def get_default_branch(): + gitreview_file = os.path.join(PROJECT_ROOT, '.gitreview') + if not os.path.exists(gitreview_file): + return + with open(gitreview_file, 'r') as gitreview: + for line in gitreview: + branches = RE_DEFAULT_BRANCH.findall(line) + if branches: + return branches[0] -def diff_link(project, old_ref, new_ref): - return "https://github.com/openstack/{}/compare/{}...{}".format( - project, old_ref, new_ref) +def load_all_info(openstack_release): + projects = {} + release_path = os.path.join(TARGET, 'deliverables', openstack_release) -def compare_versions(): - up_to_date = True - result = prettytable.PrettyTable(["Project", "Current version", - "Latest version", "Comparing changes"]) - result.align = "l" + if not os.path.exists(release_path): + raise ValueError( + 'Can not find openstack release: "%s"' % openstack_release) - for project in VERSIONS['upstream']: - if project not in VERSIONS['local']: + for deliverable in os.listdir(release_path): + if not deliverable.endswith('.yaml'): continue - - upstream_version = VERSIONS['upstream'][project] - local_version = VERSIONS['local'][project] - - if more_recent(upstream_version, local_version): - result.add_row([ - project, - VERSIONS['local'][project], - VERSIONS['upstream'][project], - diff_link(project, local_version, upstream_version) - ]) - up_to_date = False - - if up_to_date: - result = "Everything is up to date" - - print(result) + with open(os.path.join(release_path, deliverable)) as f: + info = yaml.safe_load(f) + if 'releases' in info and len(info['releases']) > 0: + latest_release = info['releases'][-1] + latest_version = latest_release['version'] + for project in latest_release['projects']: + project_name = project['repo'].split('/')[-1] + projects[project_name] = latest_version + if 'tarball-base' in project: + projects[project['tarball-base']] = latest_version + return projects def main(): - conf = cfg.ConfigOpts() - common_config.parse(conf, sys.argv[1:], prog='version-check') - - if conf.debug: - LOG.setLevel(logging.DEBUG) - - retrieve_local_versions(conf) - retrieve_upstream_versions() + parser = argparse.ArgumentParser( + description='Check and update OpenStack service version.') + parser.add_argument('--openstack-release', '-r', + default=get_default_branch(), + help='OpenStack release name') + parser.add_argument('--include-independent', '-i', + default=False, action='store_true', + help='Whether update independent projects') + parser.add_argument('--check', '-c', + default=False, action='store_true', + help='Run without update config.py file') + conf = parser.parse_args(sys.argv[1:]) + + if not conf.openstack_release: + raise ValueError('Can not detect openstack release. Please assign' + ' it through "--openstack-release" parameter') + + LOG.info('Update using openstack release: "%s"', conf.openstack_release) + + if conf.check: + LOG.info('Run in check only mode') + + update_releases_repo() + + projects = load_all_info(openstack_release=conf.openstack_release) + independents_projects = load_all_info(openstack_release='_independent') + + with open(os.path.join(PROJECT_ROOT, 'kolla/common/config.py')) as f: + config_py = f.read() + + for key in sorted(config.SOURCES): + independent_project = False + value = config.SOURCES[key] + if key in SKIP_PROJECTS: + LOG.info('%s is skiped: %s', key, SKIP_PROJECTS[key]) + continue + # get project name from location + location = value['location'] + filename = os.path.basename(location) + match = RE_FILENAME.match(filename) + if match: + project_name, old_tag = match.groups() + else: + raise ValueError('Can not parse "%s"' % filename) + + latest_tag = projects.get(project_name, None) + if not latest_tag: + latest_tag = independents_projects.get(project_name, None) + if latest_tag: + independent_project = True + else: + LOG.warning('Can not find %s project release', project_name) + continue + if latest_tag and old_tag != latest_tag: + if independent_project and not conf.include_independent: + LOG.warning('%s is an independent project, please update it' + ' manually. Possible need upgrade from %s to %s', + project_name, old_tag, latest_tag) + continue + LOG.info('Update %s from %s to %s', project_name, old_tag, + latest_tag) + old_str = '{}-{}'.format(project_name, old_tag) + new_str = '{}-{}'.format(project_name, latest_tag) + config_py = config_py.replace(old_str, new_str) + + if not conf.check: + with open(os.path.join(PROJECT_ROOT, 'kolla/common/config.py'), + 'w') as f: + f.write(config_py) - compare_versions() if __name__ == '__main__': main() -- cgit v1.2.3