diff options
| -rw-r--r-- | etc/kolla/kolla-build.conf | 12 | ||||
| -rwxr-xr-x | kolla/cmd/build.py | 332 | ||||
| -rw-r--r-- | requirements.txt | 1 |
3 files changed, 178 insertions, 167 deletions
diff --git a/etc/kolla/kolla-build.conf b/etc/kolla/kolla-build.conf index 39d210925..e75788ddc 100644 --- a/etc/kolla/kolla-build.conf +++ b/etc/kolla/kolla-build.conf @@ -1,6 +1,6 @@ # Configuration for kolla build script -[kolla-build] +[DEFAULT] #The MAINTAINER field #maintainer = Kolla Project (https://launchpad.net/kolla) @@ -58,11 +58,11 @@ # Preset build profiles can be set here to easily build common sets of images [profiles] -infra = ceph,data,mariadb,haproxy,keepalived,kolla-ansible,memcached,mongodb,openvswitch,rabbitmq,rsyslog -main = cinder,ceilometer,glance,heat,horizon,keystone,neutron,nova,swift -aux = aodh,designate,gnocchi,ironic,magnum,mistral,trove,zaqar -default = data,kolla-ansible,glance,haproxy,heat,horizon,keepalived,keystone,memcached,mariadb,neutron,nova,openvswitch,rabbitmq,rsyslog -gate = ceph,cinder,data,dind,glance,haproxy,heat,horizon,keepalived,keystone,kolla-ansible,mariadb,memcached,neutron,nova,openvswitch,rabbitmq,rsyslog +#infra = ceph,data,mariadb,haproxy,keepalived,kolla-ansible,memcached,mongodb,openvswitch,rabbitmq,rsyslog +#main = cinder,ceilometer,glance,heat,horizon,keystone,neutron,nova,swift +#aux = aodh,designate,gnocchi,ironic,magnum,mistral,trove,zaqar +#default = data,kolla-ansible,glance,haproxy,heat,horizon,keepalived,keystone,memcached,mariadb,neutron,nova,openvswitch,rabbitmq,rsyslog +#gate = ceph,cinder,data,dind,glance,haproxy,heat,horizon,keepalived,keystone,kolla-ansible,mariadb,memcached,neutron,nova,openvswitch,rabbitmq,rsyslog # Provide location of sources for source install builds. # Example: diff --git a/kolla/cmd/build.py b/kolla/cmd/build.py index 842967d54..7deb01891 100755 --- a/kolla/cmd/build.py +++ b/kolla/cmd/build.py @@ -14,7 +14,6 @@ # TODO(jpeeler): Add clean up handler for SIGINT -import argparse import datetime import errno import json @@ -34,6 +33,8 @@ import time import docker import git import jinja2 +from oslo_config import cfg +from oslo_config import types from requests.exceptions import ConnectionError import six @@ -46,6 +47,7 @@ signal.signal(signal.SIGINT, signal.SIG_DFL) RDO_MIRROR = "http://trunk.rdoproject.org/centos7" DELOREAN = "{}/current/delorean.repo".format(RDO_MIRROR) DELOREAN_DEPS = "{}/delorean-deps.repo".format(RDO_MIRROR) +INSTALL_TYPE_CHOICES = ['binary', 'source', 'rdo', 'rhos'] class KollaDirNotFoundException(Exception): @@ -107,13 +109,12 @@ class PushThread(Thread): class WorkerThread(Thread): - def __init__(self, queue, push_queue, config): - self.config = config + def __init__(self, queue, push_queue, conf): + self.conf = conf self.queue = queue self.push_queue = push_queue - self.nocache = config['no_cache'] - self.forcerm = not config['keep'] - self.retries = config['retries'] + self.nocache = not conf.cache or conf.no_cache + self.forcerm = not conf.keep self.dc = docker_client() super(WorkerThread, self).__init__() @@ -133,7 +134,7 @@ class WorkerThread(Thread): while True: try: image = self.queue.get() - for _ in range(self.retries + 1): + for _ in range(self.conf.retries + 1): self.builder(image) if image['status'] in ['built', 'unmatched', 'parent_error']: @@ -273,7 +274,7 @@ class WorkerThread(Thread): image['status'] = "built" LOG.info('%s:Built', image['name']) - if self.config['push']: + if self.conf.push: self.push_queue.put(image) @@ -320,121 +321,25 @@ def find_config_file(filename): ) -def merge_args_and_config(settings_from_config_file): - parser = argparse.ArgumentParser(description='Kolla build script') - - defaults = { - "base": "centos", - "base_tag": "latest", - "install_type": "binary", - "keep": False, - "maintainer": "Kolla Project (https://launchpad.net/kolla)", - "namespace": "kollaglue", - "no_cache": False, - "push": False, - "push_threads": 1, - "registry": None, - "retries": 3, - "rpm_setup_config": '', - "tag": get_kolla_version(), - "threads": 8 - } - defaults.update(settings_from_config_file.items('kolla-build')) - parser.set_defaults(**defaults) - - parser.add_argument('-b', '--base', - help='The base distro to use when building', - type=str) - parser.add_argument('--base-tag', - help='The base distro image tag', - type=str) - parser.add_argument('-d', '--debug', - help='Turn on debugging log level', - action='store_true') - parser.add_argument('-i', '--include-header', - help=('Path to custom file to be added at ' - 'beginning of base Dockerfile'), - type=str) - parser.add_argument('-I', '--include-footer', - help=('Path to custom file to be added at ' - 'end of Dockerfiles for final images'), - type=str) - parser.add_argument('--keep', - help='Keep failed intermediate containers', - action='store_true') - parser.add_argument('-n', '--namespace', - help='Set the Docker namespace name', - type=str) - parser.add_argument('--no-cache', - help='Do not use the Docker cache when building', - action='store_true') - parser.add_argument('-p', '--profile', - help=('Build a pre-defined set of images, see ' - '[profiles] section in ' - '{}'.format( - find_config_file('kolla-build.conf'))), - type=str, - action='append') - parser.add_argument('--push', - help='Push images after building', - action='store_true') - parser.add_argument('--push-threads', - help=('The number of threads to use while pushing' - ' images.(NOTE: Docker can not handle' - 'threading push properly.)'), - type=int) - parser.add_argument('-r', '--retries', - help='The number of times to retry while building', - type=int) - parser.add_argument('regex', - help=('Build only images matching ' - 'regex and its dependencies'), - nargs='*') - parser.add_argument('--registry', - help=("the docker registry host"), - type=str) - parser.add_argument('-t', '--type', - help='The method of the Openstack install: binary,' - ' source, rdo, or rhos', - type=str, - dest='install_type') - parser.add_argument('-T', '--threads', - help='The number of threads to use while building.' - ' (Note: setting to one will allow real time' - ' logging.)', - type=int) - parser.add_argument('--tag', - help='Set the Docker tag', - type=str) - parser.add_argument('--template', - help='DEPRECATED: All Dockerfiles are templates', - action='store_true', - default=True) - parser.add_argument('--template-only', - help=("Don't build images. Generate Dockerfile only"), - action='store_true') - return vars(parser.parse_args()) - - class KollaWorker(object): - def __init__(self, config): + def __init__(self, conf): + self.conf = conf self.base_dir = os.path.abspath(find_base_dir()) LOG.debug("Kolla base directory: " + self.base_dir) self.images_dir = os.path.join(self.base_dir, 'docker') - self.registry = config['registry'] + + self.registry = conf.registry if self.registry: - self.namespace = self.registry + '/' + config['namespace'] + self.namespace = self.registry + '/' + conf.namespace else: - self.namespace = config['namespace'] - self.base = config['base'] - self.base_tag = config['base_tag'] - self.install_type = config['install_type'] - self.tag = config['tag'] + self.namespace = conf.namespace + self.base = conf.base + self.base_tag = conf.base_tag + self.install_type = conf.install_type + self.tag = conf.tag self.images = list() - rpm_setup_config = [cfg.strip() - for cfg in config['rpm_setup_config'].split(',') - if cfg] + rpm_setup_config = filter(None, conf.rpm_setup_config) self.rpm_setup = self.build_rpm_setup(rpm_setup_config) if self.install_type == 'binary': @@ -454,17 +359,13 @@ class KollaWorker(object): self.image_prefix = self.base + '-' + self.install_type + '-' - self.tag = config['tag'] - self.include_header = config['include_header'] - self.include_footer = config['include_footer'] - self.regex = config['regex'] - self.profile = config['profile'] - self.source_location = six.moves.configparser.SafeConfigParser() - self.source_location.read(find_config_file('kolla-build.conf')) + self.include_header = conf.include_header + self.include_footer = conf.include_footer + self.regex = conf.regex self.image_statuses_bad = dict() self.image_statuses_good = dict() self.image_statuses_unmatched = dict() - self.maintainer = config['maintainer'] + self.maintainer = conf.maintainer def build_rpm_setup(self, rpm_setup_config): """Generates a list of docker commands based on provided configuration. @@ -474,9 +375,6 @@ class KollaWorker(object): """ rpm_setup = list() - if not rpm_setup_config: - rpm_setup_config = [DELOREAN, DELOREAN_DEPS] - for config in rpm_setup_config: if config.endswith('.rpm'): # RPM files can be installed with yum from file path or url @@ -566,19 +464,9 @@ class KollaWorker(object): if self.regex: filter_ += self.regex - if self.profile: - for profile in self.profile: - try: - filter_ += self.source_location.get('profiles', - profile - ).split(',') - except six.moves.configparser.NoSectionError: - LOG.error('No [profiles] section found in %s', - find_config_file('kolla-build.conf')) - except six.moves.configparser.NoOptionError: - LOG.error('No profile named "%s" found in %s', - self.profile, - find_config_file('kolla-build.conf')) + if self.conf.profile: + for profile in self.conf.profile: + filter_ += self.conf.profiles[profile] if filter_: patterns = re.compile(r"|".join(filter_).join('()')) @@ -650,17 +538,14 @@ class KollaWorker(object): def build_image_list(self): def process_source_installation(image, section): installation = dict() - try: - installation['type'] = \ - self.source_location.get(section, 'type') - installation['source'] = \ - self.source_location.get(section, 'location') + if section not in self.conf.list_all_sections(): + LOG.debug('%s:No source location found', section) + else: + installation['type'] = self.conf[section]['type'] + installation['source'] = self.conf[section]['location'] installation['name'] = section if installation['type'] == 'git': - installation['reference'] = \ - self.source_location.get(section, 'reference') - except six.moves.configparser.NoSectionError: - LOG.debug('%s:No source location found', section) + installation['reference'] = self.conf[section]['reference'] return installation for path in self.docker_build_paths: @@ -681,12 +566,17 @@ class KollaWorker(object): image['plugins'] = list() if self.install_type == 'source': - image['source'] = \ - process_source_installation(image, image['name']) + self.conf.register_opts([ + cfg.StrOpt('type'), + cfg.StrOpt('location'), + cfg.StrOpt('reference') + ], image['name']) + image['source'] = process_source_installation(image, + image['name']) for plugin in [match.group(0) for match in (re.search('{}-plugin-.+'.format(image['name']), section) for section in - self.source_location.sections()) if match]: + self.conf.list_all_sections()) if match]: image['plugins'].append( process_source_installation(image, plugin)) @@ -725,19 +615,138 @@ class KollaWorker(object): return queue +def get_conf(): + conf = cfg.ConfigOpts() + + _kolla_profile_opts = [ + cfg.ListOpt('infra', + default=['ceph', 'data', 'mariadb', 'haproxy', + 'keepalived', 'kolla-ansible', 'memcached', + 'mongodb', 'openvswitch', 'rabbitmq', 'rsyslog']), + cfg.ListOpt('main', + default=['cinder', 'ceilometer', 'glance', 'heat', + 'horizon', 'keystone', 'neutron', 'nova', + 'swift']), + cfg.ListOpt('aux', + default=['aodh', 'designate', 'gnocchi', 'ironic', + 'magnum', 'mistral', 'trove,' 'zaqar']), + cfg.ListOpt('default', + default=['data', 'kolla-ansible', 'glance', 'haproxy', + 'heat', 'horizon', 'keepalived', 'keystone', + 'memcached', 'mariadb', 'neutron', 'nova', + 'openvswitch', 'rabbitmq', 'rsyslog']), + cfg.ListOpt('gate', + default=['ceph', 'cinder', 'data', 'dind', 'glance', + 'haproxy', 'heat', 'horizon', 'keepalived', + 'keystone', 'kolla-ansible', 'mariadb', + 'memcached', 'neutron', 'nova', 'openvswitch', + 'rabbitmq', 'rsyslog']) + ] + + _kolla_cli_opts = [ + cfg.StrOpt('base', short='b', default='centos', + deprecated_group='kolla-build', + help='The base distro to use when building'), + cfg.StrOpt('base_tag', default='latest', + deprecated_group='kolla-build', + help='The base distro image tag'), + cfg.BoolOpt('debug', short='d', default=False, + deprecated_group='kolla-build', + help='Turn on debugging log level'), + cfg.StrOpt('include-header', short='i', + deprecated_group='kolla-build', + help=('Path to custom file to be added at ' + 'beginning of base Dockerfile')), + cfg.StrOpt('include-footer', short='I', + deprecated_group='kolla-build', + help=('Path to custom file to be added at ' + 'end of Dockerfiles for final images')), + cfg.BoolOpt('keep', default=False, + deprecated_group='kolla-build', + help='Keep failed intermediate containers'), + cfg.StrOpt('namespace', short='n', default='kollaglue', + deprecated_group='kolla-build', + help='The Docker namespace name'), + cfg.BoolOpt('cache', default=True, + help='Use the Docker cache when building', + ), + cfg.BoolOpt('no-cache', default=False, + help='Do not use the Docker cache when building', + deprecated_for_removal=True), + cfg.MultiOpt('profile', types.String(), short='p', + deprecated_group='kolla-build', + help=('Build a pre-defined set of images, see [profiles]' + ' section in {}. The default profiles are:' + ' {}'.format( + find_config_file('kolla-build.conf'), + ', '.join( + [opt.name for opt in _kolla_profile_opts]) + ))), + cfg.BoolOpt('push', default=False, + deprecated_group='kolla-build', + help='Push images after building'), + cfg.IntOpt('push-threads', default=1, min=1, + deprecated_group='kolla-build', + help=('The number of threads to user while pushing' + ' Images. Note: Docker can not handle threading' + ' push properly.')), + cfg.IntOpt('retries', short='r', default=3, min=0, + deprecated_group='kolla-build', + help='The number of times to retry while building'), + cfg.MultiOpt('regex', types.String(), positional=True, + help=('Build only images matching regex and its' + ' dependencies')), + cfg.StrOpt('registry', deprecated_group='kolla-build', + help=('The docker registry host. The default registry host' + ' is Docker Hub')), + cfg.StrOpt('type', short='t', default='binary', + choices=INSTALL_TYPE_CHOICES, + dest='install_type', deprecated_group='kolla-build', + help=('The method of the Openstack install. The valid' + ' types are: {}'.format( + ', '.join(INSTALL_TYPE_CHOICES)))), + cfg.IntOpt('threads', short='T', default=8, min=1, + deprecated_group='kolla-build', + help=('The number of threads to use while building.' + ' (Note: setting to one will allow real time' + ' logging.)')), + cfg.StrOpt('tag', default=get_kolla_version(), + deprecated_group='kolla-build', + help='The Docker tag'), + cfg.BoolOpt('template-only', default=False, + deprecated_group='kolla-build', + help=("Don't build images. Generate Dockerfile only")), + ] + + _kolla_base_opts = [ + cfg.StrOpt('maintainer', deprecated_group='kolla-build', + default='Kolla Project (https://launchpad.net/kolla)', + help='The MAINTAINER field'), + cfg.ListOpt('rpm_setup_config', default=[DELOREAN, DELOREAN_DEPS], + deprecated_group='kolla-build', + help=('Comma separated list of .rpm or .repo file(s)' + 'or URL(s) to install before building containers')) + ] + conf.register_cli_opts(_kolla_cli_opts) + conf.register_opts(_kolla_profile_opts, group='profiles') + conf.register_opts(_kolla_base_opts) + conf(sys.argv[1:], + default_config_files=[find_config_file('kolla-build.conf')]) + return conf + + def main(): - build_config = six.moves.configparser.SafeConfigParser() - build_config.read(find_config_file('kolla-build.conf')) - config = merge_args_and_config(build_config) - if config['debug']: + conf = get_conf() + + if conf.debug: LOG.setLevel(logging.DEBUG) - kolla = KollaWorker(config) + kolla = KollaWorker(conf) kolla.setup_working_dir() kolla.find_dockerfiles() kolla.create_dockerfiles() - if config['template_only']: + if conf.template_only: LOG.info('Dockerfiles are generated in %s', kolla.working_dir) return @@ -748,13 +757,13 @@ def main(): queue = kolla.build_queue() push_queue = six.moves.queue.Queue() - for x in six.moves.xrange(config['threads']): - worker = WorkerThread(queue, push_queue, config) + for x in six.moves.xrange(conf.threads): + worker = WorkerThread(queue, push_queue, conf) worker.setDaemon(True) worker.start() - for x in six.moves.xrange(config['push_threads']): - push_thread = PushThread(config, push_queue) + for x in six.moves.xrange(conf.push_threads): + push_thread = PushThread(conf, push_queue) push_thread.start() # block until queue is empty @@ -766,5 +775,6 @@ def main(): return kolla.get_image_statuses() + if __name__ == '__main__': main() diff --git a/requirements.txt b/requirements.txt index f9c5e75d1..a1b735afd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,4 @@ Jinja2>=2.8 # BSD License (3 clause) gitdb>=0.6.4 # BSD License (3 clause) GitPython>=1.0.1 # BSD License (3 clause) six>=1.9.0 +oslo.config>=2.7.0 # Apache-2.0 |