diff options
| -rwxr-xr-x | kolla/cmd/build.py | 912 | ||||
| -rw-r--r-- | kolla/image/__init__.py | 0 | ||||
| -rwxr-xr-x | kolla/image/build.py | 924 | ||||
| -rw-r--r-- | kolla/tests/base.py | 2 | ||||
| -rw-r--r-- | kolla/tests/test_build.py | 11 | ||||
| -rw-r--r-- | tests/test_build.py | 2 |
6 files changed, 934 insertions, 917 deletions
diff --git a/kolla/cmd/build.py b/kolla/cmd/build.py index a84a49f6b..97dbafc67 100755 --- a/kolla/cmd/build.py +++ b/kolla/cmd/build.py @@ -12,919 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - -import datetime -import errno -import graphviz -import json -import logging -import os -import pprint -import re -import requests -import shutil -import signal -import sys -import tarfile -import tempfile -import threading -import time - -import docker -import git -import jinja2 -from oslo_config import cfg -from requests import exceptions as requests_exc -import six - -PROJECT_ROOT = os.path.abspath(os.path.join( - os.path.dirname(os.path.realpath(__file__)), '../..')) - -# NOTE(SamYaple): Update the search patch to prefer PROJECT_ROOT as the source -# of packages to import if we are using local tools/build.py -# instead of pip installed kolla-build tool -if PROJECT_ROOT not in sys.path: - sys.path.insert(0, PROJECT_ROOT) - -from kolla.common import config as common_config -from kolla.common import task -from kolla import version - -logging.basicConfig() -LOG = logging.getLogger(__name__) -LOG.setLevel(logging.INFO) - - -def handle_ctrlc(single, frame): - kollaobj = frame.f_locals['kolla'] - kollaobj.cleanup() - sys.exit(1) - -signal.signal(signal.SIGINT, handle_ctrlc) - - -class KollaDirNotFoundException(Exception): - pass - - -class KollaUnknownBuildTypeException(Exception): - pass - - -class KollaMismatchBaseTypeException(Exception): - pass - - -class KollaRpmSetupUnknownConfig(Exception): - pass - - -# Image status constants. -# -# TODO(harlowja): use enum lib in the future?? -STATUS_CONNECTION_ERROR = 'connection_error' -STATUS_PUSH_ERROR = 'push_error' -STATUS_ERROR = 'error' -STATUS_PARENT_ERROR = 'parent_error' -STATUS_BUILT = 'built' -STATUS_BUILDING = 'building' -STATUS_UNMATCHED = 'unmatched' -STATUS_MATCHED = 'matched' -STATUS_UNPROCESSED = 'unprocessed' - -# All error status constants. -STATUS_ERRORS = (STATUS_CONNECTION_ERROR, STATUS_PUSH_ERROR, - STATUS_ERROR, STATUS_PARENT_ERROR) - - -class Recorder(object): - """Recorder/buffer of (unicode) log lines for eventual display.""" - - def __init__(self): - self._lines = [] - - def write(self, text=""): - if isinstance(text, six.text_type): - self._lines.append(text) - elif isinstance(text, six.binary_type): - self._lines.append(text.decode('utf8')) - elif isinstance(text, Recorder): - self._lines.extend(text._lines) - else: - self.write(text=str(text)) - - def clear(self): - self._lines = [] - - def __iter__(self): - for line in self._lines: - yield line - - def __str__(self): - return u"\n".join(self._lines) - - -def docker_client(): - try: - docker_kwargs = docker.utils.kwargs_from_env() - return docker.Client(version='auto', **docker_kwargs) - except docker.errors.DockerException: - LOG.exception('Can not communicate with docker service.' - 'Please check docker service is running without errors') - sys.exit(1) - - -class Image(object): - def __init__(self, name, canonical_name, path, parent_name='', - status=STATUS_UNPROCESSED, parent=None, source=None): - self.name = name - self.canonical_name = canonical_name - self.path = path - self.status = status - self.parent = parent - self.source = source - self.parent_name = parent_name - self.logs = Recorder() - self.push_logs = Recorder() - self.children = [] - self.plugins = [] - - def __repr__(self): - return ("Image(%s, %s, %s, parent_name=%s," - " status=%s, parent=%s, source=%s)") % ( - self.name, self.canonical_name, self.path, - self.parent_name, self.status, self.parent, self.source) - - -class PushIntoQueueTask(task.Task): - """Task that pushes some other task into a queue.""" - - def __init__(self, push_task, push_queue): - super(PushIntoQueueTask, self).__init__() - self.push_task = push_task - self.push_queue = push_queue - - @property - def name(self): - return 'PushIntoQueueTask(%s=>%s)' % (self.push_task.name, - self.push_queue) - - def run(self): - self.push_queue.put(self.push_task) - self.success = True - - -class PushTask(task.Task): - """Task that pushes a image to a docker repository.""" - - def __init__(self, conf, image): - super(PushTask, self).__init__() - self.dc = docker_client() - self.conf = conf - self.image = image - - @property - def name(self): - return 'PushTask(%s)' % self.image.name - - def run(self): - image = self.image - try: - LOG.debug('%s:Try to push the image', image.name) - self.push_image(image) - except requests_exc.ConnectionError: - LOG.exception('%s:Make sure Docker is running and that you' - ' have the correct privileges to run Docker' - ' (root)', image.name) - image.status = STATUS_CONNECTION_ERROR - except Exception: - LOG.exception('%s:Unknown error when pushing', image.name) - image.status = STATUS_PUSH_ERROR - finally: - if (image.status not in STATUS_ERRORS - and image.status != STATUS_UNPROCESSED): - LOG.info('%s:Pushed successfully', image.name) - self.success = True - else: - self.success = False - - def push_image(self, image): - image.push_logs.clear() - for response in self.dc.push(image.canonical_name, - stream=True, - insecure_registry=True): - stream = json.loads(response) - if 'stream' in stream: - image.push_logs.write(image.logs) - image.push_logs.write(stream['stream']) - LOG.info('%s', stream['stream']) - elif 'errorDetail' in stream: - image.status = STATUS_ERROR - LOG.error(stream['errorDetail']['message']) - - -class BuildTask(task.Task): - """Task that builds out an image.""" - - def __init__(self, conf, image, push_queue): - super(BuildTask, self).__init__() - self.conf = conf - self.image = image - self.dc = docker_client() - self.push_queue = push_queue - self.forcerm = not conf.keep - - @property - def name(self): - return 'BuildTask(%s)' % self.image.name - - def run(self): - self.builder(self.image) - if self.image.status == STATUS_BUILT: - self.success = True - - @property - def followups(self): - followups = [] - if self.conf.push and self.success: - followups.extend([ - # If we are supposed to push the image into a docker - # repository, then make sure we do that... - PushIntoQueueTask( - PushTask(self.conf, self.image), - self.push_queue), - ]) - if self.image.children and self.success: - for image in self.image.children: - followups.append(BuildTask(self.conf, image, self.push_queue)) - return followups - - def process_source(self, image, source): - dest_archive = os.path.join(image.path, source['name'] + '-archive') - - if source.get('type') == 'url': - LOG.debug("%s:Getting archive from %s", image.name, - source['source']) - try: - r = requests.get(source['source'], timeout=self.conf.timeout) - except requests_exc.Timeout: - LOG.exception('Request timed out while getting archive' - ' from %s', source['source']) - image.status = STATUS_ERROR - image.logs.clear() - return - - if r.status_code == 200: - with open(dest_archive, 'wb') as f: - f.write(r.content) - else: - LOG.error('%s:Failed to download archive: status_code %s', - image.name, r.status_code) - image.status = STATUS_ERROR - return - - elif source.get('type') == 'git': - clone_dir = '{}-{}'.format(dest_archive, - source['reference'].replace('/', '-')) - try: - LOG.debug("%s:Cloning from %s", image.name, - source['source']) - git.Git().clone(source['source'], clone_dir) - git.Git(clone_dir).checkout(source['reference']) - reference_sha = git.Git(clone_dir).rev_parse('HEAD') - LOG.debug("%s:Git checkout by reference %s (%s)", - image.name, source['reference'], reference_sha) - except Exception as e: - LOG.error("%s:Failed to get source from git", image.name) - LOG.error("%s:Error:%s", image.name, str(e)) - # clean-up clone folder to retry - shutil.rmtree(clone_dir) - image.status = STATUS_ERROR - return - - with tarfile.open(dest_archive, 'w') as tar: - tar.add(clone_dir, arcname=os.path.basename(clone_dir)) - - elif source.get('type') == 'local': - LOG.debug("%s:Getting local archive from %s", image.name, - source['source']) - if os.path.isdir(source['source']): - with tarfile.open(dest_archive, 'w') as tar: - tar.add(source['source'], - arcname=os.path.basename(source['source'])) - else: - shutil.copyfile(source['source'], dest_archive) - - else: - LOG.error("%s:Wrong source type '%s'", image.name, - source.get('type')) - image.status = STATUS_ERROR - return - - # Set time on destination archive to epoch 0 - os.utime(dest_archive, (0, 0)) - - return dest_archive - - def update_buildargs(self): - buildargs = dict() - if self.conf.build_args: - buildargs = dict(self.conf.build_args) - - proxy_vars = ('HTTP_PROXY', 'http_proxy', 'HTTPS_PROXY', - 'https_proxy', 'FTP_PROXY', 'ftp_proxy', - 'NO_PROXY', 'no_proxy') - - for proxy_var in proxy_vars: - if proxy_var in os.environ and proxy_var not in buildargs: - buildargs[proxy_var] = os.environ.get(proxy_var) - - if not buildargs: - return None - return buildargs - - def builder(self, image): - LOG.debug('%s:Processing', image.name) - if image.status == STATUS_UNMATCHED: - return - - if (image.parent is not None and - image.parent.status in STATUS_ERRORS): - LOG.error('%s:Parent image error\'d with message "%s"', - image.name, image.parent.status) - image.status = STATUS_PARENT_ERROR - return - - image.status = STATUS_BUILDING - LOG.info('%s:Building', image.name) - - if image.source and 'source' in image.source: - self.process_source(image, image.source) - if image.status in STATUS_ERRORS: - return - - plugin_archives = list() - plugins_path = os.path.join(image.path, 'plugins') - for plugin in image.plugins: - archive_path = self.process_source(image, plugin) - if image.status in STATUS_ERRORS: - return - plugin_archives.append(archive_path) - if plugin_archives: - for plugin_archive in plugin_archives: - with tarfile.open(plugin_archive, 'r') as plugin_archive_tar: - plugin_archive_tar.extractall(path=plugins_path) - else: - try: - os.mkdir(plugins_path) - except OSError as e: - if e.errno == errno.EEXIST: - LOG.info('Directory %s already exist. Skipping.', - plugins_path) - else: - LOG.error('Failed to create directory %s: %s', - plugins_path, e) - image.status = STATUS_CONNECTION_ERROR - return - with tarfile.open(os.path.join(image.path, 'plugins-archive'), - 'w') as tar: - tar.add(plugins_path, arcname='plugins') - - # Pull the latest image for the base distro only - pull = True if image.parent is None else False - - image.logs.clear() - buildargs = self.update_buildargs() - for response in self.dc.build(path=image.path, - tag=image.canonical_name, - nocache=not self.conf.cache, - rm=True, - pull=pull, - forcerm=self.forcerm, - buildargs=buildargs): - stream = json.loads(response.decode('utf-8')) - - if 'stream' in stream: - image.logs.write(stream['stream']) - for line in stream['stream'].split('\n'): - if line: - LOG.info('%s:%s', image.name, line) - - if 'errorDetail' in stream: - image.status = STATUS_ERROR - LOG.error('%s:Error\'d with the following message', - image.name) - for line in stream['errorDetail']['message'].split('\n'): - if line: - LOG.error('%s:%s', image.name, line) - return - - image.status = STATUS_BUILT - - LOG.info('%s:Built', image.name) - - -class WorkerThread(threading.Thread): - """Thread that executes tasks until the queue provides a tombstone.""" - - #: Object to be put on worker queues to get them to die. - tombstone = object() - - def __init__(self, conf, queue): - super(WorkerThread, self).__init__() - self.queue = queue - self.conf = conf - - def run(self): - while True: - task = self.queue.get() - if task is self.tombstone: - # Ensure any other threads also get the tombstone. - self.queue.put(task) - break - try: - for attempt in six.moves.range(self.conf.retries + 1): - if attempt > 0: - LOG.debug("Attempting to run task %s for the %s time", - task.name, attempt + 1) - else: - LOG.debug("Attempting to run task %s for the first" - " time", task.name) - try: - task.run() - if task.success: - break - except Exception: - LOG.exception('Unhandled error when running %s', - task.name) - # try again... - task.reset() - if task.success: - for next_task in task.followups: - LOG.debug('Added next task %s to queue', - next_task.name) - self.queue.put(next_task) - finally: - self.queue.task_done() - - -class KollaWorker(object): - - def __init__(self, conf): - self.conf = conf - self.images_dir = self._get_images_dir() - self.registry = conf.registry - if self.registry: - self.namespace = self.registry + '/' + conf.namespace - else: - self.namespace = conf.namespace - self.base = conf.base - self.base_tag = conf.base_tag - self.install_type = conf.install_type - self.tag = conf.tag - self.images = list() - rpm_setup_config = filter(None, conf.rpm_setup_config) - self.rpm_setup = self.build_rpm_setup(rpm_setup_config) - - rh_base = ['fedora', 'centos', 'oraclelinux', 'rhel'] - rh_type = ['source', 'binary', 'rdo', 'rhos'] - deb_base = ['ubuntu', 'debian'] - deb_type = ['source', 'binary'] - - if not ((self.base in rh_base and self.install_type in rh_type) or - (self.base in deb_base and self.install_type in deb_type)): - raise KollaMismatchBaseTypeException( - '{} is unavailable for {}'.format(self.install_type, self.base) - ) - - if self.install_type == 'binary': - self.install_metatype = 'rdo' - elif self.install_type == 'source': - self.install_metatype = 'mixed' - elif self.install_type == 'rdo': - self.install_type = 'binary' - self.install_metatype = 'rdo' - elif self.install_type == 'rhos': - self.install_type = 'binary' - self.install_metatype = 'rhos' - else: - raise KollaUnknownBuildTypeException( - 'Unknown install type' - ) - - self.image_prefix = self.base + '-' + self.install_type + '-' - - self.include_header = conf.include_header - self.include_footer = conf.include_footer - self.regex = conf.regex - self.image_statuses_bad = dict() - self.image_statuses_good = dict() - self.image_statuses_unmatched = dict() - self.maintainer = conf.maintainer - - def _get_images_dir(self): - possible_paths = ( - PROJECT_ROOT, - os.path.join(sys.prefix, 'share/kolla'), - os.path.join(sys.prefix, 'local/share/kolla')) - - for path in possible_paths: - image_path = os.path.join(path, 'docker') - # NOTE(SamYaple): We explicty check for the base folder to ensure - # this is the correct path - # TODO(SamYaple): Improve this to make this safer - if os.path.exists(os.path.join(image_path, 'base')): - LOG.info('Found the docker image folder at %s', image_path) - return image_path - else: - raise KollaDirNotFoundException('Image dir can not be found') - - def build_rpm_setup(self, rpm_setup_config): - """Generates a list of docker commands based on provided configuration. - - :param rpm_setup_config: A list of .rpm or .repo paths or URLs - :return: A list of docker commands - """ - rpm_setup = list() - - for config in rpm_setup_config: - if config.endswith('.rpm'): - # RPM files can be installed with yum from file path or url - cmd = "RUN yum -y install {}".format(config) - elif config.endswith('.repo'): - if config.startswith('http'): - # Curl http://url/etc.repo to /etc/yum.repos.d/etc.repo - name = config.split('/')[-1] - cmd = "RUN curl {} -o /etc/yum.repos.d/{}".format(config, - name) - else: - # Copy .repo file from filesystem - cmd = "COPY {} /etc/yum.repos.d/".format(config) - else: - raise KollaRpmSetupUnknownConfig( - 'RPM setup must be provided as .rpm or .repo files.' - ' Attempted configuration was {}'.format(config) - ) - - rpm_setup.append(cmd) - - return rpm_setup - - def copy_apt_files(self): - if self.conf.apt_sources_list: - shutil.copyfile( - self.conf.apt_sources_list, - os.path.join(self.working_dir, "base", "sources.list") - ) - - if self.conf.apt_preferences: - shutil.copyfile( - self.conf.apt_preferences, - os.path.join(self.working_dir, "base", "apt_preferences") - ) - - def setup_working_dir(self): - """Creates a working directory for use while building""" - ts = time.time() - ts = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d_%H-%M-%S_') - self.temp_dir = tempfile.mkdtemp(prefix='kolla-' + ts) - self.working_dir = os.path.join(self.temp_dir, 'docker') - shutil.copytree(self.images_dir, self.working_dir) - self.copy_apt_files() - LOG.debug('Created working dir: %s', self.working_dir) - - def set_time(self): - for root, dirs, files in os.walk(self.working_dir): - for file_ in files: - os.utime(os.path.join(root, file_), (0, 0)) - for dir_ in dirs: - os.utime(os.path.join(root, dir_), (0, 0)) - LOG.debug('Set atime and mtime to 0 for all content in working dir') - - def create_dockerfiles(self): - kolla_version = version.version_info.cached_version_string() - supported_distro_release = common_config.DISTRO_RELEASE.get( - self.base) - for path in self.docker_build_paths: - template_name = "Dockerfile.j2" - env = jinja2.Environment( # nosec: not used to render HTML - loader=jinja2.FileSystemLoader(path)) - template = env.get_template(template_name) - values = {'base_distro': self.base, - 'base_image': self.conf.base_image, - 'base_distro_tag': self.base_tag, - 'supported_distro_release': supported_distro_release, - 'install_metatype': self.install_metatype, - 'image_prefix': self.image_prefix, - 'install_type': self.install_type, - 'namespace': self.namespace, - 'tag': self.tag, - 'maintainer': self.maintainer, - 'kolla_version': kolla_version, - 'rpm_setup': self.rpm_setup} - if self.include_header: - with open(self.include_header, 'r') as f: - values['include_header'] = f.read() - if self.include_footer: - with open(self.include_footer, 'r') as f: - values['include_footer'] = f.read() - content = template.render(values) - with open(os.path.join(path, 'Dockerfile'), 'w') as f: - f.write(content) - - def find_dockerfiles(self): - """Recursive search for Dockerfiles in the working directory""" - self.docker_build_paths = list() - path = self.working_dir - filename = 'Dockerfile.j2' - - for root, dirs, names in os.walk(path): - if filename in names: - self.docker_build_paths.append(root) - LOG.debug('Found %s', root.split(self.working_dir)[1]) - - LOG.debug('Found %d Dockerfiles', len(self.docker_build_paths)) - - def cleanup(self): - """Remove temp files""" - shutil.rmtree(self.temp_dir) - - def filter_images(self): - """Filter which images to build""" - filter_ = list() - - if self.regex: - filter_ += self.regex - elif self.conf.profile: - for profile in self.conf.profile: - if profile not in self.conf.profiles: - self.conf.register_opt(cfg.ListOpt(profile, - default=[]), - 'profiles') - if len(self.conf.profiles[profile]) == 0: - msg = 'Profile: {} does not exist'.format(profile) - raise ValueError(msg) - else: - filter_ += self.conf.profiles[profile] - - if filter_: - patterns = re.compile(r"|".join(filter_).join('()')) - for image in self.images: - if image.status == STATUS_MATCHED: - continue - if re.search(patterns, image.name): - image.status = STATUS_MATCHED - while (image.parent is not None and - image.parent.status != STATUS_MATCHED): - image = image.parent - image.status = STATUS_MATCHED - LOG.debug('%s:Matched regex', image.name) - else: - image.status = STATUS_UNMATCHED - else: - for image in self.images: - image.status = STATUS_MATCHED - - def summary(self): - """Walk the dictionary of images statuses and print results""" - # For debug we print the logs again if the image error'd. This is to - # to help us debug and it will be extra helpful in the gate. - for image in self.images: - if image.status in STATUS_ERRORS: - LOG.debug("%s:Failed with the following logs", image.name) - for line in image.logs: - if line: - LOG.debug("%s:%s", image.name, line) - - self.get_image_statuses() - - if self.image_statuses_good: - LOG.info("Successfully built images") - LOG.info("=========================") - for name in self.image_statuses_good.keys(): - LOG.info(name) - - if self.image_statuses_bad: - LOG.info("Images that failed to build") - LOG.info("===========================") - for name, status in self.image_statuses_bad.items(): - LOG.error('%s Failed with status: %s', name, status) - - if self.image_statuses_unmatched: - LOG.debug("Images not matched for build by regex") - LOG.debug("=====================================") - for name in self.image_statuses_unmatched.keys(): - LOG.debug(name) - - def get_image_statuses(self): - if any([self.image_statuses_bad, - self.image_statuses_good, - self.image_statuses_unmatched]): - return (self.image_statuses_bad, - self.image_statuses_good, - self.image_statuses_unmatched) - for image in self.images: - if image.status == STATUS_BUILT: - self.image_statuses_good[image.name] = image.status - elif image.status == STATUS_UNMATCHED: - self.image_statuses_unmatched[image.name] = image.status - else: - self.image_statuses_bad[image.name] = image.status - return (self.image_statuses_bad, - self.image_statuses_good, - self.image_statuses_unmatched) - - def build_image_list(self): - def process_source_installation(image, section): - installation = dict() - # NOTE(jeffrey4l): source is not needed when the type is None - if self.conf._get('type', self.conf._get_group(section)) is None: - LOG.debug('%s:No source location found', section) - else: - installation['type'] = self.conf[section]['type'] - installation['source'] = self.conf[section]['location'] - installation['name'] = section - if installation['type'] == 'git': - installation['reference'] = self.conf[section]['reference'] - return installation - - for path in self.docker_build_paths: - # Reading parent image name - with open(os.path.join(path, 'Dockerfile')) as f: - content = f.read() - - image_name = os.path.basename(path) - canonical_name = (self.namespace + '/' + self.image_prefix + - image_name + ':' + self.tag) - image = Image(image_name, canonical_name, path, - parent_name=content.split(' ')[1].split('\n')[0]) - - if self.install_type == 'source': - # NOTE(jeffrey4l): register the opts if the section didn't - # register in the kolla/common/config.py file - if image.name not in self.conf._groups: - self.conf.register_opts(common_config.get_source_opts(), - image.name) - image.source = process_source_installation(image, image.name) - for plugin in [match.group(0) for match in - (re.search('{}-plugin-.+'.format(image.name), - section) for section in - self.conf.list_all_sections()) if match]: - self.conf.register_opts(common_config.get_source_opts(), - plugin) - image.plugins.append( - process_source_installation(image, plugin)) - - self.images.append(image) - - def save_dependency(self, to_file): - dot = graphviz.Digraph(comment='Docker Images Dependency') - dot.body.extend(['rankdir=LR']) - for image in self.images: - if image.status not in [STATUS_MATCHED]: - continue - dot.node(image.name) - if image.parent is not None: - dot.edge(image.parent.name, image.name) - - with open(to_file, 'w') as f: - f.write(dot.source) - - def list_images(self): - for count, image in enumerate(self.images): - print(count + 1, ':', image.name) - - def list_dependencies(self): - match = False - for image in self.images: - if image.status in [STATUS_MATCHED]: - match = True - if image.parent is None: - base = image - if not match: - print('Nothing matched!') - return - - def list_children(images, ancestry): - children = ancestry.values()[0] - for item in images: - if item.status not in [STATUS_MATCHED]: - continue - if not item.children: - children.append(item.name) - else: - newparent = {item.name: []} - children.append(newparent) - list_children(item.children, newparent) - - ancestry = {base.name: []} - list_children(base.children, ancestry) - pprint.pprint(ancestry) - - def find_parents(self): - """Associate all images with parents and children""" - sort_images = dict() - - for image in self.images: - sort_images[image.canonical_name] = image - - for parent_name, parent in sort_images.items(): - for image in sort_images.values(): - if image.parent_name == parent_name: - parent.children.append(image) - image.parent = parent - - def build_queue(self, push_queue): - """Organizes Queue list - - Return a list of Queues that have been organized into a hierarchy - based on dependencies - """ - self.build_image_list() - self.find_parents() - self.filter_images() - - queue = six.moves.queue.Queue() - - for image in self.images: - if image.parent is None: - queue.put(BuildTask(self.conf, image, push_queue)) - LOG.debug('%s:Added image to queue', image.name) - - return queue - - -def run_build(): - """Build container images. - - :return: A 3-tuple containing bad, good, and unmatched container image - status dicts, or None if no images were built. - """ - conf = cfg.ConfigOpts() - common_config.parse(conf, sys.argv[1:], prog='kolla-build') - - if conf.debug: - LOG.setLevel(logging.DEBUG) - - kolla = KollaWorker(conf) - kolla.setup_working_dir() - kolla.find_dockerfiles() - kolla.create_dockerfiles() - - if conf.template_only: - LOG.info('Dockerfiles are generated in %s', kolla.working_dir) - return - - # We set the atime and mtime to 0 epoch to preserve allow the Docker cache - # to work like we want. A different size or hash will still force a rebuild - kolla.set_time() - - if conf.save_dependency: - kolla.save_dependency(conf.save_dependency) - LOG.info('Docker images dependency is saved in %s', - conf.save_dependency) - return - if conf.list_images: - kolla.list_images() - return - if conf.list_dependencies: - kolla.list_dependencies() - return - - push_queue = six.moves.queue.Queue() - queue = kolla.build_queue(push_queue) - workers = [] - - for x in six.moves.range(conf.threads): - worker = WorkerThread(conf, queue) - worker.setDaemon(True) - worker.start() - workers.append(worker) - - for x in six.moves.range(conf.push_threads): - worker = WorkerThread(conf, push_queue) - worker.start() - workers.append(worker) - - # sleep until queue is empty - while queue.unfinished_tasks or push_queue.unfinished_tasks: - time.sleep(3) - - kolla.summary() - kolla.cleanup() - - # ensure all threads exited happily - queue.put(WorkerThread.tombstone) - push_queue.put(WorkerThread.tombstone) - for w in workers: - w.join() - - return kolla.get_image_statuses() +from kolla.image import build def main(): - statuses = run_build() + statuses = build.run_build() if statuses: bad_results, good_results, unmatched_results = statuses if bad_results: diff --git a/kolla/image/__init__.py b/kolla/image/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/kolla/image/__init__.py diff --git a/kolla/image/build.py b/kolla/image/build.py new file mode 100755 index 000000000..40601f0e8 --- /dev/null +++ b/kolla/image/build.py @@ -0,0 +1,924 @@ +#!/usr/bin/env python + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import datetime +import errno +import graphviz +import json +import logging +import os +import pprint +import re +import requests +import shutil +import signal +import sys +import tarfile +import tempfile +import threading +import time + +import docker +import git +import jinja2 +from oslo_config import cfg +from requests import exceptions as requests_exc +import six + +PROJECT_ROOT = os.path.abspath(os.path.join( + os.path.dirname(os.path.realpath(__file__)), '../..')) + +# NOTE(SamYaple): Update the search patch to prefer PROJECT_ROOT as the source +# of packages to import if we are using local tools/build.py +# instead of pip installed kolla-build tool +if PROJECT_ROOT not in sys.path: + sys.path.insert(0, PROJECT_ROOT) + +from kolla.common import config as common_config +from kolla.common import task +from kolla import version + +logging.basicConfig() +LOG = logging.getLogger(__name__) +LOG.setLevel(logging.INFO) + + +def handle_ctrlc(single, frame): + kollaobj = frame.f_locals['kolla'] + kollaobj.cleanup() + sys.exit(1) + +signal.signal(signal.SIGINT, handle_ctrlc) + + +class KollaDirNotFoundException(Exception): + pass + + +class KollaUnknownBuildTypeException(Exception): + pass + + +class KollaMismatchBaseTypeException(Exception): + pass + + +class KollaRpmSetupUnknownConfig(Exception): + pass + + +# Image status constants. +# +# TODO(harlowja): use enum lib in the future?? +STATUS_CONNECTION_ERROR = 'connection_error' +STATUS_PUSH_ERROR = 'push_error' +STATUS_ERROR = 'error' +STATUS_PARENT_ERROR = 'parent_error' +STATUS_BUILT = 'built' +STATUS_BUILDING = 'building' +STATUS_UNMATCHED = 'unmatched' +STATUS_MATCHED = 'matched' +STATUS_UNPROCESSED = 'unprocessed' + +# All error status constants. +STATUS_ERRORS = (STATUS_CONNECTION_ERROR, STATUS_PUSH_ERROR, + STATUS_ERROR, STATUS_PARENT_ERROR) + + +class Recorder(object): + """Recorder/buffer of (unicode) log lines for eventual display.""" + + def __init__(self): + self._lines = [] + + def write(self, text=""): + if isinstance(text, six.text_type): + self._lines.append(text) + elif isinstance(text, six.binary_type): + self._lines.append(text.decode('utf8')) + elif isinstance(text, Recorder): + self._lines.extend(text._lines) + else: + self.write(text=str(text)) + + def clear(self): + self._lines = [] + + def __iter__(self): + for line in self._lines: + yield line + + def __str__(self): + return u"\n".join(self._lines) + + +def docker_client(): + try: + docker_kwargs = docker.utils.kwargs_from_env() + return docker.Client(version='auto', **docker_kwargs) + except docker.errors.DockerException: + LOG.exception('Can not communicate with docker service.' + 'Please check docker service is running without errors') + sys.exit(1) + + +class Image(object): + def __init__(self, name, canonical_name, path, parent_name='', + status=STATUS_UNPROCESSED, parent=None, source=None): + self.name = name + self.canonical_name = canonical_name + self.path = path + self.status = status + self.parent = parent + self.source = source + self.parent_name = parent_name + self.logs = Recorder() + self.push_logs = Recorder() + self.children = [] + self.plugins = [] + + def __repr__(self): + return ("Image(%s, %s, %s, parent_name=%s," + " status=%s, parent=%s, source=%s)") % ( + self.name, self.canonical_name, self.path, + self.parent_name, self.status, self.parent, self.source) + + +class PushIntoQueueTask(task.Task): + """Task that pushes some other task into a queue.""" + + def __init__(self, push_task, push_queue): + super(PushIntoQueueTask, self).__init__() + self.push_task = push_task + self.push_queue = push_queue + + @property + def name(self): + return 'PushIntoQueueTask(%s=>%s)' % (self.push_task.name, + self.push_queue) + + def run(self): + self.push_queue.put(self.push_task) + self.success = True + + +class PushTask(task.Task): + """Task that pushes a image to a docker repository.""" + + def __init__(self, conf, image): + super(PushTask, self).__init__() + self.dc = docker_client() + self.conf = conf + self.image = image + + @property + def name(self): + return 'PushTask(%s)' % self.image.name + + def run(self): + image = self.image + try: + LOG.debug('%s:Try to push the image', image.name) + self.push_image(image) + except requests_exc.ConnectionError: + LOG.exception('%s:Make sure Docker is running and that you' + ' have the correct privileges to run Docker' + ' (root)', image.name) + image.status = STATUS_CONNECTION_ERROR + except Exception: + LOG.exception('%s:Unknown error when pushing', image.name) + image.status = STATUS_PUSH_ERROR + finally: + if (image.status not in STATUS_ERRORS + and image.status != STATUS_UNPROCESSED): + LOG.info('%s:Pushed successfully', image.name) + self.success = True + else: + self.success = False + + def push_image(self, image): + image.push_logs.clear() + for response in self.dc.push(image.canonical_name, + stream=True, + insecure_registry=True): + stream = json.loads(response) + if 'stream' in stream: + image.push_logs.write(image.logs) + image.push_logs.write(stream['stream']) + LOG.info('%s', stream['stream']) + elif 'errorDetail' in stream: + image.status = STATUS_ERROR + LOG.error(stream['errorDetail']['message']) + + +class BuildTask(task.Task): + """Task that builds out an image.""" + + def __init__(self, conf, image, push_queue): + super(BuildTask, self).__init__() + self.conf = conf + self.image = image + self.dc = docker_client() + self.push_queue = push_queue + self.nocache = not conf.cache or conf.no_cache + self.forcerm = not conf.keep + + @property + def name(self): + return 'BuildTask(%s)' % self.image.name + + def run(self): + self.builder(self.image) + if self.image.status == STATUS_BUILT: + self.success = True + + @property + def followups(self): + followups = [] + if self.conf.push and self.success: + followups.extend([ + # If we are supposed to push the image into a docker + # repository, then make sure we do that... + PushIntoQueueTask( + PushTask(self.conf, self.image), + self.push_queue), + ]) + if self.image.children and self.success: + for image in self.image.children: + followups.append(BuildTask(self.conf, image, self.push_queue)) + return followups + + def process_source(self, image, source): + dest_archive = os.path.join(image.path, source['name'] + '-archive') + + if source.get('type') == 'url': + LOG.debug("%s:Getting archive from %s", image.name, + source['source']) + try: + r = requests.get(source['source'], timeout=self.conf.timeout) + except requests_exc.Timeout: + LOG.exception('Request timed out while getting archive' + ' from %s', source['source']) + image.status = STATUS_ERROR + image.logs.clear() + return + + if r.status_code == 200: + with open(dest_archive, 'wb') as f: + f.write(r.content) + else: + LOG.error('%s:Failed to download archive: status_code %s', + image.name, r.status_code) + image.status = STATUS_ERROR + return + + elif source.get('type') == 'git': + clone_dir = '{}-{}'.format(dest_archive, + source['reference'].replace('/', '-')) + try: + LOG.debug("%s:Cloning from %s", image.name, + source['source']) + git.Git().clone(source['source'], clone_dir) + git.Git(clone_dir).checkout(source['reference']) + reference_sha = git.Git(clone_dir).rev_parse('HEAD') + LOG.debug("%s:Git checkout by reference %s (%s)", + image.name, source['reference'], reference_sha) + except Exception as e: + LOG.error("%s:Failed to get source from git", image.name) + LOG.error("%s:Error:%s", image.name, str(e)) + # clean-up clone folder to retry + shutil.rmtree(clone_dir) + image.status = STATUS_ERROR + return + + with tarfile.open(dest_archive, 'w') as tar: + tar.add(clone_dir, arcname=os.path.basename(clone_dir)) + + elif source.get('type') == 'local': + LOG.debug("%s:Getting local archive from %s", image.name, + source['source']) + if os.path.isdir(source['source']): + with tarfile.open(dest_archive, 'w') as tar: + tar.add(source['source'], + arcname=os.path.basename(source['source'])) + else: + shutil.copyfile(source['source'], dest_archive) + + else: + LOG.error("%s:Wrong source type '%s'", image.name, + source.get('type')) + image.status = STATUS_ERROR + return + + # Set time on destination archive to epoch 0 + os.utime(dest_archive, (0, 0)) + + return dest_archive + + def update_buildargs(self): + buildargs = dict() + if self.conf.build_args: + buildargs = dict(self.conf.build_args) + + proxy_vars = ('HTTP_PROXY', 'http_proxy', 'HTTPS_PROXY', + 'https_proxy', 'FTP_PROXY', 'ftp_proxy', + 'NO_PROXY', 'no_proxy') + + for proxy_var in proxy_vars: + if proxy_var in os.environ and proxy_var not in buildargs: + buildargs[proxy_var] = os.environ.get(proxy_var) + + if not buildargs: + return None + return buildargs + + def builder(self, image): + LOG.debug('%s:Processing', image.name) + if image.status == STATUS_UNMATCHED: + return + + if (image.parent is not None and + image.parent.status in STATUS_ERRORS): + LOG.error('%s:Parent image error\'d with message "%s"', + image.name, image.parent.status) + image.status = STATUS_PARENT_ERROR + return + + image.status = STATUS_BUILDING + LOG.info('%s:Building', image.name) + + if image.source and 'source' in image.source: + self.process_source(image, image.source) + if image.status in STATUS_ERRORS: + return + + plugin_archives = list() + plugins_path = os.path.join(image.path, 'plugins') + for plugin in image.plugins: + archive_path = self.process_source(image, plugin) + if image.status in STATUS_ERRORS: + return + plugin_archives.append(archive_path) + if plugin_archives: + for plugin_archive in plugin_archives: + with tarfile.open(plugin_archive, 'r') as plugin_archive_tar: + plugin_archive_tar.extractall(path=plugins_path) + else: + try: + os.mkdir(plugins_path) + except OSError as e: + if e.errno == errno.EEXIST: + LOG.info('Directory %s already exist. Skipping.', + plugins_path) + else: + LOG.error('Failed to create directory %s: %s', + plugins_path, e) + image.status = STATUS_CONNECTION_ERROR + return + with tarfile.open(os.path.join(image.path, 'plugins-archive'), + 'w') as tar: + tar.add(plugins_path, arcname='plugins') + + # Pull the latest image for the base distro only + pull = True if image.parent is None else False + + image.logs.clear() + buildargs = self.update_buildargs() + for response in self.dc.build(path=image.path, + tag=image.canonical_name, + nocache=not self.conf.cache, + rm=True, + pull=pull, + forcerm=self.forcerm, + buildargs=buildargs): + stream = json.loads(response.decode('utf-8')) + + if 'stream' in stream: + image.logs.write(stream['stream']) + for line in stream['stream'].split('\n'): + if line: + LOG.info('%s:%s', image.name, line) + + if 'errorDetail' in stream: + image.status = STATUS_ERROR + LOG.error('%s:Error\'d with the following message', + image.name) + for line in stream['errorDetail']['message'].split('\n'): + if line: + LOG.error('%s:%s', image.name, line) + return + + image.status = STATUS_BUILT + + LOG.info('%s:Built', image.name) + + +class WorkerThread(threading.Thread): + """Thread that executes tasks until the queue provides a tombstone.""" + + #: Object to be put on worker queues to get them to die. + tombstone = object() + + def __init__(self, conf, queue): + super(WorkerThread, self).__init__() + self.queue = queue + self.conf = conf + + def run(self): + while True: + task = self.queue.get() + if task is self.tombstone: + # Ensure any other threads also get the tombstone. + self.queue.put(task) + break + try: + for attempt in six.moves.range(self.conf.retries + 1): + if attempt > 0: + LOG.debug("Attempting to run task %s for the %s time", + task.name, attempt + 1) + else: + LOG.debug("Attempting to run task %s for the first" + " time", task.name) + try: + task.run() + if task.success: + break + except Exception: + LOG.exception('Unhandled error when running %s', + task.name) + # try again... + task.reset() + if task.success: + for next_task in task.followups: + LOG.debug('Added next task %s to queue', + next_task.name) + self.queue.put(next_task) + finally: + self.queue.task_done() + + +class KollaWorker(object): + + def __init__(self, conf): + self.conf = conf + self.images_dir = self._get_images_dir() + self.registry = conf.registry + if self.registry: + self.namespace = self.registry + '/' + conf.namespace + else: + self.namespace = conf.namespace + self.base = conf.base + self.base_tag = conf.base_tag + self.install_type = conf.install_type + self.tag = conf.tag + self.images = list() + rpm_setup_config = filter(None, conf.rpm_setup_config) + self.rpm_setup = self.build_rpm_setup(rpm_setup_config) + + rh_base = ['fedora', 'centos', 'oraclelinux', 'rhel'] + rh_type = ['source', 'binary', 'rdo', 'rhos'] + deb_base = ['ubuntu', 'debian'] + deb_type = ['source', 'binary'] + + if not ((self.base in rh_base and self.install_type in rh_type) or + (self.base in deb_base and self.install_type in deb_type)): + raise KollaMismatchBaseTypeException( + '{} is unavailable for {}'.format(self.install_type, self.base) + ) + + if self.install_type == 'binary': + self.install_metatype = 'rdo' + elif self.install_type == 'source': + self.install_metatype = 'mixed' + elif self.install_type == 'rdo': + self.install_type = 'binary' + self.install_metatype = 'rdo' + elif self.install_type == 'rhos': + self.install_type = 'binary' + self.install_metatype = 'rhos' + else: + raise KollaUnknownBuildTypeException( + 'Unknown install type' + ) + + self.image_prefix = self.base + '-' + self.install_type + '-' + + self.include_header = conf.include_header + self.include_footer = conf.include_footer + self.regex = conf.regex + self.image_statuses_bad = dict() + self.image_statuses_good = dict() + self.image_statuses_unmatched = dict() + self.maintainer = conf.maintainer + + def _get_images_dir(self): + possible_paths = ( + PROJECT_ROOT, + os.path.join(sys.prefix, 'share/kolla'), + os.path.join(sys.prefix, 'local/share/kolla')) + + for path in possible_paths: + image_path = os.path.join(path, 'docker') + # NOTE(SamYaple): We explicty check for the base folder to ensure + # this is the correct path + # TODO(SamYaple): Improve this to make this safer + if os.path.exists(os.path.join(image_path, 'base')): + LOG.info('Found the docker image folder at %s', image_path) + return image_path + else: + raise KollaDirNotFoundException('Image dir can not be found') + + def build_rpm_setup(self, rpm_setup_config): + """Generates a list of docker commands based on provided configuration. + + :param rpm_setup_config: A list of .rpm or .repo paths or URLs + :return: A list of docker commands + """ + rpm_setup = list() + + for config in rpm_setup_config: + if config.endswith('.rpm'): + # RPM files can be installed with yum from file path or url + cmd = "RUN yum -y install {}".format(config) + elif config.endswith('.repo'): + if config.startswith('http'): + # Curl http://url/etc.repo to /etc/yum.repos.d/etc.repo + name = config.split('/')[-1] + cmd = "RUN curl {} -o /etc/yum.repos.d/{}".format(config, + name) + else: + # Copy .repo file from filesystem + cmd = "COPY {} /etc/yum.repos.d/".format(config) + else: + raise KollaRpmSetupUnknownConfig( + 'RPM setup must be provided as .rpm or .repo files.' + ' Attempted configuration was {}'.format(config) + ) + + rpm_setup.append(cmd) + + return rpm_setup + + def copy_apt_files(self): + if self.conf.apt_sources_list: + shutil.copyfile( + self.conf.apt_sources_list, + os.path.join(self.working_dir, "base", "sources.list") + ) + + if self.conf.apt_preferences: + shutil.copyfile( + self.conf.apt_preferences, + os.path.join(self.working_dir, "base", "apt_preferences") + ) + + def setup_working_dir(self): + """Creates a working directory for use while building""" + ts = time.time() + ts = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d_%H-%M-%S_') + self.temp_dir = tempfile.mkdtemp(prefix='kolla-' + ts) + self.working_dir = os.path.join(self.temp_dir, 'docker') + shutil.copytree(self.images_dir, self.working_dir) + self.copy_apt_files() + LOG.debug('Created working dir: %s', self.working_dir) + + def set_time(self): + for root, dirs, files in os.walk(self.working_dir): + for file_ in files: + os.utime(os.path.join(root, file_), (0, 0)) + for dir_ in dirs: + os.utime(os.path.join(root, dir_), (0, 0)) + LOG.debug('Set atime and mtime to 0 for all content in working dir') + + def create_dockerfiles(self): + kolla_version = version.version_info.cached_version_string() + supported_distro_release = common_config.DISTRO_RELEASE.get( + self.base) + for path in self.docker_build_paths: + template_name = "Dockerfile.j2" + env = jinja2.Environment( # nosec: not used to render HTML + loader=jinja2.FileSystemLoader(path)) + template = env.get_template(template_name) + values = {'base_distro': self.base, + 'base_image': self.conf.base_image, + 'base_distro_tag': self.base_tag, + 'supported_distro_release': supported_distro_release, + 'install_metatype': self.install_metatype, + 'image_prefix': self.image_prefix, + 'install_type': self.install_type, + 'namespace': self.namespace, + 'tag': self.tag, + 'maintainer': self.maintainer, + 'kolla_version': kolla_version, + 'rpm_setup': self.rpm_setup} + if self.include_header: + with open(self.include_header, 'r') as f: + values['include_header'] = f.read() + if self.include_footer: + with open(self.include_footer, 'r') as f: + values['include_footer'] = f.read() + content = template.render(values) + with open(os.path.join(path, 'Dockerfile'), 'w') as f: + f.write(content) + + def find_dockerfiles(self): + """Recursive search for Dockerfiles in the working directory""" + self.docker_build_paths = list() + path = self.working_dir + filename = 'Dockerfile.j2' + + for root, dirs, names in os.walk(path): + if filename in names: + self.docker_build_paths.append(root) + LOG.debug('Found %s', root.split(self.working_dir)[1]) + + LOG.debug('Found %d Dockerfiles', len(self.docker_build_paths)) + + def cleanup(self): + """Remove temp files""" + shutil.rmtree(self.temp_dir) + + def filter_images(self): + """Filter which images to build""" + filter_ = list() + + if self.regex: + filter_ += self.regex + elif self.conf.profile: + for profile in self.conf.profile: + if profile not in self.conf.profiles: + self.conf.register_opt(cfg.ListOpt(profile, + default=[]), + 'profiles') + if len(self.conf.profiles[profile]) == 0: + msg = 'Profile: {} does not exist'.format(profile) + raise ValueError(msg) + else: + filter_ += self.conf.profiles[profile] + + if filter_: + patterns = re.compile(r"|".join(filter_).join('()')) + for image in self.images: + if image.status == STATUS_MATCHED: + continue + if re.search(patterns, image.name): + image.status = STATUS_MATCHED + while (image.parent is not None and + image.parent.status != STATUS_MATCHED): + image = image.parent + image.status = STATUS_MATCHED + LOG.debug('%s:Matched regex', image.name) + else: + image.status = STATUS_UNMATCHED + else: + for image in self.images: + image.status = STATUS_MATCHED + + def summary(self): + """Walk the dictionary of images statuses and print results""" + # For debug we print the logs again if the image error'd. This is to + # to help us debug and it will be extra helpful in the gate. + for image in self.images: + if image.status in STATUS_ERRORS: + LOG.debug("%s:Failed with the following logs", image.name) + for line in image.logs: + if line: + LOG.debug("%s:%s", image.name, line) + + self.get_image_statuses() + + if self.image_statuses_good: + LOG.info("Successfully built images") + LOG.info("=========================") + for name in self.image_statuses_good.keys(): + LOG.info(name) + + if self.image_statuses_bad: + LOG.info("Images that failed to build") + LOG.info("===========================") + for name, status in self.image_statuses_bad.items(): + LOG.error('%s Failed with status: %s', name, status) + + if self.image_statuses_unmatched: + LOG.debug("Images not matched for build by regex") + LOG.debug("=====================================") + for name in self.image_statuses_unmatched.keys(): + LOG.debug(name) + + def get_image_statuses(self): + if any([self.image_statuses_bad, + self.image_statuses_good, + self.image_statuses_unmatched]): + return (self.image_statuses_bad, + self.image_statuses_good, + self.image_statuses_unmatched) + for image in self.images: + if image.status == STATUS_BUILT: + self.image_statuses_good[image.name] = image.status + elif image.status == STATUS_UNMATCHED: + self.image_statuses_unmatched[image.name] = image.status + else: + self.image_statuses_bad[image.name] = image.status + return (self.image_statuses_bad, + self.image_statuses_good, + self.image_statuses_unmatched) + + def build_image_list(self): + def process_source_installation(image, section): + installation = dict() + # NOTE(jeffrey4l): source is not needed when the type is None + if self.conf._get('type', self.conf._get_group(section)) is None: + LOG.debug('%s:No source location found', section) + else: + installation['type'] = self.conf[section]['type'] + installation['source'] = self.conf[section]['location'] + installation['name'] = section + if installation['type'] == 'git': + installation['reference'] = self.conf[section]['reference'] + return installation + + for path in self.docker_build_paths: + # Reading parent image name + with open(os.path.join(path, 'Dockerfile')) as f: + content = f.read() + + image_name = os.path.basename(path) + canonical_name = (self.namespace + '/' + self.image_prefix + + image_name + ':' + self.tag) + image = Image(image_name, canonical_name, path, + parent_name=content.split(' ')[1].split('\n')[0]) + + if self.install_type == 'source': + # NOTE(jeffrey4l): register the opts if the section didn't + # register in the kolla/common/config.py file + if image.name not in self.conf._groups: + self.conf.register_opts(common_config.get_source_opts(), + image.name) + image.source = process_source_installation(image, image.name) + for plugin in [match.group(0) for match in + (re.search('{}-plugin-.+'.format(image.name), + section) for section in + self.conf.list_all_sections()) if match]: + self.conf.register_opts(common_config.get_source_opts(), + plugin) + image.plugins.append( + process_source_installation(image, plugin)) + + self.images.append(image) + + def save_dependency(self, to_file): + dot = graphviz.Digraph(comment='Docker Images Dependency') + dot.body.extend(['rankdir=LR']) + for image in self.images: + if image.status not in [STATUS_MATCHED]: + continue + dot.node(image.name) + if image.parent is not None: + dot.edge(image.parent.name, image.name) + + with open(to_file, 'w') as f: + f.write(dot.source) + + def list_images(self): + for count, image in enumerate(self.images): + print(count + 1, ':', image.name) + + def list_dependencies(self): + match = False + for image in self.images: + if image.status in [STATUS_MATCHED]: + match = True + if image.parent is None: + base = image + if not match: + print('Nothing matched!') + return + + def list_children(images, ancestry): + children = ancestry.values()[0] + for item in images: + if item.status not in [STATUS_MATCHED]: + continue + if not item.children: + children.append(item.name) + else: + newparent = {item.name: []} + children.append(newparent) + list_children(item.children, newparent) + + ancestry = {base.name: []} + list_children(base.children, ancestry) + pprint.pprint(ancestry) + + def find_parents(self): + """Associate all images with parents and children""" + sort_images = dict() + + for image in self.images: + sort_images[image.canonical_name] = image + + for parent_name, parent in sort_images.items(): + for image in sort_images.values(): + if image.parent_name == parent_name: + parent.children.append(image) + image.parent = parent + + def build_queue(self, push_queue): + """Organizes Queue list + + Return a list of Queues that have been organized into a hierarchy + based on dependencies + """ + self.build_image_list() + self.find_parents() + self.filter_images() + + queue = six.moves.queue.Queue() + + for image in self.images: + if image.parent is None: + queue.put(BuildTask(self.conf, image, push_queue)) + LOG.debug('%s:Added image to queue', image.name) + + return queue + + +def run_build(): + """Build container images. + + :return: A 3-tuple containing bad, good, and unmatched container image + status dicts, or None if no images were built. + """ + conf = cfg.ConfigOpts() + common_config.parse(conf, sys.argv[1:], prog='kolla-build') + + if conf.debug: + LOG.setLevel(logging.DEBUG) + + kolla = KollaWorker(conf) + kolla.setup_working_dir() + kolla.find_dockerfiles() + kolla.create_dockerfiles() + + if conf.template_only: + LOG.info('Dockerfiles are generated in %s', kolla.working_dir) + return + + # We set the atime and mtime to 0 epoch to preserve allow the Docker cache + # to work like we want. A different size or hash will still force a rebuild + kolla.set_time() + + if conf.save_dependency: + kolla.save_dependency(conf.save_dependency) + LOG.info('Docker images dependency is saved in %s', + conf.save_dependency) + return + if conf.list_images: + kolla.list_images() + return + if conf.list_dependencies: + kolla.list_dependencies() + return + + push_queue = six.moves.queue.Queue() + queue = kolla.build_queue(push_queue) + workers = [] + + for x in six.moves.range(conf.threads): + worker = WorkerThread(conf, queue) + worker.setDaemon(True) + worker.start() + workers.append(worker) + + for x in six.moves.range(conf.push_threads): + worker = WorkerThread(conf, push_queue) + worker.start() + workers.append(worker) + + # sleep until queue is empty + while queue.unfinished_tasks or push_queue.unfinished_tasks: + time.sleep(3) + + kolla.summary() + kolla.cleanup() + + # ensure all threads exited happily + queue.put(WorkerThread.tombstone) + push_queue.put(WorkerThread.tombstone) + for w in workers: + w.join() + + return kolla.get_image_statuses() diff --git a/kolla/tests/base.py b/kolla/tests/base.py index 0ac1467cc..1c43b7619 100644 --- a/kolla/tests/base.py +++ b/kolla/tests/base.py @@ -36,7 +36,7 @@ class TestCase(oslotest_base.BaseTestCase): # NOTE(jeffrey4l): mock the _get_image_dir method to return a fake # docker images dir self.useFixture(fixtures.MockPatch( - 'kolla.cmd.build.KollaWorker._get_images_dir', + 'kolla.image.build.KollaWorker._get_images_dir', mock.Mock(return_value=os.path.join(TESTS_ROOT, 'docker')))) def get_default_config_files(self): diff --git a/kolla/tests/test_build.py b/kolla/tests/test_build.py index 98c240d6e..98ffebf99 100644 --- a/kolla/tests/test_build.py +++ b/kolla/tests/test_build.py @@ -17,7 +17,8 @@ import mock import os import requests -from kolla.cmd import build +from kolla.cmd import build as build_cmd +from kolla.image import build from kolla.tests import base @@ -233,22 +234,22 @@ class MainTest(base.TestCase): def test_images_built(self, mock_run_build): image_statuses = ({}, {'img': 'built'}, {}) mock_run_build.return_value = image_statuses - result = build.main() + result = build_cmd.main() self.assertEqual(0, result) def test_images_unmatched(self, mock_run_build): image_statuses = ({}, {}, {'img': 'unmatched'}) mock_run_build.return_value = image_statuses - result = build.main() + result = build_cmd.main() self.assertEqual(0, result) def test_no_images_built(self, mock_run_build): mock_run_build.return_value = None - result = build.main() + result = build_cmd.main() self.assertEqual(0, result) def test_bad_images(self, mock_run_build): image_statuses = ({'img': 'error'}, {}, {}) mock_run_build.return_value = image_statuses - result = build.main() + result = build_cmd.main() self.assertEqual(1, result) diff --git a/tests/test_build.py b/tests/test_build.py index f2e802bf6..075063f87 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -23,7 +23,7 @@ import testtools sys.path.append( os.path.abspath(os.path.join(os.path.dirname(__file__), '../tools'))) -from kolla.cmd import build +from kolla.image import build LOG = logging.getLogger(__name__) |