aboutsummaryrefslogtreecommitdiff
path: root/kolla
diff options
context:
space:
mode:
authorJoshua Harlow <jxharlow@godaddy.com>2016-05-26 13:38:21 -0700
committerJoshua Harlow <jxharlow@godaddy.com>2016-06-02 11:21:24 -0700
commit7c6990ccec16e55334376f696eb591891fc1fd7e (patch)
tree33abcd9a8f09e28d99160946b0cd8b3dfa554ac1 /kolla
parente78ae9bc618dff1ed38db1f5013d7f1735124790 (diff)
downloadkolla-7c6990ccec16e55334376f696eb591891fc1fd7e.tar.gz
kolla-7c6990ccec16e55334376f696eb591891fc1fd7e.zip
Use an image object, recorder object and status constants
Instead of using raw dicts and passing data around via dictionaries (which makes it really hard to figure out what is in those dictionaries at any point) prefer to use objects. That way people can actually understand what the object is supposed to be, vs guessing and/or having to decipher its usage. The same goes for raw string constants, prefer using named constants instead. Closes-Bug: #1586475 Change-Id: Ide179dc6593c50696d47a2d3d4cd000f343855d4
Diffstat (limited to 'kolla')
-rwxr-xr-xkolla/cmd/build.py292
-rw-r--r--kolla/tests/test_build.py48
2 files changed, 197 insertions, 143 deletions
diff --git a/kolla/cmd/build.py b/kolla/cmd/build.py
index dee1bb790..30c9dbd51 100755
--- a/kolla/cmd/build.py
+++ b/kolla/cmd/build.py
@@ -80,6 +80,51 @@ class KollaRpmSetupUnknownConfig(Exception):
pass
+# Image status constants.
+#
+# TODO(harlowja): use enum lib in the future??
+STATUS_CONNECTION_ERROR = 'connection_error'
+STATUS_PUSH_ERROR = 'push_error'
+STATUS_ERROR = 'error'
+STATUS_PARENT_ERROR = 'parent_error'
+STATUS_BUILT = 'built'
+STATUS_BUILDING = 'building'
+STATUS_UNMATCHED = 'unmatched'
+STATUS_MATCHED = 'matched'
+STATUS_UNPROCESSED = 'unprocessed'
+
+# All error status constants.
+STATUS_ERRORS = (STATUS_CONNECTION_ERROR, STATUS_PUSH_ERROR,
+ STATUS_ERROR, STATUS_PARENT_ERROR)
+
+
+class Recorder(object):
+ """Recorder/buffer of (unicode) log lines for eventual display."""
+
+ def __init__(self):
+ self._lines = []
+
+ def write(self, text=""):
+ if isinstance(text, six.text_type):
+ self._lines.append(text)
+ elif isinstance(text, six.binary_type):
+ self._lines.append(text.decode('utf8'))
+ elif isinstance(text, Recorder):
+ self._lines.extend(text._lines)
+ else:
+ self.write(text=str(text))
+
+ def clear(self):
+ self._lines = []
+
+ def __iter__(self):
+ for line in self._lines:
+ yield line
+
+ def __str__(self):
+ return u"\n".join(self._lines)
+
+
def docker_client():
try:
docker_kwargs = docker.utils.kwargs_from_env()
@@ -90,6 +135,28 @@ def docker_client():
sys.exit(1)
+class Image(object):
+ def __init__(self, name, canonical_name, path, parent_name='',
+ status=STATUS_UNPROCESSED, parent=None, source=None):
+ self.name = name
+ self.canonical_name = canonical_name
+ self.path = path
+ self.status = status
+ self.parent = parent
+ self.source = source
+ self.parent_name = parent_name
+ self.logs = Recorder()
+ self.push_logs = Recorder()
+ self.children = []
+ self.plugins = []
+
+ def __repr__(self):
+ return ("Image(%s, %s, %s, parent_name=%s,"
+ " status=%s, parent=%s, source=%s)") % (
+ self.name, self.canonical_name, self.path,
+ self.parent_name, self.status, self.parent, self.source)
+
+
class PushIntoQueueTask(task.Task):
"""Task that pushes some other task into a queue."""
@@ -119,41 +186,41 @@ class PushTask(task.Task):
@property
def name(self):
- return 'PushTask(%s)' % self.image['name']
+ return 'PushTask(%s)' % self.image.name
def run(self):
image = self.image
try:
- LOG.debug('%s:Try to push the image', image['name'])
+ LOG.debug('%s:Try to push the image', image.name)
self.push_image(image)
except requests_exc.ConnectionError:
LOG.exception('%s:Make sure Docker is running and that you'
' have the correct privileges to run Docker'
- ' (root)', image['name'])
- image['status'] = "connection_error"
+ ' (root)', image.name)
+ image.status = STATUS_CONNECTION_ERROR
except Exception:
- LOG.exception('%s:Unknown error when pushing', image['name'])
- image['status'] = "push_error"
+ LOG.exception('%s:Unknown error when pushing', image.name)
+ image.status = STATUS_PUSH_ERROR
finally:
- if "error" not in image['status']:
- LOG.info('%s:Pushed successfully', image['name'])
+ if (image.status not in STATUS_ERRORS
+ and image.status != STATUS_UNPROCESSED):
+ LOG.info('%s:Pushed successfully', image.name)
self.success = True
else:
self.success = False
def push_image(self, image):
- image['push_logs'] = str()
-
- for response in self.dc.push(image['fullname'],
+ image.push_logs.clear()
+ for response in self.dc.push(image.canonical_name,
stream=True,
insecure_registry=True):
stream = json.loads(response)
-
if 'stream' in stream:
- image['push_logs'] = image['logs'] + stream['stream']
+ image.push_logs.write(image.logs)
+ image.push_logs.write(stream['stream'])
LOG.info('%s', stream['stream'])
elif 'errorDetail' in stream:
- image['status'] = "error"
+ image.status = STATUS_ERROR
LOG.error(stream['errorDetail']['message'])
@@ -171,11 +238,11 @@ class BuildTask(task.Task):
@property
def name(self):
- return 'BuildTask(%s)' % self.image['name']
+ return 'BuildTask(%s)' % self.image.name
def run(self):
self.builder(self.image)
- if self.image['status'] == 'built':
+ if self.image.status == STATUS_BUILT:
self.success = True
@property
@@ -189,24 +256,24 @@ class BuildTask(task.Task):
PushTask(self.conf, self.image),
self.push_queue),
])
- if self.image['children'] and self.success:
- for image in self.image['children']:
+ if self.image.children and self.success:
+ for image in self.image.children:
followups.append(BuildTask(self.conf, image, self.push_queue))
return followups
def process_source(self, image, source):
- dest_archive = os.path.join(image['path'], source['name'] + '-archive')
+ dest_archive = os.path.join(image.path, source['name'] + '-archive')
if source.get('type') == 'url':
- LOG.debug("%s:Getting archive from %s", image['name'],
+ LOG.debug("%s:Getting archive from %s", image.name,
source['source'])
try:
r = requests.get(source['source'], timeout=self.conf.timeout)
except requests_exc.Timeout:
LOG.exception('Request timed out while getting archive'
' from %s', source['source'])
- image['status'] = "error"
- image['logs'] = str()
+ image.status = STATUS_ERROR
+ image.logs.clear()
return
if r.status_code == 200:
@@ -214,34 +281,34 @@ class BuildTask(task.Task):
f.write(r.content)
else:
LOG.error('%s:Failed to download archive: status_code %s',
- image['name'], r.status_code)
- image['status'] = "error"
+ image.name, r.status_code)
+ image.status = STATUS_ERROR
return
elif source.get('type') == 'git':
clone_dir = '{}-{}'.format(dest_archive,
source['reference'].replace('/', '-'))
try:
- LOG.debug("%s:Cloning from %s", image['name'],
+ LOG.debug("%s:Cloning from %s", image.name,
source['source'])
git.Git().clone(source['source'], clone_dir)
git.Git(clone_dir).checkout(source['reference'])
reference_sha = git.Git(clone_dir).rev_parse('HEAD')
LOG.debug("%s:Git checkout by reference %s (%s)",
- image['name'], source['reference'], reference_sha)
+ image.name, source['reference'], reference_sha)
except Exception as e:
- LOG.error("%s:Failed to get source from git", image['name'])
- LOG.error("%s:Error:%s", image['name'], str(e))
+ LOG.error("%s:Failed to get source from git", image.name)
+ LOG.error("%s:Error:%s", image.name, str(e))
# clean-up clone folder to retry
shutil.rmtree(clone_dir)
- image['status'] = "error"
+ image.status = STATUS_ERROR
return
with tarfile.open(dest_archive, 'w') as tar:
tar.add(clone_dir, arcname=os.path.basename(clone_dir))
elif source.get('type') == 'local':
- LOG.debug("%s:Getting local archive from %s", image['name'],
+ LOG.debug("%s:Getting local archive from %s", image.name,
source['source'])
if os.path.isdir(source['source']):
with tarfile.open(dest_archive, 'w') as tar:
@@ -251,9 +318,9 @@ class BuildTask(task.Task):
shutil.copyfile(source['source'], dest_archive)
else:
- LOG.error("%s:Wrong source type '%s'", image['name'],
+ LOG.error("%s:Wrong source type '%s'", image.name,
source.get('type'))
- image['status'] = "error"
+ image.status = STATUS_ERROR
return
# Set time on destination archive to epoch 0
@@ -279,31 +346,30 @@ class BuildTask(task.Task):
return buildargs
def builder(self, image):
- LOG.debug('%s:Processing', image['name'])
- if image['status'] == 'unmatched':
+ LOG.debug('%s:Processing', image.name)
+ if image.status == STATUS_UNMATCHED:
return
- if (image['parent'] is not None and
- image['parent']['status'] in ['error', 'parent_error',
- 'connection_error']):
+ if (image.parent is not None and
+ image.parent.status in STATUS_ERRORS):
LOG.error('%s:Parent image error\'d with message "%s"',
- image['name'], image['parent']['status'])
- image['status'] = "parent_error"
+ image.name, image.parent.status)
+ image.status = STATUS_PARENT_ERROR
return
- image['status'] = "building"
- LOG.info('%s:Building', image['name'])
+ image.status = STATUS_BUILDING
+ LOG.info('%s:Building', image.name)
- if 'source' in image and 'source' in image['source']:
- self.process_source(image, image['source'])
- if image['status'] == "error":
+ if image.source and 'source' in image.source:
+ self.process_source(image, image.source)
+ if image.status in STATUS_ERRORS:
return
plugin_archives = list()
- plugins_path = os.path.join(image['path'], 'plugins')
- for plugin in image['plugins']:
+ plugins_path = os.path.join(image.path, 'plugins')
+ for plugin in image.plugins:
archive_path = self.process_source(image, plugin)
- if image['status'] == "error":
+ if image.status in STATUS_ERRORS:
return
plugin_archives.append(archive_path)
if plugin_archives:
@@ -320,19 +386,19 @@ class BuildTask(task.Task):
else:
LOG.error('Failed to create directory %s: %s',
plugins_path, e)
- image['status'] = "error"
+ image.status = STATUS_CONNECTION_ERROR
return
- with tarfile.open(os.path.join(image['path'], 'plugins-archive'),
+ with tarfile.open(os.path.join(image.path, 'plugins-archive'),
'w') as tar:
tar.add(plugins_path, arcname='plugins')
# Pull the latest image for the base distro only
- pull = True if image['parent'] is None else False
+ pull = True if image.parent is None else False
- image['logs'] = str()
+ image.logs.clear()
buildargs = self.update_buildargs()
- for response in self.dc.build(path=image['path'],
- tag=image['fullname'],
+ for response in self.dc.build(path=image.path,
+ tag=image.canonical_name,
nocache=self.nocache,
rm=True,
pull=pull,
@@ -341,23 +407,23 @@ class BuildTask(task.Task):
stream = json.loads(response.decode('utf-8'))
if 'stream' in stream:
- image['logs'] = image['logs'] + stream['stream']
+ image.logs.write(stream['stream'])
for line in stream['stream'].split('\n'):
if line:
- LOG.info('%s:%s', image['name'], line)
+ LOG.info('%s:%s', image.name, line)
if 'errorDetail' in stream:
- image['status'] = "error"
+ image.status = STATUS_ERROR
LOG.error('%s:Error\'d with the following message',
- image['name'])
+ image.name)
for line in stream['errorDetail']['message'].split('\n'):
if line:
- LOG.error('%s:%s', image['name'], line)
+ LOG.error('%s:%s', image.name, line)
return
- image['status'] = "built"
+ image.status = STATUS_BUILT
- LOG.info('%s:Built', image['name'])
+ LOG.info('%s:Built', image.name)
class WorkerThread(threading.Thread):
@@ -606,31 +672,31 @@ class KollaWorker(object):
if filter_:
patterns = re.compile(r"|".join(filter_).join('()'))
for image in self.images:
- if image['status'] == 'matched':
+ if image.status == STATUS_MATCHED:
continue
- if re.search(patterns, image['name']):
- image['status'] = 'matched'
- while (image['parent'] is not None and
- image['parent']['status'] != 'matched'):
- image = image['parent']
- image['status'] = 'matched'
- LOG.debug('%s:Matched regex', image['name'])
+ if re.search(patterns, image.name):
+ image.status = STATUS_MATCHED
+ while (image.parent is not None and
+ image.parent.status != STATUS_MATCHED):
+ image = image.parent
+ image.status = STATUS_MATCHED
+ LOG.debug('%s:Matched regex', image.name)
else:
- image['status'] = 'unmatched'
+ image.status = STATUS_UNMATCHED
else:
for image in self.images:
- image['status'] = 'matched'
+ image.status = STATUS_MATCHED
def summary(self):
"""Walk the dictionary of images statuses and print results"""
# For debug we print the logs again if the image error'd. This is to
# to help us debug and it will be extra helpful in the gate.
for image in self.images:
- if image['status'] == 'error':
- LOG.debug("%s:Failed with the following logs", image['name'])
- for line in image['logs'].split('\n'):
+ if image.status in STATUS_ERRORS:
+ LOG.debug("%s:Failed with the following logs", image.name)
+ for line in image.logs:
if line:
- LOG.debug("%s:%s", image['name'], ''.join(line))
+ LOG.debug("%s:%s", image.name, line)
self.get_image_statuses()
@@ -660,12 +726,12 @@ class KollaWorker(object):
self.image_statuses_good,
self.image_statuses_unmatched)
for image in self.images:
- if image['status'] == "built":
- self.image_statuses_good[image['name']] = image['status']
- elif image['status'] == "unmatched":
- self.image_statuses_unmatched[image['name']] = image['status']
+ if image.status == STATUS_BUILT:
+ self.image_statuses_good[image.name] = image.status
+ elif image.status == STATUS_UNMATCHED:
+ self.image_statuses_unmatched[image.name] = image.status
else:
- self.image_statuses_bad[image['name']] = image['status']
+ self.image_statuses_bad[image.name] = image.status
return (self.image_statuses_bad,
self.image_statuses_good,
self.image_statuses_unmatched)
@@ -689,33 +755,26 @@ class KollaWorker(object):
with open(os.path.join(path, 'Dockerfile')) as f:
content = f.read()
- image = dict()
- image['status'] = "unprocessed"
- image['name'] = os.path.basename(path)
- image['fullname'] = self.namespace + '/' + self.image_prefix + \
- image['name'] + ':' + self.tag
- image['path'] = path
- image['parent_name'] = content.split(' ')[1].split('\n')[0]
- if not image['parent_name'].startswith(self.namespace + '/'):
- image['parent'] = None
- image['children'] = list()
- image['plugins'] = list()
+ image_name = os.path.basename(path)
+ canonical_name = (self.namespace + '/' + self.image_prefix +
+ image_name + ':' + self.tag)
+ image = Image(image_name, canonical_name, path,
+ parent_name=content.split(' ')[1].split('\n')[0])
if self.install_type == 'source':
# NOTE(jeffrey4l): register the opts if the section didn't
# register in the kolla/common/config.py file
- if image['name'] not in self.conf._groups:
+ if image.name not in self.conf._groups:
self.conf.register_opts(common_config.get_source_opts(),
- image['name'])
- image['source'] = process_source_installation(image,
- image['name'])
+ image.name)
+ image.source = process_source_installation(image, image.name)
for plugin in [match.group(0) for match in
- (re.search('{}-plugin-.+'.format(image['name']),
+ (re.search('{}-plugin-.+'.format(image.name),
section) for section in
self.conf.list_all_sections()) if match]:
self.conf.register_opts(common_config.get_source_opts(),
plugin)
- image['plugins'].append(
+ image.plugins.append(
process_source_installation(image, plugin))
self.images.append(image)
@@ -724,25 +783,25 @@ class KollaWorker(object):
dot = graphviz.Digraph(comment='Docker Images Dependency')
dot.body.extend(['rankdir=LR'])
for image in self.images:
- if image['status'] not in ['matched']:
+ if image.status not in [STATUS_MATCHED]:
continue
- dot.node(image['name'])
- if image['parent'] is not None:
- dot.edge(image['parent']['name'], image['name'])
+ dot.node(image.name)
+ if image.parent is not None:
+ dot.edge(image.parent.name, image.name)
with open(to_file, 'w') as f:
f.write(dot.source)
def list_images(self):
for count, image in enumerate(self.images):
- print(count + 1, ':', image['name'])
+ print(count + 1, ':', image.name)
def list_dependencies(self):
match = False
for image in self.images:
- if image['status'] in ['matched']:
+ if image.status in [STATUS_MATCHED]:
match = True
- if image['parent'] is None:
+ if image.parent is None:
base = image
if not match:
print('Nothing matched!')
@@ -751,18 +810,17 @@ class KollaWorker(object):
def list_children(images, ancestry):
children = ancestry.values()[0]
for item in images:
- if item['status'] not in ['matched']:
+ if item.status not in [STATUS_MATCHED]:
continue
-
- if not item['children']:
- children.append(item['name'])
+ if not item.children:
+ children.append(item.name)
else:
- newparent = {item['name']: []}
+ newparent = {item.name: []}
children.append(newparent)
- list_children(item['children'], newparent)
+ list_children(item.children, newparent)
- ancestry = {base['name']: []}
- list_children(base['children'], ancestry)
+ ancestry = {base.name: []}
+ list_children(base.children, ancestry)
pprint.pprint(ancestry)
def find_parents(self):
@@ -770,13 +828,13 @@ class KollaWorker(object):
sort_images = dict()
for image in self.images:
- sort_images[image['fullname']] = image
+ sort_images[image.canonical_name] = image
for parent_name, parent in sort_images.items():
for image in sort_images.values():
- if image['parent_name'] == parent_name:
- parent['children'].append(image)
- image['parent'] = parent
+ if image.parent_name == parent_name:
+ parent.children.append(image)
+ image.parent = parent
def build_queue(self, push_queue):
"""Organizes Queue list
@@ -791,9 +849,9 @@ class KollaWorker(object):
queue = six.moves.queue.Queue()
for image in self.images:
- if image['parent'] is None:
+ if image.parent is None:
queue.put(BuildTask(self.conf, image, push_queue))
- LOG.debug('%s:Added image to queue', image['name'])
+ LOG.debug('%s:Added image to queue', image.name)
return queue
diff --git a/kolla/tests/test_build.py b/kolla/tests/test_build.py
index d92906ced..98c240d6e 100644
--- a/kolla/tests/test_build.py
+++ b/kolla/tests/test_build.py
@@ -10,6 +10,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import copy
import fixtures
import itertools
import mock
@@ -20,24 +21,18 @@ from kolla.cmd import build
from kolla.tests import base
-FAKE_IMAGE = {
- 'name': 'image-base',
- 'status': 'matched',
- 'parent': None,
- 'parent_name': None,
- 'path': '/fake/path',
- 'plugins': [],
- 'fullname': 'image-base:latest',
-}
+FAKE_IMAGE = build.Image('image-base', 'image-base:latest',
+ '/fake/path', parent_name=None,
+ parent=None, status=build.STATUS_MATCHED)
class TasksTest(base.TestCase):
def setUp(self):
super(TasksTest, self).setUp()
- self.image = FAKE_IMAGE.copy()
+ self.image = copy.deepcopy(FAKE_IMAGE)
# NOTE(jeffrey4l): use a real, temporary dir
- self.image['path'] = self.useFixture(fixtures.TempDir()).path
+ self.image.path = self.useFixture(fixtures.TempDir()).path
@mock.patch.dict(os.environ, clear=True)
@mock.patch('docker.Client')
@@ -45,7 +40,7 @@ class TasksTest(base.TestCase):
pusher = build.PushTask(self.conf, self.image)
pusher.run()
mock_client().push.assert_called_once_with(
- self.image['fullname'], stream=True, insecure_registry=True)
+ self.image.canonical_name, stream=True, insecure_registry=True)
@mock.patch.dict(os.environ, clear=True)
@mock.patch('docker.Client')
@@ -55,7 +50,7 @@ class TasksTest(base.TestCase):
builder.run()
mock_client().build.assert_called_once_with(
- path=self.image['path'], tag=self.image['fullname'],
+ path=self.image.path, tag=self.image.canonical_name,
nocache=False, rm=True, pull=True, forcerm=True,
buildargs=None)
@@ -74,7 +69,7 @@ class TasksTest(base.TestCase):
builder.run()
mock_client().build.assert_called_once_with(
- path=self.image['path'], tag=self.image['fullname'],
+ path=self.image.path, tag=self.image.canonical_name,
nocache=False, rm=True, pull=True, forcerm=True,
buildargs=build_args)
@@ -92,7 +87,7 @@ class TasksTest(base.TestCase):
builder.run()
mock_client().build.assert_called_once_with(
- path=self.image['path'], tag=self.image['fullname'],
+ path=self.image.path, tag=self.image.canonical_name,
nocache=False, rm=True, pull=True, forcerm=True,
buildargs=build_args)
@@ -112,7 +107,7 @@ class TasksTest(base.TestCase):
builder.run()
mock_client().build.assert_called_once_with(
- path=self.image['path'], tag=self.image['fullname'],
+ path=self.image.path, tag=self.image.canonical_name,
nocache=False, rm=True, pull=True, forcerm=True,
buildargs=build_args)
@@ -121,7 +116,7 @@ class TasksTest(base.TestCase):
@mock.patch('docker.Client')
@mock.patch('requests.get')
def test_requests_get_timeout(self, mock_get, mock_client):
- self.image['source'] = {
+ self.image.source = {
'source': 'http://fake/source',
'type': 'url',
'name': 'fake-image-base'
@@ -129,12 +124,12 @@ class TasksTest(base.TestCase):
push_queue = mock.Mock()
builder = build.BuildTask(self.conf, self.image, push_queue)
mock_get.side_effect = requests.exceptions.Timeout
- get_result = builder.process_source(self.image, self.image['source'])
+ get_result = builder.process_source(self.image, self.image.source)
self.assertIsNone(get_result)
- self.assertEqual(self.image['status'], 'error')
- self.assertEqual(self.image['logs'], str())
- mock_get.assert_called_once_with(self.image['source']['source'],
+ self.assertEqual(self.image.status, build.STATUS_ERROR)
+ self.assertEqual(str(self.image.logs), str())
+ mock_get.assert_called_once_with(self.image.source['source'],
timeout=120)
self.assertFalse(builder.success)
@@ -146,8 +141,8 @@ class KollaWorkerTest(base.TestCase):
def setUp(self):
super(KollaWorkerTest, self).setUp()
- image = FAKE_IMAGE.copy()
- image['status'] = None
+ image = copy.deepcopy(FAKE_IMAGE)
+ image.status = None
self.images = [image]
def test_supported_base_type(self):
@@ -188,14 +183,15 @@ class KollaWorkerTest(base.TestCase):
'type': 'git'
}
for image in kolla.images:
- if image['name'] == 'neutron-server':
- self.assertEqual(image['plugins'][0], expected_plugin)
+ if image.name == 'neutron-server':
+ self.assertEqual(image.plugins[0], expected_plugin)
break
else:
self.fail('Can not find the expected neutron arista plugin')
def _get_matched_images(self, images):
- return [image for image in images if image['status'] == 'matched']
+ return [image for image in images
+ if image.status == build.STATUS_MATCHED]
def test_without_profile(self):
kolla = build.KollaWorker(self.conf)