From f79df0f25e096f984c1c83b7048979ca2b167555 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Thu, 19 Sep 2019 17:20:50 +0100 Subject: Fix retries when pushing images Currently the retry mechanism is broken for pushing because the image state gets set to an error state, and is never reset to 'built'. This prevents the PushTask from setting success to True. This change sets the image state to 'built' if the push succeeds, ensuring it overrides any previous failures. Change-Id: I93fc0e383da8fec6b3ca31f8094321c2a0c3af71 Closes-Bug: #1844697 (cherry picked from commit f8ded663891f79c289156ee61d4e02197d80ce7a) --- kolla/image/build.py | 3 +++ kolla/tests/test_build.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/kolla/image/build.py b/kolla/image/build.py index 8e7b546e3..f70f1b445 100755 --- a/kolla/image/build.py +++ b/kolla/image/build.py @@ -368,6 +368,9 @@ class PushTask(DockerTask): image.status = STATUS_ERROR self.logger.error(response['errorDetail']['message']) + # Reset any previous errors. + image.status = STATUS_BUILT + class BuildTask(DockerTask): """Task that builds out an image.""" diff --git a/kolla/tests/test_build.py b/kolla/tests/test_build.py index 1fe0f27d3..b0ce009a9 100644 --- a/kolla/tests/test_build.py +++ b/kolla/tests/test_build.py @@ -77,6 +77,40 @@ class TasksTest(base.TestCase): pusher.run() mock_client().push.assert_called_once_with( self.image.canonical_name, decode=True, stream=True) + self.assertTrue(pusher.success) + + @mock.patch('docker.version', '3.0.0') + @mock.patch.dict(os.environ, clear=True) + @mock.patch('docker.APIClient') + def test_push_image_failure(self, mock_client): + self.dc = mock_client + mock_client().push.side_effect = Exception + pusher = build.PushTask(self.conf, self.image) + pusher.run() + mock_client().push.assert_called_once_with( + self.image.canonical_name, decode=True, stream=True) + self.assertFalse(pusher.success) + self.assertEqual(build.STATUS_PUSH_ERROR, self.image.status) + + @mock.patch('docker.version', '3.0.0') + @mock.patch.dict(os.environ, clear=True) + @mock.patch('docker.APIClient') + def test_push_image_failure_retry(self, mock_client): + self.dc = mock_client + mock_client().push.side_effect = [Exception, []] + pusher = build.PushTask(self.conf, self.image) + pusher.run() + mock_client().push.assert_called_once_with( + self.image.canonical_name, decode=True, stream=True) + self.assertFalse(pusher.success) + self.assertEqual(build.STATUS_PUSH_ERROR, self.image.status) + + # Try again, this time without exception. + pusher.reset() + pusher.run() + self.assertEqual(2, mock_client().push.call_count) + self.assertTrue(pusher.success) + self.assertEqual(build.STATUS_BUILT, self.image.status) @mock.patch.dict(os.environ, clear=True) @mock.patch('docker.APIClient') -- cgit v1.2.3