Merge "Fix retries when pushing images" into stable/stein

2 files changed, 37 insertions, 0 deletions

diff --git a/kolla/image/build.py b/kolla/image/build.py
index 8e7b546e3..f70f1b445 100755
--- a/kolla/image/build.py
+++ b/kolla/image/build.py
@@ -368,6 +368,9 @@ class PushTask(DockerTask):
                 image.status = STATUS_ERROR
                 self.logger.error(response['errorDetail']['message'])
 
+        # Reset any previous errors.
+        image.status = STATUS_BUILT
+
 
 class BuildTask(DockerTask):
     """Task that builds out an image."""
diff --git a/kolla/tests/test_build.py b/kolla/tests/test_build.py
index 1fe0f27d3..b0ce009a9 100644
--- a/kolla/tests/test_build.py
+++ b/kolla/tests/test_build.py
@@ -77,6 +77,40 @@ class TasksTest(base.TestCase):
         pusher.run()
         mock_client().push.assert_called_once_with(
             self.image.canonical_name, decode=True, stream=True)
+        self.assertTrue(pusher.success)
+
+    @mock.patch('docker.version', '3.0.0')
+    @mock.patch.dict(os.environ, clear=True)
+    @mock.patch('docker.APIClient')
+    def test_push_image_failure(self, mock_client):
+        self.dc = mock_client
+        mock_client().push.side_effect = Exception
+        pusher = build.PushTask(self.conf, self.image)
+        pusher.run()
+        mock_client().push.assert_called_once_with(
+            self.image.canonical_name, decode=True, stream=True)
+        self.assertFalse(pusher.success)
+        self.assertEqual(build.STATUS_PUSH_ERROR, self.image.status)
+
+    @mock.patch('docker.version', '3.0.0')
+    @mock.patch.dict(os.environ, clear=True)
+    @mock.patch('docker.APIClient')
+    def test_push_image_failure_retry(self, mock_client):
+        self.dc = mock_client
+        mock_client().push.side_effect = [Exception, []]
+        pusher = build.PushTask(self.conf, self.image)
+        pusher.run()
+        mock_client().push.assert_called_once_with(
+            self.image.canonical_name, decode=True, stream=True)
+        self.assertFalse(pusher.success)
+        self.assertEqual(build.STATUS_PUSH_ERROR, self.image.status)
+
+        # Try again, this time without exception.
+        pusher.reset()
+        pusher.run()
+        self.assertEqual(2, mock_client().push.call_count)
+        self.assertTrue(pusher.success)
+        self.assertEqual(build.STATUS_BUILT, self.image.status)
 
     @mock.patch.dict(os.environ, clear=True)
     @mock.patch('docker.APIClient')