Fix retries when pushing images

Currently the retry mechanism is broken for pushing because the image
state gets set to an error state, and is never reset to 'built'. This
prevents the PushTask from setting success to True.

This change sets the image state to 'built' if the push succeeds,
ensuring it overrides any previous failures.

Change-Id: I93fc0e383da8fec6b3ca31f8094321c2a0c3af71
Closes-Bug: #1844697

2 files changed, 37 insertions, 0 deletions

diff --git a/kolla/image/build.py b/kolla/image/build.py
index 0ad5cd6d3..914f08870 100755
--- a/kolla/image/build.py
+++ b/kolla/image/build.py
@@ -340,6 +340,9 @@ class PushTask(DockerTask):
                 image.status = STATUS_ERROR
                 self.logger.error(response['errorDetail']['message'])
 
+        # Reset any previous errors.
+        image.status = STATUS_BUILT
+
 
 class BuildTask(DockerTask):
     """Task that builds out an image."""
diff --git a/kolla/tests/test_build.py b/kolla/tests/test_build.py
index ef309613b..be78a6939 100644
--- a/kolla/tests/test_build.py
+++ b/kolla/tests/test_build.py
@@ -77,6 +77,40 @@ class TasksTest(base.TestCase):
         pusher.run()
         mock_client().push.assert_called_once_with(
             self.image.canonical_name, decode=True, stream=True)
+        self.assertTrue(pusher.success)
+
+    @mock.patch('docker.version', '3.0.0')
+    @mock.patch.dict(os.environ, clear=True)
+    @mock.patch('docker.APIClient')
+    def test_push_image_failure(self, mock_client):
+        self.dc = mock_client
+        mock_client().push.side_effect = Exception
+        pusher = build.PushTask(self.conf, self.image)
+        pusher.run()
+        mock_client().push.assert_called_once_with(
+            self.image.canonical_name, decode=True, stream=True)
+        self.assertFalse(pusher.success)
+        self.assertEqual(build.STATUS_PUSH_ERROR, self.image.status)
+
+    @mock.patch('docker.version', '3.0.0')
+    @mock.patch.dict(os.environ, clear=True)
+    @mock.patch('docker.APIClient')
+    def test_push_image_failure_retry(self, mock_client):
+        self.dc = mock_client
+        mock_client().push.side_effect = [Exception, []]
+        pusher = build.PushTask(self.conf, self.image)
+        pusher.run()
+        mock_client().push.assert_called_once_with(
+            self.image.canonical_name, decode=True, stream=True)
+        self.assertFalse(pusher.success)
+        self.assertEqual(build.STATUS_PUSH_ERROR, self.image.status)
+
+        # Try again, this time without exception.
+        pusher.reset()
+        pusher.run()
+        self.assertEqual(2, mock_client().push.call_count)
+        self.assertTrue(pusher.success)
+        self.assertEqual(build.STATUS_BUILT, self.image.status)
 
     @mock.patch.dict(os.environ, clear=True)
     @mock.patch('docker.APIClient')