diff options
| author | Matt Hart <matthew.hart@linaro.org> | 2018-07-16 17:46:45 +0100 |
|---|---|---|
| committer | Neil Williams <neil.williams@linaro.org> | 2018-07-24 09:05:47 +0000 |
| commit | baf61b4651593964b5cbc7c56cea6154c07c9e8a (patch) | |
| tree | 17a80b758031ca76175c55e0231e5085b03dfb83 | |
| parent | ca74a19397579ce57b622126d22253f1fc1fcfe7 (diff) | |
Check downloaded file size against expected
If the data stream closes during a download, don't just assume the
download is finised. If we have an expected size for calculating
the progress, use that to check the file has fully downloaded.
Change-Id: I7d6790d867151c991286f46deb0ce320bbe39a37
| -rw-r--r-- | lava_dispatcher/actions/deploy/download.py | 9 | ||||
| -rw-r--r-- | lava_dispatcher/test/test_compression.py | 4 |
2 files changed, 13 insertions, 0 deletions
diff --git a/lava_dispatcher/actions/deploy/download.py b/lava_dispatcher/actions/deploy/download.py index 38c238e53..a3502c7ab 100644 --- a/lava_dispatcher/actions/deploy/download.py +++ b/lava_dispatcher/actions/deploy/download.py @@ -317,6 +317,13 @@ class DownloadHandler(Action): # pylint: disable=too-many-instance-attributes (downloaded_size / (1024 * 1024), round(ending - beginning, 2), round(downloaded_size / (1024 * 1024 * (ending - beginning)), 2))) + # If the remote server uses "Content-Encoding: gzip", this calculation will be wrong + # because requests will decompress the file on the fly, creating a larger file than + # LAVA expects. + if self.size: + if self.size != downloaded_size: + raise InfrastructureError("Download finished (%i bytes) but was not expected size (%i bytes), check your networking." % (downloaded_size, self.size)) + # set the dynamic data into the context self.set_namespace_data(action='download-action', label=self.key, key='file', value=fname) self.set_namespace_data(action='download-action', label=self.key, key='md5', value=md5.hexdigest()) @@ -464,6 +471,8 @@ class HttpDownloadAction(DownloadHandler): def reader(self): res = None try: + # FIXME: When requests 3.0 is released, use the enforce_content_length + # parameter to raise an exception the file is not fully downloaded res = requests.get(self.url.geturl(), allow_redirects=True, stream=True) if res.status_code != requests.codes.OK: # pylint: disable=no-member diff --git a/lava_dispatcher/test/test_compression.py b/lava_dispatcher/test/test_compression.py index ae63047ee..cd5d4b7b8 100644 --- a/lava_dispatcher/test/test_compression.py +++ b/lava_dispatcher/test/test_compression.py @@ -65,6 +65,8 @@ class TestDecompression(StdoutTestCase): outputmd5 = md5sumhash.hexdigest() outputsha = sha256hash.hexdigest() outputsize = os.path.getsize(os.path.join(httpaction.path, output)) + self.assertIsInstance(httpaction.size, int) + self.assertIsNot(httpaction.size, -1) if httpaction.key == 'testzip': # zipfiles are NOT decompressed on the fly self.assertEqual(outputmd5, md5zipsum) @@ -72,6 +74,8 @@ class TestDecompression(StdoutTestCase): self.assertEqual(outputsize, zipsize) # zipfiles aren't decompressed, so shouldn't change name self.assertEqual(outputfile, '10MB.zip') + # we know it's 10MB.zip for download size test + self.assertEqual(httpaction.size, 10109) else: self.assertEqual(outputmd5, md5sum) self.assertEqual(outputsha, sha256sum) |