From 8df04de1eac9a7e060b5f6a5b4e164c151531cc6 Mon Sep 17 00:00:00 2001
From: rlaphoenix <rlaphoenix@pm.me>
Date: Fri, 19 May 2023 22:11:05 +0100
Subject: [PATCH] Remove file size check from Requests downloader
We cannot actually do this check. The Content-Length value will be the size after being further encoded or compressed. While we can find out what it was compressed with via the Content-Encoding header, we cannot match the downloaded length with the Content-Length header as requests will automatically decompress/decode according to the Content-Encoding header.
---
devine/core/downloaders/requests.py | 4 ----
1 file changed, 4 deletions(-)
diff --git a/devine/core/downloaders/requests.py b/devine/core/downloaders/requests.py
index d2d2fa5..1ddf599 100644
--- a/devine/core/downloaders/requests.py
+++ b/devine/core/downloaders/requests.py
@@ -57,7 +57,6 @@ def requests(
for url, out_path in uri:
out_path.parent.mkdir(parents=True, exist_ok=True)
stream = session.get(url, stream=True)
- file_size = stream.headers.get("Content-Length")
with open(out_path, "wb") as f:
written = 0
for chunk in stream.iter_content(chunk_size=1024):
@@ -77,9 +76,6 @@ def requests(
progress(downloaded=f"{filesize.decimal(download_speed)}/s")
last_speed_refresh = now
download_sizes.clear()
- if file_size and written < int(file_size):
- raise ValueError(
- f"{url} finished downloading unexpectedly, got {decimal(written)}/{decimal(int(file_size))}")
return 0