From ce53a1b636b6eb3bc29b1f29be061eddb98c50d3 Mon Sep 17 00:00:00 2001
From: rlaphoenix <rlaphoenix@pm.me>
Date: Tue, 28 Feb 2023 08:18:30 +0000
Subject: [PATCH] Don't run aria2c under asyncio, further improve progress
updates
I've removed asyncio usage as it's generally unnecessary. If you want to run aria2c under a thread, run it under a thread. In the case for devine, this would take another thread, and would be another thread layer deep. Pointless. Would affect speed.
With this change I've been able to improve the aria2c progress capture code quite a bit.
---
devine/commands/dl.py | 13 +++--
devine/core/downloaders/aria2c.py | 88 +++++++++++++------------------
devine/core/manifests/dash.py | 13 +++--
devine/core/manifests/hls.py | 13 +++--
4 files changed, 54 insertions(+), 73 deletions(-)
diff --git a/devine/commands/dl.py b/devine/commands/dl.py
index d34a6db..3462bb5 100644
--- a/devine/commands/dl.py
+++ b/devine/commands/dl.py
@@ -1,6 +1,5 @@
from __future__ import annotations
-import asyncio
import html
import logging
import math
@@ -756,13 +755,13 @@ class dl:
# no else-if as DASH may convert the track to URL descriptor
if track.descriptor == track.Descriptor.URL:
- asyncio.run(aria2c(
- track.url,
- save_path,
- service.session.headers,
- proxy if track.needs_proxy else None,
+ aria2c(
+ uri=track.url,
+ out=save_path,
+ headers=service.session.headers,
+ proxy=proxy if track.needs_proxy else None,
progress=progress
- ))
+ )
track.path = save_path
if not track.drm and isinstance(track, (Video, Audio)):
diff --git a/devine/core/downloaders/aria2c.py b/devine/core/downloaders/aria2c.py
index 486e6f7..d0886e3 100644
--- a/devine/core/downloaders/aria2c.py
+++ b/devine/core/downloaders/aria2c.py
@@ -1,6 +1,4 @@
-import asyncio
import subprocess
-from asyncio import IncompleteReadError
from functools import partial
from pathlib import Path
from typing import Optional, Union
@@ -12,7 +10,7 @@ from devine.core.console import console
from devine.core.utilities import get_binary_path, start_pproxy
-async def aria2c(
+def aria2c(
uri: Union[str, list[str]],
out: Path,
headers: Optional[dict] = None,
@@ -81,68 +79,54 @@ async def aria2c(
if proxy.lower().split(":")[0] != "http":
# HTTPS proxies are not supported by aria2(c).
# Proxy the proxy via pproxy to access it as an HTTP proxy.
- async with start_pproxy(proxy) as pproxy_:
- return await aria2c(uri, out, headers, pproxy_)
+ with start_pproxy(proxy) as pproxy_:
+ return aria2c(uri, out, headers, pproxy_)
arguments += ["--all-proxy", proxy]
- p = await asyncio.create_subprocess_exec(
- executable,
- *arguments,
+ p = subprocess.Popen(
+ [executable, *arguments],
stdin=subprocess.PIPE,
stderr=[None, subprocess.DEVNULL][silent],
stdout=(
subprocess.PIPE if progress else
subprocess.DEVNULL if silent else
None
- )
+ ),
+ universal_newlines=True
)
- p.stdin.write(uri.encode())
- await p.stdin.drain()
- p.stdin.close()
+ p._stdin_write(uri) # noqa
if progress:
- def update_progress_bar(data: str):
- if "%" in data:
- # id, dledMiB/totalMiB(x%), CN:xx, DL:xxMiB, ETA:Xs
- # eta may not always be available
- data_parts = data[1:-1].split()
- perc_parts = data_parts[1].split("(")
- if len(perc_parts) == 2:
- # might otherwise be e.g., 0B/0B, with no % symbol provided
- progress(
- total=100,
- completed=int(perc_parts[1][:-2]),
- downloaded=f"{data_parts[3].split(':')[1]}/s"
- )
+ is_dl_summary = False
+ for line in iter(p.stdout.readline, ""):
+ line = line.strip()
+ if line:
+ if line.startswith("[") and line.endswith("]"):
+ if "%" in line:
+ # id, dledMiB/totalMiB(x%), CN:xx, DL:xxMiB, ETA:Xs
+ # eta may not always be available
+ data_parts = line[1:-1].split()
+ perc_parts = data_parts[1].split("(")
+ if len(perc_parts) == 2:
+ # might otherwise be e.g., 0B/0B, with no % symbol provided
+ progress(
+ total=100,
+ completed=int(perc_parts[1][:-2]),
+ downloaded=f"{data_parts[3].split(':')[1]}/s"
+ )
+ elif line.startswith("Download Results"):
+ # we know it's 100% downloaded, but let's use the avg dl speed value
+ is_dl_summary = True
+ elif is_dl_summary and "OK" in line and "|" in line:
+ gid, status, avg_speed, path_or_uri = line.split("|")
+ progress(total=100, completed=100, downloaded=avg_speed.strip())
+ elif not is_dl_summary:
+ buffer_msg = line.split(" ", maxsplit=2)
+ buffer_msg = f"[Aria2c]: {buffer_msg[-1].strip()}"
+ console.log(Text.from_ansi(buffer_msg))
- # I'm sorry for this shameful code, aria2(c) is annoying as f!!!
- while not p.stdout.at_eof():
- try:
- buffer = await p.stdout.readuntil(b"\r")
- except IncompleteReadError as e:
- buffer = e.partial
-
- buffer = buffer.decode().strip()
- if buffer:
- buffer_lines = buffer.splitlines()
- is_dl_summary = False
- for line in buffer_lines:
- if line:
- if line.startswith("[") and line.endswith("]"):
- update_progress_bar(line)
- elif line.startswith("Download Results"):
- # we know it's 100% downloaded, but let's use the avg dl speed value
- is_dl_summary = True
- elif is_dl_summary and "OK" in line and "|" in line:
- gid, status, avg_speed, path_or_uri = line.split("|")
- progress(total=100, completed=100, downloaded=avg_speed.strip())
- elif not is_dl_summary:
- buffer_msg = line.split(" ", maxsplit=2)
- buffer_msg = f"[Aria2c]: {buffer_msg[-1].strip()}"
- console.log(Text.from_ansi(buffer_msg))
-
- await p.wait()
+ p.wait()
if p.returncode != 0:
raise subprocess.CalledProcessError(p.returncode, arguments)
diff --git a/devine/core/manifests/dash.py b/devine/core/manifests/dash.py
index d0bb47e..852d5a6 100644
--- a/devine/core/manifests/dash.py
+++ b/devine/core/manifests/dash.py
@@ -1,6 +1,5 @@
from __future__ import annotations
-import asyncio
import base64
import logging
import math
@@ -469,13 +468,13 @@ class DASH:
segment_save_path.parent.mkdir(parents=True, exist_ok=True)
segment_save_path.write_bytes(res.content)
else:
- asyncio.run(aria2c(
- segment_uri,
- segment_save_path,
- session.headers,
- proxy,
+ aria2c(
+ uri=segment_uri,
+ out=segment_save_path,
+ headers=session.headers,
+ proxy=proxy,
silent=True
- ))
+ )
data_size = segment_save_path.stat().st_size
diff --git a/devine/core/manifests/hls.py b/devine/core/manifests/hls.py
index 8431188..eedc613 100644
--- a/devine/core/manifests/hls.py
+++ b/devine/core/manifests/hls.py
@@ -1,6 +1,5 @@
from __future__ import annotations
-import asyncio
import logging
import re
import sys
@@ -289,13 +288,13 @@ class HLS:
segment_save_path.parent.mkdir(parents=True, exist_ok=True)
segment_save_path.write_bytes(res.content)
else:
- asyncio.run(aria2c(
- segment.uri,
- segment_save_path,
- session.headers,
- proxy,
+ aria2c(
+ uri=segment.uri,
+ out=segment_save_path,
+ headers=session.headers,
+ proxy=proxy,
silent=True
- ))
+ )
data_size = segment_save_path.stat().st_size