From ce53a1b636b6eb3bc29b1f29be061eddb98c50d3 Mon Sep 17 00:00:00 2001
From: rlaphoenix <rlaphoenix@pm.me>
Date: Tue, 28 Feb 2023 08:18:30 +0000
Subject: [PATCH] Don't run aria2c under asyncio, further improve progress
 updates

I've removed asyncio usage as it's generally unnecessary. If you want to run aria2c under a thread, run it under a thread. In the case for devine, this would take another thread, and would be another thread layer deep. Pointless. Would affect speed.

With this change I've been able to improve the aria2c progress capture code quite a bit.
---
 devine/commands/dl.py             | 13 +++--
 devine/core/downloaders/aria2c.py | 88 +++++++++++++------------------
 devine/core/manifests/dash.py     | 13 +++--
 devine/core/manifests/hls.py      | 13 +++--
 4 files changed, 54 insertions(+), 73 deletions(-)

diff --git a/devine/commands/dl.py b/devine/commands/dl.py
index d34a6db..3462bb5 100644
--- a/devine/commands/dl.py
+++ b/devine/commands/dl.py
@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import asyncio
 import html
 import logging
 import math
@@ -756,13 +755,13 @@ class dl:
 
         # no else-if as DASH may convert the track to URL descriptor
         if track.descriptor == track.Descriptor.URL:
-            asyncio.run(aria2c(
-                track.url,
-                save_path,
-                service.session.headers,
-                proxy if track.needs_proxy else None,
+            aria2c(
+                uri=track.url,
+                out=save_path,
+                headers=service.session.headers,
+                proxy=proxy if track.needs_proxy else None,
                 progress=progress
-            ))
+            )
             track.path = save_path
 
             if not track.drm and isinstance(track, (Video, Audio)):
diff --git a/devine/core/downloaders/aria2c.py b/devine/core/downloaders/aria2c.py
index 486e6f7..d0886e3 100644
--- a/devine/core/downloaders/aria2c.py
+++ b/devine/core/downloaders/aria2c.py
@@ -1,6 +1,4 @@
-import asyncio
 import subprocess
-from asyncio import IncompleteReadError
 from functools import partial
 from pathlib import Path
 from typing import Optional, Union
@@ -12,7 +10,7 @@ from devine.core.console import console
 from devine.core.utilities import get_binary_path, start_pproxy
 
 
-async def aria2c(
+def aria2c(
     uri: Union[str, list[str]],
     out: Path,
     headers: Optional[dict] = None,
@@ -81,68 +79,54 @@ async def aria2c(
         if proxy.lower().split(":")[0] != "http":
             # HTTPS proxies are not supported by aria2(c).
             # Proxy the proxy via pproxy to access it as an HTTP proxy.
-            async with start_pproxy(proxy) as pproxy_:
-                return await aria2c(uri, out, headers, pproxy_)
+            with start_pproxy(proxy) as pproxy_:
+                return aria2c(uri, out, headers, pproxy_)
         arguments += ["--all-proxy", proxy]
 
-    p = await asyncio.create_subprocess_exec(
-        executable,
-        *arguments,
+    p = subprocess.Popen(
+        [executable, *arguments],
         stdin=subprocess.PIPE,
         stderr=[None, subprocess.DEVNULL][silent],
         stdout=(
             subprocess.PIPE if progress else
             subprocess.DEVNULL if silent else
             None
-        )
+        ),
+        universal_newlines=True
     )
 
-    p.stdin.write(uri.encode())
-    await p.stdin.drain()
-    p.stdin.close()
+    p._stdin_write(uri)  # noqa
 
     if progress:
-        def update_progress_bar(data: str):
-            if "%" in data:
-                # id, dledMiB/totalMiB(x%), CN:xx, DL:xxMiB, ETA:Xs
-                # eta may not always be available
-                data_parts = data[1:-1].split()
-                perc_parts = data_parts[1].split("(")
-                if len(perc_parts) == 2:
-                    # might otherwise be e.g., 0B/0B, with no % symbol provided
-                    progress(
-                        total=100,
-                        completed=int(perc_parts[1][:-2]),
-                        downloaded=f"{data_parts[3].split(':')[1]}/s"
-                    )
+        is_dl_summary = False
+        for line in iter(p.stdout.readline, ""):
+            line = line.strip()
+            if line:
+                if line.startswith("[") and line.endswith("]"):
+                    if "%" in line:
+                        # id, dledMiB/totalMiB(x%), CN:xx, DL:xxMiB, ETA:Xs
+                        # eta may not always be available
+                        data_parts = line[1:-1].split()
+                        perc_parts = data_parts[1].split("(")
+                        if len(perc_parts) == 2:
+                            # might otherwise be e.g., 0B/0B, with no % symbol provided
+                            progress(
+                                total=100,
+                                completed=int(perc_parts[1][:-2]),
+                                downloaded=f"{data_parts[3].split(':')[1]}/s"
+                            )
+                elif line.startswith("Download Results"):
+                    # we know it's 100% downloaded, but let's use the avg dl speed value
+                    is_dl_summary = True
+                elif is_dl_summary and "OK" in line and "|" in line:
+                    gid, status, avg_speed, path_or_uri = line.split("|")
+                    progress(total=100, completed=100, downloaded=avg_speed.strip())
+                elif not is_dl_summary:
+                    buffer_msg = line.split(" ", maxsplit=2)
+                    buffer_msg = f"[Aria2c]: {buffer_msg[-1].strip()}"
+                    console.log(Text.from_ansi(buffer_msg))
 
-        # I'm sorry for this shameful code, aria2(c) is annoying as f!!!
-        while not p.stdout.at_eof():
-            try:
-                buffer = await p.stdout.readuntil(b"\r")
-            except IncompleteReadError as e:
-                buffer = e.partial
-
-            buffer = buffer.decode().strip()
-            if buffer:
-                buffer_lines = buffer.splitlines()
-                is_dl_summary = False
-                for line in buffer_lines:
-                    if line:
-                        if line.startswith("[") and line.endswith("]"):
-                            update_progress_bar(line)
-                        elif line.startswith("Download Results"):
-                            # we know it's 100% downloaded, but let's use the avg dl speed value
-                            is_dl_summary = True
-                        elif is_dl_summary and "OK" in line and "|" in line:
-                            gid, status, avg_speed, path_or_uri = line.split("|")
-                            progress(total=100, completed=100, downloaded=avg_speed.strip())
-                        elif not is_dl_summary:
-                            buffer_msg = line.split(" ", maxsplit=2)
-                            buffer_msg = f"[Aria2c]: {buffer_msg[-1].strip()}"
-                            console.log(Text.from_ansi(buffer_msg))
-
-    await p.wait()
+    p.wait()
 
     if p.returncode != 0:
         raise subprocess.CalledProcessError(p.returncode, arguments)
diff --git a/devine/core/manifests/dash.py b/devine/core/manifests/dash.py
index d0bb47e..852d5a6 100644
--- a/devine/core/manifests/dash.py
+++ b/devine/core/manifests/dash.py
@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import asyncio
 import base64
 import logging
 import math
@@ -469,13 +468,13 @@ class DASH:
                     segment_save_path.parent.mkdir(parents=True, exist_ok=True)
                     segment_save_path.write_bytes(res.content)
                 else:
-                    asyncio.run(aria2c(
-                        segment_uri,
-                        segment_save_path,
-                        session.headers,
-                        proxy,
+                    aria2c(
+                        uri=segment_uri,
+                        out=segment_save_path,
+                        headers=session.headers,
+                        proxy=proxy,
                         silent=True
-                    ))
+                    )
 
                 data_size = segment_save_path.stat().st_size
 
diff --git a/devine/core/manifests/hls.py b/devine/core/manifests/hls.py
index 8431188..eedc613 100644
--- a/devine/core/manifests/hls.py
+++ b/devine/core/manifests/hls.py
@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import asyncio
 import logging
 import re
 import sys
@@ -289,13 +288,13 @@ class HLS:
                 segment_save_path.parent.mkdir(parents=True, exist_ok=True)
                 segment_save_path.write_bytes(res.content)
             else:
-                asyncio.run(aria2c(
-                    segment.uri,
-                    segment_save_path,
-                    session.headers,
-                    proxy,
+                aria2c(
+                    uri=segment.uri,
+                    out=segment_save_path,
+                    headers=session.headers,
+                    proxy=proxy,
                     silent=True
-                ))
+                )
 
             data_size = segment_save_path.stat().st_size