Updated videobundler to add concurrency and caching

videobundler/README.md

@@ -22,6 +22,5 @@ Takes 2 input streams, downloads them, and spits out a combined file.
 ## Endpoints
 
 - `/`: Will return `{success:true}` if alive.
-- `/merge?id=VIDEO_ID&audio_itag=AUDIO_ITAG&video_itag=VIDEO_ITAG`: Starts the merging process. ID is the youtube video ID, and itags are self explanatory. As a response, you will get a job ID that you will be able to use in future requests to query the video or its status. When this process is finished, the inactive autodelete counter will start, which will allow you to fetch the video until the countdown is over.
+- `/[ANYTHING]?id=VIDEO_ID&audio_itag=AUDIO_ITAG&video_itag=VIDEO_ITAG`: Starts the merging process. ID is the youtube video ID, and itags are self explanatory. As a response, you will get a job ID that you will be able to use in future requests to query the video or its status. When this process is finished, the inactive autodelete counter will start, which will allow you to fetch the video until the countdown is over.
-- `/get?id=JOB_ID`: Queries a merged video and sends it to you. If the video is successfully and fully merged you will get a 200 response with a video. However, if it isn't finished, you will get a `success: false` 404 response. If the video indeed exists and is sent to you, the get autodelete counter will start, which will allow you to fetch it until this countdown is over.
+> Replace `[ANYTHING]` with absolutely anything, however it has to be unique to the request. Preferably use an UUID
-- `/check?id=JOB_ID`: Queries a merged video's status. If the video is successfully and fully merged you will get a 200 response with `success:true`. However, if it isn't finished, you will get a `success: false` 404 response. Useful if you want to poll the status without triggering the get autodelete counter.

videobundler/main.py

@@ -1,48 +1,13 @@
-from datetime import datetime
+import asyncio
-from dotenv import load_dotenv
+import aiohttp
-from flask import Flask, request, Response, send_file
+from aiohttp import web
-from threading import Thread
+import string
-from time import sleep
-
-import io
-import json
 import os
 import random
-import string
 import subprocess
-import uuid
 
-load_dotenv()
+app = web.Application()
-
+app.router._frozen = False
-app = Flask(__name__)
-
-def autodelete(job_id: str):
-	f = open(f"pendingDelete.{job_id}", "a")
-	f.write(":3")
-	f.close()
-	sleep(int(os.getenv("TIME_BEFORE_DELETE")))
-	try:
-		os.remove(f"done.{job_id}")
-		os.remove(f"{job_id}.mp4")
-		os.remove(f"{job_id}.m4a")
-		os.remove(f"output.{job_id}.mp4")
-		os.remove(f"pendingDelete.{job_id}")
-	except Exception:
-		_ = 0
-
-def inactive_autodelete(job_id: str):
-	pwd = os.getcwd()
-	sleep(int(os.getenv("INACTIVE_TIME_BEFORE_DELETE")))
-	if not os.path.isfile(f"{pwd}/done.{job_id}"):
-		return
-	try:
-		os.remove(f"done.{job_id}")
-		os.remove(f"{job_id}.mp4")
-		os.remove(f"{job_id}.m4a")
-		os.remove(f"output.{job_id}.mp4")
-		os.remove(f"pendingDelete.{job_id}")
-	except Exception:
-		_ = 0
 
 def get_random_string(length):
 	# choose from all lowercase letter
@@ -50,73 +15,57 @@ def get_random_string(length):
 	result_str = "".join(random.choice(letters) for i in range(length))
 	return result_str
 
-def merge_video(job_id: str, video_id: str, audio_itag: str, video_itag: str):
+async def merge(request):
-	pwd = os.getcwd()
+	# register params
-	# Download both audio and video
+	try:
-	subprocess.run(["wget", f"-O{job_id}.m4a", f"{os.getenv("PROXY_URL")}/latest_version?id={video_id}&itag={audio_itag}&local=true"], check=True)
+		job_id = request.rel_url.query["id"]
-	subprocess.run(["wget", f"-O{job_id}.mp4", f"{os.getenv("PROXY_URL")}/latest_version?id={video_id}&itag={video_itag}&local=true"], check=True)
+		video_id: str = request.rel_url.query["id"]
-	# Merge both files
+		audio_itag: str = request.rel_url.query["audio_itag"]
-	subprocess.run(f"ffmpeg -i {pwd}/{job_id}.m4a -i {pwd}/{job_id}.mp4 -c copy {pwd}/output.{job_id}.mp4", shell=True, check=True)
+		video_itag: str = request.rel_url.query["video_itag"]
+	except:
+		# no one gives a fuck
+		_ = 0
+	# validate
+	if " " in video_id or len(video_id) > 11:
+		print(f"Video {video_id} flagged as invalid, dropping request")
+		return
+	if not audio_itag.isdigit():
+		print(f"Audio itag {audio_itag} flagged as invalid, dropping request")
+		return
+	if not video_itag.isdigit():
+		print(f"Video itag {video_itag} flagged as invalid, dropping request")
+		return
+	if os.path.isfile(f"done.{job_id}"):
+		return web.FileResponse(
+			path=f"output.{job_id}.mp4"
+		)
+	proc_audio = await asyncio.create_subprocess_shell(
+		f"wget -O{job_id}.m4a \"https://eu-proxy.poketube.fun/latest_version?id={video_id}&itag={audio_itag}&local=true\"",
+	)
+	proc_video = await asyncio.create_subprocess_shell(
+		f"wget -O{job_id}.mp4 \"https://eu-proxy.poketube.fun/latest_version?id={video_id}&itag={video_itag}&local=true\""
+	)
+	await asyncio.gather(proc_audio.wait(), proc_video.wait())
+	proc_ffmpeg = await asyncio.create_subprocess_shell(
+		f"ffmpeg -i {job_id}.m4a -i {job_id}.mp4 -c copy output.{job_id}.mp4"
+	)
+	await proc_ffmpeg.wait()
 	f = open(f"done.{job_id}", "a")
 	f.write(":3")
 	f.close()
-	thread = Thread(target=inactive_autodelete, args = (job_id, ))
+	return web.FileResponse(
-	thread.start()
+		path=f"output.{job_id}.mp4"
+	)
 
-@app.route("/")
+async def ping(request):
-def ping():
+	return web.Response(body='{"success": true}', content_type="application/json")
-	return json.loads("""
-	{
-		"success": true
-	}
-	""")
 
-@app.route("/merge")
+async def init_app():
-def merge():
+	app.router.add_get("/{id:.+}", merge)
-	job_id = get_random_string(10)
+	app.router.add_get("/", ping)
-	thread = Thread(target=merge_video, args = (job_id, request.args.get("id"), request.args.get("audio_itag"), request.args.get("video_itag")))
+	return app
-	thread.start()
-	return json.loads('{"success":true,"job_id":"' + job_id + '"}')
 
-@app.route("/get")
+if __name__ == '__main__':
-def get():
+	loop = asyncio.get_event_loop()
-	pwd = os.getcwd()
+	app = loop.run_until_complete(init_app())
-	job_id = request.args.get("job_id")
+	web.run_app(app, port=3030)
-	if os.path.isfile(f"{pwd}/done.{job_id}"):
-		if not os.path.isfile(f"{pwd}/pendingDelete.{job_id}"):
-			thread = Thread(target=autodelete, args = (job_id, ))
-			thread.start()
-		with open(f"output.{job_id}.mp4", "rb") as bytes:
-			return send_file(
-				io.BytesIO(bytes.read()),
-				mimetype="video/mp4",
-				download_name=f"output.{job_id}.mp4",
-				as_attachment=True
-			)
-	return json.loads('{"success":false}'), 404
-
-@app.route("/check")
-def check():
-	pwd = os.getcwd()
-	job_id = request.args.get("job_id")
-	if os.path.isfile(f"{pwd}/done.{job_id}"):
-		return json.loads('{"success":true}')
-	return json.loads('{"success":false}'), 404
-
-
-if __name__ == "__main__":
-	from waitress import serve
-	serve(app, host="0.0.0.0", port=os.getenv("PORT"))
-
-#with open(f"output.{job_id}.mp4", "rb") as bytes:
-		#return send_file(
-	  #      io.BytesIO(bytes.read()),
-		#    mimetype="video/mp4",
-	   #     download_name=f"output.{job_id}.mp4",
-	  #      as_attachment=True
-	 #   )
-#
-#
-#
-#
-#