celery workers

App/Worker.py

@@ -10,15 +10,15 @@ from asgiref.sync import async_to_sync
 import json
 import os
 
-# celery = Celery()
-# celery.config_from_object(celery_config)
+celery = Celery()
+celery.config_from_object(celery_config)
 # celery.conf.update(
 #     # Other Celery configuration settings
 #     CELERYD_LOG_LEVEL="DEBUG",  # Set log level to DEBUG for the worker
 # )
 
 
-# @celery.task(name="CreateFile")
+@celery.task(name="CreateFile")
 def create_json_file(assets: List[Assets], asset_dir: str):
     for asset in assets:
         filename = f"{asset.type.capitalize()}Sequences.json"
@@ -59,7 +59,7 @@ def download_with_wget(link, download_dir, filename):
     subprocess.run(["aria2c", link, "-d", download_dir, "-o", filename])
 
 
-# @celery.task(name="CopyRemotion")
+@celery.task(name="CopyRemotion")
 def copy_remotion_app(src: str, dest: str):
     shutil.copytree(src, dest)
 
@@ -68,7 +68,7 @@ def copy_remotion_app(src: str, dest: str):
     # create_symlink(source_dir, target_dir=dest, symlink_name="node_module")
 
 
-# @celery.task(name="Unsilence")
+@celery.task(name="Unsilence")
 def unsilence(directory: str):
     output_dir = os.path.join(directory, "out/video.mp4")
     shortered_dir = os.path.join(directory, "out/temp.mp4")
@@ -77,13 +77,13 @@ def unsilence(directory: str):
     os.rename(shortered_dir, output_dir)
 
 
-# @celery.task(name="InstallDependency")
+@celery.task(name="InstallDependency")
 def install_dependencies(directory: str):
     os.chdir(directory)
     os.system("npm install")
 
 
-# @celery.task(name="DownloadAssets")
+@celery.task(name="DownloadAssets")
 def download_assets(links: List[LinkInfo], temp_dir: str):
     public_dir = f"{temp_dir}/public"
     for link in links:
@@ -92,14 +92,14 @@ def download_assets(links: List[LinkInfo], temp_dir: str):
         download_with_wget(file_link, public_dir, file_name)
 
 
-# @celery.task(name="RenderFile")
+@celery.task(name="RenderFile")
 def render_video(directory: str, output_directory: str):
     os.chdir(directory)
     os.system(f"npm run build --output {output_directory}")
     print("complete")
 
 
-# @celery.task(name="send")
+@celery.task(name="send")
 async def cleanup_temp_directory(
     temp_dir: str, output_dir: str, chat_id: int = -1002069945904
 ):
@@ -114,7 +114,7 @@ async def cleanup_temp_directory(
         shutil.rmtree(temp_dir, ignore_errors=True)
 
 
-# @celery.task(name="All")
+@celery.task(name="All")
 async def celery_task(video_task: EditorRequest):
     remotion_app_dir = os.path.join("/srv", "Remotion-app")
     project_id = str(uuid.uuid4())
@@ -123,26 +123,26 @@ async def celery_task(video_task: EditorRequest):
 
     assets_dir = os.path.join(temp_dir, "src/HelloWorld/Assets")
 
-    copy_remotion_app(remotion_app_dir, temp_dir),
-    install_dependencies(temp_dir),
-    create_constants_json_file(video_task.constants, assets_dir),
-    create_json_file(video_task.assets, assets_dir),
-    download_assets(video_task.links, temp_dir) if video_task.links else None,
-    render_video(temp_dir, output_dir),
-    unsilence(temp_dir),
-    await cleanup_temp_directory(temp_dir, output_dir),
-
-    # chain(
-    #     copy_remotion_app.si(remotion_app_dir, temp_dir),
-    #     install_dependencies.si(temp_dir),
-    #     create_json_file.si(video_task.assets, assets_dir),
-    #     download_assets.si(video_task.links, temp_dir) if video_task.links else None,
-    #     render_video.si(temp_dir, output_dir),
-    #     # unsilence.si(temp_dir),
-    #     cleanup_temp_directory.si(temp_dir, output_dir),
-    # ).apply_async(
-    #     # link_error=handle_error
-    # )  # Link the tasks and handle errors
+    # copy_remotion_app(remotion_app_dir, temp_dir),
+    # install_dependencies(temp_dir),
+    # create_constants_json_file(video_task.constants, assets_dir),
+    # create_json_file(video_task.assets, assets_dir),
+    # download_assets(video_task.links, temp_dir) if video_task.links else None,
+    # render_video(temp_dir, output_dir),
+    # unsilence(temp_dir),
+    # await cleanup_temp_directory(temp_dir, output_dir),
+
+    chain(
+        copy_remotion_app.si(remotion_app_dir, temp_dir),
+        install_dependencies.si(temp_dir),
+        create_json_file.si(video_task.assets, assets_dir),
+        download_assets.si(video_task.links, temp_dir) if video_task.links else None,
+        render_video.si(temp_dir, output_dir),
+        unsilence.si(temp_dir),
+        cleanup_temp_directory.si(temp_dir, output_dir),
+    ).apply_async(
+        # link_error=handle_error
+    )  # Link the tasks and handle errors
 
 
 def handle_error(task_id, err, *args, **kwargs):

App/celery_config.py

@@ -9,13 +9,13 @@ result_accept_content = ["application/json", "application/x-python-serialize"]
 timezone = "Europe/Oslo"
 enable_utc = True
 
-broker_url = f"rediss://default:157885c870c9fe1154ac9ae5d0ab97b6@master.llm-cache--6h5xnbs2d8rh.addon.code.run:6379?ssl_cert_reqs=none"
+broker_url = f"amqps://llmgzyix:WZZdL_6mmwvoawt58_gYYJV4veF8dOZm@beaver.rmq.cloudamqp.com/llmgzyix"
 result_backend = f"db+postgresql+psycopg2://postgres:[email protected]:5432/postgres"
 
 # SSL/TLS and SNI configuration
-broker_use_ssl = {
-    "ssl_cert_reqs": ssl.CERT_NONE,
-    "ssl_match_hostname": False,
-    "ssl_check_hostname": False,
-    "ssl_sni": "master.cache--j5zxzwppzvjs.addon.code.run",
-}
+# broker_use_ssl = {
+#     "ssl_cert_reqs": ssl.CERT_NONE,
+#     "ssl_match_hostname": False,
+#     "ssl_check_hostname": False,
+#     "ssl_sni": "master.cache--j5zxzwppzvjs.addon.code.run",
+# }

Dockerfile

@@ -9,7 +9,7 @@ RUN chmod -R 755 /srv
 
 # Install dependencies
 RUN apt-get update && \
-  apt-get install -y libu2f-udev libvulkan1 mesa-vulkan-drivers wget ffmpeg curl aria2
+  apt-get install -y   wget ffmpeg curl aria2
 
 RUN apt-get install -y \
   fonts-liberation \
@@ -26,6 +26,7 @@ RUN apt-get install -y \
   libvulkan1 \
   libxcomposite1 \
   libxdamage1 \
+  mesa-vulkan-drivers\
   libxfixes3 \
   libasound2 \
   libxkbcommon0 \
@@ -69,9 +70,9 @@ RUN pipx install unsilence
 
 
 # Command to run the application
-# CMD python -m uvicorn App.app:app --host 0.0.0.0 --port 7860 &  python -m celery -A App.Worker.celery worker -c 4  --max-tasks-per-child=1 
+CMD python -m uvicorn App.app:app --host 0.0.0.0 --port 7860 &  python -m celery -A App.Worker.celery worker -c 5  --max-tasks-per-child=1  --without-heartbeat 
 
-CMD python -m uvicorn App.app:app --host 0.0.0.0  --port 7860  --workers 2
+# CMD python -m uvicorn App.app:app --host 0.0.0.0  --port 7860  --workers 2
 
 
 EXPOSE 7860

requirements.txt

@@ -10,7 +10,7 @@ Werkzeug==2.2.2
 uvicorn==0.21.1
 gunicorn
 requests
-celery
+celery==5.3.0
 pyrogram
 Telethon
 TgCrypto