<reponame>pkavumba/arxiv-vanity-restful-api
import datetime
import docker.errors
from django.conf import settings
from django.db import models
from django.contrib.postgres.fields import ArrayField, JSONField
from django.core.files.storage import default_storage
from django.urls import reverse
from django.utils import timezone
import os
from ..scraper.query import query_single_paper
from ..storage import storage_delete_path
from ..utils import log_exception
from .downloader import download_source_file
from .processor import process_render
from .renderer import render_paper, create_client, TooManyRendersRunningError


class RenderError(Exception):
    pass


class PaperIsNotRenderableError(RenderError):
    """Paper cannot be rendered."""


class RenderAlreadyStartedError(RenderError):
    """Render started when render has already been started."""


class RenderWrongStateError(RenderError):
    """The state of a render is being updated when it has not been started."""


class PaperQuerySet(models.QuerySet):
    def _with_has_successful_render_annotation(self):
        renders = Render.objects.filter(
            paper=models.OuterRef("pk"), state=Render.STATE_SUCCESS
        )
        return self.annotate(has_successful_render=models.Exists(renders))

    def has_successful_render(self):
        qs = self._with_has_successful_render_annotation()
        return qs.filter(has_successful_render=True)

    def has_no_successful_render(self):
        qs = self._with_has_successful_render_annotation()
        return qs.filter(has_successful_render=False)

    def _with_has_not_deleted_render_annotation(self):
        renders = Render.objects.filter(paper=models.OuterRef("pk"), is_deleted=False)
        return self.annotate(has_not_deleted_render=models.Exists(renders))

    def has_not_deleted_render(self):
        qs = self._with_has_not_deleted_render_annotation()
        return qs.filter(has_not_deleted_render=True)

    def downloaded(self):
        return self.filter(source_file__isnull=False)

    def not_downloaded(self):
        return self.filter(source_file__isnull=True)

    def update_or_create_from_api(self, result):
        return self.update_or_create(arxiv_id=result["arxiv_id"], defaults=result)

    def update_or_create_from_arxiv_id(self, arxiv_id):
        """
        Query the arXiv API and create a Paper from it.

        Raises:
            `arxiv_vanity.scraper.query.PaperNotFoundError`: If paper does not exist on arxiv.
        """
        return self.update_or_create_from_api(query_single_paper(arxiv_id))

    def machine_learning(self):
        """
        Return only machine learning papers.
        """
        return self.filter(
            categories__overlap=settings.PAPERS_MACHINE_LEARNING_CATEGORIES
        )


class PaperManager(models.Manager):
    def get_queryset(self):
        return super().get_queryset().filter(is_deleted=False)

    def deleted(self):
        return super().get_queryset().filter(is_deleted=True)


class Paper(models.Model):
    # ArXiV fields
    arxiv_id = models.CharField(max_length=50, unique=True)
    arxiv_version = models.IntegerField()
    title = models.TextField()
    published = models.DateTimeField()
    updated = models.DateTimeField(db_index=True)
    summary = models.TextField()
    authors = JSONField()
    arxiv_url = models.URLField()
    pdf_url = models.URLField()
    primary_category = models.CharField(max_length=100)
    categories = ArrayField(models.CharField(max_length=100))
    comment = models.TextField(null=True, blank=True)
    doi = models.CharField(null=True, blank=True, max_length=255)
    journal_ref = models.TextField(null=True, blank=True, max_length=255)

    # arXiv Vanity fields
    source_file = models.ForeignKey(
        "SourceFile", on_delete=models.SET_NULL, null=True, blank=True
    )
    is_deleted = models.BooleanField(default=False)

    objects = PaperManager.from_queryset(PaperQuerySet)()

    class Meta:
        get_latest_by = "updated"

    def __str__(self):
        return self.title

    def get_absolute_url(self):
        return reverse("paper_detail", args=(self.arxiv_id,))

    def get_https_arxiv_url(self):
        url = self.arxiv_url
        if url.startswith("http://arxiv.org"):
            url = "https" + url.lstrip("http")
        return url

    def get_https_pdf_url(self):
        url = self.pdf_url
        if url.startswith("http://arxiv.org"):
            url = "https" + url.lstrip("http")
        return url

    def is_renderable(self):
        """
        Returns whether it is possible to render this paper.
        """
        return self.source_file and self.source_file.is_renderable()

    def get_or_download_source_file(self):
        """
        Attempts to get the source file from a bulk data download, otherwise
        downloads it and creates it.
        """
        self.source_file = SourceFile.objects.get_or_download(self.arxiv_id)
        self.save()
        return self.source_file

    def get_render_to_display_and_render_if_needed(self, force_render=False):
        """
        Returns the render that should display for this paper, and kicks off
        a new render if need be.
        """
        try:
            render = self.renders.not_deleted().latest()
        except Render.DoesNotExist:
            render = None
        if not render:
            return self.render()

        # Stuck for some reason, give it a boot
        # This normally happens if there is an exception raised in render()
        if render.state == Render.STATE_UNSTARTED:
            # This will put it into running state
            render = self.render()

        # There is already a render running, so try and get the most recent one that isn't running
        if render.state == Render.STATE_RUNNING:
            try:
                return self.renders.not_deleted().succeeded().latest()
            except Render.DoesNotExist:
                # If this is the only render or the other renders errored, display loading render
                return render

        elif render.state == Render.STATE_FAILURE:
            # Kick off render if this one has expired
            if render.is_expired() or force_render:
                try:
                    render = self.render()
                except TooManyRendersRunningError:
                    pass
                except:
                    #  Don't block displaying render if kicking off failed
                    log_exception()
            # Try and display a successful render
            try:
                return self.renders.not_deleted().succeeded().latest()
            # Otherwise, display the failed or running render
            except Render.DoesNotExist:
                return render

        elif render.state == Render.STATE_SUCCESS:
            # Kick off render in background if it has expired
            if render.is_expired() or force_render:
                try:
                    self.render()
                except TooManyRendersRunningError:
                    pass
                except:
                    #  Don't block displaying render if kicking off failed
                    log_exception()
            return render

        else:
            raise Exception(f"Unknown render state: {render.state}")

    def render(self):
        """
        Make a new render of this paper. Will download the source file and save
        itself if it hasn't already.
        """
        if not self.source_file:
            self.get_or_download_source_file()
        if not self.source_file.is_renderable():
            raise PaperIsNotRenderableError("This paper is not renderable.")
        render = Render.objects.create(paper=self)
        render.run()
        return render


def _get_expired_date():
    expired_delta = datetime.timedelta(days=settings.PAPERS_EXPIRED_DAYS)
    return timezone.now() - expired_delta


class RenderQuerySet(models.QuerySet):
    def running(self):
        return self.filter(state=Render.STATE_RUNNING)

    def succeeded(self):
        return self.filter(state=Render.STATE_SUCCESS)

    def failed(self):
        return self.filter(state=Render.STATE_FAILURE)

    def not_deleted(self):
        return self.filter(is_deleted=False)

    def deleted(self):
        return self.filter(is_deleted=True)

    def update_state(self):
        """
        Update the state of renders that have a container.
        """
        qs = self.exclude(state=Render.STATE_UNSTARTED).filter(
            container_is_removed=False
        )
        for render in qs.iterator(chunk_size=100):
            try:
                render.update_state()
                render.delete_older_renders_if_successful()
            except:
                log_exception()
        return self

    def expired(self):
        """
        Returns renders that ran more than PAPERS_EXPIRED_DAYS ago.
        """
        return self.filter(created_at__lte=_get_expired_date())

    def not_expired(self):
        """
        Returns renders than ran less than PAPERS_EXPIRED_DAYS ago.
        """
        return self.filter(created_at__gt=_get_expired_date())

    def mark_as_deleted(self):
        """
        Mark renders as deleted. Useful for forcing re-rendering.
        """
        for render in self.iterator():
            render.mark_as_deleted()
        return self


class Render(models.Model):
    STATE_UNSTARTED = "unstarted"
    STATE_RUNNING = "running"
    STATE_SUCCESS = "success"
    STATE_FAILURE = "failure"

    paper = models.ForeignKey(Paper, on_delete=models.CASCADE, related_name="renders")
    created_at = models.DateTimeField(auto_now_add=True)
    state = models.CharField(
        max_length=20,
        default=STATE_UNSTARTED,
        choices=(
            (STATE_UNSTARTED, "Unstarted"),
            (STATE_RUNNING, "Running"),
            (STATE_SUCCESS, "Success"),
            (STATE_FAILURE, "Failure"),
        ),
    )
    is_deleted = models.BooleanField(default=False)
    container_id = models.CharField(max_length=64, null=True, blank=True)
    container_inspect = JSONField(null=True, blank=True)
    container_logs = models.TextField(null=True, blank=True)
    container_is_removed = models.BooleanField(default=False)

    objects = RenderQuerySet.as_manager()

    class Meta:
        get_latest_by = "created_at"

    def __str__(self):
        return self.paper.title

    def save(self, *args, **kwargs):
        super(Render, self).save(*args, **kwargs)

    def get_output_path(self):
        """
        Path to the directory that this render is in.
        """
        return os.path.join("render-output", str(self.id))

    def get_html_path(self):
        """
        Path to the HTML file of this render.
        """
        return os.path.join(self.get_output_path(), "index.html")

    def get_output_url(self):
        """
        Returns the URL to the output path.
        """
        return settings.MEDIA_URL + self.get_output_path()

    def short_container_id(self):
        if self.container_id:
            return self.container_id[:12]

    def get_webhook_url(self):
        """
        Get the webhook to call when the Engrafo job ends to update render
        state.
        """
        path = reverse("render_update_state", args=(self.id,))
        return settings.ENGRAFO_WEBHOOK_URL_PREFIX + path

    def run(self):
        """
        Start running this render.
        """
        if self.state != Render.STATE_UNSTARTED:
            raise RenderAlreadyStartedError(
                f"Render {self.id} has already been started"
            )
        self.container_id = render_paper(
            self.paper.source_file.file.name,
            self.get_output_path(),
            webhook_url=self.get_webhook_url(),
        ).id
        self.state = Render.STATE_RUNNING
        self.save()

    def update_state(self, exit_code=None):
        """
        Update state of this render from the container.

        This is used for two purposes:

        1. Called by the webhook to update the state. In this case, exit_code
            is passed through instead of using the containers exit code
            because the container is still running (it's sending the webhook!).

        2. Called by the update_render_state cron job. In this case, it'll
            sync all of the details about the containers that exist, and remove
            any containers that have stopped. This sweeps up containers
            that have reported their state in (1).
        """
        if self.state == Render.STATE_UNSTARTED:
            # The create container timeed out, but container ran anyway
            # Clean up after it...
            self.mark_as_deleted()
            return

        client = create_client()

        try:
            container = client.containers.get(self.container_id)
            self.container_inspect = container.attrs
            self.container_logs = str(container.logs(), "utf-8").replace("\x00", "")
        except docker.errors.NotFound:
            # Container has been removed for some reason, so mark it as
            # removed so we don't try to update its state again
            self.container_is_removed = True
            # Give it a failed state if the render was still running
            if self.state == Render.STATE_RUNNING:
                self.state = Render.STATE_FAILURE
            self.save()
            return

        if exit_code is None and container.status == "exited":
            exit_code = container.attrs["State"]["ExitCode"]

        if exit_code is not None:
            # Safer to convert int to str than other way round
            if str(exit_code) == "0":
                self.state = Render.STATE_SUCCESS
            else:
                self.state = Render.STATE_FAILURE

        # Deleting containers on Hyper.sh often fails, so save here so we at
        # least save the state. If remove fails, this will get retried by
        # the update_render_state cron job.
        self.save()

        if container.status == "exited":
            try:
                # Force, or Hyper.sh often throws a 500
                container.remove(force=True)
            except docker.errors.NotFound:
                # Somebody got in there before us. Oh well.
                pass
            self.container_is_removed = True
            self.save()

    def get_processed_render(self):
        """
        Do final processing on this render and returns it as a dictionary of
        {"body", "script", "styles"}.
        """
        context = {"render": self, "paper": self.paper}
        with default_storage.open(self.get_html_path()) as fh:
            return process_render(fh, self.get_output_url(), context=context)

    def is_expired(self):
        """
        Returns True if this render was run more than PAPERS_EXPIRED_DAYS ago.
        """
        return self.created_at <= _get_expired_date()

    def mark_as_deleted(self):
        """
        Delete render's output and mark is as deleted.
        """
        try:
            self.delete_output()
        except FileNotFoundError:
            log_exception()
        self.is_deleted = True
        self.save()

    def delete_output(self):
        """
        Delete output path.
        """
        storage_delete_path(default_storage, self.get_output_path())

    def delete_older_renders_if_successful(self):
        """
        Delete any older undeleted renders if this render was successful.
        """
        if not self.state == Render.STATE_SUCCESS:
            return
        for render in self.paper.renders.not_deleted().filter(pk__lt=self.pk):
            render.mark_as_deleted()


class SourceFileBulkTarball(models.Model):
    """
    A tarball of sources that is listed in arXiv's bulk sources manifest.

    We keep track of these so we know which tarballs we have already
    downloaded. They're quite big so we don't want to get them every time.
    """

    filename = models.CharField(max_length=255, unique=True)

    content_md5sum = models.TextField()
    first_item = models.TextField()
    last_item = models.TextField()
    md5sum = models.TextField()
    num_items = models.IntegerField()
    seq_num = models.IntegerField()
    size = models.IntegerField()
    timestamp = models.TextField()
    yymm = models.TextField()

    def __str__(self):
        return self.filename

    def has_correct_number_of_files(self):
        """
        Number of items specified in the manifest matches how many we have
        in the database. If this is false, it suggests there was some error
        in downloading source files.
        """
        return self.num_items == self.sourcefile_set.count()


class SourceFileQuerySet(models.QuerySet):
    def get_or_download(self, arxiv_id):
        """
        Returns a source file for an arxiv ID if it exists (probably created
        from a bulk source download), otherwise downloads and creates it.
        """
        try:
            return self.get(arxiv_id=arxiv_id)
        except SourceFile.DoesNotExist:
            return self.download_and_create(arxiv_id)

    def download_and_create(self, arxiv_id):
        """
        Download the LaTeX source of this paper, save to storage, and create
        SourceFile.
        """
        file = download_source_file(arxiv_id)
        return self.create(arxiv_id=arxiv_id, file=file)

    def filename_exists(self, fn):
        return self.filter(file=f"source-files/{fn}").exists()


class SourceFile(models.Model):
    """
    Represents a paper source file from arXiv.

    NOTE: This is a new system, not yet used by the `Paper` model. Initially,
    we are downloading arXiv's bulk papers into this model, then we can switch
    `Paper` to use this model.
    """

    arxiv_id = models.CharField(max_length=50, unique=True)
    file = models.FileField(upload_to="source-files/", unique=True)
    bulk_tarball = models.ForeignKey(
        SourceFileBulkTarball,
        on_delete=models.SET_NULL,
        null=True,
        blank=True,
        help_text="If this source file is from arXiv's bulk download service, this is the tarball it was in. If null, this source file was downloaded individually.",
    )

    objects = SourceFileQuerySet.as_manager()

    def __str__(self):
        return str(self.file)

    def is_pdf(self):
        return self.file.name.endswith(".pdf")

    def is_renderable(self):
        """
        Returns whether it is possible to render this file.
        """
        name = self.file.name
        return (
            name is not None
            and name.endswith(".gz")
            and not name.endswith(".ps.gz")
            and not name.endswith(".dvi.gz")
        )

import datetime
import docker.errors
from django.conf import settings
from django.db import models
from django.contrib.postgres.fields import ArrayField, JSONField
from django.core.files.storage import default_storage
from django.urls import reverse
from django.utils import timezone
import os
from ..scraper.query import query_single_paper
from ..storage import storage_delete_path
from ..utils import log_exception
from .downloader import download_source_file
from .processor import process_render
from .renderer import render_paper, create_client, TooManyRendersRunningError


class RenderError(Exception):
    pass


class PaperIsNotRenderableError(RenderError):
    """Paper cannot be rendered."""


class RenderAlreadyStartedError(RenderError):
    """Render started when render has already been started."""


class RenderWrongStateError(RenderError):
    """The state of a render is being updated when it has not been started."""


class PaperQuerySet(models.QuerySet):
    def _with_has_successful_render_annotation(self):
        renders = Render.objects.filter(
            paper=models.OuterRef("pk"), state=Render.STATE_SUCCESS
        )
        return self.annotate(has_successful_render=models.Exists(renders))

    def has_successful_render(self):
        qs = self._with_has_successful_render_annotation()
        return qs.filter(has_successful_render=True)

    def has_no_successful_render(self):
        qs = self._with_has_successful_render_annotation()
        return qs.filter(has_successful_render=False)

    def _with_has_not_deleted_render_annotation(self):
        renders = Render.objects.filter(paper=models.OuterRef("pk"), is_deleted=False)
        return self.annotate(has_not_deleted_render=models.Exists(renders))

    def has_not_deleted_render(self):
        qs = self._with_has_not_deleted_render_annotation()
        return qs.filter(has_not_deleted_render=True)

    def downloaded(self):
        return self.filter(source_file__isnull=False)

    def not_downloaded(self):
        return self.filter(source_file__isnull=True)

    def update_or_create_from_api(self, result):
        return self.update_or_create(arxiv_id=result["arxiv_id"], defaults=result)

    def update_or_create_from_arxiv_id(self, arxiv_id):
        """
        Query the arXiv API and create a Paper from it.

        Raises:
            `arxiv_vanity.scraper.query.PaperNotFoundError`: If paper does not exist on arxiv.
        """
        return self.update_or_create_from_api(query_single_paper(arxiv_id))

    def machine_learning(self):
        """
        Return only machine learning papers.
        """
        return self.filter(
            categories__overlap=settings.PAPERS_MACHINE_LEARNING_CATEGORIES
        )


class PaperManager(models.Manager):
    def get_queryset(self):
        return super().get_queryset().filter(is_deleted=False)

    def deleted(self):
        return super().get_queryset().filter(is_deleted=True)


class Paper(models.Model):
    # ArXiV fields
    arxiv_id = models.CharField(max_length=50, unique=True)
    arxiv_version = models.IntegerField()
    title = models.TextField()
    published = models.DateTimeField()
    updated = models.DateTimeField(db_index=True)
    summary = models.TextField()
    authors = JSONField()
    arxiv_url = models.URLField()
    pdf_url = models.URLField()
    primary_category = models.CharField(max_length=100)
    categories = ArrayField(models.CharField(max_length=100))
    comment = models.TextField(null=True, blank=True)
    doi = models.CharField(null=True, blank=True, max_length=255)
    journal_ref = models.TextField(null=True, blank=True, max_length=255)

    # arXiv Vanity fields
    source_file = models.ForeignKey(
        "SourceFile", on_delete=models.SET_NULL, null=True, blank=True
    )
    is_deleted = models.BooleanField(default=False)

    objects = PaperManager.from_queryset(PaperQuerySet)()

    class Meta:
        get_latest_by = "updated"

    def __str__(self):
        return self.title

    def get_absolute_url(self):
        return reverse("paper_detail", args=(self.arxiv_id,))

    def get_https_arxiv_url(self):
        url = self.arxiv_url
        if url.startswith("http://arxiv.org"):
            url = "https" + url.lstrip("http")
        return url

    def get_https_pdf_url(self):
        url = self.pdf_url
        if url.startswith("http://arxiv.org"):
            url = "https" + url.lstrip("http")
        return url

    def is_renderable(self):
        """
        Returns whether it is possible to render this paper.
        """
        return self.source_file and self.source_file.is_renderable()

    def get_or_download_source_file(self):
        """
        Attempts to get the source file from a bulk data download, otherwise
        downloads it and creates it.
        """
        self.source_file = SourceFile.objects.get_or_download(self.arxiv_id)
        self.save()
        return self.source_file

    def get_render_to_display_and_render_if_needed(self, force_render=False):
        """
        Returns the render that should display for this paper, and kicks off
        a new render if need be.
        """
        try:
            render = self.renders.not_deleted().latest()
        except Render.DoesNotExist:
            render = None
        if not render:
            return self.render()

        # Stuck for some reason, give it a boot
        # This normally happens if there is an exception raised in render()
        if render.state == Render.STATE_UNSTARTED:
            # This will put it into running state
            render = self.render()

        # There is already a render running, so try and get the most recent one that isn't running
        if render.state == Render.STATE_RUNNING:
            try:
                return self.renders.not_deleted().succeeded().latest()
            except Render.DoesNotExist:
                # If this is the only render or the other renders errored, display loading render
                return render

        elif render.state == Render.STATE_FAILURE:
            # Kick off render if this one has expired
            if render.is_expired() or force_render:
                try:
                    render = self.render()
                except TooManyRendersRunningError:
                    pass
                except:
                    #  Don't block displaying render if kicking off failed
                    log_exception()
            # Try and display a successful render
            try:
                return self.renders.not_deleted().succeeded().latest()
            # Otherwise, display the failed or running render
            except Render.DoesNotExist:
                return render

        elif render.state == Render.STATE_SUCCESS:
            # Kick off render in background if it has expired
            if render.is_expired() or force_render:
                try:
                    self.render()
                except TooManyRendersRunningError:
                    pass
                except:
                    #  Don't block displaying render if kicking off failed
                    log_exception()
            return render

        else:
            raise Exception(f"Unknown render state: {render.state}")

    def render(self):
        """
        Make a new render of this paper. Will download the source file and save
        itself if it hasn't already.
        """
        if not self.source_file:
            self.get_or_download_source_file()
        if not self.source_file.is_renderable():
            raise PaperIsNotRenderableError("This paper is not renderable.")
        render = Render.objects.create(paper=self)
        render.run()
        return render


def _get_expired_date():
    expired_delta = datetime.timedelta(days=settings.PAPERS_EXPIRED_DAYS)
    return timezone.now() - expired_delta


class RenderQuerySet(models.QuerySet):
    def running(self):
        return self.filter(state=Render.STATE_RUNNING)

    def succeeded(self):
        return self.filter(state=Render.STATE_SUCCESS)

    def failed(self):
        return self.filter(state=Render.STATE_FAILURE)

    def not_deleted(self):
        return self.filter(is_deleted=False)

    def deleted(self):
        return self.filter(is_deleted=True)

    def update_state(self):
        """
        Update the state of renders that have a container.
        """
        qs = self.exclude(state=Render.STATE_UNSTARTED).filter(
            container_is_removed=False
        )
        for render in qs.iterator(chunk_size=100):
            try:
                render.update_state()
                render.delete_older_renders_if_successful()
            except:
                log_exception()
        return self

    def expired(self):
        """
        Returns renders that ran more than PAPERS_EXPIRED_DAYS ago.
        """
        return self.filter(created_at__lte=_get_expired_date())

    def not_expired(self):
        """
        Returns renders than ran less than PAPERS_EXPIRED_DAYS ago.
        """
        return self.filter(created_at__gt=_get_expired_date())

    def mark_as_deleted(self):
        """
        Mark renders as deleted. Useful for forcing re-rendering.
        """
        for render in self.iterator():
            render.mark_as_deleted()
        return self


class Render(models.Model):
    STATE_UNSTARTED = "unstarted"
    STATE_RUNNING = "running"
    STATE_SUCCESS = "success"
    STATE_FAILURE = "failure"

    paper = models.ForeignKey(Paper, on_delete=models.CASCADE, related_name="renders")
    created_at = models.DateTimeField(auto_now_add=True)
    state = models.CharField(
        max_length=20,
        default=STATE_UNSTARTED,
        choices=(
            (STATE_UNSTARTED, "Unstarted"),
            (STATE_RUNNING, "Running"),
            (STATE_SUCCESS, "Success"),
            (STATE_FAILURE, "Failure"),
        ),
    )
    is_deleted = models.BooleanField(default=False)
    container_id = models.CharField(max_length=64, null=True, blank=True)
    container_inspect = JSONField(null=True, blank=True)
    container_logs = models.TextField(null=True, blank=True)
    container_is_removed = models.BooleanField(default=False)

    objects = RenderQuerySet.as_manager()

    class Meta:
        get_latest_by = "created_at"

    def __str__(self):
        return self.paper.title

    def save(self, *args, **kwargs):
        super(Render, self).save(*args, **kwargs)

    def get_output_path(self):
        """
        Path to the directory that this render is in.
        """
        return os.path.join("render-output", str(self.id))

    def get_html_path(self):
        """
        Path to the HTML file of this render.
        """
        return os.path.join(self.get_output_path(), "index.html")

    def get_output_url(self):
        """
        Returns the URL to the output path.
        """
        return settings.MEDIA_URL + self.get_output_path()

    def short_container_id(self):
        if self.container_id:
            return self.container_id[:12]

    def get_webhook_url(self):
        """
        Get the webhook to call when the Engrafo job ends to update render
        state.
        """
        path = reverse("render_update_state", args=(self.id,))
        return settings.ENGRAFO_WEBHOOK_URL_PREFIX + path

    def run(self):
        """
        Start running this render.
        """
        if self.state != Render.STATE_UNSTARTED:
            raise RenderAlreadyStartedError(
                f"Render {self.id} has already been started"
            )
        self.container_id = render_paper(
            self.paper.source_file.file.name,
            self.get_output_path(),
            webhook_url=self.get_webhook_url(),
        ).id
        self.state = Render.STATE_RUNNING
        self.save()

    def update_state(self, exit_code=None):
        """
        Update state of this render from the container.

        This is used for two purposes:

        1. Called by the webhook to update the state. In this case, exit_code
            is passed through instead of using the containers exit code
            because the container is still running (it's sending the webhook!).

        2. Called by the update_render_state cron job. In this case, it'll
            sync all of the details about the containers that exist, and remove
            any containers that have stopped. This sweeps up containers
            that have reported their state in (1).
        """
        if self.state == Render.STATE_UNSTARTED:
            # The create container timeed out, but container ran anyway
            # Clean up after it...
            self.mark_as_deleted()
            return

        client = create_client()

        try:
            container = client.containers.get(self.container_id)
            self.container_inspect = container.attrs
            self.container_logs = str(container.logs(), "utf-8").replace("\x00", "")
        except docker.errors.NotFound:
            # Container has been removed for some reason, so mark it as
            # removed so we don't try to update its state again
            self.container_is_removed = True
            # Give it a failed state if the render was still running
            if self.state == Render.STATE_RUNNING:
                self.state = Render.STATE_FAILURE
            self.save()
            return

        if exit_code is None and container.status == "exited":
            exit_code = container.attrs["State"]["ExitCode"]

        if exit_code is not None:
            # Safer to convert int to str than other way round
            if str(exit_code) == "0":
                self.state = Render.STATE_SUCCESS
            else:
                self.state = Render.STATE_FAILURE

        # Deleting containers on Hyper.sh often fails, so save here so we at
        # least save the state. If remove fails, this will get retried by
        # the update_render_state cron job.
        self.save()

        if container.status == "exited":
            try:
                # Force, or Hyper.sh often throws a 500
                container.remove(force=True)
            except docker.errors.NotFound:
                # Somebody got in there before us. Oh well.
                pass
            self.container_is_removed = True
            self.save()

    def get_processed_render(self):
        """
        Do final processing on this render and returns it as a dictionary of
        {"body", "script", "styles"}.
        """
        context = {"render": self, "paper": self.paper}
        with default_storage.open(self.get_html_path()) as fh:
            return process_render(fh, self.get_output_url(), context=context)

    def is_expired(self):
        """
        Returns True if this render was run more than PAPERS_EXPIRED_DAYS ago.
        """
        return self.created_at <= _get_expired_date()

    def mark_as_deleted(self):
        """
        Delete render's output and mark is as deleted.
        """
        try:
            self.delete_output()
        except FileNotFoundError:
            log_exception()
        self.is_deleted = True
        self.save()

    def delete_output(self):
        """
        Delete output path.
        """
        storage_delete_path(default_storage, self.get_output_path())

    def delete_older_renders_if_successful(self):
        """
        Delete any older undeleted renders if this render was successful.
        """
        if not self.state == Render.STATE_SUCCESS:
            return
        for render in self.paper.renders.not_deleted().filter(pk__lt=self.pk):
            render.mark_as_deleted()


class SourceFileBulkTarball(models.Model):
    """
    A tarball of sources that is listed in arXiv's bulk sources manifest.

    We keep track of these so we know which tarballs we have already
    downloaded. They're quite big so we don't want to get them every time.
    """

    filename = models.CharField(max_length=255, unique=True)

    content_md5sum = models.TextField()
    first_item = models.TextField()
    last_item = models.TextField()
    md5sum = models.TextField()
    num_items = models.IntegerField()
    seq_num = models.IntegerField()
    size = models.IntegerField()
    timestamp = models.TextField()
    yymm = models.TextField()

    def __str__(self):
        return self.filename

    def has_correct_number_of_files(self):
        """
        Number of items specified in the manifest matches how many we have
        in the database. If this is false, it suggests there was some error
        in downloading source files.
        """
        return self.num_items == self.sourcefile_set.count()


class SourceFileQuerySet(models.QuerySet):
    def get_or_download(self, arxiv_id):
        """
        Returns a source file for an arxiv ID if it exists (probably created
        from a bulk source download), otherwise downloads and creates it.
        """
        try:
            return self.get(arxiv_id=arxiv_id)
        except SourceFile.DoesNotExist:
            return self.download_and_create(arxiv_id)

    def download_and_create(self, arxiv_id):
        """
        Download the LaTeX source of this paper, save to storage, and create
        SourceFile.
        """
        file = download_source_file(arxiv_id)
        return self.create(arxiv_id=arxiv_id, file=file)

    def filename_exists(self, fn):
        return self.filter(file=f"source-files/{fn}").exists()


class SourceFile(models.Model):
    """
    Represents a paper source file from arXiv.

    NOTE: This is a new system, not yet used by the `Paper` model. Initially,
    we are downloading arXiv's bulk papers into this model, then we can switch
    `Paper` to use this model.
    """

    arxiv_id = models.CharField(max_length=50, unique=True)
    file = models.FileField(upload_to="source-files/", unique=True)
    bulk_tarball = models.ForeignKey(
        SourceFileBulkTarball,
        on_delete=models.SET_NULL,
        null=True,
        blank=True,
        help_text="If this source file is from arXiv's bulk download service, this is the tarball it was in. If null, this source file was downloaded individually.",
    )

    objects = SourceFileQuerySet.as_manager()

    def __str__(self):
        return str(self.file)

    def is_pdf(self):
        return self.file.name.endswith(".pdf")

    def is_renderable(self):
        """
        Returns whether it is possible to render this file.
        """
        name = self.file.name
        return (
            name is not None
            and name.endswith(".gz")
            and not name.endswith(".ps.gz")
            and not name.endswith(".dvi.gz")
        )
Paper cannot be rendered. Render started when render has already been started. The state of a render is being updated when it has not been started. Query the arXiv API and create a Paper from it.          Raises:             `arxiv_vanity.scraper.query.PaperNotFoundError`: If paper does not exist on arxiv. Return only machine learning papers. # ArXiV fields # arXiv Vanity fields Returns whether it is possible to render this paper. Attempts to get the source file from a bulk data download, otherwise         downloads it and creates it. Returns the render that should display for this paper, and kicks off         a new render if need be. # Stuck for some reason, give it a boot # This normally happens if there is an exception raised in render() # This will put it into running state # There is already a render running, so try and get the most recent one that isn't running # If this is the only render or the other renders errored, display loading render # Kick off render if this one has expired #  Don't block displaying render if kicking off failed # Try and display a successful render # Otherwise, display the failed or running render # Kick off render in background if it has expired #  Don't block displaying render if kicking off failed Make a new render of this paper. Will download the source file and save         itself if it hasn't already. Update the state of renders that have a container. Returns renders that ran more than PAPERS_EXPIRED_DAYS ago. Returns renders than ran less than PAPERS_EXPIRED_DAYS ago. Mark renders as deleted. Useful for forcing re-rendering. Path to the directory that this render is in. Path to the HTML file of this render. Returns the URL to the output path. Get the webhook to call when the Engrafo job ends to update render         state. Start running this render. Update state of this render from the container.          This is used for two purposes:          1. Called by the webhook to update the state. In this case, exit_code             is passed through instead of using the containers exit code             because the container is still running (it's sending the webhook!).          2. Called by the update_render_state cron job. In this case, it'll             sync all of the details about the containers that exist, and remove             any containers that have stopped. This sweeps up containers             that have reported their state in (1). # The create container timeed out, but container ran anyway # Clean up after it... # Container has been removed for some reason, so mark it as # removed so we don't try to update its state again # Give it a failed state if the render was still running # Safer to convert int to str than other way round # Deleting containers on Hyper.sh often fails, so save here so we at # least save the state. If remove fails, this will get retried by # the update_render_state cron job. # Force, or Hyper.sh often throws a 500 # Somebody got in there before us. Oh well. Do final processing on this render and returns it as a dictionary of         {"body", "script", "styles"}. Returns True if this render was run more than PAPERS_EXPIRED_DAYS ago. Delete render's output and mark is as deleted. Delete output path. Delete any older undeleted renders if this render was successful. A tarball of sources that is listed in arXiv's bulk sources manifest.      We keep track of these so we know which tarballs we have already     downloaded. They're quite big so we don't want to get them every time. Number of items specified in the manifest matches how many we have         in the database. If this is false, it suggests there was some error         in downloading source files. Returns a source file for an arxiv ID if it exists (probably created         from a bulk source download), otherwise downloads and creates it. Download the LaTeX source of this paper, save to storage, and create         SourceFile. Represents a paper source file from arXiv.      NOTE: This is a new system, not yet used by the `Paper` model. Initially,     we are downloading arXiv's bulk papers into this model, then we can switch     `Paper` to use this model. Returns whether it is possible to render this file.