Create source.py

source.py

@@ -0,0 +1,51 @@
+# source.py provides an abstract dataclass for a data source
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from collections import namedtuple
+from typing import List, Optional
+
+Summary = namedtuple('Summary',['source','cluster_list','link_ext','hed','dek','date','authors','original_length','summary_text','summary_length','chunk_time', 'query_time', 'mean_query_time', 'summary_time'])
+Summary.__doc__ = f"""
+                Summary: a namedtuple for storing Summaries and relevant metadata.
+                
+                • Source: A Source object for the source of the summarized document. 
+                • cluster_list: A list of the NER entities detected in this article's hed (headline).
+                • link_ext: The link extension of the article (on the base url, source's source_url)
+                • hed, dek: headline and subheader. These are standard industry terms. 
+                            Dek is None if not applicable.
+                • date: Date of publication/update listed in article.
+                • authors: list of authors, currently a string containing the byline.
+                • original_length: length of the original article
+                • cluster_num: Number of clusters the source article appears in
+                • summary_text: List of summarized chunks.
+                • summary_length: Length of summary text
+                • stats for stats
+                """
+
+@dataclass
+class Source(ABC):
+    source_name: Optional[str] = ""
+    source_url: Optional[str] = ""
+    # Checkpoint str encourages use of source-appropriate models.
+    source_summarization_checkpoint: Optional[str] = "" 
+    source_ner_checkpoint: Optional[str] = ""
+
+    """
+    User must implement a source-dependent method
+    to retrieve data used to create clusters.  
+
+    This gets called when clustering is performed.
+    """
+    @abstractmethod
+    def retrieve_cluster_data(self) -> List[namedtuple]:
+        pass
+
+    """
+    User must implement a source-dependent method 
+    to retrieve texts for summarization.
+
+    This gets called once topics for digestion have been selected.  
+    """
+    @abstractmethod
+    def retrieve_article(self) -> List[namedtuple]:
+        pass