small refactor, simplifying app

app.py

@@ -27,27 +27,150 @@ def initialize():
     logging.basicConfig(level=logging.INFO)
     load_dotenv()
 
-    st.session_state.setdefault('model_names', ['SurveyBot3000'])
-    st.session_state.setdefault('input_model_name', st.session_state['model_names'][0])
-    st.session_state.setdefault('loaded_model_name', None)
-    st.session_state.setdefault('search_query', None)
+    st.session_state.setdefault('config', None)
+    
+    st.session_state.setdefault('encryption_key', None)
+    st.session_state.setdefault('is_authenticated', False)
+    
     st.session_state.setdefault('db', None)
+
+    st.session_state.setdefault('search_query', None)
     st.session_state.setdefault('search_results', pd.DataFrame())
-    st.session_state.setdefault('explore_plot', None)
-    st.session_state.setdefault('is_authenticated', False)
 
-    with open('config.yaml', 'r') as stream:
-        st.session_state['config'] = yaml.safe_load(stream)
+    if st.session_state['config'] is None:
+        with open('config.yaml', 'r') as stream:
+            st.session_state['config'] = yaml.safe_load(stream)
+
+def show_authentication():
+
+    with st.container(height=400, border=None, key=None):
+
+        with open('tos.md', 'r', encoding='utf-8') as f:
+            tos_content = f.read()
+        st.write(tos_content)
+
+    checkbox1 = "I agree to use this application **solely for non-commercial research purposes**. Any other usage is **strictly prohibited**!"
+    checkbox2 = "I have **read**, **understood**, and **agree** to be bound by the Terms of Service and Privacy Policy."
+
+
+    if st.checkbox(label=checkbox1) & st.checkbox(label=checkbox2):
+
+        with st.form("authentication_form", border=False):
+            st.markdown("""
+                ## Authentication
+                This app is a research preview and requires authentication.
+                All data is encrypted. Please use your 32-byte encryption key to proceed!
+            """)
+
+            st.text_input(
+                label="🔑 Encryption key",
+                value="",
+                max_chars=None,
+                key='encryption_key',
+                placeholder="A URL-safe base64-encoded 32-byte key"
+            )
+
+            submitted = st.form_submit_button(
+                label="Authenticate",
+                type="primary",
+                use_container_width=True
+            )
+
+            if submitted:
+                try:
+                    modeling.load_db()
+                    st.rerun()
+                except InvalidToken:
+                    error = f"Error: The encryption key you have entered is invalid!"
+                    st.error(body=error, icon="🔑")
+                    logging.error(error)
+                    st.session_state['is_authenticated'] = False
+                    return
+                except ValueError as error:
+                    st.error(body=error, icon="🔑")
+                    logging.error(error)
+                    st.session_state['is_authenticated'] = False
+                    return
+    # with placeholder:
+
+    #     with st.container():
+
+    #         with st.container(height=200, border=None, key=None):
+
+    #             with open('tos.md', 'r', encoding='utf-8') as f:
+    #                 tos_content = f.read()
+    #             st.write(tos_content)
+
+    #         checkbox1 = "I agree to use this application **solely for non-commercial research purposes**. Any other usage is **strictly prohibited**!"
+    #         checkbox2 = "I have **read**, **understood**, and **agree** to be bound by the Terms of Service and Privacy Policy."
+
+
+    #         if st.checkbox(label=checkbox1) & st.checkbox(label=checkbox2):
+    #             with st.form("authentication_form"):
+
+    #                 st.markdown("""
+    #                     ## Authentication
+    #                     This app is a research preview and requires authentication.
+    #                     All data is encrypted. Please use your 32-byte encryption key to proceed!
+    #                 """)
+
 
-    if os.environ.get('encryption_key'):
-        encryption_key = os.environ.get('encryption_key')
-        st.session_state.setdefault('encryption_key', encryption_key)    
-    else:
-        st.session_state.setdefault('encryption_key', None)
 
 def main():
-    st.set_page_config(page_title='Synth-Net')
 
+    with st.container():
+        st.divider()
+        st.markdown("""
+            ## Try it yourself!
+            Define a scale by entering individual items in YAML format.
+            After form submission, a vector representation for the scale is calculated using the selected encoder model.
+            Cosine similarities between this vector and the representations of existing scales are then computed.
+            The resulting table outputs measures with high semantic overlap.
+        """)
+
+        with st.container():
+            if 'input_items' not in st.session_state:
+                st.session_state['input_items'] = dict_to_yaml(st.session_state['config']['input_items'])
+
+            with st.form("submission_form"):
+                st.text_area(
+                    label="Search for similar measures by entering items that constitute the scale (YAML-Formatted):",
+                    height=175,
+                    key='input_items'
+                )
+
+
+                submitted = st.form_submit_button(
+                    label="Search Synth-Net",
+                    type="primary",
+                    use_container_width=True
+                )
+
+                if submitted:
+
+                    try:
+                        st.session_state['search_query'] = yaml_to_dict(st.session_state['input_items'])
+                    except yaml.YAMLError as e:
+                        st.error(f"Yikes, you better get your YAML straight! Check https://yaml.org/ for help! \n {e}")
+                        return
+
+                    if not st.session_state.get('model'):
+                        modeling.load_model()
+
+                    modeling.search()
+
+                with st.container():
+                    if not st.session_state['search_results'].empty:
+                        with st.spinner('Rendering search results...'):
+                            df = st.session_state['search_results'].style.format({
+                                'Match': '{:.2f}'.format,
+                                'Scale': str.capitalize,
+                                'Instrument': str.capitalize,
+                            })
+                            st.dataframe(df, use_container_width=True, hide_index=True)
+
+if __name__ == '__main__':
+    st.set_page_config(page_title='Synth-Net')
     st.markdown("# The Synthetic Nomological Net")
     st.markdown("""
         Psychological science is experiencing rapid growth in constructs and measures, partly due to refinement and new research areas,
@@ -58,141 +181,9 @@ def main():
         It analyzes textual data from over 21,000 scales (containing more than 330,000 items) in an effort to reduce redundancies in measures used in the behavioral sciences.
     """, unsafe_allow_html=True)
 
-    placeholder_authentication = st.empty()
-    placeholder_demo = st.empty()
+    initialize()
 
     if st.session_state['is_authenticated']:
-        show_demo(placeholder_demo)
+        main()
     else:
-        show_authentication(placeholder_authentication)
-
-def show_authentication(placeholder):
-    with placeholder:
-
-        with st.container():            
-
-            with st.container(height=200, border=None, key=None):
-
-                with open('tos.md', 'r', encoding='utf-8') as f:
-                    tos_content = f.read()
-                st.write(tos_content)
-
-            checkbox1 = "I agree to use this application **solely for non-commercial research purposes**. Any other usage is **strictly prohibited**!"
-            checkbox2 = "I have **read**, **understood**, and **agree** to be bound by the Terms of Service and Privacy Policy."
-
-
-            if st.checkbox(label=checkbox1) & st.checkbox(label=checkbox2):
-                with st.form("authentication_form"):
-
-                    st.markdown("""
-                        ## Authentication
-                        This app is a research preview and requires authentication. 
-                        All data is encrypted. Please use your 32-byte encryption key to proceed!
-                    """)
-
-                    st.text_input(
-                        label="🔑 Encryption key",
-                        value="",
-                        max_chars=None,
-                        key='encryption_key',
-                        placeholder="A URL-safe base64-encoded 32-byte key"
-                    )
-                    
-                    submitted = st.form_submit_button(
-                        label="Authenticate",
-                        type="primary",
-                        use_container_width=True
-                    )
-
-                    if submitted:
-                        try:
-                            modeling.load_db()
-                            st.rerun()
-                        except InvalidToken:
-                            error = f"Error: The encryption key you have entered is invalid (**{st.session_state['encryption_key']}**)!"
-                            st.error(body=error, icon="🔑")
-                            logging.error(error)
-                            st.session_state['is_authenticated'] = False
-                            return
-                        except ValueError as error:
-                            st.error(body=error, icon="🔑")
-                            logging.error(error)
-                            st.session_state['is_authenticated'] = False
-                            return
-
-def show_demo(placeholder):
-
-    with placeholder:
-        with st.container():
-            st.divider()
-            st.markdown("""
-                ## Try it yourself!
-                Define a scale by entering individual items in YAML format.
-                After form submission, a vector representation for the scale is calculated using the selected encoder model.
-                Cosine similarities between this vector and the representations of existing scales are then computed.
-                The resulting table outputs measures with high semantic overlap.
-            """)
-
-            if st.session_state['loaded_model_name'] is not None:
-                input_model_index = st.session_state['model_names'].index(st.session_state['input_model_name'])
-            else:
-                input_model_index = 0
-
-            st.selectbox(
-                label="Select model",
-                options=st.session_state['model_names'],
-                index=input_model_index,
-                placeholder="Choose a model",
-                key='input_model_name'
-            )
-
-            
-            with st.container():
-                if 'input_items' not in st.session_state:
-                    st.session_state['input_items'] = dict_to_yaml(st.session_state['config']['input_items'])
-
-                with st.form("submission_form"):
-                    st.text_area(
-                        label="Search for similar measures by entering items that constitute the scale (YAML-Formatted):",
-                        height=175,
-                        key='input_items'
-                    )
-
-                    
-                    submitted = st.form_submit_button(
-                        label="Search Synth-Net",
-                        type="primary",
-                        use_container_width=True
-                    )
-
-                    if submitted:
-
-                        try:
-                            st.session_state['search_query'] = yaml_to_dict(st.session_state['input_items'])
-                        except yaml.YAMLError as e:
-                            st.error(f"Yikes, you better get your YAML straight! Check https://yaml.org/ for help! \n {e}")
-                            return
-
-                        no_model = st.session_state.get('model') is None
-                        swap_model = st.session_state.get('input_model_name') != st.session_state['loaded_model_name']
-
-                        if swap_model or no_model:
-                            modeling.load_db()
-                            modeling.load_model()
-
-                        modeling.search()
-
-
-                    with st.container():
-                        if not st.session_state['search_results'].empty:
-                            with st.spinner('Rendering search results...'):
-                                df = st.session_state['search_results'].style.format({
-                                    'Match': '{:.2f}'.format,
-                                    'Scale': str.capitalize,
-                                    'Instrument': str.capitalize,
-                                })
-                                st.dataframe(df, use_container_width=True, hide_index=True)
-
-if __name__ == '__main__':
-    initialize()
-    main()
+        show_authentication()

modeling.py

@@ -15,17 +15,15 @@ from pdb import set_trace as trace
 def load_db():
 
     with st.spinner('Loading pre-computed embeddings...'):
-        if st.session_state['input_model_name']:
-            file_path = f"./{st.session_state['input_model_name'].lower()}.enc"
-        else:
-            file_path = f"./{st.session_state['model_names'][0].lower()}.enc"
 
+        file_path = "./surveybot3000.enc"
         logging.info(f"Loading data from {file_path}!")
 
         with open(file_path, 'rb') as f:
             encrypted_data = f.read()
 
         encryption_key = st.session_state['encryption_key']
+
         if isinstance(encryption_key, str):
             encryption_key = encryption_key.encode('utf-8')
 
@@ -40,15 +38,9 @@ def load_db():
 def load_model():
 
     with st.spinner('Loading the model...'):
-        env_local = st.session_state['input_model_name'].lower() + '_path'
-        env_remote = st.session_state['input_model_name'].lower(
-        ) + '_remote_path'
 
-        if os.environ.get(env_remote):
-            model_path = os.environ.get(env_remote)
-        else:
-            model_path = os.getenv(env_local)
 
+        model_path = os.environ.get('surveybot3000_path') or os.getenv('surveybot3000_path')
         logging.info(f"Loading model from {model_path}!")
 
         auth_token = os.environ.get('read_models') or True
@@ -58,9 +50,7 @@ def load_model():
             token=auth_token
         )
 
-        st.session_state['loaded_model_name'] = st.session_state['input_model_name']
-
-        logging.info(f"Loaded {st.session_state['input_model_name']}!")
+        logging.info(f"Loaded model!")
 
 
 def search():
@@ -81,40 +71,4 @@ def search():
             'Scale': st.session_state['db']['ScaleName'],
             'Instrument': st.session_state['db']['InstrumentName'],
             'Reference': st.session_state['db']['psyctest_doi'],
-        }).sort_values(by='Match', ascending=False)
-
-
-def explore():
-
-    df = st.session_state['db']
-    message = f'Modeling synthetic construct space for {df.shape[0]} scales...'
-    logging.info(message)
-
-    with st.spinner(message):
-
-        documents = [f'{x}\n{y}' for x, y in zip(
-            df.ScaleName.tolist(), df.InstrumentName.tolist())]
-        embeddings = np.stack(df.ItemStemEmbeddings.to_numpy())
-
-        topic_model = BERTopic().fit(
-            documents=documents,
-            embeddings=embeddings
-        )
-
-        reduced_embeddings = UMAP(
-            n_neighbors=10,
-            n_components=2,
-            min_dist=0.0,
-            metric='cosine'
-        ).fit_transform(embeddings)
-
-        st.session_state['explore_plot'] = topic_model.visualize_documents(
-            docs=documents,
-            reduced_embeddings=reduced_embeddings,
-            hide_annotations=True,
-            hide_document_hover=False,
-            custom_labels=False,
-            title="The Synthetic Nomological Net",
-            width=1500,
-            height=1500
-        )
+        }).sort_values(by='Match', ascending=False)

tos.md

@@ -1,18 +1,17 @@
-# Terms of Service
-## The Synthetic Nomological Net
+## Terms of Service
 
 **Effective Date:** June, 13h, 2025
 **Last Updated:** June, 13h, 2025
 
-## 1. Acceptance of Terms
+### 1. Acceptance of Terms
 
 By accessing or using The Synthetic Nomological Net web application ("Service"), you agree to be bound by these Terms of Service ("Terms"). If you do not agree to these Terms, please do not use the Service.
 
-## 2. Description of Service
+### 2. Description of Service
 
 The Synthetic Nomological Net is a research tool that uses natural language processing to analyze semantic overlap in psychological measures. The Service provides access to analyzed data from over 21,000 scales containing more than 330,000 items for academic and research purposes.
 
-## 3. Noncommercial Use Only
+### 3. Noncommercial Use Only
 
 **IMPORTANT RESTRICTION:** This Service and all data contained within it may only be used for noncommercial purposes. The underlying dataset has been provided by the American Psychological Association (APA) as a research preview with strict noncommercial use limitations.
 
@@ -21,7 +20,7 @@ You are expressly prohibited from:
 - Downloading, saving or storing any data associated with this application
 - Using any data or code to develop commercial applications or tools
 
-## 4. Permitted Uses
+### 4. Permitted Uses
 
 The Service may be used for:
 - Academic research and scholarly inquiry
@@ -30,11 +29,11 @@ The Service may be used for:
 - Peer-reviewed publication and scientific communication
 - Conference presentations and academic discussions
 
-## 5. Data Attribution and Source
+### 5. Data Attribution and Source
 
 The underlying psychological measurement data is provided by the American Psychological Association (APA) as a research preview. Users must acknowledge this data source in any research, publications, or presentations that utilize findings from this Service.
 
-## 6. User Responsibilities
+### 6. User Responsibilities
 
 You agree to:
 - Use the Service only for its intended research and academic purposes
@@ -44,7 +43,7 @@ You agree to:
 - Comply with all applicable laws and regulations
 - Use the Service in a manner that does not interfere with its normal operation
 
-## 7. Prohibited Activities
+### 7. Prohibited Activities
 
 You may not:
 - Violate the noncommercial use restriction
@@ -56,59 +55,60 @@ You may not:
 
 **Data Security Notice**: The underlying APA dataset is encrypted to protect the data integrity and comply with usage restrictions. Any attempts to decrypt, break, or circumvent this encryption constitute a clear and serious violation of these Terms and may result in immediate termination of access and potential legal action.
 
-## 8. Intellectual Property Rights
+### 8. Intellectual Property Rights
 
 The Service, its design, functionality, and presentation are owned by the Service provider. The underlying psychological measurement data remains the property of the American Psychological Association and/or that of the authors of the respective publication from which they were obtained. Users do not acquire any ownership rights in the Service or underlying data through use.
 
-## 9. Privacy and Data Collection
+### 9. Privacy and Data Collection
 
 Your use of the Service is also governed by our Privacy Policy, which is incorporated into these Terms by reference.
 
-## 10. Disclaimers
+### 10. Disclaimers
 
 THE SERVICE IS PROVIDED "AS IS" WITHOUT WARRANTIES OF ANY KIND. We disclaim all warranties, express or implied, including but not limited to warranties of merchantability, fitness for a particular purpose, and non-infringement.
 
 The Service is provided for research purposes. We make no representations about the accuracy, completeness, or reliability of any research findings or data analysis provided by the Service.
 
-## 11. Limitation of Liability
+### 11. Limitation of Liability
 
 To the maximum extent permitted by law, we shall not be liable for any indirect, incidental, special, consequential, or punitive damages, including but not limited to loss of profits, data, or other intangible losses resulting from your use of the Service.
 
-## 12. Termination
+### 12. Termination
 
 We reserve the right to terminate or suspend your access to the Service at any time, with or without notice, for any reason, including violation of these Terms or the noncommercial use restriction.
 
-## 13. Changes to Terms
+### 13. Changes to Terms
 
 We reserve the right to modify these Terms at any time. Changes will be effective immediately upon posting. Your continued use of the Service after changes constitutes acceptance of the new Terms.
 
-## 14. Governing Law
+### 14. Governing Law
 
-These Terms shall be governed by and construed in accordance with the laws of [Insert Jurisdiction], without regard to conflict of law principles.
+These Terms shall be governed by and construed in accordance with the laws of the European Union, without regard to conflict of law principles.
 
-## 15. Research Ethics
+### 15. Research Ethics
 
 Users are encouraged to follow appropriate research ethics guidelines when using the Service, including obtaining necessary institutional approvals for research involving human subjects data analysis.
 
-## 16. Contact Information
+### 16. Contact Information
 
 For questions about these Terms of Service, please contact:  
 - Björn E. Hommel, PhD; [email protected]
+- Ruben C. Lennartz (Arslan), PhD; [email protected]
 - Malte Elson, PhD; [email protected]
 
 ---
 
-# Privacy Policy
-## The Synthetic Nomological Net
+## Privacy Policy
+### The Synthetic Nomological Net
 
 **Effective Date:** June, 13h, 2025
 **Last Updated:** June, 13h, 2025
 
-## Data Collection by This Application
+### Data Collection by This Application
 
 **We do not collect, store, or process any personal data or user information through this application.** The Synthetic Nomological Net operates as a research tool that provides access to pre-analyzed psychological measurement data without requiring user registration, tracking, or data collection.
 
-## Third-Party Platform Data Collection
+### Third-Party Platform Data Collection
 
 **Important Notice:** This application is hosted on Hugging Face Spaces, a third-party platform operated by Hugging Face, Inc. **We are not responsible for any data collection, processing, or privacy practices of Hugging Face Spaces.** 
 
@@ -118,20 +118,21 @@ Hugging Face may collect certain information about your use of their platform, i
 - Cookies and similar tracking technologies
 - Any other data as described in Hugging Face's own privacy policy
 
-## Your Rights and Responsibilities
+### Your Rights and Responsibilities
 
 Since we do not collect your data, we cannot provide access, correction, or deletion of personal information. For any privacy concerns related to the hosting platform, you must contact Hugging Face directly and review their privacy policy at https://huggingface.co/privacy.
 
 Users are advised to review Hugging Face's privacy policy and terms of service to understand what data may be collected during your use of this application.
 
-## Data Processing Disclaimer
+### Data Processing Disclaimer
 
 Any research queries or interactions you perform within this application are processed locally and are not stored or transmitted by us. However, such interactions may be subject to Hugging Face's data processing practices over which we have no control.
 
-## Contact for Privacy Questions
+### Contact for Privacy Questions
 
 For questions specifically about this Privacy Policy, please contact:  
 - Björn E. Hommel, PhD; [email protected]
+- Ruben C. Lennartz (Arslan), PhD; [email protected]
 - Malte Elson, PhD; [email protected]
 
 For questions about data collection by the hosting platform, please contact Hugging Face directly through their official channels.