Update app.py

app.py

@@ -1,29 +1,30 @@
+import requests
+from bs4 import BeautifulSoup
 import pandas as pd
 import folium
-from folium.plugins import MarkerCluster, HeatMap
+from folium.plugins import MarkerCluster
 import plotly.graph_objects as go
-import plotly.express as px
+import plotly.express as px  # Add this import
 from geopy.geocoders import Nominatim
-from geopy.exc import GeocoderInsufficientPrivileges
 import re
 import streamlit as st
-import time
 
 # Streamlit title and description
-st.title("米其林餐廳指南分析")
-st.write("分析餐廳數據，可視化區域分佈，並在地圖上顯示位置和餐廳數量熱力圖。")
+st.title("米其林餐廳指南爬蟲")
+st.write("Extract restaurant data, visualize with a pie chart and bar chart, and display locations on a map.")
 
 # Read data from Google Sheets
 sheet_id = "1xUfnD1WCF5ldqECI8YXIko1gCpaDDCwTztL17kjI42U"
-df = pd.read_csv(f"https://docs.google.com/spreadsheets/d/{sheet_id}/export?format=csv")
+df1 = pd.read_csv(f"https://docs.google.com/spreadsheets/d/{sheet_id}/export?format=csv")
 
-# Print column names and first few rows
-st.write("資料框的列名：", df.columns.tolist())
-st.write("資料預覽：")
-st.dataframe(df.head())
+# Convert "網址" column to a Python list
+urls = df1["網址"].tolist()
+
+# Create a DataFrame to store all restaurant data
+df = pd.DataFrame(columns=["Store Name", "Address", "Phone", "Latitude", "Longitude", "Region"])
 
 # Initialize Nominatim geocoder
-geolocator = Nominatim(user_agent="my_unique_app/3.0")
+geolocator = Nominatim(user_agent="my_app")
 
 # Function to extract region (區域) from the address using regex
 def extract_region(address):
@@ -33,61 +34,108 @@ def extract_region(address):
     else:
         return "Unknown"
 
-# Function to get latitude and longitude with caching
-@st.cache_data
-def get_lat_lon(district):
-    try:
-        location = geolocator.geocode(f"台南市{district}")
-        if location:
-            time.sleep(1)  # Delay to avoid rate limiting
-            return location.latitude, longitude
-    except GeocoderInsufficientPrivileges:
-        st.error("地理編碼器遇到權限問題，請稍後再試。")
-    return None, None
-
-# Assuming we have a column that represents the region or can be used to derive it
-# If we don't have such a column, we'll need to skip this part
-if '區域' in df.columns:
-    region_column = '區域'
-elif '地址' in df.columns:
-    df['區域'] = df['地址'].apply(extract_region)
-    region_column = '區域'
-else:
-    st.error("無法找到區域資訊，某些分析將無法進行。")
-    region_column = None
-
-# Group the data by region and count the number of restaurants
-if region_column:
-    region_group = df.groupby(region_column).size().reset_index(name='Count')
-
-    # Create hierarchical data for the Sunburst chart
-    region_group['Total'] = 'All Regions'  # Add a root level
-    hierarchical_data = region_group[['Total', region_column, 'Count']]
-
-    # Plot interactive Sunburst chart
-    sunburst = go.Figure(go.Sunburst(
-        labels=hierarchical_data[region_column].tolist() + hierarchical_data['Total'].tolist(),
-        parents=hierarchical_data['Total'].tolist() + [''],
-        values=hierarchical_data['Count'].tolist() + [hierarchical_data['Count'].sum()],
-        branchvalues="total",
-        hovertemplate='<b>%{label}</b><br>餐廳數量: %{value}<extra></extra>',
-        maxdepth=2,
+# Function to fetch and parse data
+def fetch_data():
+    global df
+    # Progress bar in Streamlit
+    progress_bar = st.progress(0)
+    total_urls = len(urls)
+
+    # Iterate through each URL
+    for idx, url in enumerate(urls):
+        response = requests.get(url)
+        soup = BeautifulSoup(response.content, "html.parser")
+
+        try:
+            store_name = soup.find("h2", class_="restaurant-details__heading--title").text.strip()
+        except AttributeError:
+            store_name = None
+
+        try:
+            address = soup.find("li", class_="restaurant-details__heading--address").text.strip()
+            region = extract_region(address)
+        except AttributeError:
+            address = None
+            region = "Unknown"
+        
+        # Try to extract phone number
+        try:
+            phone = soup.find("a", {"data-event": "CTA_tel"}).get("href").replace("tel:", "")
+        except AttributeError:
+            phone = None
+
+        try:
+            location = geolocator.geocode(address)
+            if location:
+                latitude = location.latitude
+                longitude = location.longitude
+            else:
+                latitude = None
+                longitude = None
+        except:
+            latitude = None
+            longitude = None
+
+        new_row = pd.DataFrame({
+            "Store Name": [store_name],
+            "Address": [address],
+            "Phone": [phone],
+            "Latitude": [latitude],
+            "Longitude": [longitude],
+            "Region": [region]
+        })
+
+        df = pd.concat([df, new_row], ignore_index=True)
+
+        # Update progress bar
+        progress_bar.progress((idx + 1) / total_urls)
+
+# Button to trigger data fetching
+if st.button("爬取餐廳資料"):
+    fetch_data()
+    
+    # Save the DataFrame to CSV with UTF-8 encoding, including latitude and longitude
+    csv_file = "restaurants_data.csv"
+    df.to_csv(csv_file, encoding="utf-8-sig", index=False)
+
+    # Display the DataFrame as a table at the top
+    st.subheader("Restaurant Data")
+    st.dataframe(df)
+
+    # Display download button for the CSV
+    st.download_button(
+        label="Download restaurant data as CSV",
+        data=open(csv_file, "rb").read(),
+        file_name=csv_file,
+        mime="text/csv"
+    )
+
+    # Group the data by region and sum the number of restaurants
+    region_group = df.groupby("Region").size().reset_index(name='Count')
+
+    # Plot enlarged pie chart with custom colors and labels
+    pie_chart = go.Figure(go.Pie(
+        labels=region_group["Region"],
+        values=region_group["Count"],
+        textinfo="label+percent",
+        hoverinfo="label+value",
+        textfont=dict(size=18),
+        marker=dict(colors=px.colors.qualitative.Set3, line=dict(color='#000000', width=2))
     ))
 
-    sunburst.update_layout(
-        title="餐廳分佈（點擊可放大查看）",
+    pie_chart.update_layout(
+        title="Restaurant Distribution by Region",
         title_x=0.5,
         title_font=dict(size=24, family="Arial"),
         height=600,
-        margin=dict(t=50, b=50, l=0, r=0)
+        margin=dict(t=50, b=50, l=50, r=50)
     )
-
-    st.subheader("餐廳分佈（Sunburst 圖）")
-    st.plotly_chart(sunburst, use_container_width=True)
+    st.subheader("Restaurant Distribution by Region (Enlarged Pie Chart)")
+    st.plotly_chart(pie_chart)
 
     # Plot bar chart with custom colors and labels
     bar_chart = go.Figure(go.Bar(
-        x=region_group[region_column],
+        x=region_group["Region"],
         y=region_group["Count"],
         text=region_group["Count"],
         textposition='auto',
@@ -95,61 +143,33 @@ if region_column:
     ))
 
     bar_chart.update_layout(
-        title="各區域餐廳數量",
+        title="Restaurant Count by Region",
         title_x=0.5,
         title_font=dict(size=24, family="Arial"),
         height=400,
         margin=dict(t=50, b=50, l=50, r=50),
-        xaxis_title="區域",
-        yaxis_title="餐廳數量",
+        xaxis_title="Region",
+        yaxis_title="Number of Restaurants",
         xaxis=dict(tickangle=-45)
     )
-    st.subheader("各區域餐廳數量（條形圖）")
+    st.subheader("Restaurant Count by Region (Bar Chart)")
     st.plotly_chart(bar_chart)
 
-# Display a map using Folium if we have latitude and longitude
-if '緯度' in df.columns and '經度' in df.columns:
-    st.subheader("餐廳位置地圖（含數量熱力圖）")
+    # Display a map using Folium
+    st.subheader("Restaurant Locations Map")
 
-    # Create map centered around the mean latitude and longitude
-    center_lat = df['緯度'].mean()
-    center_lon = df['經度'].mean()
-    m = folium.Map(location=[center_lat, center_lon], zoom_start=12)
+    # Create map centered around Tainan
+    m = folium.Map(location=[23.0, 120.2], zoom_start=12)
 
     # Add marker cluster to the map
     marker_cluster = MarkerCluster().add_to(m)
-
-    # Prepare data for heatmap
-    heat_data = []
-
     for index, row in df.iterrows():
-        if pd.notnull(row["緯度"]) and pd.notnull(row["經度"]):
+        if pd.notnull(row["Latitude"]) and pd.notnull(row["Longitude"]):
             folium.Marker(
-                location=[row["緯度"], row["經度"]],
-                popup=f"{row.get('店名', 'Unknown')}",
-                tooltip=row.get('地址', 'Unknown')
+                location=[row["Latitude"], row["Longitude"]],
+                popup=f"{row['Store Name']} ({row['Phone']})",
+                tooltip=row["Address"]
             ).add_to(marker_cluster)
-            heat_data.append([row["緯度"], row["經度"], 1])  # Weight of 1 for each restaurant
-
-    # Add heatmap layer
-    HeatMap(heat_data, radius=15, blur=10, max_zoom=1, name="餐廳數量熱力圖").add_to(m)
-
-    # Add layer control
-    folium.LayerControl().add_to(m)
 
     # Display the map in Streamlit
     st.components.v1.html(m._repr_html_(), height=600)
-else:
-    st.error("無法顯示地圖，因為缺少緯度和經度資訊。")
-
-# Save the DataFrame to CSV with UTF-8 encoding
-csv_file = "restaurants_data.csv"
-df.to_csv(csv_file, encoding="utf-8-sig", index=False)
-
-# Display download button for the CSV
-st.download_button(
-    label="下載餐廳數據 CSV 檔案",
-    data=open(csv_file, "rb").read(),
-    file_name=csv_file,
-    mime="text/csv"
-)