Initial Commit

Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.10
+
+WORKDIR /code
+
+COPY ./requirements.txt /code/requirements.txt
+
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+
+COPY . .
+
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

Excel/Noonfood_excel.py

@@ -0,0 +1,173 @@
+from fastapi import FastAPI, File, UploadFile,HTTPException
+from fastapi.responses import StreamingResponse
+import pandas as pd
+import numpy as np
+import requests
+import os
+import io 
+import zipfile
+from urllib.parse import urljoin
+from typing import List
+from pydantic import BaseModel
+from io import BytesIO
+
+# app = FastAPI()
+
+# Input model
+
+
+
+
+# Extract items with categories
+def extract_items_with_categories(menu):
+    items_list = []
+    categories_seen = set()  # Set to keep track of categories that have been added
+    for category in menu['categories']:
+        category_name = category['name']
+        category_position = category['position'] if category['position'] != -1 else 0
+        if category_name not in categories_seen:
+            items_list.append({
+                'category': category_name,
+                'position': category_position,
+                'Item': None,  # Placeholder for item
+                'itemCode': None,
+                'item_position': None,
+                'img_url': None,
+                'Description': None,
+                'price_dine': None
+                
+            })
+            categories_seen.add(category_name)
+        for item_code in category['items']:
+            item = next((item for item in menu['items'] if item['itemCode'] == item_code), None)
+            if item:
+                items_list.append({
+                    'category': '',  # Empty string for subsequent items in the same category
+                    'position': category['position'],
+                    'Item': item['name'],
+                    'itemCode': item.get("itemCode",""),
+                    'item_position': item['position'],
+                    'img_url': "https://f.nooncdn.com/food_production/"+item['image'],
+                    'Description': item['itemDesc'],
+                    'price_dine': item['price']
+                    
+                })
+    return items_list
+
+# Extract options
+def extract_options(menu):
+    options_dict = {}
+    for item in menu['items']:
+        if 'modifiers' in item:
+            for modifier_code in item['modifiers']:
+                modifier = next((modifier for modifier in menu['modifiers'] if modifier['modifierCode'] == modifier_code), None)
+                if modifier:
+                    if item['itemCode'] not in options_dict:
+                        options_dict[item['itemCode']] = {}
+                    if modifier['name'] not in options_dict[item['itemCode']]:
+                        options_dict[item['itemCode']][modifier['name']] = {
+                            'Min': modifier.get('minTotalOptions'),
+                            'Max': modifier.get('maxTotalOptions'),
+                            'Options': []
+                        }
+                    for option in modifier['options']:
+                        option_item = next((i for i in menu['items'] if i['itemCode'] == option['itemCode']), None)
+                        if option_item:
+                            options_dict[item['itemCode']][modifier['name']]['Options'].append({
+                                'Option name': option_item['name'],
+                                'Option price': option['price']
+                            })
+    return options_dict
+
+# Process data for a single URL
+def process_url(url, latitude, longitude):
+    outlet_code = url.split('/')[-2]
+
+    # Make the request
+    api_url = "https://food.noon.com/_svc/mp-food-api-mpnoon/consumer/restaurant/outlet/details/guest"
+    payload = {
+        "addressLat": latitude,
+        "addressLng": longitude,
+        "deliveryType": "default",
+        "outletCode": outlet_code
+    }
+    headers = {
+        'Connection': 'keep-alive',
+        "Accept": "application/json, text/plain, */*",
+        "Accept-Encoding": "gzip, deflate, br, zstd",
+        "Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8,gu;q=0.7",
+        "Cache-Control": "no-cache, max-age=0, must-revalidate, no-store",
+        "Content-Type": "application/json",
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
+        "Cookie": "bm_mi=791533C8E67CE8E7DA98E80ADED70F69~YAAQRK0cuOep9tGPAQAAUYKw3RcGDAVhD+mtWU8IH76wZL29zl4qqCjMwGv8sKtYlQWZNaFftSOvHFOvQU4+3CY2uHZyjjK6I3GeNdKEn+XHupISeNc0K16GOXLqcPOwu4sADTmxE7PYQvSQE7eimhqsBiJVRd96R8W0D2hl31FlY/4rl+NPZvM3iXjrn2GO50VMv+HhGfCnDMBwApBxgpMWFLfs0u6EYy44mg/FXbom5s5pa3cro8AS35nYHbdUbi61K9fnWRVaF8L/4z0xh7V1AEQETevb5fdGF8aB9m2UG29p2W6KSMb8DyFZLpG3vl5+IRECqZdFxaUMnykO8G/ynRHG~1; Domain=.noon.com; Path=/; Expires=Mon, 03 Jun 2024 12:41:22 GMT; Max-Age=7199; Secure"
+    }
+    response = requests.post(api_url, headers=headers, json=payload)
+    json_data = response.json()
+
+    # Extract items and options
+    items = extract_items_with_categories(json_data['data']['menu'])
+    options = extract_options(json_data['data']['menu'])
+
+    # Create a DataFrame for items
+    items_df = pd.DataFrame(items)
+
+    options_list = []
+    for item_code, option_groups in options.items():
+        for group_name, group_data in option_groups.items():
+            row = {
+                'itemCode': item_code,
+                'Option Group Name': group_name,
+                'Min': group_data.get('Min'),
+                'Max': group_data.get('Max')
+            }
+            for i, option in enumerate(group_data['Options']):
+                row[f'Option name {i+1}'] = option['Option name']
+                row[f'Option price {i+1}'] = option['Option price']
+            options_list.append(row)
+
+    # Create DataFrame for options
+    options_df = pd.DataFrame(options_list)
+
+    # Merge DataFrames on 'itemCode'
+    merged_df = items_df.merge(options_df, on='itemCode', how='left')
+
+    merged_df['category'] = merged_df['category'].replace('', np.nan).ffill()
+    merged_df['Item'] = merged_df['Item'].replace('', np.nan)
+    merged_df.iloc[:, :7] = merged_df.iloc[:, :7].mask(merged_df.iloc[:, :7].duplicated(), '')
+    merged_df = merged_df.dropna(how='all')
+    non_cat_pos_columns = merged_df.columns.difference(['category', 'position'])
+    mask = merged_df[non_cat_pos_columns].isna().all(axis=1) & merged_df[['category', 'position']].notna().all(axis=1)
+    merged_df = merged_df[~mask]
+
+    if 'Max' in merged_df.columns:
+            max_column_index = merged_df.columns.get_loc('Max')
+            for i in range(max_column_index + 1, len(merged_df.columns)):
+                merged_df.rename(columns={merged_df.columns[i]: ''}, inplace=True)
+    merged_df = merged_df.drop(columns=['itemCode'])
+
+
+    output = BytesIO()
+    with pd.ExcelWriter(output, engine='openpyxl') as writer:
+            merged_df.to_excel(writer, index=False, sheet_name='Menu')
+    output.seek(0)
+    
+    return output,f"{outlet_code}_menu.xlsx"
+# @app.post("/generate_and_download", response_class=StreamingResponse)
+# def generate_and_download(details: RestaurantDetailsRequest):
+#     files = []
+#     for url in details.urls:
+#         output_filename = process_url(url, details.latitude, details.longitude)
+#         files.append(output_filename)
+    
+#     zip_buffer = io.BytesIO()
+#     with zipfile.ZipFile(zip_buffer, 'w') as zip_file:
+#         for file in files:
+#             zip_file.write(file, os.path.basename(file))
+#     zip_buffer.seek(0)
+    
+#     return StreamingResponse(zip_buffer, media_type="application/x-zip-compressed", headers={"Content-Disposition": "attachment;filename=output_files.zip"})
+
+# if __name__ == "__main__":
+#     import uvicorn
+#     uvicorn.run(app, host="127.0.0.1", port=8000)
+

Excel/Noonfood_excel_final.py

@@ -0,0 +1,129 @@
+import requests
+import json
+import pandas as pd
+import numpy as np
+from fastapi import FastAPI, Query
+
+app=FastAPI()
+
+latitude=int(input("Enter latitude:"))
+longitude=int(input("Enter longitude:"))
+url1=input("Enter the restaurant url:")
+outlet_code = url1.split('/')[-2]
+
+def extract_items_with_categories(menu):
+    items_list = []
+    categories_seen = set()  # Set to keep track of categories that have been added
+    for category in menu['categories']:
+        category_name = category['name']
+        category_position = category['position'] if category['position'] != -1 else 0
+        if category_name not in categories_seen:
+            items_list.append({
+                'category': category_name,
+                'item': None,  # Placeholder for item
+                'itemCode': None,
+                'item-position': None,
+                'img-url': None,
+                'price': None,
+                'Description': None,
+                'position': category_position
+            })
+            categories_seen.add(category_name)
+        for item_code in category['items']:
+            item = next((item for item in menu['items'] if item['itemCode'] == item_code), None)
+            if item:
+                items_list.append({
+                    'category': '',  # Empty string for subsequent items in the same category
+                    'item': item['name'],
+                    'itemCode': item['itemCode'],
+                    'item-position': item['position'],
+                    'img-url': "https://f.nooncdn.com/food_production/"+item['image'],
+                    'price': item['price'],
+                    'Description': item['itemDesc'],
+                    'position': category['position']
+                })
+    return items_list
+
+def extract_options(menu):
+    options_dict = {}
+    for item in menu['items']:
+        if 'modifiers' in item:
+            for modifier_code in item['modifiers']:
+                modifier = next((modifier for modifier in menu['modifiers'] if modifier['modifierCode'] == modifier_code), None)
+                if modifier:
+                    if item['itemCode'] not in options_dict:
+                        options_dict[item['itemCode']] = {}
+                    if modifier['name'] not in options_dict[item['itemCode']]:
+                        options_dict[item['itemCode']][modifier['name']] = {
+                            'Min': modifier.get('minTotalOptions'),
+                            'Max': modifier.get('maxTotalOptions'),
+                            'Options': []
+                        }
+                    for option in modifier['options']:
+                        option_item = next((i for i in menu['items'] if i['itemCode'] == option['itemCode']), None)
+                        if option_item:
+                            options_dict[item['itemCode']][modifier['name']]['Options'].append({
+                                'Option name': option_item['name'],
+                                'Option price': option['price']
+                            })
+    return options_dict
+
+# Make the request
+url = "https://food.noon.com/_svc/mp-food-api-mpnoon/consumer/restaurant/outlet/details/guest"
+payload = {
+    "addressLat": latitude,
+    "addressLng": longitude,
+    "deliveryType": "default",
+    "outletCode": outlet_code
+}
+headers = {
+    'Connection': 'keep-alive',
+    "Accept": "application/json, text/plain, */*",
+    "Accept-Encoding": "gzip, deflate, br, zstd",
+    "Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8,gu;q=0.7",
+    "Cache-Control": "no-cache, max-age=0, must-revalidate, no-store",
+    "Content-Type": "application/json",
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
+    "Cookie": "bm_mi=791533C8E67CE8E7DA98E80ADED70F69~YAAQRK0cuOep9tGPAQAAUYKw3RcGDAVhD+mtWU8IH76wZL29zl4qqCjMwGv8sKtYlQWZNaFftSOvHFOvQU4+3CY2uHZyjjK6I3GeNdKEn+XHupISeNc0K16GOXLqcPOwu4sADTmxE7PYQvSQE7eimhqsBiJVRd96R8W0D2hl31FlY/4rl+NPZvM3iXjrn2GO50VMv+HhGfCnDMBwApBxgpMWFLfs0u6EYy44mg/FXbom5s5pa3cro8AS35nYHbdUbi61K9fnWRVaF8L/4z0xh7V1AEQETevb5fdGF8aB9m2UG29p2W6KSMb8DyFZLpG3vl5+IRECqZdFxaUMnykO8G/ynRHG~1; Domain=.noon.com; Path=/; Expires=Mon, 03 Jun 2024 12:41:22 GMT; Max-Age=7199; Secure"
+}
+response = requests.post(url, headers=headers, json=payload)
+json_data = response.json()
+
+
+# Extract items and options
+items = extract_items_with_categories(json_data['data']['menu'])
+options = extract_options(json_data['data']['menu'])
+
+# Create a DataFrame for items
+items_df = pd.DataFrame(items)
+
+options_list = []
+for item_code, option_groups in options.items():
+    for group_name, group_data in option_groups.items():
+        row = {
+            'itemCode': item_code,
+            'Option Group Name': group_name,
+            'Min': group_data.get('Min'),
+            'Max': group_data.get('Max')
+        }
+        for i, option in enumerate(group_data['Options']):
+            row[f'Option name {i+1}'] = option['Option name']
+            row[f'Option price {i+1}'] = option['Option price']
+        options_list.append(row)
+
+# Create DataFrame for options
+options_df = pd.DataFrame(options_list)
+
+# Merge DataFrames on 'itemCode'
+merged_df = items_df.merge(options_df, on='itemCode', how='left')
+
+merged_df['category'] = merged_df['category'].replace('', np.nan).ffill()
+merged_df['item'] = merged_df['item'].replace('', np.nan)
+#merged_df.iloc[:, :7] = merged_df.groupby('category').apply(lambda x: x.ffill().mask(x.duplicated(), '')).reset_index(level=0, drop=True)
+
+#merged_df['category'] = merged_df['category'].replace('', pd.NA).ffill()
+merged_df.iloc[:, :7] = merged_df.iloc[:, :7].mask(merged_df.iloc[:, :7].duplicated(), '')
+merged_df = merged_df.dropna(subset=['item', 'itemCode', 'item-position', 'img-url', 'price', 'Description'], how='all')
+
+
+merged_df.to_excel("output3.xlsx", index=False)

Excel/excel.py

@@ -0,0 +1,112 @@
+import json
+import requests
+from bs4 import BeautifulSoup
+import pandas as pd
+
+def Excel_final(url):
+    def fetch_restaurant_data(url):
+        headers = {
+            'Cookie': '__cf_bm=_AOZtAiObnqBHPy4zhGRgBLW9xg9WiaDCRzg5E0sbMk-1715757967-1.0.1.1-xZNMBsnAqy_tfjUveujgfzT4Usw5ur4u7L0JlCcNXAQIC6Cq6wj46vPH7RLTh0Gq90JENxl7kbzjyOUFaBr8yCkmRGmt7APITEk0kkXzLTs; roo_guid=c40617a7-76f7-432c-b780-f2653cd2edfe; roo_session_guid=5846d6f0-5b7f-4598-8c6d-82b8023fd4fc'
+        }
+        response = requests.get(url, headers=headers)
+        if response.status_code != 200:
+            print(f"Failed to fetch the URL: {url}")
+            return None
+        soup = BeautifulSoup(response.content, 'html.parser')
+        script_tag = soup.find('script', id='__NEXT_DATA__')
+        if not script_tag:
+            print("Script tag not found")
+            return None
+        json_data = json.loads(script_tag.string)
+        json_data = json_data['props']['initialState']['menuPage']['menu']['meta']
+        items = json_data['items']
+        categories = json_data['categories']
+        category_map = {category['id']: category['name'] for category in categories}
+        modifier_groups = json_data['modifierGroups']
+        modifier_groups_dict = {modifier_group['id']: modifier_group for modifier_group in modifier_groups}
+        items_with_modifiers = []
+        current_category = None
+        current_category_position = 0
+        for item in items:
+            category_id = item['categoryId']
+            category_name = category_map.get(category_id, 'Unknown')
+            if category_name == "Unknown":
+                continue
+            if category_name != current_category:
+                current_category = category_name
+                current_category_position += 1
+                item_position = 1
+            else:
+                item_position += 1
+            item_with_modifiers = {
+                "id": item['id'],
+                "category_id": category_id,
+                "category_name": category_name,
+                "category_position": current_category_position,
+                "item_position": item_position,
+                "name": item['name'],
+                "description": item.get('description', ''),
+                "price": item['price']['formatted'],
+                "img_url": item.get('image').get('url', '') if item.get('image') else '',
+                "modifier_groups": [modifier_groups_dict.get(modifier_group_id, {}) for modifier_group_id in item.get('modifierGroupIds', [])],
+            }
+            items_with_modifiers.append(item_with_modifiers)
+        return items_with_modifiers
+    def save_data_to_excel(data):
+        writer = pd.ExcelWriter("restaurant_data.xlsx", engine='xlsxwriter')
+        rows = []
+        max_options = 0
+        # Find the maximum number of options for any modifier group
+        for item in data:
+            for modifier_group in item['modifier_groups']:
+                num_options = len(modifier_group.get('modifierOptions', []))
+                if num_options > max_options:
+                    max_options = num_options
+        for item in data:
+            base_row = [
+                item['category_name'],
+                item['category_position'],
+                item['item_position'],
+                item['name'],
+                item['description'],
+                item['price'],
+                item['img_url'],
+            ]
+            first_modifier_group = True
+            for modifier_group in item['modifier_groups']:
+                modifier_group_row = base_row + [
+                    modifier_group.get('name', ''),
+                    modifier_group.get('minSelection', ''),
+                    modifier_group.get('maxSelection', '')
+                ]
+                options = modifier_group.get('modifierOptions', [])
+                for option in options:
+                    modifier_group_row += [
+                        option.get('name', ''),
+                        option['price']['formatted'] if option.get('price') else ''
+                    ]
+                # Fill in the remaining columns with empty strings if there are fewer options than max_options
+                modifier_group_row += [''] * (max_options * 2 - len(options) * 2)
+                if first_modifier_group:
+                    rows.append(modifier_group_row)
+                    first_modifier_group = False
+                else:
+                    rows.append([''] * len(base_row) + modifier_group_row[len(base_row):])
+            if not item['modifier_groups']:
+                rows.append(base_row + [''] * (max_options * 2 + 3))
+        # Create column headers
+        columns = [
+            'Category Name', 'Category Position', 'Item Position', 'Item Name', 'Description', 'Item Price', 'Image URL', 'Modifier Group Name', 'Min Selection', 'Max Selection'
+        ]
+        for i in range(1, max_options + 1):
+            columns += [f'Option {i} Name', f'Option {i} Price']
+        df = pd.DataFrame(rows, columns=columns)
+        if 'Max Selection' in df.columns:
+            max_column_index = df.columns.get_loc('Max_quantity')
+            for i in range(max_column_index + 1, len(df.columns)):
+                df.rename(columns={df.columns[i]: ''}, inplace=True)
+        df.to_excel(writer, sheet_name='Sheet1', index=False)
+        writer.close()
+        print("Data saved to restaurant_data.xlsx")
+    data = fetch_restaurant_data(url)
+    save_data_to_excel(data)

Excel/excel_final.py

@@ -0,0 +1,119 @@
+import json
+import requests
+from bs4 import BeautifulSoup
+import pandas as pd
+from io import BytesIO
+def Excel_final(url):
+        def fetch_restaurant_data(url):
+            headers = {
+                'Cookie': '__cf_bm=_AOZtAiObnqBHPy4zhGRgBLW9xg9WiaDCRzg5E0sbMk-1715757967-1.0.1.1-xZNMBsnAqy_tfjUveujgfzT4Usw5ur4u7L0JlCcNXAQIC6Cq6wj46vPH7RLTh0Gq90JENxl7kbzjyOUFaBr8yCkmRGmt7APITEk0kkXzLTs; roo_guid=c40617a7-76f7-432c-b780-f2653cd2edfe; roo_session_guid=5846d6f0-5b7f-4598-8c6d-82b8023fd4fc'
+            }
+            response = requests.get(url, headers=headers)
+            if response.status_code != 200:
+                print(f"Failed to fetch the URL: {url}")
+                return None
+            soup = BeautifulSoup(response.content, 'html.parser')
+            script_tag = soup.find('script', id='__NEXT_DATA__')
+            if not script_tag:
+                print("Script tag not found")
+                return None
+            json_data = json.loads(script_tag.string)
+            json_data = json_data['props']['initialState']['menuPage']['menu']['meta']
+            items = json_data['items']
+            categories = json_data['categories']
+            category_map = {category['id']: category['name'] for category in categories}
+            modifier_groups = json_data['modifierGroups']
+            modifier_groups_dict = {modifier_group['id']: modifier_group for modifier_group in modifier_groups}
+            items_with_modifiers = []
+            current_category = None
+            current_category_position = 0
+            for item in items:
+                category_id = item['categoryId']
+                category_name = category_map.get(category_id, 'Unknown')
+                if category_name == "Unknown":
+                    continue
+                if category_name != current_category:
+                    current_category = category_name
+                    current_category_position += 1
+                    item_position = 1
+                else:
+                    item_position += 1
+                item_with_modifiers = {
+                    "id": item['id'],
+                    "category_id": category_id,
+                    "category_name": category_name,
+                    "category_position": current_category_position,
+                    "item_position": item_position,
+                    "name": item['name'],
+                    "description": item.get('description', ''),
+                    "price": item['price']['formatted'],
+                    "img_url": item.get('image').get('url', '') if item.get('image') else '',
+                    "modifier_groups": [modifier_groups_dict.get(modifier_group_id, {}) for modifier_group_id in item.get('modifierGroupIds', [])],
+                }
+                items_with_modifiers.append(item_with_modifiers)
+            return items_with_modifiers
+
+        def save_data_to_excel(data):
+            rows = []
+            max_options = 0
+            # Find the maximum number of options for any modifier group
+            for item in data:
+                for modifier_group in item['modifier_groups']:
+                    num_options = len(modifier_group.get('modifierOptions', []))
+                    if num_options > max_options:
+                        max_options = num_options
+            for item in data:
+                base_row = [
+                    item['category_name'],
+                    item['category_position'],
+                    item['item_position'],
+                    item['name'],
+                    item['description'],
+                    item['price'],
+                    item['img_url'],
+                ]
+                first_modifier_group = True
+                for modifier_group in item['modifier_groups']:
+                    modifier_group_row = base_row + [
+                        modifier_group.get('name', ''),
+                        modifier_group.get('minSelection', ''),
+                        modifier_group.get('maxSelection', '')
+                    ]
+                    options = modifier_group.get('modifierOptions', [])
+                    for option in options:
+                        modifier_group_row += [
+                            option.get('name', ''),
+                            option['price']['formatted'] if option.get('price') else ''
+                        ]
+                    # Fill in the remaining columns with empty strings if there are fewer options than max_options
+                    modifier_group_row += [''] * (max_options * 2 - len(options) * 2)
+                    if first_modifier_group:
+                        rows.append(modifier_group_row)
+                        first_modifier_group = False
+                    else:
+                        rows.append([''] * len(base_row) + modifier_group_row[len(base_row):])
+                if not item['modifier_groups']:
+                    rows.append(base_row + [''] * (max_options * 2 + 3))
+            # Create column headers
+            columns = [
+                'Category Name', 'Category Position', 'Item Position', 'Item Name', 'Description', 'Item Price', 'Image URL', 'Modifier Group Name', 'Min Selection', 'Max Selection'
+            ]
+            for i in range(1, max_options + 1):
+                columns += [f'Option {i} Name', f'Option {i} Price']
+            df = pd.DataFrame(rows, columns=columns)
+            if 'Max Selection' in df.columns:
+                max_column_index = df.columns.get_loc('Max Selection')
+                for i in range(max_column_index + 1, len(df.columns)):
+                    df.rename(columns={df.columns[i]: ''}, inplace=True)
+            return df
+        outlet_code = "Deliveroo_restro"
+        data = fetch_restaurant_data(url)
+        df = save_data_to_excel(data)
+        
+        output = BytesIO()
+        with pd.ExcelWriter(output, engine='openpyxl') as writer:
+                df.to_excel(writer, index=False, sheet_name='Menu')
+        output.seek(0)
+        
+        return output,f"{outlet_code}_menu.xlsx"
+

Excel/links_excel.py

@@ -0,0 +1,152 @@
+import json
+import requests
+from bs4 import BeautifulSoup
+import pandas as pd
+
+def fetch_restaurant_links(city, location):
+    base_url = "https://deliveroo.ae"
+    url = f"{base_url}/restaurants/{city}/{location}/?collection=restaurants"
+
+    headers = {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+        'Cookie': '__cf_bm=oakl46sJ3V9vwmnIIbfXWfkHbGmmC2pH56GyTI33b4U-1715931048-1.0.1.1-4XOcSGSThZV_INfpn3aptlo8jpZtLFbYoLsZxP9BpQ8LIjq3wBIe8CPlSf0AomuniXy4TZWyVlBQBTlrm.CPiSfI1jzx18y9zxwc9GX0fmo; roo_guid=c40617a7-76f7-432c-b780-f2653cd2edfe; roo_session_guid=2e989653-2776-4ede-a52e-b610f1ad64a2'
+    }
+
+    response = requests.get(url, headers=headers)
+    if response.status_code == 200:
+        soup = BeautifulSoup(response.content, 'html.parser')
+
+        if "We couldn't find" in soup.text or "No restaurants" in soup.text:
+            print("No restaurants found for the specified location.")
+            return []
+
+        divs = soup.find_all('div', class_=["HomeFeedScrollTracker-bd9a6ffea8a4b4b7", "HomeFeedUICard-157f7be5d7b2fa7b"])
+
+        hrefs = [a_tag['href'] for div in divs for a_tag in div.find_all('a', href=True)]
+        hrefs = hrefs[:20]
+        return [f"{base_url}{href}" for href in hrefs]
+    else:
+        print("Response timed out.")
+        return []
+
+def Excel_final(urls):
+    def fetch_restaurant_data(url):
+        headers = {
+            'Cookie': '__cf_bm=_AOZtAiObnqBHPy4zhGRgBLW9xg9WiaDCRzg5E0sbMk-1715757967-1.0.1.1-xZNMBsnAqy_tfjUveujgfzT4Usw5ur4u7L0JlCcNXAQIC6Cq6wj46vPH7RLTh0Gq90JENxl7kbzjyOUFaBr8yCkmRGmt7APITEk0kkXzLTs; roo_guid=c40617a7-76f7-432c-b780-f2653cd2edfe; roo_session_guid=5846d6f0-5b7f-4598-8c6d-82b8023fd4fc'
+        }
+        response = requests.get(url, headers=headers)
+        if response.status_code != 200:
+            print(f"Failed to fetch the URL: {url}")
+            return None
+        soup = BeautifulSoup(response.content, 'html.parser')
+        script_tag = soup.find('script', id='__NEXT_DATA__')
+        if not script_tag:
+            print("Script tag not found")
+            return None
+        json_data = json.loads(script_tag.string)
+        json_data = json_data['props']['initialState']['menuPage']['menu']['meta']
+        items = json_data['items']
+        categories = json_data['categories']
+        category_map = {category['id']: category['name'] for category in categories}
+        modifier_groups = json_data['modifierGroups']
+        modifier_groups_dict = {modifier_group['id']: modifier_group for modifier_group in modifier_groups}
+        items_with_modifiers = []
+        current_category = None
+        current_category_position = 0
+        for item in items:
+            category_id = item['categoryId']
+            category_name = category_map.get(category_id, 'Unknown')
+            if category_name == "Unknown":
+                continue
+            if category_name != current_category:
+                current_category = category_name
+                current_category_position += 1
+                item_position = 1
+            else:
+                item_position += 1
+            item_with_modifiers = {
+                "id": item['id'],
+                "category_id": category_id,
+                "category_name": category_name,
+                "category_position": current_category_position,
+                "item_position": item_position,
+                "name": item['name'],
+                "description": item.get('description', ''),
+                "price": item['price']['formatted'],
+                "img_url": item.get('image').get('url', '') if item.get('image') else '',
+                "modifier_groups": [modifier_groups_dict.get(modifier_group_id, {}) for modifier_group_id in item.get('modifierGroupIds', [])],
+            }
+            items_with_modifiers.append(item_with_modifiers)
+        return items_with_modifiers
+
+    def save_data_to_excel(data, sheet_name, writer):
+        rows = []
+        max_options = 0
+        # Find the maximum number of options for any modifier group
+        for item in data:
+            for modifier_group in item['modifier_groups']:
+                num_options = len(modifier_group.get('modifierOptions', []))
+                if num_options > max_options:
+                    max_options = num_options
+        for item in data:
+            base_row = [
+                item['category_name'],
+                item['category_position'],
+                item['item_position'],
+                item['name'],
+                item['description'],
+                item['price'],
+                item['img_url'],
+            ]
+            first_modifier_group = True
+            for modifier_group in item['modifier_groups']:
+                modifier_group_row = base_row + [
+                    modifier_group.get('name', ''),
+                    modifier_group.get('minSelection', ''),
+                    modifier_group.get('maxSelection', '')
+                ]
+                options = modifier_group.get('modifierOptions', [])
+                for option in options:
+                    modifier_group_row += [
+                        option.get('name', ''),
+                        option['price']['formatted'] if option.get('price') else ''
+                    ]
+                # Fill in the remaining columns with empty strings if there are fewer options than max_options
+                modifier_group_row += [''] * (max_options * 2 - len(options) * 2)
+                if first_modifier_group:
+                    rows.append(modifier_group_row)
+                    first_modifier_group = False
+                else:
+                    rows.append([''] * len(base_row) + modifier_group_row[len(base_row):])
+            if not item['modifier_groups']:
+                rows.append(base_row + [''] * (max_options * 2 + 3))
+        # Create column headers
+        columns = [
+            'Category Name', 'Category Position', 'Item Position', 'Item Name', 'Description', 'Item Price', 'Image URL', 'Modifier Group Name', 'Min Selection', 'Max Selection'
+        ]
+        for i in range(1, max_options + 1):
+            columns += [f'Option {i} Name', f'Option {i} Price']
+        df = pd.DataFrame(rows, columns=columns)
+        if 'Max Selection' in df.columns:
+            max_column_index = df.columns.get_loc('Max Selection')
+            for i in range(max_column_index + 1, len(df.columns)):
+                df.rename(columns={df.columns[i]: ''}, inplace=True)
+        df.to_excel(writer, sheet_name=sheet_name, index=False)
+
+    with pd.ExcelWriter("restaurant_data.xlsx", engine='xlsxwriter') as writer:
+        for idx, url in enumerate(urls):
+            data = fetch_restaurant_data(url)
+            if data:
+                save_data_to_excel(data, f'Sheet{idx+1}', writer)
+    print("Data saved to restaurant_data.xlsx")
+
+if __name__ == "__main__":
+    city = input("Enter the city: ")
+    location = input("Enter the location: ")
+
+    urls = fetch_restaurant_links(city, location)
+    
+    if urls:
+        Excel_final(urls)
+    else:
+        print("No restaurant links found or unable to fetch data.")

Excel/streamlit_excel.py

@@ -0,0 +1,301 @@
+import pandas as pd
+import requests
+import json
+from bs4 import BeautifulSoup
+import streamlit as st
+import mongoengine as me
+import certifi
+
+# Streamlit app setup
+st.title("Restaurant Menu Data Extraction")
+url1 = st.text_input("Please enter the restaurant URL:")
+
+if url1:
+    # Extract outlet code from URL
+    outlet_code = url1.split('/')[-2]
+    st.write(f"Extracted outlet code: {outlet_code}")
+
+    # API request setup
+    url = "https://food.noon.com/_svc/mp-food-api-mpnoon/consumer/restaurant/outlet/details/guest"
+    payload = {
+        "addressLat": 244538840,
+        "addressLng": 543773438,
+        "deliveryType": "default",
+        "outletCode": outlet_code
+    }
+    headers = {
+        'Connection': 'keep-alive',
+        "Accept": "application/json, text/plain, */*",
+        "Accept-Encoding": "gzip, deflate, br, zstd",
+        "Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8,gu;q=0.7",
+        "Cache-Control": "no-cache, max-age=0, must-revalidate, no-store",
+        "Content-Type": "application/json",
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
+        "Cookie":"bm_mi=791533C8E67CE8E7DA98E80ADED70F69~YAAQRK0cuOep9tGPAQAAUYKw3RcGDAVhD+mtWU8IH76wZL29zl4qqCjMwGv8sKtYlQWZNaFftSOvHFOvQU4+3CY2uHZyjjK6I3GeNdKEn+XHupISeNc0K16GOXLqcPOwu4sADTmxE7PYQvSQE7eimhqsBiJVRd96R8W0D2hl31FlY/4rl+NPZvM3iXjrn2GO50VMv+HhGfCnDMBwApBxgpMWFLfs0u6EYy44mg/FXbom5s5pa3cro8AS35nYHbdUbi61K9fnWRVaF8L/4z0xh7V1AEQETevb5fdGF8aB9m2UG29p2W6KSMb8DyFZLpG3vl5+IRECqZdFxaUMnykO8G/ynRHG~1; Domain=.noon.com; Path=/; Expires=Mon, 03 Jun 2024 12:41:22 GMT; Max-Age=7199; Secure"
+    }
+
+    response = requests.post(url, headers=headers, json=payload)
+
+    if response.status_code == 200:
+        json_data = response.json()
+        with open(f'{outlet_code}.json', 'w') as json_file:
+            json.dump(json_data, json_file, indent=4)
+        st.success("JSON data has been written to json file.")
+    else:
+        st.error(f"Failed to retrieve content from the URL: {response.status_code}")
+
+    # Extract items and options
+    def extract_items_with_categories(menu):
+        items_list = []
+        for category in menu['categories']:
+            category_name = category['name']
+            category_position = category['position']
+            for item_code in category['items']:
+                item = next((item for item in menu['items'] if item['itemCode'] == item_code), None)
+                if item:
+                    items_list.append({
+                        'category': category_name,
+                        'item': item['name'],
+                        'itemCode': item['itemCode'],
+                        'item-position': item['position'],
+                        'img-url': item['image'],
+                        'price': item['price'],
+                        'Description': item['itemDesc'],
+                        'position': category_position,
+                    })
+        return items_list
+
+    def extract_options(menu):
+        options_list = []
+        for item in menu['items']:
+            if 'modifiers' in item:
+                for modifier_code in item['modifiers']:
+                    modifier = next((modifier for modifier in menu['modifiers'] if modifier['modifierCode'] == modifier_code), None)
+                    if modifier:
+                        for option in modifier['options']:
+                            option_item = next((item for item in menu['items'] if item['itemCode'] == option['itemCode']), None)
+                            if option_item:
+                                options_list.append({
+                                    'itemCode': item['itemCode'],
+                                    'itemName': item['name'],
+                                    'Option Group Name': modifier['name'],
+                                    'Min': modifier.get('minTotalOptions'),
+                                    'Max': modifier.get('maxTotalOptions'),
+                                    'Option name': option_item['name'],
+                                    'Option price': option['price']
+                                })
+        return options_list
+
+    items_list = extract_items_with_categories(json_data['data']['menu'])
+    options_list = extract_options(json_data['data']['menu'])
+
+    def join_with_newline(x):
+        return '\n'.join(str(value) for value in x)
+
+    # Creating DataFrames with 'itemCode' column
+    df_items = pd.DataFrame(items_list)
+    df_options = pd.DataFrame(options_list)
+
+    # Merge the DataFrames based on 'itemCode'
+    final_df = pd.merge(df_items, df_options, on='itemCode', how='left')
+
+    final_df = final_df.groupby(['item', 'category'], as_index=False).agg({
+        'Option Group Name': join_with_newline,
+        'Option name': join_with_newline,
+        'Option price': join_with_newline,
+        'Min': join_with_newline,
+        'Max': join_with_newline,
+        'item-position': 'first',
+        'img-url': 'first',
+        'price': 'first',
+        'Description': 'first',
+        'item-position': 'first',
+        'position': 'first'
+    })
+
+    final_df = final_df.drop_duplicates().reset_index(drop=True)
+
+    def split_rows(df):
+        rows = []
+        for idx, row in df.iterrows():
+            min_values = row['Min'].split('\n')
+            max_values = row['Max'].split('\n')
+            option_groups = row['Option Group Name'].split('\n')
+            option_names = row['Option name'].split('\n')
+            option_prices = row['Option price'].split('\n')
+
+            for i in range(len(option_groups)):
+                current_row = {
+                    'category': row['category'],
+                    'item': row['item'],
+                    'item-position': row['item-position'],
+                    'img-url': row['img-url'],
+                    'price': row['price'],
+                    'Description': row['Description'],
+                    'position': row['position'],
+                    'Min': min_values[i] if i < len(min_values) else '',
+                    'Max': max_values[i] if i < len(max_values) else '',
+                    'Option Group Name': option_groups[i] if i < len(option_groups) else '',
+                    'Option name': option_names[i] if i < len(option_names) else '',
+                    'Option price': option_prices[i] if i < len(option_prices) else ''
+                }
+                rows.append(current_row)
+        return pd.DataFrame(rows)
+
+    split_df = split_rows(final_df)
+    split_df.fillna('', inplace=True)
+    split_df = split_df.replace('nan', '')
+    split_df.to_excel(f'{outlet_code}_1.xlsx', index=False)
+
+    # Final processing to JSON
+    df = pd.read_excel(f'{outlet_code}_1.xlsx')
+    json_data = df.to_json(orient='records')
+    with open(f'{outlet_code}_1.json', 'w') as f:
+        f.write(json_data)
+
+    # Process JSON data for final output
+    with open(f'{outlet_code}_1.json', 'r') as file:
+        data = json.load(file)
+
+    def process_item(item):
+        common_fields = {
+            'category': item['category'],
+            'category-position': item['position'],
+            'item': item['item'],
+            'item-position': item['item-position'],
+            'Description': item['Description'],
+            'img-url': item['img-url'],
+            'price': item['price']
+        }
+
+        options = []
+        for i in range(1, 38):
+            option_group_name = f"Option Group {i} Name"
+            option_name_key = f"Option {i} Name"
+            option_price_key = f"Option {i} Price"
+            min_values_key = f"Min{i}"
+            max_values_key = f"Max{i}"
+
+            if option_group_name in item and item[option_group_name]:
+                options.append({
+                    'Option Group Name': item[option_group_name],
+                    'Option Name': item[option_name_key],
+                    'Option Price': item[option_price_key],
+                    'Min': item[min_values_key],
+                    'Max': item[max_values_key]
+                })
+
+        return {**common_fields, 'Options': options}
+
+    processed_items = [process_item(item) for item in data]
+    with open(f'processed_items_{outlet_code}.json', 'w') as outfile:
+        json.dump(processed_items, outfile, indent=4)
+
+    # Extract options
+    def extract_options(item):
+        option_data = {}
+        for option in item["Options"]:
+            option_group_name = option["Option Group Name"]
+            if option_group_name not in option_data:
+                option_data[option_group_name] = {
+                    "Names": [],
+                    "Prices": [],
+                    "Mins": [],
+                    "Maxs": [],
+                    "GroupMin": option["Min"],
+                    "GroupMax": option["Max"]
+                }
+            option_data[option_group_name]["Names"].append(option["Option Name"])
+            option_data[option_group_name]["Prices"].append(option["Option Price"])
+            option_data[option_group_name]["Mins"].append(option["Min"])
+            option_data[option_group_name]["Maxs"].append(option["Max"])
+        return option_data
+
+    with open(f'processed_items_{outlet_code}.json', 'r') as file:
+        data = json.load(file)
+
+    all_rows_data = []
+    for item in data:
+        item_info = {
+            'category': item['category'],
+            'category-position': item['category-position'],
+            'item': item['item'],
+            'item-position': item['item-position'],
+            'Description': item['Description'],
+            'img-url': item['img-url'],
+            'price': item['price']
+        }
+        options = extract_options(item)
+        max_options = 0
+        for group in options.values():
+            max_options = max(max_options, len(group["Names"]))
+
+        for i in range(max_options):
+            row = item_info.copy()
+            for group_name, group_data in options.items():
+                row[f"Option Group {group_name}"] = group_name
+                row[f"Option {group_name} Name"] = group_data["Names"][i] if i < len(group_data["Names"]) else ""
+                row[f"Option {group_name} Price"] = group_data["Prices"][i] if i < len(group_data["Prices"]) else ""
+                row[f"Min {group_name}"] = group_data["Mins"][i] if i < len(group_data["Mins"]) else ""
+                row[f"Max {group_name}"] = group_data["Maxs"][i] if i < len(group_data["Maxs"]) else ""
+            all_rows_data.append(row)
+
+    final_df = pd.DataFrame(all_rows_data)
+    final_df.to_excel(f'{outlet_code}_final_output.xlsx', index=False)
+
+    # MongoDB setup
+    mongo_host = st.secrets["mongo"]["host"]
+    mongo_username = st.secrets["mongo"]["username"]
+    mongo_password = st.secrets["mongo"]["password"]
+
+    # Connect to MongoDB
+    connection_str = f"mongodb+srv://{mongo_username}:{mongo_password}@{mongo_host}/test?retryWrites=true&w=majority"
+    me.connect(host=connection_str, tlsCAFile=certifi.where())
+
+    class Item(me.Document):
+        category = me.StringField()
+        category_position = me.IntField()
+        item = me.StringField()
+        item_position = me.IntField()
+        description = me.StringField()
+        img_url = me.StringField()
+        price = me.FloatField()
+        options = me.ListField()
+
+    items_collection = []
+
+    for index, row in final_df.iterrows():
+        options = []
+        for i in range(1, 38):
+            option_group = f"Option Group {i}"
+            option_name_key = f"Option {i} Name"
+            option_price_key = f"Option {i} Price"
+            min_key = f"Min {i}"
+            max_key = f"Max {i}"
+
+            if pd.notna(row[option_group]) and row[option_group]:
+                options.append({
+                    "group_name": row[option_group],
+                    "name": row[option_name_key],
+                    "price": row[option_price_key],
+                    "min": row[min_key],
+                    "max": row[max_key]
+                })
+
+        item_doc = Item(
+            category=row['category'],
+            category_position=row['category-position'],
+            item=row['item'],
+            item_position=row['item-position'],
+            description=row['Description'],
+            img_url=row['img-url'],
+            price=row['price'],
+            options=options
+        )
+        items_collection.append(item_doc)
+
+    if items_collection:
+        Item.objects.insert(items_collection)
+        st.success("Data has been saved to MongoDB.")
+    else:
+        st.warning("No data to save to MongoDB.")

Mongo/Deliveroo_excel.py

@@ -0,0 +1,5 @@
+from Mongo.mongo_one import fetch_restaurant_data
+
+def Deliveroo_excel_multy(urls,location):
+    for url in urls:
+        fetch_restaurant_data(url,location)

Mongo/Noonfood_Mongo_URL_From_location.py

@@ -0,0 +1,250 @@
+import pandas as pd
+import requests
+import json
+import logging
+import mongoengine as me
+from mongoengine import connect
+import certifi
+from bson.objectid import ObjectId
+from fastapi import FastAPI, Query
+from pydantic import BaseModel
+
+db_name = 'NoonFood'
+host = 'ac-ut7ghe7-shard-00-00.299tz43.mongodb.net'
+port = 27017
+username = 'sahlhubnoonfood'
+password = '5JiDKBl6IPyH9Ff4'
+
+# Connect to the MongoDB database
+connect(db_name, host=host, port=port, username=username, password=password, ssl=True, tlsCAFile=certifi.where())
+
+class NoonFoodLink(me.Document):
+    initial_response = me.DictField(required=True)
+    url_list = me.ListField(me.StringField(), required=True)
+
+class MenuOption(me.Document):
+    name = me.StringField(required=True)
+    price = me.FloatField(required=True)
+    #location = me.StringField(required=True)
+    #restro_ref_id = me.ReferenceField(NoonFoodLink)
+
+class MenuOptionGroup(me.Document):
+    group_name = me.StringField(required=True)
+    options = me.ListField(me.ReferenceField(MenuOption))
+    min_quantity = me.IntField(required=True)
+    max_quantity = me.IntField(required=True)
+    #restro_ref_id = me.ReferenceField(NoonFoodLink)
+
+class MenuItem(me.Document):
+    category_name = me.StringField(required=True)
+    category_position = me.IntField(required=True)
+    name = me.StringField(required=True)
+    position = me.IntField(required=True)
+    description = me.StringField()
+    image_url = me.StringField()
+    price = me.FloatField(required=True)
+    option_groups = me.ListField(me.ReferenceField(MenuOptionGroup))
+    #restro_ref_id = me.ReferenceField(NoonFoodLink)
+
+class MenuCategory(me.Document):
+    name = me.StringField(required=True)
+    items = me.ListField(me.ReferenceField(MenuItem))
+    #restro_ref_id = me.ReferenceField(NoonFoodLink)
+
+# app = FastAPI()
+
+class RestaurantDetailsRequest(BaseModel):
+    latitude: float
+    longitude: float
+    restaurant_url: str
+
+class StoreDataRequest(BaseModel):
+    initial_response: dict
+    url_list: list
+
+def Mongo_location_URLS(url1,latitude,longitude):
+    outlet_code = url1.split('/')[-2]
+    def extract_items_with_categories(menu):
+        items_list = []
+        categories_seen = set()
+        for category in menu['categories']:
+            category_name = category['name']
+            if category_name not in categories_seen:
+                items_list.append({
+                    'category': category_name,
+                    'item': None,
+                    'itemCode': None,
+                    'item-position': None,
+                    'img-url': None,
+                    'price': None,
+                    'Description': None,
+                    'position': category['position']
+                })
+                categories_seen.add(category_name)
+            for item_code in category['items']:
+                item = next((item for item in menu['items'] if item['itemCode'] == item_code), None)
+                if item:
+                    items_list.append({
+                        'category': category_name,
+                        'item': item['name'],
+                        'itemCode': item['itemCode'],
+                        'item-position': item['position'],
+                        'img-url': item.get('image', ''),
+                        'price': item.get('price', 0.0),
+                        'Description': item.get('itemDesc', ''),
+                        'position': category['position']
+                    })
+        return items_list
+
+    # Extract options with proper formatting
+    def extract_options(menu):
+        options_dict = {}
+        for item in menu['items']:
+            if 'modifiers' in item:
+                for modifier_code in item['modifiers']:
+                    modifier = next((modifier for modifier in menu['modifiers'] if modifier['modifierCode'] == modifier_code), None)
+                    if modifier:
+                        if item['itemCode'] not in options_dict:
+                            options_dict[item['itemCode']] = {}
+                        if modifier['name'] not in options_dict[item['itemCode']]:
+                            options_dict[item['itemCode']][modifier['name']] = {
+                                'Min': modifier.get('minTotalOptions'),
+                                'Max': modifier.get('maxTotalOptions'),
+                                'Options': []
+                            }
+                        for option in modifier['options']:
+                            option_item = next((i for i in menu['items'] if i['itemCode'] == option['itemCode']), None)
+                            if option_item:
+                                options_dict[item['itemCode']][modifier['name']]['Options'].append({
+                                    'Option name': option_item['name'],
+                                    'Option price': option.get('price', 0.0)
+                                })
+        return options_dict
+
+    def process_json(data, options, outlet_code):
+        def process_item(item):
+            common_fields = {
+                'category': item['category'],
+                'category-position': item['position'],
+                'item': item['item'],
+                'item-position': item['item-position'],
+                'Description': item['Description'],
+                'img-url': item['img-url'],
+                'price': item['price']
+            }
+            item_options = options.get(item['itemCode'], {})
+            option_groups = []
+            for group_name, group_data in item_options.items():
+                min_value = group_data['Min']
+                max_value = group_data['Max']
+                options_list = []
+                for option in group_data['Options']:
+                    options_list.append({
+                        'name': option['Option name'],
+                        'price': option['Option price'],
+                    })
+                option_groups.append({
+                    'group_name': group_name,
+                    'min_quantity': min_value,
+                    'max_quantity': max_value,
+                    'options': options_list,
+                })
+            return {
+                'item': common_fields,
+                'option_groups': option_groups
+            }
+
+        processed_items = []
+        for item in data:
+            processed_item = process_item(item)
+            processed_items.append(processed_item)
+        
+        return processed_items
+
+    def store_in_db(processed_items):
+        categories_dict = {}
+
+        for item in processed_items:
+            item_common_fields = item['item']
+            option_groups = item['option_groups']
+
+            # Ensure required fields are present
+            if not item_common_fields['item'] or not item_common_fields['item-position'] or not item_common_fields['price']:
+                logging.error(f"Missing required fields in item: {item_common_fields}")
+                continue
+
+            # Create MenuItem
+            item_document = MenuItem(
+                category_name=item_common_fields['category'],
+                category_position=item_common_fields['category-position'],
+                name=item_common_fields['item'],
+                position=item_common_fields['item-position'],
+                description=item_common_fields['Description'],
+                image_url=item_common_fields['img-url'],
+                price=item_common_fields['price'],
+                option_groups=[]
+            )
+            item_document.save()
+
+            for group in option_groups:
+                options_list = []
+                for option_data in group['options']:
+                    menu_option = MenuOption(
+                        name=option_data['name'],
+                        price=option_data['price'],
+                    )
+                    menu_option.save()
+                    options_list.append(menu_option)
+
+                option_group = MenuOptionGroup(
+                    group_name=group['group_name'],
+                    min_quantity=group['min_quantity'],
+                    max_quantity=group['max_quantity'],
+                    options=options_list,
+                )
+                option_group.save()
+                item_document.option_groups.append(option_group)
+
+            item_document.save()
+
+            # Manage MenuCategory
+            category_name = item_common_fields['category']
+            if category_name not in categories_dict:
+                categories_dict[category_name] = MenuCategory(name=category_name, items=[])
+            categories_dict[category_name].items.append(item_document)
+
+        for category in categories_dict.values():
+            category.save()
+
+
+    url = "https://food.noon.com/_svc/mp-food-api-mpnoon/consumer/restaurant/outlet/details/guest"
+    payload = {
+        "addressLat": latitude,
+        "addressLng": longitude,
+        "deliveryType": "default",
+        "outletCode": outlet_code
+    }
+    headers = {
+        'Connection': 'keep-alive',
+        "Accept": "application/json, text/plain, */*",
+        "Accept-Encoding": "gzip, deflate, br, zstd",
+        "Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8,gu;q=0.7",
+        "Cache-Control": "no-cache, max-age=0, must-revalidate, no-store",
+        "Content-Type": "application/json",
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
+        "Cookie": "bm_mi=791533C8E67CE8E7DA98E80ADED70F69~YAAQRK0cuOep9tGPAQAAUYKw3RcGDAVhD+mtWU8IH76wZL29zl4qqCjMwGv8sKtYlQWZNaFftSOvHFOvQU4+3CY2uHZyjjK6I3GeNdKEn+XHupISeNc0K16GOXLqcPOwu4sADTmxE7PYQvSQE7eimhqsBiJVRd96R8W0D2hl31FlY/4rl+NPZvM3iXjrn2GO50VMv+HhGfCnDMBwApBxgpMWFLfs0u6EYy44mg/FXbom5s5pa3cro8AS35nYHbdUbi61K9fnWRVaF8L/4z0xh7V1AEQETevb5fdGF8aB9m2UG29p2W6KSMb8DyFZLpG3vl5+IRECqZdFxaUMnykO8G/ynRHG~1; Domain=.noon.com; Path=/; Expires=Mon, 03 Jun 2024 12:41:22 GMT; Max-Age=7199; Secure"
+    }
+
+    try:
+        response = requests.post(url, headers=headers, json=payload)
+        json_data = response.json()
+        items = extract_items_with_categories(json_data['data']['menu'])
+        options = extract_options(json_data['data']['menu'])
+        processed_items = process_json(items, options, outlet_code)
+        store_in_db(processed_items)
+
+        return {"message": "Restaurant details extracted and stored successfully."}
+
+    except Exception as e:
+        logging.error(f"An error occurred: {str(e)}")
+        return {"error": "An error occurred while processing the request."}

Mongo/Noonfood_location.py

@@ -0,0 +1,153 @@
+import requests
+import json
+from urllib.parse import urljoin
+import pymongo
+from pymongo import MongoClient
+import certifi
+from Mongo.Noonfood_Mongo_URL_From_location import Mongo_location_URLS
+
+
+def mutiple_url_location(lat , lng):
+    def get_initial_data(lat, lng):
+        initial_url = "https://food.noon.com/_svc/customer-v1/customer/public-area-serviceable"
+        initial_payload = {"lat": str(lat), "lng": str(lng)}
+        initial_headers = {
+            "Accept": "application/json, text/plain, */*",
+            "Accept-Encoding": "gzip, deflate, br, zstd",
+            "Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8,gu;q=0.7",
+            "Cache-Control": "no-cache, max-age=0, must-revalidate, no-store",
+            "Content-Type": "application/json",
+            "Origin": "https://food.noon.com",
+            "Referer": "https://food.noon.com/",
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
+            "X-Content": "desktop",
+            "X-Experience": "food",
+            "X-Locale": "en-ae",
+            "X-Mp": "noon",
+            "X-Platform": "web",
+            "X-Visitor-Id": "1072f8c2-cfd6-4734-8795-d637de61fba1",
+        }
+
+        try:
+            response = requests.post(initial_url, json=initial_payload, headers=initial_headers)
+            response.raise_for_status()
+            json_data = response.json()
+            print(f"Initial Response JSON: {json_data}")
+
+            new_lat = json_data.get('lat')
+            new_lng = json_data.get('lng')
+            
+            if new_lat and new_lng:
+                return json_data, new_lat, new_lng
+            else:
+                print("lat or lng not found in the initial response.")
+                return None, None, None
+        except requests.exceptions.RequestException as e:
+            print(f"Request failed: {e}")
+            return None, None, None
+
+    def get_urls_from_json(json_data):
+        url_list = []
+        try:
+            results = json_data.get("results", [])
+            if len(results) > 5:
+                banners = results[5]['modules'][0].get("banners", [])
+                for n in range(min(20, len(banners))):
+                    try:
+                        base_url = 'https://food.noon.com/'
+                        want_data = banners[n].get("linkUrl")
+                        if want_data:
+                            full_url = urljoin(base_url, want_data)
+                            url_list.append(full_url)
+                    except Exception as e:
+                        print(f"Error occurred while extracting URLs: {e}")
+                        break
+            else:
+                print("Insufficient data in 'results'")
+        except Exception as e:
+            print(f"Error parsing JSON data: {e}")
+        return url_list
+
+    def store_data_in_mongo(json_data, url_list):
+        try:
+            # MongoDB connection settings
+            client = MongoClient(
+                "mongodb+srv://dipenigenerate:[email protected]",
+                tlsCAFile=certifi.where()
+            )
+            db = client['Restaurants_in_dubai']
+            collection = db['noonfood_link']
+
+            # Create the document to insert
+            document = {
+                "initial_response": json_data,
+                "url_list": url_list
+            }
+
+            # Insert the document into the collection
+            result = collection.insert_one(document)
+            print(f"Data inserted with id: {result.inserted_id}")
+        except Exception as e:
+            print(f"Failed to store data in MongoDB: {e}")
+
+    def fetch_urls(lat, lng):
+        json_data, new_lat, new_lng = get_initial_data(lat, lng)
+        
+        if new_lat and new_lng:
+            new_url = "https://food.noon.com/_svc/mp-food-api-catalog/api/"
+            new_headers = {
+                "method": "GET",
+                "Accept": "application/json, text/plain, */*",
+                "Accept-Encoding": "gzip, deflate, br, zstd",
+                "Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8,gu;q=0.7",
+                "Cache-Control": "no-cache, max-age=0, must-revalidate, no-store",
+                "Content-Type": "application/json",
+                "Origin": "https://food.noon.com",
+                "Referer": "https://food.noon.com/",
+                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
+                "X-Content": "desktop",
+                "X-Experience": "food",
+                "X-Locale": "en-ae",
+                "X-Mp": "noon",
+                "X-Platform": "web",
+                "X-lat": f"{new_lat}",
+                "X-lng": f"{new_lng}"
+            }
+
+            try:
+                new_response = requests.get(new_url, headers=new_headers)
+                new_response.raise_for_status()
+                new_json_data = new_response.json()
+
+                # Extract URLs from the JSON data
+                url_list = get_urls_from_json(new_json_data)
+                
+                # Store the initial JSON response and URLs in MongoDB
+                store_data_in_mongo(json_data, url_list)
+                
+                return url_list
+
+            except requests.exceptions.RequestException as e:
+                print(f"Failed to retrieve new content from the URL: {e}")
+                return []
+        else:
+            return []
+    urls = fetch_urls(lat, lng)
+    if urls:
+        print("Fetched URLs:" , urls)
+        for url in urls:
+            url = url+"/"
+            Mongo_location_URLS(url,lat,lng)
+
+# if __name__ == "__main__":
+#     lat = int(input("Enter the latitude: "))
+#     lng = int(input("Enter the longitude: "))
+
+#     urls = fetch_urls(lat, lng)
+#     if urls:
+#         print("Fetched URLs:" , urls)
+#         for url in urls:
+#             url = url+"/"
+#             Mongo_location_URLS(url,lat,lng)
+#     else:
+#         print("No URLs fetched.")

Mongo/Noonfood_mongo_Single_URL.py

@@ -0,0 +1,274 @@
+import pandas as pd
+import requests
+import json
+import logging
+import mongoengine as me
+from mongoengine import connect
+import certifi
+from bson.objectid import ObjectId
+from fastapi import FastAPI, Query
+from pydantic import BaseModel
+
+# Define the connection settings
+db_name = 'NoonFood'
+host = 'ac-ut7ghe7-shard-00-00.299tz43.mongodb.net'
+port = 27017
+username = 'sahlhubnoonfood'
+password = '5JiDKBl6IPyH9Ff4'
+alias = 'default'
+
+me.disconnect(alias)
+me.connect(db_name, host=host, port=port, username=username, password=password, ssl=True, tlsCAFile=certifi.where())
+
+class NoonFoodLink(me.Document):
+    initial_response = me.DictField(required=True)
+    url_list = me.ListField(me.StringField(), required=True)
+
+class MenuOption(me.Document):
+    name = me.StringField(required=True)
+    price = me.FloatField(required=True)
+    #location = me.StringField(required=True)
+    #restro_ref_id = me.ReferenceField(NoonFoodLink)
+
+class MenuOptionGroup(me.Document):
+    group_name = me.StringField(required=True)
+    options = me.ListField(me.ReferenceField(MenuOption))
+    min_quantity = me.IntField(required=True)
+    max_quantity = me.IntField(required=True)
+    #restro_ref_id = me.ReferenceField(NoonFoodLink)
+
+class MenuItem(me.Document):
+    category_name = me.StringField(required=True)
+    category_position = me.IntField(required=True)
+    name = me.StringField(required=True)
+    position = me.IntField(required=True)
+    description = me.StringField()
+    image_url = me.StringField()
+    price = me.FloatField(required=True)
+    option_groups = me.ListField(me.ReferenceField(MenuOptionGroup))
+    #restro_ref_id = me.ReferenceField(NoonFoodLink)
+
+class MenuCategory(me.Document):
+    name = me.StringField(required=True)
+    items = me.ListField(me.ReferenceField(MenuItem))
+    #restro_ref_id = me.ReferenceField(NoonFoodLink)
+
+# app = FastAPI()
+
+class RestaurantDetailsRequest(BaseModel):
+    latitude: float
+    longitude: float
+    restaurant_url: str
+
+class StoreDataRequest(BaseModel):
+    initial_response: dict
+    url_list: list
+
+# Extract items with categories
+def extract_items_with_categories(menu):
+    items_list = []
+    categories_seen = set()
+    for category in menu['categories']:
+        category_name = category['name']
+        if category_name not in categories_seen:
+            items_list.append({
+                'category': category_name,
+                'item': None,
+                'itemCode': None,
+                'item-position': None,
+                'img-url': None,
+                'price': None,
+                'Description': None,
+                'position': category['position']
+            })
+            categories_seen.add(category_name)
+        for item_code in category['items']:
+            item = next((item for item in menu['items'] if item['itemCode'] == item_code), None)
+            if item:
+                items_list.append({
+                    'category': category_name,
+                    'item': item['name'],
+                    'itemCode': item['itemCode'],
+                    'item-position': item['position'],
+                    'img-url': item.get('image', ''),
+                    'price': item.get('price', 0.0),
+                    'Description': item.get('itemDesc', ''),
+                    'position': category['position']
+                })
+    return items_list
+
+# Extract options with proper formatting
+def extract_options(menu):
+    options_dict = {}
+    for item in menu['items']:
+        if 'modifiers' in item:
+            for modifier_code in item['modifiers']:
+                modifier = next((modifier for modifier in menu['modifiers'] if modifier['modifierCode'] == modifier_code), None)
+                if modifier:
+                    if item['itemCode'] not in options_dict:
+                        options_dict[item['itemCode']] = {}
+                    if modifier['name'] not in options_dict[item['itemCode']]:
+                        options_dict[item['itemCode']][modifier['name']] = {
+                            'Min': modifier.get('minTotalOptions'),
+                            'Max': modifier.get('maxTotalOptions'),
+                            'Options': []
+                        }
+                    for option in modifier['options']:
+                        option_item = next((i for i in menu['items'] if i['itemCode'] == option['itemCode']), None)
+                        if option_item:
+                            options_dict[item['itemCode']][modifier['name']]['Options'].append({
+                                'Option name': option_item['name'],
+                                'Option price': option.get('price', 0.0)
+                            })
+    return options_dict
+
+def process_json(data, options, outlet_code):
+    def process_item(item):
+        common_fields = {
+            'category': item['category'],
+            'category-position': item['position'],
+            'item': item['item'],
+            'item-position': item['item-position'],
+            'Description': item['Description'],
+            'img-url': item['img-url'],
+            'price': item['price']
+        }
+        item_options = options.get(item['itemCode'], {})
+        option_groups = []
+        for group_name, group_data in item_options.items():
+            min_value = group_data['Min']
+            max_value = group_data['Max']
+            options_list = []
+            for option in group_data['Options']:
+                options_list.append({
+                    'name': option['Option name'],
+                    'price': option['Option price'],
+                })
+            option_groups.append({
+                'group_name': group_name,
+                'min_quantity': min_value,
+                'max_quantity': max_value,
+                'options': options_list,
+            })
+        return {
+            'item': common_fields,
+            'option_groups': option_groups
+        }
+
+    processed_items = []
+    for item in data:
+        processed_item = process_item(item)
+        processed_items.append(processed_item)
+    
+    return processed_items
+
+def store_in_db(processed_items):
+    categories_dict = {}
+
+    for item in processed_items:
+        item_common_fields = item['item']
+        option_groups = item['option_groups']
+
+        # Ensure required fields are present
+        if not item_common_fields['item'] or not item_common_fields['item-position'] or not item_common_fields['price']:
+            logging.error(f"Missing required fields in item: {item_common_fields}")
+            continue
+
+        # Create MenuItem
+        item_document = MenuItem(
+            category_name=item_common_fields['category'],
+            category_position=item_common_fields['category-position'],
+            name=item_common_fields['item'],
+            position=item_common_fields['item-position'],
+            description=item_common_fields['Description'],
+            image_url=item_common_fields['img-url'],
+            price=item_common_fields['price'],
+            option_groups=[]
+        )
+        item_document.save()
+
+        for group in option_groups:
+            options_list = []
+            for option_data in group['options']:
+                menu_option = MenuOption(
+                    name=option_data['name'],
+                    price=option_data['price'],
+                )
+                menu_option.save()
+                options_list.append(menu_option)
+
+            option_group = MenuOptionGroup(
+                group_name=group['group_name'],
+                min_quantity=group['min_quantity'],
+                max_quantity=group['max_quantity'],
+                options=options_list,
+            )
+            option_group.save()
+            item_document.option_groups.append(option_group)
+
+        item_document.save()
+
+        # Manage MenuCategory
+        category_name = item_common_fields['category']
+        if category_name not in categories_dict:
+            categories_dict[category_name] = MenuCategory(name=category_name, items=[])
+        categories_dict[category_name].items.append(item_document)
+
+    for category in categories_dict.values():
+        category.save()
+
+
+def get_restaurant_details(latitude,longitude,url1):
+    outlet_code = url1.split('/')[-2]
+
+    # Make the request to fetch restaurant details
+    url = "https://food.noon.com/_svc/mp-food-api-mpnoon/consumer/restaurant/outlet/details/guest"
+    payload = {
+        "addressLat": latitude,
+        "addressLng": longitude,
+        "deliveryType": "default",
+        "outletCode": outlet_code
+    }
+    headers = {
+        'Connection': 'keep-alive',
+        "Accept": "application/json, text/plain, */*",
+        "Accept-Encoding": "gzip, deflate, br, zstd",
+        "Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8,gu;q=0.7",
+        "Cache-Control": "no-cache, max-age=0, must-revalidate, no-store",
+        "Content-Type": "application/json",
+        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
+        "Cookie": "bm_mi=791533C8E67CE8E7DA98E80ADED70F69~YAAQRK0cuOep9tGPAQAAUYKw3RcGDAVhD+mtWU8IH76wZL29zl4qqCjMwGv8sKtYlQWZNaFftSOvHFOvQU4+3CY2uHZyjjK6I3GeNdKEn+XHupISeNc0K16GOXLqcPOwu4sADTmxE7PYQvSQE7eimhqsBiJVRd96R8W0D2hl31FlY/4rl+NPZvM3iXjrn2GO50VMv+HhGfCnDMBwApBxgpMWFLfs0u6EYy44mg/FXbom5s5pa3cro8AS35nYHbdUbi61K9fnWRVaF8L/4z0xh7V1AEQETevb5fdGF8aB9m2UG29p2W6KSMb8DyFZLpG3vl5+IRECqZdFxaUMnykO8G/ynRHG~1; Domain=.noon.com; Path=/; Expires=Mon, 03 Jun 2024 12:41:22 GMT; Max-Age=7199; Secure"
+    }
+
+    try:
+        response = requests.post(url, headers=headers, json=payload)
+        json_data = response.json()
+        items = extract_items_with_categories(json_data['data']['menu'])
+        options = extract_options(json_data['data']['menu'])
+        processed_items = process_json(items, options, outlet_code)
+        store_in_db(processed_items)
+
+        return {"message": "Restaurant details extracted and stored successfully."}
+
+    except Exception as e:
+        logging.error(f"An error occurred: {str(e)}")
+        return {"error": "An error occurred while processing the request."}
+
+# @app.post("/store_data")
+# def store_data(details: StoreDataRequest):
+#     try:
+#         # Create a document from the request data
+#         document = NoonFoodLink(
+#             initial_response=details.initial_response,
+#             url_list=details.url_list
+#         )
+#         document.save()
+#         return {"message": "Data stored successfully."}
+#     except Exception as e:
+#         return {"error": str(e)}
+
+# if __name__ == "__main__":
+#     import uvicorn
+
+#     # Run FastAPI application using Uvicorn
+#     uvicorn.run(app, host="127.0.0.1", port=8000)

Mongo/Noonfood_multy.py

@@ -0,0 +1,6 @@
+from Mongo.Noonfood_mongo_Single_URL import get_restaurant_details
+
+def Noonfood_multy_urls(latitude,longitude,urls):
+    for url in urls:
+        get_restaurant_details(latitude,longitude,url)
+

Mongo/mongo_final.py

@@ -0,0 +1,292 @@
+import json
+import requests
+from bs4 import BeautifulSoup
+import mongoengine as me
+import certifi
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+from typing import List
+from urllib.parse import urljoin
+import uvicorn
+
+# MongoDB connection settings
+db_name = 'Deliveroo'
+host = 'ac-xvwfer4-shard-00-00.xrv1il9.mongodb.net'
+port = 27017
+username = 'sahlhubdeliveroo'
+password = 'vp5bLCjufKdarP1W'
+alias = 'default'
+
+me.disconnect(alias)
+me.connect(db_name, host=host, port=port, username=username, password=password, ssl=True, tlsCAFile=certifi.where())
+
+# Define the MongoDB document models
+class DeliverooLink(me.Document):
+    href = me.StringField(required=True)
+ 
+class MenuOption(me.Document):
+    name = me.StringField(required=True)
+    price = me.StringField(required=True)
+    option_groups = me.ListField(me.ReferenceField('MenuOptionGroup'))
+    location = me.StringField(required=True)
+    restro_ref_id = me.ReferenceField(DeliverooLink, required=True)
+
+class MenuOptionGroup(me.Document):
+    group_name = me.StringField(required=True)
+    options = me.ListField(me.ReferenceField(MenuOption))
+    min_quantity = me.IntField(required=True)
+    max_quantity = me.IntField(required=True)
+    location = me.StringField(required=True)
+    restro_ref_id = me.ReferenceField(DeliverooLink, required=True)
+
+class MenuItem(me.Document):
+    category_name = me.StringField(required=True)
+    name = me.StringField(required=True)
+    description = me.StringField()
+    image_url = me.StringField()
+    price = me.StringField(required=True)
+    option_groups = me.ListField(me.ReferenceField(MenuOptionGroup))
+    location = me.StringField(required=True)
+    restro_ref_id = me.ReferenceField(DeliverooLink, required=True)
+
+class MenuCategory(me.Document):
+    name = me.StringField(required=True)
+    items = me.ListField(me.ReferenceField(MenuItem))
+    location = me.StringField(required=True)
+    restro_ref_id = me.ReferenceField(DeliverooLink, required=True)
+
+# FastAPI app
+# app = FastAPI()
+
+# Pydantic model for input validation
+# class LocationRequest(BaseModel):
+#     city: str
+#     location: str
+
+# @app.post("/fetch_and_process_data")
+# async def fetch_and_process_data(request: LocationRequest):
+#     city = request.city.lower()
+#     location = request.location.lower()
+#     print(f"Fetching restaurant links for {city}, {location}...")
+
+#     links = fetch_restaurant_links(city, location)
+#     if not links:
+#         raise HTTPException(status_code=404, detail="No restaurants found for the specified location.")
+
+#     print(f"Found {len(links)} links. Processing restaurant data...")
+#     data = process_links(links, location)
+
+#     print("Saving data to MongoDB...")
+#     save_data_to_db(data, location)
+
+#     print("Data has been processed and saved successfully.")
+#     return {"message": "Data has been processed and saved successfully."}
+
+def fetch_restaurant_links(city, location):
+    base_url = "https://deliveroo.ae"
+    url = f"{base_url}/restaurants/{city}/{location}/?collection=restaurants"
+
+    headers = {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+        'Cookie': '__cf_bm=oakl46sJ3V9vwmnIIbfXWfkHbGmmC2pH56GyTI33b4U-1715931048-1.0.1.1-4XOcSGSThZV_INfpn3aptlo8jpZtLFbYoLsZxP9BpQ8LIjq3wBIe8CPlSf0AomuniXy4TZWyVlBQBTlrm.CPiSfI1jzx18y9zxwc9GX0fmo; roo_guid=c40617a7-76f7-432c-b780-f2653cd2edfe; roo_session_guid=2e989653-2776-4ede-a52e-b610f1ad64a2'
+    }
+
+    response = requests.get(url, headers=headers)
+    if response.status_code == 200:
+        soup = BeautifulSoup(response.content, 'html.parser')
+
+        if "We couldn't find" in soup.text or "No restaurants" in soup.text:
+            print("No restaurants found for the specified location.")
+            return []
+
+        divs = soup.find_all('div', class_=["HomeFeedScrollTracker-bd9a6ffea8a4b4b7", "HomeFeedUICard-157f7be5d7b2fa7b"])
+
+        hrefs = [a_tag['href'] for div in divs for a_tag in div.find_all('a', href=True)]
+        print(hrefs)
+        hrefs = hrefs[:20]
+
+        full_urls = []
+        for href in hrefs:
+            full_url = urljoin(base_url, href)
+            link = DeliverooLink(href=full_url)
+            link.save()
+            full_urls.append(full_url)
+
+        return full_urls
+    else:
+        print("Response timed out.")
+        return []
+
+def fetch_restaurant_data(url, location):
+    headers = {
+        'Cookie': '__cf_bm=_AOZtAiObnqBHPy4zhGRgBLW9xg9WiaDCRzg5E0sbMk-1715757967-1.0.1.1-xZNMBsnAqy_tfjUveujgfzT4Usw5ur4u7L0JlCcNXAQIC6Cq6wj46vPH7RLTh0Gq90JENxl7kbzjyOUFaBr8yCkmRGmt7APITEk0kkXzLTs; roo_guid=c40617a7-76f7-432c-b780-f2653cd2edfe; roo_session_guid=5846d6f0-5b7f-4598-8c6d-82b8023fd4fc'
+    }
+
+    response = requests.get(url, headers=headers)
+
+    if response.status_code != 200:
+        print(f"Failed to fetch the URL: {url}")
+        return None
+
+    soup = BeautifulSoup(response.content, 'html.parser')
+    script_tag = soup.find('script', id='__NEXT_DATA__')
+
+    if not script_tag:
+        print("Script tag not found")
+        return None
+
+    json_data = json.loads(script_tag.string)
+    json_data = json_data['props']['initialState']['menuPage']['menu']['meta']
+
+    items = json_data['items']
+    categories = json_data['categories']
+    category_map = {category['id']: category['name'] for category in categories}
+    modifier_groups = json_data['modifierGroups']
+
+    """modifier_groups_dict = {item['id']: [] for item in items}
+    for modifier_group in modifier_groups:
+        modifier_groups_dict[modifier_group['id']] = modifier_group"""
+    modifier_groups_dict = {modifier_group['id']: modifier_group for modifier_group in modifier_groups}
+
+    items_with_modifiers = []
+    for item in items:
+        category_id = item['categoryId']
+        category_name = category_map.get(category_id, 'Unknown')
+        
+        # Skip items with the "Unknown" category
+        if category_name == 'Unknown':
+            continue
+
+        item_with_modifiers = {
+            "id": item['id'],
+            "category_id": category_id,
+            "category_name": category_name,
+            "name": item['name'],
+            "description": item.get('description', ''),
+            "price": item['price']['formatted'],
+            "img_url": item.get('image').get('url', '') if item.get('image') else '',
+            "modifier_groups": [modifier_groups_dict.get(modifier_group_id, {}) for modifier_group_id in item.get('modifierGroupIds', [])],
+            "location": location,
+            "restro_ref_id": url
+        }
+        items_with_modifiers.append(item_with_modifiers)
+
+    return items_with_modifiers
+
+# def save_data_to_json(data, filename='restaurants.json'):
+#     with open(filename, 'w') as f:
+#         json.dump(data, f, indent=4)
+
+def process_links(urls, location):
+    all_data = []
+    for url in urls:
+        data = fetch_restaurant_data(url, location)
+        if data:
+            all_data.extend(data)
+    # save_data_to_json(all_data)
+    return all_data
+
+def save_data_to_db(data, location):
+    # Dictionary to hold categories and associated items
+    category_dict = {}
+
+    for item_data in data:
+        category_name = item_data['category_name']
+        name = item_data['name']
+        description = item_data.get('description', '')
+        image_url = item_data.get('img_url', '')
+        price = item_data['price']
+        option_groups_data = item_data.get('modifier_groups', [])
+        restro_ref_id_url = item_data['restro_ref_id']
+
+        restro_link = DeliverooLink.objects(href=restro_ref_id_url).first()
+        if not restro_link:
+            restro_link = DeliverooLink(href=restro_ref_id_url)
+            restro_link.save()
+
+        # Create or retrieve MenuOption instances
+        option_groups = []
+        for group_data in option_groups_data:
+            group_name = group_data['name']
+            min_quantity = group_data.get('minSelection', 0)
+            max_quantity = group_data.get('maxSelection', 1)
+            options_data = group_data.get('modifierOptions', [])
+
+            option_group = MenuOptionGroup.objects(
+                group_name=group_name, min_quantity=min_quantity, max_quantity=max_quantity, location=location, restro_ref_id=restro_link).first()
+
+            if not option_group:
+                option_group = MenuOptionGroup(
+                    group_name=group_name,
+                    min_quantity=min_quantity,
+                    max_quantity=max_quantity,
+                    location=location,
+                    restro_ref_id=restro_link
+                )
+                option_group.save()
+
+            options = []
+            for option_data in options_data:
+                option_name = option_data['name']
+                option_price = option_data['price']['formatted']
+
+                option = MenuOption.objects(name=option_name, price=option_price, location=location, restro_ref_id=restro_link).first()
+                if not option:
+                    option = MenuOption(name=option_name, price=option_price, location=location, restro_ref_id=restro_link)
+                    option.save()
+
+                if option not in option_group.options:
+                    option_group.options.append(option)
+                    option_group.save()
+
+                if option_group not in option.option_groups:
+                    option.option_groups.append(option_group)
+                    option.save()
+
+                options.append(option)
+
+            option_groups.append(option_group)
+        
+        # Create or retrieve MenuItem instance
+        menu_item = MenuItem.objects(
+            category_name=category_name, name=name, location=location, restro_ref_id=restro_link).first()
+        if not menu_item:
+            menu_item = MenuItem(
+                category_name=category_name,
+                name=name,
+                description=description,
+                image_url=image_url,
+                price=price,
+                option_groups=option_groups,
+                location=location,
+                restro_ref_id=restro_link
+            )
+            menu_item.save()
+        else:
+            # Ensure option groups are updated for existing menu items
+            for option_group in option_groups:
+                if option_group not in menu_item.option_groups:
+                    menu_item.option_groups.append(option_group)
+            menu_item.save()
+
+        # Store the menu item under its category
+        if category_name not in category_dict:
+            category_dict[category_name] = []
+        category_dict[category_name].append(menu_item)
+
+    # Create or retrieve MenuCategory instances and associate items
+    for category_name, items in category_dict.items():
+        menu_category = MenuCategory.objects(name=category_name, location=location, restro_ref_id=restro_link).first()
+        if not menu_category:
+            menu_category = MenuCategory(name=category_name, location=location, restro_ref_id=restro_link)
+            menu_category.save()
+
+        for menu_item in items:
+            if menu_item not in menu_category.items:
+                menu_category.items.append(menu_item)
+                menu_category.save()
+
+        print(f"Saved item '{name}' in category '{category_name}' for location '{location}'")
+
+# if __name__ == "__main__":
+#     uvicorn.run(app, host="0.0.0.0", port=8000)

Mongo/mongo_one.py

@@ -0,0 +1,243 @@
+import json
+import requests
+from bs4 import BeautifulSoup
+import mongoengine as me
+import certifi
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+from typing import List
+from urllib.parse import urljoin
+import uvicorn
+
+# MongoDB connection settings
+db_name = 'Deliveroo'
+host = 'ac-xvwfer4-shard-00-00.xrv1il9.mongodb.net'
+port = 27017
+username = 'sahlhubdeliveroo'
+password = 'vp5bLCjufKdarP1W'
+
+
+me.disconnect()
+# Connect to the MongoDB database
+me.connect(db_name, host=host, port=port, username=username, password=password, ssl=True, tlsCAFile=certifi.where())
+
+# Define the MongoDB document models
+class DeliverooLink(me.Document):
+    href = me.StringField(required=True)
+ 
+class MenuOption(me.Document):
+    name = me.StringField(required=True)
+    price = me.StringField(required=True)
+    option_groups = me.ListField(me.ReferenceField('MenuOptionGroup'))
+    location = me.StringField(required=True)
+    restro_ref_id = me.ReferenceField(DeliverooLink, required=True)
+
+class MenuOptionGroup(me.Document):
+    group_name = me.StringField(required=True)
+    options = me.ListField(me.ReferenceField(MenuOption))
+    min_quantity = me.IntField(required=True)
+    max_quantity = me.IntField(required=True)
+    location = me.StringField(required=True)
+    restro_ref_id = me.ReferenceField(DeliverooLink, required=True)
+
+class MenuItem(me.Document):
+    category_name = me.StringField(required=True)
+    name = me.StringField(required=True)
+    description = me.StringField()
+    image_url = me.StringField()
+    price = me.StringField(required=True)
+    option_groups = me.ListField(me.ReferenceField(MenuOptionGroup))
+    location = me.StringField(required=True)
+    restro_ref_id = me.ReferenceField(DeliverooLink, required=True)
+
+class MenuCategory(me.Document):
+    name = me.StringField(required=True)
+    items = me.ListField(me.ReferenceField(MenuItem))
+    location = me.StringField(required=True)
+    restro_ref_id = me.ReferenceField(DeliverooLink, required=True)
+
+# FastAPI app
+
+# Pydantic model for input validation
+# class URLRequest(BaseModel):
+#     city: str
+#     location: str
+#     url: str
+
+# @app.post("/fetch_and_process_data")
+# async def fetch_and_process_data(request: URLRequest):
+#     city = request.city.lower()
+#     location = request.location.lower()
+#     url = request.url
+#     print(f"Fetching restaurant data for {city}, {location}...")
+
+#     print(f"Processing restaurant data from URL: {url}...")
+#     data = fetch_restaurant_data(url, location)
+
+#     if not data:
+#         raise HTTPException(status_code=404, detail="No data found for the specified URL.")
+
+#     print("Saving data to MongoDB...")
+#     save_data_to_db(data, location)
+
+#     print("Data has been processed and saved successfully.")
+#     return {"message": "Data has been processed and saved successfully."}
+
+def fetch_restaurant_data(url, location):
+    headers = {
+        'Cookie': '__cf_bm=_AOZtAiObnqBHPy4zhGRgBLW9xg9WiaDCRzg5E0sbMk-1715757967-1.0.1.1-xZNMBsnAqy_tfjUveujgfzT4Usw5ur4u7L0JlCcNXAQIC6Cq6wj46vPH7RLTh0Gq90JENxl7kbzjyOUFaBr8yCkmRGmt7APITEk0kkXzLTs; roo_guid=c40617a7-76f7-432c-b780-f2653cd2edfe; roo_session_guid=5846d6f0-5b7f-4598-8c6d-82b8023fd4fc'
+    }
+
+    response = requests.get(url, headers=headers)
+
+    if response.status_code != 200:
+        print(f"Failed to fetch the URL: {url}")
+        return None
+
+    soup = BeautifulSoup(response.content, 'html.parser')
+    script_tag = soup.find('script', id='__NEXT_DATA__')
+
+    if not script_tag:
+        print("Script tag not found")
+        return None
+
+    json_data = json.loads(script_tag.string)
+    json_data = json_data['props']['initialState']['menuPage']['menu']['meta']
+
+    items = json_data['items']
+    categories = json_data['categories']
+    category_map = {category['id']: category['name'] for category in categories}
+    modifier_groups = json_data['modifierGroups']
+
+    modifier_groups_dict = {modifier_group['id']: modifier_group for modifier_group in modifier_groups}
+
+    items_with_modifiers = []
+    for item in items:
+        category_id = item['categoryId']
+        category_name = category_map.get(category_id, 'Unknown')
+        
+        # Skip items with the "Unknown" category
+        if category_name == 'Unknown':
+            continue
+
+        item_with_modifiers = {
+            "id": item['id'],
+            "category_id": category_id,
+            "category_name": category_name,
+            "name": item['name'],
+            "description": item.get('description', ''),
+            "price": item['price']['formatted'],
+            "img_url": item.get('image').get('url', '') if item.get('image') else '',
+            "modifier_groups": [modifier_groups_dict.get(modifier_group_id, {}) for modifier_group_id in item.get('modifierGroupIds', [])],
+            "location": location,
+            "restro_ref_id": url
+        }
+        items_with_modifiers.append(item_with_modifiers)
+
+    return items_with_modifiers
+
+def save_data_to_json(data, filename='restaurants.json'):
+    with open(filename, 'w') as f:
+        json.dump(data, f, indent=4)
+
+def save_data_to_db(data, location):
+    # Dictionary to hold categories and associated items
+    category_dict = {}
+
+    for item_data in data:
+        category_name = item_data['category_name']
+        name = item_data['name']
+        description = item_data.get('description', '')
+        image_url = item_data.get('img_url', '')
+        price = item_data['price']
+        option_groups_data = item_data.get('modifier_groups', [])
+        restro_ref_id_url = item_data['restro_ref_id']
+
+        restro_link = DeliverooLink.objects(href=restro_ref_id_url).first()
+        if not restro_link:
+            restro_link = DeliverooLink(href=restro_ref_id_url)
+            restro_link.save()
+
+        # Create or retrieve MenuOption instances
+        option_groups = []
+        for group_data in option_groups_data:
+            group_name = group_data['name']
+            min_quantity = group_data.get('minSelection', 0)
+            max_quantity = group_data.get('maxSelection', 1)
+            options_data = group_data.get('modifierOptions', [])
+
+            option_group = MenuOptionGroup.objects(
+                group_name=group_name, min_quantity=min_quantity, max_quantity=max_quantity, location=location, restro_ref_id=restro_link).first()
+
+            if not option_group:
+                option_group = MenuOptionGroup(
+                    group_name=group_name,
+                    min_quantity=min_quantity,
+                    max_quantity=max_quantity,
+                    location=location,
+                    restro_ref_id=restro_link
+                )
+                option_group.save()
+
+            options = []
+            for option_data in options_data:
+                option_name = option_data['name']
+                option_price = option_data['price']['formatted']
+
+                option = MenuOption.objects(name=option_name, price=option_price, location=location, restro_ref_id=restro_link).first()
+                if not option:
+                    option = MenuOption(name=option_name, price=option_price, location=location, restro_ref_id=restro_link)
+                    option.save()
+
+                if option not in option_group.options:
+                    option_group.options.append(option)
+                    option_group.save()
+
+                if option_group not in option.option_groups:
+                    option.option_groups.append(option_group)
+                    option.save()
+
+                options.append(option)
+
+            option_groups.append(option_group)
+        
+        # Create or retrieve MenuItem instance
+        menu_item = MenuItem.objects(
+            category_name=category_name, name=name, location=location, restro_ref_id=restro_link).first()
+        if not menu_item:
+            menu_item = MenuItem(
+                category_name=category_name,
+                name=name,
+                description=description,
+                image_url=image_url,
+                price=price,
+                option_groups=option_groups,
+                location=location,
+                restro_ref_id=restro_link
+            )
+            menu_item.save()
+        else:
+            # Ensure option groups are updated for existing menu items
+            for option_group in option_groups:
+                if option_group not in menu_item.option_groups:
+                    menu_item.option_groups.append(option_group)
+            menu_item.save()
+
+        # Store the menu item under its category
+        if category_name not in category_dict:
+            category_dict[category_name] = []
+        category_dict[category_name].append(menu_item)
+
+    # Create or retrieve MenuCategory instances and associate items
+    for category_name, items in category_dict.items():
+        menu_category = MenuCategory.objects(name=category_name, location=location, restro_ref_id=restro_link).first()
+        if not menu_category:
+            menu_category = MenuCategory(name=category_name, location=location, restro_ref_id=restro_link)
+            menu_category.save()
+
+        for menu_item in items:
+            if menu_item not in menu_category.items:
+                menu_category.items.append(menu_item)
+                menu_category.save()
+
+        print(f"Saved item '{name}' in category '{category_name}' for location '{location}'")

app.py

@@ -0,0 +1,151 @@
+from fastapi import FastAPI, HTTPException,Response
+from pydantic import BaseModel
+import uvicorn
+from Mongo.mongo_one import fetch_restaurant_data, save_data_to_db
+from Mongo.mongo_final import fetch_restaurant_links,save_data_to_db,process_links
+from Excel.excel_final import Excel_final
+from Mongo.Noonfood_mongo_Single_URL import get_restaurant_details
+from Mongo.Noonfood_location import mutiple_url_location
+from fastapi.responses import StreamingResponse
+import zipfile
+import os
+from Excel.Noonfood_excel import process_url
+import io 
+from Mongo.Deliveroo_excel import Deliveroo_excel_multy
+from Mongo.Noonfood_multy import Noonfood_multy_urls
+from starlette.responses import RedirectResponse
+
+# FastAPI app
+app = FastAPI()
+
+# Pydantic model for input validation
+class URLRequest(BaseModel):
+    location: str
+    url: list
+class LocationRequest(BaseModel):
+    city: str
+    location: str
+class Excel_From_URL(BaseModel):
+    url:str
+
+
+class FetchAndStoreRequest(BaseModel):
+    latitude: int
+    longitude: int
+
+class GenerateAndDownload(BaseModel):
+    latitude:int
+    longitude : int
+    url:list
+
+class RestaurantDetailsRequest(BaseModel):
+    latitude: float
+    longitude: float
+    url: str
+
+
+@app.get("/")
+def Docsmain():
+    return RedirectResponse(url="/docs")
+
+
+
+@app.post("/Deliveroo_Excel_From_URL",tags=["Deliveroo"])
+def Excel_From_URL(request: Excel_From_URL):
+
+    try:
+        output, filename = Excel_final(request.url)
+        headers = {
+            'Content-Disposition': f'attachment; filename="{filename}"'
+        }
+        return Response(content=output.getvalue(), media_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', headers=headers)
+    except HTTPException as e:
+        raise e
+    except Exception as e:  
+        raise HTTPException(status_code=500, detail=str(e))
+
+@app.post("/MongoDB_Data_Store__From_One_URL",tags=["Deliveroo"])
+def MongoDB_Data_Store__From_One_URL(request: URLRequest):
+    location = request.location.lower()
+    url = request.url
+
+    print(f"Processing restaurant data from URL: {url}...")
+    data = Deliveroo_excel_multy(url, location)
+
+    if not data:
+        raise HTTPException(status_code=404, detail="No data found for the specified URL.")
+
+    print("Saving data to MongoDB...")
+    save_data_to_db(data, location)
+
+    print("Data has been processed and saved successfully.")
+    return {"message": "Data has been processed and saved successfully."}
+
+@app.post("/MongoDB_Data_Store__From_Location",tags=["Deliveroo"])
+def MongoDB_Data_Store__From_Location(request: LocationRequest):
+    city = request.city.lower()
+    location = request.location.lower()
+    print(f"Fetching restaurant links for {city}, {location}...")
+
+    links = fetch_restaurant_links(city, location)
+    if not links:
+        raise HTTPException(status_code=404, detail="No restaurants found for the specified location.")
+
+    print(f"Found {len(links)} links. Processing restaurant data...")
+    data = process_links(links, location)
+
+    print("Saving data to MongoDB...")
+    save_data_to_db(data, location)
+
+    print("Data has been processed and saved successfully.")
+    return {"message": "Data has been processed and saved successfully."}
+
+
+
+@app.post("/Mongo_From_Url",tags=["Noonfood"])
+def Mongo_url_data(request :GenerateAndDownload):
+    latitude = request.latitude
+    longitude = request.longitude
+    url=request.url
+    try:
+        data =Noonfood_multy_urls(latitude,longitude,url)
+        return {"message": "Extraction process completed successfully", "data": data}
+    except HTTPException as e:
+        raise e
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+    
+
+
+@app.post("/MONGO_DB_FROM_LOCATION",tags=["Noonfood"])
+def MOngo_DB_LOCATION(request: FetchAndStoreRequest):
+    latitude = request.latitude
+    longitude = request.longitude
+
+    try:
+        data =mutiple_url_location(latitude,longitude)
+        return {"message": "Extraction process completed successfully", "data": data}
+    except HTTPException as e:
+        raise e
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+    
+
+@app.post("/Noon_Food_EXCEL",tags=["Noonfood"], response_class=StreamingResponse)
+def Noon_Food_EXCEL(details: RestaurantDetailsRequest):
+    files = []
+    try:
+        output, filename = process_url(details.url, details.latitude, details.longitude)
+        headers = {
+            'Content-Disposition': f'attachment; filename="{filename}"'
+        }
+        return Response(content=output.getvalue(), media_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', headers=headers)
+    except HTTPException as e:
+        raise e
+    except Exception as e:  
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=8000)

requirements.txt

@@ -0,0 +1,11 @@
+mongoengine
+requests
+beautifulsoup4
+certifi
+fastapi
+requests
+beautifulsoup4
+pandas
+openpyxl
+pydantic
+uvicorn