ND-Scrapper / app.py
viraj
Initial Commit
9e6917b
from fastapi import FastAPI, HTTPException,Response
from pydantic import BaseModel
import uvicorn
from Mongo.mongo_one import fetch_restaurant_data, save_data_to_db
from Mongo.mongo_final import fetch_restaurant_links,save_data_to_db,process_links
from Excel.excel_final import Excel_final
from Mongo.Noonfood_mongo_Single_URL import get_restaurant_details
from Mongo.Noonfood_location import mutiple_url_location
from fastapi.responses import StreamingResponse
import zipfile
import os
from Excel.Noonfood_excel import process_url
import io
from Mongo.Deliveroo_excel import Deliveroo_excel_multy
from Mongo.Noonfood_multy import Noonfood_multy_urls
from starlette.responses import RedirectResponse
# FastAPI app
app = FastAPI()
# Pydantic model for input validation
class URLRequest(BaseModel):
location: str
url: list
class LocationRequest(BaseModel):
city: str
location: str
class Excel_From_URL(BaseModel):
url:str
class FetchAndStoreRequest(BaseModel):
latitude: int
longitude: int
class GenerateAndDownload(BaseModel):
latitude:int
longitude : int
url:list
class RestaurantDetailsRequest(BaseModel):
latitude: float
longitude: float
url: str
@app.get("/")
def Docsmain():
return RedirectResponse(url="/docs")
@app.post("/Deliveroo_Excel_From_URL",tags=["Deliveroo"])
def Excel_From_URL(request: Excel_From_URL):
try:
output, filename = Excel_final(request.url)
headers = {
'Content-Disposition': f'attachment; filename="{filename}"'
}
return Response(content=output.getvalue(), media_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', headers=headers)
except HTTPException as e:
raise e
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@app.post("/MongoDB_Data_Store__From_One_URL",tags=["Deliveroo"])
def MongoDB_Data_Store__From_One_URL(request: URLRequest):
location = request.location.lower()
url = request.url
print(f"Processing restaurant data from URL: {url}...")
data = Deliveroo_excel_multy(url, location)
if not data:
raise HTTPException(status_code=404, detail="No data found for the specified URL.")
print("Saving data to MongoDB...")
save_data_to_db(data, location)
print("Data has been processed and saved successfully.")
return {"message": "Data has been processed and saved successfully."}
@app.post("/MongoDB_Data_Store__From_Location",tags=["Deliveroo"])
def MongoDB_Data_Store__From_Location(request: LocationRequest):
city = request.city.lower()
location = request.location.lower()
print(f"Fetching restaurant links for {city}, {location}...")
links = fetch_restaurant_links(city, location)
if not links:
raise HTTPException(status_code=404, detail="No restaurants found for the specified location.")
print(f"Found {len(links)} links. Processing restaurant data...")
data = process_links(links, location)
print("Saving data to MongoDB...")
save_data_to_db(data, location)
print("Data has been processed and saved successfully.")
return {"message": "Data has been processed and saved successfully."}
@app.post("/Mongo_From_Url",tags=["Noonfood"])
def Mongo_url_data(request :GenerateAndDownload):
latitude = request.latitude
longitude = request.longitude
url=request.url
try:
data =Noonfood_multy_urls(latitude,longitude,url)
return {"message": "Extraction process completed successfully", "data": data}
except HTTPException as e:
raise e
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@app.post("/MONGO_DB_FROM_LOCATION",tags=["Noonfood"])
def MOngo_DB_LOCATION(request: FetchAndStoreRequest):
latitude = request.latitude
longitude = request.longitude
try:
data =mutiple_url_location(latitude,longitude)
return {"message": "Extraction process completed successfully", "data": data}
except HTTPException as e:
raise e
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@app.post("/Noon_Food_EXCEL",tags=["Noonfood"], response_class=StreamingResponse)
def Noon_Food_EXCEL(details: RestaurantDetailsRequest):
files = []
try:
output, filename = process_url(details.url, details.latitude, details.longitude)
headers = {
'Content-Disposition': f'attachment; filename="{filename}"'
}
return Response(content=output.getvalue(), media_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', headers=headers)
except HTTPException as e:
raise e
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
if __name__ == "__main__":
uvicorn.run(app, host="0.0.0.0", port=8000)