Spaces:
Running
on
CPU Upgrade
Running
on
CPU Upgrade
Fixes "invalid url" error in websearch (#675)
Browse files* Fixed #656
* lint
---------
Co-authored-by: muhammadbaqir1327 <[email protected]>
Co-authored-by: Nathan Sarrazin <[email protected]>
src/lib/server/websearch/runWebSearch.ts
CHANGED
|
@@ -48,11 +48,17 @@ export async function runWebSearch(
|
|
| 48 |
webSearch.results =
|
| 49 |
(results.organic_results &&
|
| 50 |
results.organic_results.map((el: { title?: string; link: string; text?: string }) => {
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 54 |
})) ??
|
| 55 |
[];
|
|
|
|
| 56 |
webSearch.results = webSearch.results
|
| 57 |
.filter(({ link }) => !DOMAIN_BLOCKLIST.some((el) => link.includes(el))) // filter out blocklist links
|
| 58 |
.slice(0, MAX_N_PAGES_SCRAPE); // limit to first 10 links only
|
|
|
|
| 48 |
webSearch.results =
|
| 49 |
(results.organic_results &&
|
| 50 |
results.organic_results.map((el: { title?: string; link: string; text?: string }) => {
|
| 51 |
+
try {
|
| 52 |
+
const { title, link, text } = el;
|
| 53 |
+
const { hostname } = new URL(link);
|
| 54 |
+
return { title, link, hostname, text };
|
| 55 |
+
} catch (e) {
|
| 56 |
+
// Ignore Errors
|
| 57 |
+
return null;
|
| 58 |
+
}
|
| 59 |
})) ??
|
| 60 |
[];
|
| 61 |
+
webSearch.results = webSearch.results.filter((value) => value !== null);
|
| 62 |
webSearch.results = webSearch.results
|
| 63 |
.filter(({ link }) => !DOMAIN_BLOCKLIST.some((el) => link.includes(el))) // filter out blocklist links
|
| 64 |
.slice(0, MAX_N_PAGES_SCRAPE); // limit to first 10 links only
|