Update index.js

index.js

@@ -6,16 +6,32 @@ const { minifyHtml } = require('./minify');
 const { removeMedia } = require('./removeMedia');
 
 const app = express();
-const upload = multer();
 
-app.use(express.static('public'));
-app.use(bodyParser.json());
-app.use(bodyParser.urlencoded({ extended: true }));
-// Increase JSON body parser limit
-app.use(express.json({limit: '50mb'}));
-// If you're also using URL-encoded parser
-app.use(express.urlencoded({limit: '50mb', extended: true}));
+// Configure size limits
+const MAX_SIZE = '50mb';
+
+// Configure multer with size limits
+const upload = multer({
+  limits: {
+    fileSize: 50 * 1024 * 1024, // 50MB limit
+    fieldSize: 50 * 1024 * 1024 // 50MB limit for fields
+  }
+});
 
+// Configure body parsers with consistent limits
+app.use(express.static('public'));
+app.use(bodyParser.json({limit: MAX_SIZE}));
+app.use(bodyParser.urlencoded({
+  extended: true,
+  limit: MAX_SIZE,
+  parameterLimit: 50000
+}));
+app.use(express.json({limit: MAX_SIZE}));
+app.use(express.urlencoded({
+  limit: MAX_SIZE,
+  extended: true,
+  parameterLimit: 50000
+}));
 
 function compressHtmlForLlm(html, options = {}) {
   const operationStatus = {
@@ -87,24 +103,23 @@ function compressHtmlForLlm(html, options = {}) {
     }
   }
 
-
-// Step 5: Remove media
-if (options.removeMedia) {
+  // Step 5: Remove media
+  if (options.removeMedia) {
     try {
-    const mediaResult = removeMedia($);
-    if (mediaResult.success) {
+      const mediaResult = removeMedia($);
+      if (mediaResult.success) {
         operationStatus.mediaRemoval.success = true;
-    } else {
+      } else {
         operationStatus.mediaRemoval.error = mediaResult.error.substring(0, 100);
         console.warn('Media removal failed:', mediaResult.error);
-    }
+      }
     } catch (err) {
-    operationStatus.mediaRemoval.error = err.message.substring(0, 100);
-    console.warn('Media removal failed:', err);
+      operationStatus.mediaRemoval.error = err.message.substring(0, 100);
+      console.warn('Media removal failed:', err);
     }
-}
+  }
 
-  // Step 5: Clean head
+  // Step 6: Clean head
   if (options.cleanHead) {
     try {
       cleanHead($);
@@ -114,7 +129,7 @@ if (options.removeMedia) {
     }
   }
 
-  // Step 6: Handle repeating elements
+  // Step 7: Handle repeating elements
   if (options.handleRepeatingElements) {
     try {
       handleRepeatingElements($);
@@ -124,7 +139,7 @@ if (options.removeMedia) {
     }
   }
 
-  // Step 7: Truncate text
+  // Step 8: Truncate text
   if (options.truncateText) {
     try {
       truncateText($, options.truncateLength);
@@ -292,6 +307,47 @@ function computeStats(html, processed) {
   }
 }
 
+function validateScript(scriptContent) {
+  if (!scriptContent.includes('function extract(')) {
+    throw new Error('Script must contain a function named "extract"');
+  }
+}
+
+function executeCheerioScript(html, scriptContent) {
+  try {
+    validateScript(scriptContent);
+    
+    const context = {
+      cheerio,
+      input: html
+    };
+
+    const extractorFunction = new Function('input', 'cheerio', `
+      ${scriptContent}
+      return extract(input, cheerio);
+    `);
+
+    const result = extractorFunction(html, cheerio);
+
+    if (!result || typeof result !== 'object') {
+      throw new Error('Extract function must return an object');
+    }
+
+    if (!('success' in result && 'data' in result && 'error' in result)) {
+      throw new Error('Return object must contain success, data, and error fields');
+    }
+
+    return result;
+
+  } catch (err) {
+    return {
+      success: false,
+      data: null,
+      error: err.message
+    };
+  }
+}
+
 app.post('/process', upload.single('htmlFile'), (req, res) => {
   try {
     const startTime = Date.now();
@@ -315,7 +371,6 @@ app.post('/process', upload.single('htmlFile'), (req, res) => {
     };
 
     const processed = compressHtmlForLlm(htmlContent, options);
-    
     const stats = computeStats(htmlContent, processed.html);
 
     return res.json({
@@ -343,55 +398,6 @@ app.post('/process', upload.single('htmlFile'), (req, res) => {
   }
 });
 
-// Helper function to validate script structure
-function validateScript(scriptContent) {
-  // Check if script contains function declaration
-  if (!scriptContent.includes('function extract(')) {
-    throw new Error('Script must contain a function named "extract"');
-  }
-}
-
-// Modified function to execute the extract function
-function executeCheerioScript(html, scriptContent) {
-  try {
-    validateScript(scriptContent);
-    
-    // Create a safe context for the script
-    const context = {
-      cheerio,
-      input: html
-    };
-
-    // Create the function from script content
-    const extractorFunction = new Function('input', 'cheerio', `
-      ${scriptContent}
-      return extract(input, cheerio);
-    `);
-
-    // Execute the script with provided parameters
-    const result = extractorFunction(html, cheerio);
-
-    // Validate result structure
-    if (!result || typeof result !== 'object') {
-      throw new Error('Extract function must return an object');
-    }
-
-    if (!('success' in result && 'data' in result && 'error' in result)) {
-      throw new Error('Return object must contain success, data, and error fields');
-    }
-
-    return result;
-
-  } catch (err) {
-    return {
-      success: false,
-      data: null,
-      error: err.message
-    };
-  }
-}
-
-// Updated endpoint
 app.post('/extract', upload.single('htmlFile'), (req, res) => {
   try {
     const startTime = Date.now();