pocket2pdf.js
// Process the supplied URL and dump out files asynchronously

Process the supplied URL and dump out files asynchronously
Process the supplied URL and dump out files asynchronously
async function process_url(url, date, whenDone) {
	const browser = await puppeteer.launch();
	const page = await browser.newPage();
	try {
		await page.goto(url, {
			waitUntil: 'networkidle'
		});
		var title = await page.title();
		var sanitized_title = get_date(date) + ' - ' + sanitize(title);
		if (fs.existsSync(out_dir + '/' + sanitized_title + '.html')) {
			console.log('Saved ' + num_processed_links++ + '/' + num_total_links + ' (' + num_error_links + ' failed) (SKIP):', sanitized_title);
			browser.close();
			whenDone(); 
			return;
		} else {
			await page.pdf({
				path: out_dir + '/' + sanitized_title + '.raw.pdf',
				format: 'Letter',
				displayHeaderFooter: true,
				margin: {
					top: '1in',
					right: '1in',
					bottom: '1in',
					left: '1in'
				}
			});
			fs.writeFile(out_dir + '/' + sanitized_title + '.txt', await page.plainText(), function () {});
			reader(url, function (err, article, meta) {
				if (article == undefined) {
					return;
				}
				if (article.content.length > 0) {
					fs.writeFile(out_dir + '/' + sanitized_title + '.html', article.content, function () {});
					if (false) {
						pdf.create(article.content, {
							format: 'Letter'
						}).toFile(out_dir + '/' + sanitized_title + '.pdf', function (err, res) {
							if (err) {
								console.log('Error writing PDF:', out_dir + '/' + sanitized_title + '.pdf : ', err)
							};
							console.log('Done writing PDF:', out_dir + '/' + sanitized_title + '.pdf');
							article.close();
						});
					}
				} else {
					console.log('ERROR: Invalid length for cleaned up article:', article.content.length);
					article.close();
				}
			});
			fs.writeFile(out_dir + '/' + date + '.done', '', function () {});
			console.log('Saved ' + num_processed_links++ + '/' + num_total_links + ' (' + num_error_links + ' failed) :', sanitized_title);
			browser.close();
			whenDone(); 
		}
	} catch (err) {
		console.log('Failed to retrieve URL:', url);
		num_error_links++;
		whenDone();
	};
}
async function process_url ( url , date , whenDone ) { const browser = await puppeteer . launch ( ) ; const page = await browser . newPage ( ) ; try { await page . goto ( url , { waitUntil : 'networkidle' } ) ; var title = await page . title ( ) ; var sanitized_title = get_date ( date ) + ' - ' + sanitize ( title ) ; if ( fs . existsSync ( out_dir + '/' + sanitized_title + '.html' ) ) { console . log ( 'Saved ' + num_processed_links ++ + '/' + num_total_links + ' (' + num_error_links + ' failed) (SKIP):' , sanitized_title ) ; browser . close ( ) ; whenDone ( ) ; return ; } else { await page . pdf ( { path : out_dir + '/' + sanitized_title + '.raw.pdf' , format : 'Letter' , displayHeaderFooter : true , margin : { top : '1in' , right : '1in' , bottom : '1in' , left : '1in' } } ) ; fs . writeFile ( out_dir + '/' + sanitized_title + '.txt' , await page . plainText ( ) , function ( ) { } ) ; reader ( url , function ( err , article , meta ) { if ( article == undefined ) { return ; } if ( article . content . length > 0 ) { fs . writeFile ( out_dir + '/' + sanitized_title + '.html' , article . content , function ( ) { } ) ; if ( false ) { pdf . create ( article . content , { format : 'Letter' } ) . toFile ( out_dir + '/' + sanitized_title + '.pdf' , function ( err , res ) { if ( err ) { console . log ( 'Error writing PDF:' , out_dir + '/' + sanitized_title + '.pdf : ' , err ) } ; console . log ( 'Done writing PDF:' , out_dir + '/' + sanitized_title + '.pdf' ) ; article . close ( ) ; } ) ; } } else { console . log ( 'ERROR: Invalid length for cleaned up article:' , article . content . length ) ; article . close ( ) ; } } ) ; fs . writeFile ( out_dir + '/' + date + '.done' , '' , function ( ) { } ) ; console . log ( 'Saved ' + num_processed_links ++ + '/' + num_total_links + ' (' + num_error_links + ' failed) :' , sanitized_title ) ; browser . close ( ) ; whenDone ( ) ; } } catch ( err ) { console . log ( 'Failed to retrieve URL:' , url ) ; num_error_links ++ ; whenDone ( ) ; } ; }
Process the supplied URL and dump out files asynchronously
Process the supplied URL and dump out files asynchronously
// If the URL was already captured, skip it
 // You MUST call this to signal to async that your operation is done
 // Store the raw PDF of the page
 // Store the plain text content
 // Store the cleaned up webpage
 // Reject empty cleaned up articles
 // Store the article content as HTML
 // Skip PDF generation from HTML for now - broken on Windows
 // Main Article 
 // console.log('Cleaned up article length:', article.content.length);
 // Close article to clean up jsdom and prevent leaks
 // Close article to clean up jsdom and prevent leaks 
 // Create a touchfile with the epoch date to allow faster skips in future runs
 // You MUST call this to signal to async that your operation is done
