sketch.js

// Alan Ren @NYU ITP
// Spring 2024

// Global variables for the detector, image elements, and status paragraph
let detector;
let imgElement,
	defaultImg1,
	defaultImg1DataURL,
	defaultImg2,
	defaultImg2DataURL;
let statusP;

// Preload function to load the default image and convert it to a base64 data URL
function preload() {
	defaultImg1 = loadImage('https://cors-anywhere-ajr.up.railway.app/https://hdwallpaperim.com/wp-content/uploads/2017/08/25/452511-street-street_view-cityscape-city.jpg', (img) => {
		// Create an offscreen graphics object to draw the image for conversion
		let offscreenGraphics = createGraphics(img.width, img.height);
		offscreenGraphics.image(img, 0, 0);
		// Convert the drawn image on canvas to Data URL (base64)
		defaultImg1DataURL = offscreenGraphics.elt.toDataURL();
	});

	defaultImg2 = loadImage('https://cors-anywhere-ajr.up.railway.app/https://external-content.duckduckgo.com/iu/?u=https%3A%2F%2Fdirectoffice.com%2Fwp-content%2Fuploads%2F2024%2F06%2FOffice-Storage-Cabinets-2048x1366.jpg&f=1&nofb=1&ipt=f6b591aded06e6451a47bac25c3b6fe04aa285fb7796c7bc35b881e063a07a5f&ipo=images', (img) => {
		// Create an offscreen graphics object to draw the image for conversion
		let offscreenGraphics = createGraphics(img.width, img.height);
		offscreenGraphics.image(img, 0, 0);
		// Convert the drawn image on canvas to Data URL (base64)
		defaultImg2DataURL = offscreenGraphics.elt.toDataURL();
	});
}

// Setup function initializes the p5 sketch
function setup() {
	// Create a paragraph element to show status messages
	statusP = createP('Loading model...').style('color', 'blue');

	// Check if the transformersPipeline is loaded and available
	if (typeof window.transformersPipeline === 'undefined') {
		console.error('Transformers pipeline not available!');
		statusP.html('Pipeline loading failed.');
		return;
	}

	// Initialize the object detection model using the transformersPipeline
	window
		.transformersPipeline('object-detection', 'Xenova/yolos-tiny', {
			device: 'webgpu',
		})
		.then((d) => {
			detector = d;
			statusP.html('Model ready. Upload an image.');
		})
		.catch((error) => {
			console.error('Error loading the model: ', error);
			statusP.html('Model loading failed.');
		});

	// Create a file input for image upload
	createFileInput(imageUploaded).attribute('accept', 'image/*');
	createButton('Example 1').mousePressed(loadImg1);
	createButton('Example 2').mousePressed(loadImg2);
}

function loadImg1() {
	if (imgElement) {
		imgElement.remove();
	}
	// Initialize the imgElement with the preloaded default image #2
	imgElement = createImg(defaultImg1DataURL, '').hide();
	// Immediately perform detection on the default image
	detect(imgElement);
}

function loadImg2() {
	if (imgElement) {
		imgElement.remove();
	}
	// Initialize the imgElement with the preloaded default image #2
	imgElement = createImg(defaultImg2DataURL, '').hide();
	// Immediately perform detection on the default image
	detect(imgElement);
}

// Function to handle uploaded images
function imageUploaded(file) {
	if (file.type === 'image') {
		if (imgElement) {
			imgElement.remove(); // Remove the previous image if exists
		}
		// Create an image element with the uploaded file
		imgElement = createImg(file.data, '').hide();
		// Detect objects in the uploaded image
		detect(imgElement);
	} else {
		statusP.html('Please upload an image file.');
	}
}

// Function to perform object detection
async function detect(image) {
	statusP.html('Analysing...');
	const results = await detector(image.elt.src, {
		threshold: 0.5,
		percentage: true,
	});

	// Display the image and bounding boxes once analysis is complete
	displayImageAndBoxes(image, results);
	statusP.html('Image Processed');
}

// Function to display the image and bounding boxes on the canvas
function displayImageAndBoxes(img, results) {
	// Calculate scale to fit the image to the screen width while maintaining aspect ratio
	let scaleX = windowWidth / img.width;
	let scaleY = scaleX; // Maintain aspect ratio

	// Adjust the canvas to fit the scaled image
	let scaledWidth = img.width * scaleX;
	let scaledHeight = img.height * scaleY;

	// Resize the canvas to fit the scaled image and provide space for status
	resizeCanvas(windowWidth, scaledHeight + 200);

	// Draw the scaled image on the canvas
	image(img, 0, 100, scaledWidth, scaledHeight);

	// Iterate over each result to draw bounding boxes and labels
	for (const result of results) {
		const { box, label } = result;
		const { xmax, xmin, ymax, ymin } = box;

		// Calculate coordinates and dimensions for scaled bounding boxes
		let rectX = xmin * scaledWidth;
		let rectY = ymin * scaledHeight + 100; // Offset by 100 pixels from top
		let rectWidth = (xmax - xmin) * scaledWidth;
		let rectHeight = (ymax - ymin) * scaledHeight;

		// Draw bounding box
		stroke(255, 0, 0);
		noFill();
		rect(rectX, rectY, rectWidth, rectHeight);

		// Draw label above the bounding box
		fill(255);
		strokeWeight(1);
		textSize(16);
		text(label, rectX, rectY - 10);
	}
}

// The draw function is left empty because updates only occur during detection
function draw() {
	// Leave this function empty if updates only occur during detection
}