uploaded all project files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+example.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,82 @@
+---
+language:
+  - en
+base_model:
+  - stabilityai/stable-diffusion-2-1
+tags:
+  - Beam-Search
+  - stable-diffusion
+  - Diffusers
+  - Latent-Space
+pipeline_tag: text-to-image
+---
+# BeamDiffusion: Latent Beam Diffusion Models for Decoding Image Sequences
+**BeamDiffusion** introduces a novel approach for generating coherent image sequences from text prompts by employing beam search in latent space. Unlike traditional methods that generate images independently, BeamDiffusion iteratively explores latent representations, ensuring smooth transitions and visual continuity across frames. A cross-attention mechanism efficiently scores and prunes search paths, optimizing both textual alignment and visual coherence.
+BeamDiffusion addresses the challenge of maintaining visual consistency in image sequences generated from text prompts. By leveraging a beam search strategy in the latent space, it refines the generation process to produce sequences with enhanced coherence and alignment with textual descriptions, as outlined in the [paper](https://arxiv.org/abs/2503.20429).
+
+## Quickstart Guide
+
+Here's a basic example of how to use BeamDiffusion with the `transformers` library to generate an image sequence based on a series of text prompts:
+
+```python
+from huggingface_hub import snapshot_download
+# Download the model snapshot
+snapshot_download(repo_id="Gui28F/BeamDiffusion", local_dir="BeamDiffusionModel")
+from BeamDiffusionModel.beam_diffusion import BeamDiffusionPipeline, BeamDiffusionConfig,BeamDiffusionModel
+
+# Initialize the configuration, model, and pipeline
+config = BeamDiffusionConfig()
+model = BeamDiffusionModel(config)
+pipe = BeamDiffusionPipeline(model)
+                             
+# Define the input parameters
+input_data = {
+    "steps": ["A lively outdoor celebration with guests gathered around, everyone excited to support the event.",
+              "A chef in a cooking uniform raises one hand dramatically, signaling it's time to serve the food.",
+              "Guests chat and laugh in a vibrant setting, with people gathered around tables, enjoying the event."],
+    "latents_idx": [0, 1, 2, 3], 
+    "n_seeds": 4,
+    "steps_back": 2,
+    "beam_width": 2,
+    "window_size": 2,
+    "use_rand": True
+}
+
+# Generate the sequence of images
+sequence_imgs = pipe(input_data)
+```
+
+**Result:**
+
+![Generated Image Sequence](./example.png)
+
+## Input Parameters Explained
+
+- **`steps`** (`list of strings`): Descriptions for each step in the image generation process. The model generates one image per step, forming a sequence that aligns with these descriptions.
+
+- **`latents_idx`** (`list of integers`): Indices referring to specific positions in the latent space to be used during image generation. This allows the model to leverage different latent representations for diverse outputs.
+
+- **`n_seeds`** (`int`): Number of random seeds to use for the generation process. Each seed provides a different starting point for the randomness in the first step, influencing the diversity of generated sequences.
+
+- **`seeds`** (`list of integers`): Specific seeds to use for the generation process. If provided, these seeds override the `n_seeds` parameter, allowing for controlled randomness.
+
+- **`steps_back`** (`int`): Number of previous steps to consider during the beam search process. This parameter helps refine the current generation by incorporating information from earlier steps.
+
+- **`beam_width`** (`int`): Number of candidate sequences to maintain during inference. Beam search evaluates multiple potential outputs and keeps the most probable ones based on the defined criteria.
+
+- **`window_size`** (`int`): Size of the "window" for beam search pruning. Determines after how many steps pruning starts, helping the model focus on more probable options as the generation progresses.
+
+- **`use_rand`** (`bool`): Flag to introduce randomness in the inference process. If set to `True`, the model generates more varied and creative results; if `False`, it produces more deterministic outputs.
+
+## Citation
+
+If you use BeamDiffusion in your research or projects, please cite the following paper:
+
+```
+@article{fernandes2025latent,
+  title={Latent Beam Diffusion Models for Decoding Image Sequences},
+  author={Fernandes, Guilherme and Ramos, Vasco and Cohen, Regev and Szpektor, Idan and Magalh{\~a}es, Jo{\~a}o},
+  journal={arXiv preprint arXiv:2503.20429},
+  year={2025}
+}
+```

beamInference.py

@@ -0,0 +1,71 @@
+import random
+
+from BeamDiffusionModel.models.CoSeD.cross_attention import get_softmax
+from BeamDiffusionModel.models.diffusionModel.configs.config_loader import CONFIG
+from BeamDiffusionModel.tree.tree import BeamSearchTree
+from BeamDiffusionModel.utils.utils import gen_img
+
+
+def set_softmax(nodes, softmax, n_latents, n_max_latents):
+    for node, softmax_value in zip(nodes, softmax):
+        node.set_softmax(softmax_value, n_latents, n_max_latents)
+
+def beam_inference(steps, latents_idx, n_seeds= 1, seeds=[], steps_back=2, beam_width=4, window_size=2, use_rand=True):
+    while len(seeds) < n_seeds:
+        seeds.append(random.randint(0, 10**6))
+    captions = steps
+    tree = BeamSearchTree(steps_back,beam_width,latents_idx,len(captions))
+    nodes_to_explore = []
+    for i, caption in enumerate(captions):
+        if i == 0:
+            for seed in seeds:
+                latents, img = gen_img(caption, seed=seed)
+                new_node = tree.add_node(tree.root, caption, i + 1, "Rand Seed", "Rand Seed",
+                                         img, latents, None)
+                nodes_to_explore.append(new_node)
+        else:
+            next_nodes = []
+            for child, parent_node in enumerate(nodes_to_explore):
+                parent_childs = []
+                current_step_embeddings, current_image_embeddings = [], []
+                if use_rand:
+                    seed = random.randint(0, 10 ** 6)
+                    latents, img  = gen_img(caption, seed=seed)
+                    new_node = tree.add_node(parent_node, caption, i + 1, "Rand Seed", "Rand Seed",
+                                             img, latents, None)
+                    parent_childs.append(new_node)
+                    current_step_embedding, current_image_embedding = new_node.get_features()
+                    current_step_embeddings.append(current_step_embedding)
+                    current_image_embeddings.append(current_image_embedding)
+
+                ancestors = parent_node.get_ancestors(steps_back-1)
+
+                for ancestor_idx, ancestor in enumerate(ancestors):
+                    for latent in latents_idx:
+                        ancestor_latent = ancestor.get_latent(latent)
+                        latents, img = gen_img(caption, latent=ancestor_latent)
+                        new_node = tree.add_node(parent_node, caption, i + 1, ancestor.step, latent,img, latents, None)
+                        parent_childs.append(new_node)
+                        current_step_embedding, current_image_embedding = new_node.get_features()
+                        current_step_embeddings.append(current_step_embedding)
+                        current_image_embeddings.append(current_image_embedding)
+
+                if current_step_embeddings != []:
+                    previous_steps_embeddings, previous_images_embeddings = tree.get_previous_steps_features(parent_childs[-1])
+                    softmax = get_softmax(previous_steps_embeddings, previous_images_embeddings,
+                                          current_step_embeddings,
+                                          current_image_embeddings)
+                    set_softmax(parent_childs, softmax, len(latents_idx), CONFIG["stable_diffusion"]["diffusion_settings"]["steps"])
+                next_nodes += parent_childs
+
+            if i >= window_size:
+                print("-----------------------------------Cleaning some nodes-----------------------------------")
+                best_paths = tree.get_n_best_paths(beam_width, i + 1)
+                new_next_nodes = []
+                for node in next_nodes:
+                    for node_path in best_paths:
+                        if node in node_path:
+                            new_next_nodes.append(node)
+                next_nodes = new_next_nodes
+            nodes_to_explore = next_nodes
+    return tree.best_path_imgs()

beam_diffusion.py

@@ -0,0 +1,65 @@
+from transformers import PretrainedConfig, PreTrainedModel, Pipeline
+import torch
+
+from BeamDiffusionModel.beamInference import beam_inference
+# Your custom configuration for the BeamDiffusion model
+class BeamDiffusionConfig(PretrainedConfig):
+    model_type = "beam_diffusion"
+    def __init__(self, latents_idx=None, n_seeds=4, seeds=None, steps_back=2, beam_width=4, window_size=2, use_rand=True, **kwargs):
+        super().__init__(**kwargs)
+        self.latents_idx = latents_idx if latents_idx else [0, 1, 2, 3]
+        self.n_seeds = n_seeds
+        self.seeds = seeds if seeds else []
+        self.steps_back = steps_back
+        self.beam_width = beam_width
+        self.window_size = window_size
+        self.use_rand = use_rand
+
+import torch.nn as nn
+from huggingface_hub import ModelHubMixin
+# Custom BeamDiffusionModel that performs inference for each step
+class BeamDiffusionModel(PreTrainedModel, ModelHubMixin):
+    config_class = BeamDiffusionConfig
+    model_type = "beam_diffusion"
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.config = config
+        self.dummy_param = nn.Parameter(torch.zeros(1))  # Ensure at least one parameter
+
+    def forward(self, input_data):
+        images = beam_inference(
+            steps=input_data.get('steps', []),
+            latents_idx=input_data.get("latents_idx", [0, 1, 2, 3]),
+            n_seeds=input_data.get("n_seeds", 4),
+            seeds=input_data.get("seeds", []),
+            steps_back=input_data.get("steps_back", 2),
+            beam_width=input_data.get("beam_width", 4),
+            window_size=input_data.get("window_size", 2),
+            use_rand=input_data.get("use_rand", True)
+        )
+        return {"images": images}
+
+
+# Custom pipeline to handle inference
+class BeamDiffusionPipeline(Pipeline, ModelHubMixin):
+    def __init__(self, model, tokenizer=None, device="cuda", framework="pt"):
+        super().__init__(model=model, tokenizer=tokenizer, device=device, framework=framework)
+
+    def __call__(self, inputs):
+        return self._forward(inputs)
+
+    def preprocess(self, inputs):
+        """Converts raw input data into model-ready format."""
+        return inputs  # Keep as-is
+
+    def postprocess(self, model_outputs):
+        """Processes model output into a user-friendly format."""
+        return model_outputs["images"]  # Ensure this matches expected output
+
+    def _sanitize_parameters(self, **kwargs):
+        """Handles unused parameters gracefully."""
+        return {}, {}, {}
+
+    def _forward(self, model_inputs):
+        return self.model(model_inputs)

datasets/DIYs.json

@@ -0,0 +1,173 @@
+{
+  "0": {
+    "title": "How to Clean a Sharpening Stone",
+    "steps": [
+      "Rub a quarter-size amount of honing oil onto your stone. Honing oil is a mineral oil that is commonly used in machine shops to carry away metal flecks from stone. Apply honing oil to your sharpening stone with your finger or a soft toothbrush in small, circular motions. Continue rubbing until you see metal flecks rise up from the pores of the stone.\nHoning oil can also be used to provide lubrication during the sharpening process.\nPurchase honing oil in hardware stores or online.\nUse honing oil to flush out your sharpening stone whenever you see metal flecks embedded in the surface, as they may reduce its effectiveness.",
+      "Wipe away the metal flecks with a damp cloth or paper towel. Wet a clean cloth or paper towel and wring it out. Gently wipe the surface of the sharpening stone with it. Be sure to remove all the metal filings that emerged from the stone.",
+      "Rinse the stone thoroughly under running water and dry it. Washing the sharpening stone will remove any remaining metal filings and debris from your stone. Run the stone under warm water for 1-2 minutes to ensure that it is clean. Dry the stone with a clean, dry cloth or paper towel. \nDo not use soap or anything other than water to clean it."
+    ],
+    "captions": [
+      "A sharpening stone with a small pool of honing oil on its surface.",
+      "A hand holding a damp cloth wiping the surface of a sharpening stone. Metal flecks are visible on the damp cloth.",
+      "A sharpening stone being rinsed under running water."
+    ]
+  },
+  "1": {
+    "title": "How to Clean Silk Rugs",
+    "steps": [
+      "Vacuum silk rugs regularly. Use a brushless suction head to vacuum silk rugs. Beater or roller brushes may pull out fibers, reduce the pile and damage the rug.",
+      "Sweep silk rugs with a broom. Gently sweep the rug to avoid damaging the fibers."
+    ],
+    "captions": [
+      "A person using a vacuum cleaner on a silk rug.",
+      "A person sweeping a silk rug with a broom."
+    ]
+  },
+  "2": {
+    "title": "How to Preserve Basil",
+    "steps": [
+      "Clean the basil leaves and allow to dry.",
+      "Place in a suitable storage container.",
+      "Sprinkle with salt. Shake through to cover in salt.",
+      "Fill the container with olive oil.",
+      "Cover the container. Place in the fridge and leave. Stored in this manner, the basil leaves will remain in great condition for use over the coming months.",
+      "Use as usual according to your recipe. Both the leaves and the oil (now infused with basil) can be used for cooking."
+    ],
+    "captions": [
+      "Basil leaves being washed and dried.",
+      "A storage container filled with basil leaves.",
+      "A storage container filled with basil leaves being sprinkled with salt.",
+      "A storage container with basil leaves being filled with olive oil.",
+      "A storage container with basil leaves in the fridge.",
+      "A pan with chicken and basil leaves being cooked."
+    ]
+  },
+  "3": {
+    "title": "How to Steam Clean Carpet",
+    "steps": [
+      "Remove everything from the room. Steam cleaning is most effective when you remove all toys, papers, and general clutter from the floor. Move all tables, chairs, and furniture out of the room. Clear as much of the floor space as you can.\nIf some furniture is too heavy to move, place squares of wax paper, foil, wood blocks, or plastic film under the legs to protect them from the moisture of the steam cleaner. You can\u2019t clean the carpet completely if you leave furniture in the room.\nIf you don\u2019t have room to move the furniture to another room, move it all to half of the room and do the room in two separate sections. Let the carpet dry before you move furniture back to the already cleaned half.",
+      "Dust the baseboards. As you run the steam cleaner, you might knock dust off the baseboards if you don\u2019t dust them beforehand. Use a dust cloth with wood polish or a long handled duster to remove as much of the dust as you can.\nDust ceiling fans and corners of the ceiling to prevent extra dust from falling on the carpet after you have steam cleaned them.",
+      "Vacuum the entire carpet carefully. Steam cleaners are meant to pull up small dirt particles that are down in the fibers of the carpet. They are not for removing hair and larger dirt pieces. Vacuum the entire room more slowly than you usually would. It is best to go over the room twice to get as much of the larger debris as possible. The second time, go over the room in the opposite direction to pick up more dirt.\nFor extra cleaning, use a nozzle attachment to get right up to the baseboards and edges of the room.\nVacuuming also fluffs up the carpet so that the steamer can clean the fibers more effectively.",
+      "Use stain remover on specific bad spots. Steamers can\u2019t always lift deep, set-in stains, so it\u2019s always helpful to treat stains before steaming the carpet. Use a carpet stain remover, or other natural solutions if you prefer. Blot the stain remover up with a cloth or leave it to be sucked up by the machine.\nWhen you do this, dab at the stains with a cloth. Never scrub at stains as this can cause them to work deeper into the carpet."
+    ],
+    "captions": [
+      "A room with all furniture removed.",
+      "A person dusting the baseboards.",
+      "A person vacuuming a carpet.",
+      "A person using a stain remover on a carpet."
+    ]
+  },
+  "4": {
+    "title": "How to Polish Wood",
+    "steps": [
+      "Wipe down your furniture with the wax remover. Dampen your dry microfiber cloth with the wax remover and wipe along the grain of the wood.  Wait until it is completely dry to ensure that you won\u2019t damage the wood, then wipe off any excess dirt or wax with a dry microfiber cloth.  Use 0000 steel wool to gently sand away any remaining marks or stains.\nIt is important to remove any excess wax buildup before polishing as any remaining wax will degrade your polish.\nTest your wax remover on an inconspicuous area before covering the entire piece of furniture.\nYou can purchase wax remover in any store, but you can also use a mixture of .5 cups (0.12\u00a0L) cups of water to .5 cups (0.12\u00a0L) of white vinegar if you would prefer a homemade option.",
+      "Apply thin layers of polish along the grain of the wood.  Place a dry microfiber cloth on the open cap of polish and flip.  This will allow the polish to absorb into the cloth without applying so much that it will begin pooling on the furniture.  Rub the cloth along the grain of the wood to work in the polish.\nYou can continue to apply layers depending on how dry the furniture is and the level of shine you would like to achieve.\nMake sure to get in all crevices and corners.  Open cabinets or drawers to polish the joints and interior spaces.\nBe sure to test the polish in an inconspicuous area before covering the entire piece of furniture.",
+      "Repeat the polishing process as needed. Once complete, your furniture should be glossy and shiny, but you can continue adding layers to achieve the finish you want.  You can repeat the polishing process on a routine basis to keep your furniture looking great, but unless you apply a new layer of wax you won\u2019t need to use wax remover again."
+    ],
+    "captions": [
+      "A person wiping down a piece of wood furniture with a wax remover.",
+      "A person applying polish to a piece of wood furniture.",
+      "A person polishing a piece of wood furniture."
+    ]
+  },
+  "5": {
+    "title": "How to Make a Door",
+    "steps": [
+      "Measure the doorway where you\u2019ll be installing your door. Before you get to sawing, gluing, and sanding, you\u2019ll need to know exactly how big your door needs to be. Find the height and width of your empty doorway by running a tape measure down one side, then stretching it across the top. Make sure you jot your measurements down. You\u2019ll need to refer back to them later while you\u2019re cutting the panel for your door.",
+      "Acquire a 4 ft (1.2 m) x 8 ft (2.4 m) sheet of 1\u20442 in (1.3 cm) plywood. Run down to your local hardware store or home improvement center and purchase a sheet of plywood to use as the main panel for your new door. For a standard interior door, 1\u20442 in (1.3 cm) plywood will work best. A solid piece of plywood will create a door with a much sturdier construction than the hollow-bodied variety installed in most homes.",
+      "Mark the doorway dimensions on your plywood sheet with a pencil. Using the measurements you recorded earlier, draw one line down the length of the plywood corresponding to the height of the doorway and another line across the top to indicate the width. This will create a rough outline for your door panel. Use a ruler or straight edge to confirm that your lines are straight and precise. Otherwise, you could end up with a door that doesn\u2019t fit!",
+      "Cut the plywood to the correct dimensions using a circular saw. Guide the saw blade slowly over the plywood sheet along the height and width lines you just drew to trim the excess material from the edges. Depending on how your work bench is set up, you\u2019ll need to reposition either the plywood or the saw when it comes time to make your second cut. Laying a separate piece of lumber along your measurement lines will ensure cleaner cuts and help prevent mistakes. The standard size of an interior door is 80 inches (200 cm) x 24\u201330 inches (61\u201376 cm).",
+      "Sand the entire door panel. Run an electric sander or sheet of high-grit sandpaper over both sides of the panel, using even pressure to ensure that the outer surface is smooth and level. Once you\u2019ve sanded both faces, turn your attention to the edges of the panel. You may have to clamp or brace the panel against another object to hold it steady while you sand the edges. At this point, you can either move on to painting and installing the mounting hardware if you\u2019re satisfied with a plain, flat door, or cut a few more pieces of plywood to add textural accents to your blank panel."
+    ],
+    "captions": [
+      "A person holding a tape measure against an empty doorway. Numbers are visible on the tape measure.",
+      "A stack of plywood sheets at a hardware store with a person reaching for one.",
+      "A sheet of plywood lying flat with a pencil and ruler beside it. Two straight lines are drawn on the plywood with measurement markings next to them.",
+      "A person using a circular saw to cut along a marked line on a sheet of plywood. Sparks are flying from the saw blade. A straight piece of lumber is positioned next to the cut line as a guide.",
+      "A person using an electric sander on a flat plywood door panel. Dust is visible in the air."
+    ]
+  },
+  "6": {
+    "title": "How to Clean Natural Stone",
+    "steps": [
+      "Sweep off surface debris. If you are cleaning a natural stone floor, you'll want to use a fluffy mop or brush that has soft, natural bristles. This will prevent the stone from getting scratched by brushes with abrasive bristles. If you are cleaning natural stone countertops, a clean cotton rag or a handheld duster can remove surface debris. Sweep all the dirt and dust into a dustpan and dispose of it.\nIf you are cleaning natural stone floors, you can also use a vacuum.\nSweep or dust your natural stone at least twice a week.",
+      "Mix warm water and a mild dish detergent in a bucket. Fill up a bucket with five cups (1.1 l) of warm water from your faucet and put two to three drops of a mild, pH neutral dish soap into the bucket. Mix the water and soap together until bubbles start to form on the top. Read the label of the dish soap that you are purchasing to make sure that it doesn't contain any acid, which could damage the natural stone.\nAlcohol, citrus, and vinegar all contain acids that are corrosive to natural stone.\nYou can also ask a stone retailer to direct you towards a stone-specific cleaner.",
+      "Wipe down the natural stone with the solution. Use a soft mop or a rag and dip it into the solution that you just created. Work in circular motions to lift up any stuck on soap scum, dirt, or gunk that is on your stone. If you are using a mop, go left and right in large sweeping motions to remove dirt and gunk from the floor.",
+      "Rinse off the solution with distilled water. Rinse the stone with distilled water until there is no more soap or cleaning solution left on the stone. Regular water from your tap may contain minerals that could discolor the stone. The cleaner can also be absorbed by the stone and cause discoloration, so make sure that you rinse it thoroughly.\nIf you don't have distilled water, you can boil tap water instead.",
+      "Dry the stone. Dry the stone with a soft, synthetic or cotton rag until all of the moisture is gone. Do not leave any cleaner or liquid on the surface of the stone. Continue to buff the area until your natural stone looks shiny and clean."
+    ],
+    "captions": [
+      "A person using a soft-bristled mop to sweep dust and debris off a natural stone floor.",
+      "A bucket filled with warm water and a few drops of dish soap. Bubbles are visible on the water's surface.",
+      "A person using a soft cloth dipped in cleaning solution to wipe a natural stone countertop in circular motions.",
+      "A person pouring distilled water from a watering can onto a natural stone surface.",
+      "A person buffing a natural stone surface with a dry, clean cloth."
+    ]
+  },
+  "7": {
+    "title": "How to Frame an Oil Painting",
+    "steps": [
+      "Select a floating or baguette frame for a minimalistic, modern look. Baguette frames refer to flat panels of wood or metal that fit along the edges of your painting without obscuring any portion of the image. Float frames use a mat to set your image inside a simple frame. These frames provide a modern look, and differ from traditional frames by adding very little visual impact on their own. They look sort of like siding or open borders for a piece of art.\nFloat and baguette frames tend to emphasize the work itself, since the frames tend to contain few details on their own.\nFloat frames can make a work appear more striking. The mat helps isolate the work in the center of the wall, making it stand out more.\nBaguette frames can make a work look dramatic. The minimalistic frame lifts the image off of the wall a few inches, which can make it seem like the work is literally popping out.\nThe difference between a baguette and float frame is whether there is any space between the canvas and the frame itself. Float frames have a space, which is sometimes filled in with a mat, while baguette frames leave no room between the frame and the painting.",
+      "Use a traditional frame for classic landscapes and portraits. Traditional frames come in all types of shapes, sizes, and designs. They tend to be metal or wood, and add an additional visual element to a painting\u2019s presentation. Choose a frame that you think works well alongside your image without drawing the viewer\u2019s attention away from what\u2019s important.\nTraditional frames cover the edges of a canvas. They also tend to have minor details and patterns built into the frame. These minor elements change the way that a viewer perceives a work.\nWhen in doubt, opt for a simpler traditional frame. You can always upgrade the frame later when you need a change of pace.",
+      "Get an antique frame for high-end or older oil paintings. For older works or paintings with a high price tag, choose an antique frame to really make your work stand out. Antique frames tend to have high levels of detail and create a striking impression on their own, which can demand attention as soon as the viewer enters the room.\nAntique frames tend to be have expressive designs and strange colors. These can help stress the specialness of a historical or expensive piece of art.\nLook for antique frames at antique shops.",
+      "Display your art without a frame if the sides of the canvas are painted. If the canvas comes with a prebuilt frame and the artist elected to paint details along the sides of the canvas, you probably don\u2019t need a frame. The artist made a conscious decision to extend the image past the edge of the canvas, and covering it could reduce the impact of an image.\nYou can tell if a framed canvas was meant to be hung on its own if the artist installed hangers along the backside of the frame."
+    ],
+    "captions": [
+      "A modern oil painting with a float frame. A thin border surrounds the painting, creating a clean and contemporary look.",
+      "A traditional oil painting (e.g., landscape or portrait) with a decorative wooden frame. The frame complements the artwork's style without overwhelming it.",
+      "An antique oil painting with an ornate frame. The frame features rich details and flourishes, enhancing the prestige of the artwork.",
+      "A close-up of a painted canvas edge. The sides of the canvas continue the artwork, creating a seamless flow from the front."
+    ]
+  },
+  "8": {
+    "title": "How to Remove Oil Stains from Paper",
+    "steps": [
+      "Create a cleaning solution with half plain water, half white vinegar. Combine 1\u20442\u00a0\u00a0c (120\u00a0ml) of white vinegar and 1\u20442\u00a0\u00a0c (120\u00a0ml) of plain water in a cup or bowl. Set your solution aside for now until you are ready to clean.\nVinegar is a natural cleaner that works like a mild bleach and can be used to clean many types of materials.",
+      "Lay the affected piece of paper out flat on a hard, waterproof surface. Flatten it out as much as possible. You can place heavy objects on the corners to stretch it out and keep it flat and stable.\nRemember that the sooner you start working on the oil stain, the easier it will be to remove.",
+      "Moisten a cotton ball or swab with the cleaning solution and clean the spot. Gently and carefully dab the oil stain with the moistened cotton ball or swab. Be careful not to use too much of the cleaning solution as you could damage the paper you are cleaning if it gets too wet.\nIf the oil spot looks like it is getting too wet, dab it dry with a paper towel or let it air dry so that you don\u2019t create a hole in the paper.\nKeep dabbing rather than letting the vinegar soak in, until the stain appears to be lifted, then you can dry the area.",
+      "Dry the cleaned area by dabbing it with a dry paper towel and letting it air dry. Once the spot is completely dry you can see if the stain was completely removed. If there is still an oil stain, repeat the process to remove as much of it as possible.\nKeep in mind that this method will work better the fresher the stain is, older stains may not come out completely but you can reduce their visibility."
+    ],
+    "captions": [
+      "A cup with equal parts white vinegar and water.",
+      "A piece of paper with an oil stain laid out flat on a table.",
+      "A person using a cotton swab to dab a cleaning solution onto an oil stain on a piece of paper.",
+      "A person drying a cleaned oil stain on a piece of paper with a paper towel."
+    ]
+  },
+  "9": {
+    "title": "How to Paint Plastic Models",
+    "steps": [
+      "Paint your base coats using as few strokes as possible. To start painting a smaller or intricate model, grab the color that you\u2019re going to use for the base of your model. Load your brush with your base color and wipe it back and forth on your easel or palette to remove the excess paint. Apply your base coat over the areas that you want to paint using as few strokes as possible. Hold your brush at a slight angle and make each brush stroke as large as you can without brushing a part of the model that you aren\u2019t trying to cover.\nFor larger surfaces, use a flat brush. The tip of the brush must be smaller than the surface that you\u2019re painting. For models smaller than 1 foot (30\u00a0cm), a 1\u20132\u00a0in (2.5\u20135.1\u00a0cm) brush is appropriate. The size of the brush is largely a matter of personal preference, though.\nIf your base coat is a lighter color, feel free to paint the entire model. The other paints will easily cover the surfaces you\u2019ve painted. If the base coat is a darker color, like a black or blue, try to avoid painting other surfaces of the model. It will take multiple coats to cover a darker base coat.\nTo figure out what your base coat\u2019s color is, determine what color will cover the highest percentage of your model\u2019s surface. This is the color of your base coat.\nSome models have instructions for which colors you should use, but you can always pick your colors out yourself. There are no right or wrong choices when it comes to picking colors. It\u2019s totally up to you!",
+      "Overlap your brush strokes to ensure an even coat of color. When painting your model, overlap each brush stroke to avoid leaving gaps between them. If you try to place each brush stroke perfectly against one another, you\u2019ll naturally miss small areas in between the paint strokes. Overlap each stroke to ensure that your colors come out evenly.\nThis will also prevent smearing, since you\u2019ll be spreading the paint out before it has an opportunity to start drying. If you have to go in and add paint to spaces that you\u2019ve missed, you\u2019ll have to wait for the coat to dry entirely before you can add more color.",
+      "Wait 24-72 hours in between coats to let them dry. After you finish your base coat, wait 24-72 hours to give the paint time to settle. Acrylic paint will dry closer to the 24-48 hour range, while enamel usually takes 48-72 hours to dry. The thickness of your coat will also impact the amount of time it takes for your paint to dry.\nIf you\u2019re trying to blend colors, feel free to continue painting while your paint is wet.",
+      "Work your way down to the smaller layers after painting the base coat. Once you finish your base coat, add your next biggest layer of color. Work from larger layers to smaller layers to avoid having to work around smaller brushstrokes. Use a smaller brush to cover smaller layers, and continue to use as few brush strokes as possible. Overlap each application of color to ensure that the coat comes out evenly.\nFor example, if you\u2019re painting a model police officer, your base layer is probably the blue uniform. The next biggest section of color will be the skin tone, followed by the silver buckle and buttons and the black details on the uniform.",
+      "Apply details with your thinner brushes by using short strokes. Once you\u2019ve added your key layers of paint, start adding your details. Use the thinnest brush that you have to carefully apply small lines, textures, or shadows. Use a short, flicking motion and only load the tip of your brush to avoid adding too much paint. Work slowly and be careful to avoid putting paint where it doesn\u2019t belong.\nFor some people, it\u2019s easier to keep their hand steady if they brace it on a table or small stack of books. Maintaining a steady hand is the best way to ensure that you don\u2019t miss a stroke.\nDon\u2019t be too hard on yourself. It can take a lot of practice to develop the skills required to paint small details. The more you practice though, the better you\u2019ll get at it.\nFor smaller miniatures, use a 0, 00, or 000 round brush. These brushes are smaller than 1\u204432\u00a0in (0.079\u00a0cm). For larger models, details may be painted with a 1 or 2 brush, which are only slightly larger than 1\u204432\u00a0in (0.079\u00a0cm).",
+      "Thin your paint before applying textures or translucent colors. If you want to add dirt, blood, dust, or some other texture, thin your paint before applying it. Acrylic paint is thinned with water, while enamel requires a special modelling solvent. Start with a drop or two of thinner when mixing your paint. Continue to add thinner until you\u2019ve achieved the opacity and look that you desire.\nTest your mixed paint on a blank sheet of paper to see how it looks. If you don\u2019t like the texture, add paint or thinner to change the opacity as desired.\nUse droppers to mix small amounts if you\u2019re painting a smaller model. You usually don\u2019t need more than a drop or two to adequately thin out paint for a texture.",
+      "Use a toothpick to carve grooves or add tiny details. For extremely intricate details, grab a toothpick with a sharp point. Use the toothpick to carefully remove small sections of paint and add details. You can also dip the tip of your toothpick into paint to apply small specks of color where you want them.\nIf you\u2019re trying to add shadows, use a cotton swab instead of a toothpick. This usually produces a shadowy effect without adding too much paint.",
+      "Clean your brushes as soon as you\u2019re done painting. When you\u2019ve finished painting, fill a cup with 2\u20133\u00a0in (5.1\u20137.6\u00a0cm) of your cleaning solution. Use water to clean your acrylic brushes and use paint thinner or enamel cleaner for enamel brushes. Wipe your bristles against the side of the cup repeatedly to remove the majority of the paint. Once most of the paint is gone, rinse your enamel or acrylic brushes under a stream of warm water.\nNever push your bristles directly into the bottom of the cup. This can damage your brushes by fraying the bristles.\nAfter you\u2019ve dried your brushes, you can rub petroleum jelly on the bristles to preserve them.\nIf you don\u2019t clean your brushes, you\u2019re going to waste a lot of money replacing them."
+    ],
+    "captions": [
+      "A person holding a paintbrush at an angle, applying paint to a plastic model. The brush is loaded with a small amount of paint, and the person is using long strokes to cover a large area.",
+      "Close-up of a person painting a plastic model with a brush. The brush is overlapping the previous stroke slightly, ensuring there are no gaps between the painted areas.",
+      "A painted plastic model sitting on a table. A clock with an arrow pointing between 24 and 72 hours is next to the model.",
+      "A partially painted plastic model. The base coat is complete, and the person is now applying a different color to a larger section of the model.",
+      "A person carefully painting a small detail on a plastic model with a very thin brush. The person's other hand is braced on a stack of books for stability.",
+      "A palette with two drops of paint and a small amount of water mixed together. A paintbrush rests in the palette.",
+      "A person holding a toothpick and carefully scraping away a small section of paint on a plastic model to create a detail.",
+      "A cup filled with cleaning solution and a paintbrush being rinsed under warm water."
+    ]
+  },
+  "10": {
+    "title": "Make a Decorative Jar",
+    "steps": [
+      "Take an empty glass jar.",
+      "Paint or decorate it with stickers, twine, or washi tape.",
+      "Use it as a vase, a pen holder, or a candle holder."
+    ],
+    "captions": [
+      "A clean, empty glass jar with paint, stickers, twine, and other decorative supplies beside it.",
+      "A person painting the jar, wrapping twine around it, or applying stickers.",
+      "The finished decorative jar being used as a vase, pen holder, or candle holder."
+    ]
+  }
+}

datasets/VIST.json

@@ -0,0 +1,512 @@
+{
+  "0": {
+    "title": "Fireworks - 4th July in Washington, DC",
+    "steps": [
+      "The two friends went to a new location",
+      "Large crowds had gathered on the steps of the Lincoln Memorial",
+      "We had our own little spot on the grass. ",
+      "As the evening grew darker the crowd was gearing up to enjoy the show, with a great view of the Washington Monument.",
+      "The fireworks were the best that we had ever seen"
+    ],
+    "captions": [
+      "Two friends walk through a busy area, making their way toward the Lincoln Memorial on a bright day.",
+      "A large crowd sits and stands on the steps of the Lincoln Memorial, engaged in conversation and waiting for the event.",
+      "Two friends sit on the grass among many others, settling into their spot for the evening.",
+      "As the sky darkens, people sit facing the Washington Monument, eagerly awaiting the show.",
+      "Fireworks light up the night sky, reflecting over the Washington Monument as the crowd watches in awe."
+    ]
+  },
+  "1": {
+    "title": "Klamath Project Construction",
+    "steps": [
+      "it was a simpler time indeed ",
+      "We fired up the machinery and got to work.",
+      "We were building a new waterway for the reservoir to spill into.",
+      "the men weeped as they buried their brothers",
+      "Finally, we added some chemicals to the new water filtration system."
+    ],
+    "captions": [
+      "A flume stretches over a river, surrounded by scattered debris, capturing a moment of industrial progress.",
+      "Workers operate heavy machinery at an unfinished bridge construction site, dust rising as they labor.",
+      "Water rushes over the newly completed spillway of the Gerber Dam for the first time, marking a milestone.",
+      "A somber group of men stand by a freshly dug ditch, heads bowed as they pay tribute to their fallen comrades.",
+      "A man carefully pours liquid into a large gallon, finalizing the water filtration system with chemical treatment."
+    ]
+  },
+  "2": {
+    "title": "Isfahan: 2nd Iranian flickr Gathering",
+    "steps": [
+      "It's nice to be able to look back and see how things are, how they were, and how they've changed to get here.",
+      "We visited many places, some so beautiful in very subtle ways.",
+      "He encountered many great people on his visit.",
+      "Afterwards, we met a nice woman who explained that she was much too shy to have her photo taken.",
+      "Heritage and history are important things to the world."
+    ],
+    "captions": [
+      "A young boy stands confidently, looking at the camera as others around him reflect on their journey.",
+      "A softly lit scene showcases a historic church with its lights glowing, surrounded by a peaceful atmosphere.",
+      "A gathering of people is seen observing an important object, their faces filled with curiosity and admiration.",
+      "A woman in a headscarf stands slightly apart, gesturing gently with her hand while speaking to someone nearby.",
+      "A detailed mural filled with vivid imagery depicts a historical battle, capturing the essence of heritage and history."
+    ]
+  },
+  "3": {
+    "title": "Yosemite Weekend",
+    "steps": [
+      "It was a beautiful day in the forest.",
+      "We were lucky enough to see some wildlife, like this deer.",
+      "I fed a squirrel and it seemed very hungry.",
+      "A snake came out of nowhere and scared me. ",
+      "The scenery from the top of the mountain was breathtaking. "
+    ],
+    "captions": [
+      "Sunlight filters through towering trees, casting dappled shadows on the forest floor.",
+      "A deer stands cautiously in a forest clearing, ears perked up as it listens to the sounds around.",
+      "A plump squirrel eagerly nibbles on food, its tiny paws clutching a morsel.",
+      "A snake slithers out from beneath the undergrowth, its eyes fixed intently ahead.",
+      "From the mountain peak, the landscape unfolds into a vast, breathtaking view of valleys and waterfalls."
+    ]
+  },
+  "4": {
+    "title": "4th of July 2006",
+    "steps": [
+      "This was the first time my sister saw my son. ",
+      "He seemed to like the company of his new relatives. ",
+      "The kids enjoyed their self and I am happy.",
+      "He couldn't walk, let alone jump but he was having a good time regardless. ",
+      "The loving man in our lives reminded us of the importance of family. "
+    ],
+    "captions": [
+      "A woman arrives at a park with her baby, meeting her sister for the first time in a long while.",
+      "A baby sits on a picnic blanket, curiously observing his new relatives gathered around him.",
+      "Children laugh and play together on a bounce house, enjoying the family gathering.",
+      "A baby sits on the grass, unable to walk but joyfully clapping as the kids jump nearby.",
+      "A loving father embraces his family, reminding everyone of the warmth and bond they share."
+    ]
+  },
+  "5": {
+    "title": "07.04.06 Fourth of July",
+    "steps": [
+      "Some are raised to believe that the world takes all kinds of people to make it go 'round. ",
+      "Our group of friends certainly demonstrates that adage to be true. ",
+      "There are always interesting moments to be found for those hanging around any of us.",
+      "Moments that you aren't quite sure what is going on, and aren't really sure you want to know either.",
+      "But everything is always in good fun, and we enjoy hanging out."
+    ],
+    "captions": [
+      "A woman stands on a busy street corner, casually smoking as cars pass by.",
+      "A group of friends sits together under a patio umbrella, laughing and enjoying drinks.",
+      "A woman takes a bite of her frozen drink while chatting with her friends.",
+      "A man with food on his lips gestures animatedly, mid-conversation, as others react with amusement.",
+      "A few friends walk down a rainy street, still laughing and enjoying the night together."
+    ]
+  },
+  "6": {
+    "title": "July4",
+    "steps": [
+      "There was a parade going on.",
+      "A another man saying a speech to the crowd.",
+      "Everyone looked like they were having fun. ",
+      "These kids were fighting over the flag but finally stopped. ",
+      "Later on that night there was fireworks! God bless America!"
+    ],
+    "captions": [
+      "A patriotic parade moves down the street with men in red berets and white gloves carrying flags.",
+      "A man dressed in a Civil War uniform stands on a decorated float, addressing the excited crowd.",
+      "People of all ages line the streets, smiling and cheering as the parade continues.",
+      "Two kids stand in front of an American flag, their hands raised as they playfully argue over it.",
+      "As night falls, bright blue and yellow fireworks explode in the sky, marking the grand finale of the celebration."
+    ]
+  },
+  "7": {
+    "title": "4th of July",
+    "steps": [
+      "The waiting was almost too much for the Kid Town citizens.",
+      "Of course, when evening came and my wife and I wanted to nap, the kids wanted to run around.",
+      "The kid's especially loved that they were able to use sparklers while they waited for the show.",
+      "Finally, everyone settled to watch the fireworks. ",
+      "The fireworks were fantastic."
+    ],
+    "captions": [
+      "A lively town square filled with eager children and families waiting excitedly.",
+      "A couple sits on a bench, trying to rest as children run and play around them.",
+      "A young boy in a robe joyfully waves a sparkler, its glow illuminating his face.",
+      "Families and children lie on blankets, gazing up in anticipation of the fireworks.",
+      "Brilliant fireworks explode in the night sky, casting colorful light over the crowd."
+    ]
+  },
+  "8": {
+    "title": "July 4th Barbecue - Technology Teams",
+    "steps": [
+      "Today is the big celebration and everyone is here to support.",
+      "The chef put his hand in the air, because it was time to surrender the food. He is so funny.",
+      "Nobody wanted to miss this wonderful event.",
+      "Cindi loved her boss's cooking but was especially fond of the strawberry dessert he made every year.",
+      "She got too much on her plate and fed some to a fellow coworker."
+    ],
+    "captions": [
+      "A lively outdoor celebration with guests gathered around, everyone excited to support the event.",
+      "A chef in a cooking uniform raises one hand dramatically, signaling it's time to serve the food.",
+      "Guests chat and laugh in a vibrant setting, with people gathered around tables, enjoying the event.",
+      "A woman admires a beautiful strawberry dessert being served by her boss at the celebration.",
+      "A woman, smiling, offers a strawberry to a coworker sitting next to her, sharing the dessert."
+    ]
+  },
+  "9": {
+    "title": "fourth of july",
+    "steps": [
+      "We look forward to the recycling parade every year. ",
+      "We pulled them along the road.",
+      "Others stand by and watch.",
+      "The race for the children at the end was so much fun!",
+      "Thankfully all the fun is kept safe by our local police force."
+    ],
+    "captions": [
+      "A colorful recycling parade takes place on a lively residential street with families and children participating.",
+      "A decorated wagon rolls down the road, with a little girl smiling as she's pulled along by her parents.",
+      "A group of onlookers stand on the sidewalk, watching the parade with excitement and interest.",
+      "Children race joyfully at the end of the parade, their laughter and energy filling the air.",
+      "Local police officers on horseback ensure safety as they patrol the streets during the event."
+    ]
+  },
+  "10": {
+    "title": "Independence Day Parade",
+    "steps": [
+      "The festival was a blast. ",
+      "It was quite a performance to watch.",
+      "then it started to get interesting and elaborate ",
+      "Horses were used for tricks,",
+      "Next came the old cars."
+    ],
+    "captions": [
+      "A festival scene filled with people gathered around, enjoying the lively atmosphere.",
+      "A captivating performance unfolds with vibrant displays of skill and flair.",
+      "The excitement grows as the tricks become more elaborate, with horses performing intricate maneuvers.",
+      "A majestic horse prances gracefully as a rider performs impressive stunts.",
+      "An old car drives through the parade, commemorating fallen soldiers as spectators watch."
+    ]
+  },
+  "11": {
+    "title": "Fourth of July, 2007",
+    "steps": [
+      "Today, there was a parade held out on the grounds.",
+      "There were all kinds of people there.",
+      "It's nice to see the participants enjoying themselves.",
+      "Next in line came the flag team.",
+      "Before long, it was already nighttime and we went home."
+    ],
+    "captions": [
+      "Soldiers march in flawless formation, their uniforms sharp and the rhythm of their footsteps echoing through the streets.",
+      "The parade is alive with color, dancers and musicians filling the streets with joy and energy.",
+      "A flamenco dancer spins elegantly, her colorful dress flowing with every movement, captivating the onlookers.",
+      "A woman holds a red flag high, her face beaming with pride as her group marches past.",
+      "The vibrant city quiets as the day winds down, the lights dimming as the final notes of the parade fade away."
+    ]
+  },
+  "12": {
+    "title": "Chapel Hill Fireworks",
+    "steps": [
+      "It was fourth of july night.",
+      "It was a sight to behold!",
+      "Some weren't as big or bright as others.",
+      "Everyone had a good time watching them.",
+      "The show was so much fun and I wish I could go again. "
+    ],
+    "captions": [
+      "The streets are bathed in the warm glow of streetlights as the crowd eagerly awaits the fireworks display.",
+      "The night sky bursts with bright blue and white fireworks, sparkling in the cool evening air.",
+      "Explosions of red, green, and gold create intricate patterns as the fireworks explode high above.",
+      "Families stand together, faces illuminated by the glow of the fireworks, eyes wide in amazement.",
+      "The grand finale fills the sky with a cascade of dazzling lights, leaving the crowd in awe."
+    ]
+  },
+  "13": {
+    "title": "4th of July, 2007, Breckenridge",
+    "steps": [
+      "The owner took a picture with the dog.",
+      "...and my husband!",
+      "And both know just how to get what they want.",
+      "The dog rolled around and enjoyed the heat.",
+      "And they are in no way spoiled....Um yeah...ok."
+    ],
+    "captions": [
+      "A woman smiles brightly, cradling her furry companion in her arms, both basking in the sunlight.",
+      "A man sits leisurely on a park bench, his dog lying contentedly by his side, enjoying the serene day.",
+      "The woman carefully adjusts the straps on her dog's harness, ensuring comfort before they set off again.",
+      "A joyful black dog rolls around in the soft grass, soaking in the warm rays of the sun.",
+      "The two dogs lie peacefully together, curled up in a contented slumber, completely at ease."
+    ]
+  },
+  "14": {
+    "title": "4th of July",
+    "steps": [
+      "Then multiple fireworks were let off at different times.",
+      "The fireworks were huge and we had a really great view of them.",
+      "The white fireworks were the most clear in the sky.",
+      "As the finale approached, more and more fireworks filled the sky.",
+      "I had a great time, the finale of the fireworks was awesome!"
+    ],
+    "captions": [
+      "A series of fireworks light up the sky, each burst more brilliant than the last, as the crowd watches in awe.",
+      "Massive white fireworks explode against the dark canvas of the night, casting shimmering reflections on the ground below.",
+      "The sky is alight with a dazzling array of white fireworks, each explosion more stunning than the last.",
+      "The finale draws near, and the number of fireworks exploding in the sky intensifies, filling the crowd with excitement.",
+      "The grand finale is a riot of color, with fireworks bursting in every direction, leaving everyone stunned and delighted."
+    ]
+  },
+  "15": {
+    "title": "Independance Day 2007",
+    "steps": [
+      "The fireworks begin to go off.",
+      "The explosions also got more complex.",
+      "Look at all of those fireworks!",
+      "It seemed much brighter up there, ",
+      "in the dark night sky."
+    ],
+    "captions": [
+      "The sky lights up as the first fireworks burst, signaling the start of a spectacular display.",
+      "Intricate fireworks patterns unfold in the sky, each explosion more complex than the last.",
+      "The fireworks intensify, filling the sky with a brilliant array of lights and colors that seem to dance in the night.",
+      "Two giant fireworks explode at once, their glowing trails creating a breathtaking spectacle for the crowd below.",
+      "The deep night sky provides the perfect backdrop for the fireworks, their vibrant colors creating a stunning contrast."
+    ]
+  },
+  "16": {
+    "title": "LSU: July 2007",
+    "steps": [
+      "I always enjoy paying a visit to Spitz Laboratories.",
+      "Seeing the control panel on some of the old equipment can be fascinating.",
+      "Look at this old knob!",
+      "Some more lab equipment",
+      "We'll celebrate in the hope of one day finding more out there, and to enjoy what we have now."
+    ],
+    "captions": [
+      "The dark surface is engraved with the name of Spitz Laboratories, a symbol of innovation and progress.",
+      "An antique electronic device rests open, its control panel full of complex dials and switches, hinting at its once-advanced technology.",
+      "A hand turns the dial of a vintage machine, setting it to the highest setting, reminiscent of the experimental days gone by.",
+      "A collection of historic lab equipment stands in neat rows, a testament to the advancements made over the years.",
+      "As a fitting tribute, fireworks explode in the sky, celebrating the spirit of discovery and achievement."
+    ]
+  },
+  "17": {
+    "title": "Sleepin' and Eatin'",
+    "steps": [
+      "The dog is being held up, and loved.",
+      "The dog will put anything in his mouth",
+      "Oh look how cute she is napping. ",
+      "The dog is playing with a clear bottle.",
+      "The dog is very tired from all the playing."
+    ],
+    "captions": [
+      "The puppy gazes up with wide, soulful eyes, being lovingly held in someone's arms.",
+      "The dog lies comfortably beside a container, its paw resting gently on top, as if claiming it.",
+      "The dog sleeps contentedly on a cozy blanket, its eyes closed in peaceful slumber.",
+      "The playful puppy rolls a clear plastic bottle around, its tiny paws having a blast.",
+      "Exhausted from play, the dog curls up on the floor and drifts off into a deep sleep."
+    ]
+  },
+  "18": {
+    "title": "Ribfest - Naperville, IL 4th of July, 2007",
+    "steps": [
+      "The man and his kid were on the ride.",
+      "He was excited.",
+      "The kid was not.",
+      "What a wild ride this is. So much excitement",
+      "The man was having a great time."
+    ],
+    "captions": [
+      "The man scrunches up his face in exaggerated fun as the ride begins to lift.",
+      "The two men prepare for the thrilling ride ahead, full of excitement and anticipation.",
+      "With their hands gripping the restraints, the man and boy brace themselves for the roller coaster.",
+      "The two riders sit side by side, preparing for the wild twists and turns of the ride.",
+      "The man is thrilled as he enjoys the roller coaster, laughing and screaming with joy."
+    ]
+  },
+  "19": {
+    "title": "4th of July",
+    "steps": [
+      "The grill is getting hot.",
+      "They had beer to go with it.",
+      "and various meat to keep them coming to his house",
+      "He cooked them way too long but we ate them anyways. ",
+      "The party went into the night."
+    ],
+    "captions": [
+      "The grill crackles as flames lick the edges of the barbecue, heating up the meat.",
+      "The man sips his beer contentedly, chatting with friends around the grill.",
+      "The meat on the plate is still raw, ready to be thrown onto the sizzling barbecue.",
+      "Hot dogs and hamburgers sizzle on the grill, the smell filling the air as they cook.",
+      "The group of friends laughs and enjoys drinks as the party continues into the night."
+    ]
+  },
+  "20": {
+    "title": "4th of July",
+    "steps": [
+      "We waited for sundown so that we could watch the fireworks show.",
+      "Finally, the show started!",
+      "Bright lights filled the sky!",
+      "The finale was spectacular and left us in awe.",
+      "There was a lot of smoke in the sky afterwards."
+    ],
+    "captions": [
+      "The cross rises against the sky, standing tall as the sun begins to set.",
+      "A burst of colorful fireworks lights up the sky, marking the beginning of the show.",
+      "Brilliant white fireworks shine in the dark, illuminating the night with their glow.",
+      "The grand finale arrives, with two giant fireworks exploding in perfect harmony.",
+      "The smoke lingers in the sky, a remnant of the dazzling fireworks that just filled the air."
+    ]
+  },
+  "21": {
+    "title": "4th of July, 2007 in Sausalito, CA",
+    "steps": [
+      "The baby was having a great time. He even had a cute hat.",
+      "People were starting to arrive in town already.",
+      "Mommy then held him up so he could see.",
+      "A yellow bug makes it's way past",
+      "People rode on horses and were dressed in all sorts of things."
+    ],
+    "captions": [
+      "The baby beams with joy, sitting in a stroller, wearing a tiny, adorable hat.",
+      "The busy street is filled with pedestrians, as people begin arriving in town for the event.",
+      "The woman lifts her son into the air, letting him see all the excitement unfolding around them.",
+      "The bright yellow buggy, adorned with flags, moves through the crowd, capturing everyone\u2019s attention.",
+      "A rider in elaborate, elegant attire steers their horse through the parade, creating a striking impression."
+    ]
+  },
+  "22": {
+    "title": "Born on the 4th of July",
+    "steps": [
+      "It was the 4th of July, and a party was soon to begin.",
+      "man about to eat the food ",
+      "Sarah and Steve stopped by and ate little burgers too. ",
+      "I brought my crazy weird girlfriend home for the first time and sure enough, weirdness ensued. ",
+      "Later on a band played. Where they came from I have no idea but it was fun!"
+    ],
+    "captions": [
+      "A festive banner hangs in the air, signaling the start of a joyous 4th birthday celebration.",
+      "The man prepares to take a large, satisfying bite of his sandwich, eager for the delicious treat.",
+      "Sarah and Steve walk hand in hand, enjoying the sights and sounds of the lively street.",
+      "The couple playfully poses for the camera, their goofy antics standing out among the crowd.",
+      "A talented musician captivates the crowd with their performance, adding a special touch to the event."
+    ]
+  },
+  "23": {
+    "title": "7/4/7 Fireworks San Diego Bay",
+    "steps": [
+      "The first firework was let off.",
+      "With a boom, two red fireworks burst in the night sky.",
+      "The fireworks were beautiful.",
+      "Another red burst of fireworks in the sky.",
+      "The last was a bit of everything and it all lit the sky. "
+    ],
+    "captions": [
+      "The first firework crackles and bursts above the town, lighting up the night.",
+      "Two brilliant red fireworks explode simultaneously, their bright trails illuminating the sky.",
+      "The dark night sky is filled with dazzling bursts of light from the fireworks above.",
+      "The sky comes alive with red and yellow bursts of light, a brilliant display of fireworks.",
+      "The final explosion of fireworks fills the air with a grand, awe-inspiring display."
+    ]
+  },
+  "24": {
+    "title": "4th of July Fireworks - 2007",
+    "steps": [
+      "The fireworks were bright and beautiful, ",
+      "then it became a little more intense with bigger fireworks",
+      "they exploded in the sky, ",
+      "and then the fireworks were going everywhere when they turned the show up to a 6",
+      "then things got even crazier as the show ended"
+    ],
+    "captions": [
+      "A series of red fireworks explode, their brilliant light filling the night sky.",
+      "The fireworks intensify, with larger explosions lighting up the night in a blur of color.",
+      "A single firework bursts in the sky, its vibrant colors bright against the dark background.",
+      "The fireworks continue to escalate, lighting up the night with a dazzling display.",
+      "The grand finale is a cascade of bright fireworks, filling the sky with their brilliance."
+    ]
+  },
+  "25": {
+    "title": "Red, White and Boom [Jul '08]",
+    "steps": [
+      "There was a crowd of people waiting already. ",
+      "The clock tower was very tall.",
+      "Finally it got dark out, and everyone looked to the skies in anticipation.",
+      "the fireworks has stared and it was well worth the time. ",
+      "the show is over and now everyone is on there way home. "
+    ],
+    "captions": [
+      "A crowd gathers, chatting excitedly as they wait for the event to start.",
+      "The clock tower looms tall in front of surrounding buildings, a towering landmark.",
+      "As night descends, people look up in awe at the darkening sky, waiting for the fireworks.",
+      "Explosions of color light up the sky as the fireworks burst in a dazzling display.",
+      "With the show over, people begin to head home, reflecting on the magical evening."
+    ]
+  },
+  "26": {
+    "title": "Greenville, SC 7/4/08",
+    "steps": [
+      "The fireworks didn't look very good on the TV.",
+      "John had an Idea. He would go see the in person.",
+      "Then my dog started barking because he heard the noises too. ",
+      "Watching them alone instead",
+      "After they were finished we came back in and finished watching our tv show. "
+    ],
+    "captions": [
+      "The fireworks appear distant and blurry on the TV screen, failing to capture their true beauty.",
+      "Inspired, the young boy decides to head out and see the fireworks live.",
+      "As the fireworks explode outside, the dog barks loudly, reacting to the noise.",
+      "The night sky glows with vibrant colors as fireworks light up the horizon in a stunning display.",
+      "Once the fireworks show concludes, they return to the living room to finish their TV show."
+    ]
+  },
+  "27": {
+    "title": "Williamsburg, July 2008",
+    "steps": [
+      "The garden was full of herbs and beautiful plants.",
+      "and the gift was a flower, this thing has been growing for years ",
+      "The pumpkin was starting to grow larger with each passing week. It was going to be perfect in the Fall.",
+      "there was even soliders from the other side there",
+      "It was all beautiful and full of life."
+    ],
+    "captions": [
+      "The market is filled with an assortment of beautiful plants and fragrant herbs, creating a vibrant atmosphere.",
+      "A delicate flower blooms on its stem, adding a touch of elegance to the garden.",
+      "The pumpkin grows larger by the day, soon to be ready for harvest in time for Fall.",
+      "Soldiers in red coats stand proudly in a field, preparing for their performance with their instruments.",
+      "A woman in a gorgeous colonial costume stands in front of the old yellow house, radiating grace and history."
+    ]
+  },
+  "28": {
+    "title": "4th of July Family BBQ",
+    "steps": [
+      "I was invited to an outdoor lunch party with lots of people and food. ",
+      "The children had fun playing with each other.",
+      "Even the dogs were excited. ",
+      "Everyone was hanging around and having a great time.",
+      "We remember to pose for plenty of pictures."
+    ],
+    "captions": [
+      "A lively picnic scene unfolds as many people gather, enjoying food and conversation in the open air.",
+      "The young boy and the younger child play together, their laughter filling the air with joy.",
+      "Small dogs roam excitedly around the group of people sitting on the patio, adding to the lively atmosphere.",
+      "A small group of friends gathers inside, smiling and enjoying each other's company as they chat and laugh.",
+      "Before leaving, everyone poses for a group photo, capturing the memories of the fun day together."
+    ]
+  },
+  "29": {
+    "title": "Fourth of July 08",
+    "steps": [
+      "The women attend to their children.",
+      "But they have other ideas",
+      "Let's try this again, maybe if we reposition the kids.",
+      "Two out of three is pretty good, but the little guy is still not having it",
+      "We ended up only taking one picture. Oh well, pictures aren't for everyone. "
+    ],
+    "captions": [
+      "The women look after their children as they spend time together outdoors, enjoying the warm weather.",
+      "The children run and play on the grass, laughing and having fun despite their different ideas.",
+      "The woman attempts to arrange the children for a photo, trying to get everyone in position.",
+      "The children sit together, some more cooperative than others, in a serene moment on the grass.",
+      "After much effort, only one picture is taken as the children decide they\u2019ve had enough of the photo session."
+    ]
+  }
+}

datasets/recipes.json

@@ -0,0 +1,471 @@
+{
+  "0": {
+    "steps": [
+      "Heat the Coconut Oil in a wide pan over a medium flame, then add the Onion, Garlic, Scallion, and Ground Black Pepper. Reduce the heat to low for about 3-4 minutes.",
+      "Add the Small Shrimp, stir well and cook for another 3 minutes.",
+      "Turn the heat up to medium high and add the Jamaican Callaloo, Tomato, Scotch Bonnet Pepper, Fresh Thyme, and Sea Salt. After a couple minutes, add the Water and cook until tender.",
+      "After about 10-12 minutes, taste for salt and adjust accordingly."
+    ],
+    "title": "Jamaican Callaloo With Shrimp",
+    "captions": [
+      "Pan with sizzling coconut oil, onions, garlic, scallions, and black pepper",
+      "Pan with saut\u00e9ed aromatics and added shrimp",
+      "Pan with saut\u00e9ed aromatics, shrimp, Jamaican callaloo, tomato, scotch bonnet pepper, thyme, and sea salt",
+      "Pan with simmering Jamaican callaloo shrimp dish"
+    ]
+  },
+  "2": {
+    "steps": [
+      "To a medium bowl, add Pecans, Vegan Cream Cheese, Daiya\u00ae Mozzarella Cheese, Sun-Dried Tomatoes in Olive Oil, Garlic, Ground Black Pepper, and Vegan Basil Pesto. Stir until well combined.",
+      "Add Fresh Spinach to the mixture and continue stirring until all ingredients are well mixed.",
+      "Lay out Tortilla, divide cheese/pesto/tomato mixture evenly among the four tortillas, spreading in a thin layer until it coats the whole tortilla up to the outer edge. Tear and sprinkle approximately 5 Fresh Basil Leaf over each tortilla.",
+      "Gently roll tortillas, keeping them fairly tight, then slice each into 8 pieces. Serve immediately or store in the refrigerator for up to 3 days."
+    ],
+    "title": "Vegan Basil Pesto and Sun-Dried Tomato Pinwheels",
+    "captions": [
+      "Bowl with pecans, vegan cream cheese, Daiya mozzarella, sun-dried tomatoes, garlic, black pepper, and vegan basil pesto",
+      "Bowl with filling ingredients and added spinach",
+      "Tortilla with spread filling and basil leaves",
+      "Rolled and sliced vegan quesadilla"
+    ]
+  },
+  "3": {
+    "steps": [
+      "Melt butter in a small pot over medium heat.",
+      "Add egg and sugar to a bowl and whisk until combined. Stir in honey and add lemon zest. Stir to combine until a smooth batter forms.",
+      "Sieve flour and baking powder into the batter, then add melted butter and stir to combine. Cover bowl with plastic wrap and let rest in the fridge for at least 3 hrs.",
+      "Pre-heat oven to 210\u00b0C/410\u00b0F. Grease baking pan with butter. Remove batter from the fridge and transfer into a piping bag. Cut off the tip and pipe batter into baking pan up to three quarters, so the madeleines have enough space to rise.",
+      "Reduce baking temperature to 170\u00b0C/350\u00b0F and bake madeleines for approx. 12 \u2013 15 min. until golden brown. Dust with confectioner\u2019s sugar and"
+    ],
+    "title": "Madeleines",
+    "captions": [
+      "Small pot with melting butter",
+      "Bowl with whisked eggs, sugar, honey, and lemon zest",
+      "Bowl with combined batter",
+      "Covered bowl of batter in the fridge",
+      "Baking pan with piped madeleine batter"
+    ]
+  },
+  "4": {
+    "steps": [
+      "Preheat oven to 180\u00b0C/350\u00b0F and grease springform pan with coconut oil. For the crust, pulse oats in a food processor to a fine flour. Add buckwheat flour, coconut oil, maple syrup, some cinnamon, baking powder, and a pinch of salt, and pulse to form a crust. If the mixture is too dry, carefully add water a tbsp. at a time to make the crust moldable.",
+      "Use hands to press two-thirds of the dough evenly into springform pan along the base and border. Lightly flour work surface and remaining dough. Roll out, using a rolling pin. Use a cookie cutter to cut out decorative shapes, or roll out dough into a round the size of the pan, if you prefer.",
+      "For the filling, peel and quarter apples. Remove core and slice.",
+      "In a large bowl, thoroughly mix apple slices, some cinnamon, nutmeg, lemon juice, agave syrup, arrowroot flour, coconut sugar, speculaas spice, and salt.",
+      "Transfer filling to prepared springform pan and spread evenly.",
+      "Cover filling with either with cookie cut-outs or a round of dough. For the latter, pierce top with a fork to allow steam to escape while baking. Place in preheated oven and bake at 180\u00b0C/350\u00b0F for approx. 30 min. Cover with aluminum foil and press down lightly so that the apple juice rises to the top and soaks the entire cake. Bake for approx. 45 \u2013 60 min. more, until golden brown. Allow to cool completely on a cooling rack before cutting."
+    ],
+    "title": "Gluten-free apple cake",
+    "captions": [
+      "Close-up view of a food processor blending ingredients for a cake crust.",
+      "A rolling pin rolling out dough on a wooden surface.",
+      "Sliced apples on a wooden cutting board.",
+      "A large bowl filled with apple slices, cinnamon, nutmeg, lemon juice, agave syrup, arrowroot flour, coconut sugar, speculaas spice, and salt, being stirred.",
+      "A layer of apple filling being spread evenly inside a springform pan.",
+      "A hand using a fork to prick the top of an apple cake in a springform pan before baking."
+    ]
+  },
+  "7": {
+    "steps": [
+      "Preheat oven to 200\u00b0C/400\u00b0F. Pierce skin of sweet potatoes approx. 5 to 7 times with a fork. Then, place on a parchment-lined baking sheet, transfer to oven, and bake at 200\u00b0C/400\u00b0F for approx. 45 \u2013 60 min. Once baking time is finished, turn off oven and leave potatoes in oven until the other steps are complete.",
+      "Mince garlic. Cut lemon into wedges and halve pomegranate. Remove seeds from pomegranate.",
+      "In a large, grease-free frying pan, toast pine nuts over medium heat for approx. 1 \u20132 min. until fragrant and golden brown. Remove from pan and set aside.",
+      "Add some of the coconut oil to pan, saut\u00e9 garlic, and wilt spinach over medium heat for approx. 1 \u2013 2 min. Drizzle with part of the lemon juice. Add toasted pine nuts to spinach and set aside.",
+      "Remove potatoes from oven and cut in half. Add rest of coconut oil to potato halves and mash with fork.",
+      "Spoon spinach and pine nut mixture on top of potatoes. Add some more lemon juice. Drizzle with yogurt on top and garnish with pomegranate seeds. Season with salt and pepper to taste."
+    ],
+    "title": "Baked sweet potatoes with spinach and pine nuts",
+    "captions": [
+      "A baking sheet lined with parchment paper holding several sweet potatoes, pierced with forks.",
+      "A close-up shot of minced garlic, lemon wedges, and a halved pomegranate with its seeds visible.",
+      "A frying pan with toasted pine nuts, golden brown in color.",
+      "A frying pan with saut\u00e9ed garlic and wilted spinach, along with toasted pine nuts.",
+      "Baked sweet potatoes cut in half, with coconut oil added and being mashed with a fork.",
+      "Baked sweet potatoes topped with a spinach and pine nut mixture, a drizzle of lemon juice and yogurt, pomegranate seeds, and a sprinkle of salt and pepper."
+    ]
+  },
+  "8": {
+    "steps": [
+      "Place eggs and egg white, flour, evaporated milk, sugar, lemon peel, vanilla extract and salt in blender; cover. Blend until smooth. Scrape down sides and blend again if needed. Let mixture sit for approx. 10 min.",
+      "Meanwhile, place an ovenproof frying pan in oven and preheat to 220\u00b0C/425\u00b0F. Remove frying pan from oven.",
+      "Add butter to frying pan and swirl to coat all sides. Add batter to pan.",
+      "Bake for 15 \u2013 20 min. at 220\u00b0C/425\u00b0F, or until puffed and golden. Edges should be deep golden brown and crisp.",
+      "Slide Dutch baby onto serving platter or cutting board, or serve straight from pan, if desired. Top with fresh berries and sprinkle with lemon juice. Dust with confectioner's sugar and cut into wedges for serving. Serve warm with maple syrup."
+    ],
+    "title": "Dutch baby",
+    "captions": [
+      "Close-up view of a blender filled with a creamy mixture of eggs, flour, evaporated milk, sugar, lemon peel, vanilla extract, and salt.",
+      "An ovenproof frying pan is carefully placed into a preheating oven set to 220\u00b0C/425\u00b0F.",
+      "A person is pouring a smooth batter into a buttered frying pan.",
+      "A golden-brown Dutch baby pancake is baking in a hot oven at 220\u00b0C/425\u00b0F.",
+      "A woman uses a knife to cut a delicious-looking Dutch baby pancake into even wedges on a wooden cutting board.\n"
+    ]
+  },
+  "9": {
+    "steps": [
+      "Preheat the oven to 120\u00b0C/250\u00b0F. Heat (don't boil!) heavy cream with some of the sugar and vanilla seeds until the sugar has dissolved.",
+      "Next, whisk the warm mixture into the egg yolks in a large bowl.",
+      "Pour the mixture into br\u00fbl\u00e9e tins and place these into a tall baking dish. Pour hot water into the baking dish, creating a water bath in which to cook the cr\u00e8me br\u00fbl\u00e9e.",
+      "Bake in a preheated oven at 120\u00b0C/250\u00b0F for approx. 60 min. Next, remove the tins from the water bath and leave to cool for 30 min. Chill for at least four hours or overnight before serving.",
+      "Caramelize rest of the sugar in a small frying pan. Add a pinch of salt, pour in water and reduce into a sauce. Pour over the Crema Catalana to serve."
+    ],
+    "title": "Crema Catalana with salted caramel sauce",
+    "captions": [
+      "Close-up view of a pot of heavy cream simmering gently on a stovetop burner.",
+      "A woman vigorously whisking a creamy mixture into egg yolks in a large mixing bowl.",
+      "A woman carefully pouring a custard mixture into individual ramekins, then arranging them in a larger baking dish.  Hot water is being added to the baking dish, creating a water bath around the ramekins.",
+      "A woman removing a baking dish containing ramekins from a preheated oven. The ramekins are slightly browned on top.",
+      "A woman sprinkling granulated sugar into a small, hot frying pan.\n"
+    ]
+  },
+  "10": {
+    "steps": [
+      "Preheat oven to 180\u00b0C/350\u00b0F. Grease and flour the cake pan. Line the bottom with parchment paper if it does not have a removable bottom. Zest all lemons and juice half of them. Set zest aside. In a medium bowl, combine flour, baking powder, and salt. In a measuring cup, mix together milk and lemon juice.",
+      "Beat together butter, sugar, lemon zest, and some of the vanilla sugar until smooth. Add eggs one at a time, beating after each incorporation. Add half of the flour, mix to incorporate, then add the milk mixture and remaining flour. Stir until fully combined.",
+      "Transfer batter to cake pan, smooth evenly, then bake at 180\u00b0C/350\u00b0F for approx. 20 \u2013 30 min. Cool for approx. 10 min., then transfer to a wire rack to cool completely.",
+      "Meanwhile, make frosting: Add cr\u00e8me fra\u00eeche, powdered sugar, and remaining vanilla sugar to a medium mixing bowl. Beat until combined and thickened slightly. Add a drop or two of food coloring until the desired shade of pink is reached. Stir until evenly incorporated.",
+      "Smooth frosting over cake, decorate with sprinkles if desired, and"
+    ],
+    "title": "Tangy lemon sheet cake",
+    "captions": [
+      "Close-up view of a woman pouring milk into a measuring cup next to a small bowl of lemon juice.",
+      "A woman beating a mixture of butter, sugar, lemon zest, and vanilla sugar in a mixing bowl.  Eggs are added one at a time.  Flour and milk mixture are gradually incorporated.",
+      "A woman carefully pouring batter into a cake pan and smoothing the top with a spatula.",
+      "A woman mixing cr\u00e8me fra\u00eeche, powdered sugar, and vanilla sugar in a bowl.",
+      "A woman decorating a frosted cake with sprinkles and frosting.\n"
+    ]
+  },
+  "13": {
+    "steps": [
+      "Finely chop rosemary. Combine buttermilk, lime zest, chopped rosemary, paprika, salt, and pepper. Add pressed garlic and stir to combine.",
+      "Pour buttermilk mixture into a freezer bag. Add chicken wings and mix well. Transfer to refrigerator and let marinate for approx. 2 hours.",
+      "Add bread crumbs to a large bowl. Coat chicken wings with bread crumbs, using tongs to make sure they are coated on all sides.",
+      "Heat a generous amount of vegetable oil in a frying pan. Once oil is hot, add chicken wings and cook until browned and cooked through, approx. 8 \u2013 10 min. Rotate to cook on all sides. Enjoy with your favorite dipping sauce."
+    ],
+    "title": "Buttermilk chicken wings",
+    "captions": [
+      "A bowl containing finely chopped rosemary, buttermilk, lime zest, paprika, salt, pepper, and pressed garlic.",
+      "A freezer bag filled with chicken wings submerged in a buttermilk mixture.",
+      "Chicken wings coated in breadcrumbs in a large bowl.",
+      "A frying pan with chicken wings being fried in hot vegetable oil."
+    ]
+  },
+  "14": {
+    "steps": [
+      "Cut the Vanilla Bean Pod in half, then halve them lengthwise as well.",
+      "Prepare a 100-milliliter jar that has a sealable lid, and put the vanilla in it. Fill up the jar to the brim with Vodka.",
+      "Seal the jar, and store the mix in a cool, dark place for about three months. This will give time to the vanilla to infuse the vodka with its taste. The longer you leave it, the more infused the flavors will be."
+    ],
+    "title": "Home Made Vanilla Extract",
+    "captions": [
+      "A vanilla bean pod being halved with a sharp knife.",
+      "A jar with a sealable lid, filled partially with a substance and a wooden spoon resting inside.",
+      "A sealed jar rests on a wooden table next to a bottle of vodka.\n"
+    ]
+  },
+  "15": {
+    "steps": [
+      "In a bowl, add Milk, Granulated Sugar, Cream, and Vanilla Extract. Whisk until combined.",
+      "Pour into ice cream churner. Churn for 12-15 minutes.",
+      "Refrigerate for 30 minutes.",
+      "Add M&M's\u00ae Milk Chocolate Minis. Mix through."
+    ],
+    "title": "Homemade McDonald's McFlurry",
+    "captions": [
+      "A bowl containing milk, granulated sugar, cream, and vanilla extract.",
+      "Milk being poured into a blender.",
+      "A bowl of ice cream being placed in a refrigerator.",
+      "A bowl of chocolate M&Ms on a white table.\n"
+    ]
+  },
+  "18": {
+    "steps": [
+      "Using a mandolin with the julienne attachment, slice the Zucchini on all four sides into spaghetti-like strands. Stop before you reach the seeds.",
+      "Tear the Fresh Mint Leaves or thinly slice with a knife by dragging it across to prevent bruising.",
+      "Toss the zucchini in a bowl with the Extra-Virgin Olive Oil, Lemon, and torn mint leaves. Season with Kosher Salt and freshly ground Ground Black Pepper.",
+      "Using a vegetable peeler, make shavings of the Pecorino Romano Cheese. You can also finely grate the cheese with a microplane."
+    ],
+    "title": "Zucchini, Pecorino & Mint Salad",
+    "captions": [
+      "Close-up view of a woman using a mandolin slicer with the julienne blade to create thin zucchini noodles.",
+      "A hand using a knife to finely chop fresh mint leaves on a wooden cutting board.",
+      "A glass bowl filled with zucchini noodles being tossed with olive oil, lemon juice, torn mint leaves, salt, and pepper.",
+      "A hand using a vegetable peeler to create thin shavings of Pecorino Romano cheese.\n"
+    ]
+  },
+  "22": {
+    "steps": [
+      "Chop pistachios. Peel and fillet oranges, mandarins, and grapefruit, reserving the juice. Set aside in a bowl",
+      "Mix yogurt, cinnamon, and vanilla sugar in a bowl and set aside. In another mixing bowl, whisk together eggs and milk. Cut panettone into 2-cm/0.66-in. thick slices and soak each side in egg mixture.",
+      "In a frying pan, melt butter over medium heat, and fry panettone for approx. 3 min. on each side, or until golden brown.",
+      "Serve French toast warm with yogurt and citrus fruits. Sprinkle with pistachios."
+    ],
+    "title": "Panettone French toast",
+    "captions": [
+      "Close-up view of a woman's hands skillfully chopping pistachios on a wooden cutting board using a sharp knife.",
+      "A woman carefully cuts a slice of panettone, then dips it into a bowl of lightly beaten egg mixture.",
+      "Golden-brown slices of panettone are frying in a hot pan on a stovetop.",
+      "A plate of delicious-looking panettone French toast, garnished with creamy yogurt and segments of fresh citrus fruit.\n"
+    ]
+  },
+  "24": {
+    "steps": [
+      "Halve pomegranate, deseed, and set aside. Finely chop garlic and set aside. Finely chop mint. Add yogurt to a bowl and mix with chopped mint and cumin. Add salt and sugar to taste.",
+      "Preheat oven to 180\u00b0C/350\u00b0F. Heat olive oil in a pan over medium-high heat. Season lamb with salt and pepper and sear for approx. 2 min. on both sides along with chopped garlic. Remove from pan and set aside. In another pan, toast pistachios over medium-high heat for approx. 2 min. Set aside.",
+      "Transfer pita bread to oven and toast for approx. 2 min. on 180\u00b0C/350\u00b0F. Thinly slice lamb fillet.",
+      "Remove pita bread from oven and fill with yogurt sauce, pomegranate seeds, toasted pistachios, and thinly sliced lamb. Serve immediately."
+    ],
+    "title": "Lamb and pomegranate sandwich",
+    "captions": [
+      "Close-up view of hands using a knife to chop a pomegranate on a wooden cutting board.",
+      "A pan on a stovetop with lamb cooking, showing some browning and sizzling.",
+      "A person slicing a lamb fillet on a wooden cutting board with a knife.",
+      "A pita bread being filled with yogurt sauce, pomegranate seeds, toasted pistachios, and thinly sliced lamb.\n"
+    ]
+  },
+  "26": {
+    "steps": [
+      "Cut the upper half off the Bread Boule. Carve out the bread from the bottom half to form a bowl.",
+      "Add the Campbell's New England Clam Chowder, Milk and Cream into a pan. Season with Ground Black Pepper.",
+      "Heat over medium high heat and stir everything together. Heat to a boil, and it is ready to serve.",
+      "Ladle soup into the bread bowl. Finely chop some Fresh Parsley and/or Fresh Chives for garnish."
+    ],
+    "title": "New England Clam Chowder",
+    "captions": [
+      "Close-up of hands carefully carving out the inside of a bread boule, creating a bread bowl.",
+      "A cast iron pan containing a bread bowl filled with a dark, seasoned soup.",
+      "A cast iron pan containing a bread bowl filled with a dark, seasoned soup.",
+      "A rustic wooden table with a bread bowl filled with soup, a spoon resting inside, and a piece of bread on the side.\n"
+    ]
+  },
+  "27": {
+    "steps": [
+      "Cook pasta in plenty of salted boiling water, according to package instructions, for approx. 10 \u2013 12 min. until al dente. Drain, save some pasta water, and set aside.",
+      "Peel onion and garlic cloves and chop finely. Dice eggplant into cubes.",
+      "Heat up some olive oil in a large frying pan and saut\u00e9 onion and garlic for about 2 \u2013 4 min. until translucent. Add eggplant to pan and fry for approx. 6 \u2013 8 min. until browned on all sides and somewhat soft.",
+      "Add whole peeled tomatoes and thyme leaves. Boil everything over medium heat for about 5 \u2013 8 min. Season generously with salt and pepper. Toss cooked pasta in the sauce and serve in deep plates with a dollop of ricotta and fresh basil."
+    ],
+    "title": "Whole wheat pasta alla norma",
+    "captions": [
+      "Close-up view of a pot of boiling pasta on a stovetop.",
+      "A person chopping an onion and eggplant on a wooden cutting board.",
+      "A frying pan on a stovetop with eggplant cooking in it.",
+      "A frying pan on a stovetop with pasta cooking in it.\n"
+    ]
+  },
+  "28": {
+    "steps": [
+      "Bring a pot of water to a boil on the stove top. Add Elbow Macaroni, and cook until al dente, about 8-10 minutes.",
+      "In a separate pan, over medium heat, melt the Butter. Once melted, pour in Milk and plain All-Purpose Flour. Whisk until flour is completely dissolved. Continue stirring until mixture comes to a boil.",
+      "Turn heat down to a simmer, stirring constantly, and cook for 2 minutes. Add Cheddar Cheese, and mix until cheese is fully melted.",
+      "Add drained macaroni, and mix until thoroughly combined."
+    ],
+    "title": "Macaroni and Cheese",
+    "captions": [
+      "A pot of boiling water with elbow macaroni cooking on a stovetop.",
+      "A saucepan with milk and flour simmering on a stovetop.",
+      "A saucepan with cheese sauce, made from cheese and flour, bubbling on a stovetop.",
+      "A pan containing macaroni and cheese, ready to be served, on a stovetop.\n"
+    ]
+  },
+  "31": {
+    "steps": [
+      "Add rolled oats, coconut milk, almond milk, vanilla extract, ground cinnamon, and ground cardamom to a pot over medium-high heat and bring to a simmer. Reduce heat to medium-low, cover, and let cook for approx. 10 min., or until the oats are soft and almost all liquid is absorbed.",
+      "In the meantime, toast cashews in a frying pan until fragrant. Remove from heat and set aside.",
+      "As soon as the porridge is done, remove from heat, add toasted cashews and honey, and stir to combine.",
+      "Peel and slice mango. Serve warm porridge in bowls and top with sliced mango. Sprinkle with coconut flakes, cocoa nibs, and black sesame seeds. Drizzle with some more honey, if desired, and"
+    ],
+    "title": "Honey-coconut porridge",
+    "captions": [
+      "A pot on a stovetop containing oatmeal simmering in coconut milk, almond milk, vanilla extract, ground cinnamon, and ground cardamom.",
+      "A frying pan with toasted cashews.",
+      "A pot of oatmeal being stirred with a spoon and a wooden spoon.",
+      "A bowl of oatmeal topped with sliced mango, coconut flakes, cocoa nibs, and black sesame seeds.\n"
+    ]
+  },
+  "32": {
+    "steps": [
+      "Add Beef Rump Roast, Pepperoncini Pepper, Dry Italian Salad Dressing Mix, Brown Gravy Mix, and Water to the Instant Pot.",
+      "Close and lock lid. Cook on Manual High Pressure for 60 minutes, then natural release pressure.",
+      "Rest for 5 minutes. Shred big chunks of beef to allow juice and peppers to mix in with the meat.",
+      "Serve alone, with side dishes, on a sandwich or any way you want!"
+    ],
+    "title": "Instant Pot Pepperoncini Italian Beef",
+    "captions": [
+      "A woman adds beef rump roast, pepperoncini peppers, dry Italian salad dressing mix, brown gravy mix, and water to an Instant Pot.",
+      "An Instant Pot cooking a beef roast submerged in water.",
+      "A woman shreds large chunks of beef, allowing the juices and peppers to mix with the meat.",
+      "A bowl of shredded beef soup served on a wooden table.\n"
+    ]
+  },
+  "35": {
+    "steps": [
+      "Add Chicken Backbones, Fennel Bulb, Onion, Bay Leaf, Black Peppercorns, Fresh Ginger, Garlic, and Jalape\u00f1o Pepper to a stock pot. Cover with Water.",
+      "Bring to boil. Reduce to a light simmer. Cover. Cook 24-48 hours on lowest setting.",
+      "Season withSoy Sauce to taste before serving!"
+    ],
+    "title": "Bone Broth",
+    "captions": [
+      "A stock pot filled with chicken backbones, fennel bulb, onion, bay leaf, black peppercorns, fresh ginger, garlic, and jalape\u00f1o pepper.",
+      "A pot of soup simmering on a stove top.",
+      "A bowl of finished soup with a spoon resting in it.\n"
+    ]
+  },
+  "39": {
+    "steps": [
+      "Slice off both ends of the oranges. Then, cut off the remaining peel and remaining pith, if necessary. Fillet them over a small bowl by cutting down along the sides of the inner membrane to get segments. Add juice from the segmented oranges to the bowl.",
+      "Caramelize sugar in a frying pan over medium heat and deglaze with orange segments and juice. Let simmer for approx. 2 min.",
+      "In a heatproof bowl over a pot of simmering water, melt white chocolate. Add heavy cream to a bowl and whip with a hand mixer with beaters until soft peaks form. In another bowl, mix Greek yogurt with melted white chocolate. Carefully fold the whipped cream into the yogurt- mixture, one half at a time. Transfer to a piping bag and chill for approx. 1 hr.",
+      "Layer yogurt-white chocolate cream, oranges, and chopped pistachios in serving glasses. Garnish with white chocolate shavings and"
+    ],
+    "title": "Creamy yogurt-white chocolate parfait with orange",
+    "captions": [
+      "A woman carefully peeling an orange.",
+      "A frying pan with glistening caramelized sugar.",
+      "A bowl of melted white chocolate and another bowl of whipped cream; a third bowl contains a mixture of Greek yogurt and melted white chocolate.",
+      "Elegant serving glasses filled with layers of yogurt-white chocolate cream, orange segments, and chopped pistachios, garnished with white chocolate shavings.\n"
+    ]
+  },
+  "41": {
+    "steps": [
+      "Measure out semolina and flour onto a clean working surface. Form a hollow in the middle and add eggs. Whisk eggs with a fork and incorporate flour little by little, then knead to combine. If needed, add some water. Knead dough for approx. 10 min., wrap in plastic wrap, and let rest for approx. 30 min.",
+      "In the meantime, add potatoes to a large pot and cover with water, then season with salt. Cook for approx. 20 min., or until fork-tender. In the meantime, brown butter in a small pot and let cool down. Slice mint into thin ribbons.",
+      "Drain potatoes, add to a bowl, and mash with a fork. Add grated Parmesan cheese, browned butter, salt, pepper, mint, and stir to combine.",
+      "Quarter pasta dough and use a pasta machine to roll out the dough. Stamp out discs of dough for the ravioli with a round cookie cutter. Add a spoonful of potato and mint filling to one disc and brush the dough with some water. Place a second disc of dough on top and press around the edges to seal. Repeat the process with remaining dough and filling.",
+      "Bring a large pot water to a boil, salt well, and cook ravioli for approx. 5 min. In the meantime, melt butter in a large frying pan and add sage leaves and a ladle of cooking water from the ravioli. Add ravioli to the frying pan and toss to coat. Serve ravioli with fresh mint and black pepper."
+    ],
+    "title": "Potato and mint ravioli with sage butter",
+    "captions": [
+      "A clean work surface with measured semolina and flour, a well made in the center, eggs added to the well, and a fork whisking the eggs while incorporating the flour.",
+      "A large pot of boiling water with potatoes and salt.",
+      "A bowl of fluffy mashed potatoes being stirred with a fork.",
+      "A pasta machine rolling out dough, round ravioli cutouts brushed with water, a spoonful of potato and mint filling added to one disc, and the edges sealed.",
+      "A pot of boiling ravioli, a frying pan with melted butter and sage, and cooked ravioli tossed in the butter and sage, ready to be served.\n"
+    ]
+  },
+  "42": {
+    "steps": [
+      "Pre-heat oven to 210\u00b0C/410\u00b0F. Slice sweet potato approx. 1-in. thick. Peel and cut onion into wedges. Peel and finely dice garlic. Melt coconut oil in a small pot over low heat.",
+      "Add sliced sweet potato, onion, shredded coconut, melted coconut oil, garlic, sea salt, and pepper to a baking sheet and toss to coat. Bake at 210\u00b0C/410\u00b0F for approx. 20 min. Then remove from the oven and let cool down a bit.",
+      "Peel and chop ginger. Transfer sweet potato mixture to a blender. Add ginger, lemon zest, hot vegetable stock, and orange juice. Blend for approx. 40 sec. on high, or until the mixture is smooth. Season with salt and pepper to taste.",
+      "Add coconut flakes to a pan set over medium heat and toast until they are slightly browned. Transfer soup to serving bowls. Serve with diced mango, toasted coconut flakes, sliced green onion, chopped cilantro, and lime zest on top.",
+      "This and many more quick recipes can be found in Sebastian Copien\u00b4s cookbook \u201cFit-Mix\u201d (ZS Verlag)."
+    ],
+    "title": "Sweet potato and orange soup",
+    "captions": [
+      "Close-up view of hands slicing a sweet potato on a wooden cutting board.",
+      "A baking sheet filled with sliced sweet potatoes, onions, shredded coconut, and seasonings.",
+      "A blender filled with cooked sweet potato, creating a smooth puree.",
+      "A pan with toasted coconut flakes, lightly browned and fragrant.\n"
+    ]
+  },
+  "43": {
+    "steps": [
+      "In a large bowl, mix together flour, water, oil, and salt and knead until a smooth dough forms. Cover with kitchen towel and let rest for approx. 15 min.",
+      "In a small bowl, season cr\u00e8me fra\u00eeche with nutmeg and pepper. Dice bacon and thinly slice onions. Finely chop chives and set aside.",
+      "Preheat oven to 220\u00b0C/425\u00b0F. Roll out dough onto a sheet of parchment paper. Transfer to a baking sheet covered with parchment paper. Spread cr\u00e8me fra\u00eeche over dough and top with onion rings and diced bacon.",
+      "Bake at 220\u00b0C/425\u00b0F for approx. 15 \u2013 20 min. Garnish with chopped chives and season with more pepper to taste."
+    ],
+    "title": "Shortcut tarte flamb\u00e9e",
+    "captions": [
+      "Close-up view of hands kneading dough in a large bowl using a wooden spoon.",
+      "A small bowl containing cr\u00e8me fra\u00eeche, with a sprinkle of nutmeg and pepper nearby.  Diced bacon, thinly sliced onions, and finely chopped chives are arranged separately on a surface.",
+      "A person rolling out a circle of dough on a sheet of parchment paper, then carefully transferring it to a baking sheet lined with parchment paper.",
+      "A person gently placing a round cake onto a baking sheet lined with parchment paper.\n"
+    ]
+  },
+  "44": {
+    "steps": [
+      "Preheat oven to 180\u00b0C/350\u00b0F. Add flour, gingerbread spice, cinnamon, salt, and baking powder to a bowl and combine.",
+      "In another bowl, beat soft butter, cane sugar and egg for approx. 5 min. until smooth. Stir in applesauce.",
+      "Gradually add the dry ingredients and mix into a smooth dough.",
+      "Grease loaf pan and line with parchment paper. Transfer dough into loaf pan and bake at 180\u00b0C/350\u00b0F for approx. 50 - 60 min., or until a toothpick inserted into the center comes out clean.",
+      "Remove from oven and let cool completely. For the frosting, beat soft butter, confectioner's sugar, cream cheese, and vanilla extract until fluffy.",
+      "Unmold cake and spread the frosting on top. Dust with cinnamon."
+    ],
+    "title": "Gingerbread loaf cake",
+    "captions": [
+      "Close-up view of a bowl containing flour, gingerbread spice, cinnamon, salt, and baking powder, being mixed together.",
+      "A woman stirring a bowl of applesauce with a spoon.",
+      "A woman adding dry ingredients to a bowl of applesauce.",
+      "A woman carefully placing a prepared loaf of dough into a loaf pan, which is then shown in an oven baking at 180\u00b0C/350\u00b0F.",
+      "A woman beating a bowl of frosting with a whisk.",
+      "A woman spreading frosting on top of a baked loaf cake with a spatula.\n"
+    ]
+  },
+  "46": {
+    "steps": [
+      "Preheat oven to 200\u00b0C/400\u00b0F and line a baking sheet with parchment paper, if needed. Peel carrots and roughy dice. Add to a large mixing bowl along with garlic, olive oil, and cumin and stir to coat.",
+      "Add carrot mixture to the prepared baking sheet, transfer to oven, and roast for approx. 20 min. at 200\u00b0C/400\u00b0F until carrots are tender.",
+      "Drain chickpeas and reserve the liquid. Transfer carrot mixture, chickpeas, lemon juice, lemon zest, and paprika powder to a food processor and blend until creamy. Add reserved liquid from chickpeas, a little at a time while blending, until smooth and silky in texture. Season with salt and pepper to taste.",
+      "Transfer hummus to a serving bowl, drizzle with olive oil, and sprinkle with paprika powder and Ras el Hanout to taste. Cut pita bread into triangles and toast. Serve alongside hummus and"
+    ],
+    "title": "Carrot hummus with pita bread",
+    "captions": [
+      "Close-up view of hands using a knife to finely chop carrots on a parchment-lined baking sheet.",
+      "A spatula smoothly spreading a mixture of chopped carrots onto a baking sheet.",
+      "A food processor blending a vibrant mixture of carrots, chickpeas, lemon juice, lemon zest, and paprika powder.",
+      "A bowl filled with creamy hummus, topped with a sprinkle of paprika powder and Ras el Hanout.\n"
+    ]
+  },
+  "47": {
+    "steps": [
+      "Prepare the Onion, Habanero Pepper, Tomato, Scallion, and Bell Pepper. Normally bell peppers and scallions are not used like this in its original form is very rustic, but using up the stuff you have in the fridge when you cook reduces waste.",
+      "Heat a pan on medium heat, then empty the contents of the Corned Beef Lunch Meat into it and break it apart. The corned beef itself is very fatty, so basically there\u2019s no real need for oil.",
+      "Toss everything into the pan with the corned beef and hit it with a dash of Ground Black Pepper and Ketchup. Don\u2019t cover or you\u2019ll risk everything going soggy and into a \u201cmush\u201d. Cook on medium heat while stirring for 5-7 minutes.",
+      "Serve with sliced cucumber on the side and some rice. This can also be enjoyed on sandwiches, as a topping for crackers, roti, sliced bread.. even stuffed in pitas."
+    ],
+    "title": "Classic Caribbean Corned Beef",
+    "captions": [
+      "Close-up view of hands chopping a bell pepper and scallions on a wooden cutting board.",
+      "A pan sits on a stovetop, its contents\u2014corned beef\u2014being gently broken apart with a spoon.",
+      "A pan sits on a stovetop, its contents\u2014corned beef\u2014being gently broken apart with a spoon.",
+      "A bowl filled with a refreshing cucumber salad; a spoon rests inside.\n"
+    ]
+  },
+  "50": {
+    "steps": [
+      "Combine the Frozen Pineapple, Frozen Strawberries, Pear, Frozen Blueberries, and Water in a blender and blend until smooth.",
+      "Transfer the slushy fruits to a large punch bowl and add the Sprite just before serving.",
+      "Your punch is ready to serve! Scoop into glasses."
+    ],
+    "title": "Sparkling Fruit Slushy",
+    "captions": [
+      "Blender filled with frozen pineapple, strawberries, pear, blueberries, and water.",
+      "Large punch bowl filled with a slushy mixture of fruits and Sprite.",
+      "Punch bowl filled with a slushy mixture of fruits and Sprite.\n"
+    ]
+  },
+  "51": {
+    "steps": [
+      "Heat oil in a pan. Add Chili Powder, Ground Black Pepper, Ground Turmeric, Ground Coriander, Salt, and Curry Leaf. Mix well for 5 minutes over a low flame.",
+      "Halve the Egg and place them in the masalas. Let it fry for 1 minute.",
+      "Flip the eggs carefully without damaging the yolk and roast for another minute. Switch off the flame. Yummy egg fry is ready."
+    ],
+    "title": "Simple Egg Fry",
+    "captions": [
+      "A pan with chili powder, ground black pepper, ground turmeric, ground coriander, salt, and curry leaf.",
+      "Halved eggs frying gently in a pan of spices.",
+      "A person flipping eggs in a pan on a stove.\n"
+    ]
+  },
+  "54": {
+    "steps": [
+      "Pre-heat oven to 220\u00b0C/430\u00b0F. Cut sweet potatoes into matchsticks and transfer them to a large bowl. Add cornstarch and toss to coat. Spread sweet potato fries over a baking sheet. Add some oil and smoked ground paprika and toss to coat. Bake for approx. 20 min. or until golden-brown and crisp. Remove from oven and season with salt to taste.",
+      "Finely slice white cabbage and transfer to a bowl. Add cashew cream, white balsamic vinegar, coriander, salt, and pepper, and stir to combine. Let sit until serving to marinate.",
+      "Add vegan mayonnaise, cinnamon, and chili powder to a small bowl and stir to combine. Refrigerate until serving.",
+      "Heat remaining oil in a frying pan. Add oyster mushrooms and fry over medium heat for approx. 8 min. Remove from heat and season with salt to taste.",
+      "Spread some ketchup and vegan spicy cinnamon mayonnaise on each toasted white bread slice. Top half of the slices with lamb\u2019s lettuce, sweet potato fries, white cabbage slaw, fried oyster mushrooms, and some sliced red onion. Place remaining white bread slices on top.",
+      "This and more delicious recipes can be found in Sebastian Copien\u00b4s cookbook \u201cDie vegane Kochschule\u201c (Christian Verlag)."
+    ],
+    "title": "Winter sandwich with spicy cinnamon mayo",
+    "captions": [
+      "Woman chopping sweet potatoes into matchsticks, tossing them with cornstarch, and arranging them on a baking sheet with oil and smoked paprika.",
+      "Bowl of white cabbage with white balsamic vinegar being poured over it.",
+      "Small bowl with vegan mayonnaise, cinnamon, and chili powder being stirred together.",
+      "Oyster mushrooms cooking in a frying pan.",
+      "Two slices of toasted white bread topped with ketchup, vegan spicy cinnamon mayonnaise, lamb's lettuce, sweet potato fries, white cabbage slaw, fried oyster mushrooms, and sliced red onion.",
+      "A cookbook titled \"Die vegane Kochschule\" rests on a wooden table.\n"
+    ]
+  }
+}

models/CoSeD/checkpoints/epoch=19-step=140.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79988a3d86f30cf7ea8c738b7c195be72d19fc075a8172a06fd239cd20d0ddc1
+size 10531819

models/CoSeD/cross_attention.py

@@ -0,0 +1,26 @@
+from BeamDiffusionModel.models.CoSeD.sequence_predictor import SoftAttention
+import torch
+
+model = SoftAttention.load_from_checkpoint(
+    "./BeamDiffusionModel/models/CoSeD/checkpoints/epoch=19-step=140.ckpt")
+# "/user/home/vcc.ramos/latent_training/sft/reference_training/9q3eu8vi/checkpoints/epoch=7-step=15.ckpt"
+
+model.eval()
+
+
+def get_softmax(previous_steps_embeddings, previous_images_embeddings, current_steps_embeddings,
+                current_images_embeddings):
+    previous_steps_tensor = torch.cat(previous_steps_embeddings, dim=0).to("cpu").unsqueeze(0)
+    previous_images_tensor = torch.cat(previous_images_embeddings, dim=0).to("cpu").unsqueeze(0)
+    current_steps_tensor = torch.cat(current_steps_embeddings).to("cpu").unsqueeze(0)
+    current_images_tensor = torch.cat(current_images_embeddings).to("cpu").unsqueeze(0)
+    with torch.no_grad():
+        softmax, logit = model(current_steps_tensor,
+                               current_images_tensor,
+                               previous_steps_tensor,
+                               previous_images_tensor
+                               )
+        if len(softmax.shape) <= 1:
+            return softmax
+        # sum and normalize the softmax values
+        return torch.sum(softmax, dim=-1) / softmax.shape[-1]

models/CoSeD/sequence_predictor.py

@@ -0,0 +1,675 @@
+import math
+import pickle
+import sys
+
+import torch
+from torch import nn, optim
+from torch.utils.data import DataLoader
+import torch.nn.functional as F
+
+import lightning as L
+from lightning.pytorch.tuner import Tuner
+from lightning.pytorch.callbacks import LearningRateMonitor
+import wandb
+from pytorch_lightning.loggers import WandbLogger
+from transformers import CLIPTokenizer, CLIPTextModelWithProjection
+
+
+
+class SoftAttention(L.LightningModule):
+    def __init__(self, learning_rate=0.001, batch_size=10, unfreeze=0, random_text=False, random_everything=False,
+                 fixed_text=False, random_images=False):
+        super(SoftAttention, self).__init__()
+        self.my_optimizer = None
+        self.my_scheduler = None
+        self.save_hyperparameters()
+        self.learning_rate = learning_rate
+        self.batch_size = batch_size
+        self.frozen = False
+        self.unfreeze_epoch = unfreeze
+        self.loss_method = torch.nn.CrossEntropyLoss()
+        self.train_sum_precision = 0
+        self.train_sum_accuracy = 0
+        self.train_sum_recall = 0
+        self.train_sum_runs = 0
+        self.val_sum_precision = 0
+        self.val_sum_accuracy = 0
+        self.val_sum_recall = 0
+        self.val_sum_runs = 0
+
+        # NETWORK
+        # Linear layers to reduce dimensionality
+        self.text_reduction = torch.nn.Linear(512, 256)
+        self.image_reduction = torch.nn.Linear(512, 256)
+
+        # Soft attention weights
+        self.W_query_text_half_dim = torch.nn.Linear(256, 256)
+        self.W_query_image_half_dim = torch.nn.Linear(256, 256)
+        self.W_query_text_full_dim = torch.nn.Linear(512, 512)
+        self.W_query_image_full_dim = torch.nn.Linear(512, 512)
+
+        self.W_key_text_half_dim = torch.nn.Linear(256, 256)
+        self.W_key_image_half_dim = torch.nn.Linear(256, 256)
+        self.W_key_image_full_dim = torch.nn.Linear(512, 512)
+        self.W_key_text_full_dim = torch.nn.Linear(512, 512)
+
+        # TO TEST THE MODEL WITH SAME TEXT
+        self.fixed_text = torch.tensor([2.2875e-01, 2.3762e-02, 1.3448e-01, 6.5997e-02, 2.5605e-01,
+                                        -1.6183e-01, 7.1169e-03, -1.6895e+00, 1.8110e-01, 1.7249e-01,
+                                        7.0582e-02, -6.3566e-02, -1.5862e-01, -2.3586e-01, 6.9382e-02,
+                                        9.4649e-02, 6.3127e-01, -4.1287e-02, -4.9883e-02, -2.1821e-01,
+                                        5.8677e-01, -2.5353e-01, 1.4792e-01, 2.2195e-02, -6.8436e-02,
+                                        -1.5512e-01, -9.8894e-02, 6.3377e-02, -2.3078e-01, 9.3588e-02,
+                                        5.2875e-02, -5.1388e-01, -7.0461e-02, 2.4253e-02, -7.8069e-02,
+                                        7.6921e-02, -1.1610e-01, -1.3345e-01, 7.8038e-03, -2.0226e-01,
+                                        1.1381e-01, -9.6335e-02, -2.2195e-02, -6.5028e-02, 1.4025e-01,
+                                        2.6969e-01, -1.0758e-01, 3.6736e-02, 3.2893e-01, -1.9067e-01,
+                                        4.9070e-02, 8.0207e-02, 7.2942e-02, 7.7496e-03, 2.0883e-01,
+                                        1.7339e-01, 1.0072e-01, -1.7874e-01, -4.6898e-02, -6.2682e-02,
+                                        5.9596e-02, 5.2925e-02, 2.4633e-01, -7.2811e-02, -1.4157e-01,
+                                        8.8013e-03, -4.6815e-02, -7.4260e-02, 8.6530e-03, -1.8174e-01,
+                                        1.6101e-01, -4.8832e-02, -5.8030e-02, -3.2518e-02, -6.2896e-02,
+                                        -2.3472e-01, -8.0996e-02, 1.1261e-01, -2.1039e-01, -2.3837e-01,
+                                        -2.6827e-02, -2.3075e-01, -2.2087e-02, 5.4009e-01, 3.7671e-02,
+                                        3.3140e-01, -4.2569e-02, -1.6946e-01, 1.7165e-01, 3.0887e-01,
+                                        4.9847e-02, 1.2438e-02, -2.0701e+00, 2.7104e-01, 1.9001e-01,
+                                        3.1907e-01, -9.1116e-02, -8.3141e-02, 4.5765e-03, -2.5675e-01,
+                                        -2.2119e-02, 3.4949e-02, 2.8192e-01, 7.9688e-02, -2.1810e-01,
+                                        8.1565e-02, 3.3208e-01, -9.1857e-02, -2.1145e-01, -1.6843e-01,
+                                        6.7942e-02, 5.1067e-01, -1.6835e-01, 2.2090e-02, 1.8061e-02,
+                                        -2.1313e-01, 2.6867e-02, -2.2734e-01, 8.4164e-02, -4.7868e-02,
+                                        2.0980e-02, -2.1424e-01, -2.2919e-02, 1.7554e-01, 5.2253e-02,
+                                        -2.2049e-01, 6.9408e-02, 7.0811e-02, -1.1892e-02, -4.7958e-02,
+                                        7.9476e-02, 1.8851e-01, 2.2516e-02, 8.6119e+00, -7.8583e-02,
+                                        1.0218e-01, 1.6675e-01, -4.0961e-01, 4.5291e-02, 7.9783e-02,
+                                        -1.1764e-01, -2.3162e-01, -2.7717e-02, 1.2963e-01, -3.0165e-01,
+                                        -2.1588e-02, -1.2324e-01, 1.9732e-02, -1.9312e-01, -7.1229e-02,
+                                        2.5102e-01, -4.1674e-01, -1.5610e-01, -6.1321e-03, -4.5332e-02,
+                                        6.1500e-02, -1.5942e-01, 3.5142e-01, -2.1119e-01, 4.5057e-02,
+                                        -5.6277e-02, -3.4298e-01, -1.6499e-01, -2.9384e-02, -2.7163e-01,
+                                        6.5339e-03, 2.7674e-02, -1.1302e-01, -2.6373e-02, -1.4370e-01,
+                                        2.1936e-01, 1.3103e-01, 2.5538e-01, 1.9502e-01, -1.5278e-01,
+                                        1.4978e-01, -2.5552e-01, 2.2397e-01, -1.0369e-01, -1.0491e-01,
+                                        5.1112e-01, 2.4879e-01, 7.0940e-02, 1.7351e-01, -3.6831e-02,
+                                        1.5027e-01, -1.9452e-01, 2.0322e-01, 8.5931e-02, -2.8588e-03,
+                                        3.1146e-02, -3.3307e-01, 1.1595e-01, 1.9435e-01, -3.4536e-02,
+                                        2.5245e-01, 4.5388e-02, 2.1197e-02, 4.2232e-02, 4.2436e-02,
+                                        4.9622e-02, -2.0907e-01, 1.2264e-01, -7.3529e-02, -2.1788e-01,
+                                        -1.2429e-01, -8.1422e-02, 1.6572e-01, -6.0989e-02, 8.0322e-02,
+                                        3.3477e-01, -7.2207e-02, -8.8658e-02, -2.4944e-01, 9.9211e-02,
+                                        8.6244e-02, 8.8807e-02, -1.9676e-01, -4.5365e-03, -3.7754e-01,
+                                        -1.7204e-01, -1.3001e-01, 6.4961e-02, -5.8192e-03, 2.4670e-01,
+                                        -8.3591e-02, -3.0810e-01, -3.4549e-02, -1.4452e-01, -5.5416e-02,
+                                        1.0527e-02, 3.1159e-01, -1.3857e-01, -2.2676e-01, 1.4768e-01,
+                                        3.2650e-01, 2.3971e-01, 6.8196e-02, -2.6235e-02, -2.9741e-01,
+                                        4.7721e-02, -1.2859e-02, 2.0340e-01, 1.7823e-02, -1.1337e-01,
+                                        4.4077e-02, -1.3949e-01, 2.9229e-01, 1.7425e-01, -5.0722e-03,
+                                        -6.3722e-02, 1.0181e-01, 2.3344e-02, 2.2200e-01, 3.5022e-02,
+                                        1.5361e-01, -1.0702e-03, 2.9319e-02, 1.8938e-01, -7.2263e-02,
+                                        2.2192e-02, 9.5394e-02, -4.4459e-03, 7.6698e-02, -1.7830e-01,
+                                        1.0213e-01, -8.8493e-02, -1.6439e-01, -1.1085e-01, 1.2938e-01,
+                                        2.3929e-01, -4.9047e-02, -1.2814e-01, -2.1075e-01, 2.4423e-01,
+                                        -4.4565e-02, -5.1225e-02, -4.0214e-02, -1.4033e-01, 6.3284e-02,
+                                        4.7094e-01, -2.6821e-02, 2.1138e-02, 1.1590e-01, -2.0023e-02,
+                                        1.7200e-01, 3.8215e-01, -2.4871e-01, -1.5359e-01, 2.4691e-01,
+                                        1.4904e-01, -1.0636e-01, 2.4185e-01, 1.7119e-03, 1.4618e-01,
+                                        -1.6813e-01, -4.4372e-01, -1.7475e-01, -6.9891e-02, -4.5553e-02,
+                                        9.3102e-02, 1.7686e-02, -1.1781e-01, 6.9423e-02, 1.0211e-02,
+                                        3.2742e-01, 7.5272e-02, 8.5080e-02, -1.7731e-01, 1.4030e-01,
+                                        2.7764e-01, -6.5041e-02, 8.5968e+00, 2.5900e-01, -2.0825e-01,
+                                        9.6241e-02, -1.5257e-01, -3.4269e-01, -1.1251e-01, 3.0549e-01,
+                                        3.1628e-01, 6.1856e-01, 1.5791e-03, 6.5656e-02, 1.8862e-02,
+                                        -7.1927e-02, 1.3239e-01, -1.1126e-01, 1.1135e-02, -3.2411e+00,
+                                        -4.7349e-02, 1.4775e-01, -9.7712e-02, 4.5727e-02, -1.3868e-01,
+                                        2.1260e-01, 1.5465e-01, 1.1308e-01, -8.0110e-02, -1.3123e-01,
+                                        1.8527e-01, -8.6424e-02, -1.9778e-01, -1.3295e-01, -1.5880e-01,
+                                        2.0800e-01, -3.6513e-02, 2.6472e-02, 2.7275e-01, 1.8995e-01,
+                                        -7.7340e-02, 1.2059e-02, 3.5163e-02, 1.5442e-02, 5.1417e-02,
+                                        5.0993e-01, 1.2994e-01, 2.3873e-01, -7.2816e-02, 1.5850e-01,
+                                        -2.0404e-01, -2.2941e-01, 2.3660e-01, 2.0418e-01, 6.7775e-02,
+                                        -3.9195e-01, 3.6655e-01, 1.6498e-01, 6.4065e-02, 4.9579e-02,
+                                        2.8265e-01, -5.9919e-03, 4.0163e-02, 8.9072e-02, 1.5125e-01,
+                                        9.0711e-02, -1.2608e-01, -1.0413e-01, -2.1931e-01, 5.0183e-02,
+                                        -3.4841e-02, -8.1449e-02, -1.1225e-01, -4.5787e-02, -7.8871e-02,
+                                        3.8858e-02, 9.2660e-02, 1.5991e-01, -6.7528e-02, -6.3166e-02,
+                                        -4.7824e-03, -1.3528e-01, 1.4845e-01, 2.0460e-01, -9.3238e-02,
+                                        1.4902e-03, 1.1896e-01, -3.1337e-01, 2.1637e-02, 1.4990e-01,
+                                        -2.1179e-03, -8.1374e-02, -1.0241e-01, -8.0754e-02, -1.4449e-01,
+                                        -1.3549e-01, -7.5588e-02, -8.0083e-02, -1.4114e-01, 2.9467e-03,
+                                        3.5340e-01, -4.3351e-02, 9.6934e-02, 1.3625e-01, 1.3339e-01,
+                                        -1.2059e-02, -1.4325e-01, -2.1202e-01, 3.8758e-02, 2.5965e-01,
+                                        -7.8454e-02, 1.5983e-01, 1.0115e-02, 2.2192e-01, -1.4043e-01,
+                                        6.7966e-02, -1.4672e-01, -1.8846e-01, 1.9488e-01, 1.2942e-01,
+                                        -1.3165e-02, -1.6099e-01, -9.6146e-02, 1.3439e-01, -5.0560e-02,
+                                        8.2779e-02, -2.4827e-01, -7.8047e-02, -3.1163e-01, -1.7481e-01,
+                                        2.1450e-01, -7.6112e-02, -1.9967e-02, 5.7099e-02, 7.7664e-02,
+                                        -7.9647e-02, 3.3941e-02, 2.9551e-02, 1.4231e-01, 2.3480e-02,
+                                        1.5209e-01, -2.0011e-01, 1.1153e-01, 1.2694e-01, 8.7853e-02,
+                                        2.6997e-01, 1.3525e-01, 1.9541e-01, 3.4429e-03, -9.6446e-02,
+                                        7.6708e-02, -3.0698e-02, -1.8507e-01, 2.5645e-01, 2.8182e-01,
+                                        -1.2282e-01, -1.1017e-01, 2.2249e-01, 2.1966e-01, 3.5795e-01,
+                                        1.6279e-01, 1.7276e-01, 2.1410e-01, -3.2499e-01, 5.0327e-02,
+                                        7.9813e-02, -1.5915e-01, -3.6175e-02, 1.4376e-01, 2.9565e-01,
+                                        6.9097e-02, -8.0661e-01, 4.9966e-02, 6.2506e-02, 1.8852e-02,
+                                        -8.6921e-02, 6.0899e-02, 2.2442e-01, -1.4272e-01, -4.0656e-04,
+                                        -1.2531e-01, 1.5240e-01, -6.8841e-02, 4.2114e-01, -4.4379e-02,
+                                        -3.5105e-02, 1.4931e-01, -8.3358e-02, -1.0498e-01, 1.4575e-01,
+                                        -1.6491e-01, 4.7820e-02, 2.5958e-01, 1.1974e-01, 1.8271e-01,
+                                        1.7439e-02, -1.5855e-01, -9.0135e-02, -2.6199e-01, -2.5709e-01,
+                                        6.3203e-03, 7.5823e-02])
+
+        self.random_text_flag = random_text
+        self.random_everything_flag = random_everything
+        self.fixed_text_flag = fixed_text
+        self.random_image_flag = random_images
+
+        # Weight Stacks
+        self.W_query = {
+            "multimodal": [self.text_reduction, self.image_reduction, self.W_query_text_half_dim,
+                           self.W_query_image_half_dim],
+            "image": [self.W_query_image_full_dim],
+        }
+
+        self.W_key = {
+            "multimodal": [self.text_reduction, self.image_reduction, self.W_key_text_half_dim,
+                           self.W_key_image_half_dim],
+            "image": [self.W_key_image_full_dim]
+        }
+
+    def weight_pass(self, query_text, query_image, key_text, key_image):
+        inference_functions = [
+            (True, True, True, True),  # Input: text and image Context: text and image
+            (False, True, False, True),  # Input: image Context: image
+            (False, True, True, True)  # Input: image Context: text and image
+        ]
+
+        if None in (query_image, key_image):
+            raise ValueError("Query and Key image cannot be None")
+
+        if (query_text is not None, query_image is not None, key_text is not None,
+            key_image is not None) in inference_functions:
+            query = self._queries_inference(query_text, query_image)
+            key = self._keys_inference(key_text, key_image)
+            return query, key
+        else:
+            raise ValueError("Invalid input")
+
+    def _queries_inference(self, query_text, query_image):
+        if query_text is None:
+            output = self.W_query_image_full_dim(query_image)
+        elif query_image is None:
+            raise ValueError("Query image cannot be None")
+        else:
+            text_reduction = self.text_reduction(query_text)
+            image_reduction = self.image_reduction(query_image)
+            query_text_half_dim = self.W_query_text_half_dim(text_reduction)
+            query_image_half_dim = self.W_query_image_half_dim(image_reduction)
+            output = torch.cat((query_text_half_dim, query_image_half_dim), dim=-1)
+        return output
+
+    def _keys_inference(self, key_text, key_image):
+        if key_text is None:
+            output = self.W_key_image_full_dim(key_image)
+        elif key_image is None:
+            raise ValueError("Key image cannot be None")
+        else:
+            text_reduction = self.text_reduction(key_text)
+            image_reduction = self.image_reduction(key_image)
+            key_text_half_dim = self.W_key_text_half_dim(text_reduction)
+            key_image_half_dim = self.W_key_image_half_dim(image_reduction)
+            output = torch.cat((key_text_half_dim, key_image_half_dim), dim=-1)
+        return output
+
+    def forward(self, query_text, query_image, key_text, key_image):
+
+        query_text = query_text.to(self.device)
+        query_image = query_image.to(self.device)
+        key_text = key_text.to(self.device)
+        key_image = key_image.to(self.device)
+        query, key = self.weight_pass(query_text, query_image, key_text, key_image)
+
+        d_k = key.size()[-1]  # Get the size of the key
+
+        key_transposed = key.transpose(1, 2)
+        logits = torch.matmul(query, key_transposed) / math.sqrt(d_k)
+        logits = logits.squeeze()
+
+        if len(logits.shape) <= 2:
+            softmax = F.softmax(logits, dim=0)
+        else:
+            softmax = F.softmax(logits, dim=1)
+
+
+        return softmax, logits
+
+    def training_step(self, train_batch, batch_idx):
+
+        if self.current_epoch == 0 and not self.frozen and self.unfreeze_epoch != 0:
+            print("Freezing....................................................")
+            for param in self.image_reduction.parameters():
+                param.requires_grad = False
+            self.frozen = True
+
+        if self.current_epoch == self.unfreeze_epoch and self.frozen:
+            print("Unfreezing....................................................")
+            for param in self.image_reduction.parameters():
+                param.requires_grad = True
+            self.frozen = False
+
+        # Unpack the batch data
+        queries = train_batch['queries']
+        keys = train_batch['keys']
+        real_labels = train_batch['real_index']
+
+        keys_text = []
+        keys_image = []
+        for batch in keys:
+            temp_key_text = []
+            temp_key_image = []
+
+            for key_text, key_image in batch:
+                temp_key_text.append(key_text)
+                temp_key_image.append(key_image)
+            keys_text.append(torch.stack(temp_key_text))
+            keys_image.append(torch.stack(temp_key_image))
+
+        queries_text = []
+        queries_image = []
+        for batch in queries:
+            temp_query_text = []
+            temp_query_image = []
+            for query_text, query_image in batch:
+                temp_query_text.append(query_text)
+                temp_query_image.append(query_image)
+            queries_text.append(torch.stack(temp_query_text))
+            queries_image.append(torch.stack(temp_query_image))
+
+        queries_text = torch.stack(queries_text)
+        queries_image = torch.stack(queries_image)
+        keys_text = torch.stack(keys_text)
+        keys_image = torch.stack(keys_image)
+
+        if self.fixed_text_flag:
+            print("Fixed text flag")
+            queries_text_shape = queries_text.shape
+            keys_text_shape = keys_text.shape
+            queries_text = self.fixed_text.expand(*queries_text_shape).to(queries_text.device)
+            keys_text = self.fixed_text.expand(*keys_text_shape).to(keys_text.device)
+
+        if self.random_text_flag:
+            print("Random text flag")
+            old_queries_text = queries_text.clone()
+            old_keys_text = keys_text.clone()
+            queries_text = torch.randn(queries_text.shape).to(queries_text.device)
+            keys_text = torch.randn(keys_text.shape).to(keys_text.device)
+            if torch.equal(queries_text, old_queries_text):
+                print("Queries text are equal")
+            if torch.equal(keys_text, old_keys_text):
+                print("Keys text are equal")
+
+        if self.random_image_flag:
+            print("Random image flag")
+            old_queries_image = queries_image.clone()
+            old_keys_image = keys_image.clone()
+            queries_image = torch.randn(queries_image.shape).to(queries_image.device)
+            keys_image = torch.randn(keys_image.shape).to(keys_image.device)
+            if torch.equal(queries_image, old_queries_image):
+                print("Queries image are equal")
+            if torch.equal(keys_image, old_keys_image):
+                print("Keys image are equal")
+
+        if self.random_everything_flag:
+            print("Random everything flag")
+            old_queries_text = queries_text.clone()
+            old_keys_text = keys_text.clone()
+            old_queries_image = queries_image.clone()
+            old_keys_image = keys_image.clone()
+            queries_text = torch.randn(queries_text.shape).to(queries_text.device)
+            keys_text = torch.randn(keys_text.shape).to(keys_text.device)
+            queries_image = torch.randn(queries_image.shape).to(queries_image.device)
+            keys_image = torch.randn(keys_image.shape).to(keys_image.device)
+            if torch.equal(queries_text, old_queries_text):
+                print("Queries text are equal")
+            if torch.equal(keys_text, old_keys_text):
+                print("Keys text are equal")
+            if torch.equal(queries_image, old_queries_image):
+                print("Queries image are equal")
+            if torch.equal(keys_image, old_keys_image):
+                print("Keys image are equal")
+
+        # Forward pass
+        softmax, logits = self.forward(queries_text, queries_image, keys_text, keys_image)
+
+        softmax = softmax.squeeze()
+        real_labels = real_labels.squeeze()
+        logits = logits.squeeze()
+        real_labels = real_labels.float()
+
+        if real_labels.dim() < 3:
+            real_labels = real_labels.unsqueeze(0)
+            softmax = softmax.unsqueeze(0)
+            logits = logits.unsqueeze(0)
+
+        temp_real_labels = []
+        temp_logits = []
+        global_padding = 0
+        for batch_l, batch_r in zip(logits, real_labels):
+            padding = torch.nonzero(batch_r[0] == -100)
+            if padding.nelement() == 0:
+                temp_real_labels.append(batch_r)
+                temp_logits.append(batch_l)
+                continue
+            global_padding = global_padding + padding.nelement()
+            padding_index = padding[0]
+            temp_r = batch_r.clone()
+            temp_r[:, padding_index:] = 0
+            temp_l = batch_l.clone()
+            temp_l[:, padding_index:] = -100
+            temp_real_labels.append(temp_r)
+            temp_logits.append(temp_l)
+
+        for_loss_real_labels = torch.stack(temp_real_labels).float()
+        for_loss_logits = torch.stack(temp_logits)
+        loss = self.loss_method(for_loss_logits.mT, for_loss_real_labels.mT)
+
+        batched_precision = []
+        batched_accuracy = []
+        batched_recall = []
+
+        for batch_s, batch_r in zip(softmax, real_labels):
+            padding = torch.nonzero(batch_r[0] == -100)
+            if padding.nelement() > 0:
+                padding_index = padding[0]
+                batch_r = batch_r[:, :padding_index]
+                batch_s = batch_s[:, :padding_index]
+            max_indices = batch_s.argmax(dim=0)
+            # print("Max indices: ", max_indices)
+            target_index = batch_r.argmax(dim=0)
+            # print("Target index: ", target_index)
+            subtraction = max_indices - target_index
+            # print("Subtraction: ", subtraction)
+            different_values = torch.count_nonzero(subtraction)
+            # print("Different values: ", different_values)
+            # print("Sample size: ", target_index.shape)
+            # print("Len target index: ", len(target_index))
+            samples = batch_s.shape[1] * batch_s.shape[0]
+
+            TP = len(target_index) - different_values
+            FP = different_values
+            FN = different_values
+            TN = samples - TP - FP - FN
+
+            precision = TP / (TP + FP)
+            accuracy = (TP + TN) / samples
+            recall = TP / (TP + FN)
+
+            batched_precision.append(precision.item())
+            batched_accuracy.append(accuracy.item())
+            batched_recall.append(recall.item())
+
+        precision = sum(batched_precision) / len(batched_precision)
+        accuracy = sum(batched_accuracy) / len(batched_accuracy)
+        recall = sum(batched_recall) / len(batched_recall)
+
+        self.train_sum_precision += precision
+        self.train_sum_accuracy += accuracy
+        self.train_sum_recall += recall
+        self.train_sum_runs += 1
+        self.log("train_loss", loss, on_epoch=True, on_step=False, prog_bar=True, logger=True)
+        self.log("train_precision", precision, on_epoch=True, on_step=False, prog_bar=True, logger=True)
+        self.log("train_accuracy", accuracy, on_epoch=True, on_step=False, prog_bar=True, logger=True)
+        self.log("train_recall", recall, on_epoch=True, on_step=False, prog_bar=True, logger=True)
+
+        return loss
+
+    def on_train_epoch_end(self) -> None:
+        self.log("train_precision_epoch", self.train_sum_precision / self.train_sum_runs)
+        self.log("train_accuracy_epoch", self.train_sum_accuracy / self.train_sum_runs)
+        self.log("train_recall_epoch", self.train_sum_recall / self.train_sum_runs)
+        self.train_sum_precision = 0
+        self.train_sum_accuracy = 0
+        self.train_sum_recall = 0
+        self.train_sum_runs = 0
+
+    def configure_optimizers(self):
+        self.my_optimizer = torch.optim.Adam(params=self.parameters(), lr=self.learning_rate)
+        optimizer = self.my_optimizer
+        """self.my_scheduler = torch.optim.lr_scheduler.CyclicLR(self.my_optimizer, base_lr=0.01, max_lr=0.05,step_size_up=100,cycle_momentum=False)
+        scheduler = {
+            'scheduler': self.my_scheduler,
+            'interval': 'step',
+            'frequency': 1,
+            'name': 'learning_rate'
+        }"""
+        return [optimizer]
+
+    def validation_step(self, val_batch, batch_idx):
+        # Unpack the batch data
+        queries = val_batch['queries']
+        keys = val_batch['keys']
+        real_labels = val_batch['real_index']
+
+        keys_text = []
+        keys_image = []
+        for batch in keys:
+            temp_key_text = []
+            temp_key_image = []
+
+            for key_text, key_image in batch:
+                temp_key_text.append(key_text)
+                temp_key_image.append(key_image)
+            keys_text.append(torch.stack(temp_key_text))
+            keys_image.append(torch.stack(temp_key_image))
+
+        queries_text = []
+        queries_image = []
+        for batch in queries:
+            temp_query_text = []
+            temp_query_image = []
+            for query_text, query_image in batch:
+                temp_query_text.append(query_text)
+                temp_query_image.append(query_image)
+            queries_text.append(torch.stack(temp_query_text))
+            queries_image.append(torch.stack(temp_query_image))
+
+        queries_text = torch.stack(queries_text)
+        queries_image = torch.stack(queries_image)
+        keys_text = torch.stack(keys_text)
+        keys_image = torch.stack(keys_image)
+
+        # Forward pass
+        softmax, logits = self.forward(queries_text, queries_image, keys_text, keys_image)
+
+        softmax = softmax.squeeze()
+        real_labels = real_labels.squeeze()
+
+        if real_labels.dim() < 3:
+            real_labels = real_labels.unsqueeze(0)
+            softmax = softmax.unsqueeze(0)
+            logits = logits.unsqueeze(0)
+
+        temp_real_labels = []
+        temp_logits = []
+        for batch_l, batch_r in zip(logits, real_labels):
+            padding = torch.nonzero(batch_r[0] == -100)
+            if padding.nelement() == 0:
+                continue
+            padding_index = padding[0]
+            temp_r = batch_r.clone()
+            temp_r[:, padding_index:] = 0
+            temp_l = batch_l.clone()
+            temp_l[:, padding_index:] = -100
+            temp_real_labels.append(temp_r)
+            temp_logits.append(temp_l)
+
+        if padding.nelement() > 0:
+            for_loss_real_labels = torch.stack(temp_real_labels).float()
+            for_loss_logits = torch.stack(temp_logits)
+            loss = self.loss_method(for_loss_logits.mT, for_loss_real_labels.mT)
+        else:
+            loss = self.loss_method(logits.mT, real_labels.mT)
+
+        if loss < 0:
+            print("Padding: ", padding.nelement())
+            print("Loss: ", loss)
+            print("Logits: ", logits)
+            print("Real labels: ", real_labels)
+            exit()
+
+        batched_precision = []
+        batched_accuracy = []
+        batched_recall = []
+        for batch_s, batch_r in zip(softmax, real_labels):
+            padding = torch.nonzero(batch_r[0] == -100)
+            if padding.nelement() > 0:
+                padding_index = padding[0]
+                batch_r = batch_r[:, :padding_index]
+                batch_s = batch_s[:, :padding_index]
+            max_indices = batch_s.argmax(dim=0)
+            # print("Max indices: ", max_indices)
+            target_index = batch_r.argmax(dim=0)
+            # print("Target index: ", target_index)
+            subtraction = max_indices - target_index
+            # print("Subtraction: ", subtraction)
+            different_values = torch.count_nonzero(subtraction)
+            # print("Different values: ", different_values)
+            # print("Sample size: ", target_index.shape)
+            # print("Len target index: ", len(target_index))
+            samples = batch_s.shape[1] * batch_s.shape[0]
+            TP = len(target_index) - different_values
+            FP = different_values
+            FN = different_values
+            TN = samples - TP - FP - FN
+
+            precision = TP / (TP + FP)
+            accuracy = (TP + TN) / samples
+            recall = TP / (TP + FN)
+
+            batched_precision.append(precision.item())
+            batched_accuracy.append(accuracy.item())
+            batched_recall.append(recall.item())
+
+        precision = sum(batched_precision) / len(batched_precision)
+        accuracy = sum(batched_accuracy) / len(batched_accuracy)
+        recall = sum(batched_recall) / len(batched_recall)
+        self.val_sum_precision += precision
+        self.val_sum_accuracy += accuracy
+        self.val_sum_recall += recall
+        self.val_sum_runs += 1
+        self.log("val_loss", loss, on_epoch=True, on_step=False, prog_bar=True, logger=True)
+        self.log("val_precision", precision, on_epoch=True, on_step=False, prog_bar=True, logger=True)
+        self.log("val_accuracy", accuracy, on_epoch=True, on_step=False, prog_bar=True, logger=True)
+        self.log("val_recall", recall, on_epoch=True, on_step=False, prog_bar=True, logger=True)
+
+    def on_validation_epoch_end(self) -> None:
+        self.log("val_precision_epoch", self.val_sum_precision / self.val_sum_runs)
+        self.log("val_accuracy_epoch", self.val_sum_accuracy / self.val_sum_runs)
+        self.log("val_recall_epoch", self.val_sum_recall / self.val_sum_runs)
+        self.val_sum_precision = 0
+        self.val_sum_accuracy = 0
+        self.val_sum_recall = 0
+        self.val_sum_runs = 0
+
+
+if __name__ == '__main__':
+
+    if len(sys.argv) > 1:
+        print("Using arguments")
+        batch_size = int(sys.argv[1])
+        learning_rate = float(sys.argv[2])
+        epochs = int(sys.argv[3])
+        if sys.argv[4] == "True":
+            wandb_flag = True
+        else:
+            wandb_flag = False
+        if sys.argv[5] == "True":
+            find_lr = True
+        else:
+            find_lr = False
+        unfreeze = int(sys.argv[6])
+
+    else:
+        print("Using default values")
+        batch_size = 500
+        learning_rate = 0.01
+        epochs = 50
+        wandb_flag = True
+        find_lr = False
+        unfreeze = 10
+        random_text = False
+        random_everything = False
+        random_images = False
+        fixed_text = False
+
+    print("Batch size: ", batch_size)
+    print("Learning rate: ", learning_rate)
+    print("Epochs: ", epochs)
+    print("Wandb flag: ", wandb_flag)
+    print("Find lr: ", find_lr)
+    print("Unfreeze: ", unfreeze)
+
+    train_path = "./recipe_dataset_3500_real_1.pkl"
+    val_path = "./recipe_dataset_3500_real_2.pkl"
+    train = pickle.load(open(train_path, "rb"))
+    val = pickle.load(open(val_path, "rb"))
+
+    if "wrong" in train_path and "wrong" in val_path:
+        print("Using dataset with false positives")
+        string_wrong = "WRONG_"
+    elif "wrong" in train_path or "wrong" in val_path:
+        raise ValueError("One of the datasets is wrong")
+    else:
+        print("Using normal dataset")
+        string_wrong = ""
+    if random_text:
+        string_wrong += "RANDOM_TEXT_"
+    elif random_everything:
+        string_wrong += "RANDOM_EVERYTHING_"
+    elif random_images:
+        string_wrong += "RANDOM_IMAGES_"
+    elif fixed_text:
+        string_wrong += "FIXED_TEXT_"
+
+
+
+    # remove fields that are not needed
+    for batch in train:
+        batch.pop('ids_queries')
+        batch.pop('ids_keys')
+
+    for batch in val:
+        batch.pop('ids_queries')
+        batch.pop('ids_keys')
+
+    train_dataset = DataLoader(train, num_workers=0, shuffle=False, batch_size=batch_size)
+    print("Train dataset size:", len(train_dataset))
+
+    val_dataset = DataLoader(val, num_workers=0, shuffle=False, batch_size=batch_size)
+    print("Val dataset size:", len(val_dataset))
+
+    model = SoftAttention(learning_rate=learning_rate, batch_size=batch_size, unfreeze=unfreeze,
+                          random_text=random_text, random_everything=random_everything, fixed_text=fixed_text,
+                          random_images=random_images)
+
+    lr_monitor = LearningRateMonitor(logging_interval='step')
+    if wandb_flag:
+        run_name = f"{string_wrong}MORE_RECIPES_{len(train_dataset)}_batch_{batch_size}_lr_{learning_rate}_epochs_{epochs}_unfreeze_{unfreeze}"
+        wandb_logger = WandbLogger(project='reference_training', name=run_name, log_model="all")
+        wandb_logger.experiment.config["batch_size"] = batch_size
+        wandb_logger.experiment.config["max_epochs"] = epochs
+        wandb_logger.experiment.config["learning_rate"] = learning_rate
+
+        trainer = L.Trainer(max_epochs=epochs, detect_anomaly=False, logger=wandb_logger, callbacks=[lr_monitor])
+    else:
+        trainer = L.Trainer(max_epochs=epochs, default_root_dir="./", callbacks=[lr_monitor])
+
+    if find_lr:
+        tuner = Tuner(trainer)
+        lr_finder = tuner.lr_find(model, train_dataloaders=train_dataset, val_dataloaders=val_dataset)
+        print(lr_finder.suggestion())
+    else:
+        trainer.fit(model, train_dataloaders=train_dataset, val_dataloaders=val_dataset)
+        # trainer.fit(model, train_dataloaders=train_dataset)
+
+    if wandb_flag:
+        wandb.finish()

models/clip/clip.py

@@ -0,0 +1,65 @@
+from transformers import AutoTokenizer, AutoProcessor, CLIPModel
+from PIL import Image
+import torch
+
+
+class Clip():
+    _instance = None
+
+    def __new__(cls, *args, **kwargs):
+        if not cls._instance:
+            cls._instance = super(Clip, cls).__new__(cls, *args, **kwargs)
+        return cls._instance
+
+    def __init__(self):
+        if not hasattr(self, 'initialized'):  # To ensure __init__ is only called once
+            self.processor = AutoProcessor.from_pretrained("openai/clip-vit-base-patch32")
+            self.tokenizer = AutoTokenizer.from_pretrained("openai/clip-vit-base-patch32")
+            self.model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32").to("cuda")
+            self.initialized = True
+
+    def get_model(self):
+        return self.model
+
+    def get_preprocess(self):
+        return self.processor
+
+    def get_tokenizer(self):
+        return self.tokenizer
+
+    def similarity(self, text, image_or_images):
+        text_embedding = self.generate_embedding(text)
+        image_embeddings = self.generate_embedding(image_or_images)
+
+        if isinstance(image_embeddings, torch.Tensor):
+            # Single image case
+            similarity = (text_embedding @ image_embeddings.T)
+        else:
+            # Multiple images case, image_embeddings is a list of tensors
+            similarity = torch.stack([(text_embedding @ image_embedding.T) for image_embedding in image_embeddings])
+            # apply softmax to get the similarity score
+            similarity = torch.nn.functional.softmax(similarity, dim=0)
+        return similarity
+
+    def generate_embedding(self, data):
+        if isinstance(data, Image.Image):
+            image_processed = self.processor(images=data, return_tensors="pt").to("cuda")
+            image_embedding = self.model.get_image_features(**image_processed)
+            return image_embedding
+
+        elif isinstance(data, list) and all(isinstance(img, Image.Image) for img in data):
+            # Multiple images case
+            image_embeddings = []
+            for img in data:
+                image_processed = self.processor(images=img, return_tensors="pt").to("cuda")
+                image_embedding = self.model.get_image_features(**image_processed)
+                image_embeddings.append(image_embedding)
+            return image_embeddings
+
+        elif isinstance(data, str):
+            data = data[0:77]
+            text = self.tokenizer(data, return_tensors="pt").to("cuda")
+            text_embedding = self.model.get_text_features(**text)
+            return text_embedding
+
+clip = Clip()

models/diffusionModel/Latents_Singleton.py

@@ -0,0 +1,30 @@
+import torch
+from typing import List
+
+
+class Singleton(type):
+    _instances = {}
+
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._instances:
+            cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
+        return cls._instances[cls]
+
+
+class Latents(metaclass=Singleton):
+    def __init__(self) -> None:
+        self.history: List[torch.FloatTensor] = []
+
+    def is_empty(self) -> bool:
+        return self.history is None
+
+    def add_latents(self, latents: torch.FloatTensor):
+        self.history.append(latents)
+
+    def clear(self):
+        self.history = []
+
+    def dump_and_clear(self):
+        history = self.history
+        self.clear()
+        return history

models/diffusionModel/StableDiffusion.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+
+import torch
+from diffusers import StableDiffusionPipeline
+from BeamDiffusionModel.models.diffusionModel.configs.config_loader import CONFIG
+from functools import partial
+from BeamDiffusionModel.models.diffusionModel.Latents_Singleton import Latents
+
+
+class StableDiffusion:
+    def __init__(self):
+        self.device = "cuda" if CONFIG.get("stable_diffusion", {}).get("use_cuda", True) and torch.cuda.is_available() else "cpu"
+        self.torch_dtype = torch.float16 if CONFIG.get("stable_diffusion", {}).get("precision") == "float16" else torch.float32
+
+        print(f"Loading model: {CONFIG['stable_diffusion']['id']} on {self.device}")
+
+        self.pipe = StableDiffusionPipeline.from_pretrained(CONFIG["stable_diffusion"]["id"], torch_dtype=self.torch_dtype)
+        self.pipe.to(self.device)
+
+        self.unet = self.pipe.unet
+        self.vae = self.pipe.vae
+
+        print("Model loaded successfully!")
+
+
+    def capture_latents(self, latents_store: Latents, pipe, step, timestep, callback_kwargs):
+        latents = callback_kwargs["latents"]
+        latents_store.add_latents(latents)
+        return callback_kwargs
+
+    def generate_image(self, prompt: str, latent=None, generator=None):
+        latents = Latents()
+        callback = partial(self.capture_latents, latents)
+        img = self.pipe(prompt, latents=latent, callback_on_step_end=callback,
+                             generator=generator, callback_on_step_end_tensor_inputs=["latents"],
+                             num_inference_steps=CONFIG["stable_diffusion"]["diffusion_settings"]["steps"]).images[0]
+
+        return img, latents.dump_and_clear()

models/diffusionModel/configs/config_loader.py

@@ -0,0 +1,34 @@
+import yaml
+import os
+import glob
+
+CONFIG_DIR = "./BeamDiffusionModel/models/diffusionModel/configs"  # Directory containing config files
+
+def load_all_configs():
+    config = {}  # Start with an empty config
+
+    # Find all YAML files in the directory (sorted for consistency)
+    config_files = sorted(glob.glob(os.path.join(CONFIG_DIR, "*.yml")))
+
+    for file_path in config_files:
+        with open(file_path, "r") as file:
+            new_config = yaml.safe_load(file) or {}  # Load config (avoid None)
+            config = deep_merge(config, new_config)  # Merge into main config
+
+    return config
+
+def deep_merge(dict1, dict2):
+    """Recursively merges two dictionaries (deep merge)."""
+    for key, value in dict2.items():
+        if isinstance(value, dict) and key in dict1 and isinstance(dict1[key], dict):
+            dict1[key] = deep_merge(dict1[key], value)  # Recursively merge
+        else:
+            dict1[key] = value  # Overwrite
+    return dict1
+
+# Load and merge all configs automatically
+CONFIG = load_all_configs()
+
+print(os.getcwd())
+print("Final Merged Config:")
+print(CONFIG)  # Display merged config

models/diffusionModel/configs/diffusionModel.yml

@@ -0,0 +1,6 @@
+stable_diffusion:
+  use_cuda: True
+  precision: float16
+  id: stabilityai/stable-diffusion-2-1
+  diffusion_settings:
+    steps: 50

tree/node.py

@@ -0,0 +1,59 @@
+from BeamDiffusionModel.models.clip.clip import clip
+import numpy as np
+
+class Node:
+    def __init__(self, step_text, image_path, latents, heat_map, step, generated_from_step, generated_from_latent, parent=None):
+        self.step_text = step_text
+        self.step = step
+        self.generated_from_step = generated_from_step
+        self.generated_from_latent = generated_from_latent
+        self.image = image_path
+        self.latents = latents
+        self.heat_map = heat_map
+        self.parent = parent
+        self.softmax = 0.
+        self.children = []
+
+
+    def get_ancestors(self, n):
+        ancestors = [self]
+        current_node = self.parent
+        while current_node is not None and current_node.parent is not None and n > 0:
+            ancestors.append(current_node)
+            current_node = current_node.parent
+            n -= 1
+        return ancestors
+
+    def set_softmax(self, softmax, n_latents, n_max_latents, epsilon=1e-6):
+        self.softmax = np.log((softmax.cpu() * (1 / n_max_latents)) / (1 / n_latents) + epsilon)
+
+    def get_softmax(self):
+        return self.softmax
+
+    def add_child(self, node):
+        self.children.append(node)
+
+    def find_all_paths(self):
+        all_paths = []
+        stack = [(self, [self])]  # Stack holds tuples of (current_node, path_to_current_node)
+
+        while stack:
+            current_node, path = stack.pop()
+
+            # If the current node is a leaf, add the path to all_paths
+            if not current_node.children:
+                all_paths.append(path)
+            else:
+                # Add all children to the stack with the updated path
+                for child in current_node.children:
+                    stack.append((child, path + [child]))
+
+        return all_paths
+
+    def get_latent(self, idx):
+        return self.latents[idx]
+
+    def get_features(self):
+        text_embedding = clip.generate_embedding(self.step_text)
+        img_embedding = clip.generate_embedding(self.image)
+        return text_embedding, img_embedding

tree/tree.py

@@ -0,0 +1,84 @@
+from BeamDiffusionModel.tree.node import Node
+import numpy as np
+
+class BeamSearchTree:
+    def __init__(self, steps_back, beam_width, fixed_goals, n_steps):
+        self.root = Node(None, None, None,None, "Rand Seed", None, None)
+        self.nodes = [self.root]
+        self.n_steps = n_steps
+        self.steps_back = steps_back
+        self.fixed_goals = fixed_goals
+        self.beam_width = beam_width
+
+    def add_node(self, parent, step_text, step, generated_from_step, generated_from_latent, image, latents, heat_map):
+        new_node = Node(step_text, image, latents, heat_map, step, generated_from_step, generated_from_latent, parent)
+        parent.add_child(new_node)
+        self.nodes.append(new_node)
+        return new_node
+
+    def all_paths(self):
+        paths = []
+        # iterate over all the subtrees and get all the paths
+        for root in self.root.children:
+            paths.append(root.find_all_paths())
+        return paths
+
+    def get_previous_steps_features(self, node):
+        previous_steps = []
+        previous_images = []
+        current = node.parent
+
+        while current is not None and current.parent is not None:
+            text_embedding, img_embedding = current.get_features()
+            previous_steps = [text_embedding] + previous_steps
+            previous_images = [img_embedding] + previous_images
+            current = current.parent
+        return previous_steps, previous_images
+
+    def best_path(self):
+        all_paths = self.all_paths()
+        best_path = None
+        best_score = float('-inf')
+        for paths in all_paths:
+            for path in paths:
+                if len(path) < self.n_steps:
+                    continue
+                score = 0.0
+                for node in path:
+                    score += np.log(node.softmax) if node.softmax > 0 else node.softmax
+
+                if score > best_score:
+                    best_score = score
+                    best_path = path
+        return best_path, best_score
+
+    def best_path_imgs(self):
+        best_path, _ = self.best_path()
+        seq = []
+        for node in best_path:
+            seq.append(node.image)
+        return seq
+
+    def get_n_best_paths(self, n, path_length):
+        all_paths = self.all_paths()
+        best_paths = []
+        for paths in all_paths:
+            for path in paths:
+                if len(path) < path_length:
+                    continue
+                score = float('-inf')
+                for node in path:
+                    score += np.log(node.softmax) if node.softmax > 0 else node.softmax
+                if len(best_paths) < n:
+                    best_paths.append((path, score))
+                else:
+                    min_score = min(best_paths, key=lambda x: x[1])
+                    if score > min_score[1]:
+                        best_paths.remove(min_score)
+                        best_paths.append((path, score))
+
+        new_best_paths = []
+        for (path, score) in best_paths:
+            new_best_paths.append(path)
+
+        return new_best_paths

utils/utils.py

@@ -0,0 +1,42 @@
+import PIL
+from BeamDiffusionModel.models.diffusionModel.StableDiffusion import StableDiffusion
+
+from BeamDiffusionModel.models.diffusionModel.configs.config_loader import CONFIG
+from BeamDiffusionModel.models.clip.clip import Clip
+import torch
+import json
+
+sd = StableDiffusion()
+clip = Clip()
+
+
+def read_json(path):
+    with open(path, 'r') as f:
+        data = json.load(f)
+        return data
+
+def get_img(path):
+    img = PIL.Image.open(path)
+    return img
+
+def clip_score(step, imgs_path):
+    imgs = []
+    if isinstance(imgs_path, list):
+        for img_path in imgs_path:
+            img = get_img(img_path)
+            imgs.append(img)
+    else:
+        img = get_img(imgs_path)
+        imgs.append(img)
+
+    return clip.similarity(step, imgs)
+
+def gen_img(caption, latent= None, seed=None):
+    system = "cuda" if CONFIG.get("diffusion_model", {}).get("use_cuda", True) and torch.cuda.is_available() else "cpu"
+    if seed:
+        generator = torch.Generator(system).manual_seed(seed)
+    else:
+        generator = None
+    img, latents = sd.generate_image(caption, generator=generator, latent=latent)
+    return latents, img
+