ate_command)

conn.commit()

register_vector(conn)# Insert data only if it doesn't already exist

for doc in documents:

content = doc.page_content

token_count = doc.token_count

embedding = doc.embedding.tolist()

filename = doc.filename

parent_section = doc.parent_section

url = doc.url

cur.execute(

"SELECT COUNT(*) FROM embeddings WHERE content = %s",

(content,),

count = cur.fetchone()[0]

if count == 0:

cur.execute(

"INSERT INTO embeddings (content, token_count, embedding, filename, parent_section, url) VALUES (%s, %s, %s, %s, %s, %s)",

content,

token_count,

embedding,

filename,

parent_section,

url,

),

conn.commit()

cur.execute("SELECT COUNT(*) as cnt FROM embeddings;")

num_records = cur.fetchone()[0]

logger.info(f"Number of vector records in table: {num_records}")

# calculate the index parameters according to best practices

num_lists = max(num_records / 1000, 10)

if num_records > 1000000:

num_lists = math.sqrt(num_records)

# use the cosine distance measure, which is what we'll later use for querying

cur.execute(

f"CREATE INDEX IF NOT EXISTS embeddings_idx ON embeddings USING ivfflat (embedding vector_cosine_ops) WITH (lists = {num_lists});"

conn.commit()

except Exception as e:

logger.error(f"Error in index_generator: {e}")

raise

finally:

if conn:

conn.close()

We use some utility functions, but what we do here is:

connect to the database

create the vector extension if it doesn't already exist (this is to enable the vector data type in PostgreSQL)

create the embeddings table if it doesn't exist

insert the embeddings and documents into the table

calculate the index parameters according to best practices

create an index on the embeddings

Note that we're inserting the documents into the embeddings table as well as the embeddings themselves. This is so that we can retrieve the documents based on their embeddings later on. It also helps with debugging from within the Supabase interface or wherever else we're examining the contents of the database.