movie_data = dataset("ggplot2", "movies")
function get_movie_minibatch(x, y, batch_size)
    x_out = [CuArray{Float32}([NaN for _ in 1:length(x[1, :])]) for _ in 1:batch_size]
    y_out = [NaN for _ in 1:batch_size]
    for i in 1:batch_size
        rn = rand(1:length(y))
        while (any(x->ismissing(x), x[rn, :]) || any(x->ismissing(x), y[rn]))
            rn = rand(1:length(y))
        end

        x_out[i] = CuArray{Float32}(collect(Iterators.flatten(x[rn, :])))
        y_out[i] = y[rn]
    end
    return x_out, y_out
end
convert_names(titles) = begin
    out = zeros(length(titles), maximum(titles .|> length))
    for i in eachindex(titles)
        for j in eachindex(titles[i])
            out[i, j] = titles[i][j] |> Int
        end
    end
    return out
end

TRAIN_PERCENTAGE = 70 # how much data will be used for training

titles = convert_names(movie_data.Title)
normalized_budget = movie_data.Budget ./ (sum(skipmissing(movie_data.Budget)) / length(movie_data.Budget))
x_data = cat(normalized_budget, movie_data.Length, titles; dims=2)
y_data = movie_data.Rating

# print number of rows with missing values in x_data
no_missing_rows = map((x->any(y->ismissing(y), x)), [x_data[i,:] for i in 1:length(x_data[:, 1])]) |> sum
println("number of rows with missing values: ", no_missing_rows)
println("usable number of rows: ", length(y_data) - no_missing_rows)


DATA_DIVIDION_INDEX = (length(x_data[:, 1])/100) * TRAIN_PERCENTAGE |> round |> Int
train_x = x_data[1:DATA_DIVIDION_INDEX, :]
train_y = y_data[1:DATA_DIVIDION_INDEX]
test_x = x_data[DATA_DIVIDION_INDEX+1:end, :]
test_y = y_data[DATA_DIVIDION_INDEX+1:end]
