module TestData
    importall Base # so that we get warnings for conflicts
    using Base.Test
    using DataFrames

    #test_group("DataVector creation")
    dvint = @data([1, 2, NA, 4])
    dvint2 = data([5:8])
    dvint3 = data(5:8)
    dvflt = @data([1.0, 2, NA, 4])
    dvstr = @data(["one", "two", NA, "four"])
    dvdict = DataArray(Dict, 4)    # for issue #199

    #test_group("constructors")
    df1 = DataFrame({dvint, dvstr}, [:Ints, :Strs])
    df2 = DataFrame({dvint, dvstr}) 
    df3 = DataFrame({dvint})
    df4 = convert(DataFrame, [1:4 1:4])
    df5 = DataFrame({@data([1,2,3,4]), dvstr})
    df6 = DataFrame({dvint, dvint, dvstr}, [:A, :B, :C])
    df7 = DataFrame(x = dvint, y = dvstr)
    @test size(df7) == (4, 2)
    @test isequal(df7[:x], dvint)

    #test_group("description functions")
    @test size(df6, 1) == 4
    @test size(df6, 2) == 3
    @test all(names(df6) .== [:A, :B, :C])
    @test all(names(df2) .== [:x1, :x2])
    @test all(names(df7) .== [:x, :y])

    #test_group("ref")
    @test df6[2, 3] == "two"
    @test isna(df6[3, 3])
    @test df6[2, :C] == "two"
    @test isequal(df6[:B], dvint)
    @test size(df6[[2,3]], 2) == 2
    @test size(df6[2,:], 1) == 1
    @test size(df6[[1, 3], [1, 3]]) == (2, 2)
    @test size(df6[1:2, 1:2]) == (2, 2)
    @test size(head(df6,2)) == (2, 3)
    # lots more to do

    #test_group("combining")

    dfc = hcat(df3, df4)
    @test size(dfc, 2) == 3
    @test all(names(dfc) .== [:x1, :x1_1, :x2])
    @test isequal(dfc[:x1], df3[:x1])

    @test isequal(dfc, [df3 df4])

    dfr = vcat(df4, df4)
    @test size(dfr, 1) == 8
    @test all(names(df4) .== names(dfr))
    @test isequal(dfr, [df4, df4])

    dfr = vcat(df2, df3)
    @test size(dfr) == (8,2)
    @test all(names(df2) .== names(dfr))
    @test isna(dfr[8,:x2])

    #test_group("assign")
    df6[3] = @data(["un", "deux", "troix", "quatre"])
    @test df6[1, 3] == "un"
    df6[:B] = [4, 3, 2, 1]
    @test df6[1,2] == 4
    df6[:D] = [true, false, true, false]
    @test df6[1,4] == true
    delete!(df6, :D)
    @test all(names(df6) .== [:A, :B, :C])
    @test size(df6, 2) == 3

    #test_group("NA handling")
    @test nrow(df5[complete_cases(df5), :]) == 3

    #test_context("SubDataFrames")

    #test_group("constructors")
    # single index is rows
    sdf6a = sub(df6, 1)
    sdf6b = sub(df6, 2:3)
    sdf6c = sub(df6, [true, false, true, false])
    @test size(sdf6a) == (1,3)
    sdf6d = sub(df6, [1,3], :B)
    @test size(sdf6d) == (2,1)

    #test_group("ref")
    @test sdf6a[1,2] == 4

    #test_context("Within")
    #test_group("Associative")

    #test_group("DataFrame")
    srand(1)
    N = 20
    #Cast to int64 as rand() behavior differs between Int32/64
    d1 = pdata(rand(int64(1:2), N))
    d2 = (@pdata ["A", "B", NA])[rand(int64(1:3), N)]
    d3 = data(randn(N))
    d4 = data(randn(N))
    df7 = DataFrame({d1, d2, d3}, [:d1, :d2, :d3])

    #test_group("groupby")
    gd = groupby(df7, :d1)
    @test length(gd) == 2
    # @test isequal(gd[2]["d2"], PooledDataVector["A", "B", NA, "A", NA, NA, NA, NA])
    @test sum(gd[2][:d3]) == sum(df7[:d3][dropna(df7[:d1] .== 2)])

    g1 = groupby(df7, [:d1, :d2])
    g2 = groupby(df7, [:d2, :d1])
    @test sum(g1[1][:d3]) == sum(g2[1][:d3])

    res = 0.0
    for x in g1
        res += sum(x[:d1])
    end
    @test res == sum(df7[:d1])

    # TODO: Don't use symbols here
    df8 = colwise(df7[[1, 3]], :sum)
    @test df8[1, :d1_sum] == sum(df7[:d1])

    df8 = colwise(groupby(df7, :d2), [:sum, :length])
    @test size(df8, 1) == 3
    @test size(df8, 2) == 5
    @test df8[2, :d1_length] == 8

    df9 = df7 |> groupby([:d2]) |> [:sum, :length]
    @test isequal(df9, df8)
    df9 = by(df7, :d2, [:sum, :length])
    @test isequal(df9, df8)

    #test_group("reshape")
    d1 = DataFrame(a = repeat([1:3], inner = [4]),
                   b = repeat([1:4], inner = [3]),
                   c = randn(12),
                   d = randn(12))

    d1s = stack(d1, [:a, :b])
    d1s2 = stack(d1, [:c, :d])
    d1s3 = melt(d1, [:c, :d])
    @test isequal(d1s[1:12, :c], d1[:c])
    @test isequal(d1s[13:24, :c], d1[:c])
    @test all(names(d1s) .== [:variable, :value, :c, :d])
    @test isequal(d1s, d1s3)
    d1s_df = stackdf(d1, [:a, :b])
    @test isequal(d1s[:variable], d1s_df[:variable][:])
    @test isequal(d1s[:value], d1s_df[:value][:])
    @test isequal(d1s[:c], d1s_df[:c][:])
    @test isequal(d1s[1,:], d1s_df[1,:])

    d1s[:idx] = [1:12, 1:12]
    d1s2[:idx] = [1:12, 1:12]
    d1us = unstack(d1s, :variable, :idx, :value)
    d1us2 = unstack(d1s2, :variable, :idx, :value)
    @test isequal(d1us[:a], d1[:a])
    @test isequal(d1us2[:d], d1[:d])

    const letters = convert(Vector{ASCIIString}, split("abcdefghijklmnopqrstuvwxyz", ""))
    const LETTERS = convert(Vector{ASCIIString}, split("ABCDEFGHIJKLMNOPQRSTUVWXYZ", ""))

    #test_group("merge")

    srand(1)
    df1 = DataFrame(a = shuffle!([1:10]),
                    b = [:A,:B][rand(1:2, 10)],
                    v1 = randn(10))

    df2 = DataFrame(a = shuffle!(reverse([1:5])),
                    b2 = [:A,:B,:C][rand(1:3, 5)],
                    v2 = randn(5))

    m1 = join(df1, df2, on = :a)
    @test isequal(m1[:a], @data([1, 2, 3, 4, 5]))
    # TODO: Re-enable
    # m2 = join(df1, df2, on = :a, kind = :outer)
    # @test isequal(m2[:b2], DataVector["A", "B", "B", "B", "B", NA, NA, NA, NA, NA])
    # @test isequal(m2[:b2], DataVector["B", "B", "B", "C", "B", NA, NA, NA, NA, NA])

    df1 = DataFrame(a = [1, 2, 3],
                    b = ["America", "Europe", "Africa"])
    df2 = DataFrame(a = [1, 2, 4],
                    c = ["New World", "Old World", "New World"])

    m1 = join(df1, df2, on = :a, kind = :inner)
    @test isequal(m1[:a], @data([1, 2]))

    m2 = join(df1, df2, on = :a, kind = :left)
    @test isequal(m2[:a], @data([1, 2, 3]))

    m3 = join(df1, df2, on = :a, kind = :right)
    @test isequal(m3[:a], @data([1, 2, 4]))

    m4 = join(df1, df2, on = :a, kind = :outer)
    @test isequal(m4[:a], @data([1, 2, 3, 4]))

    # test with NAs (issue #185)
    df1 = DataFrame()
    df1[:A] = @data(["a", "b", "a", NA])
    df1[:B] = @data([1, 2, 1, 3])

    df2 = DataFrame()
    df2[:A] = @data(["a", NA, "c"])
    df2[:C] = @data([1, 2, 4])

    m1 = join(df1, df2, on = :A)
    @test size(m1) == (3,3) 
    @test isequal(m1[:A], @data([NA,"a","a"]))

    m2 = join(df1, df2, on = :A, kind = :outer)
    @test size(m2) == (5,3) 
    @test isequal(m2[:A], @data([NA,"a","a","b","c"]))

    srand(1)
    df1 = DataFrame(
        a = [:x,:y][rand(1:2, 10)],
        b = [:A,:B][rand(1:2, 10)],
        v1 = randn(10)
    )

    df2 = DataFrame(
        a = [:x,:y][[1,2,1,1,2]],
        b = [:A,:B,:C][[1,1,1,2,3]],
        v2 = randn(5)
    )
    df2[1,:a] = NA

    # # TODO: Restore this functionality
    # m1 = join(df1, df2, on = [:a,:b])
    # @test isequal(m1[:a], DataArray(["x", "x", "y", "y", fill("x", 5)]))
    # m2 = join(df1, df2, on = ["a","b"], kind = :outer)
    # @test isequal(m2[10,:v2], NA)
    # @test isequal(m2[:a], DataVector["x", "x", "y", "y", "x", "x", "x", "x", "x", "y", NA, "y"])

    srand(1)
    function spltdf(d)
        d[:x1] = map(x -> x[1], d[:a])
        d[:x2] = map(x -> x[2], d[:a])
        d[:x3] = map(x -> x[3], d[:a])
        d
    end
    df1 = DataFrame(
        a = ["abc", "abx", "axz", "def", "dfr"],
        v1 = randn(5)
    )
    df1 = spltdf(df1)
    df2 = DataFrame(
        a = ["def", "abc","abx", "axz", "xyz"],
        v2 = randn(5)    
    )
    df2 = spltdf(df2)

    # m1 = join(df1, df2, on = :a)
    # m2 = join(df1, df2, on = [:x1, :x2, :x3])
    # @test isequal(sort(m1[:a]), sort(m2[:a]))

    #test_group("New DataVector constructors")
    dv = DataArray(Int, 5)
    @test all(isna(dv))
    dv = DataArray(Float64, 5)
    @test all(isna(dv))
    dv = @data(zeros(5))
    @test all(dv .== 0.0)
    dv = @data(ones(5))
    @test all(dv .== 1.0)

    # No more NA corruption
    dv = @data(ones(10_000))
    @test !any(isna(dv))

    PooledDataArray(falses(2), falses(2))
    PooledDataArray(falses(2), trues(2))

    # Test vectorized comparisons work for DataVector's and PooledDataVector's
    @data([1, 2, NA]) .== 1
    @pdata([1, 2, NA]) .== 1
    @data(["1", "2", NA]) .== "1"
    @pdata(["1", "2", NA]) .== "1"

    # Test unique()
    #test_group("unique()")
    # TODO: Restore this
    # dv = DataArray(1:4)
    # dv[4] = NA
    # @test (1 in unique(dv))
    # @test (2 in unique(dv))
    # @test (3 in unique(dv))
    # @test (NA in unique(dv))

    #test_group("find()")
    dv = DataArray([true, false, true])
    @test isequal(find(dv), [1, 3])

    pdv = PooledDataArray([true, false, true])
    @test isequal(find(pdv), [1, 3])

    dv[1] = NA
    @test isequal(find(dv), [3])

    pdv[1] = NA
    @test isequal(find(pdv), [3])
end
