<filename>experiments/task_datasets.py
TASK_DATASETS=dict(
        POS=dict(ALL=dict(
            TRAIN=[
                "/datadrive/Datasets/Twitter/TweeboParser/ark-tweet-nlp-0.3.2/data/twpos-data-v0.3/oct27.splits/oct27.train",
                "/datadrive/Datasets/Twitter/TweeboParser/ark-tweet-nlp-0.3.2/data/twpos-data-v0.3/oct27.splits/oct27.dev",
                ],
            DEV="SplitTrain20",
            OWOPUTI="/datadrive/Datasets/Twitter/TweeboParser/ark-tweet-nlp-0.3.2/data/twpos-data-v0.3/oct27.splits/oct27.test",
            OWOPUTI_DAILY547="/datadrive/Datasets/Twitter/TweeboParser/ark-tweet-nlp-0.3.2/data/twpos-data-v0.3/daily547.conll",
            FOSTER="/datadrive/Datasets/lowlands-data/ACL2014/crowdsourced_POS/data/foster-twitter.test",
            RITTER="/datadrive/Datasets/lowlands-data/ACL2014/crowdsourced_POS/data/ritter.test",
            LOWLANDS="/datadrive/Datasets/lowlands-data/ACL2014/crowdsourced_POS/data/lowlands.test"
            ),
            OWOPUTI=dict(
                TRAIN="/datadrive/Datasets/Twitter/TweeboParser/ark-tweet-nlp-0.3.2/data/twpos-data-v0.3/oct27.splits/oct27.train",
                DEV="/datadrive/Datasets/Twitter/TweeboParser/ark-tweet-nlp-0.3.2/data/twpos-data-v0.3/oct27.splits/oct27.dev",
                TEST="/datadrive/Datasets/Twitter/TweeboParser/ark-tweet-nlp-0.3.2/data/twpos-data-v0.3/oct27.splits/oct27.test",
                DAILY547="/datadrive/Datasets/Twitter/TweeboParser/ark-tweet-nlp-0.3.2/data/twpos-data-v0.3/daily547.conll"
                )
            ),
        ## NER DATASETS
        NER=dict(
            ALL=dict(
                TRAIN=[
                    "/datadrive/Datasets/lowlands-data/LREC2014/twitter_ner/data/finin.train.tsv",
                    "/datadrive/Datasets/Twitter/MSM2013/data/msm2013-ce_challenge_gs/TweetsTrainingSetCH.tsv.conll",
                    "/datadrive/Codes/multi-task-nlp-keras/data/WNUT_NER/train.tsv",
                    "/datadrive/Codes/multi-task-nlp-keras/data/WNUT_NER/dev.tsv",
                    "/datadrive/Codes/multi-task-nlp-keras/data/WNUT_2017/wnut17train.conll.tsv",
                    "/datadrive/Codes/multi-task-nlp-keras/data/WNUT_2017/emerging.dev.conll.tsv",
                    "/datadrive/Datasets/Twitter/broad_twitter_corpus/a.conll",
                    "/datadrive/Datasets/Twitter/broad_twitter_corpus/b.conll",
                    "/datadrive/Datasets/Twitter/broad_twitter_corpus/e.conll",
                    "/datadrive/Datasets/Twitter/broad_twitter_corpus/g.conll",
                    "/datadrive/Datasets/Twitter/broad_twitter_corpus/h.conll",
                    ],
                DEV="SplitTrain20",
                FININ="/datadrive/Datasets/lowlands-data/LREC2014/twitter_ner/data/finin.test.tsv.utf8",
                MSM2013="/datadrive/Datasets/Twitter/MSM2013/data/msm2013-ce_challenge_gs/goldStandard.tsv.conll",
                WNUT2016="/datadrive/Codes/multi-task-nlp-keras/data/WNUT_NER/test.tsv",
                WNUT2017="/datadrive/Codes/multi-task-nlp-keras/data/WNUT_2017/emerging.test.annotated.tsv",
                RITTER="/datadrive/Datasets/lowlands-data/LREC2014/twitter_ner/data/ritter.test.tsv",
                HEGE="/datadrive/Datasets/lowlands-data/LREC2014/twitter_ner/data/hege.test.tsv",
                BROAD="/datadrive/Datasets/Twitter/broad_twitter_corpus/f.conll",
                ),
            ## FININ is also called UMBC dataset
            FININ=dict(
                TRAIN="/datadrive/Datasets/lowlands-data/LREC2014/twitter_ner/data/finin.train.tsv",
                DEV="SplitTrain20",
                TEST="/datadrive/Datasets/lowlands-data/LREC2014/twitter_ner/data/finin.test.tsv.utf8",
                ),

            MSM2013=dict(
                TRAIN="/datadrive/Datasets/Twitter/MSM2013/data/msm2013-ce_challenge_gs/TweetsTrainingSetCH.tsv.conll",
                DEV="SplitTrain20",
                TEST="/datadrive/Datasets/Twitter/MSM2013/data/msm2013-ce_challenge_gs/goldStandard.tsv.conll",
                ),

            WNUT2016=dict(
                TRAIN="/datadrive/Codes/multi-task-nlp-keras/data/WNUT_NER/train.tsv",
                DEV="/datadrive/Codes/multi-task-nlp-keras/data/WNUT_NER/dev.tsv",
                TEST="/datadrive/Codes/multi-task-nlp-keras/data/WNUT_NER/test.tsv",
                ),

            WNUT2017=dict(
                TRAIN="/datadrive/Codes/multi-task-nlp-keras/data/WNUT_2017/wnut17train.conll.tsv",
                DEV="/datadrive/Codes/multi-task-nlp-keras/data/WNUT_2017/emerging.dev.conll.tsv",
                TEST="/datadrive/Codes/multi-task-nlp-keras/data/WNUT_2017/emerging.test.annotated.tsv",
                ),
            BROAD=dict(
                TRAIN=[
                    "/datadrive/Datasets/Twitter/broad_twitter_corpus/a.conll",
                    "/datadrive/Datasets/Twitter/broad_twitter_corpus/b.conll",
                    "/datadrive/Datasets/Twitter/broad_twitter_corpus/e.conll",
                    "/datadrive/Datasets/Twitter/broad_twitter_corpus/g.conll",
                    "/datadrive/Datasets/Twitter/broad_twitter_corpus/h.conll",
                    ],
                DEV="SplitTrain20",
                TEST="/datadrive/Datasets/Twitter/broad_twitter_corpus/f.conll",
                ),
            ),

        DIMSUM=dict(
                DIMSUM16=dict(
                    TRAIN="/datadrive/Datasets/Twitter/dimsum-data/conll/dimsum16.train",
                    DEV="SplitTrain20",
                    TEST="/datadrive/Datasets/Twitter/dimsum-data/conll/dimsum16.test"
                    )
                ),
        SUPERSENSE=dict(
                ALLSST=dict(
                    TRAIN="/datadrive/Datasets/Twitter/supersense-data-twitter/ritter-train.tsv",
                    DEV="/datadrive/Datasets/Twitter/supersense-data-twitter/ritter-dev.tsv",
                    RITTER="/datadrive/Datasets/Twitter/supersense-data-twitter/ritter-eval.tsv",
                    JOHANSON="/datadrive/Datasets/Twitter/supersense-data-twitter/in-house-eval.tsv"
                    )
                )
        )

<filename>experiments/task_datasets.py
TASK_DATASETS=dict(
        POS=dict(ALL=dict(
            TRAIN=[
                "/datadrive/Datasets/Twitter/TweeboParser/ark-tweet-nlp-0.3.2/data/twpos-data-v0.3/oct27.splits/oct27.train",
                "/datadrive/Datasets/Twitter/TweeboParser/ark-tweet-nlp-0.3.2/data/twpos-data-v0.3/oct27.splits/oct27.dev",
                ],
            DEV="SplitTrain20",
            OWOPUTI="/datadrive/Datasets/Twitter/TweeboParser/ark-tweet-nlp-0.3.2/data/twpos-data-v0.3/oct27.splits/oct27.test",
            OWOPUTI_DAILY547="/datadrive/Datasets/Twitter/TweeboParser/ark-tweet-nlp-0.3.2/data/twpos-data-v0.3/daily547.conll",
            FOSTER="/datadrive/Datasets/lowlands-data/ACL2014/crowdsourced_POS/data/foster-twitter.test",
            RITTER="/datadrive/Datasets/lowlands-data/ACL2014/crowdsourced_POS/data/ritter.test",
            LOWLANDS="/datadrive/Datasets/lowlands-data/ACL2014/crowdsourced_POS/data/lowlands.test"
            ),
            OWOPUTI=dict(
                TRAIN="/datadrive/Datasets/Twitter/TweeboParser/ark-tweet-nlp-0.3.2/data/twpos-data-v0.3/oct27.splits/oct27.train",
                DEV="/datadrive/Datasets/Twitter/TweeboParser/ark-tweet-nlp-0.3.2/data/twpos-data-v0.3/oct27.splits/oct27.dev",
                TEST="/datadrive/Datasets/Twitter/TweeboParser/ark-tweet-nlp-0.3.2/data/twpos-data-v0.3/oct27.splits/oct27.test",
                DAILY547="/datadrive/Datasets/Twitter/TweeboParser/ark-tweet-nlp-0.3.2/data/twpos-data-v0.3/daily547.conll"
                )
            ),
        ## NER DATASETS
        NER=dict(
            ALL=dict(
                TRAIN=[
                    "/datadrive/Datasets/lowlands-data/LREC2014/twitter_ner/data/finin.train.tsv",
                    "/datadrive/Datasets/Twitter/MSM2013/data/msm2013-ce_challenge_gs/TweetsTrainingSetCH.tsv.conll",
                    "/datadrive/Codes/multi-task-nlp-keras/data/WNUT_NER/train.tsv",
                    "/datadrive/Codes/multi-task-nlp-keras/data/WNUT_NER/dev.tsv",
                    "/datadrive/Codes/multi-task-nlp-keras/data/WNUT_2017/wnut17train.conll.tsv",
                    "/datadrive/Codes/multi-task-nlp-keras/data/WNUT_2017/emerging.dev.conll.tsv",
                    "/datadrive/Datasets/Twitter/broad_twitter_corpus/a.conll",
                    "/datadrive/Datasets/Twitter/broad_twitter_corpus/b.conll",
                    "/datadrive/Datasets/Twitter/broad_twitter_corpus/e.conll",
                    "/datadrive/Datasets/Twitter/broad_twitter_corpus/g.conll",
                    "/datadrive/Datasets/Twitter/broad_twitter_corpus/h.conll",
                    ],
                DEV="SplitTrain20",
                FININ="/datadrive/Datasets/lowlands-data/LREC2014/twitter_ner/data/finin.test.tsv.utf8",
                MSM2013="/datadrive/Datasets/Twitter/MSM2013/data/msm2013-ce_challenge_gs/goldStandard.tsv.conll",
                WNUT2016="/datadrive/Codes/multi-task-nlp-keras/data/WNUT_NER/test.tsv",
                WNUT2017="/datadrive/Codes/multi-task-nlp-keras/data/WNUT_2017/emerging.test.annotated.tsv",
                RITTER="/datadrive/Datasets/lowlands-data/LREC2014/twitter_ner/data/ritter.test.tsv",
                HEGE="/datadrive/Datasets/lowlands-data/LREC2014/twitter_ner/data/hege.test.tsv",
                BROAD="/datadrive/Datasets/Twitter/broad_twitter_corpus/f.conll",
                ),
            ## FININ is also called UMBC dataset
            FININ=dict(
                TRAIN="/datadrive/Datasets/lowlands-data/LREC2014/twitter_ner/data/finin.train.tsv",
                DEV="SplitTrain20",
                TEST="/datadrive/Datasets/lowlands-data/LREC2014/twitter_ner/data/finin.test.tsv.utf8",
                ),

            MSM2013=dict(
                TRAIN="/datadrive/Datasets/Twitter/MSM2013/data/msm2013-ce_challenge_gs/TweetsTrainingSetCH.tsv.conll",
                DEV="SplitTrain20",
                TEST="/datadrive/Datasets/Twitter/MSM2013/data/msm2013-ce_challenge_gs/goldStandard.tsv.conll",
                ),

            WNUT2016=dict(
                TRAIN="/datadrive/Codes/multi-task-nlp-keras/data/WNUT_NER/train.tsv",
                DEV="/datadrive/Codes/multi-task-nlp-keras/data/WNUT_NER/dev.tsv",
                TEST="/datadrive/Codes/multi-task-nlp-keras/data/WNUT_NER/test.tsv",
                ),

            WNUT2017=dict(
                TRAIN="/datadrive/Codes/multi-task-nlp-keras/data/WNUT_2017/wnut17train.conll.tsv",
                DEV="/datadrive/Codes/multi-task-nlp-keras/data/WNUT_2017/emerging.dev.conll.tsv",
                TEST="/datadrive/Codes/multi-task-nlp-keras/data/WNUT_2017/emerging.test.annotated.tsv",
                ),
            BROAD=dict(
                TRAIN=[
                    "/datadrive/Datasets/Twitter/broad_twitter_corpus/a.conll",
                    "/datadrive/Datasets/Twitter/broad_twitter_corpus/b.conll",
                    "/datadrive/Datasets/Twitter/broad_twitter_corpus/e.conll",
                    "/datadrive/Datasets/Twitter/broad_twitter_corpus/g.conll",
                    "/datadrive/Datasets/Twitter/broad_twitter_corpus/h.conll",
                    ],
                DEV="SplitTrain20",
                TEST="/datadrive/Datasets/Twitter/broad_twitter_corpus/f.conll",
                ),
            ),

        DIMSUM=dict(
                DIMSUM16=dict(
                    TRAIN="/datadrive/Datasets/Twitter/dimsum-data/conll/dimsum16.train",
                    DEV="SplitTrain20",
                    TEST="/datadrive/Datasets/Twitter/dimsum-data/conll/dimsum16.test"
                    )
                ),
        SUPERSENSE=dict(
                ALLSST=dict(
                    TRAIN="/datadrive/Datasets/Twitter/supersense-data-twitter/ritter-train.tsv",
                    DEV="/datadrive/Datasets/Twitter/supersense-data-twitter/ritter-dev.tsv",
                    RITTER="/datadrive/Datasets/Twitter/supersense-data-twitter/ritter-eval.tsv",
                    JOHANSON="/datadrive/Datasets/Twitter/supersense-data-twitter/in-house-eval.tsv"
                    )
                )
        )

## NER DATASETS ## FININ is also called UMBC dataset