Fail to download c4 english corpus

Fail to download c4 english corpus
 i run following code to download c4 English corpus.

```
dataset = nlp.load_dataset('c4', 'en', beam_runner='DirectRunner'
, data_dir='/mypath')
```

and i met failure as follows

```
Downloading and preparing dataset c4/en (download: Unknown size, generated: Unknown size, total: Unknown size) to /home/adam/.cache/huggingface/datasets/c4/en/2.3.0...
Traceback (most recent call last):
  File "download_corpus.py", line 38, in <module>
    , data_dir='/home/adam/data/corpus/en/c4')
  File "/home/adam/anaconda3/envs/adam/lib/python3.7/site-packages/nlp/load.py", line 520, in load_dataset
    save_infos=save_infos,
  File "/home/adam/anaconda3/envs/adam/lib/python3.7/site-packages/nlp/builder.py", line 420, in download_and_prepare
    dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
  File "/home/adam/anaconda3/envs/adam/lib/python3.7/site-packages/nlp/builder.py", line 816, in _download_and_prepare
    dl_manager, verify_infos=False, pipeline=pipeline,
  File "/home/adam/anaconda3/envs/adam/lib/python3.7/site-packages/nlp/builder.py", line 457, in _download_and_prepare
    split_generators = self._split_generators(dl_manager, **split_generators_kwargs)
  File "/home/adam/anaconda3/envs/adam/lib/python3.7/site-packages/nlp/datasets/c4/f545de9f63300d8d02a6795e2eb34e140c47e62a803f572ac5599e170ee66ecc/c4.py", line 175, in _split_generators
    dl_manager.download_checksums(_CHECKSUMS_URL)
AttributeError: 'DownloadManager' object has no attribute 'download_checksums

```
can i get any advice?
 @lhoestq I am facing `IsADirectoryError` while downloading with this command.
Can you pls look into it & help me.
I'm using version 0.4.0 of `nlp`.

```
dataset = load_dataset("c4", 'en', data_dir='.', beam_runner='DirectRunner')
```

Here's the complete stack trace.

```
Downloading and preparing dataset c4/en (download: Unknown size, generated: Unknown size, post-processed: Unknown sizetotal: Unknown size) to /home/devops/.cache/huggingface/datasets/c4/en/2.3.0/096df5a27756d51957c959a2499453e60a08154971fceb017bbb29f54b11bef7...

---------------------------------------------------------------------------
IsADirectoryError                         Traceback (most recent call last)
<ipython-input-11-f622e6705e03> in <module>
----> 1 dataset = load_dataset("c4", 'en', data_dir='.', beam_runner='DirectRunner')

/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/load.py in load_dataset(path, name, version, data_dir, data_files, split, cache_dir, features, download_config, download_mode, ignore_verifications, save_infos, **config_kwargs)
    547     # Download and prepare data
    548     builder_instance.download_and_prepare(
--> 549         download_config=download_config, download_mode=download_mode, ignore_verifications=ignore_verifications,
    550     )
    551

/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/builder.py in download_and_prepare(self, download_config, download_mode, ignore_verifications, try_from_hf_gcs, dl_manager, **download_and_prepare_kwargs)
    461                 if not downloaded_from_gcs:
    462                     self._download_and_prepare(
--> 463                         dl_manager=dl_manager, verify_infos=verify_infos, **download_and_prepare_kwargs
    464                     )
    465                 # Sync info

/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/builder.py in _download_and_prepare(self, dl_manager, verify_infos)
    964         pipeline = beam_utils.BeamPipeline(runner=beam_runner, options=beam_options,)
    965         super(BeamBasedBuilder, self)._download_and_prepare(
--> 966             dl_manager, verify_infos=False, pipeline=pipeline,
    967         )  # TODO handle verify_infos in beam datasets
    968         # Run pipeline

/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/builder.py in _download_and_prepare(self, dl_manager, verify_infos, **prepare_split_kwargs)
    516         split_dict = SplitDict(dataset_name=self.name)
    517         split_generators_kwargs = self._make_split_generators_kwargs(prepare_split_kwargs)
--> 518         split_generators = self._split_generators(dl_manager, **split_generators_kwargs)
    519         # Checksums verification
    520         if verify_infos:

/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/datasets/c4/096df5a27756d51957c959a2499453e60a08154971fceb017bbb29f54b11bef7/c4.py in _split_generators(self, dl_manager, pipeline)
    187         if self.config.realnewslike:
    188             files_to_download["realnews_domains"] = _REALNEWS_DOMAINS_URL
--> 189         file_paths = dl_manager.download_and_extract(files_to_download)
    190
    191         if self.config.webtextlike:

/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/utils/download_manager.py in download_and_extract(self, url_or_urls)
    218             extracted_path(s): `str`, extracted paths of given URL(s).
    219         """
--> 220         return self.extract(self.download(url_or_urls))
    221
    222     def get_recorded_sizes_checksums(self):

/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/utils/download_manager.py in download(self, url_or_urls)
    156             lambda url: cached_path(url, download_config=self._download_config,), url_or_urls,
    157         )
--> 158         self._record_sizes_checksums(url_or_urls, downloaded_path_or_paths)
    159         return downloaded_path_or_paths
    160

/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/utils/download_manager.py in _record_sizes_checksums(self, url_or_urls, downloaded_path_or_paths)
    106         flattened_downloaded_path_or_paths = flatten_nested(downloaded_path_or_paths)
    107         for url, path in zip(flattened_urls_or_urls, flattened_downloaded_path_or_paths):
--> 108             self._recorded_sizes_checksums[url] = get_size_checksum_dict(path)
    109
    110     def download_custom(self, url_or_urls, custom_download):

/data/anaconda/envs/hf/lib/python3.6/site-packages/nlp/utils/info_utils.py in get_size_checksum_dict(path)
     77     """Compute the file size and the sha256 checksum of a file"""
     78     m = sha256()
---> 79     with open(path, "rb") as f:
     80         for chunk in iter(lambda: f.read(1 << 20), b""):
     81             m.update(chunk)

IsADirectoryError: [Errno 21] Is a directory: '/'

```

Can anyone please try to see what I am doing wrong or is this a bug?


Not able to access the XNLI dataset

Not able to access the XNLI dataset
 When I try to access the XNLI dataset, I get the following error. The option of plain_text get selected automatically and then I get the following error.

```
FileNotFoundError: [Errno 2] No such file or directory: '/home/sasha/.cache/huggingface/datasets/xnli/plain_text/1.0.0/dataset_info.json'
Traceback:
File "/home/sasha/.local/lib/python3.7/site-packages/streamlit/ScriptRunner.py", line 322, in _run_script
    exec(code, module.__dict__)
File "/home/sasha/nlp_viewer/run.py", line 86, in <module>
    dts, fail = get(str(option.id), str(conf_option.name) if conf_option else None)
File "/home/sasha/.local/lib/python3.7/site-packages/streamlit/caching.py", line 591, in wrapped_func
    return get_or_create_cached_value()
File "/home/sasha/.local/lib/python3.7/site-packages/streamlit/caching.py", line 575, in get_or_create_cached_value
    return_value = func(*args, **kwargs)
File "/home/sasha/nlp_viewer/run.py", line 72, in get
    builder_instance = builder_cls(name=conf)
File "/home/sasha/.local/lib/python3.7/site-packages/nlp/builder.py", line 610, in __init__
    super(GeneratorBasedBuilder, self).__init__(*args, **kwargs)
File "/home/sasha/.local/lib/python3.7/site-packages/nlp/builder.py", line 152, in __init__
    self.info = DatasetInfo.from_directory(self._cache_dir)
File "/home/sasha/.local/lib/python3.7/site-packages/nlp/info.py", line 157, in from_directory
    with open(os.path.join(dataset_info_dir, DATASET_INFO_FILENAME), "r") as f:
```

Is it possible to see if the dataset_info.json is correctly placed?
 Thanks for reporting this bug !
As it seems to be just a cache problem, I closed your PR.
I think we might just need to clear and reload the `xnli` cache @srush ?


Not able to access the XNLI dataset

Not able to access the XNLI dataset
 When I try to access the XNLI dataset, I get the following error. The option of plain_text get selected automatically and then I get the following error.

```
FileNotFoundError: [Errno 2] No such file or directory: '/home/sasha/.cache/huggingface/datasets/xnli/plain_text/1.0.0/dataset_info.json'
Traceback:
File "/home/sasha/.local/lib/python3.7/site-packages/streamlit/ScriptRunner.py", line 322, in _run_script
    exec(code, module.__dict__)
File "/home/sasha/nlp_viewer/run.py", line 86, in <module>
    dts, fail = get(str(option.id), str(conf_option.name) if conf_option else None)
File "/home/sasha/.local/lib/python3.7/site-packages/streamlit/caching.py", line 591, in wrapped_func
    return get_or_create_cached_value()
File "/home/sasha/.local/lib/python3.7/site-packages/streamlit/caching.py", line 575, in get_or_create_cached_value
    return_value = func(*args, **kwargs)
File "/home/sasha/nlp_viewer/run.py", line 72, in get
    builder_instance = builder_cls(name=conf)
File "/home/sasha/.local/lib/python3.7/site-packages/nlp/builder.py", line 610, in __init__
    super(GeneratorBasedBuilder, self).__init__(*args, **kwargs)
File "/home/sasha/.local/lib/python3.7/site-packages/nlp/builder.py", line 152, in __init__
    self.info = DatasetInfo.from_directory(self._cache_dir)
File "/home/sasha/.local/lib/python3.7/site-packages/nlp/info.py", line 157, in from_directory
    with open(os.path.join(dataset_info_dir, DATASET_INFO_FILENAME), "r") as f:
```

Is it possible to see if the dataset_info.json is correctly placed?
 Update: The dataset_info.json error is gone, but we have a new one instead:
```
ConnectionError: Couldn't reach https://www.nyu.edu/projects/bowman/xnli/XNLI-1.0.zip
```
I am not able to reproduce on my side unfortunately. Any idea @srush ?


[ROUGE] Different scores with `files2rouge`
 It seems that the ROUGE score of `nlp` is lower than the one of `files2rouge`.

Here is a self-contained notebook to reproduce both scores : https://colab.research.google.com/drive/14EyAXValB6UzKY9x4rs_T3pyL7alpw_F?usp=sharing

---

`nlp` : (Only mid F-scores)

>rouge1 0.33508031962733364
rouge2 0.14574333776191592
rougeL 0.2321187823256159

`files2rouge` :

>Running ROUGE...
===========================
1 ROUGE-1 Average_R: 0.48873 (95%-conf.int. 0.41192 - 0.56339)
1 ROUGE-1 Average_P: 0.29010 (95%-conf.int. 0.23605 - 0.34445)
1 ROUGE-1 Average_F: 0.34761 (95%-conf.int. 0.29479 - 0.39871)
===========================
1 ROUGE-2 Average_R: 0.20280 (95%-conf.int. 0.14969 - 0.26244)
1 ROUGE-2 Average_P: 0.12772 (95%-conf.int. 0.08603 - 0.17752)
1 ROUGE-2 Average_F: 0.14798 (95%-conf.int. 0.10517 - 0.19240)
===========================
1 ROUGE-L Average_R: 0.32960 (95%-conf.int. 0.26501 - 0.39676)
1 ROUGE-L Average_P: 0.19880 (95%-conf.int. 0.15257 - 0.25136)
1 ROUGE-L Average_F: 0.23619 (95%-conf.int. 0.19073 - 0.28663)

---

When using longer predictions/gold, the difference is bigger.
**How can I reproduce same score as `files2rouge` ?**

@lhoestq

 @Colanim unfortunately there are different implementations of the ROUGE metric floating around online which yield different results, and we had to chose one for the package :) We ended up including the one from the google-research repository, which does minimal post-processing before computing the P/R/F scores. If I recall correctly, files2rouge relies on the Perl, script, which among other things normalizes all numbers to a special token: in the case you presented, this should account for a good chunk of the difference.

We may end up adding in more versions of the metric, but probably not for a while (@lhoestq correct me if I'm wrong). However, feel free to take a stab at adding it in yourself and submitting a PR if you're interested!


[ROUGE] Different scores with `files2rouge`
 It seems that the ROUGE score of `nlp` is lower than the one of `files2rouge`.

Here is a self-contained notebook to reproduce both scores : https://colab.research.google.com/drive/14EyAXValB6UzKY9x4rs_T3pyL7alpw_F?usp=sharing

---

`nlp` : (Only mid F-scores)

>rouge1 0.33508031962733364
rouge2 0.14574333776191592
rougeL 0.2321187823256159

`files2rouge` :

>Running ROUGE...
===========================
1 ROUGE-1 Average_R: 0.48873 (95%-conf.int. 0.41192 - 0.56339)
1 ROUGE-1 Average_P: 0.29010 (95%-conf.int. 0.23605 - 0.34445)
1 ROUGE-1 Average_F: 0.34761 (95%-conf.int. 0.29479 - 0.39871)
===========================
1 ROUGE-2 Average_R: 0.20280 (95%-conf.int. 0.14969 - 0.26244)
1 ROUGE-2 Average_P: 0.12772 (95%-conf.int. 0.08603 - 0.17752)
1 ROUGE-2 Average_F: 0.14798 (95%-conf.int. 0.10517 - 0.19240)
===========================
1 ROUGE-L Average_R: 0.32960 (95%-conf.int. 0.26501 - 0.39676)
1 ROUGE-L Average_P: 0.19880 (95%-conf.int. 0.15257 - 0.25136)
1 ROUGE-L Average_F: 0.23619 (95%-conf.int. 0.19073 - 0.28663)

---

When using longer predictions/gold, the difference is bigger.
**How can I reproduce same score as `files2rouge` ?**

@lhoestq

 Thank you for your kind answer.

As a side question : Isn't it better to have a package that normalize more ?

I understand to idea of having a package that does minimal post-processing for transparency.

But it means that people using different architecture (with different tokenizers for example) will have difference in ROUGE scores even if their predictions are actually similar.
The goal of `nlp` is to have _one package to rule them all_, right ?

I will look into it but I'm not sure I have the required skill for this ^^


[ROUGE] Different scores with `files2rouge`
 It seems that the ROUGE score of `nlp` is lower than the one of `files2rouge`.

Here is a self-contained notebook to reproduce both scores : https://colab.research.google.com/drive/14EyAXValB6UzKY9x4rs_T3pyL7alpw_F?usp=sharing

---

`nlp` : (Only mid F-scores)

>rouge1 0.33508031962733364
rouge2 0.14574333776191592
rougeL 0.2321187823256159

`files2rouge` :

>Running ROUGE...
===========================
1 ROUGE-1 Average_R: 0.48873 (95%-conf.int. 0.41192 - 0.56339)
1 ROUGE-1 Average_P: 0.29010 (95%-conf.int. 0.23605 - 0.34445)
1 ROUGE-1 Average_F: 0.34761 (95%-conf.int. 0.29479 - 0.39871)
===========================
1 ROUGE-2 Average_R: 0.20280 (95%-conf.int. 0.14969 - 0.26244)
1 ROUGE-2 Average_P: 0.12772 (95%-conf.int. 0.08603 - 0.17752)
1 ROUGE-2 Average_F: 0.14798 (95%-conf.int. 0.10517 - 0.19240)
===========================
1 ROUGE-L Average_R: 0.32960 (95%-conf.int. 0.26501 - 0.39676)
1 ROUGE-L Average_P: 0.19880 (95%-conf.int. 0.15257 - 0.25136)
1 ROUGE-L Average_F: 0.23619 (95%-conf.int. 0.19073 - 0.28663)

---

When using longer predictions/gold, the difference is bigger.
**How can I reproduce same score as `files2rouge` ?**

@lhoestq

 You're right, there's a pretty interesting trade-off here between robustness and sensitivity :) The flip side of your argument is that we also still want the metric to be sensitive to model mistakes. How we think about number normalization for example has evolved a fair bit since the Perl script was written: at the time, ROUGE was used mostly to evaluate short-medium text summarization systems, where there were only a few numbers in the input and it was assumed that the most popular methods in use at the time would get those right. However, as your example showcases, that assumption does not hold any more, and we do want to be able to penalize a model that generates a wrong numerical value.

Also, we think that abstracting away tokenization differences is the role of the model/tokenizer: if you use the 🤗Tokenizers library for example, it will handle that for you ;)

Finally, there is a lot of active research on developing model-powered metrics that are both more sensitive and more robust than ROUGE. Check out for example BERTscore, which is implemented in this library!


[Feature Request/Help] BLEURT model -> PyTorch
 Hi, I am interested in porting google research's new BLEURT learned metric to PyTorch (because I wish to do something experimental with language generation and backpropping through BLEURT). I noticed that you guys don't have it yet so I am partly just asking if you plan to add it (@thomwolf said you want to do so on Twitter).

I had a go of just like manually using the checkpoint that they publish which includes the weights. It seems like the architecture is exactly aligned with the out-of-the-box BertModel in transformers just with a single linear layer on top of the CLS embedding. I loaded all the weights to the PyTorch model but I am not able to get the same numbers as the BLEURT package's python api. Here is my colab notebook where I tried  https://colab.research.google.com/drive/1Bfced531EvQP_CpFvxwxNl25Pj6ptylY?usp=sharing . If you have any pointers on what might be going wrong that would be much appreciated!

Thank you muchly!
 Hitting this error when using bleurt with PyTorch ...

```
UnrecognizedFlagError: Unknown command line flag 'f'
```
... and I'm assuming because it was built for TF specifically.  Is there a way to use this metric in PyTorch?


[Feature Request/Help] BLEURT model -> PyTorch
 Hi, I am interested in porting google research's new BLEURT learned metric to PyTorch (because I wish to do something experimental with language generation and backpropping through BLEURT). I noticed that you guys don't have it yet so I am partly just asking if you plan to add it (@thomwolf said you want to do so on Twitter).

I had a go of just like manually using the checkpoint that they publish which includes the weights. It seems like the architecture is exactly aligned with the out-of-the-box BertModel in transformers just with a single linear layer on top of the CLS embedding. I loaded all the weights to the PyTorch model but I am not able to get the same numbers as the BLEURT package's python api. Here is my colab notebook where I tried  https://colab.research.google.com/drive/1Bfced531EvQP_CpFvxwxNl25Pj6ptylY?usp=sharing . If you have any pointers on what might be going wrong that would be much appreciated!

Thank you muchly!
 We currently provide a wrapper on the TensorFlow implementation: https://huggingface.co/metrics/bleurt

We have long term plans to better handle model-based metrics, but they probably won't be implemented right away

@adamwlev it would still be cool to add the BLEURT checkpoints to the transformers repo if you're interested, but that would best be discussed there :)

closing for now


[Feature Request/Help] BLEURT model -> PyTorch
 Hi, I am interested in porting google research's new BLEURT learned metric to PyTorch (because I wish to do something experimental with language generation and backpropping through BLEURT). I noticed that you guys don't have it yet so I am partly just asking if you plan to add it (@thomwolf said you want to do so on Twitter).

I had a go of just like manually using the checkpoint that they publish which includes the weights. It seems like the architecture is exactly aligned with the out-of-the-box BertModel in transformers just with a single linear layer on top of the CLS embedding. I loaded all the weights to the PyTorch model but I am not able to get the same numbers as the BLEURT package's python api. Here is my colab notebook where I tried  https://colab.research.google.com/drive/1Bfced531EvQP_CpFvxwxNl25Pj6ptylY?usp=sharing . If you have any pointers on what might be going wrong that would be much appreciated!

Thank you muchly!
 Hi there. We ran into the same problem this year (converting BLEURT to PyTorch) and thanks to @adamwlev found his colab notebook which didn't work but served as a good starting point. Finally, we **made it work** by doing just two simple conceptual fixes:

1. Transposing 'kernel' layers instead of 'dense' ones when copying params from the original model;
2. Taking pooler_output as a cls_state in forward function of the BleurtModel class.

Plus few minor syntactical fixes for the outdated parts. The result is still not exactly the same, but is very close to the expected one (1.0483 vs 1.0474).

Find the fixed version here (fixes are commented): https://colab.research.google.com/drive/1KsCUkFW45d5_ROSv2aHtXgeBa2Z98r03?usp=sharing


[Feature request] Add FLUE dataset

[Feature request] Add FLUE dataset
 Hi,

I think it would be interesting to add the FLUE dataset for francophones or anyone wishing to work on French.

In other requests, I read that you are already working on some datasets, and I was wondering if FLUE was planned.

If it is not the case, I can provide each of the cleaned FLUE datasets (in the form of a directly exploitable dataset rather than in the original xml formats which require additional processing, with the French part for cases where the dataset is based on a multilingual dataframe, etc.).
 @mariamabarham
I put all the datasets on this drive: https://1drv.ms/u/s!Ao2Rcpiny7RFinDypq7w-LbXcsx9?e=iVsEDh

Some information :
• For FLUE, the quote used is

> @misc{le2019flaubert,
>     title={FlauBERT: Unsupervised Language Model Pre-training for French},
>     author={Hang Le and Loïc Vial and Jibril Frej and Vincent Segonne and Maximin Coavoux and Benjamin Lecouteux and Alexandre Allauzen and Benoît Crabbé and Laurent Besacier and Didier Schwab},
>     year={2019},
>     eprint={1912.05372},
>     archivePrefix={arXiv},
>     primaryClass={cs.CL}
> }

• The Github repo of FLUE is avaible here : https://github.com/getalp/Flaubert/tree/master/flue

Information related to the different tasks of FLUE :

**1. Classification**
Three dataframes are available:
- Book
- DVD
- Music
For each of these dataframes is available a set of training and test data, and a third one containing unlabelled data.

Citation  :
>@dataset{prettenhofer_peter_2010_3251672,
  author       = {Prettenhofer, Peter and
                  Stein, Benno},
  title        = {{Webis Cross-Lingual Sentiment Dataset 2010 (Webis-
                   CLS-10)}},
  month        = jul,
  year         = 2010,
  publisher    = {Zenodo},
  doi          = {10.5281/zenodo.3251672},
  url          = {https://doi.org/10.5281/zenodo.3251672}
}

**2. Paraphrasing**
French part of the PAWS-X dataset (https://github.com/google-research-datasets/paws).
Three dataframes are available:
- train
- dev
- test

Citation :
> @InProceedings{pawsx2019emnlp,
>   title = {{PAWS-X: A Cross-lingual Adversarial Dataset for Paraphrase Identification}},
>   author = {Yang, Yinfei and Zhang, Yuan and Tar, Chris and Baldridge, Jason},
>   booktitle = {Proc. of EMNLP},
>   year = {2019}
> }

**3. Natural Language Inference**
French part of the XNLI dataset (https://github.com/facebookresearch/XNLI).
Three dataframes are available:
- train
- dev
- test

For the dev and test datasets, extra columns compared to the train dataset were available so I left them in the dataframe (I didn't know if these columns could be useful for other tasks or not).
In the context of the FLUE benchmark, only the columns gold_label, sentence1 and sentence2 are useful.

Citation :

> @InProceedings{conneau2018xnli,
>   author = "Conneau, Alexis
>         and Rinott, Ruty
>         and Lample, Guillaume
>         and Williams, Adina
>         and Bowman, Samuel R.
>         and Schwenk, Holger
>         and Stoyanov, Veselin",
>   title = "XNLI: Evaluating Cross-lingual Sentence Representations",
>   booktitle = "Proceedings of the 2018 Conference on Empirical Methods
>                in Natural Language Processing",
>   year = "2018",
>   publisher = "Association for Computational Linguistics",
>   location = "Brussels, Belgium",

**4. Parsing**
The dataset used by the FLUE authors for this task is not freely available.
Users of your library will therefore not be able to access it.
Nevertheless, I think maybe it is useful to add a link to the site where to request this dataframe: http://ftb.linguist.univ-paris-diderot.fr/telecharger.php?langue=en
(personally it was sent to me less than 48 hours after I requested it).

**5. Word Sense Disambiguation Tasks**
5.1 Verb Sense Disambiguation

Two dataframes are available: train and test
For both dataframes, 4 columns are available: document, sentence, lemma and word.
I created the document column thinking that there were several documents in the dataset but afterwards it turns out that there were not: several sentences but only one document. It's up to you to keep it or not when importing these two dataframes.

The sentence column is used to determine to which sentence the word in the word column belongs. It is in the form of a dictionary {'id': 'd000.s001', 'idx': '1'}. I thought for a while to keep only the idx because the id doesn't matter any more information. Nevertheless for the test dataset, the dictionary has an extra value indicating the source of the sentence. I don't know if it's useful or not, that's why I left the dictionary just in case. The user is free to do what he wants with it.

Citation :

> Segonne, V., Candito, M., and Crabb ́e, B. (2019). Usingwiktionary as a resource for wsd: the case of frenchverbs. InProceedings of the 13th International Confer-ence on Computational Semantics-Long Papers, pages259–270

5.2 Noun Sense Disambiguation
Two dataframes are available:  2 train and  1 test

I confess I didn't fully understand the procedure for this task.

Citation :

> @dataset{loic_vial_2019_3549806,
>   author       = {Loïc Vial},
>   title        = {{French Word Sense Disambiguation with Princeton
>                    WordNet Identifiers}},
>   month        = nov,
>   year         = 2019,
>   publisher    = {Zenodo},
>   version      = {1.0},
>   doi          = {10.5281/zenodo.3549806},
>   url          = {https://doi.org/10.5281/zenodo.3549806}
> }

Finally, additional information about FLUE is available in the FlauBERT publication :
https://arxiv.org/abs/1912.05372 (p. 4).

Hoping to have provided you with everything you need to add this benchmark  :)


Colab Notebook breaks when downloading the squad dataset
 When I run the notebook in Colab
https://colab.research.google.com/github/huggingface/nlp/blob/master/notebooks/Overview.ipynb
breaks when running this cell:
![image](https://user-images.githubusercontent.com/338917/83311709-ffd1b800-a1dd-11ea-8394-3a87df0d7f8b.png)

 The notebook forces version 0.1.0. If I use the latest, things work, I'll run the whole notebook and create a PR.

But in the meantime, this issue gets fixed by changing:
`!pip install nlp==0.1.0`
to
`!pip install nlp`


Colab Notebook breaks when downloading the squad dataset
 When I run the notebook in Colab
https://colab.research.google.com/github/huggingface/nlp/blob/master/notebooks/Overview.ipynb
breaks when running this cell:
![image](https://user-images.githubusercontent.com/338917/83311709-ffd1b800-a1dd-11ea-8394-3a87df0d7f8b.png)

 When you install `nlp` for the first time on a Colab runtime, it updates the `pyarrow` library that was already on colab. This update shows this message on colab:
```
WARNING: The following packages were previously imported in this runtime:
  [pyarrow]
You must restart the runtime in order to use newly installed versions.
```
You just have to restart the runtime and it should be fine.
If you don't restart, then it breaks like in your first message


Colab Notebook breaks when downloading the squad dataset
 When I run the notebook in Colab
https://colab.research.google.com/github/huggingface/nlp/blob/master/notebooks/Overview.ipynb
breaks when running this cell:
![image](https://user-images.githubusercontent.com/338917/83311709-ffd1b800-a1dd-11ea-8394-3a87df0d7f8b.png)

 Thanks for reporting the second one ! We'll update the notebook to fix this one :)


Colab Notebook breaks when downloading the squad dataset
 When I run the notebook in Colab
https://colab.research.google.com/github/huggingface/nlp/blob/master/notebooks/Overview.ipynb
breaks when running this cell:
![image](https://user-images.githubusercontent.com/338917/83311709-ffd1b800-a1dd-11ea-8394-3a87df0d7f8b.png)

 This trick from @thomwolf seems to be the most reliable solution to fix this colab notebook issue:

```python
# install nlp
!pip install -qq nlp==0.2.0

# Make sure that we have a recent version of pyarrow in the session before we continue - otherwise reboot Colab to activate it
import pyarrow
if int(pyarrow.__version__.split('.')[1]) < 16:
    import os
    os.kill(os.getpid(), 9)
```


Multi-task dataset mixing

Multi-task dataset mixing
 It seems like many of the best performing models on the GLUE benchmark make some use of multitask learning (simultaneous training on multiple tasks).

The [T5 paper](https://arxiv.org/pdf/1910.10683.pdf) highlights multiple ways of mixing the tasks together during finetuning:
- **Examples-proportional mixing** - sample from tasks proportionally to their dataset size
- **Equal mixing** - sample uniformly from each task
- **Temperature-scaled mixing** - The generalized approach used by multilingual BERT which uses a temperature T, where the mixing rate of each task is raised to the power 1/T and renormalized. When T=1 this is equivalent to equal mixing, and becomes closer to equal mixing with increasing T.

Following this discussion https://github.com/huggingface/transformers/issues/4340 in [transformers](https://github.com/huggingface/transformers), @enzoampil suggested that the `nlp` library might be a better place for this functionality.

Some method for combining datasets could be implemented ,e.g.
```
dataset = nlp.load_multitask(['squad','imdb','cnn_dm'], temperature=2.0, ...)
```

We would need a few additions:
- Method of identifying the tasks - how can we support adding a string to each task as an identifier: e.g. 'summarisation: '?
- Method of combining the metrics - a standard approach is to use the specific metric for each task and add them together for a combined score.

It would be great to support common use cases such as pretraining on the GLUE benchmark before fine-tuning on each GLUE task in turn.

I'm willing to write bits/most of this I just need some guidance on the interface and other library details so I can integrate it properly.

 I like this feature! I think the first question we should decide on is how to convert all datasets into the same format. In T5, the authors decided to format every dataset into a text-to-text format. If the dataset had "multiple" inputs like MNLI, the inputs were concatenated. So in MNLI the input:

> - **Hypothesis**: The St. Louis Cardinals have always won.
>
> - **Premise**: yeah well losing is i mean i’m i’m originally from Saint Louis and Saint Louis Cardinals when they were there were uh a mostly a losing team but

was flattened to a single input:

> mnli hypothesis: The St. Louis Cardinals have always won. premise:
> yeah well losing is i mean i’m i’m originally from Saint Louis and Saint Louis Cardinals
> when they were there were uh a mostly a losing team but.

This flattening is actually a very simple operation in `nlp` already. You would just need to do the following:

```python
def flatten_inputs(example):
    return {"input": "mnli hypothesis: " + example['hypothesis'] + " premise: " + example['premise']}

t5_ready_mnli_ds = mnli_ds.map(flatten_inputs, remove_columns=[<all columns except output>])
```

So I guess converting the datasets into the same format can be left to the user for now.
Then the question is how we can merge the datasets. I would probably be in favor of a simple

```python
dataset.add()
```

function that checks if the dataset is of the same format and if yes merges the two datasets. Finally, how should the sampling be implemented? **Examples-proportional mixing** corresponds to just merging the datasets and shuffling. For the other two sampling approaches we would need some higher-level features, maybe even a `dataset.sample()` function for merged datasets.

What are your thoughts on this @thomwolf @lhoestq  @ghomasHudson @enzoampil ?


Multi-task dataset mixing

Multi-task dataset mixing
 It seems like many of the best performing models on the GLUE benchmark make some use of multitask learning (simultaneous training on multiple tasks).

The [T5 paper](https://arxiv.org/pdf/1910.10683.pdf) highlights multiple ways of mixing the tasks together during finetuning:
- **Examples-proportional mixing** - sample from tasks proportionally to their dataset size
- **Equal mixing** - sample uniformly from each task
- **Temperature-scaled mixing** - The generalized approach used by multilingual BERT which uses a temperature T, where the mixing rate of each task is raised to the power 1/T and renormalized. When T=1 this is equivalent to equal mixing, and becomes closer to equal mixing with increasing T.

Following this discussion https://github.com/huggingface/transformers/issues/4340 in [transformers](https://github.com/huggingface/transformers), @enzoampil suggested that the `nlp` library might be a better place for this functionality.

Some method for combining datasets could be implemented ,e.g.
```
dataset = nlp.load_multitask(['squad','imdb','cnn_dm'], temperature=2.0, ...)
```

We would need a few additions:
- Method of identifying the tasks - how can we support adding a string to each task as an identifier: e.g. 'summarisation: '?
- Method of combining the metrics - a standard approach is to use the specific metric for each task and add them together for a combined score.

It would be great to support common use cases such as pretraining on the GLUE benchmark before fine-tuning on each GLUE task in turn.

I'm willing to write bits/most of this I just need some guidance on the interface and other library details so I can integrate it properly.

 I agree that we should leave the flattening of the dataset to the user for now. Especially because although the T5 framing seems obvious, there are slight variations on how the T5 authors do it in comparison to other approaches such as gpt-3 and decaNLP.

In terms of sampling, Examples-proportional mixing does seem the simplest to implement so would probably be a good starting point.

Temperature-scaled mixing would probably most useful, offering flexibility as it can simulate the other 2 methods by setting the temperature parameter. There is a [relevant part of the T5 repo](https://github.com/google-research/text-to-text-transfer-transformer/blob/03c94165a7d52e4f7230e5944a0541d8c5710788/t5/data/utils.py#L889-L1118) which should help with implementation.

According to the T5 authors, equal-mixing performs worst. Among the other two methods, tuning the K value (the artificial dataset size limit) has a large impact.


Multi-task dataset mixing

Multi-task dataset mixing
 It seems like many of the best performing models on the GLUE benchmark make some use of multitask learning (simultaneous training on multiple tasks).

The [T5 paper](https://arxiv.org/pdf/1910.10683.pdf) highlights multiple ways of mixing the tasks together during finetuning:
- **Examples-proportional mixing** - sample from tasks proportionally to their dataset size
- **Equal mixing** - sample uniformly from each task
- **Temperature-scaled mixing** - The generalized approach used by multilingual BERT which uses a temperature T, where the mixing rate of each task is raised to the power 1/T and renormalized. When T=1 this is equivalent to equal mixing, and becomes closer to equal mixing with increasing T.

Following this discussion https://github.com/huggingface/transformers/issues/4340 in [transformers](https://github.com/huggingface/transformers), @enzoampil suggested that the `nlp` library might be a better place for this functionality.

Some method for combining datasets could be implemented ,e.g.
```
dataset = nlp.load_multitask(['squad','imdb','cnn_dm'], temperature=2.0, ...)
```

We would need a few additions:
- Method of identifying the tasks - how can we support adding a string to each task as an identifier: e.g. 'summarisation: '?
- Method of combining the metrics - a standard approach is to use the specific metric for each task and add them together for a combined score.

It would be great to support common use cases such as pretraining on the GLUE benchmark before fine-tuning on each GLUE task in turn.

I'm willing to write bits/most of this I just need some guidance on the interface and other library details so I can integrate it properly.

 I agree with going with temperature-scaled mixing for its flexibility!

For the function that combines the datasets, I also find `dataset.add()` okay while also considering that users may want it to be easy to combine a list of say 10 data sources in one go.

`dataset.sample()` should also be good. By the looks of it, we're planning to have as main parameters: `temperature`, and `K`.

On converting the datasets to the same format, I agree that we can leave these to the users for now. But, I do imagine it'd be an awesome feature for the future to have this automatically handled, based on a chosen *approach* to formatting :smile:

E.g. T5, GPT-3, decaNLP, original raw formatting, or a contributed way of formatting in text-to-text.


Multi-task dataset mixing

Multi-task dataset mixing
 It seems like many of the best performing models on the GLUE benchmark make some use of multitask learning (simultaneous training on multiple tasks).

The [T5 paper](https://arxiv.org/pdf/1910.10683.pdf) highlights multiple ways of mixing the tasks together during finetuning:
- **Examples-proportional mixing** - sample from tasks proportionally to their dataset size
- **Equal mixing** - sample uniformly from each task
- **Temperature-scaled mixing** - The generalized approach used by multilingual BERT which uses a temperature T, where the mixing rate of each task is raised to the power 1/T and renormalized. When T=1 this is equivalent to equal mixing, and becomes closer to equal mixing with increasing T.

Following this discussion https://github.com/huggingface/transformers/issues/4340 in [transformers](https://github.com/huggingface/transformers), @enzoampil suggested that the `nlp` library might be a better place for this functionality.

Some method for combining datasets could be implemented ,e.g.
```
dataset = nlp.load_multitask(['squad','imdb','cnn_dm'], temperature=2.0, ...)
```

We would need a few additions:
- Method of identifying the tasks - how can we support adding a string to each task as an identifier: e.g. 'summarisation: '?
- Method of combining the metrics - a standard approach is to use the specific metric for each task and add them together for a combined score.

It would be great to support common use cases such as pretraining on the GLUE benchmark before fine-tuning on each GLUE task in turn.

I'm willing to write bits/most of this I just need some guidance on the interface and other library details so I can integrate it properly.

 This is an interesting discussion indeed and it would be nice to make multi-task easier.

Probably the best would be to have a new type of dataset especially designed for that in order to easily combine and sample from the multiple datasets.

This way we could probably handle the combination of datasets with differing schemas as well (unlike T5).


Multi-task dataset mixing

Multi-task dataset mixing
 It seems like many of the best performing models on the GLUE benchmark make some use of multitask learning (simultaneous training on multiple tasks).

The [T5 paper](https://arxiv.org/pdf/1910.10683.pdf) highlights multiple ways of mixing the tasks together during finetuning:
- **Examples-proportional mixing** - sample from tasks proportionally to their dataset size
- **Equal mixing** - sample uniformly from each task
- **Temperature-scaled mixing** - The generalized approach used by multilingual BERT which uses a temperature T, where the mixing rate of each task is raised to the power 1/T and renormalized. When T=1 this is equivalent to equal mixing, and becomes closer to equal mixing with increasing T.

Following this discussion https://github.com/huggingface/transformers/issues/4340 in [transformers](https://github.com/huggingface/transformers), @enzoampil suggested that the `nlp` library might be a better place for this functionality.

Some method for combining datasets could be implemented ,e.g.
```
dataset = nlp.load_multitask(['squad','imdb','cnn_dm'], temperature=2.0, ...)
```

We would need a few additions:
- Method of identifying the tasks - how can we support adding a string to each task as an identifier: e.g. 'summarisation: '?
- Method of combining the metrics - a standard approach is to use the specific metric for each task and add them together for a combined score.

It would be great to support common use cases such as pretraining on the GLUE benchmark before fine-tuning on each GLUE task in turn.

I'm willing to write bits/most of this I just need some guidance on the interface and other library details so I can integrate it properly.

 @thomwolf Are you suggesting making a wrapper class which can take existing datasets as arguments and do all the required sampling/combining, to present the same interface as a normal dataset?

That doesn't seem too complicated to implement.


Multi-task dataset mixing

Multi-task dataset mixing
 It seems like many of the best performing models on the GLUE benchmark make some use of multitask learning (simultaneous training on multiple tasks).

The [T5 paper](https://arxiv.org/pdf/1910.10683.pdf) highlights multiple ways of mixing the tasks together during finetuning:
- **Examples-proportional mixing** - sample from tasks proportionally to their dataset size
- **Equal mixing** - sample uniformly from each task
- **Temperature-scaled mixing** - The generalized approach used by multilingual BERT which uses a temperature T, where the mixing rate of each task is raised to the power 1/T and renormalized. When T=1 this is equivalent to equal mixing, and becomes closer to equal mixing with increasing T.

Following this discussion https://github.com/huggingface/transformers/issues/4340 in [transformers](https://github.com/huggingface/transformers), @enzoampil suggested that the `nlp` library might be a better place for this functionality.

Some method for combining datasets could be implemented ,e.g.
```
dataset = nlp.load_multitask(['squad','imdb','cnn_dm'], temperature=2.0, ...)
```

We would need a few additions:
- Method of identifying the tasks - how can we support adding a string to each task as an identifier: e.g. 'summarisation: '?
- Method of combining the metrics - a standard approach is to use the specific metric for each task and add them together for a combined score.

It would be great to support common use cases such as pretraining on the GLUE benchmark before fine-tuning on each GLUE task in turn.

I'm willing to write bits/most of this I just need some guidance on the interface and other library details so I can integrate it properly.

 I guess we're looking at the end user writing something like:
``` python
ds = nlp.load_dataset('multitask-t5',datasets=["squad","cnn_dm",...], k=1000, t=2.0)
```
Using the t5 method of combining here (or this could be a function passed in as an arg)

Passing kwargs to each 'sub-dataset' might become tricky.


Multi-task dataset mixing

Multi-task dataset mixing
 It seems like many of the best performing models on the GLUE benchmark make some use of multitask learning (simultaneous training on multiple tasks).

The [T5 paper](https://arxiv.org/pdf/1910.10683.pdf) highlights multiple ways of mixing the tasks together during finetuning:
- **Examples-proportional mixing** - sample from tasks proportionally to their dataset size
- **Equal mixing** - sample uniformly from each task
- **Temperature-scaled mixing** - The generalized approach used by multilingual BERT which uses a temperature T, where the mixing rate of each task is raised to the power 1/T and renormalized. When T=1 this is equivalent to equal mixing, and becomes closer to equal mixing with increasing T.

Following this discussion https://github.com/huggingface/transformers/issues/4340 in [transformers](https://github.com/huggingface/transformers), @enzoampil suggested that the `nlp` library might be a better place for this functionality.

Some method for combining datasets could be implemented ,e.g.
```
dataset = nlp.load_multitask(['squad','imdb','cnn_dm'], temperature=2.0, ...)
```

We would need a few additions:
- Method of identifying the tasks - how can we support adding a string to each task as an identifier: e.g. 'summarisation: '?
- Method of combining the metrics - a standard approach is to use the specific metric for each task and add them together for a combined score.

It would be great to support common use cases such as pretraining on the GLUE benchmark before fine-tuning on each GLUE task in turn.

I'm willing to write bits/most of this I just need some guidance on the interface and other library details so I can integrate it properly.

 From thinking upon @thomwolf 's suggestion, I've started experimenting:
```python
class MultitaskDataset(DatasetBuilder):
    def __init__(self, *args, **kwargs):
        super(MultitaskDataset, self).__init__(*args, **kwargs)
        self._datasets = kwargs.get("datasets")

    def _info(self):
        return nlp.DatasetInfo(
            description=_DESCRIPTION,
            features=nlp.Features({
                    "source": nlp.Value("string"),
                    "target": nlp.Sequence(nlp.Value("string"))
                })
        )

    def _get_common_splits(self):
        '''Finds the common splits present in all self._datasets'''
        min_set = None
        for dataset in self._datasets:
            if min_set != None:
                min_set.intersection(set(dataset.keys()))
            else:
                min_set = set(dataset.keys())
        return min_set

....

# Maybe this?:
squad = nlp.load_dataset("squad")
cnn_dm = nlp.load_dataset("cnn_dailymail","3.0.0")
multitask_dataset = nlp.load_dataset(
    'multitask_dataset',
    datasets=[squad,cnn_dailymail],
    k=1000,
    t=2.0
)

```

Does anyone know what methods of `MultitaskDataset` I would need to implement? Maybe `as_dataset` and `download_and_prepare`? Most of these should be just calling the methods of the sub-datasets.

I'm assuming DatasetBuilder is better than the more specific `GeneratorBasedBuilder`, `BeamBasedBuilder`, etc....

One of the other problems is that the dataset size is unknown till you construct it (as you can pick the sub-datasets). Am hoping not to need to make changes to `nlp.load_dataset` just for this class.

I'd appreciate it if anyone more familiar with nlp's internal workings could tell me if I'm on the right track!


Multi-task dataset mixing

Multi-task dataset mixing
 It seems like many of the best performing models on the GLUE benchmark make some use of multitask learning (simultaneous training on multiple tasks).

The [T5 paper](https://arxiv.org/pdf/1910.10683.pdf) highlights multiple ways of mixing the tasks together during finetuning:
- **Examples-proportional mixing** - sample from tasks proportionally to their dataset size
- **Equal mixing** - sample uniformly from each task
- **Temperature-scaled mixing** - The generalized approach used by multilingual BERT which uses a temperature T, where the mixing rate of each task is raised to the power 1/T and renormalized. When T=1 this is equivalent to equal mixing, and becomes closer to equal mixing with increasing T.

Following this discussion https://github.com/huggingface/transformers/issues/4340 in [transformers](https://github.com/huggingface/transformers), @enzoampil suggested that the `nlp` library might be a better place for this functionality.

Some method for combining datasets could be implemented ,e.g.
```
dataset = nlp.load_multitask(['squad','imdb','cnn_dm'], temperature=2.0, ...)
```

We would need a few additions:
- Method of identifying the tasks - how can we support adding a string to each task as an identifier: e.g. 'summarisation: '?
- Method of combining the metrics - a standard approach is to use the specific metric for each task and add them together for a combined score.

It would be great to support common use cases such as pretraining on the GLUE benchmark before fine-tuning on each GLUE task in turn.

I'm willing to write bits/most of this I just need some guidance on the interface and other library details so I can integrate it properly.

 I think I would probably go for a `MultiDataset` wrapper around a list of `Dataset`.

I'm not sure we need to give it `k` and `t` parameters at creation, it can maybe be something along the lines of:
```python
squad = nlp.load_dataset("squad")
cnn_dm = nlp.load_dataset("cnn_dailymail","3.0.0")

multitask_dataset = nlp.MultiDataset(squad, cnn_dm)

batch = multitask_dataset.sample(10, temperature=2.0, k=1000)
```

The first proof-of-concept for multi-task datasets could definitely require that the provided datasets have the same name/type for columns (if needed you easily rename/cast a column prior to instantiating the `MultiDataset`).

It's good to think about it for some time though and don't overfit too much on the T5 examples (in particular for the ways/kwargs for sampling among datasets).


Multi-task dataset mixing

Multi-task dataset mixing
 It seems like many of the best performing models on the GLUE benchmark make some use of multitask learning (simultaneous training on multiple tasks).

The [T5 paper](https://arxiv.org/pdf/1910.10683.pdf) highlights multiple ways of mixing the tasks together during finetuning:
- **Examples-proportional mixing** - sample from tasks proportionally to their dataset size
- **Equal mixing** - sample uniformly from each task
- **Temperature-scaled mixing** - The generalized approach used by multilingual BERT which uses a temperature T, where the mixing rate of each task is raised to the power 1/T and renormalized. When T=1 this is equivalent to equal mixing, and becomes closer to equal mixing with increasing T.

Following this discussion https://github.com/huggingface/transformers/issues/4340 in [transformers](https://github.com/huggingface/transformers), @enzoampil suggested that the `nlp` library might be a better place for this functionality.

Some method for combining datasets could be implemented ,e.g.
```
dataset = nlp.load_multitask(['squad','imdb','cnn_dm'], temperature=2.0, ...)
```

We would need a few additions:
- Method of identifying the tasks - how can we support adding a string to each task as an identifier: e.g. 'summarisation: '?
- Method of combining the metrics - a standard approach is to use the specific metric for each task and add them together for a combined score.

It would be great to support common use cases such as pretraining on the GLUE benchmark before fine-tuning on each GLUE task in turn.

I'm willing to write bits/most of this I just need some guidance on the interface and other library details so I can integrate it properly.

 The problem with changing `k` and `t` per sampling is that you'd have to somehow remember which examples you'd already returned while re-weighting the remaining examples based on the new `k` and `t`values. It seems possible but complicated (I can't really see a reason why you'd want to change the weighting of datasets after you constructed the multidataset).

Wouldn't it be convenient if it implemented the dataset interface? Then if someone has code using a single nlp dataset, they can replace it with a multitask combination of more datasets without having to change other code. We would at least need to be able to pass it into a `DataLoader`.


Multi-task dataset mixing

Multi-task dataset mixing
 It seems like many of the best performing models on the GLUE benchmark make some use of multitask learning (simultaneous training on multiple tasks).

The [T5 paper](https://arxiv.org/pdf/1910.10683.pdf) highlights multiple ways of mixing the tasks together during finetuning:
- **Examples-proportional mixing** - sample from tasks proportionally to their dataset size
- **Equal mixing** - sample uniformly from each task
- **Temperature-scaled mixing** - The generalized approach used by multilingual BERT which uses a temperature T, where the mixing rate of each task is raised to the power 1/T and renormalized. When T=1 this is equivalent to equal mixing, and becomes closer to equal mixing with increasing T.

Following this discussion https://github.com/huggingface/transformers/issues/4340 in [transformers](https://github.com/huggingface/transformers), @enzoampil suggested that the `nlp` library might be a better place for this functionality.

Some method for combining datasets could be implemented ,e.g.
```
dataset = nlp.load_multitask(['squad','imdb','cnn_dm'], temperature=2.0, ...)
```

We would need a few additions:
- Method of identifying the tasks - how can we support adding a string to each task as an identifier: e.g. 'summarisation: '?
- Method of combining the metrics - a standard approach is to use the specific metric for each task and add them together for a combined score.

It would be great to support common use cases such as pretraining on the GLUE benchmark before fine-tuning on each GLUE task in turn.

I'm willing to write bits/most of this I just need some guidance on the interface and other library details so I can integrate it properly.

 A very janky (but working) implementation of  `multitask_dataset.sample()` could be something like this:
```python
import nlp
import torch

class MultiDataset():
    def __init__(self, *args, temperature=2.0, k=1000, maximum=None, scale=1):
        self.datasets = args
        self._dataloaders = {}
        for split in self._get_common_splits():
            split_datasets = [ds[split] for ds in self.datasets]
            mixing_rates = self._calc_mixing_rates(split_datasets,temperature, k, maximum, scale)
            weights = []
            for i in range(len(self.datasets)):
                weights += [mixing_rates[i]]*len(self.datasets[i][split])
            self._dataloaders[split] = torch.utils.data.DataLoader(torch.utils.data.ConcatDataset(split_datasets),
                                                        sampler=torch.utils.data.sampler.WeightedRandomSampler(
                                                            num_samples=len(weights),
                                                            weights = weights,
                                                            replacement=True),
                                                        shuffle=False)

    def _get_common_splits(self):
        '''Finds the common splits present in all self.datasets'''
        min_set = None
        for dataset in self.datasets:
            if min_set != None:
                min_set.intersection(set(dataset.keys()))
            else:
                min_set = set(dataset.keys())
        return min_set

    def _calc_mixing_rates(self,datasets, temperature=2.0, k=1000, maximum=None, scale=1):
       '''Work out the weighting of each dataset based on t and k'''
        mixing_rates = []
        for dataset in datasets:
            rate = len(dataset)
            rate *= scale
            if maximum:
                rate = min(rate, maximum)
            if temperature != 1.0:
                rate = rate ** (1.0/temperature)
            mixing_rates.append(rate)
        return mixing_rates

    def sample(self,n,split):
        batch = []
        for example in self._dataloaders[split]:
            batch.append(example)
            n -= 1
            if n == 0:
                return batch

def flatten(dataset,flatten_fn):
    for k in dataset.keys():
        if isinstance(dataset[k],nlp.Dataset):
            dataset[k] = dataset[k].map(flatten_fn,remove_columns=dataset[k].column_names)

# Squad
def flatten_squad(example):
    return {"source": "squad context: " + example['context'] + " question: " + example['question'],"target":example["answers"]["text"]}
squad = nlp.load_dataset("squad")
flatten(squad,flatten_squad)

# CNN_DM
def flatten_cnn_dm(example):
    return {"source": "cnn_dm: " + example['article'],"target":[example["highlights"]]}
cnn_dm = nlp.load_dataset("cnn_dailymail", "3.0.0")
flatten(cnn_dm,flatten_cnn_dm)

multitask_dataset = MultiDataset(squad, cnn_dm)
batch = multitask_dataset.sample(100,"train")
```

There's definitely a more sensible way than embedding `DataLoader`s inside.


Multi-task dataset mixing

Multi-task dataset mixing
 It seems like many of the best performing models on the GLUE benchmark make some use of multitask learning (simultaneous training on multiple tasks).

The [T5 paper](https://arxiv.org/pdf/1910.10683.pdf) highlights multiple ways of mixing the tasks together during finetuning:
- **Examples-proportional mixing** - sample from tasks proportionally to their dataset size
- **Equal mixing** - sample uniformly from each task
- **Temperature-scaled mixing** - The generalized approach used by multilingual BERT which uses a temperature T, where the mixing rate of each task is raised to the power 1/T and renormalized. When T=1 this is equivalent to equal mixing, and becomes closer to equal mixing with increasing T.

Following this discussion https://github.com/huggingface/transformers/issues/4340 in [transformers](https://github.com/huggingface/transformers), @enzoampil suggested that the `nlp` library might be a better place for this functionality.

Some method for combining datasets could be implemented ,e.g.
```
dataset = nlp.load_multitask(['squad','imdb','cnn_dm'], temperature=2.0, ...)
```

We would need a few additions:
- Method of identifying the tasks - how can we support adding a string to each task as an identifier: e.g. 'summarisation: '?
- Method of combining the metrics - a standard approach is to use the specific metric for each task and add them together for a combined score.

It would be great to support common use cases such as pretraining on the GLUE benchmark before fine-tuning on each GLUE task in turn.

I'm willing to write bits/most of this I just need some guidance on the interface and other library details so I can integrate it properly.

 Good spot! Here are my thoughts:

- Aside: Adding `MultitaskModel` to transformers might be a thing to raise - even though having task-specific heads has become unfashionable in recent times in favour of text-to-text type models.
- Adding the task name as an extra field also seems useful for these kind of models which have task-specific heads
- There is some validation of our approach that the user should be expected to `map` datasets into a common form.
- The size-proportional sampling (also called "Examples-proportional mixing") used here doesn't perform too badly in the T5 paper (it's comparable to temperature-scaled mixing in many cases but less flexible. This is only reasonable with a `K` maximum size parameter to prevent very large datasets dominating). This might be good for a first prototype using:
    ```python
    def __iter__(self):
        """
        For each batch, sample a task, and yield a batch from the respective
        task Dataloader.

        We use size-proportional sampling, but you could easily modify this
        to sample from some-other distribution.
        """
        task_choice_list = []
        for i, task_name in enumerate(self.task_name_list):
            task_choice_list += [i] * self.num_batches_dict[task_name]
        task_choice_list = np.array(task_choice_list)
        np.random.shuffle(task_choice_list)

        dataloader_iter_dict = {
            task_name: iter(dataloader)
            for task_name, dataloader in self.dataloader_dict.items()
        }
        for task_choice in task_choice_list:
            task_name = self.task_name_list[task_choice]
            yield next(dataloader_iter_dict[task_name])
    ```
    We'd just need to pull samples from the raw datasets and not from `DataLoader`s for each task. We can assume the user has done `dataset.shuffle()` if they want to.

    Other sampling methods can later be implemented by changing how the `task_choice_list` is generated. This should allow more flexibility and not tie us to specific methods for sampling among datasets.


Multi-task dataset mixing

Multi-task dataset mixing
 It seems like many of the best performing models on the GLUE benchmark make some use of multitask learning (simultaneous training on multiple tasks).

The [T5 paper](https://arxiv.org/pdf/1910.10683.pdf) highlights multiple ways of mixing the tasks together during finetuning:
- **Examples-proportional mixing** - sample from tasks proportionally to their dataset size
- **Equal mixing** - sample uniformly from each task
- **Temperature-scaled mixing** - The generalized approach used by multilingual BERT which uses a temperature T, where the mixing rate of each task is raised to the power 1/T and renormalized. When T=1 this is equivalent to equal mixing, and becomes closer to equal mixing with increasing T.

Following this discussion https://github.com/huggingface/transformers/issues/4340 in [transformers](https://github.com/huggingface/transformers), @enzoampil suggested that the `nlp` library might be a better place for this functionality.

Some method for combining datasets could be implemented ,e.g.
```
dataset = nlp.load_multitask(['squad','imdb','cnn_dm'], temperature=2.0, ...)
```

We would need a few additions:
- Method of identifying the tasks - how can we support adding a string to each task as an identifier: e.g. 'summarisation: '?
- Method of combining the metrics - a standard approach is to use the specific metric for each task and add them together for a combined score.

It would be great to support common use cases such as pretraining on the GLUE benchmark before fine-tuning on each GLUE task in turn.

I'm willing to write bits/most of this I just need some guidance on the interface and other library details so I can integrate it properly.

 Another thought: Multitasking over benchmarks (represented as Meta-datasets in nlp) is probably a common use case. Would be nice to pass an entire benchmark to our `MultiDataset` wrapper rather than having to pass individual components.


Multi-task dataset mixing

Multi-task dataset mixing
 It seems like many of the best performing models on the GLUE benchmark make some use of multitask learning (simultaneous training on multiple tasks).

The [T5 paper](https://arxiv.org/pdf/1910.10683.pdf) highlights multiple ways of mixing the tasks together during finetuning:
- **Examples-proportional mixing** - sample from tasks proportionally to their dataset size
- **Equal mixing** - sample uniformly from each task
- **Temperature-scaled mixing** - The generalized approach used by multilingual BERT which uses a temperature T, where the mixing rate of each task is raised to the power 1/T and renormalized. When T=1 this is equivalent to equal mixing, and becomes closer to equal mixing with increasing T.

Following this discussion https://github.com/huggingface/transformers/issues/4340 in [transformers](https://github.com/huggingface/transformers), @enzoampil suggested that the `nlp` library might be a better place for this functionality.

Some method for combining datasets could be implemented ,e.g.
```
dataset = nlp.load_multitask(['squad','imdb','cnn_dm'], temperature=2.0, ...)
```

We would need a few additions:
- Method of identifying the tasks - how can we support adding a string to each task as an identifier: e.g. 'summarisation: '?
- Method of combining the metrics - a standard approach is to use the specific metric for each task and add them together for a combined score.

It would be great to support common use cases such as pretraining on the GLUE benchmark before fine-tuning on each GLUE task in turn.

I'm willing to write bits/most of this I just need some guidance on the interface and other library details so I can integrate it properly.

 Here's a fully working implementation based on the `__iter__` function of @zphang.

- I've generated the task choice list in the constructor as it allows us to index into the MultiDataset just like a normal dataset. I'm changing `task_choice_list` into a list of `(dataset_idx, example_idx)` so each entry references a unique dataset example. The shuffling has to be done before this as we don't want to shuffle within each task (we assume this is done by the user if this is what they intend).
-  I'm slightly concerned this list could become very large if many large datasets were used. Can't see a way round it at the moment though.
- I've used `task.info.builder_name` as the dataset name. Not sure if this is correct.
- I'd love to add some of the other `Dataset` methods (map, slicing by column, etc...). Would be great to implement the whole interface so a single dataset can be simply replaced by this.
- This does everything on the individual example-level. If some application required batches all from a single task in turn we can't really do that.

```python
import nlp
import numpy as np

class MultiDataset:
    def __init__(self,tasks):
        self.tasks = tasks

        # Create random order of tasks
        # Using size-proportional sampling
        task_choice_list = []
        for i, task in enumerate(self.tasks):
            task_choice_list += [i] * len(task)
        task_choice_list = np.array(task_choice_list)
        np.random.shuffle(task_choice_list)

        # Add index into each dataset
        # - We don't want to shuffle within each task
        counters = {}
        self.task_choice_list = []
        for i in range(len(task_choice_list)):
            idx = counters.get(task_choice_list[i],0)
            self.task_choice_list.append((task_choice_list[i],idx))
            counters[task_choice_list[i]] = idx + 1

    def __len__(self):
        return np.sum([len(t) for t in self.tasks])

    def __repr__(self):
        task_str = ", ".join([str(t) for t in self.tasks])
        return f"MultiDataset(tasks: {task_str})"

    def __getitem__(self,key):
        if isinstance(key, int):
            task_idx, example_idx = self.task_choice_list[key]
            task = self.tasks[task_idx]
            example = task[example_idx]
            example["task_name"] = task.info.builder_name
            return example
        elif isinstance(key, slice):
            raise NotImplementedError()

    def __iter__(self):
        for i in range(len(self)):
            yield self[i]

def load_multitask(*datasets):
    '''Create multitask datasets per split'''

    def _get_common_splits(datasets):
        '''Finds the common splits present in all self.datasets'''
        min_set = None
        for dataset in datasets:
            if min_set != None:
                min_set.intersection(set(dataset.keys()))
            else:
                min_set = set(dataset.keys())
        return min_set

    common_splits = _get_common_splits(datasets)
    out = {}
    for split in common_splits:
        out[split] = MultiDataset([d[split] for d in datasets])
    return out

##########################################
# Dataset Flattening

def flatten(dataset,flatten_fn):
    for k in dataset.keys():
        if isinstance(dataset[k],nlp.Dataset):
            dataset[k] = dataset[k].map(flatten_fn,remove_columns=dataset[k].column_names)

# Squad
def flatten_squad(example):
    return {"source": "squad context: " + example['context'] + " question: " + example['question'],
          "target":example["answers"]["text"]}
squad = nlp.load_dataset("squad")
flatten(squad,flatten_squad)

# CNN_DM
def flatten_cnn_dm(example):
    return {"source": "cnn_dm: " + example['article'],"target":[example["highlights"]]}
cnn_dm = nlp.load_dataset("cnn_dailymail", "3.0.0")
flatten(cnn_dm,flatten_cnn_dm)

#############################################

mtds = load_multitask(squad,cnn_dm)

for example in mtds["train"]:
    print(example["task_name"],example["target"])
```
Let me know if you have any thoughts. I've started using this in some of my projects and it seems to work. If people are happy with the general approach for a first version, I can make a pull request.


Multi-task dataset mixing

Multi-task dataset mixing
 It seems like many of the best performing models on the GLUE benchmark make some use of multitask learning (simultaneous training on multiple tasks).

The [T5 paper](https://arxiv.org/pdf/1910.10683.pdf) highlights multiple ways of mixing the tasks together during finetuning:
- **Examples-proportional mixing** - sample from tasks proportionally to their dataset size
- **Equal mixing** - sample uniformly from each task
- **Temperature-scaled mixing** - The generalized approach used by multilingual BERT which uses a temperature T, where the mixing rate of each task is raised to the power 1/T and renormalized. When T=1 this is equivalent to equal mixing, and becomes closer to equal mixing with increasing T.

Following this discussion https://github.com/huggingface/transformers/issues/4340 in [transformers](https://github.com/huggingface/transformers), @enzoampil suggested that the `nlp` library might be a better place for this functionality.

Some method for combining datasets could be implemented ,e.g.
```
dataset = nlp.load_multitask(['squad','imdb','cnn_dm'], temperature=2.0, ...)
```

We would need a few additions:
- Method of identifying the tasks - how can we support adding a string to each task as an identifier: e.g. 'summarisation: '?
- Method of combining the metrics - a standard approach is to use the specific metric for each task and add them together for a combined score.

It would be great to support common use cases such as pretraining on the GLUE benchmark before fine-tuning on each GLUE task in turn.

I'm willing to write bits/most of this I just need some guidance on the interface and other library details so I can integrate it properly.

 Hey! Happy to jump into the discussion here. I'm still getting familiar with bits of this code, but the reasons I sampled over data loaders rather than datasets is 1) ensuring that each sampled batch corresponds to only 1 task (in case of different inputs formats/downstream models) and 2) potentially having different batch sizes per task (e.g. some tasks have very long/short inputs). How are you currently dealing with these in your PR?


Multi-task dataset mixing

Multi-task dataset mixing
 It seems like many of the best performing models on the GLUE benchmark make some use of multitask learning (simultaneous training on multiple tasks).

The [T5 paper](https://arxiv.org/pdf/1910.10683.pdf) highlights multiple ways of mixing the tasks together during finetuning:
- **Examples-proportional mixing** - sample from tasks proportionally to their dataset size
- **Equal mixing** - sample uniformly from each task
- **Temperature-scaled mixing** - The generalized approach used by multilingual BERT which uses a temperature T, where the mixing rate of each task is raised to the power 1/T and renormalized. When T=1 this is equivalent to equal mixing, and becomes closer to equal mixing with increasing T.

Following this discussion https://github.com/huggingface/transformers/issues/4340 in [transformers](https://github.com/huggingface/transformers), @enzoampil suggested that the `nlp` library might be a better place for this functionality.

Some method for combining datasets could be implemented ,e.g.
```
dataset = nlp.load_multitask(['squad','imdb','cnn_dm'], temperature=2.0, ...)
```

We would need a few additions:
- Method of identifying the tasks - how can we support adding a string to each task as an identifier: e.g. 'summarisation: '?
- Method of combining the metrics - a standard approach is to use the specific metric for each task and add them together for a combined score.

It would be great to support common use cases such as pretraining on the GLUE benchmark before fine-tuning on each GLUE task in turn.

I'm willing to write bits/most of this I just need some guidance on the interface and other library details so I can integrate it properly.

 The short answer is - I'm not! Everything is currently on a per-example basis. It would be fairly simple to add a `batch_size` argument which would ensure that every `batch_size` examples come from the same task. That should suit most use-cases (unless you wanted to ensure batches all came from the same task and apply something like `SortishSampler` on each task first)

Your notebook was really inspiring by the way - thanks!


Multi-task dataset mixing

Multi-task dataset mixing
 It seems like many of the best performing models on the GLUE benchmark make some use of multitask learning (simultaneous training on multiple tasks).

The [T5 paper](https://arxiv.org/pdf/1910.10683.pdf) highlights multiple ways of mixing the tasks together during finetuning:
- **Examples-proportional mixing** - sample from tasks proportionally to their dataset size
- **Equal mixing** - sample uniformly from each task
- **Temperature-scaled mixing** - The generalized approach used by multilingual BERT which uses a temperature T, where the mixing rate of each task is raised to the power 1/T and renormalized. When T=1 this is equivalent to equal mixing, and becomes closer to equal mixing with increasing T.

Following this discussion https://github.com/huggingface/transformers/issues/4340 in [transformers](https://github.com/huggingface/transformers), @enzoampil suggested that the `nlp` library might be a better place for this functionality.

Some method for combining datasets could be implemented ,e.g.
```
dataset = nlp.load_multitask(['squad','imdb','cnn_dm'], temperature=2.0, ...)
```

We would need a few additions:
- Method of identifying the tasks - how can we support adding a string to each task as an identifier: e.g. 'summarisation: '?
- Method of combining the metrics - a standard approach is to use the specific metric for each task and add them together for a combined score.

It would be great to support common use cases such as pretraining on the GLUE benchmark before fine-tuning on each GLUE task in turn.

I'm willing to write bits/most of this I just need some guidance on the interface and other library details so I can integrate it properly.

 @zphang is having different batch sizes per task actually helpful? Would be interesting to know as it's not something I've come across as a technique used by any MTL papers.


Multi-task dataset mixing

Multi-task dataset mixing
 It seems like many of the best performing models on the GLUE benchmark make some use of multitask learning (simultaneous training on multiple tasks).

The [T5 paper](https://arxiv.org/pdf/1910.10683.pdf) highlights multiple ways of mixing the tasks together during finetuning:
- **Examples-proportional mixing** - sample from tasks proportionally to their dataset size
- **Equal mixing** - sample uniformly from each task
- **Temperature-scaled mixing** - The generalized approach used by multilingual BERT which uses a temperature T, where the mixing rate of each task is raised to the power 1/T and renormalized. When T=1 this is equivalent to equal mixing, and becomes closer to equal mixing with increasing T.

Following this discussion https://github.com/huggingface/transformers/issues/4340 in [transformers](https://github.com/huggingface/transformers), @enzoampil suggested that the `nlp` library might be a better place for this functionality.

Some method for combining datasets could be implemented ,e.g.
```
dataset = nlp.load_multitask(['squad','imdb','cnn_dm'], temperature=2.0, ...)
```

We would need a few additions:
- Method of identifying the tasks - how can we support adding a string to each task as an identifier: e.g. 'summarisation: '?
- Method of combining the metrics - a standard approach is to use the specific metric for each task and add them together for a combined score.

It would be great to support common use cases such as pretraining on the GLUE benchmark before fine-tuning on each GLUE task in turn.

I'm willing to write bits/most of this I just need some guidance on the interface and other library details so I can integrate it properly.

 > @zphang is having different batch sizes per task actually helpful? Would be interesting to know as it's not something I've come across as a technique used by any MTL papers.

I think having different batch sizes per task is particularly helpful in some scenarios where each task has different amount of data. For example, the problem I'm currently facing is one task has tens of thousands of samples while one task has a couple hundreds. I think in this case different batch size could help. But if using the same batch size is a lot simpler to implement, I guess it makes sense to go with that.


Multi-task dataset mixing

Multi-task dataset mixing
 It seems like many of the best performing models on the GLUE benchmark make some use of multitask learning (simultaneous training on multiple tasks).

The [T5 paper](https://arxiv.org/pdf/1910.10683.pdf) highlights multiple ways of mixing the tasks together during finetuning:
- **Examples-proportional mixing** - sample from tasks proportionally to their dataset size
- **Equal mixing** - sample uniformly from each task
- **Temperature-scaled mixing** - The generalized approach used by multilingual BERT which uses a temperature T, where the mixing rate of each task is raised to the power 1/T and renormalized. When T=1 this is equivalent to equal mixing, and becomes closer to equal mixing with increasing T.

Following this discussion https://github.com/huggingface/transformers/issues/4340 in [transformers](https://github.com/huggingface/transformers), @enzoampil suggested that the `nlp` library might be a better place for this functionality.

Some method for combining datasets could be implemented ,e.g.
```
dataset = nlp.load_multitask(['squad','imdb','cnn_dm'], temperature=2.0, ...)
```

We would need a few additions:
- Method of identifying the tasks - how can we support adding a string to each task as an identifier: e.g. 'summarisation: '?
- Method of combining the metrics - a standard approach is to use the specific metric for each task and add them together for a combined score.

It would be great to support common use cases such as pretraining on the GLUE benchmark before fine-tuning on each GLUE task in turn.

I'm willing to write bits/most of this I just need some guidance on the interface and other library details so I can integrate it properly.

 I think that instead of proportional to size sampling you should specify weights or probabilities for drawing a batch from each dataset. We should also ensure that the smaller datasets are repeated so that the encoder layer doesn't overtrain on the largest dataset.


Multi-task dataset mixing

Multi-task dataset mixing
 It seems like many of the best performing models on the GLUE benchmark make some use of multitask learning (simultaneous training on multiple tasks).

The [T5 paper](https://arxiv.org/pdf/1910.10683.pdf) highlights multiple ways of mixing the tasks together during finetuning:
- **Examples-proportional mixing** - sample from tasks proportionally to their dataset size
- **Equal mixing** - sample uniformly from each task
- **Temperature-scaled mixing** - The generalized approach used by multilingual BERT which uses a temperature T, where the mixing rate of each task is raised to the power 1/T and renormalized. When T=1 this is equivalent to equal mixing, and becomes closer to equal mixing with increasing T.

Following this discussion https://github.com/huggingface/transformers/issues/4340 in [transformers](https://github.com/huggingface/transformers), @enzoampil suggested that the `nlp` library might be a better place for this functionality.

Some method for combining datasets could be implemented ,e.g.
```
dataset = nlp.load_multitask(['squad','imdb','cnn_dm'], temperature=2.0, ...)
```

We would need a few additions:
- Method of identifying the tasks - how can we support adding a string to each task as an identifier: e.g. 'summarisation: '?
- Method of combining the metrics - a standard approach is to use the specific metric for each task and add them together for a combined score.

It would be great to support common use cases such as pretraining on the GLUE benchmark before fine-tuning on each GLUE task in turn.

I'm willing to write bits/most of this I just need some guidance on the interface and other library details so I can integrate it properly.

 Are there any references for people doing different batch sizes per task in the literature? I've only seen constant batch sizes with differing numbers of batches for each task which seems sufficient to prevent the impact of large datasets (Read 3.5.3 of the [T5 paper](https://arxiv.org/pdf/1910.10683.pdf) for example).


Multi-task dataset mixing

Multi-task dataset mixing
 It seems like many of the best performing models on the GLUE benchmark make some use of multitask learning (simultaneous training on multiple tasks).

The [T5 paper](https://arxiv.org/pdf/1910.10683.pdf) highlights multiple ways of mixing the tasks together during finetuning:
- **Examples-proportional mixing** - sample from tasks proportionally to their dataset size
- **Equal mixing** - sample uniformly from each task
- **Temperature-scaled mixing** - The generalized approach used by multilingual BERT which uses a temperature T, where the mixing rate of each task is raised to the power 1/T and renormalized. When T=1 this is equivalent to equal mixing, and becomes closer to equal mixing with increasing T.

Following this discussion https://github.com/huggingface/transformers/issues/4340 in [transformers](https://github.com/huggingface/transformers), @enzoampil suggested that the `nlp` library might be a better place for this functionality.

Some method for combining datasets could be implemented ,e.g.
```
dataset = nlp.load_multitask(['squad','imdb','cnn_dm'], temperature=2.0, ...)
```

We would need a few additions:
- Method of identifying the tasks - how can we support adding a string to each task as an identifier: e.g. 'summarisation: '?
- Method of combining the metrics - a standard approach is to use the specific metric for each task and add them together for a combined score.

It would be great to support common use cases such as pretraining on the GLUE benchmark before fine-tuning on each GLUE task in turn.

I'm willing to write bits/most of this I just need some guidance on the interface and other library details so I can integrate it properly.

 Hi,
regarding building T5 dataset , I think we can use datasets https://github.com/huggingface/datasets and then need something similar to tf.data.experimental.sample_from_datasets, do you know if similar functionality exist in pytorch? Which can sample multiple datasets with the given rates. thanks.


❓ How to get ROUGE-2 with the ROUGE metric ?
 I'm trying to use ROUGE metric, but I don't know how to get the ROUGE-2 metric.

---

I compute scores with :

```python
import nlp

rouge = nlp.load_metric('rouge')
with open("pred.txt") as p, open("ref.txt") as g:
    for lp, lg in zip(p, g):
        rouge.add([lp], [lg])
score = rouge.compute()
```

then : _(print only the F-score for readability)_

```python
for k, s in score.items():
    print(k, s.mid.fmeasure)
```

It gives :

>rouge1 0.7915168355671788
rougeL 0.7915168355671788

---

**How can I get the ROUGE-2 score ?**

Also, it's seems weird that ROUGE-1 and ROUGE-L scores are the same. Did I made a mistake ?

@lhoestq
 For the rouge2 metric you can do

```python
rouge = nlp.load_metric('rouge')
with open("pred.txt") as p, open("ref.txt") as g:
    for lp, lg in zip(p, g):
        rouge.add(lp, lg)
score = rouge.compute(rouge_types=["rouge2"])
```

Note that I just did a PR to have both `.add` and `.add_batch` for metrics, that's why now this is `rouge.add(lp, lg)` and not `rouge.add([lp], [lg])`


❓ How to get ROUGE-2 with the ROUGE metric ?
 I'm trying to use ROUGE metric, but I don't know how to get the ROUGE-2 metric.

---

I compute scores with :

```python
import nlp

rouge = nlp.load_metric('rouge')
with open("pred.txt") as p, open("ref.txt") as g:
    for lp, lg in zip(p, g):
        rouge.add([lp], [lg])
score = rouge.compute()
```

then : _(print only the F-score for readability)_

```python
for k, s in score.items():
    print(k, s.mid.fmeasure)
```

It gives :

>rouge1 0.7915168355671788
rougeL 0.7915168355671788

---

**How can I get the ROUGE-2 score ?**

Also, it's seems weird that ROUGE-1 and ROUGE-L scores are the same. Did I made a mistake ?

@lhoestq
 Well I just tested with the official script and both rouge1 and rougeL return exactly the same thing for the input you gave, so this is actually fine ^^

I hope it helped :)


NonMatchingSplitsSizesError when loading blog_authorship_corpus
 Getting this error when i run `nlp.load_dataset('blog_authorship_corpus')`.

```
raise NonMatchingSplitsSizesError(str(bad_splits))
nlp.utils.info_utils.NonMatchingSplitsSizesError: [{'expected': SplitInfo(name='train',
num_bytes=610252351, num_examples=532812, dataset_name='blog_authorship_corpus'),
'recorded': SplitInfo(name='train', num_bytes=616473500, num_examples=536323,
dataset_name='blog_authorship_corpus')}, {'expected': SplitInfo(name='validation',
num_bytes=37500394, num_examples=31277, dataset_name='blog_authorship_corpus'),
'recorded': SplitInfo(name='validation', num_bytes=30786661, num_examples=27766,
dataset_name='blog_authorship_corpus')}]
```

Upon checking it seems like there is a disparity between the information in `datasets/blog_authorship_corpus/dataset_infos.json` and what was downloaded. Although I can get away with this by passing `ignore_verifications=True` in `load_dataset`, I'm thinking doing so might give problems later on.
 I just ran it on colab and got this
```
[{'expected': SplitInfo(name='train', num_bytes=610252351, num_examples=532812,
dataset_name='blog_authorship_corpus'), 'recorded': SplitInfo(name='train',
num_bytes=611607465, num_examples=533285, dataset_name='blog_authorship_corpus')},
{'expected': SplitInfo(name='validation', num_bytes=37500394, num_examples=31277,
dataset_name='blog_authorship_corpus'), 'recorded': SplitInfo(name='validation',
num_bytes=35652716, num_examples=30804, dataset_name='blog_authorship_corpus')}]
```
which is different from the `dataset_infos.json` and also different from yours.

It looks like the script for generating examples is not consistent


NonMatchingSplitsSizesError when loading blog_authorship_corpus
 Getting this error when i run `nlp.load_dataset('blog_authorship_corpus')`.

```
raise NonMatchingSplitsSizesError(str(bad_splits))
nlp.utils.info_utils.NonMatchingSplitsSizesError: [{'expected': SplitInfo(name='train',
num_bytes=610252351, num_examples=532812, dataset_name='blog_authorship_corpus'),
'recorded': SplitInfo(name='train', num_bytes=616473500, num_examples=536323,
dataset_name='blog_authorship_corpus')}, {'expected': SplitInfo(name='validation',
num_bytes=37500394, num_examples=31277, dataset_name='blog_authorship_corpus'),
'recorded': SplitInfo(name='validation', num_bytes=30786661, num_examples=27766,
dataset_name='blog_authorship_corpus')}]
```

Upon checking it seems like there is a disparity between the information in `datasets/blog_authorship_corpus/dataset_infos.json` and what was downloaded. Although I can get away with this by passing `ignore_verifications=True` in `load_dataset`, I'm thinking doing so might give problems later on.
 The files provided by the authors are corrupted and the script seems to ignore the xml files that can't be decoded (it does `try:... except UnicodeDecodeError`). Maybe depending of the environment some files can be opened and some others don't but not sure why


NonMatchingSplitsSizesError when loading blog_authorship_corpus
 Getting this error when i run `nlp.load_dataset('blog_authorship_corpus')`.

```
raise NonMatchingSplitsSizesError(str(bad_splits))
nlp.utils.info_utils.NonMatchingSplitsSizesError: [{'expected': SplitInfo(name='train',
num_bytes=610252351, num_examples=532812, dataset_name='blog_authorship_corpus'),
'recorded': SplitInfo(name='train', num_bytes=616473500, num_examples=536323,
dataset_name='blog_authorship_corpus')}, {'expected': SplitInfo(name='validation',
num_bytes=37500394, num_examples=31277, dataset_name='blog_authorship_corpus'),
'recorded': SplitInfo(name='validation', num_bytes=30786661, num_examples=27766,
dataset_name='blog_authorship_corpus')}]
```

Upon checking it seems like there is a disparity between the information in `datasets/blog_authorship_corpus/dataset_infos.json` and what was downloaded. Although I can get away with this by passing `ignore_verifications=True` in `load_dataset`, I'm thinking doing so might give problems later on.
 Feel free to do `ignore_verifications=True` for now... The verifications only include a check on the checksums of the downloaded files, and a check on the number of examples in each splits.


NonMatchingSplitsSizesError when loading blog_authorship_corpus
 Getting this error when i run `nlp.load_dataset('blog_authorship_corpus')`.

```
raise NonMatchingSplitsSizesError(str(bad_splits))
nlp.utils.info_utils.NonMatchingSplitsSizesError: [{'expected': SplitInfo(name='train',
num_bytes=610252351, num_examples=532812, dataset_name='blog_authorship_corpus'),
'recorded': SplitInfo(name='train', num_bytes=616473500, num_examples=536323,
dataset_name='blog_authorship_corpus')}, {'expected': SplitInfo(name='validation',
num_bytes=37500394, num_examples=31277, dataset_name='blog_authorship_corpus'),
'recorded': SplitInfo(name='validation', num_bytes=30786661, num_examples=27766,
dataset_name='blog_authorship_corpus')}]
```

Upon checking it seems like there is a disparity between the information in `datasets/blog_authorship_corpus/dataset_infos.json` and what was downloaded. Although I can get away with this by passing `ignore_verifications=True` in `load_dataset`, I'm thinking doing so might give problems later on.
 I'm getting this same issue when loading the `imdb` corpus via `dataset = load_dataset("imdb")`. When I try `ignore_verifications=True`, no examples are read into the `train` portion of the dataset.


NonMatchingSplitsSizesError when loading blog_authorship_corpus
 Getting this error when i run `nlp.load_dataset('blog_authorship_corpus')`.

```
raise NonMatchingSplitsSizesError(str(bad_splits))
nlp.utils.info_utils.NonMatchingSplitsSizesError: [{'expected': SplitInfo(name='train',
num_bytes=610252351, num_examples=532812, dataset_name='blog_authorship_corpus'),
'recorded': SplitInfo(name='train', num_bytes=616473500, num_examples=536323,
dataset_name='blog_authorship_corpus')}, {'expected': SplitInfo(name='validation',
num_bytes=37500394, num_examples=31277, dataset_name='blog_authorship_corpus'),
'recorded': SplitInfo(name='validation', num_bytes=30786661, num_examples=27766,
dataset_name='blog_authorship_corpus')}]
```

Upon checking it seems like there is a disparity between the information in `datasets/blog_authorship_corpus/dataset_infos.json` and what was downloaded. Although I can get away with this by passing `ignore_verifications=True` in `load_dataset`, I'm thinking doing so might give problems later on.
 > I'm getting this same issue when loading the `imdb` corpus via `dataset = load_dataset("imdb")`. When I try `ignore_verifications=True`, no examples are read into the `train` portion of the dataset.

When the checksums don't match, it may mean that the file you downloaded is corrupted. In this case you can try to load the dataset again `load_dataset("imdb", download_mode="force_redownload")`

Also I just checked on my side and it worked fine:

```python
from datasets import load_dataset

dataset = load_dataset("imdb")
print(len(dataset["train"]))
# 25000
```

Let me know if redownloading fixes your issue @EmilyAlsentzer .
If not, feel free to open a separate issue.


NonMatchingSplitsSizesError when loading blog_authorship_corpus
 Getting this error when i run `nlp.load_dataset('blog_authorship_corpus')`.

```
raise NonMatchingSplitsSizesError(str(bad_splits))
nlp.utils.info_utils.NonMatchingSplitsSizesError: [{'expected': SplitInfo(name='train',
num_bytes=610252351, num_examples=532812, dataset_name='blog_authorship_corpus'),
'recorded': SplitInfo(name='train', num_bytes=616473500, num_examples=536323,
dataset_name='blog_authorship_corpus')}, {'expected': SplitInfo(name='validation',
num_bytes=37500394, num_examples=31277, dataset_name='blog_authorship_corpus'),
'recorded': SplitInfo(name='validation', num_bytes=30786661, num_examples=27766,
dataset_name='blog_authorship_corpus')}]
```

Upon checking it seems like there is a disparity between the information in `datasets/blog_authorship_corpus/dataset_infos.json` and what was downloaded. Although I can get away with this by passing `ignore_verifications=True` in `load_dataset`, I'm thinking doing so might give problems later on.
 I wasn't aware of the "force_redownload" option and manually removed the '/home/me/.cache/huggingface/datasets/' dir, this worked for me (dataset 'cnn_dailymail')


NonMatchingSplitsSizesError when loading blog_authorship_corpus
 Getting this error when i run `nlp.load_dataset('blog_authorship_corpus')`.

```
raise NonMatchingSplitsSizesError(str(bad_splits))
nlp.utils.info_utils.NonMatchingSplitsSizesError: [{'expected': SplitInfo(name='train',
num_bytes=610252351, num_examples=532812, dataset_name='blog_authorship_corpus'),
'recorded': SplitInfo(name='train', num_bytes=616473500, num_examples=536323,
dataset_name='blog_authorship_corpus')}, {'expected': SplitInfo(name='validation',
num_bytes=37500394, num_examples=31277, dataset_name='blog_authorship_corpus'),
'recorded': SplitInfo(name='validation', num_bytes=30786661, num_examples=27766,
dataset_name='blog_authorship_corpus')}]
```

Upon checking it seems like there is a disparity between the information in `datasets/blog_authorship_corpus/dataset_infos.json` and what was downloaded. Although I can get away with this by passing `ignore_verifications=True` in `load_dataset`, I'm thinking doing so might give problems later on.
 Yes I think this might not be documented well enough. Let’s add it to the doc @lhoestq @SBrandeis.
And everything on how to control the cache behavior better (removing, overriding, changing the path, etc)


[Arrow writer, Trivia_qa] Could not convert TagMe with type str: converting to null type
 Running the following code

```
import nlp
ds = nlp.load_dataset("trivia_qa", "rc", split="validation[:1%]")  # this might take 2.3 min to download but it's cached afterwards...
ds.map(lambda x: x, load_from_cache_file=False)
```

triggers a `ArrowInvalid: Could not convert TagMe with type str: converting to null type` error.

On the other hand if we remove a certain column of `trivia_qa` which seems responsible for the bug, it works:

```
import nlp
ds = nlp.load_dataset("trivia_qa", "rc", split="validation[:1%]")  # this might take 2.3 min to download but it's cached afterwards...
ds.map(lambda x: x, remove_columns=["entity_pages"], load_from_cache_file=False)
```

. Seems quite hard to debug what's going on here... @lhoestq @thomwolf - do you have a good first guess what the problem could be?

**Note** BTW: I think this could be a good test to check that the datasets work correctly: Take a tiny portion of the dataset and check that it can be written correctly.
 Here the full error trace:

```
ArrowInvalid                              Traceback (most recent call last)
<ipython-input-1-7aaf3f011358> in <module>
      1 import nlp
      2 ds = nlp.load_dataset("trivia_qa", "rc", split="validation[:1%]")  # this might take 2.3 min to download but it's cached afterwards...
----> 3 ds.map(lambda x: x, load_from_cache_file=False)

~/python_bin/nlp/arrow_dataset.py in map(self, function, with_indices, batched, batch_size, remove_columns, keep_in_memory, load_from_cache_file, cache_file_name, writer_batch_size, arrow_schema, disable_nullable)
    549
    550         if update_data:
--> 551             writer.finalize()  # close_stream=bool(buf_writer is None))  # We only close if we are writing in a file
    552
    553             # Create new Dataset from buffer or file

~/python_bin/nlp/arrow_writer.py in finalize(self, close_stream)
    182     def finalize(self, close_stream=True):
    183         if self.pa_writer is not None:
--> 184             self.write_on_file()
    185             self.pa_writer.close()
    186         if close_stream:

~/python_bin/nlp/arrow_writer.py in write_on_file(self)
    104         """
    105         if self.current_rows:
--> 106             pa_array = pa.array(self.current_rows, type=self._type)
    107             first_example = pa.array(self.current_rows[0:1], type=self._type)[0]
    108             # Sanity check

~/hugging_face/venv_3.7/lib/python3.7/site-packages/pyarrow/array.pxi in pyarrow.lib.array()

~/hugging_face/venv_3.7/lib/python3.7/site-packages/pyarrow/array.pxi in pyarrow.lib._sequence_to_array()

~/hugging_face/venv_3.7/lib/python3.7/site-packages/pyarrow/error.pxi in pyarrow.lib.check_status()

ArrowInvalid: Could not convert TagMe with type str: converting to null type
```


[Arrow writer, Trivia_qa] Could not convert TagMe with type str: converting to null type
 Running the following code

```
import nlp
ds = nlp.load_dataset("trivia_qa", "rc", split="validation[:1%]")  # this might take 2.3 min to download but it's cached afterwards...
ds.map(lambda x: x, load_from_cache_file=False)
```

triggers a `ArrowInvalid: Could not convert TagMe with type str: converting to null type` error.

On the other hand if we remove a certain column of `trivia_qa` which seems responsible for the bug, it works:

```
import nlp
ds = nlp.load_dataset("trivia_qa", "rc", split="validation[:1%]")  # this might take 2.3 min to download but it's cached afterwards...
ds.map(lambda x: x, remove_columns=["entity_pages"], load_from_cache_file=False)
```

. Seems quite hard to debug what's going on here... @lhoestq @thomwolf - do you have a good first guess what the problem could be?

**Note** BTW: I think this could be a good test to check that the datasets work correctly: Take a tiny portion of the dataset and check that it can be written correctly.
 Actually thinking a bit more about it, it's probably a data sample that is not correct in `trivia_qa`. But I'm a bit surprised though that we managed to write it in .arrow format and now cannot write it anymore after an "identity" mapping.