New HF datasets version causes unit test failures when using filtering_lambda

[yoavkatz]

Localized an temporarily disabled test in test_loaders.

@unittest.skip("Currently this fails from datasets 2.20") def test_load_from_HF_multiple_innvocation_with_filter(self): loader = LoadHF( path="CohereForAI/aya_evaluation_suite", name="aya_human_annotated", filtering_lambda='lambda instance: instance["language"]=="eng"', ) ms = loader.process() dataset = ms.to_dataset() self.assertEqual( list(dataset.keys()), ["test"] ) # that HF dataset only has the 'test' split self.assertEqual(dataset["test"][0]["language"], "eng")

    ms = loader.process()
    dataset = ms.to_dataset()
    self.assertEqual(
        list(dataset.keys()), ["test"]
    )  # that HF dataset only has the 'test' split
    self.assertEqual(dataset["test"][0]["language"], "eng")