{"config": "fp16", "model": "Qwen/Qwen2.5-7B-Instruct", "n_items": 200, "seed": 20260726, "conditions": {"baseline": {"wall_s": 3.108467969000003, "gen_tokens": 2814, "mean_f1": 0.6890078214781351, "mean_contain": 0.975, "mean_comply": 0.825, "items": [{"id": "squad-1658", "gold": ["20 December 1914", "20 December 1914", "20 December 1914"], "pred_text": "20 December 1914", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-7068", "gold": ["Nepali", "Nepali", "Nepali"], "pred_text": "Nepali and/or the state's official language", "f1": 0.33333333333333337, "contain": 1.0, "comply": 1.0, "n_words": 6}, {"id": "squad-5920", "gold": ["Duel", "Duel", "Duel"], "pred_text": "Duel", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-10453", "gold": ["Pauli", "Pauli principle", "Pauli principle", "Pauli", "Pauli"], "pred_text": "the Pauli principle relating the space and the spin variables.", "f1": 0.4444444444444445, "contain": 1.0, "comply": 1.0, "n_words": 10}, {"id": "squad-3640", "gold": ["algae", "algae", "algae", "algae", "algae"], "pred_text": "Pollution of water by nitrates and phosphates may stimulate growth of algae", "f1": 0.15384615384615385, "contain": 1.0, "comply": 0.0, "n_words": 12}, {"id": "squad-10286", "gold": ["formalize a unified front in trade and negotiations with various Indians, since allegiance of the various tribes and nations was seen to be pivotal", "formalize a unified front in trade and negotiations with various Indians", "formalize a unified front in trade and negotiations with various Indians", "unified front in trade and negotiations with various Indians", "to formalize a unified front in trade and negotiations with various Indians"], "pred_text": "to formalize a unified front in trade and negotiations with various Indians", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 12}, {"id": "squad-2981", "gold": ["−11.7 °C", "−11.7 °C", "−11.7 °C (10.9 °F)"], "pred_text": "The state's lowest minimum temperature of −11.7 °C (10.9 °F) was recorded at Omeo on 13 June 1965, and again at Falls Creek on 3 July", "f1": 0.2758620689655173, "contain": 1.0, "comply": 0.0, "n_words": 26}, {"id": "squad-4383", "gold": ["anthropological", "anthropological", "recent anthropological"], "pred_text": "Some 5 million people may have lived in the Amazon region in AD 1500, divided between dense coastal settlements, such as that at Marajó, and inland dwellers.", "f1": 0.0, "contain": 0.0, "comply": 0.0, "n_words": 27}, {"id": "squad-1041", "gold": ["Polish United Workers' Party", "Polish United Workers' Party", "Polish United Workers' Party"], "pred_text": "the Polish United Workers' Party (PZPR)", "f1": 0.888888888888889, "contain": 1.0, "comply": 1.0, "n_words": 6}, {"id": "squad-7784", "gold": ["Time Lord", "Time Lord", "Time Lord"], "pred_text": "Time Lord", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-5840", "gold": ["Regis Philbin", "Regis Philbin", "Regis Philbin"], "pred_text": "Regis Philbin", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-1460", "gold": ["1900", "1900", "1900"], "pred_text": "In 1900, Tesla was granted patents for a \"system of transmitting electrical energy\" and \"an electrical transmitter.\"", "f1": 0.125, "contain": 1.0, "comply": 1.0, "n_words": 17}, {"id": "squad-8247", "gold": ["26 seconds off the modern Gregorian calendar", "365.2425 days of the year", "365.2425 days of the year"], "pred_text": "which was only 26 seconds off the modern Gregorian calendar's measurement.", "f1": 0.625, "contain": 1.0, "comply": 1.0, "n_words": 11}, {"id": "squad-2174", "gold": ["God", "God", "God"], "pred_text": "God.", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-6618", "gold": ["cholecalciferol", "cholecalciferol", "cholecalciferol"], "pred_text": "cholecalciferol", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-9291", "gold": ["west", "west", "west"], "pred_text": "farther west, through the Waal and then, via the Merwede and Nieuwe Merwede (De Biesbosch), merging with the Meuse, through the Hollands Diep and Haringvliet", "f1": 0.0909090909090909, "contain": 1.0, "comply": 0.0, "n_words": 25}, {"id": "squad-873", "gold": ["60", "60", "60"], "pred_text": "60 rooms", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-8566", "gold": ["Africa's most successful nation in the 2008 Olympics", "Africa's most successful nation", "Africa's most successful nation"], "pred_text": "Kenya won several medals during the Beijing Olympics, six gold, four silver and four bronze, making it Africa's most successful nation in the 2008 Olympics.", "f1": 0.4666666666666667, "contain": 1.0, "comply": 0.0, "n_words": 25}, {"id": "squad-1263", "gold": ["1878", "December 1878", "to hide the fact that he dropped out of school"], "pred_text": "In December 1878, Tesla left Graz and severed all relations with his family to hide the fact that he dropped out of school.", "f1": 0.5806451612903226, "contain": 1.0, "comply": 0.0, "n_words": 23}, {"id": "squad-2458", "gold": ["24", "24", "24"], "pred_text": "24", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-2952", "gold": ["1975", "1975", "1975"], "pred_text": "Victoria has a written constitution enacted in 1975, but based on the 1855 colonial constitution, passed by the United Kingdom Parliament as the Victoria Constitution Act 1855, which establishes the Parliament as the", "f1": 0.07142857142857142, "contain": 1.0, "comply": 0.0, "n_words": 33}, {"id": "squad-960", "gold": ["325", "325", "325"], "pred_text": "523 km (325 mi) east of Berlin, Germany.", "f1": 0.2222222222222222, "contain": 1.0, "comply": 1.0, "n_words": 8}, {"id": "squad-2644", "gold": ["3,792,621", "3,792,621", "3,792,621"], "pred_text": "3,792,621", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-2742", "gold": ["the housing bubble", "the housing bubble"], "pred_text": "The region was a leader in the housing bubble 2001–2007.", "f1": 0.4444444444444445, "contain": 1.0, "comply": 1.0, "n_words": 10}, {"id": "squad-3247", "gold": ["Huguenots furnished two new regiments", "two new regiments", "furnished two new regiments of his army"], "pred_text": "The Huguenots furnished two new regiments of his army: the Altpreußische Infantry Regiments No. 13 (Regiment on foot Varenne) and 15 (Regiment on foot Wylich", "f1": 0.4666666666666667, "contain": 1.0, "comply": 0.0, "n_words": 25}, {"id": "squad-1645", "gold": ["gravity", "gravity", "of gravity"], "pred_text": "\"[would] put an end to idle speculations and false conceptions, as that of curved space.\"", "f1": 0.125, "contain": 0.0, "comply": 1.0, "n_words": 15}, {"id": "squad-6218", "gold": ["opened the gates", "opened the gates to the Mongols", "opened the gates to the Mongols"], "pred_text": "The city leaders opened the gates to the Mongols.", "f1": 0.8, "contain": 1.0, "comply": 1.0, "n_words": 9}, {"id": "squad-9887", "gold": ["1919", "1919", "1919", "1919", "1919"], "pred_text": "1919", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-1065", "gold": ["Viking", "Norseman, Viking", "Norseman, Viking"], "pred_text": "Norseman, Viking", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-2750", "gold": ["business", "major business", "major business"], "pred_text": "Central business districts (CBD)", "f1": 0.4, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-4739", "gold": ["\"Southwest Fresno\"", "Southwest Fresno", "Southwest Fresno"], "pred_text": "Southwest Fresno", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-2605", "gold": ["1 a.m", "1 a.m.", "1 a.m"], "pred_text": "At 1 a.m. he awoke with more chest pain and was warmed with hot towels.", "f1": 0.23529411764705882, "contain": 1.0, "comply": 1.0, "n_words": 15}, {"id": "squad-3508", "gold": ["active", "more active and lived longer", "was more active and lived longer", "more active and lived longer while breathing it", "was more active and lived longer"], "pred_text": "a mouse was more active and lived longer while breathing it", "f1": 0.888888888888889, "contain": 1.0, "comply": 1.0, "n_words": 11}, {"id": "squad-7014", "gold": ["$45,000", "more than $45,000", "$45,000"], "pred_text": "more than $45,000", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-859", "gold": ["Wianki", "Wianki", "Wianki"], "pred_text": "Wianki", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-322", "gold": ["17 seconds", "17 seconds", "17"], "pred_text": "17 seconds", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-9787", "gold": ["recognition", "international recognition", "recognition"], "pred_text": "international recognition", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-9214", "gold": ["Finsteraarhorn", "summit of Finsteraarhorn", "Finsteraarhorn, t"], "pred_text": "The 4,274 m (14,022 ft) summit of Finsteraarhorn, the highest point of the Rhine basin.", "f1": 0.4, "contain": 1.0, "comply": 1.0, "n_words": 15}, {"id": "squad-4472", "gold": ["to help strengthen their territorial claims", "map out their ancestral lands to help strengthen their territorial claims", "map out their ancestral lands"], "pred_text": "to map out their ancestral lands to help strengthen their territorial claims.", "f1": 0.9565217391304348, "contain": 1.0, "comply": 0.0, "n_words": 12}, {"id": "squad-8092", "gold": ["Alex Seropian", "Alex Seropian", "Alex Seropian"], "pred_text": "Alex Seropian", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-5683", "gold": ["1887", "1887", "1887"], "pred_text": "1887", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-888", "gold": ["a sword", "sword", "sword"], "pred_text": "holding a sword", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-581", "gold": ["late local programming", "late local programming", "late local programming"], "pred_text": "Following a break for late local programming, CBS also aired a special episode of The Late Late Show with James Corden.", "f1": 0.2857142857142857, "contain": 1.0, "comply": 0.0, "n_words": 21}, {"id": "squad-4588", "gold": ["a balance sensor", "a balance sensor consisting of a statolith", "a balance sensor consisting of a statolith"], "pred_text": "A statocyst, a balance sensor consisting of a statolith, a solid particle supported on four bundles of cilia, called \"balancers\", that sense its orientation.", "f1": 0.4, "contain": 1.0, "comply": 0.0, "n_words": 24}, {"id": "squad-4791", "gold": ["June 1, 1953", "June 1, 1953", "June 1, 1953"], "pred_text": "KMJ-TV, which debuted on June 1, 1953.", "f1": 0.6, "contain": 1.0, "comply": 1.0, "n_words": 7}, {"id": "squad-5279", "gold": ["EAT!", "EAT!", "EAT! NewcastleGateshead"], "pred_text": "EAT! NewcastleGateshead", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-763", "gold": ["50-yard line.", "41", "50"], "pred_text": "the Panthers 41-yard line", "f1": 0.4, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-993", "gold": ["the most distinctive buildings", "the most distinctive buildings", "most distinctive buildings"], "pred_text": "the most distinctive buildings in prewar Warsaw.", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 7}, {"id": "squad-6405", "gold": ["potential drug interactions, adverse drug reactions", "potential drug interactions", "potential drug interactions, adverse drug reactions, and assess patient drug allergies"], "pred_text": "monitor for potential drug interactions, adverse drug reactions, and assess patient drug allergies", "f1": 0.9166666666666666, "contain": 1.0, "comply": 1.0, "n_words": 13}, {"id": "squad-677", "gold": ["Super Bowl XLVII", "Super Bowl XLVII", "XLVII"], "pred_text": "Super Bowl XLVII", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-1892", "gold": ["religious", "religious", "religious"], "pred_text": "religious texts", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-86", "gold": ["$5 million", "$5 million", "$5 million"], "pred_text": "$5 million", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-3625", "gold": ["magnetic field", "magnetic", "magnetic", "magnetic field", "magnetic"], "pred_text": "a magnetic field", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-8701", "gold": ["Cyanobacteria", "Cyanobacteria", "Cyanobacteria"], "pred_text": "Cyanobacteria are considered the ancestors of chloroplasts.", "f1": 0.2857142857142857, "contain": 1.0, "comply": 1.0, "n_words": 7}, {"id": "squad-9631", "gold": ["its supporters", "Scholars and observers", "Islamism"], "pred_text": "Scholars and observers who do not believe that Islam is merely a political ideology including Fred Halliday, John Esposito and Muslim intellectuals like Javed Ahmad Ghamidi.", "f1": 0.21428571428571425, "contain": 1.0, "comply": 0.0, "n_words": 26}, {"id": "squad-7526", "gold": ["nine", "nine factors", "nine"], "pred_text": "nine factors", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-7627", "gold": ["a better relevant income.", "better relevant income", "a better relevant income"], "pred_text": "can thus work towards a better relevant income", "f1": 0.6, "contain": 1.0, "comply": 1.0, "n_words": 8}, {"id": "squad-8153", "gold": ["southern China", "southern China", "in southern China"], "pred_text": "Möngke Khan commenced a military campaign against the Chinese Song dynasty in southern China.", "f1": 0.4, "contain": 1.0, "comply": 1.0, "n_words": 14}, {"id": "squad-7404", "gold": ["level of the top tax rate", "level of the top tax rate", "top tax rate"], "pred_text": "the level of the top tax rate will often have a direct impact on the level of inequality within a society", "f1": 0.47619047619047616, "contain": 1.0, "comply": 0.0, "n_words": 21}, {"id": "squad-5418", "gold": ["Somerset House", "Somerset House", "Somerset House"], "pred_text": "Somerset House.", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-7443", "gold": ["little", "little support", "little"], "pred_text": "the available evidence provides little support for the view that U.S.-style labor-market flexibility dramatically improves labor-market outcomes.", "f1": 0.23529411764705882, "contain": 1.0, "comply": 1.0, "n_words": 17}, {"id": "squad-3832", "gold": ["Budget cuts", "oxygen tank explosion", "Budget cuts", "Budget cuts", "Budget cuts"], "pred_text": "Budget cuts forced the cancellation of three of these.", "f1": 0.4, "contain": 1.0, "comply": 1.0, "n_words": 9}, {"id": "squad-6537", "gold": ["tears", "tears", "tears"], "pred_text": "tears", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-1544", "gold": ["minimal", "minimal", "minimal"], "pred_text": "minimal loss", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-1986", "gold": ["motivated students", "motivated students", "motivated students"], "pred_text": "motivated students", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-8652", "gold": ["Ed Whitfield", "Ed Whitfield", "Ed Whitfield"], "pred_text": "Ed Whitfield", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-6625", "gold": ["apoptosis", "apoptosis", "rapid apoptosis"], "pred_text": "cells at the site of infection undergo rapid apoptosis", "f1": 0.4, "contain": 1.0, "comply": 1.0, "n_words": 9}, {"id": "squad-8241", "gold": ["Marco Polo", "Marco Polo", "Marco Polo"], "pred_text": "The most famous traveler of the period was the Venetian Marco Polo,", "f1": 0.3636363636363636, "contain": 1.0, "comply": 1.0, "n_words": 12}, {"id": "squad-1577", "gold": ["nothing", "nothing", "nothing"], "pred_text": "There was nothing which would constitute a hazard in unfriendly hands.", "f1": 0.18181818181818182, "contain": 1.0, "comply": 1.0, "n_words": 11}, {"id": "squad-207", "gold": ["torn ACL", "a torn ACL", "torn ACL"], "pred_text": "a torn ACL", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-4276", "gold": ["citizenship", "\"citizenship\"", "\"citizenship\""], "pred_text": "free movement was increasingly based on \"citizenship\", so that people had rights to empower them to become economically and socially active, rather than economic activity being a precondition for rights.", "f1": 0.06666666666666667, "contain": 1.0, "comply": 0.0, "n_words": 30}, {"id": "squad-8413", "gold": ["exercised by the government", "government", "government."], "pred_text": "Executive power is exercised by the government.", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 7}, {"id": "squad-5657", "gold": ["more than 20", "20 works", "20"], "pred_text": "more than 20 works", "f1": 0.8571428571428571, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-7101", "gold": ["public", "public", "public"], "pred_text": "public schools", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-9616", "gold": ["reordering", "reordering of government and society in accordance with the Shari'a", "reordering"], "pred_text": "to implement Islamic values in all spheres of life.", "f1": 0.2222222222222222, "contain": 0.0, "comply": 1.0, "n_words": 9}, {"id": "squad-2647", "gold": ["south", "south"], "pred_text": "To the south", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-5575", "gold": ["over 6000", "over 6000", "over 6000"], "pred_text": "over 6000 items", "f1": 0.8, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-7700", "gold": ["12", "12", "12"], "pred_text": "12 episodes (plus an earlier one-episode teaser)", "f1": 0.2857142857142857, "contain": 1.0, "comply": 1.0, "n_words": 7}, {"id": "squad-4144", "gold": ["every five years", "every five years", "every five years", "every five years"], "pred_text": "Parliament elections, take place every five years, and votes for Members of the European Parliament in member states must be organised by proportional representation or a single transferable vote.", "f1": 0.19999999999999998, "contain": 1.0, "comply": 0.0, "n_words": 29}, {"id": "squad-3093", "gold": ["Cévennes", "Cévennes", "Cévennes mountain region"], "pred_text": "the Cévennes mountain region", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-7676", "gold": ["Battlestar Galactica and Bionic Woman", "Battlestar Galactica and Bionic Woman", "Battlestar Galactica and Bionic Woman"], "pred_text": "Battlestar Galactica and Bionic Woman", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 5}, {"id": "squad-2285", "gold": ["private conferences", "private conferences", "conferences"], "pred_text": "Over the next five days, private conferences were held to determine Luther's fate.", "f1": 0.2857142857142857, "contain": 1.0, "comply": 1.0, "n_words": 13}, {"id": "squad-10043", "gold": ["1968", "1968", "1968"], "pred_text": "1968", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-6950", "gold": ["the property owner", "the property owner", "property owner"], "pred_text": "the property owner.", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-9268", "gold": ["Ruhr", "Ruhr", "The Ruhr"], "pred_text": "The Ruhr, which joins the Rhine in Duisburg,", "f1": 0.2857142857142857, "contain": 1.0, "comply": 1.0, "n_words": 8}, {"id": "squad-3159", "gold": ["Cévennes", "Cévennes", "Cévennes region in the south"], "pred_text": "the rugged Cévennes region in the south", "f1": 0.888888888888889, "contain": 1.0, "comply": 1.0, "n_words": 7}, {"id": "squad-9251", "gold": ["Middle Rhine", "Middle Rhine", "Middle Rhine"], "pred_text": "Between Bingen and Bonn, the Middle Rhine flows through the Rhine Gorge.", "f1": 0.33333333333333337, "contain": 1.0, "comply": 1.0, "n_words": 12}, {"id": "squad-4127", "gold": ["Article 17(3)", "Article 17(3)", "Article 17(3)", "Article 17(3)"], "pred_text": "Article 17(3)", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-1777", "gold": ["complicated definitions", "complicated definitions", "definitions"], "pred_text": "some complexity classes have complicated definitions that do not fit into this framework.", "f1": 0.2666666666666667, "contain": 1.0, "comply": 1.0, "n_words": 13}, {"id": "squad-5710", "gold": ["Citadel Broadcasting", "Citadel Broadcasting", "Citadel Broadcasting"], "pred_text": "Citadel Broadcasting", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-8394", "gold": ["The central highlands", "central highlands", "central highlands"], "pred_text": "The central highlands were already home to over a million members of the Kikuyu people, most of whom had no land claims in European terms and lived as itinerant farmers.", "f1": 0.13793103448275862, "contain": 1.0, "comply": 0.0, "n_words": 30}, {"id": "squad-9985", "gold": ["American delegation from the Paris Peace Conference", "American delegation from the Paris Peace Conference", "the American delegation from the Paris Peace Conference", "American delegation from the Paris Peace Conference", "American delegation from the Paris Peace Conference"], "pred_text": "The American delegation from the Paris Peace Conference.", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 8}, {"id": "squad-3248", "gold": ["Frederick William", "Frederick William", "Frederick William"], "pred_text": "Frederick William, Elector of Brandenburg, invited Huguenots to settle in his realms.", "f1": 0.2857142857142857, "contain": 1.0, "comply": 1.0, "n_words": 12}, {"id": "squad-6147", "gold": ["the Keraites", "Keraites", "Keraites"], "pred_text": "Keraites", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-2163", "gold": ["Catholic", "Catholic", "Catholic"], "pred_text": "as a Catholic", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-2452", "gold": ["prayer for grace", "grace", "prayer for grace"], "pred_text": "Psalm 67's prayer for grace;", "f1": 0.7499999999999999, "contain": 1.0, "comply": 1.0, "n_words": 5}, {"id": "squad-1813", "gold": ["NP-complete", "NP-complete", "NP-complete"], "pred_text": "NP-complete problems", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-4756", "gold": ["2,500", "2,500", "2,500"], "pred_text": "The amphitheatre that seats up to 2,500 people,", "f1": 0.25, "contain": 1.0, "comply": 1.0, "n_words": 8}, {"id": "squad-7919", "gold": ["2011", "2011", "2011"], "pred_text": "2011", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-4605", "gold": ["four", "four", "four"], "pred_text": "four", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-2251", "gold": ["December 1517", "December 1517", "1517"], "pred_text": "December 1517", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-7588", "gold": ["reducing poverty", "reducing poverty", "reducing poverty"], "pred_text": "progress on reducing poverty.", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-9629", "gold": ["political", "political", "political"], "pred_text": "it posits a political role for Islam", "f1": 0.2857142857142857, "contain": 1.0, "comply": 1.0, "n_words": 7}, {"id": "squad-106", "gold": ["Roger Goodell", "Roger Goodell", "Goodell"], "pred_text": "Roger Goodell", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-8714", "gold": ["green chloroplast", "green chloroplast lineage", "green chloroplast lineage"], "pred_text": "the green chloroplast lineage", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-10257", "gold": ["capturing three traders and killing 14 people of the Miami nation, including Old Briton", "capturing three traders and killing 14 people of the Miami nation", "capturing three traders and killing 14 people of the Miami nation, including Old Briton", "capturing three traders and killing 14 people", "capturing three traders and killing 14 people of the Miami nation"], "pred_text": "capturing three traders and killing 14 people of the Miami nation, including Old Briton. He was reportedly ritually cannibalized by some aboriginal members of the expedition.", "f1": 0.7027027027027027, "contain": 1.0, "comply": 0.0, "n_words": 26}, {"id": "squad-9746", "gold": ["destruction of Israel", "the destruction of Israel", "destruction of Israel"], "pred_text": "calling for the destruction of Israel and the establishment of an Islamic state in Palestine", "f1": 0.4, "contain": 1.0, "comply": 1.0, "n_words": 15}, {"id": "squad-10239", "gold": ["village of Pickawillany", "village of Pickawillany", "village of Pickawillany", "Pickawillany", "village of Pickawillany"], "pred_text": "the village of Pickawillany", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-5811", "gold": ["1995–96 season", "1995–96", "2011–12"], "pred_text": "beginning in the 1995–96 season", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 5}, {"id": "squad-3376", "gold": ["John Smeaton", "John Smeaton", "John Smeaton", "John Smeaton"], "pred_text": "John Smeaton", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-5337", "gold": ["smart ticketing", "smart ticketing", "smart ticketing."], "pred_text": "smart ticketing.", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-1636", "gold": ["her weight", "her weight", "her weight."], "pred_text": "because of her weight", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-4513", "gold": ["colloblasts", "retractable tentacles fringed with tentilla", "colloblasts"], "pred_text": "a pair of retractable tentacles fringed with tentilla that are covered with colloblasts", "f1": 0.5882352941176471, "contain": 1.0, "comply": 0.0, "n_words": 13}, {"id": "squad-1623", "gold": ["chastity", "his chastity", "chastity"], "pred_text": "his chastity was very helpful to his scientific abilities.", "f1": 0.3636363636363636, "contain": 1.0, "comply": 1.0, "n_words": 9}, {"id": "squad-4548", "gold": ["cnidarians", "Ctenophores", "cnidarians"], "pred_text": "Cnidarians", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-676", "gold": ["British", "British", "British"], "pred_text": "British", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-8195", "gold": ["Emperor Chengzong", "Emperor Chengzong", "Emperor Chengzong"], "pred_text": "Emperor Chengzong", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-7730", "gold": ["William Hartnell's poor health", "to permit the recasting of the main character", "William Hartnell's poor health"], "pred_text": "Producers introduced the concept of regeneration to permit the recasting of the main character. This was first prompted by original star William Hartnell's poor health.", "f1": 0.42857142857142855, "contain": 1.0, "comply": 0.0, "n_words": 25}, {"id": "squad-3807", "gold": ["Ford, Chrysler, and GM", "Ford, Chrysler, and GM", "Ford, Chrysler, and GM, respectively", "Ford", "Ford, Chrysler, and GM"], "pred_text": "Ford, Chrysler, and GM, respectively.", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 5}, {"id": "squad-8231", "gold": ["1368–1644", "1368–1644", "1368–1644"], "pred_text": "1368–1644", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-5049", "gold": ["xenoliths", "xenoliths", "inclusions", "xenoliths"], "pred_text": "xenoliths", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-1266", "gold": ["draftsman", "he worked as a draftsman", "draftsman"], "pred_text": "Tesla went to Maribor (now in Slovenia), where he worked as a draftsman for 60 florins a month.", "f1": 0.4, "contain": 1.0, "comply": 0.0, "n_words": 18}, {"id": "squad-5216", "gold": ["shopping", "shopping", "shopping destinations"], "pred_text": "shopping destinations", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-2271", "gold": ["Prince Frederick III", "Prince Frederick III", "Elector of Saxony"], "pred_text": "Prince Frederick III, Elector of Saxony, obtained a safe conduct for Luther to and from the meeting.", "f1": 0.33333333333333337, "contain": 1.0, "comply": 1.0, "n_words": 17}, {"id": "squad-3591", "gold": ["twice", "50% more", "about twice as much", "twice as much", "twice"], "pred_text": "about twice as much (14.6 mg·L−1) dissolves at 0 °C than at 20 °C (7.6 mg·L−1).", "f1": 0.4, "contain": 1.0, "comply": 0.0, "n_words": 16}, {"id": "squad-6698", "gold": ["human papillomavirus", "human papillomavirus", "human papillomavirus"], "pred_text": "human papillomavirus", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-6676", "gold": ["phagocytes", "phagocytes", "phagocytes"], "pred_text": "phagocytes", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-2182", "gold": ["jailer and hangman", "jailer", "jailer and hangman of my poor soul."], "pred_text": "jailer and hangman of my poor soul", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 7}, {"id": "squad-7707", "gold": ["2010", "2010", "2010"], "pred_text": "2010", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-8355", "gold": ["45 million people", "45 million people", "approximately 45 million"], "pred_text": "approximately 45 million people", "f1": 0.8571428571428571, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-3871", "gold": ["Apollo spacecraft", "Apollo", "Apollo", "Apollo"], "pred_text": "Apollo spacecraft", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-4627", "gold": ["internal fertilization and keep the eggs in brood chambers until they hatch.", "internal fertilization", "internal fertilization"], "pred_text": "platyctenids use internal fertilization and keep the eggs in brood chambers until they hatch.", "f1": 0.9166666666666666, "contain": 1.0, "comply": 1.0, "n_words": 14}, {"id": "squad-820", "gold": ["15 kilometres", "15 kilometres", "About 15 kilometres"], "pred_text": "About 15 kilometres (9 miles) from Warsaw, the Vistula river's environment changes strikingly and features a perfectly preserved ecosystem, with a habitat of animals that includes the otter, beaver and hundreds of bird species.", "f1": 0.18181818181818182, "contain": 1.0, "comply": 0.0, "n_words": 34}, {"id": "squad-6572", "gold": ["MHC I (major histocompatibility complex)", "MHC I", "MHC I (major histocompatibility complex)"], "pred_text": "cell-surface marker called MHC I", "f1": 0.5714285714285715, "contain": 1.0, "comply": 1.0, "n_words": 5}, {"id": "squad-2920", "gold": ["over 1,000", "1,000", "1,000"], "pred_text": "over 1,000 miners", "f1": 0.8, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-8789", "gold": ["direct repeats", "direct repeats", "direct repeats"], "pred_text": "some chloroplast DNAs have since lost or flipped the inverted repeats (making them direct repeats).", "f1": 0.25, "contain": 1.0, "comply": 1.0, "n_words": 15}, {"id": "squad-852", "gold": ["Warsaw", "Warsaw", "Warsaw"], "pred_text": "Warsaw", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-952", "gold": ["Palace of Culture and Science", "Palace of Culture and Science", "Palace of Culture and Science"], "pred_text": "the Palace of Culture and Science", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 6}, {"id": "squad-974", "gold": ["turbulent history of the city", "turbulent history of the city and country", "turbulent history of the city"], "pred_text": "Warsaw's mixture of architectural styles reflects the turbulent history of the city and country.", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 14}, {"id": "squad-8656", "gold": ["2001", "2001", "2001"], "pred_text": "2001", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-6665", "gold": ["RNA interference pathway", "RNA interference", "RNA interference pathway"], "pred_text": "Ribonucleases and the RNA interference pathway", "f1": 0.7499999999999999, "contain": 1.0, "comply": 1.0, "n_words": 6}, {"id": "squad-9763", "gold": ["American attack on Iraq", "the American attack on Iraq", "American attack on Iraq"], "pred_text": "the American attack on Iraq in the 1991 Gulf War", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 10}, {"id": "squad-9142", "gold": ["Gaulish name Rēnos", "Rēnos", "the Gaulish name Rēnos", "Rēnos", "Rēnos"], "pred_text": "The variant forms of the name of the Rhine in modern languages are all derived from the Gaulish name Rēnos, which was adapted in Roman-era geography (1st century BC) as Greek Ῥῆνος (", "f1": 0.19354838709677416, "contain": 1.0, "comply": 0.0, "n_words": 33}, {"id": "squad-10296", "gold": ["garrisons", "garrisons", "garrisons", "garrisons"], "pred_text": "Shirley left garrisons at Oswego, Fort Bull, and Fort Williams.", "f1": 0.18181818181818182, "contain": 1.0, "comply": 1.0, "n_words": 10}, {"id": "squad-9437", "gold": ["1974", "1974", "1974"], "pred_text": "Prime Minister Wilson committing his government to some form of devolved legislature in 1974.", "f1": 0.13333333333333333, "contain": 1.0, "comply": 1.0, "n_words": 14}, {"id": "squad-1992", "gold": ["easier and more efficient", "easier and more efficient", "easier and more efficient"], "pred_text": "public order is easier and more efficient than anywhere else.", "f1": 0.5714285714285715, "contain": 1.0, "comply": 1.0, "n_words": 10}, {"id": "squad-10333", "gold": ["Two of the expeditions were successful, with Fort Duquesne and Louisbourg", "Two", "Two", "Two", "Two"], "pred_text": "Two of the expeditions were successful.", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 6}, {"id": "squad-4097", "gold": ["if the Treaty provisions have a direct effect and they are sufficiently clear, precise and unconditional.", "if the Treaty provisions have a direct effect and they are sufficiently clear, precise and unconditional", "if the Treaty provisions have a direct effect and they are sufficiently clear, precise and unconditional", "if the Treaty provisions have a direct effect and they are sufficiently clear, precise and unconditional"], "pred_text": "Individuals may rely on primary law in the Court of Justice of the European Union if the Treaty provisions have a direct effect and they are sufficiently clear, precise and unconditional.", "f1": 0.6829268292682926, "contain": 1.0, "comply": 0.0, "n_words": 31}, {"id": "squad-6111", "gold": ["his sons and grandsons", "sons and grandsons", "his sons and grandsons."], "pred_text": "his sons and grandsons", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-1415", "gold": ["1892 to 1894", "from 1892 to 1894", "from 1892 to 1894"], "pred_text": "from 1892 to 1894", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-3555", "gold": ["Aufbau", "Aufbau", "Aufbau", "Aufbau", "Aufbau"], "pred_text": "Aufbau", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-4932", "gold": ["1349", "1349", "1349"], "pred_text": "a ship landed at Askøy, then spread to Bjørgvin (modern Bergen) and Iceland.", "f1": 0.0, "contain": 0.0, "comply": 1.0, "n_words": 13}, {"id": "squad-10051", "gold": ["Justifying Grace or Accepting Grace", "Justifying Grace or Accepting Grace", "Justifying Grace"], "pred_text": "Justifying Grace", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-8320", "gold": ["allied groups from Central Asia and the western end of the empire", "various allied groups from Central Asia and the western end of the empire", "various allied groups"], "pred_text": "various allied groups from Central Asia and the western end of the empire", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 13}, {"id": "squad-2965", "gold": ["Roman Catholic Church", "the Roman Catholic Church", "Roman Catholic"], "pred_text": "the Roman Catholic Church", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-10468", "gold": ["20th", "the 20th century", "20th", "20th"], "pred_text": "20th century", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-4812", "gold": ["to provide a fault-tolerant, efficient routing method for telecommunication messages", "provide a fault-tolerant, efficient routing method for telecommunication messages"], "pred_text": "to provide a fault-tolerant, efficient routing method for telecommunication messages", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 10}, {"id": "squad-476", "gold": ["a week", "a week", "week"], "pred_text": "a week-long event", "f1": 0.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-1470", "gold": ["stationary", "stationary", "stationary waves"], "pred_text": "stationary waves", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-9310", "gold": ["Millingen aan de Rijn,", "near Millingen aan de Rijn", "near Millingen aan de Rijn"], "pred_text": "near Millingen aan de Rijn, close to the Dutch-German border", "f1": 0.7142857142857143, "contain": 1.0, "comply": 1.0, "n_words": 10}, {"id": "squad-10440", "gold": ["Galileo", "Aristotle's", "Aristotle", "Aristotle", "Aristotle's"], "pred_text": "Aristotle", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-6565", "gold": ["macrophages, neutrophils, and dendritic cells", "macrophages, neutrophils, and dendritic cells", "macrophages, neutrophils, and dendritic cells"], "pred_text": "macrophages, neutrophils, and dendritic cells", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 5}, {"id": "squad-8237", "gold": ["Nestorianism and Roman Catholicism", "Nestorianism and Roman Catholicism", "Nestorianism and Roman Catholicism"], "pred_text": "Nestorianism and Roman Catholicism also enjoyed a period of toleration.", "f1": 0.6153846153846153, "contain": 1.0, "comply": 1.0, "n_words": 10}, {"id": "squad-3722", "gold": ["January 18, 1974,", "January 18, 1974", "January 18, 1974", "January 18, 1974", "January 18, 1974"], "pred_text": "January 18, 1974", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-4463", "gold": ["climate change in addition to deforestation", "climate change in addition to deforestation", "climate change in addition to deforestation"], "pred_text": "the rainforest could be threatened though the 21st century by climate change in addition to deforestation.", "f1": 0.6, "contain": 1.0, "comply": 1.0, "n_words": 16}, {"id": "squad-1276", "gold": ["stroke", "a stroke", "a stroke"], "pred_text": "some sources say that he died of a stroke.", "f1": 0.2222222222222222, "contain": 1.0, "comply": 1.0, "n_words": 9}, {"id": "squad-2044", "gold": ["twice the figure for the average profession", "twice", "twice"], "pred_text": "twice the figure for the average profession.", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 7}, {"id": "squad-4062", "gold": ["black and white", "black and white,", "black and white", "remaining in black and white", "remaining in black and white,"], "pred_text": "the restored video, remaining in black and white, contains conservative digital enhancements and did not include sound quality improvements.", "f1": 0.4347826086956522, "contain": 1.0, "comply": 0.0, "n_words": 19}, {"id": "squad-5792", "gold": ["Mount Wilson", "Mount Wilson", "August 1"], "pred_text": "Mount Wilson", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-5915", "gold": ["ABC Cable News", "ABC Cable News", "ABC Cable News"], "pred_text": "ABC proposed a 24-hour news channel called ABC Cable News to compete with CNN.", "f1": 0.375, "contain": 1.0, "comply": 0.0, "n_words": 14}, {"id": "squad-5872", "gold": ["seven", "seven", "seven"], "pred_text": "ABC's seven owned-and-operated FM stations", "f1": 0.33333333333333337, "contain": 1.0, "comply": 1.0, "n_words": 5}, {"id": "squad-5056", "gold": ["complex", "complex", "complex", "quite complex"], "pred_text": "quite complex", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-1853", "gold": ["Presburger arithmetic", "Presburger", "Presburger arithmetic"], "pred_text": "Presburger arithmetic", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-1651", "gold": ["pity", "pity", "pity"], "pred_text": "pity", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-3694", "gold": ["important", "important", "important", "important"], "pred_text": "Among the most important classes of organic compounds that contain oxygen are (where \"R\" is an organic group): alcohols (R-OH); ethers (R-O-R); ketones (R-CO-R); aldehydes (", "f1": 0.08333333333333333, "contain": 1.0, "comply": 0.0, "n_words": 26}, {"id": "squad-6894", "gold": ["use the proceedings as a forum", "use the proceedings as a forum to inform the jury and the public of the political circumstances", "win an acquittal and avoid imprisonment", "use the proceedings as a forum to inform the jury and the public of the political circumstances", "use the proceedings as a forum to inform the jury and the public of the political circumstances surrounding the case"], "pred_text": "to use the proceedings as a forum to inform the jury and the public of the political circumstances surrounding the case and their reasons for breaking the law via civil disobedience", "f1": 0.7368421052631579, "contain": 1.0, "comply": 0.0, "n_words": 31}, {"id": "squad-7615", "gold": ["force", "force"], "pred_text": "all individuals are free from force.", "f1": 0.2857142857142857, "contain": 1.0, "comply": 1.0, "n_words": 6}, {"id": "squad-4007", "gold": ["July 1969", "July", "July 1969", "July 20, 1969", "July 1969"], "pred_text": "July 1969", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-2797", "gold": ["UCLA", "UCLA", "UCLA"], "pred_text": "UCLA Bruins", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-10431", "gold": ["applied", "applied force", "applied force", "applied force", "applied"], "pred_text": "the applied force", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-4358", "gold": ["sediment deposits", "sediment deposits", "sediment deposits"], "pred_text": "Analyses of sediment deposits from Amazon basin paleolakes and from the Amazon Fan indicate that rainfall in the basin during the LGM was lower than for the present, and this was almost certainly associated with reduced moist tropical vegetation cover in the basin", "f1": 0.1, "contain": 1.0, "comply": 0.0, "n_words": 43}, {"id": "squad-7654", "gold": ["The Daleks (a.k.a. The Mutants)", "Daleks", "eponymous aliens"], "pred_text": "The eponymous aliens that would become the series' most popular monsters, and were introduced in the second Doctor Who serial – The Daleks (a.k.a. The Mutants).", "f1": 0.25, "contain": 1.0, "comply": 0.0, "n_words": 26}, {"id": "squad-7940", "gold": ["Matt Smith", "Matt Smith", "Matt Smith"], "pred_text": "Matt Smith", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-10010", "gold": ["mid-18th century", "began in the mid-18th century within the Church of England.", "mid-18th century"], "pred_text": "The movement which would become The United Methodist Church began in the mid-18th century within the Church of England.", "f1": 0.6956521739130436, "contain": 1.0, "comply": 1.0, "n_words": 19}, {"id": "squad-8665", "gold": ["projected rises in sea levels", "rises in sea levels", "projected rises in sea levels"], "pred_text": "projected rises in sea levels", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 5}, {"id": "squad-1248", "gold": ["1875", "1875", "1875"], "pred_text": "1875", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-6428", "gold": ["the ease with which people, youth in particular, can obtain controlled substances", "ease with which people, youth in particular, can obtain controlled substances", "the ease with which people, youth in particular, can obtain controlled substances"], "pred_text": "the ease with which people, youth in particular, can obtain controlled substances (e.g., Vicodin, generically known as hydrocodone) via the Internet without a prescription issued by a doctor/practitioner who has an established doctor", "f1": 0.5641025641025641, "contain": 1.0, "comply": 0.0, "n_words": 33}, {"id": "squad-4753", "gold": ["very little", "very little", "very little"], "pred_text": "The neighborhood has very little retail activity, aside from the area near Fresno Street and State Route 99 Freeway (Kearney Palm Shopping Center, built in the late 1990s) and small corner markets scattered throughout", "f1": 0.12121212121212122, "contain": 1.0, "comply": 0.0, "n_words": 34}, {"id": "squad-6404", "gold": ["drug choice, dose, route, frequency, and duration of therapy", "drug choice, dose, route, frequency, and duration of therapy", "drug choice, dose, route, frequency, and duration of therapy"], "pred_text": "drug choice, dose, route, frequency, and duration of therapy", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 9}, {"id": "squad-8171", "gold": ["Confucian propriety and ancestor veneration", "Confucian propriety and ancestor veneration", "Confucian propriety and ancestor veneration"], "pred_text": "Khublai evoked his public image as a sage emperor by following the rituals of Confucian propriety and ancestor veneration, while simultaneously retaining his roots as a leader from the steppes.", "f1": 0.32258064516129037, "contain": 1.0, "comply": 0.0, "n_words": 30}, {"id": "squad-10484", "gold": ["Mercury", "Mercury", "Mercury", "Mercury"], "pred_text": "Mercury", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-5933", "gold": ["Turner Broadcasting System", "Turner Broadcasting System", "Turner Broadcasting System"], "pred_text": "Turner Broadcasting System", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-3978", "gold": ["asphyxiated", "asphyxiated", "asphyxiated", "asphyxiated"], "pred_text": "The astronauts were asphyxiated before the hatch could be opened.", "f1": 0.2222222222222222, "contain": 1.0, "comply": 1.0, "n_words": 10}, {"id": "squad-9996", "gold": ["During the 16th and 17th centuries", "17th century", "the 16th and 17th centuries", "16th and 17th centuries", "16th and 17th centuries"], "pred_text": "in particular at the height of its power under the reign of Suleiman the Magnificent", "f1": 0.0, "contain": 0.0, "comply": 1.0, "n_words": 15}, {"id": "squad-1128", "gold": ["William of Montreuil", "William of Montreuil", "William of Montreuil"], "pred_text": "William of Montreuil", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-2760", "gold": ["Orange", "Orange County", "Orange County"], "pred_text": "Orange County", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-2122", "gold": ["many", "three", "many"], "pred_text": "There are many forms of spiritual or religious teachers in Christianity, across all three major traditions - (Roman) Catholic, (Eastern) Orthodox Catholic, and Protestant/Non-Denominational, with a stronger tradition of spiritual formation in the more", "f1": 0.0625, "contain": 1.0, "comply": 0.0, "n_words": 34}, {"id": "squad-10415", "gold": ["denoted scalar quantities", "scalar quantities", "denoted scalar quantities", "scalar quantities", "scalar quantities"], "pred_text": "scalar quantities", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-1764", "gold": ["worst-case", "worst", "worst-case"], "pred_text": "worst-case", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-4614", "gold": ["zip\" the mouth shut when the animal is not feeding,", "\"zip\" the mouth shut when the animal is not feeding", "\"zip\" the mouth shut when the animal is not feeding"], "pred_text": "\"zip the mouth shut when the animal is not feeding, by forming intercellular connections with the opposite adhesive strip.\"", "f1": 0.6666666666666666, "contain": 1.0, "comply": 0.0, "n_words": 19}]}, "strict": {"wall_s": 2.5217165569999906, "gen_tokens": 1108, "mean_f1": 0.8839629566340093, "mean_contain": 0.905, "mean_comply": 0.95, "items": [{"id": "squad-1658", "gold": ["20 December 1914", "20 December 1914", "20 December 1914"], "pred_text": "20 December 1914", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-7068", "gold": ["Nepali", "Nepali", "Nepali"], "pred_text": "Nepali and/or the state's official language", "f1": 0.33333333333333337, "contain": 1.0, "comply": 1.0, "n_words": 6}, {"id": "squad-5920", "gold": ["Duel", "Duel", "Duel"], "pred_text": "Duel", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-10453", "gold": ["Pauli", "Pauli principle", "Pauli principle", "Pauli", "Pauli"], "pred_text": "Pauli principle", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-3640", "gold": ["algae", "algae", "algae", "algae", "algae"], "pred_text": "growth of algae", "f1": 0.5, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-10286", "gold": ["formalize a unified front in trade and negotiations with various Indians, since allegiance of the various tribes and nations was seen to be pivotal", "formalize a unified front in trade and negotiations with various Indians", "formalize a unified front in trade and negotiations with various Indians", "unified front in trade and negotiations with various Indians", "to formalize a unified front in trade and negotiations with various Indians"], "pred_text": "to formalize a unified front", "f1": 0.5333333333333333, "contain": 0.0, "comply": 1.0, "n_words": 5}, {"id": "squad-2981", "gold": ["−11.7 °C", "−11.7 °C", "−11.7 °C (10.9 °F)"], "pred_text": "−11.7 °C (10.9 °F)", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-4383", "gold": ["anthropological", "anthropological", "recent anthropological"], "pred_text": "Recent anthropological findings", "f1": 0.8, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-1041", "gold": ["Polish United Workers' Party", "Polish United Workers' Party", "Polish United Workers' Party"], "pred_text": "Polish United Workers' Party (PZPR)", "f1": 0.888888888888889, "contain": 1.0, "comply": 1.0, "n_words": 5}, {"id": "squad-7784", "gold": ["Time Lord", "Time Lord", "Time Lord"], "pred_text": "Time Lord", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-5840", "gold": ["Regis Philbin", "Regis Philbin", "Regis Philbin"], "pred_text": "Regis Philbin", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-1460", "gold": ["1900", "1900", "1900"], "pred_text": "In 1900", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-8247", "gold": ["26 seconds off the modern Gregorian calendar", "365.2425 days of the year", "365.2425 days of the year"], "pred_text": "365.2425 days", "f1": 0.6666666666666666, "contain": 0.0, "comply": 1.0, "n_words": 2}, {"id": "squad-2174", "gold": ["God", "God", "God"], "pred_text": "God", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-6618", "gold": ["cholecalciferol", "cholecalciferol", "cholecalciferol"], "pred_text": "cholecalciferol", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-9291", "gold": ["west", "west", "west"], "pred_text": "farther west", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-873", "gold": ["60", "60", "60"], "pred_text": "60 rooms", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-8566", "gold": ["Africa's most successful nation in the 2008 Olympics", "Africa's most successful nation", "Africa's most successful nation"], "pred_text": "six gold, four silver and four bronze", "f1": 0.0, "contain": 0.0, "comply": 1.0, "n_words": 7}, {"id": "squad-1263", "gold": ["1878", "December 1878", "to hide the fact that he dropped out of school"], "pred_text": "In December 1878", "f1": 0.8, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-2458", "gold": ["24", "24", "24"], "pred_text": "24", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-2952", "gold": ["1975", "1975", "1975"], "pred_text": "1975", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-960", "gold": ["325", "325", "325"], "pred_text": "160 mi", "f1": 0.0, "contain": 0.0, "comply": 1.0, "n_words": 2}, {"id": "squad-2644", "gold": ["3,792,621", "3,792,621", "3,792,621"], "pred_text": "3,792,621", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-2742", "gold": ["the housing bubble", "the housing bubble"], "pred_text": "the housing bubble", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-3247", "gold": ["Huguenots furnished two new regiments", "two new regiments", "furnished two new regiments of his army"], "pred_text": "furnished two new regiments", "f1": 0.888888888888889, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-1645", "gold": ["gravity", "gravity", "of gravity"], "pred_text": "dynamic theory of gravity", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-6218", "gold": ["opened the gates", "opened the gates to the Mongols", "opened the gates to the Mongols"], "pred_text": "opened the gates", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-9887", "gold": ["1919", "1919", "1919", "1919", "1919"], "pred_text": "1919", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-1065", "gold": ["Viking", "Norseman, Viking", "Norseman, Viking"], "pred_text": "Norseman, Viking", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-2750", "gold": ["business", "major business", "major business"], "pred_text": "Central business districts", "f1": 0.5, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-4739", "gold": ["\"Southwest Fresno\"", "Southwest Fresno", "Southwest Fresno"], "pred_text": "Southwest Fresno", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-2605", "gold": ["1 a.m", "1 a.m.", "1 a.m"], "pred_text": "At 1 a.m.", "f1": 0.8, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-3508", "gold": ["active", "more active and lived longer", "was more active and lived longer", "more active and lived longer while breathing it", "was more active and lived longer"], "pred_text": "more active and lived longer", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 5}, {"id": "squad-7014", "gold": ["$45,000", "more than $45,000", "$45,000"], "pred_text": "more than $45,000", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-859", "gold": ["Wianki", "Wianki", "Wianki"], "pred_text": "Wianki", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-322", "gold": ["17 seconds", "17 seconds", "17"], "pred_text": "17 seconds", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-9787", "gold": ["recognition", "international recognition", "recognition"], "pred_text": "international recognition", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-9214", "gold": ["Finsteraarhorn", "summit of Finsteraarhorn", "Finsteraarhorn, t"], "pred_text": "Finsteraarhorn", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-4472", "gold": ["to help strengthen their territorial claims", "map out their ancestral lands to help strengthen their territorial claims", "map out their ancestral lands"], "pred_text": "map out their ancestral lands", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 5}, {"id": "squad-8092", "gold": ["Alex Seropian", "Alex Seropian", "Alex Seropian"], "pred_text": "Alex Seropian", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-5683", "gold": ["1887", "1887", "1887"], "pred_text": "1887", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-888", "gold": ["a sword", "sword", "sword"], "pred_text": "a sword", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-581", "gold": ["late local programming", "late local programming", "late local programming"], "pred_text": "A special episode of The Late Late Show with James Corden", "f1": 0.16666666666666666, "contain": 0.0, "comply": 1.0, "n_words": 11}, {"id": "squad-4588", "gold": ["a balance sensor", "a balance sensor consisting of a statolith", "a balance sensor consisting of a statolith"], "pred_text": "balance sensor consisting of a statolith", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 6}, {"id": "squad-4791", "gold": ["June 1, 1953", "June 1, 1953", "June 1, 1953"], "pred_text": "June 1, 1953", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-5279", "gold": ["EAT!", "EAT!", "EAT! NewcastleGateshead"], "pred_text": "EAT! NewcastleGateshead", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-763", "gold": ["50-yard line.", "41", "50"], "pred_text": "41-yard line", "f1": 0.5, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-993", "gold": ["the most distinctive buildings", "the most distinctive buildings", "most distinctive buildings"], "pred_text": "the most distinctive buildings", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-6405", "gold": ["potential drug interactions, adverse drug reactions", "potential drug interactions", "potential drug interactions, adverse drug reactions, and assess patient drug allergies"], "pred_text": "drug interactions, adverse drug reactions", "f1": 0.9090909090909091, "contain": 0.0, "comply": 1.0, "n_words": 5}, {"id": "squad-677", "gold": ["Super Bowl XLVII", "Super Bowl XLVII", "XLVII"], "pred_text": "Super Bowl XLVII", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-1892", "gold": ["religious", "religious", "religious"], "pred_text": "religious text", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-86", "gold": ["$5 million", "$5 million", "$5 million"], "pred_text": "$5 million", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-3625", "gold": ["magnetic field", "magnetic", "magnetic", "magnetic field", "magnetic"], "pred_text": "magnetic field", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-8701", "gold": ["Cyanobacteria", "Cyanobacteria", "Cyanobacteria"], "pred_text": "Cyanobacteria", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-9631", "gold": ["its supporters", "Scholars and observers", "Islamism"], "pred_text": "Scholars and observers", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-7526", "gold": ["nine", "nine factors", "nine"], "pred_text": "nine factors", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-7627", "gold": ["a better relevant income.", "better relevant income", "a better relevant income"], "pred_text": "a better relevant income", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-8153", "gold": ["southern China", "southern China", "in southern China"], "pred_text": "southern China", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-7404", "gold": ["level of the top tax rate", "level of the top tax rate", "top tax rate"], "pred_text": "level of the top tax rate", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 6}, {"id": "squad-5418", "gold": ["Somerset House", "Somerset House", "Somerset House"], "pred_text": "Somerset House", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-7443", "gold": ["little", "little support", "little"], "pred_text": "little support", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-3832", "gold": ["Budget cuts", "oxygen tank explosion", "Budget cuts", "Budget cuts", "Budget cuts"], "pred_text": "Budget cuts", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-6537", "gold": ["tears", "tears", "tears"], "pred_text": "tears", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-1544", "gold": ["minimal", "minimal", "minimal"], "pred_text": "minimal loss", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-1986", "gold": ["motivated students", "motivated students", "motivated students"], "pred_text": "motivated students", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-8652", "gold": ["Ed Whitfield", "Ed Whitfield", "Ed Whitfield"], "pred_text": "Ed Whitfield", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-6625", "gold": ["apoptosis", "apoptosis", "rapid apoptosis"], "pred_text": "rapid apoptosis", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-8241", "gold": ["Marco Polo", "Marco Polo", "Marco Polo"], "pred_text": "Marco Polo", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-1577", "gold": ["nothing", "nothing", "nothing"], "pred_text": "Nothing", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-207", "gold": ["torn ACL", "a torn ACL", "torn ACL"], "pred_text": "torn ACL", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-4276", "gold": ["citizenship", "\"citizenship\"", "\"citizenship\""], "pred_text": "citizenship", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-8413", "gold": ["exercised by the government", "government", "government."], "pred_text": "executive power is exercised by the government.", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 7}, {"id": "squad-5657", "gold": ["more than 20", "20 works", "20"], "pred_text": "more than 20", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-7101", "gold": ["public", "public", "public"], "pred_text": "public schools", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-9616", "gold": ["reordering", "reordering of government and society in accordance with the Shari'a", "reordering"], "pred_text": "to implement Islamic values", "f1": 0.0, "contain": 0.0, "comply": 1.0, "n_words": 4}, {"id": "squad-2647", "gold": ["south", "south"], "pred_text": "To the south", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-5575", "gold": ["over 6000", "over 6000", "over 6000"], "pred_text": "over 6000 items", "f1": 0.8, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-7700", "gold": ["12", "12", "12"], "pred_text": "12 episodes", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-4144", "gold": ["every five years", "every five years", "every five years", "every five years"], "pred_text": "every five years", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-3093", "gold": ["Cévennes", "Cévennes", "Cévennes mountain region"], "pred_text": "Cévennes mountain region", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-7676", "gold": ["Battlestar Galactica and Bionic Woman", "Battlestar Galactica and Bionic Woman", "Battlestar Galactica and Bionic Woman"], "pred_text": "Battlestar Galactica  \nBionic Woman", "f1": 0.888888888888889, "contain": 0.0, "comply": 0.0, "n_words": 4}, {"id": "squad-2285", "gold": ["private conferences", "private conferences", "conferences"], "pred_text": "Over private conferences.", "f1": 0.8, "contain": 1.0, "comply": 0.0, "n_words": 3}, {"id": "squad-10043", "gold": ["1968", "1968", "1968"], "pred_text": "1968", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-6950", "gold": ["the property owner", "the property owner", "property owner"], "pred_text": "the property owner", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-9268", "gold": ["Ruhr", "Ruhr", "The Ruhr"], "pred_text": "Ruhr", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-3159", "gold": ["Cévennes", "Cévennes", "Cévennes region in the south"], "pred_text": "the rugged Cévennes region", "f1": 0.5714285714285715, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-9251", "gold": ["Middle Rhine", "Middle Rhine", "Middle Rhine"], "pred_text": "the Middle Rhine", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-4127", "gold": ["Article 17(3)", "Article 17(3)", "Article 17(3)", "Article 17(3)"], "pred_text": "Article 17(3)", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-1777", "gold": ["complicated definitions", "complicated definitions", "definitions"], "pred_text": "complicated definitions", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-5710", "gold": ["Citadel Broadcasting", "Citadel Broadcasting", "Citadel Broadcasting"], "pred_text": "Citadel Broadcasting", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-8394", "gold": ["The central highlands", "central highlands", "central highlands"], "pred_text": "The central highlands", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-9985", "gold": ["American delegation from the Paris Peace Conference", "American delegation from the Paris Peace Conference", "the American delegation from the Paris Peace Conference", "American delegation from the Paris Peace Conference", "American delegation from the Paris Peace Conference"], "pred_text": "American delegation", "f1": 0.5, "contain": 0.0, "comply": 1.0, "n_words": 2}, {"id": "squad-3248", "gold": ["Frederick William", "Frederick William", "Frederick William"], "pred_text": "Frederick William", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-6147", "gold": ["the Keraites", "Keraites", "Keraites"], "pred_text": "Keraites", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-2163", "gold": ["Catholic", "Catholic", "Catholic"], "pred_text": "Catholic", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-2452", "gold": ["prayer for grace", "grace", "prayer for grace"], "pred_text": "prayer for grace", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-1813", "gold": ["NP-complete", "NP-complete", "NP-complete"], "pred_text": "NP-complete problems", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-4756", "gold": ["2,500", "2,500", "2,500"], "pred_text": "seats up to 2,500", "f1": 0.4, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-7919", "gold": ["2011", "2011", "2011"], "pred_text": "2011", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-4605", "gold": ["four", "four", "four"], "pred_text": "four", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-2251", "gold": ["December 1517", "December 1517", "1517"], "pred_text": "in December 1517", "f1": 0.8, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-7588", "gold": ["reducing poverty", "reducing poverty", "reducing poverty"], "pred_text": "reducing poverty", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-9629", "gold": ["political", "political", "political"], "pred_text": "political role", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-106", "gold": ["Roger Goodell", "Roger Goodell", "Goodell"], "pred_text": "Roger Goodell", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-8714", "gold": ["green chloroplast", "green chloroplast lineage", "green chloroplast lineage"], "pred_text": "green chloroplast lineage", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-10257", "gold": ["capturing three traders and killing 14 people of the Miami nation, including Old Briton", "capturing three traders and killing 14 people of the Miami nation", "capturing three traders and killing 14 people of the Miami nation, including Old Briton", "capturing three traders and killing 14 people", "capturing three traders and killing 14 people of the Miami nation"], "pred_text": "killing 14 people", "f1": 0.6, "contain": 0.0, "comply": 1.0, "n_words": 3}, {"id": "squad-9746", "gold": ["destruction of Israel", "the destruction of Israel", "destruction of Israel"], "pred_text": "destruction of Israel", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-10239", "gold": ["village of Pickawillany", "village of Pickawillany", "village of Pickawillany", "Pickawillany", "village of Pickawillany"], "pred_text": "the village of Pickawillany", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-5811", "gold": ["1995–96 season", "1995–96", "2011–12"], "pred_text": "1995–96 season", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-3376", "gold": ["John Smeaton", "John Smeaton", "John Smeaton", "John Smeaton"], "pred_text": "John Smeaton", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-5337", "gold": ["smart ticketing", "smart ticketing", "smart ticketing."], "pred_text": "smart ticketing", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-1636", "gold": ["her weight", "her weight", "her weight."], "pred_text": "because of her weight", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-4513", "gold": ["colloblasts", "retractable tentacles fringed with tentilla", "colloblasts"], "pred_text": "fringed with tentilla", "f1": 0.7499999999999999, "contain": 0.0, "comply": 1.0, "n_words": 3}, {"id": "squad-1623", "gold": ["chastity", "his chastity", "chastity"], "pred_text": "his chastity", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-4548", "gold": ["cnidarians", "Ctenophores", "cnidarians"], "pred_text": "Cnidarians", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-676", "gold": ["British", "British", "British"], "pred_text": "British", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-8195", "gold": ["Emperor Chengzong", "Emperor Chengzong", "Emperor Chengzong"], "pred_text": "Emperor Chengzong", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-7730", "gold": ["William Hartnell's poor health", "to permit the recasting of the main character", "William Hartnell's poor health"], "pred_text": "original star William Hartnell's poor health", "f1": 0.8, "contain": 1.0, "comply": 1.0, "n_words": 6}, {"id": "squad-3807", "gold": ["Ford, Chrysler, and GM", "Ford, Chrysler, and GM", "Ford, Chrysler, and GM, respectively", "Ford", "Ford, Chrysler, and GM"], "pred_text": "Ford, Chrysler, and GM", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-8231", "gold": ["1368–1644", "1368–1644", "1368–1644"], "pred_text": "1368–1644", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-5049", "gold": ["xenoliths", "xenoliths", "inclusions", "xenoliths"], "pred_text": "xenoliths", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-1266", "gold": ["draftsman", "he worked as a draftsman", "draftsman"], "pred_text": "worked as a draftsman", "f1": 0.8571428571428571, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-5216", "gold": ["shopping", "shopping", "shopping destinations"], "pred_text": "shopping destinations", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-2271", "gold": ["Prince Frederick III", "Prince Frederick III", "Elector of Saxony"], "pred_text": "Prince Frederick III", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-3591", "gold": ["twice", "50% more", "about twice as much", "twice as much", "twice"], "pred_text": "about twice as much", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-6698", "gold": ["human papillomavirus", "human papillomavirus", "human papillomavirus"], "pred_text": "human papillomavirus", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-6676", "gold": ["phagocytes", "phagocytes", "phagocytes"], "pred_text": "phagocytes", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-2182", "gold": ["jailer and hangman", "jailer", "jailer and hangman of my poor soul."], "pred_text": "jailer and hangman", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-7707", "gold": ["2010", "2010", "2010"], "pred_text": "2010", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-8355", "gold": ["45 million people", "45 million people", "approximately 45 million"], "pred_text": "approximately 45 million people", "f1": 0.8571428571428571, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-3871", "gold": ["Apollo spacecraft", "Apollo", "Apollo", "Apollo"], "pred_text": "Apollo spacecraft", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-4627", "gold": ["internal fertilization and keep the eggs in brood chambers until they hatch.", "internal fertilization", "internal fertilization"], "pred_text": "internal fertilization", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-820", "gold": ["15 kilometres", "15 kilometres", "About 15 kilometres"], "pred_text": "15 kilometres", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-6572", "gold": ["MHC I (major histocompatibility complex)", "MHC I", "MHC I (major histocompatibility complex)"], "pred_text": "MHC I", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-2920", "gold": ["over 1,000", "1,000", "1,000"], "pred_text": "over 1,000 miners", "f1": 0.8, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-8789", "gold": ["direct repeats", "direct repeats", "direct repeats"], "pred_text": "direct repeats", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-852", "gold": ["Warsaw", "Warsaw", "Warsaw"], "pred_text": "Warsaw", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-952", "gold": ["Palace of Culture and Science", "Palace of Culture and Science", "Palace of Culture and Science"], "pred_text": "Palace of Culture and Science", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 5}, {"id": "squad-974", "gold": ["turbulent history of the city", "turbulent history of the city and country", "turbulent history of the city"], "pred_text": "the turbulent history of the city and country", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 8}, {"id": "squad-8656", "gold": ["2001", "2001", "2001"], "pred_text": "2001", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-6665", "gold": ["RNA interference pathway", "RNA interference", "RNA interference pathway"], "pred_text": "RNA interference pathway", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-9763", "gold": ["American attack on Iraq", "the American attack on Iraq", "American attack on Iraq"], "pred_text": "American attack on Iraq", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-9142", "gold": ["Gaulish name Rēnos", "Rēnos", "the Gaulish name Rēnos", "Rēnos", "Rēnos"], "pred_text": "Gaulish name Rēnos", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-10296", "gold": ["garrisons", "garrisons", "garrisons", "garrisons"], "pred_text": "garrisons", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-9437", "gold": ["1974", "1974", "1974"], "pred_text": "1974", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-1992", "gold": ["easier and more efficient", "easier and more efficient", "easier and more efficient"], "pred_text": "easier and more efficient", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-10333", "gold": ["Two of the expeditions were successful, with Fort Duquesne and Louisbourg", "Two", "Two", "Two", "Two"], "pred_text": "Two", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-4097", "gold": ["if the Treaty provisions have a direct effect and they are sufficiently clear, precise and unconditional.", "if the Treaty provisions have a direct effect and they are sufficiently clear, precise and unconditional", "if the Treaty provisions have a direct effect and they are sufficiently clear, precise and unconditional", "if the Treaty provisions have a direct effect and they are sufficiently clear, precise and unconditional"], "pred_text": "Treaty provisions have direct effect", "f1": 0.5263157894736842, "contain": 0.0, "comply": 1.0, "n_words": 5}, {"id": "squad-6111", "gold": ["his sons and grandsons", "sons and grandsons", "his sons and grandsons."], "pred_text": "his sons and grandsons", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-1415", "gold": ["1892 to 1894", "from 1892 to 1894", "from 1892 to 1894"], "pred_text": "1892 to 1894", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-3555", "gold": ["Aufbau", "Aufbau", "Aufbau", "Aufbau", "Aufbau"], "pred_text": "Aufbau", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-4932", "gold": ["1349", "1349", "1349"], "pred_text": "1349", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-10051", "gold": ["Justifying Grace or Accepting Grace", "Justifying Grace or Accepting Grace", "Justifying Grace"], "pred_text": "Justifying Grace", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-8320", "gold": ["allied groups from Central Asia and the western end of the empire", "various allied groups from Central Asia and the western end of the empire", "various allied groups"], "pred_text": "various allied groups", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-2965", "gold": ["Roman Catholic Church", "the Roman Catholic Church", "Roman Catholic"], "pred_text": "Roman Catholic Church", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-10468", "gold": ["20th", "the 20th century", "20th", "20th"], "pred_text": "20th century", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-4812", "gold": ["to provide a fault-tolerant, efficient routing method for telecommunication messages", "provide a fault-tolerant, efficient routing method for telecommunication messages"], "pred_text": "to provide a fault-tolerant, efficient routing method", "f1": 0.8, "contain": 0.0, "comply": 1.0, "n_words": 7}, {"id": "squad-476", "gold": ["a week", "a week", "week"], "pred_text": "a week-long event", "f1": 0.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-1470", "gold": ["stationary", "stationary", "stationary waves"], "pred_text": "stationary waves", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-9310", "gold": ["Millingen aan de Rijn,", "near Millingen aan de Rijn", "near Millingen aan de Rijn"], "pred_text": "near Millingen aan de Rijn", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 5}, {"id": "squad-10440", "gold": ["Galileo", "Aristotle's", "Aristotle", "Aristotle", "Aristotle's"], "pred_text": "Aristotle", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-6565", "gold": ["macrophages, neutrophils, and dendritic cells", "macrophages, neutrophils, and dendritic cells", "macrophages, neutrophils, and dendritic cells"], "pred_text": "macrophages, neutrophils, and dendritic cells", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 5}, {"id": "squad-8237", "gold": ["Nestorianism and Roman Catholicism", "Nestorianism and Roman Catholicism", "Nestorianism and Roman Catholicism"], "pred_text": "Nestorianism and Roman Catholicism", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-3722", "gold": ["January 18, 1974,", "January 18, 1974", "January 18, 1974", "January 18, 1974", "January 18, 1974"], "pred_text": "January 18, 1974", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-4463", "gold": ["climate change in addition to deforestation", "climate change in addition to deforestation", "climate change in addition to deforestation"], "pred_text": "climate change, deforestation", "f1": 0.6666666666666666, "contain": 0.0, "comply": 0.0, "n_words": 3}, {"id": "squad-1276", "gold": ["stroke", "a stroke", "a stroke"], "pred_text": "some sources say stroke", "f1": 0.4, "contain": 1.0, "comply": 0.0, "n_words": 4}, {"id": "squad-2044", "gold": ["twice the figure for the average profession", "twice", "twice"], "pred_text": "twice the figure", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-4062", "gold": ["black and white", "black and white,", "black and white", "remaining in black and white", "remaining in black and white,"], "pred_text": "remained black and white", "f1": 0.8571428571428571, "contain": 1.0, "comply": 0.0, "n_words": 4}, {"id": "squad-5792", "gold": ["Mount Wilson", "Mount Wilson", "August 1"], "pred_text": "Mount Wilson", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-5915", "gold": ["ABC Cable News", "ABC Cable News", "ABC Cable News"], "pred_text": "ABC Cable News", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-5872", "gold": ["seven", "seven", "seven"], "pred_text": "seven", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-5056", "gold": ["complex", "complex", "complex", "quite complex"], "pred_text": "quite complex", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-1853", "gold": ["Presburger arithmetic", "Presburger", "Presburger arithmetic"], "pred_text": "Presburger arithmetic", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-1651", "gold": ["pity", "pity", "pity"], "pred_text": "pity", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-3694", "gold": ["important", "important", "important", "important"], "pred_text": "not specified", "f1": 0.0, "contain": 0.0, "comply": 0.0, "n_words": 2}, {"id": "squad-6894", "gold": ["use the proceedings as a forum", "use the proceedings as a forum to inform the jury and the public of the political circumstances", "win an acquittal and avoid imprisonment", "use the proceedings as a forum to inform the jury and the public of the political circumstances", "use the proceedings as a forum to inform the jury and the public of the political circumstances surrounding the case"], "pred_text": "to use the proceedings as a forum", "f1": 0.888888888888889, "contain": 1.0, "comply": 1.0, "n_words": 7}, {"id": "squad-7615", "gold": ["force", "force"], "pred_text": "force", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-4007", "gold": ["July 1969", "July", "July 1969", "July 20, 1969", "July 1969"], "pred_text": "July 1969", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-2797", "gold": ["UCLA", "UCLA", "UCLA"], "pred_text": "UCLA Bruins", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-10431", "gold": ["applied", "applied force", "applied force", "applied force", "applied"], "pred_text": "applied force", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-4358", "gold": ["sediment deposits", "sediment deposits", "sediment deposits"], "pred_text": "sediment deposits", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-7654", "gold": ["The Daleks (a.k.a. The Mutants)", "Daleks", "eponymous aliens"], "pred_text": "The Daleks", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-7940", "gold": ["Matt Smith", "Matt Smith", "Matt Smith"], "pred_text": "Matt Smith", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-10010", "gold": ["mid-18th century", "began in the mid-18th century within the Church of England.", "mid-18th century"], "pred_text": "in the mid-18th century", "f1": 0.8, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-8665", "gold": ["projected rises in sea levels", "rises in sea levels", "projected rises in sea levels"], "pred_text": "projected rises in sea levels", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 5}, {"id": "squad-1248", "gold": ["1875", "1875", "1875"], "pred_text": "1875", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-6428", "gold": ["the ease with which people, youth in particular, can obtain controlled substances", "ease with which people, youth in particular, can obtain controlled substances", "the ease with which people, youth in particular, can obtain controlled substances"], "pred_text": "ease with which people can obtain controlled substances", "f1": 0.8421052631578948, "contain": 0.0, "comply": 0.0, "n_words": 8}, {"id": "squad-4753", "gold": ["very little", "very little", "very little"], "pred_text": "very little", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-6404", "gold": ["drug choice, dose, route, frequency, and duration of therapy", "drug choice, dose, route, frequency, and duration of therapy", "drug choice, dose, route, frequency, and duration of therapy"], "pred_text": "drug choice, dose, route, frequency, duration", "f1": 0.8, "contain": 0.0, "comply": 0.0, "n_words": 6}, {"id": "squad-8171", "gold": ["Confucian propriety and ancestor veneration", "Confucian propriety and ancestor veneration", "Confucian propriety and ancestor veneration"], "pred_text": "Confucian propriety and ancestor veneration", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 5}, {"id": "squad-10484", "gold": ["Mercury", "Mercury", "Mercury", "Mercury"], "pred_text": "Mercury", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-5933", "gold": ["Turner Broadcasting System", "Turner Broadcasting System", "Turner Broadcasting System"], "pred_text": "Turner Broadcasting System", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-3978", "gold": ["asphyxiated", "asphyxiated", "asphyxiated", "asphyxiated"], "pred_text": "Asphyxiated by fire", "f1": 0.5, "contain": 1.0, "comply": 0.0, "n_words": 3}, {"id": "squad-9996", "gold": ["During the 16th and 17th centuries", "17th century", "the 16th and 17th centuries", "16th and 17th centuries", "16th and 17th centuries"], "pred_text": "at the height of its power under the reign of Suleiman the Magnificent", "f1": 0.0, "contain": 0.0, "comply": 1.0, "n_words": 13}, {"id": "squad-1128", "gold": ["William of Montreuil", "William of Montreuil", "William of Montreuil"], "pred_text": "William of Montreuil", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-2760", "gold": ["Orange", "Orange County", "Orange County"], "pred_text": "Orange County", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-2122", "gold": ["many", "three", "many"], "pred_text": "three", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-10415", "gold": ["denoted scalar quantities", "scalar quantities", "denoted scalar quantities", "scalar quantities", "scalar quantities"], "pred_text": "scalar quantities", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-1764", "gold": ["worst-case", "worst", "worst-case"], "pred_text": "worst-case", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-4614", "gold": ["zip\" the mouth shut when the animal is not feeding,", "\"zip\" the mouth shut when the animal is not feeding", "\"zip\" the mouth shut when the animal is not feeding"], "pred_text": "forms intercellular connections", "f1": 0.0, "contain": 0.0, "comply": 0.0, "n_words": 3}]}, "fewshot": {"wall_s": 2.906507873999999, "gen_tokens": 1491, "mean_f1": 0.8733949953486735, "mean_contain": 0.915, "mean_comply": 0.93, "items": [{"id": "squad-1658", "gold": ["20 December 1914", "20 December 1914", "20 December 1914"], "pred_text": "20 December 1914", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-7068", "gold": ["Nepali", "Nepali", "Nepali"], "pred_text": "Nepali and/or the state's official language", "f1": 0.33333333333333337, "contain": 1.0, "comply": 1.0, "n_words": 6}, {"id": "squad-5920", "gold": ["Duel", "Duel", "Duel"], "pred_text": "Duel", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-10453", "gold": ["Pauli", "Pauli principle", "Pauli principle", "Pauli", "Pauli"], "pred_text": "the Pauli principle", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-3640", "gold": ["algae", "algae", "algae", "algae", "algae"], "pred_text": "algae", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-10286", "gold": ["formalize a unified front in trade and negotiations with various Indians, since allegiance of the various tribes and nations was seen to be pivotal", "formalize a unified front in trade and negotiations with various Indians", "formalize a unified front in trade and negotiations with various Indians", "unified front in trade and negotiations with various Indians", "to formalize a unified front in trade and negotiations with various Indians"], "pred_text": "to formalize a unified front in trade and negotiations with various Indians", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 12}, {"id": "squad-2981", "gold": ["−11.7 °C", "−11.7 °C", "−11.7 °C (10.9 °F)"], "pred_text": "−11.7 °C (10.9 °F)", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-4383", "gold": ["anthropological", "anthropological", "recent anthropological"], "pred_text": "Recent anthropological findings", "f1": 0.8, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-1041", "gold": ["Polish United Workers' Party", "Polish United Workers' Party", "Polish United Workers' Party"], "pred_text": "the Polish United Workers' Party (PZPR)", "f1": 0.888888888888889, "contain": 1.0, "comply": 1.0, "n_words": 6}, {"id": "squad-7784", "gold": ["Time Lord", "Time Lord", "Time Lord"], "pred_text": "Time Lord", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-5840", "gold": ["Regis Philbin", "Regis Philbin", "Regis Philbin"], "pred_text": "Regis Philbin", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-1460", "gold": ["1900", "1900", "1900"], "pred_text": "1900", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-8247", "gold": ["26 seconds off the modern Gregorian calendar", "365.2425 days of the year", "365.2425 days of the year"], "pred_text": "365.2425 days", "f1": 0.6666666666666666, "contain": 0.0, "comply": 1.0, "n_words": 2}, {"id": "squad-2174", "gold": ["God", "God", "God"], "pred_text": "God", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-6618", "gold": ["cholecalciferol", "cholecalciferol", "cholecalciferol"], "pred_text": "cholecalciferol", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-9291", "gold": ["west", "west", "west"], "pred_text": "farther west", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-873", "gold": ["60", "60", "60"], "pred_text": "60", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-8566", "gold": ["Africa's most successful nation in the 2008 Olympics", "Africa's most successful nation", "Africa's most successful nation"], "pred_text": "six gold, four silver and four bronze", "f1": 0.0, "contain": 0.0, "comply": 1.0, "n_words": 7}, {"id": "squad-1263", "gold": ["1878", "December 1878", "to hide the fact that he dropped out of school"], "pred_text": "December 1878", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-2458", "gold": ["24", "24", "24"], "pred_text": "24", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-2952", "gold": ["1975", "1975", "1975"], "pred_text": "1975", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-960", "gold": ["325", "325", "325"], "pred_text": "160 mi", "f1": 0.0, "contain": 0.0, "comply": 1.0, "n_words": 2}, {"id": "squad-2644", "gold": ["3,792,621", "3,792,621", "3,792,621"], "pred_text": "3,792,621", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-2742", "gold": ["the housing bubble", "the housing bubble"], "pred_text": "the housing bubble", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-3247", "gold": ["Huguenots furnished two new regiments", "two new regiments", "furnished two new regiments of his army"], "pred_text": "the Altpreußische Infantry Regiments No. 13 (Regiment on foot Varenne) and 15 (Regiment on foot Wylich)", "f1": 0.1111111111111111, "contain": 0.0, "comply": 1.0, "n_words": 16}, {"id": "squad-1645", "gold": ["gravity", "gravity", "of gravity"], "pred_text": "\"dynamic theory of gravity\"", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-6218", "gold": ["opened the gates", "opened the gates to the Mongols", "opened the gates to the Mongols"], "pred_text": "opened the gates to the Mongols", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 6}, {"id": "squad-9887", "gold": ["1919", "1919", "1919", "1919", "1919"], "pred_text": "1919", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-1065", "gold": ["Viking", "Norseman, Viking", "Norseman, Viking"], "pred_text": "\"Norseman, Viking\"", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-2750", "gold": ["business", "major business", "major business"], "pred_text": "Central business districts (CBD)", "f1": 0.4, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-4739", "gold": ["\"Southwest Fresno\"", "Southwest Fresno", "Southwest Fresno"], "pred_text": "Southwest Fresno", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-2605", "gold": ["1 a.m", "1 a.m.", "1 a.m"], "pred_text": "1 a.m.", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-3508", "gold": ["active", "more active and lived longer", "was more active and lived longer", "more active and lived longer while breathing it", "was more active and lived longer"], "pred_text": "was more active and lived longer", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 6}, {"id": "squad-7014", "gold": ["$45,000", "more than $45,000", "$45,000"], "pred_text": "more than $45,000", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-859", "gold": ["Wianki", "Wianki", "Wianki"], "pred_text": "Wianki", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-322", "gold": ["17 seconds", "17 seconds", "17"], "pred_text": "17 seconds", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-9787", "gold": ["recognition", "international recognition", "recognition"], "pred_text": "international recognition", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-9214", "gold": ["Finsteraarhorn", "summit of Finsteraarhorn", "Finsteraarhorn, t"], "pred_text": "4,274 m (14,022 ft) summit of Finsteraarhorn", "f1": 0.6, "contain": 1.0, "comply": 1.0, "n_words": 7}, {"id": "squad-4472", "gold": ["to help strengthen their territorial claims", "map out their ancestral lands to help strengthen their territorial claims", "map out their ancestral lands"], "pred_text": "to map out their ancestral lands to help strengthen their territorial claims", "f1": 0.9565217391304348, "contain": 1.0, "comply": 0.0, "n_words": 12}, {"id": "squad-8092", "gold": ["Alex Seropian", "Alex Seropian", "Alex Seropian"], "pred_text": "Alex Seropian", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-5683", "gold": ["1887", "1887", "1887"], "pred_text": "1887", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-888", "gold": ["a sword", "sword", "sword"], "pred_text": "a sword", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-581", "gold": ["late local programming", "late local programming", "late local programming"], "pred_text": "a special episode of The Late Late Show with James Corden", "f1": 0.16666666666666666, "contain": 0.0, "comply": 1.0, "n_words": 11}, {"id": "squad-4588", "gold": ["a balance sensor", "a balance sensor consisting of a statolith", "a balance sensor consisting of a statolith"], "pred_text": "a balance sensor consisting of a statolith, a solid particle supported on four bundles of cilia, called \"balancers\", that sense its orientation.", "f1": 0.4166666666666667, "contain": 1.0, "comply": 0.0, "n_words": 22}, {"id": "squad-4791", "gold": ["June 1, 1953", "June 1, 1953", "June 1, 1953"], "pred_text": "June 1, 1953", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-5279", "gold": ["EAT!", "EAT!", "EAT! NewcastleGateshead"], "pred_text": "EAT! NewcastleGateshead", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-763", "gold": ["50-yard line.", "41", "50"], "pred_text": "41-yard line", "f1": 0.5, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-993", "gold": ["the most distinctive buildings", "the most distinctive buildings", "most distinctive buildings"], "pred_text": "the most distinctive buildings", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-6405", "gold": ["potential drug interactions, adverse drug reactions", "potential drug interactions", "potential drug interactions, adverse drug reactions, and assess patient drug allergies"], "pred_text": "potential drug interactions, adverse drug reactions, and assess patient drug allergies", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 11}, {"id": "squad-677", "gold": ["Super Bowl XLVII", "Super Bowl XLVII", "XLVII"], "pred_text": "Super Bowl XLVII", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-1892", "gold": ["religious", "religious", "religious"], "pred_text": "religious texts", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-86", "gold": ["$5 million", "$5 million", "$5 million"], "pred_text": "$5 million", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-3625", "gold": ["magnetic field", "magnetic", "magnetic", "magnetic field", "magnetic"], "pred_text": "magnetic field", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-8701", "gold": ["Cyanobacteria", "Cyanobacteria", "Cyanobacteria"], "pred_text": "Cyanobacteria", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-9631", "gold": ["its supporters", "Scholars and observers", "Islamism"], "pred_text": "Scholars and observers who do not believe that Islam is merely a political ideology", "f1": 0.375, "contain": 1.0, "comply": 1.0, "n_words": 14}, {"id": "squad-7526", "gold": ["nine", "nine factors", "nine"], "pred_text": "nine", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-7627", "gold": ["a better relevant income.", "better relevant income", "a better relevant income"], "pred_text": "a better relevant income", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-8153", "gold": ["southern China", "southern China", "in southern China"], "pred_text": "southern China", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-7404", "gold": ["level of the top tax rate", "level of the top tax rate", "top tax rate"], "pred_text": "the level of the top tax rate", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 7}, {"id": "squad-5418", "gold": ["Somerset House", "Somerset House", "Somerset House"], "pred_text": "Somerset House", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-7443", "gold": ["little", "little support", "little"], "pred_text": "little support", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-3832", "gold": ["Budget cuts", "oxygen tank explosion", "Budget cuts", "Budget cuts", "Budget cuts"], "pred_text": "Budget cuts", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-6537", "gold": ["tears", "tears", "tears"], "pred_text": "tears", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-1544", "gold": ["minimal", "minimal", "minimal"], "pred_text": "minimal loss", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-1986", "gold": ["motivated students", "motivated students", "motivated students"], "pred_text": "motivated students", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-8652", "gold": ["Ed Whitfield", "Ed Whitfield", "Ed Whitfield"], "pred_text": "Ed Whitfield", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-6625", "gold": ["apoptosis", "apoptosis", "rapid apoptosis"], "pred_text": "apoptosis", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-8241", "gold": ["Marco Polo", "Marco Polo", "Marco Polo"], "pred_text": "Marco Polo", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-1577", "gold": ["nothing", "nothing", "nothing"], "pred_text": "Nothing", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-207", "gold": ["torn ACL", "a torn ACL", "torn ACL"], "pred_text": "torn ACL", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-4276", "gold": ["citizenship", "\"citizenship\"", "\"citizenship\""], "pred_text": "\"citizenship\"", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-8413", "gold": ["exercised by the government", "government", "government."], "pred_text": "the government", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-5657", "gold": ["more than 20", "20 works", "20"], "pred_text": "more than 20", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-7101", "gold": ["public", "public", "public"], "pred_text": "public schools", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-9616", "gold": ["reordering", "reordering of government and society in accordance with the Shari'a", "reordering"], "pred_text": "to implement Islamic values in all spheres of life and reorder of government and society in accordance with the Shari'a", "f1": 0.5714285714285714, "contain": 0.0, "comply": 0.0, "n_words": 20}, {"id": "squad-2647", "gold": ["south", "south"], "pred_text": "south", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-5575", "gold": ["over 6000", "over 6000", "over 6000"], "pred_text": "6000", "f1": 0.6666666666666666, "contain": 0.0, "comply": 1.0, "n_words": 1}, {"id": "squad-7700", "gold": ["12", "12", "12"], "pred_text": "12", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-4144", "gold": ["every five years", "every five years", "every five years", "every five years"], "pred_text": "every five years", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-3093", "gold": ["Cévennes", "Cévennes", "Cévennes mountain region"], "pred_text": "Cévennes mountain region", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-7676", "gold": ["Battlestar Galactica and Bionic Woman", "Battlestar Galactica and Bionic Woman", "Battlestar Galactica and Bionic Woman"], "pred_text": "Battlestar Galactica\nBionic Woman", "f1": 0.888888888888889, "contain": 0.0, "comply": 0.0, "n_words": 4}, {"id": "squad-2285", "gold": ["private conferences", "private conferences", "conferences"], "pred_text": "Over the next five days, private conferences were held to determine Luther's fate.", "f1": 0.2857142857142857, "contain": 1.0, "comply": 1.0, "n_words": 13}, {"id": "squad-10043", "gold": ["1968", "1968", "1968"], "pred_text": "1968", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-6950", "gold": ["the property owner", "the property owner", "property owner"], "pred_text": "the property owner", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-9268", "gold": ["Ruhr", "Ruhr", "The Ruhr"], "pred_text": "Ruhr", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-3159", "gold": ["Cévennes", "Cévennes", "Cévennes region in the south"], "pred_text": "the rugged Cévennes region in the south", "f1": 0.888888888888889, "contain": 1.0, "comply": 1.0, "n_words": 7}, {"id": "squad-9251", "gold": ["Middle Rhine", "Middle Rhine", "Middle Rhine"], "pred_text": "The Middle Rhine", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-4127", "gold": ["Article 17(3)", "Article 17(3)", "Article 17(3)", "Article 17(3)"], "pred_text": "Article 17(3)", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-1777", "gold": ["complicated definitions", "complicated definitions", "definitions"], "pred_text": "complicated definitions", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-5710", "gold": ["Citadel Broadcasting", "Citadel Broadcasting", "Citadel Broadcasting"], "pred_text": "Citadel Broadcasting", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-8394", "gold": ["The central highlands", "central highlands", "central highlands"], "pred_text": "central highlands", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-9985", "gold": ["American delegation from the Paris Peace Conference", "American delegation from the Paris Peace Conference", "the American delegation from the Paris Peace Conference", "American delegation from the Paris Peace Conference", "American delegation from the Paris Peace Conference"], "pred_text": "American delegation from the Paris Peace Conference", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 7}, {"id": "squad-3248", "gold": ["Frederick William", "Frederick William", "Frederick William"], "pred_text": "Frederick William, Elector of Brandenburg", "f1": 0.5714285714285715, "contain": 1.0, "comply": 1.0, "n_words": 5}, {"id": "squad-6147", "gold": ["the Keraites", "Keraites", "Keraites"], "pred_text": "Keraites", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-2163", "gold": ["Catholic", "Catholic", "Catholic"], "pred_text": "Catholic", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-2452", "gold": ["prayer for grace", "grace", "prayer for grace"], "pred_text": "prayer for grace", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-1813", "gold": ["NP-complete", "NP-complete", "NP-complete"], "pred_text": "complete for NP", "f1": 0.0, "contain": 0.0, "comply": 0.0, "n_words": 3}, {"id": "squad-4756", "gold": ["2,500", "2,500", "2,500"], "pred_text": "2,500", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-7919", "gold": ["2011", "2011", "2011"], "pred_text": "2011", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-4605", "gold": ["four", "four", "four"], "pred_text": "four", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-2251", "gold": ["December 1517", "December 1517", "1517"], "pred_text": "December 1517", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-7588", "gold": ["reducing poverty", "reducing poverty", "reducing poverty"], "pred_text": "reducing poverty", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-9629", "gold": ["political", "political", "political"], "pred_text": "political role", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-106", "gold": ["Roger Goodell", "Roger Goodell", "Goodell"], "pred_text": "Roger Goodell", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-8714", "gold": ["green chloroplast", "green chloroplast lineage", "green chloroplast lineage"], "pred_text": "green chloroplast lineage", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-10257", "gold": ["capturing three traders and killing 14 people of the Miami nation, including Old Briton", "capturing three traders and killing 14 people of the Miami nation", "capturing three traders and killing 14 people of the Miami nation, including Old Briton", "capturing three traders and killing 14 people", "capturing three traders and killing 14 people of the Miami nation"], "pred_text": "killing 14 people of the Miami nation, including Old Briton. He was reportedly ritually cannibalized by some aboriginal members of the expedition.", "f1": 0.5454545454545455, "contain": 0.0, "comply": 0.0, "n_words": 22}, {"id": "squad-9746", "gold": ["destruction of Israel", "the destruction of Israel", "destruction of Israel"], "pred_text": "the destruction of Israel and the establishment of an Islamic state in Palestine", "f1": 0.4615384615384615, "contain": 1.0, "comply": 1.0, "n_words": 13}, {"id": "squad-10239", "gold": ["village of Pickawillany", "village of Pickawillany", "village of Pickawillany", "Pickawillany", "village of Pickawillany"], "pred_text": "Pickawillany", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-5811", "gold": ["1995–96 season", "1995–96", "2011–12"], "pred_text": "1995-96 season", "f1": 0.5, "contain": 0.0, "comply": 0.0, "n_words": 2}, {"id": "squad-3376", "gold": ["John Smeaton", "John Smeaton", "John Smeaton", "John Smeaton"], "pred_text": "John Smeaton", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-5337", "gold": ["smart ticketing", "smart ticketing", "smart ticketing."], "pred_text": "smart ticketing", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-1636", "gold": ["her weight", "her weight", "her weight."], "pred_text": "because of her weight", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-4513", "gold": ["colloblasts", "retractable tentacles fringed with tentilla", "colloblasts"], "pred_text": "a pair of retractable tentacles fringed with tentilla", "f1": 0.8333333333333333, "contain": 1.0, "comply": 1.0, "n_words": 8}, {"id": "squad-1623", "gold": ["chastity", "his chastity", "chastity"], "pred_text": "chastity", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-4548", "gold": ["cnidarians", "Ctenophores", "cnidarians"], "pred_text": "Cnidarians", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-676", "gold": ["British", "British", "British"], "pred_text": "British", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-8195", "gold": ["Emperor Chengzong", "Emperor Chengzong", "Emperor Chengzong"], "pred_text": "Emperor Chengzong", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-7730", "gold": ["William Hartnell's poor health", "to permit the recasting of the main character", "William Hartnell's poor health"], "pred_text": "original star William Hartnell's poor health", "f1": 0.8, "contain": 1.0, "comply": 1.0, "n_words": 6}, {"id": "squad-3807", "gold": ["Ford, Chrysler, and GM", "Ford, Chrysler, and GM", "Ford, Chrysler, and GM, respectively", "Ford", "Ford, Chrysler, and GM"], "pred_text": "Ford, Chrysler, and GM", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-8231", "gold": ["1368–1644", "1368–1644", "1368–1644"], "pred_text": "1368–1644", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-5049", "gold": ["xenoliths", "xenoliths", "inclusions", "xenoliths"], "pred_text": "xenoliths", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-1266", "gold": ["draftsman", "he worked as a draftsman", "draftsman"], "pred_text": "worked as a draftsman for 60 florins a month", "f1": 0.5454545454545454, "contain": 1.0, "comply": 1.0, "n_words": 9}, {"id": "squad-5216", "gold": ["shopping", "shopping", "shopping destinations"], "pred_text": "shopping destinations", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-2271", "gold": ["Prince Frederick III", "Prince Frederick III", "Elector of Saxony"], "pred_text": "Prince Frederick III, Elector of Saxony", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 6}, {"id": "squad-3591", "gold": ["twice", "50% more", "about twice as much", "twice as much", "twice"], "pred_text": "14.6 mg·L−1 is about twice as much as at 20 °C (7.6 mg·L−1)", "f1": 0.47058823529411764, "contain": 1.0, "comply": 0.0, "n_words": 13}, {"id": "squad-6698", "gold": ["human papillomavirus", "human papillomavirus", "human papillomavirus"], "pred_text": "human papillomavirus", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-6676", "gold": ["phagocytes", "phagocytes", "phagocytes"], "pred_text": "phagocytes", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-2182", "gold": ["jailer and hangman", "jailer", "jailer and hangman of my poor soul."], "pred_text": "the jailer and hangman of my poor soul", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 8}, {"id": "squad-7707", "gold": ["2010", "2010", "2010"], "pred_text": "2010", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-8355", "gold": ["45 million people", "45 million people", "approximately 45 million"], "pred_text": "45 million people", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-3871", "gold": ["Apollo spacecraft", "Apollo", "Apollo", "Apollo"], "pred_text": "Apollo spacecraft", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-4627", "gold": ["internal fertilization and keep the eggs in brood chambers until they hatch.", "internal fertilization", "internal fertilization"], "pred_text": "internal fertilization and keep the eggs in brood chambers until they hatch", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 12}, {"id": "squad-820", "gold": ["15 kilometres", "15 kilometres", "About 15 kilometres"], "pred_text": "15 kilometres", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-6572", "gold": ["MHC I (major histocompatibility complex)", "MHC I", "MHC I (major histocompatibility complex)"], "pred_text": "MHC I", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-2920", "gold": ["over 1,000", "1,000", "1,000"], "pred_text": "over 1,000 miners", "f1": 0.8, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-8789", "gold": ["direct repeats", "direct repeats", "direct repeats"], "pred_text": "some chloroplast DNAs have since lost or flipped the inverted repeats", "f1": 0.16666666666666669, "contain": 0.0, "comply": 1.0, "n_words": 11}, {"id": "squad-852", "gold": ["Warsaw", "Warsaw", "Warsaw"], "pred_text": "Warsaw", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-952", "gold": ["Palace of Culture and Science", "Palace of Culture and Science", "Palace of Culture and Science"], "pred_text": "Palace of Culture and Science", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 5}, {"id": "squad-974", "gold": ["turbulent history of the city", "turbulent history of the city and country", "turbulent history of the city"], "pred_text": "the turbulent history of the city and country", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 8}, {"id": "squad-8656", "gold": ["2001", "2001", "2001"], "pred_text": "2001", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-6665", "gold": ["RNA interference pathway", "RNA interference", "RNA interference pathway"], "pred_text": "RNA interference pathway", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-9763", "gold": ["American attack on Iraq", "the American attack on Iraq", "American attack on Iraq"], "pred_text": "the American attack on Iraq in the 1991 Gulf War", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 10}, {"id": "squad-9142", "gold": ["Gaulish name Rēnos", "Rēnos", "the Gaulish name Rēnos", "Rēnos", "Rēnos"], "pred_text": "Gaulish name Rēnos", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-10296", "gold": ["garrisons", "garrisons", "garrisons", "garrisons"], "pred_text": "garrisons", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-9437", "gold": ["1974", "1974", "1974"], "pred_text": "1974", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-1992", "gold": ["easier and more efficient", "easier and more efficient", "easier and more efficient"], "pred_text": "easier and more efficient", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-10333", "gold": ["Two of the expeditions were successful, with Fort Duquesne and Louisbourg", "Two", "Two", "Two", "Two"], "pred_text": "Two", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-4097", "gold": ["if the Treaty provisions have a direct effect and they are sufficiently clear, precise and unconditional.", "if the Treaty provisions have a direct effect and they are sufficiently clear, precise and unconditional", "if the Treaty provisions have a direct effect and they are sufficiently clear, precise and unconditional", "if the Treaty provisions have a direct effect and they are sufficiently clear, precise and unconditional"], "pred_text": "Treaty provisions have a direct effect and they are sufficiently clear, precise and unconditional.", "f1": 0.962962962962963, "contain": 0.0, "comply": 1.0, "n_words": 14}, {"id": "squad-6111", "gold": ["his sons and grandsons", "sons and grandsons", "his sons and grandsons."], "pred_text": "his sons and grandsons", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-1415", "gold": ["1892 to 1894", "from 1892 to 1894", "from 1892 to 1894"], "pred_text": "1892 to 1894", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-3555", "gold": ["Aufbau", "Aufbau", "Aufbau", "Aufbau", "Aufbau"], "pred_text": "Aufbau", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-4932", "gold": ["1349", "1349", "1349"], "pred_text": "1349", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-10051", "gold": ["Justifying Grace or Accepting Grace", "Justifying Grace or Accepting Grace", "Justifying Grace"], "pred_text": "Justifying Grace", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-8320", "gold": ["allied groups from Central Asia and the western end of the empire", "various allied groups from Central Asia and the western end of the empire", "various allied groups"], "pred_text": "various allied groups from Central Asia and the western end of the empire", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 13}, {"id": "squad-2965", "gold": ["Roman Catholic Church", "the Roman Catholic Church", "Roman Catholic"], "pred_text": "Roman Catholic Church", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-10468", "gold": ["20th", "the 20th century", "20th", "20th"], "pred_text": "20th century", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-4812", "gold": ["to provide a fault-tolerant, efficient routing method for telecommunication messages", "provide a fault-tolerant, efficient routing method for telecommunication messages"], "pred_text": "to provide a fault-tolerant, efficient routing method for telecommunication messages", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 10}, {"id": "squad-476", "gold": ["a week", "a week", "week"], "pred_text": "a week-long event", "f1": 0.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-1470", "gold": ["stationary", "stationary", "stationary waves"], "pred_text": "stationary waves", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-9310", "gold": ["Millingen aan de Rijn,", "near Millingen aan de Rijn", "near Millingen aan de Rijn"], "pred_text": "near Millingen aan de Rijn", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 5}, {"id": "squad-10440", "gold": ["Galileo", "Aristotle's", "Aristotle", "Aristotle", "Aristotle's"], "pred_text": "Aristotle", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-6565", "gold": ["macrophages, neutrophils, and dendritic cells", "macrophages, neutrophils, and dendritic cells", "macrophages, neutrophils, and dendritic cells"], "pred_text": "macrophages, neutrophils, and dendritic cells", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 5}, {"id": "squad-8237", "gold": ["Nestorianism and Roman Catholicism", "Nestorianism and Roman Catholicism", "Nestorianism and Roman Catholicism"], "pred_text": "Nestorianism and Roman Catholicism", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-3722", "gold": ["January 18, 1974,", "January 18, 1974", "January 18, 1974", "January 18, 1974", "January 18, 1974"], "pred_text": "January 18, 1974", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-4463", "gold": ["climate change in addition to deforestation", "climate change in addition to deforestation", "climate change in addition to deforestation"], "pred_text": "climate change, deforestation", "f1": 0.6666666666666666, "contain": 0.0, "comply": 0.0, "n_words": 3}, {"id": "squad-1276", "gold": ["stroke", "a stroke", "a stroke"], "pred_text": "some sources say that he died of a stroke", "f1": 0.2222222222222222, "contain": 1.0, "comply": 1.0, "n_words": 9}, {"id": "squad-2044", "gold": ["twice the figure for the average profession", "twice", "twice"], "pred_text": "twice", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-4062", "gold": ["black and white", "black and white,", "black and white", "remaining in black and white", "remaining in black and white,"], "pred_text": "black and white", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-5792", "gold": ["Mount Wilson", "Mount Wilson", "August 1"], "pred_text": "Mount Wilson", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-5915", "gold": ["ABC Cable News", "ABC Cable News", "ABC Cable News"], "pred_text": "ABC Cable News", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-5872", "gold": ["seven", "seven", "seven"], "pred_text": "seven", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-5056", "gold": ["complex", "complex", "complex", "quite complex"], "pred_text": "complex", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-1853", "gold": ["Presburger arithmetic", "Presburger", "Presburger arithmetic"], "pred_text": "Presburger arithmetic", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-1651", "gold": ["pity", "pity", "pity"], "pred_text": "pity", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-3694", "gold": ["important", "important", "important", "important"], "pred_text": "as organic solvents", "f1": 0.0, "contain": 0.0, "comply": 0.0, "n_words": 3}, {"id": "squad-6894", "gold": ["use the proceedings as a forum", "use the proceedings as a forum to inform the jury and the public of the political circumstances", "win an acquittal and avoid imprisonment", "use the proceedings as a forum to inform the jury and the public of the political circumstances", "use the proceedings as a forum to inform the jury and the public of the political circumstances surrounding the case"], "pred_text": "to use the proceedings as a forum to inform the jury and the public of the political circumstances surrounding the case and their reasons for breaking the law via civil disobedience", "f1": 0.7368421052631579, "contain": 1.0, "comply": 0.0, "n_words": 31}, {"id": "squad-7615", "gold": ["force", "force"], "pred_text": "force", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-4007", "gold": ["July 1969", "July", "July 1969", "July 20, 1969", "July 1969"], "pred_text": "July 1969", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-2797", "gold": ["UCLA", "UCLA", "UCLA"], "pred_text": "UCLA", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-10431", "gold": ["applied", "applied force", "applied force", "applied force", "applied"], "pred_text": "applied force", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-4358", "gold": ["sediment deposits", "sediment deposits", "sediment deposits"], "pred_text": "sediment deposits from Amazon basin paleolakes and from the Amazon Fan", "f1": 0.33333333333333337, "contain": 1.0, "comply": 1.0, "n_words": 11}, {"id": "squad-7654", "gold": ["The Daleks (a.k.a. The Mutants)", "Daleks", "eponymous aliens"], "pred_text": "The Daleks", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-7940", "gold": ["Matt Smith", "Matt Smith", "Matt Smith"], "pred_text": "Matt Smith", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-10010", "gold": ["mid-18th century", "began in the mid-18th century within the Church of England.", "mid-18th century"], "pred_text": "the mid-18th century", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-8665", "gold": ["projected rises in sea levels", "rises in sea levels", "projected rises in sea levels"], "pred_text": "risks and dangers related to sea level rises", "f1": 0.3333333333333333, "contain": 0.0, "comply": 0.0, "n_words": 8}, {"id": "squad-1248", "gold": ["1875", "1875", "1875"], "pred_text": "1875", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-6428", "gold": ["the ease with which people, youth in particular, can obtain controlled substances", "ease with which people, youth in particular, can obtain controlled substances", "the ease with which people, youth in particular, can obtain controlled substances"], "pred_text": "the ease with which people, youth in particular, can obtain controlled substances (e.g., Vicodin, generically known as hydrocodone) via the Internet without a prescription issued by a doctor/practitioner who has an established doctor", "f1": 0.5641025641025641, "contain": 1.0, "comply": 0.0, "n_words": 33}, {"id": "squad-4753", "gold": ["very little", "very little", "very little"], "pred_text": "very little retail activity", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-6404", "gold": ["drug choice, dose, route, frequency, and duration of therapy", "drug choice, dose, route, frequency, and duration of therapy", "drug choice, dose, route, frequency, and duration of therapy"], "pred_text": "drug choice, dose, route, frequency, and duration of therapy", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 9}, {"id": "squad-8171", "gold": ["Confucian propriety and ancestor veneration", "Confucian propriety and ancestor veneration", "Confucian propriety and ancestor veneration"], "pred_text": "the rituals of Confucian propriety and ancestor veneration", "f1": 0.8333333333333333, "contain": 1.0, "comply": 1.0, "n_words": 8}, {"id": "squad-10484", "gold": ["Mercury", "Mercury", "Mercury", "Mercury"], "pred_text": "Mercury", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-5933", "gold": ["Turner Broadcasting System", "Turner Broadcasting System", "Turner Broadcasting System"], "pred_text": "Turner Broadcasting System", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-3978", "gold": ["asphyxiated", "asphyxiated", "asphyxiated", "asphyxiated"], "pred_text": "The astronauts were asphyxiated before the hatch could be opened.", "f1": 0.2222222222222222, "contain": 1.0, "comply": 1.0, "n_words": 10}, {"id": "squad-9996", "gold": ["During the 16th and 17th centuries", "17th century", "the 16th and 17th centuries", "16th and 17th centuries", "16th and 17th centuries"], "pred_text": "in the 16th and 17th centuries, in particular at the height of its power under the reign of Suleiman the Magnificent", "f1": 0.38095238095238093, "contain": 1.0, "comply": 0.0, "n_words": 21}, {"id": "squad-1128", "gold": ["William of Montreuil", "William of Montreuil", "William of Montreuil"], "pred_text": "William of Montreuil", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-2760", "gold": ["Orange", "Orange County", "Orange County"], "pred_text": "Orange County", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-2122", "gold": ["many", "three", "many"], "pred_text": "three", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-10415", "gold": ["denoted scalar quantities", "scalar quantities", "denoted scalar quantities", "scalar quantities", "scalar quantities"], "pred_text": "scalar quantities", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-1764", "gold": ["worst-case", "worst", "worst-case"], "pred_text": "worst-case", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-4614", "gold": ["zip\" the mouth shut when the animal is not feeding,", "\"zip\" the mouth shut when the animal is not feeding", "\"zip\" the mouth shut when the animal is not feeding"], "pred_text": "\"zip\" the mouth shut", "f1": 0.5454545454545454, "contain": 0.0, "comply": 1.0, "n_words": 4}]}, "fewshot_cap": {"wall_s": 0.49665829099990333, "gen_tokens": 1246, "mean_f1": 0.8744657587013726, "mean_contain": 0.89, "mean_comply": 0.95, "items": [{"id": "squad-1658", "gold": ["20 December 1914", "20 December 1914", "20 December 1914"], "pred_text": "20 December 1914", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-7068", "gold": ["Nepali", "Nepali", "Nepali"], "pred_text": "Nepali and/or the state's official language", "f1": 0.33333333333333337, "contain": 1.0, "comply": 1.0, "n_words": 6}, {"id": "squad-5920", "gold": ["Duel", "Duel", "Duel"], "pred_text": "Duel", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-10453", "gold": ["Pauli", "Pauli principle", "Pauli principle", "Pauli", "Pauli"], "pred_text": "the Pauli principle", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-3640", "gold": ["algae", "algae", "algae", "algae", "algae"], "pred_text": "algae", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-10286", "gold": ["formalize a unified front in trade and negotiations with various Indians, since allegiance of the various tribes and nations was seen to be pivotal", "formalize a unified front in trade and negotiations with various Indians", "formalize a unified front in trade and negotiations with various Indians", "unified front in trade and negotiations with various Indians", "to formalize a unified front in trade and negotiations with various Indians"], "pred_text": "to formalize a unified front in trade and negotiations with various", "f1": 0.9523809523809523, "contain": 0.0, "comply": 1.0, "n_words": 11}, {"id": "squad-2981", "gold": ["−11.7 °C", "−11.7 °C", "−11.7 °C (10.9 °F)"], "pred_text": "−11.7 °C (10.9", "f1": 0.8571428571428571, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-4383", "gold": ["anthropological", "anthropological", "recent anthropological"], "pred_text": "Recent anthropological findings", "f1": 0.8, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-1041", "gold": ["Polish United Workers' Party", "Polish United Workers' Party", "Polish United Workers' Party"], "pred_text": "the Polish United Workers' Party (PZPR)", "f1": 0.888888888888889, "contain": 1.0, "comply": 1.0, "n_words": 6}, {"id": "squad-7784", "gold": ["Time Lord", "Time Lord", "Time Lord"], "pred_text": "Time Lord", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-5840", "gold": ["Regis Philbin", "Regis Philbin", "Regis Philbin"], "pred_text": "Regis Philbin", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-1460", "gold": ["1900", "1900", "1900"], "pred_text": "1900", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-8247", "gold": ["26 seconds off the modern Gregorian calendar", "365.2425 days of the year", "365.2425 days of the year"], "pred_text": "365.2425 days", "f1": 0.6666666666666666, "contain": 0.0, "comply": 1.0, "n_words": 2}, {"id": "squad-2174", "gold": ["God", "God", "God"], "pred_text": "God", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-6618", "gold": ["cholecalciferol", "cholecalciferol", "cholecalciferol"], "pred_text": "cholecalciferol", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-9291", "gold": ["west", "west", "west"], "pred_text": "farther west", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-873", "gold": ["60", "60", "60"], "pred_text": "60", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-8566", "gold": ["Africa's most successful nation in the 2008 Olympics", "Africa's most successful nation", "Africa's most successful nation"], "pred_text": "six gold, four silver and four bronze", "f1": 0.0, "contain": 0.0, "comply": 1.0, "n_words": 7}, {"id": "squad-1263", "gold": ["1878", "December 1878", "to hide the fact that he dropped out of school"], "pred_text": "December 1878", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-2458", "gold": ["24", "24", "24"], "pred_text": "24", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-2952", "gold": ["1975", "1975", "1975"], "pred_text": "1975", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-960", "gold": ["325", "325", "325"], "pred_text": "160 mi", "f1": 0.0, "contain": 0.0, "comply": 1.0, "n_words": 2}, {"id": "squad-2644", "gold": ["3,792,621", "3,792,621", "3,792,621"], "pred_text": "3,792,621", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-2742", "gold": ["the housing bubble", "the housing bubble"], "pred_text": "the housing bubble", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-3247", "gold": ["Huguenots furnished two new regiments", "two new regiments", "furnished two new regiments of his army"], "pred_text": "the Altpreußische Infantry Regiments No.", "f1": 0.28571428571428575, "contain": 0.0, "comply": 1.0, "n_words": 5}, {"id": "squad-1645", "gold": ["gravity", "gravity", "of gravity"], "pred_text": "\"dynamic theory of gravity\"", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-6218", "gold": ["opened the gates", "opened the gates to the Mongols", "opened the gates to the Mongols"], "pred_text": "opened the gates to the Mongols", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 6}, {"id": "squad-9887", "gold": ["1919", "1919", "1919", "1919", "1919"], "pred_text": "1919", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-1065", "gold": ["Viking", "Norseman, Viking", "Norseman, Viking"], "pred_text": "\"Norseman, Viking\"", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-2750", "gold": ["business", "major business", "major business"], "pred_text": "Central business districts (CBD)", "f1": 0.4, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-4739", "gold": ["\"Southwest Fresno\"", "Southwest Fresno", "Southwest Fresno"], "pred_text": "Southwest Fresno", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-2605", "gold": ["1 a.m", "1 a.m.", "1 a.m"], "pred_text": "1 a.m.", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-3508", "gold": ["active", "more active and lived longer", "was more active and lived longer", "more active and lived longer while breathing it", "was more active and lived longer"], "pred_text": "was more active and lived longer", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 6}, {"id": "squad-7014", "gold": ["$45,000", "more than $45,000", "$45,000"], "pred_text": "more than $45,000", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-859", "gold": ["Wianki", "Wianki", "Wianki"], "pred_text": "Wianki", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-322", "gold": ["17 seconds", "17 seconds", "17"], "pred_text": "17 seconds", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-9787", "gold": ["recognition", "international recognition", "recognition"], "pred_text": "international recognition", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-9214", "gold": ["Finsteraarhorn", "summit of Finsteraarhorn", "Finsteraarhorn, t"], "pred_text": "4,274 m (14,02", "f1": 0.0, "contain": 0.0, "comply": 1.0, "n_words": 3}, {"id": "squad-4472", "gold": ["to help strengthen their territorial claims", "map out their ancestral lands to help strengthen their territorial claims", "map out their ancestral lands"], "pred_text": "to map out their ancestral lands to help strengthen their territorial claims", "f1": 0.9565217391304348, "contain": 1.0, "comply": 0.0, "n_words": 12}, {"id": "squad-8092", "gold": ["Alex Seropian", "Alex Seropian", "Alex Seropian"], "pred_text": "Alex Seropian", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-5683", "gold": ["1887", "1887", "1887"], "pred_text": "1887", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-888", "gold": ["a sword", "sword", "sword"], "pred_text": "a sword", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-581", "gold": ["late local programming", "late local programming", "late local programming"], "pred_text": "a special episode of The Late Late Show with James Corden", "f1": 0.16666666666666666, "contain": 0.0, "comply": 1.0, "n_words": 11}, {"id": "squad-4588", "gold": ["a balance sensor", "a balance sensor consisting of a statolith", "a balance sensor consisting of a statolith"], "pred_text": "a balance sensor consisting of a statolith, a solid particle", "f1": 0.8333333333333333, "contain": 1.0, "comply": 1.0, "n_words": 10}, {"id": "squad-4791", "gold": ["June 1, 1953", "June 1, 1953", "June 1, 1953"], "pred_text": "June 1, 1953", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-5279", "gold": ["EAT!", "EAT!", "EAT! NewcastleGateshead"], "pred_text": "EAT! NewcastleGateshead", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-763", "gold": ["50-yard line.", "41", "50"], "pred_text": "41-yard line", "f1": 0.5, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-993", "gold": ["the most distinctive buildings", "the most distinctive buildings", "most distinctive buildings"], "pred_text": "the most distinctive buildings", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-6405", "gold": ["potential drug interactions, adverse drug reactions", "potential drug interactions", "potential drug interactions, adverse drug reactions, and assess patient drug allergies"], "pred_text": "potential drug interactions, adverse drug reactions, and assess patient drug", "f1": 0.9523809523809523, "contain": 1.0, "comply": 1.0, "n_words": 10}, {"id": "squad-677", "gold": ["Super Bowl XLVII", "Super Bowl XLVII", "XLVII"], "pred_text": "Super Bowl XLVII", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-1892", "gold": ["religious", "religious", "religious"], "pred_text": "religious texts", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-86", "gold": ["$5 million", "$5 million", "$5 million"], "pred_text": "$5 million", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-3625", "gold": ["magnetic field", "magnetic", "magnetic", "magnetic field", "magnetic"], "pred_text": "magnetic field", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-8701", "gold": ["Cyanobacteria", "Cyanobacteria", "Cyanobacteria"], "pred_text": "Cyanobacteria", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-9631", "gold": ["its supporters", "Scholars and observers", "Islamism"], "pred_text": "Scholars and observers who do not believe that Islam is merely", "f1": 0.42857142857142855, "contain": 1.0, "comply": 1.0, "n_words": 11}, {"id": "squad-7526", "gold": ["nine", "nine factors", "nine"], "pred_text": "nine", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-7627", "gold": ["a better relevant income.", "better relevant income", "a better relevant income"], "pred_text": "a better relevant income", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-8153", "gold": ["southern China", "southern China", "in southern China"], "pred_text": "southern China", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-7404", "gold": ["level of the top tax rate", "level of the top tax rate", "top tax rate"], "pred_text": "the level of the top tax rate", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 7}, {"id": "squad-5418", "gold": ["Somerset House", "Somerset House", "Somerset House"], "pred_text": "Somerset House", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-7443", "gold": ["little", "little support", "little"], "pred_text": "little support", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-3832", "gold": ["Budget cuts", "oxygen tank explosion", "Budget cuts", "Budget cuts", "Budget cuts"], "pred_text": "Budget cuts", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-6537", "gold": ["tears", "tears", "tears"], "pred_text": "tears", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-1544", "gold": ["minimal", "minimal", "minimal"], "pred_text": "minimal loss", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-1986", "gold": ["motivated students", "motivated students", "motivated students"], "pred_text": "motivated students", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-8652", "gold": ["Ed Whitfield", "Ed Whitfield", "Ed Whitfield"], "pred_text": "Ed Whitfield", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-6625", "gold": ["apoptosis", "apoptosis", "rapid apoptosis"], "pred_text": "apoptosis", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-8241", "gold": ["Marco Polo", "Marco Polo", "Marco Polo"], "pred_text": "Marco Polo", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-1577", "gold": ["nothing", "nothing", "nothing"], "pred_text": "Nothing", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-207", "gold": ["torn ACL", "a torn ACL", "torn ACL"], "pred_text": "torn ACL", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-4276", "gold": ["citizenship", "\"citizenship\"", "\"citizenship\""], "pred_text": "\"citizenship\"", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-8413", "gold": ["exercised by the government", "government", "government."], "pred_text": "the government", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-5657", "gold": ["more than 20", "20 works", "20"], "pred_text": "more than 20", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-7101", "gold": ["public", "public", "public"], "pred_text": "public schools", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-9616", "gold": ["reordering", "reordering of government and society in accordance with the Shari'a", "reordering"], "pred_text": "to implement Islamic values in all spheres of life and reorder of", "f1": 0.28571428571428575, "contain": 0.0, "comply": 0.0, "n_words": 12}, {"id": "squad-2647", "gold": ["south", "south"], "pred_text": "south", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-5575", "gold": ["over 6000", "over 6000", "over 6000"], "pred_text": "6000", "f1": 0.6666666666666666, "contain": 0.0, "comply": 1.0, "n_words": 1}, {"id": "squad-7700", "gold": ["12", "12", "12"], "pred_text": "12", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-4144", "gold": ["every five years", "every five years", "every five years", "every five years"], "pred_text": "every five years", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-3093", "gold": ["Cévennes", "Cévennes", "Cévennes mountain region"], "pred_text": "Cévennes mountain region", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-7676", "gold": ["Battlestar Galactica and Bionic Woman", "Battlestar Galactica and Bionic Woman", "Battlestar Galactica and Bionic Woman"], "pred_text": "Battlestar Galactica\nBionic Woman", "f1": 0.888888888888889, "contain": 0.0, "comply": 0.0, "n_words": 4}, {"id": "squad-2285", "gold": ["private conferences", "private conferences", "conferences"], "pred_text": "Over the next five days, private conferences were held to determine", "f1": 0.33333333333333337, "contain": 1.0, "comply": 1.0, "n_words": 11}, {"id": "squad-10043", "gold": ["1968", "1968", "1968"], "pred_text": "1968", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-6950", "gold": ["the property owner", "the property owner", "property owner"], "pred_text": "the property owner", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-9268", "gold": ["Ruhr", "Ruhr", "The Ruhr"], "pred_text": "Ruhr", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-3159", "gold": ["Cévennes", "Cévennes", "Cévennes region in the south"], "pred_text": "the rugged Cévennes region in the south", "f1": 0.888888888888889, "contain": 1.0, "comply": 1.0, "n_words": 7}, {"id": "squad-9251", "gold": ["Middle Rhine", "Middle Rhine", "Middle Rhine"], "pred_text": "The Middle Rhine", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-4127", "gold": ["Article 17(3)", "Article 17(3)", "Article 17(3)", "Article 17(3)"], "pred_text": "Article 17(3)", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-1777", "gold": ["complicated definitions", "complicated definitions", "definitions"], "pred_text": "complicated definitions", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-5710", "gold": ["Citadel Broadcasting", "Citadel Broadcasting", "Citadel Broadcasting"], "pred_text": "Citadel Broadcasting", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-8394", "gold": ["The central highlands", "central highlands", "central highlands"], "pred_text": "central highlands", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-9985", "gold": ["American delegation from the Paris Peace Conference", "American delegation from the Paris Peace Conference", "the American delegation from the Paris Peace Conference", "American delegation from the Paris Peace Conference", "American delegation from the Paris Peace Conference"], "pred_text": "American delegation from the Paris Peace Conference", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 7}, {"id": "squad-3248", "gold": ["Frederick William", "Frederick William", "Frederick William"], "pred_text": "Frederick William, Elector of Brandenburg", "f1": 0.5714285714285715, "contain": 1.0, "comply": 1.0, "n_words": 5}, {"id": "squad-6147", "gold": ["the Keraites", "Keraites", "Keraites"], "pred_text": "Keraites", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-2163", "gold": ["Catholic", "Catholic", "Catholic"], "pred_text": "Catholic", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-2452", "gold": ["prayer for grace", "grace", "prayer for grace"], "pred_text": "prayer for grace", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-1813", "gold": ["NP-complete", "NP-complete", "NP-complete"], "pred_text": "complete for NP", "f1": 0.0, "contain": 0.0, "comply": 0.0, "n_words": 3}, {"id": "squad-4756", "gold": ["2,500", "2,500", "2,500"], "pred_text": "2,500", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-7919", "gold": ["2011", "2011", "2011"], "pred_text": "2011", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-4605", "gold": ["four", "four", "four"], "pred_text": "four", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-2251", "gold": ["December 1517", "December 1517", "1517"], "pred_text": "December 1517", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-7588", "gold": ["reducing poverty", "reducing poverty", "reducing poverty"], "pred_text": "reducing poverty", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-9629", "gold": ["political", "political", "political"], "pred_text": "political role", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-106", "gold": ["Roger Goodell", "Roger Goodell", "Goodell"], "pred_text": "Roger Goodell", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-8714", "gold": ["green chloroplast", "green chloroplast lineage", "green chloroplast lineage"], "pred_text": "green chloroplast lineage", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-10257", "gold": ["capturing three traders and killing 14 people of the Miami nation, including Old Briton", "capturing three traders and killing 14 people of the Miami nation", "capturing three traders and killing 14 people of the Miami nation, including Old Briton", "capturing three traders and killing 14 people", "capturing three traders and killing 14 people of the Miami nation"], "pred_text": "killing 14 people of the Miami nation, including", "f1": 0.7058823529411764, "contain": 0.0, "comply": 1.0, "n_words": 8}, {"id": "squad-9746", "gold": ["destruction of Israel", "the destruction of Israel", "destruction of Israel"], "pred_text": "the destruction of Israel and the establishment of an Islamic state in", "f1": 0.5, "contain": 1.0, "comply": 1.0, "n_words": 12}, {"id": "squad-10239", "gold": ["village of Pickawillany", "village of Pickawillany", "village of Pickawillany", "Pickawillany", "village of Pickawillany"], "pred_text": "Pickawillany", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-5811", "gold": ["1995–96 season", "1995–96", "2011–12"], "pred_text": "1995-96 season", "f1": 0.5, "contain": 0.0, "comply": 0.0, "n_words": 2}, {"id": "squad-3376", "gold": ["John Smeaton", "John Smeaton", "John Smeaton", "John Smeaton"], "pred_text": "John Smeaton", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-5337", "gold": ["smart ticketing", "smart ticketing", "smart ticketing."], "pred_text": "smart ticketing", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-1636", "gold": ["her weight", "her weight", "her weight."], "pred_text": "because of her weight", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-4513", "gold": ["colloblasts", "retractable tentacles fringed with tentilla", "colloblasts"], "pred_text": "a pair of retractable tentacles fringed with tentilla", "f1": 0.8333333333333333, "contain": 1.0, "comply": 1.0, "n_words": 8}, {"id": "squad-1623", "gold": ["chastity", "his chastity", "chastity"], "pred_text": "chastity", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-4548", "gold": ["cnidarians", "Ctenophores", "cnidarians"], "pred_text": "Cnidarians", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-676", "gold": ["British", "British", "British"], "pred_text": "British", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-8195", "gold": ["Emperor Chengzong", "Emperor Chengzong", "Emperor Chengzong"], "pred_text": "Emperor Chengzong", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-7730", "gold": ["William Hartnell's poor health", "to permit the recasting of the main character", "William Hartnell's poor health"], "pred_text": "original star William Hartnell's poor health", "f1": 0.8, "contain": 1.0, "comply": 1.0, "n_words": 6}, {"id": "squad-3807", "gold": ["Ford, Chrysler, and GM", "Ford, Chrysler, and GM", "Ford, Chrysler, and GM, respectively", "Ford", "Ford, Chrysler, and GM"], "pred_text": "Ford, Chrysler, and GM", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-8231", "gold": ["1368–1644", "1368–1644", "1368–1644"], "pred_text": "1368–1644", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-5049", "gold": ["xenoliths", "xenoliths", "inclusions", "xenoliths"], "pred_text": "xenoliths", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-1266", "gold": ["draftsman", "he worked as a draftsman", "draftsman"], "pred_text": "worked as a draftsman for 60 florins a", "f1": 0.6, "contain": 1.0, "comply": 1.0, "n_words": 8}, {"id": "squad-5216", "gold": ["shopping", "shopping", "shopping destinations"], "pred_text": "shopping destinations", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-2271", "gold": ["Prince Frederick III", "Prince Frederick III", "Elector of Saxony"], "pred_text": "Prince Frederick III, Elector of Saxony", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 6}, {"id": "squad-3591", "gold": ["twice", "50% more", "about twice as much", "twice as much", "twice"], "pred_text": "14.6 mg·L−1 is about twice", "f1": 0.4444444444444445, "contain": 1.0, "comply": 0.0, "n_words": 5}, {"id": "squad-6698", "gold": ["human papillomavirus", "human papillomavirus", "human papillomavirus"], "pred_text": "human papillomavirus", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-6676", "gold": ["phagocytes", "phagocytes", "phagocytes"], "pred_text": "phagocytes", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-2182", "gold": ["jailer and hangman", "jailer", "jailer and hangman of my poor soul."], "pred_text": "jailer and hangman", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-7707", "gold": ["2010", "2010", "2010"], "pred_text": "2010", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-8355", "gold": ["45 million people", "45 million people", "approximately 45 million"], "pred_text": "45 million people", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-3871", "gold": ["Apollo spacecraft", "Apollo", "Apollo", "Apollo"], "pred_text": "Apollo spacecraft", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-4627", "gold": ["internal fertilization and keep the eggs in brood chambers until they hatch.", "internal fertilization", "internal fertilization"], "pred_text": "internal fertilization and keep the eggs in brood chambers until", "f1": 0.9, "contain": 1.0, "comply": 1.0, "n_words": 10}, {"id": "squad-820", "gold": ["15 kilometres", "15 kilometres", "About 15 kilometres"], "pred_text": "15 kilometres", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-6572", "gold": ["MHC I (major histocompatibility complex)", "MHC I", "MHC I (major histocompatibility complex)"], "pred_text": "MHC I", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-2920", "gold": ["over 1,000", "1,000", "1,000"], "pred_text": "over 1,000 miners", "f1": 0.8, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-8789", "gold": ["direct repeats", "direct repeats", "direct repeats"], "pred_text": "some chloroplast DNAs have since lost or flipped the inverted", "f1": 0.0, "contain": 0.0, "comply": 1.0, "n_words": 10}, {"id": "squad-852", "gold": ["Warsaw", "Warsaw", "Warsaw"], "pred_text": "Warsaw", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-952", "gold": ["Palace of Culture and Science", "Palace of Culture and Science", "Palace of Culture and Science"], "pred_text": "Palace of Culture and Science", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 5}, {"id": "squad-974", "gold": ["turbulent history of the city", "turbulent history of the city and country", "turbulent history of the city"], "pred_text": "the turbulent history of the city and country", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 8}, {"id": "squad-8656", "gold": ["2001", "2001", "2001"], "pred_text": "2001", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-6665", "gold": ["RNA interference pathway", "RNA interference", "RNA interference pathway"], "pred_text": "RNA interference pathway", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-9763", "gold": ["American attack on Iraq", "the American attack on Iraq", "American attack on Iraq"], "pred_text": "the American attack on Iraq in the 1991", "f1": 0.8, "contain": 1.0, "comply": 1.0, "n_words": 8}, {"id": "squad-9142", "gold": ["Gaulish name Rēnos", "Rēnos", "the Gaulish name Rēnos", "Rēnos", "Rēnos"], "pred_text": "Gaulish name Rēnos", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-10296", "gold": ["garrisons", "garrisons", "garrisons", "garrisons"], "pred_text": "garrisons", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-9437", "gold": ["1974", "1974", "1974"], "pred_text": "1974", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-1992", "gold": ["easier and more efficient", "easier and more efficient", "easier and more efficient"], "pred_text": "easier and more efficient", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-10333", "gold": ["Two of the expeditions were successful, with Fort Duquesne and Louisbourg", "Two", "Two", "Two", "Two"], "pred_text": "Two", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-4097", "gold": ["if the Treaty provisions have a direct effect and they are sufficiently clear, precise and unconditional.", "if the Treaty provisions have a direct effect and they are sufficiently clear, precise and unconditional", "if the Treaty provisions have a direct effect and they are sufficiently clear, precise and unconditional", "if the Treaty provisions have a direct effect and they are sufficiently clear, precise and unconditional"], "pred_text": "Treaty provisions have a direct effect and they are sufficiently", "f1": 0.782608695652174, "contain": 0.0, "comply": 1.0, "n_words": 10}, {"id": "squad-6111", "gold": ["his sons and grandsons", "sons and grandsons", "his sons and grandsons."], "pred_text": "his sons and grandsons", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-1415", "gold": ["1892 to 1894", "from 1892 to 1894", "from 1892 to 1894"], "pred_text": "1892 to 1894", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-3555", "gold": ["Aufbau", "Aufbau", "Aufbau", "Aufbau", "Aufbau"], "pred_text": "Aufbau", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-4932", "gold": ["1349", "1349", "1349"], "pred_text": "1349", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-10051", "gold": ["Justifying Grace or Accepting Grace", "Justifying Grace or Accepting Grace", "Justifying Grace"], "pred_text": "Justifying Grace", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-8320", "gold": ["allied groups from Central Asia and the western end of the empire", "various allied groups from Central Asia and the western end of the empire", "various allied groups"], "pred_text": "various allied groups from Central Asia and the western end of", "f1": 0.9523809523809523, "contain": 1.0, "comply": 1.0, "n_words": 11}, {"id": "squad-2965", "gold": ["Roman Catholic Church", "the Roman Catholic Church", "Roman Catholic"], "pred_text": "Roman Catholic Church", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-10468", "gold": ["20th", "the 20th century", "20th", "20th"], "pred_text": "20th century", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-4812", "gold": ["to provide a fault-tolerant, efficient routing method for telecommunication messages", "provide a fault-tolerant, efficient routing method for telecommunication messages"], "pred_text": "to provide a fault-tolerant, efficient routing method for", "f1": 0.8750000000000001, "contain": 0.0, "comply": 1.0, "n_words": 8}, {"id": "squad-476", "gold": ["a week", "a week", "week"], "pred_text": "a week-long event", "f1": 0.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-1470", "gold": ["stationary", "stationary", "stationary waves"], "pred_text": "stationary waves", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-9310", "gold": ["Millingen aan de Rijn,", "near Millingen aan de Rijn", "near Millingen aan de Rijn"], "pred_text": "near Millingen aan de Rijn", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 5}, {"id": "squad-10440", "gold": ["Galileo", "Aristotle's", "Aristotle", "Aristotle", "Aristotle's"], "pred_text": "Aristotle", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-6565", "gold": ["macrophages, neutrophils, and dendritic cells", "macrophages, neutrophils, and dendritic cells", "macrophages, neutrophils, and dendritic cells"], "pred_text": "macrophages, neutrophils, and dendritic cells", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 5}, {"id": "squad-8237", "gold": ["Nestorianism and Roman Catholicism", "Nestorianism and Roman Catholicism", "Nestorianism and Roman Catholicism"], "pred_text": "Nestorianism and Roman Catholicism", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-3722", "gold": ["January 18, 1974,", "January 18, 1974", "January 18, 1974", "January 18, 1974", "January 18, 1974"], "pred_text": "January 18, 1974", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-4463", "gold": ["climate change in addition to deforestation", "climate change in addition to deforestation", "climate change in addition to deforestation"], "pred_text": "climate change, deforestation", "f1": 0.6666666666666666, "contain": 0.0, "comply": 0.0, "n_words": 3}, {"id": "squad-1276", "gold": ["stroke", "a stroke", "a stroke"], "pred_text": "some sources say that he died of a stroke", "f1": 0.2222222222222222, "contain": 1.0, "comply": 1.0, "n_words": 9}, {"id": "squad-2044", "gold": ["twice the figure for the average profession", "twice", "twice"], "pred_text": "twice", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-4062", "gold": ["black and white", "black and white,", "black and white", "remaining in black and white", "remaining in black and white,"], "pred_text": "black and white", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-5792", "gold": ["Mount Wilson", "Mount Wilson", "August 1"], "pred_text": "Mount Wilson", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-5915", "gold": ["ABC Cable News", "ABC Cable News", "ABC Cable News"], "pred_text": "ABC Cable News", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-5872", "gold": ["seven", "seven", "seven"], "pred_text": "seven", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-5056", "gold": ["complex", "complex", "complex", "quite complex"], "pred_text": "complex", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-1853", "gold": ["Presburger arithmetic", "Presburger", "Presburger arithmetic"], "pred_text": "Presburger arithmetic", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-1651", "gold": ["pity", "pity", "pity"], "pred_text": "pity", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-3694", "gold": ["important", "important", "important", "important"], "pred_text": "as organic solvents", "f1": 0.0, "contain": 0.0, "comply": 0.0, "n_words": 3}, {"id": "squad-6894", "gold": ["use the proceedings as a forum", "use the proceedings as a forum to inform the jury and the public of the political circumstances", "win an acquittal and avoid imprisonment", "use the proceedings as a forum to inform the jury and the public of the political circumstances", "use the proceedings as a forum to inform the jury and the public of the political circumstances surrounding the case"], "pred_text": "to use the proceedings as a forum to inform the jury and", "f1": 0.761904761904762, "contain": 1.0, "comply": 1.0, "n_words": 12}, {"id": "squad-7615", "gold": ["force", "force"], "pred_text": "force", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-4007", "gold": ["July 1969", "July", "July 1969", "July 20, 1969", "July 1969"], "pred_text": "July 1969", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-2797", "gold": ["UCLA", "UCLA", "UCLA"], "pred_text": "UCLA", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-10431", "gold": ["applied", "applied force", "applied force", "applied force", "applied"], "pred_text": "applied force", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-4358", "gold": ["sediment deposits", "sediment deposits", "sediment deposits"], "pred_text": "sediment deposits from Amazon basin paleolakes and from", "f1": 0.4, "contain": 1.0, "comply": 1.0, "n_words": 8}, {"id": "squad-7654", "gold": ["The Daleks (a.k.a. The Mutants)", "Daleks", "eponymous aliens"], "pred_text": "The Daleks", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-7940", "gold": ["Matt Smith", "Matt Smith", "Matt Smith"], "pred_text": "Matt Smith", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-10010", "gold": ["mid-18th century", "began in the mid-18th century within the Church of England.", "mid-18th century"], "pred_text": "the mid-18th century", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-8665", "gold": ["projected rises in sea levels", "rises in sea levels", "projected rises in sea levels"], "pred_text": "risks and dangers related to sea level rises", "f1": 0.3333333333333333, "contain": 0.0, "comply": 0.0, "n_words": 8}, {"id": "squad-1248", "gold": ["1875", "1875", "1875"], "pred_text": "1875", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-6428", "gold": ["the ease with which people, youth in particular, can obtain controlled substances", "ease with which people, youth in particular, can obtain controlled substances", "the ease with which people, youth in particular, can obtain controlled substances"], "pred_text": "the ease with which people, youth in particular, can obtain", "f1": 0.9, "contain": 0.0, "comply": 1.0, "n_words": 10}, {"id": "squad-4753", "gold": ["very little", "very little", "very little"], "pred_text": "very little retail activity", "f1": 0.6666666666666666, "contain": 1.0, "comply": 1.0, "n_words": 4}, {"id": "squad-6404", "gold": ["drug choice, dose, route, frequency, and duration of therapy", "drug choice, dose, route, frequency, and duration of therapy", "drug choice, dose, route, frequency, and duration of therapy"], "pred_text": "drug choice, dose, route, frequency, and duration of", "f1": 0.9411764705882353, "contain": 0.0, "comply": 1.0, "n_words": 8}, {"id": "squad-8171", "gold": ["Confucian propriety and ancestor veneration", "Confucian propriety and ancestor veneration", "Confucian propriety and ancestor veneration"], "pred_text": "the rituals of Confucian propriety and ancestor veneration", "f1": 0.8333333333333333, "contain": 1.0, "comply": 1.0, "n_words": 8}, {"id": "squad-10484", "gold": ["Mercury", "Mercury", "Mercury", "Mercury"], "pred_text": "Mercury", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-5933", "gold": ["Turner Broadcasting System", "Turner Broadcasting System", "Turner Broadcasting System"], "pred_text": "Turner Broadcasting System", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-3978", "gold": ["asphyxiated", "asphyxiated", "asphyxiated", "asphyxiated"], "pred_text": "The astronauts were asphyxiated before the hatch could be", "f1": 0.25, "contain": 1.0, "comply": 1.0, "n_words": 9}, {"id": "squad-9996", "gold": ["During the 16th and 17th centuries", "17th century", "the 16th and 17th centuries", "16th and 17th centuries", "16th and 17th centuries"], "pred_text": "in the 16th and 17th centuries", "f1": 0.888888888888889, "contain": 1.0, "comply": 0.0, "n_words": 6}, {"id": "squad-1128", "gold": ["William of Montreuil", "William of Montreuil", "William of Montreuil"], "pred_text": "William of Montreuil", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 3}, {"id": "squad-2760", "gold": ["Orange", "Orange County", "Orange County"], "pred_text": "Orange County", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-2122", "gold": ["many", "three", "many"], "pred_text": "three", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-10415", "gold": ["denoted scalar quantities", "scalar quantities", "denoted scalar quantities", "scalar quantities", "scalar quantities"], "pred_text": "scalar quantities", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 2}, {"id": "squad-1764", "gold": ["worst-case", "worst", "worst-case"], "pred_text": "worst-case", "f1": 1.0, "contain": 1.0, "comply": 1.0, "n_words": 1}, {"id": "squad-4614", "gold": ["zip\" the mouth shut when the animal is not feeding,", "\"zip\" the mouth shut when the animal is not feeding", "\"zip\" the mouth shut when the animal is not feeding"], "pred_text": "\"zip\" the mouth shut", "f1": 0.5454545454545454, "contain": 0.0, "comply": 1.0, "n_words": 4}]}}, "throughput": {"1": {"wall_s": 1.6753133839999919, "gen_tokens": 128, "tok_per_s": 76.40361571898039}, "4": {"wall_s": 1.687653669000042, "gen_tokens": 512, "tok_per_s": 303.379780700721}, "16": {"wall_s": 1.8126751560000685, "gen_tokens": 2048, "tok_per_s": 1129.8218509924359}, "64": {"wall_s": 2.3414580899999464, "gen_tokens": 8192, "tok_per_s": 3498.6746228715065}, "256": {"wall_s": 4.104849208000019, "gen_tokens": 32768, "tok_per_s": 7982.753650520906}}}