Skip to content

GitLab

Commit f75058c7 authored by Rayyyyy's avatar Rayyyyy
Browse files

First add.

parents
Pipeline #1411 canceled with stages
Show whitespace changes
Inline Side-by-side
Showing with 2870 additions and 0 deletions
FlagEmbedding/llm_embedder/data/toy/chat.json 0 → 100644
View file @ f75058c7
{"history": ["Speaker 1: I need some advice on where to go on vacation, have you been anywhere lately?\nSpeaker 2: I have been all over the world. I'm military.", "Speaker 1: That is good you have alot of travel experience\nSpeaker 2: Sure do. And a lot of experience blowing things up! Haha. Bora bora is nice.", "Speaker 1: I've been working non stop crazy hours and need a break.\nSpeaker 2: The best breaks are spent with cute cuddly kittens.", "Speaker 1: Bora bora sounds nice, you have been there before?\nSpeaker 2: Nope... Just sounds nice, and repetitive. Bora... Bora. Ha!", "Speaker 1: Kittens really? I rather be at the beach.\nSpeaker 2: Only if the beach was covered in kittens!", "Speaker 1: That would be a sight to see.\nSpeaker 2: Or maybe brownies... I love chocolate.", "Speaker 1: I love brownies too but I haven't quite perfected mine yet.\nSpeaker 2: Well I'm available to taste test!"], "query": "Are you still in the military?", "query_id": 0, "answers": ["No, I have no longer serve in the millitary, I had served up the full term that I signed up for, and now work outside of the millitary."], "task": "chat", "teacher_scores": [-3.171875, -2.984375, -3.03125, -3.140625, -3.140625, -3.1875, -3.171875]}
{"history": ["Speaker 1: I need some advice on where to go on vacation, have you been anywhere lately?\nSpeaker 2: I have been all over the world. I'm military.", "Speaker 1: That is good you have alot of travel experience\nSpeaker 2: Sure do. And a lot of experience blowing things up! Haha. Bora bora is nice.", "Speaker 1: I've been working non stop crazy hours and need a break.\nSpeaker 2: The best breaks are spent with cute cuddly kittens.", "Speaker 1: Bora bora sounds nice, you have been there before?\nSpeaker 2: Nope... Just sounds nice, and repetitive. Bora... Bora. Ha!", "Speaker 1: Kittens really? I rather be at the beach.\nSpeaker 2: Only if the beach was covered in kittens!", "Speaker 1: That would be a sight to see.\nSpeaker 2: Or maybe brownies... I love chocolate.", "Speaker 1: I love brownies too but I haven't quite perfected mine yet.\nSpeaker 2: Well I'm available to taste test!", "Speaker 1: Are you still in the military?\nSpeaker 2: No, I have no longer serve in the millitary, I had served up the full term that I signed up for, and now work outside of the millitary."], "query": "Oh wow that's very admirable. What do you do now?", "query_id": 1, "answers": ["I work as an electrical engineer for a private company, I was trained in the millitary as such and I found out that I not only liked it, but I was good at it as well."], "task": "chat", "teacher_scores": [-2.5625, -2.59375, -2.6875, -2.546875, -2.734375, -2.71875, -2.65625, -2.046875]}
{"history": ["Speaker 1: I need some advice on where to go on vacation, have you been anywhere lately?\nSpeaker 2: I have been all over the world. I'm military.", "Speaker 1: That is good you have alot of travel experience\nSpeaker 2: Sure do. And a lot of experience blowing things up! Haha. Bora bora is nice.", "Speaker 1: I've been working non stop crazy hours and need a break.\nSpeaker 2: The best breaks are spent with cute cuddly kittens.", "Speaker 1: Bora bora sounds nice, you have been there before?\nSpeaker 2: Nope... Just sounds nice, and repetitive. Bora... Bora. Ha!", "Speaker 1: Kittens really? I rather be at the beach.\nSpeaker 2: Only if the beach was covered in kittens!", "Speaker 1: That would be a sight to see.\nSpeaker 2: Or maybe brownies... I love chocolate.", "Speaker 1: I love brownies too but I haven't quite perfected mine yet.\nSpeaker 2: Well I'm available to taste test!", "Speaker 1: Are you still in the military?\nSpeaker 2: No, I have no longer serve in the millitary, I had served up the full term that I signed up for, and now work outside of the millitary.", "Speaker 1: Oh wow that's very admirable. What do you do now?\nSpeaker 2: I work as an electrical engineer for a private company, I was trained in the millitary as such and I found out that I not only liked it, but I was good at it as well."], "query": "Thats really impressive. I am a mechanical engineer so I can relate. What kind of stuff do you work on", "query_id": 2, "answers": ["I troubleshoot equipment that the company either sells to its customers, or that it rents to its customers. I also install and test their equipment, to ensure that it not only works, but it also is safe to use in a household."], "task": "chat", "teacher_scores": [-2.921875, -2.796875, -2.859375, -2.828125, -2.84375, -2.984375, -2.9375, -2.921875, -2.9375]}
{"history": ["Speaker 1: I need some advice on where to go on vacation, have you been anywhere lately?\nSpeaker 2: I have been all over the world. I'm military.", "Speaker 1: That is good you have alot of travel experience\nSpeaker 2: Sure do. And a lot of experience blowing things up! Haha. Bora bora is nice.", "Speaker 1: I've been working non stop crazy hours and need a break.\nSpeaker 2: The best breaks are spent with cute cuddly kittens.", "Speaker 1: Bora bora sounds nice, you have been there before?\nSpeaker 2: Nope... Just sounds nice, and repetitive. Bora... Bora. Ha!", "Speaker 1: Kittens really? I rather be at the beach.\nSpeaker 2: Only if the beach was covered in kittens!", "Speaker 1: That would be a sight to see.\nSpeaker 2: Or maybe brownies... I love chocolate.", "Speaker 1: I love brownies too but I haven't quite perfected mine yet.\nSpeaker 2: Well I'm available to taste test!", "Speaker 1: Are you still in the military?\nSpeaker 2: No, I have no longer serve in the millitary, I had served up the full term that I signed up for, and now work outside of the millitary.", "Speaker 1: Oh wow that's very admirable. What do you do now?\nSpeaker 2: I work as an electrical engineer for a private company, I was trained in the millitary as such and I found out that I not only liked it, but I was good at it as well.", "Speaker 1: Thats really impressive. I am a mechanical engineer so I can relate. What kind of stuff do you work on\nSpeaker 2: I troubleshoot equipment that the company either sells to its customers, or that it rents to its customers. I also install and test their equipment, to ensure that it not only works, but it also is safe to use in a household."], "query": "Right on, thats pretty cool. Like generators and motors or household items?", "query_id": 3, "answers": ["Generators for sure are some of the things I have worked on, but also wiring and how a system is connected as well. Its suprising how different some households can be wired than others in a given neighborhood."], "task": "chat", "teacher_scores": [-3.21875, -3.28125, -3.296875, -3.15625, -3.390625, -3.375, -3.21875, -3.234375, -3.09375, -2.9375]}
{"history": ["Speaker 1: I need some advice on where to go on vacation, have you been anywhere lately?\nSpeaker 2: I have been all over the world. I'm military.", "Speaker 1: That is good you have alot of travel experience\nSpeaker 2: Sure do. And a lot of experience blowing things up! Haha. Bora bora is nice.", "Speaker 1: I've been working non stop crazy hours and need a break.\nSpeaker 2: The best breaks are spent with cute cuddly kittens.", "Speaker 1: Bora bora sounds nice, you have been there before?\nSpeaker 2: Nope... Just sounds nice, and repetitive. Bora... Bora. Ha!", "Speaker 1: Kittens really? I rather be at the beach.\nSpeaker 2: Only if the beach was covered in kittens!", "Speaker 1: That would be a sight to see.\nSpeaker 2: Or maybe brownies... I love chocolate.", "Speaker 1: I love brownies too but I haven't quite perfected mine yet.\nSpeaker 2: Well I'm available to taste test!", "Speaker 1: Are you still in the military?\nSpeaker 2: No, I have no longer serve in the millitary, I had served up the full term that I signed up for, and now work outside of the millitary.", "Speaker 1: Oh wow that's very admirable. What do you do now?\nSpeaker 2: I work as an electrical engineer for a private company, I was trained in the millitary as such and I found out that I not only liked it, but I was good at it as well.", "Speaker 1: Thats really impressive. I am a mechanical engineer so I can relate. What kind of stuff do you work on\nSpeaker 2: I troubleshoot equipment that the company either sells to its customers, or that it rents to its customers. I also install and test their equipment, to ensure that it not only works, but it also is safe to use in a household.", "Speaker 1: Right on, thats pretty cool. Like generators and motors or household items?\nSpeaker 2: Generators for sure are some of the things I have worked on, but also wiring and how a system is connected as well. Its suprising how different some households can be wired than others in a given neighborhood."], "query": "Oh yes I can imagine. My cousin bought a house with all aluminum wiring. He spent a lot in bring it up to code.", "query_id": 4, "answers": ["That must a been some kind of endeavor. Its great that people are aware of issues that arise in their homes, otherwise it can be very problematic in the future."], "task": "chat", "teacher_scores": [-2.859375, -2.84375, -2.875, -2.875, -2.984375, -2.96875, -2.828125, -2.875, -2.984375, -2.921875, -2.75]}
{"history": ["Speaker 1: I need some advice on where to go on vacation, have you been anywhere lately?\nSpeaker 2: I have been all over the world. I'm military.", "Speaker 1: That is good you have alot of travel experience\nSpeaker 2: Sure do. And a lot of experience blowing things up! Haha. Bora bora is nice.", "Speaker 1: I've been working non stop crazy hours and need a break.\nSpeaker 2: The best breaks are spent with cute cuddly kittens.", "Speaker 1: Bora bora sounds nice, you have been there before?\nSpeaker 2: Nope... Just sounds nice, and repetitive. Bora... Bora. Ha!", "Speaker 1: Kittens really? I rather be at the beach.\nSpeaker 2: Only if the beach was covered in kittens!", "Speaker 1: That would be a sight to see.\nSpeaker 2: Or maybe brownies... I love chocolate.", "Speaker 1: I love brownies too but I haven't quite perfected mine yet.\nSpeaker 2: Well I'm available to taste test!", "Speaker 1: Are you still in the military?\nSpeaker 2: No, I have no longer serve in the millitary, I had served up the full term that I signed up for, and now work outside of the millitary.", "Speaker 1: Oh wow that's very admirable. What do you do now?\nSpeaker 2: I work as an electrical engineer for a private company, I was trained in the millitary as such and I found out that I not only liked it, but I was good at it as well.", "Speaker 1: Thats really impressive. I am a mechanical engineer so I can relate. What kind of stuff do you work on\nSpeaker 2: I troubleshoot equipment that the company either sells to its customers, or that it rents to its customers. I also install and test their equipment, to ensure that it not only works, but it also is safe to use in a household.", "Speaker 1: Right on, thats pretty cool. Like generators and motors or household items?\nSpeaker 2: Generators for sure are some of the things I have worked on, but also wiring and how a system is connected as well. Its suprising how different some households can be wired than others in a given neighborhood.", "Speaker 1: Oh yes I can imagine. My cousin bought a house with all aluminum wiring. He spent a lot in bring it up to code.\nSpeaker 2: That must a been some kind of endeavor. Its great that people are aware of issues that arise in their homes, otherwise it can be very problematic in the future."], "query": "Oh it was. I cant imagine taking something like that on. But its definitely worth it to keep his home safe. ", "query_id": 5, "answers": ["Especially if you plan on keeping it for a while, or if you have a family to take care of. I would want my house to be as safe as possible for my children."], "task": "chat", "teacher_scores": [-2.140625, -1.8984375, -1.921875, -1.765625, -1.8984375, -1.921875, -1.9609375, -2.140625, -1.984375, -1.8125, -1.6875, -1.84375]}
{"history": ["Speaker 1: Hello! What are you up to?\nSpeaker 2: I'm reading for a exam I've and you?", "Speaker 1: Just finished playing with my three dogs here at home.\nSpeaker 2: Cool 3 dogs. I'll get a dog when I'm done with school next september", "Speaker 1: Sounds good. What kind will you get?\nSpeaker 2: A pit bull. I'll get him when I find a teaching job", "Speaker 1: I teach drums and I play them really well.\nSpeaker 2: Wow my brother is in a heavy medal group and goes all around the world", "Speaker 1: That's awesome! Do you play any sports?\nSpeaker 2: No I really do not have time. College is hard lol", "Speaker 1: I bet. I work out, even though I don't like to, but need to.\nSpeaker 2: My mom and dad work out a lot. They came here when I was 5", "Speaker 1: I work out to stay healthy. I've a problem with my blood sugar.\nSpeaker 2: Oh wow I hoe you stay healthy for a long time", "Speaker 1: Thanks. I check it everyday.\nSpeaker 2: That is cool so what else do you like?"], "query": "How is the preparation for your exam going?", "query_id": 6, "answers": ["It's not.. really. I've been procrastinating a bit. Any tips on how to buckle down?"], "task": "chat", "teacher_scores": [-2.21875, -2.109375, -2.203125, -2.34375, -2.078125, -2.171875, -2.1875, -2.1875]}
{"history": ["Speaker 1: Hello! What are you up to?\nSpeaker 2: I'm reading for a exam I've and you?", "Speaker 1: Just finished playing with my three dogs here at home.\nSpeaker 2: Cool 3 dogs. I'll get a dog when I'm done with school next september", "Speaker 1: Sounds good. What kind will you get?\nSpeaker 2: A pit bull. I'll get him when I find a teaching job", "Speaker 1: I teach drums and I play them really well.\nSpeaker 2: Wow my brother is in a heavy medal group and goes all around the world", "Speaker 1: That's awesome! Do you play any sports?\nSpeaker 2: No I really do not have time. College is hard lol", "Speaker 1: I bet. I work out, even though I don't like to, but need to.\nSpeaker 2: My mom and dad work out a lot. They came here when I was 5", "Speaker 1: I work out to stay healthy. I've a problem with my blood sugar.\nSpeaker 2: Oh wow I hoe you stay healthy for a long time", "Speaker 1: Thanks. I check it everyday.\nSpeaker 2: That is cool so what else do you like?", "Speaker 1: How is the preparation for your exam going?\nSpeaker 2: It's not.. really. I've been procrastinating a bit. Any tips on how to buckle down?"], "query": "Procrastinating is a hard habit to break. Just stop doing it.", "query_id": 7, "answers": ["Haha easier said than done! Kind of like monitoring your blood sugar, how's that going for you lately? Do you feel well?"], "task": "chat", "teacher_scores": [-2.90625, -3.171875, -3.203125, -3.09375, -3.15625, -3.046875, -2.609375, -2.8125, -2.703125]}
FlagEmbedding/llm_embedder/data/toy/convsearch.json 0 → 100644
View file @ f75058c7
This source diff could not be displayed because it is too large. You can view the blob instead.
FlagEmbedding/llm_embedder/data/toy/icl.json 0 → 100644
View file @ f75058c7
{"query": "He was a journalist for the `` Greater Malling Gazette '' in Greater Malling , Yorkshire , living in York . He was a journalist for the `` Greater Malling Gazette '' at Greater Malling , York , living in Yorkshire . Do these sentences mean the same thing?", "pos": ["Chrishan was born in Toledo , Ohio and grew up in Minneapolis , Minnesota . He attended High School for Recording Arts in St. Paul , Minnesota . He was born in Toledo , Ohio , grew up in St. Paul , Minnesota , and visited the High School for Recording Arts in Minneapolis , Minnesota . Do these sentences mean the same thing?\nNo", "He was born and raised in Vancouver , British Columbia , Canada and died in Aylesbury , Saskatchewan , Canada . Born and raised in Aylesbury , Saskatchewan , Canada , he died in Vancouver , British Columbia , Canada . Do these sentences mean the same thing?\nNo", "Born in Nanhui , he attended engineering school in Nanjing and spent a year at the University of California , USA . Born in Nanjing , he attended the engineering school in Nanhui and spent one year at the University of California . Do these sentences mean the same thing?\nNo"], "pos_scores": [-5.59375, -7.40625, -7.46875], "neg": ["He was born in New Westminster and worked on the lower Fraser and Fraser River cyclists before coming to the Upper Yukon River in the early 20th century . He was born in New Westminster and worked on the lower Fraser and Yukon River sternwheelers before coming to the upper Fraser River in the early 1900s . Do these sentences mean the same thing?\nNo", "He was born in New Westminster and worked on the lower Fraser and Yukon River star cyclists before coming to the upper Fraser River in the early 1900s . He was born in New Westminster and worked on the lower Fraser and Fraser River cyclists before coming to the Upper Yukon River in the early 20th century . Do these sentences mean the same thing?\nNo", "He was born in Nanjing , attended the engineering school in Nanhui and spent a year at the University of California . Born in Nanhui , he attended engineering school in Nanjing and spent a year at the University of California , USA . Do these sentences mean the same thing?\nNo", "He was born in New Westminster and worked on the lower Fraser and Fraser River sternwheelers before coming to the upper Yukon River in the early 1900s . He was born in New Westminster and worked on the lower Fraser and Yukon River star cyclists before coming to the upper Fraser River in the early 1900s . Do these sentences mean the same thing?\nNo", "He was born in New Westminster and worked on the lower Fraser and Yukon River rear wheels before coming to the upper Fraser River in the early 1900s . He was born in New Westminster and worked on the lower Fraser and Yukon River sternwheelers before coming to the upper Fraser River in the early 1900s . Do these sentences mean the same thing?\nNo", "After completing his initial university education at Cardiff University , and in Harrogate , North Yorkshire , he was from 1996 to 1999 based in Rouen , France . After completing his first university education at Cardiff University and in Rouen , France , he was established in Harrogate , North Yorkshire , from 1996 to 1999 . Do these sentences mean the same thing?\nNo", "He was born in New York City and grew up in Athens , Greece , and then at North Grafton , Massachusetts . Born in Athens , Greece , he grew up in New York City and then in North Grafton , Massachusetts . Do these sentences mean the same thing?\nNo", "Chetrit lives in New York City . He teaches Middle Eastern language , literature , culture and Hebrew . He studies at Queens College in Flushing , New York . He lives in New York City and teaches Hebrew language , literature and culture , and Middle East studies at Queens College in Flushing , New York . Do these sentences mean the same thing?\nNo", "After completing his initial university education at Cardiff University , and in Harrogate , North Yorkshire , he was from 1996 to 1999 based in Rouen , France . After completing his university education at Cardiff University and Rouen , France , he was based in Harrogate , North Yorkshire , from 1996 to 1999 . Do these sentences mean the same thing?\nNo", "He was born and raised in Aylesbury , Saskatchewan , Canada and died in Vancouver , British Columbia , Canada . Born and raised in Vancouver , Canada , British Columbia , he died in Aylesbury , Saskatchewan , Canada . Do these sentences mean the same thing?\nNo", "Born in Athens , Greece , he grew up in New York City and then in North Grafton , Massachusetts . He was born in New York City , and grew up in Athens , Greece , and then North Grafton , Massachusetts . Do these sentences mean the same thing?\nNo", "He was born in Athens , Greece and grew up in New York City and then in North Grafton , Massachusetts . Born in New York City , he grew up in Athens , Greece , and then in North Grafton , Massachusetts . Do these sentences mean the same thing?\nNo", "Chrishan was born in Toledo , Ohio and grew up in St. Paul , Minnesota . He attended High School for Recording Arts in Minneapolis , Minnesota . He was born in Toledo , Ohio , grew up in Minneapolis , Minnesota , and visited the High School for Recording Arts in St. Paul , Minnesota . Do these sentences mean the same thing?\nNo", "He was born in Athens , Greece and grew up in New York City and then in North Grafton , Massachusetts . He was born in New York City and grew up in Athens , Greece , and then at North Grafton , Massachusetts . Do these sentences mean the same thing?\nNo", "He was born in Athens , Greece , and grew up in New York City , and then North Grafton , Massachusetts . He was born in New York City and grew up in Athens , Greece , and then at North Grafton , Massachusetts . Do these sentences mean the same thing?\nNo", "He was born in New York City , and grew up in Athens , Greece , and then North Grafton , Massachusetts . He was born in Athens , Greece and grew up in New York City and then in North Grafton , Massachusetts . Do these sentences mean the same thing?\nNo", "He was born in Athens , Greece , and grew up in New York City , and then North Grafton , Massachusetts . Born in New York City , he grew up in Athens , Greece , and then in North Grafton , Massachusetts . Do these sentences mean the same thing?\nNo"], "neg_score": [-7.53125, -7.53125, -7.71875, -7.78125, -7.84375, -7.84375, -7.96875, -8.0, -8.0625, -8.1875, -8.1875, -8.25, -8.25, -8.5, -8.5625, -8.5625, -8.5625], "teacher_scores": [-5.59375, -7.40625, -7.46875, -7.53125, -7.53125, -7.71875, -7.78125, -7.84375, -7.84375, -7.96875, -8.0, -8.0625, -8.1875, -8.1875, -8.25, -8.25, -8.5, -8.5625, -8.5625, -8.5625], "task": "icl"}
{"query": "\"Across border, cleric raises voice of reason QOM, Iran -- These are pilgrims of a different kind. Several times a week, Shi'ite Muslim clerics and community leaders from neighboring Iraq climb stairs to a little office in this Iranian city of shrines and Islamic seminaries. There, an aged and hunched figure sits on a swivel chair fitted with an electric blanket.\" What is this text about? World, Sports, Business, or Technology?", "pos": ["\"Options running out as Najaf talks collapse BAGHDAD -- After more than a week of fighting between the Mahdi Army and the combined US and Iraqi forces in Najaf, Baghdad's Sadr City, and a half-dozen other cities, analysts say neither interim Prime Minister Iyad Allawi nor radical cleric Moqtada al-Sadr had any good options left other than to talk.\" What is this text about? World, Sports, Business, or Technology?\nWorld", "\"Iraqi Cleric Calls Cease-Fire After Bloody Uprising BAGHDAD, Iraq (Reuters) - Rebel Shi'ite cleric Moqtada al-Sadr has ordered his militia to end attacks on U.S. and Iraqi government forces and will soon unveil plans to pursue his goals through politics rather than conflict, aides said Monday.\" What is this text about? World, Sports, Business, or Technology?\nWorld", "\"Militants Remove Arms From Najaf Shrine NAJAF, Iraq - Militiamen loyal to rebel Shiite cleric Muqtada al-Sadr on Friday removed weapons from the revered Imam Ali Shrine in Najaf in a step aimed at ending the 2-week-old uprising centered on the holy site. Al-Sadr's followers remained in control of the walled shrine compound, but kept their guns outside...\" What is this text about? World, Sports, Business, or Technology?\nWorld"], "pos_scores": [-5.21875, -5.34375, -5.84375], "neg": ["\"Iraqis on Mission to End Najaf Insurgency BAGHDAD, Iraq - Iraq's national conference sent a delegation bearing a peace proposal to Najaf on Tuesday, hoping to end the standoff with radical Shiite cleric Muqtada al-Sadr, which has marred the gathering meant to be a landmark step toward democracy. As the National Conference in Baghdad put together the mission, a mortar round exploded several miles from the gathering's venue, killing seven people and wounding wounding 35, according to the Health Ministry...\" What is this text about? World, Sports, Business, or Technology?\nWorld", "\"Iraq Says Offensive Vs. Cleric to Start NAJAF, Iraq - An Iraqi Cabinet minister said Thursday that Iraqi forces could begin an offensive against Muqtada al-Sadr within hours, despite the firebrand cleric's acceptance of a peace proposal. To prevent an imminent attack on his forces, who are holed up in the revered Imam Ali Shrine in Najaf, al-Sadr must immediately disarm his Mahdi Army militia and hand over its weapons to the authorities, Minister of State Qassim Dawoud said...\" What is this text about? World, Sports, Business, or Technology?\nWorld", "\"Militia Offers to Cede Control of Shrine NAJAF, Iraq - A rebel cleric's militiamen kept their guns outside a holy site Friday after issuing a surprise offer to give up control of the Imam Ali Shrine to Shiite Muslim religious leaders, but negotiators wrangled into the night over getting the militants out of the compound. The removal of weapons and pledge to hand over keys to religious authorities was seen as a big step toward a resolution of the two-week faceoff in Najaf that has killed dozens of people and wounded hundreds in fighting between Muqtada al-Sadr's militia and a joint U.S.-Iraq force...\" What is this text about? World, Sports, Business, or Technology?\nWorld", "\"Militants Turn Over Keys to Najaf Shrine NAJAF, Iraq - Thousands of pilgrims streamed into the Imam Ali Shrine on Friday, and militants who had been holed up in the site left it, handing the keys to Shiite religious authorities after Iraq's top Shiite cleric brokered a peace deal to end three weeks of fighting in this holy city. Dozens of militants piled Kalashnikov rifles in front of the offices of their leader, radical Shiite cleric Muqtada al-Sadr...\" What is this text about? World, Sports, Business, or Technology?\nWorld", "\"Iraq Cleric Urges Fighters to Drop Arms NAJAF, Iraq - Thousands of pilgrims streamed into the Imam Ali Shrine on Friday and filed out mixed with militants who had been holed up inside, leaving the holy site nearly empty after Iraq's top Shiite cleric brokered a peace deal to end three weeks of fighting in this holy city. Hours earlier, firebrand Shiite cleric Muqtada al-Sadr issued a statement broadcast over the shrine's loudspeakers, ordering his fighters to lay down their arms and leave Najaf and neighboring Kufa...\" What is this text about? World, Sports, Business, or Technology?\nWorld", "\"Al-Sadr Loyalists Agree to Hand Over Arms BAGHDAD, Iraq - Shiite militiamen loyal to radical cleric Muqtada al-Sadr agreed Saturday to begin handing in weapons, a significant step toward restoring order in Baghdad's sprawling Sadr City slum as the interim government struggles to curb Iraq's more widespread Sunni insurgency. In a sign of persistent Sunni unrest, clashes flared in several cities as the search continued for the body of British hostage Kenneth Bigley, who was decapitated by his abductors - reportedly after a failed escape attempt...\" What is this text about? World, Sports, Business, or Technology?\nWorld", "\"Radical Cleric Accepts Najaf Peace Plan NAJAF, Iraq - Radical Shiite cleric Muqtada al-Sadr accepted a peace plan Wednesday that would disarm his militia and remove them from their hideout in a revered shrine, raising hopes of resolving a crisis that has angered many of Iraq's majority Shiites and threatened to undermine the fledgling interim government. But al-Sadr has made contradictory statements in the past, and aides to the cleric said he still wanted to negotiate details of the deal to end two weeks of fighting between his forces and U.S.-led troops...\" What is this text about? World, Sports, Business, or Technology?\nWorld", "\"Mortar Attack Kills 27 at Kufa Mosque KUFA, Iraq - A mortar barrage hit the main mosque in the Iraqi city of Kufa on Thursday, killing 27 people and wounding 63 others as they prepared to march on the violence-wracked city of Najaf, hospital officials and witnesses said. Thousands of people were crowded around the mosque at the time and dead bodies lay around the mosque compound, which is a stronghold of followers of radical Shiite cleric Muqtada al-Sadr, witnesses said...\" What is this text about? World, Sports, Business, or Technology?\nWorld", "\"Al-Sadr Followers Offer to Leave Shrine NAJAF, Iraq - Followers loyal to radical Shiite cleric Muqtada al-Sadr said Friday they were prepared to hand control of the revered Imam Ali Shrine to top Shiite religious authorities, and Iraq's interim prime minister said he would not storm the holy site. The moves came after a day and night of fighting in Najaf that killed 77 people and wounded 70 others, as al-Sadr militiamen mortared a police station and U.S...\" What is this text about? World, Sports, Business, or Technology?\nWorld", "\"Mortar Attack Kills 25 at Kufa Mosque NAJAF, Iraq - Iraq's top Shiite cleric unexpectedly returned to Iraq from Britain on Wednesday armed with a new peace initiative and a call for Iraqis across the country to march on the holy city to demand an end to weeks of fighting in Najaf. But a mortar attack on the main mosque in the nearby city of Kufa killed 25 people and wounded 60 others Thursday, hospital officials and witnesses said, a blow to hopes that the crisis could be resolved peacefully...\" What is this text about? World, Sports, Business, or Technology?\nWorld", "\"Tehran official calls a Bush win preferable TEHRAN -- The head of Iran's security council said yesterday that the reelection of President Bush was in Tehran's best interests, despite the administration's axis of evil label, accusations that Iran harbors Al Qaeda terrorists, and threats of sanctions over the country's nuclear ambitions. Historically, Democrats have harmed Iran more than Republicans, said Hasan Rowhani, head of the Supreme National ...\" What is this text about? World, Sports, Business, or Technology?\nWorld", "\"Najaf Fighting Resumes Amid Peace Mission BAGHDAD, Iraq - A pared-down delegation of Iraqis arrived in Najaf by helicopter Tuesday to present radical Shiite cleric Muqtada al-Sadr with a peace proposal aimed at ending the violent insurgency wracking the holy city. Despite the delegation's presence there, fighting intensified in Najaf, with at least one U.S...\" What is this text about? World, Sports, Business, or Technology?\nWorld", "\"National Assembly elections set for Jan. 30 BAGHDAD -- Iraq's electoral commission yesterday set Jan. 30 for elections to choose a National Assembly, a vote that could deliver power to Iraq's Shi'ite Muslim majority after decades of disenfranchisement. The balloting, however, remains imperiled by calls for a Sunni Muslim boycott and a persistent insurgency that has roiled Sunni regions.\" What is this text about? World, Sports, Business, or Technology?\nWorld", "\"Sunnis drop out In the first major political backlash over the assault on Fallujah, Iraqs most prominent Sunni political party said Tuesday it was withdrawing from the interim Iraqi government, while the leading group of Sunni clerics called for Iraqis to boycott the \" What is this text about? World, Sports, Business, or Technology?\nWorld", "\"The five points of the Najaf peace agreement NAJAF: Rebel cleric Moqtada Sadr and his Mehdi Army were preparing to hand over the keys to the Imam Ali shrine on Friday after Sadr agreed a peace initiative by Iraqs top Shiite cleric, Grand Ayatollah Ali al-Sistani, to end fierce fighting in the \" What is this text about? World, Sports, Business, or Technology?\nWorld", "\"Sunni clerics threaten election boycott Baghdad -- A group representing thousands of Sunni Arab mosques around the country said Wednesday that it would call for a boycott of the elections in January if the US military and the Iraqi government continued the military operations around the \" What is this text about? World, Sports, Business, or Technology?\nWorld", "\"Najaf truce takes hold NAJAF, Iraq - Militiamen loyal to Shiite cleric Muqtada al-Sadr surrendered the sacred shrine of Imam Ali on Friday and then surrendered weapons as well, bringing a largely peaceful end to a ferocious three-week battle with US forces that challenged the \" What is this text about? World, Sports, Business, or Technology?\nWorld"], "neg_score": [-6.28125, -6.8125, -6.90625, -7.0625, -7.1875, -7.25, -7.34375, -7.375, -7.40625, -7.75, -7.90625, -8.4375, -9.25, -9.9375, -10.0, -10.375, -10.5], "teacher_scores": [-5.21875, -5.34375, -5.84375, -6.28125, -6.8125, -6.90625, -7.0625, -7.1875, -7.25, -7.34375, -7.375, -7.40625, -7.75, -7.90625, -8.4375, -9.25, -9.9375, -10.0, -10.375, -10.5], "task": "icl"}
{"query": "The album was released on the RCA Victor label in France , and on Metronome label in Germany . The album was released on the RCA Victor label in France and at Metronome Label in Germany . Do these sentences mean the same thing?", "pos": ["The songs were recorded in the Pink Studio for free and were released by Radio B92 with the label PGP-RTS . The songs were recorded for free in the Pink Studio and were released by radio B92 with the PGP-RTS label . Do these sentences mean the same thing?\nYes", "It was re-released in 2001 on Phyte Records ( CD ) and Platinum Recordings ( 7 `` Format ) and released in 2003 on Bridge 9 Records . It was relaunched in 2001 on Phyte Records ( CD ) and Platinum Recordings ( 7 `` format ) and published on Bridge 9 Records in 2003 . Do these sentences mean the same thing?\nYes", "It was released in 2001 on Phyte Records ( CD ) and Platinum Recordings ( 7 `` Format ) and re-released in 2003 on Bridge 9 Records . It was released in 2001 on Phyte Records ( CD ) and Platinum Recordings ( 7 `` format ) and will be released on Bridge 9 records in 2003 . Do these sentences mean the same thing?\nYes"], "pos_scores": [-6.9375, -7.1875, -7.25], "neg": ["The album was recorded in six different sessions at the Santa Monica Sound Records in Los Angeles and at the Westlake Recording Studios in Santa Monica , California . The album was recorded in six different sessions at both Santa Monica Sound Records in Los Angeles and Westlake Recording Studios in Santa Monica , California . Do these sentences mean the same thing?\nYes", "It was released in 2001 on Phyte Records ( CD ) and Platinum Recordings ( 7 `` format ) and was released on Bridge 9 Records in 2003 . It was re-released in 2001 on Phyte Records ( CD ) and Platinum Recordings ( 7 `` Format ) and released in 2003 on Bridge 9 Records . Do these sentences mean the same thing?\nYes", "The album features the single `` You Lose , I Win '' , which was a minor hit in Germany and also received considerable airplay in Canada . The album contains the single `` You Lose , I Win '' , which was a small hit in Germany and also received considerable airplay in Canada . Do these sentences mean the same thing?\nYes", "It was released on his album `` From Elvis in Memphis '' and was recorded in American Sound Studio in Memphis on February 18 , 1969 . It was released on his album `` From Elvis in Memphis '' and was recorded on 18 February 1969 at the American Sound Studio in Memphis . Do these sentences mean the same thing?\nYes", "It was recorded on his album `` From Elvis in Memphis '' and was released in American Sound Studio in Memphis on February 18 , 1969 . It was recorded on his album `` From Elvis in Memphis '' and was released on February 18 , 1969 at the American Sound Studio in Memphis . Do these sentences mean the same thing?\nYes", "It was released on his album `` From Elvis in Memphis '' and was recorded on February 18 , 1969 in Memphis at the American Sound Studio . It was released on his album `` From Elvis in Memphis '' and was recorded in American Sound Studio in Memphis on February 18 , 1969 . Do these sentences mean the same thing?\nYes", "It was recorded on his album `` From Elvis in Memphis '' and was released on 18 February 1969 at the American Sound Studio in Memphis . It was recorded on his album `` From Elvis in Memphis '' and was released in American Sound Studio in Memphis on February 18 , 1969 . Do these sentences mean the same thing?\nYes", "It was released in October 1971 in the US , and the following month in the UK where it reached numbers 15 and 16 respectively in the album charts . It was released in October 1971 in the United States , and the following month in the UK , where it received numbers 15 and 16 in the album charts respectively . Do these sentences mean the same thing?\nYes", "The album was recorded in six different sessions , both in Santa Monica Sound Records in Los Angeles and at Westlake Recording Studios in Santa Monica , California . The album was recorded in six different sessions at both Santa Monica Sound Records in Los Angeles and Westlake Recording Studios in Santa Monica , California . Do these sentences mean the same thing?\nYes", "The album was recorded in six different sessions at both Santa Monica Sound Records in Santa Monica , California and Westlake Recording Studios in Los Angeles . The album was recorded in six different sessions , both at Santa Monica Sound Records in Santa Monica , California , and at Westlake Recording Studios in Los Angeles . Do these sentences mean the same thing?\nYes", "The album was released on April 30 , as a limited edition , untitled album with hand drawn covers , and signed by Buckethead himself to be announced on May 31 . The album was released on April 30 as a limited , untitled album with hand drawn covers and signed by Buckethead himself to be announced on May 31 . Do these sentences mean the same thing?\nYes", "The album was released on Sony Japan , PIAS in the UK , Flying Nun in Europe , Infectious Records in the New Zealand and Festival Records in Australia . The album was released on Sony Japan , PIAS in the UK , Flying Nun in Europe , Infectious Records in New Zealand and Festival Records in Australia . Do these sentences mean the same thing?\nYes", "Free Ride is an album by trumpeter Dizzy Gillespie which was composed , arranged and conducted by Lalo Schifrin , recorded in 1977 and released on the Pablo label . Free Ride is an album by Trumpeter Dizzy Gillespie composed , arranged and conducted by Lalo Schifrin , which was recorded in 1977 and published on the label Pablo . Do these sentences mean the same thing?\nYes", "Free Ride is an album by Trumpeter Dizzy Gillespie , which was composed , arranged and conducted by Lalo Schifrin , recorded in 1977 and published on the Pablo Label . Free Ride is an album by trumpeter Dizzy Gillespie which was composed , arranged and conducted by Lalo Schifrin , recorded in 1977 and released on the Pablo label . Do these sentences mean the same thing?\nYes", "It was released in 1991 by Island Records , produced by Steve Beresford , Andy Metcalfe and Philip Oakey and re-released in 1999 by Universal Music 's Spectrum label . It was released in 1991 by Island Records , produced by Steve Beresford , Andy Metcalfe and Philip Oakey and published by Universal Music by Spectrum Label in 1999 . Do these sentences mean the same thing?\nYes", "The album was released on Sony Japan , PIAS in Europe , Flying Nun in New Zealand , Infectious Records in the UK and Festival Records in Australia . The album was released on Sony Japan , PIAS in the Europe , Flying Nun in New Zealand , Infectious Records in the UK and Festival Records in Australia . Do these sentences mean the same thing?\nYes", "Spoon in London is an album by Blues - singer Jimmy Witherspoon , which was recorded in 1965 in England and published on the Prestige label . Spoon in London is an album by blues vocalist Jimmy Witherspoon which was recorded in England in 1965 and released on the Prestige label . Do these sentences mean the same thing?\nYes"], "neg_score": [-7.46875, -7.5, -7.5625, -7.625, -7.71875, -7.71875, -7.71875, -7.84375, -7.875, -7.9375, -8.0625, -8.3125, -8.4375, -8.4375, -8.5625, -8.6875, -8.6875], "teacher_scores": [-6.9375, -7.1875, -7.25, -7.46875, -7.5, -7.5625, -7.625, -7.71875, -7.71875, -7.71875, -7.84375, -7.875, -7.9375, -8.0625, -8.3125, -8.4375, -8.4375, -8.5625, -8.6875, -8.6875], "task": "icl"}
{"query": "\"Which food should I not eat after getting a tattoo?\" \"What are some light food ideas to eat after surgery?\" Would you say that these questions are the same?", "pos": ["\"Does a lot of oil in food make it hard for you to loose fat? I'm starting to eat a lot of Indian/Pakistani food lately and I'm pretty sure it has a lot of oil (along with a lot of spices). Can I loose fat with cardio if I eat this stuff? Or should I not be eating it at all?\" \"Am I eating an unhealthy amount of junk food if I don't come near the RDA for saturated fats and cholesterol?\" Would you say that these questions are the same?\nNo", "\"Does the recommended daily values for micro-nutrients the same for, say, a 110 lb woman and a 200 lb man just like macros?\" \"If I'm eating a 3200 kcal diet, should I also be aiming for half-again as much of the micro nutrients like potassium and magnesium?\" Would you say that these questions are the same?\nNo", "\"What is more unhealthy, generally speaking, pizza or pasta?\" \"\"Whenever I hear the phrase 'Italian food' people talk about pizza, pasta, and the like. I am pretty sure this is not what Italians eat every day. What other types of foods are found in Italian cuisine that act staples of their diet?\"\" Would you say that these questions are the same?\nNo"], "pos_scores": [-5.375, -5.75, -6.375], "neg": ["\"If I have a slow metabolism do thoughts about food increase appetite?\" \"If you used to be a terrible person inside, and had extremely bad thoughts, but you changed everything about what you thought and the way you were inside, at what point can you finally say it's in the past?\" Would you say that these questions are the same?\nNo", "\"I am a stay at home mom who loves to cook and wants to run a small scale fresh food catering business from my home. What types of licenses do I need?\" \"If I decide to consume zero salt from today, eat only home cooked food, and continue for a month, what will happen to my body at the end of the month?\" Would you say that these questions are the same?\nNo", "\"Is it better to eat large meals that keep you full for a while or eat (healthy) snacks throughout the day?\" \"What's the difference between the two French sentences 'tournez a gauche' and 'tournez vous a gauche'?\" Would you say that these questions are the same?\nNo", "\"\"What is your opinion towards 'A Bite of China'?\"\" \"\"Can the dishes from the program 'A Bite of China' be made?\"\" Would you say that these questions are the same?\nNo", "\"How many calories do I need to eat (2000 is my normal intake) when I start to go road cycling for the first time?\" \"I'm 23 years old and I'm a vegan. I weigh 55 kilos and I'm 6ft tall. How many calories should I be taking daily to get into my ideal BMI range? How do I keep count of my calorie intake every time I eat?\" Would you say that these questions are the same?\nNo", "\"How do I consume food at a buffet in the most efficient way?\" \"\"Should there be a 'Buffet Rule' limiting the number of times a person can return to the buffet for more food?\"\" Would you say that these questions are the same?\nNo", "\"I just started ketogenic diet. I'm losing weight, but once I reach target weight is there a way I can go back into a more normal diet that's not super high carb of course, but perhaps a bowl of rice a day without gaining the weight back?\" \"From existing diet now days what diet gives the best result to lose weight?\" Would you say that these questions are the same?\nNo", "\"Why do children hate vegetables?\" \"I am 18 and hate all fruits and vegetables. I eat strictly meat, grains and Dairy. What will my health be if I continue to eat like this? (currently I am very fit)\" Would you say that these questions are the same?\nNo", "\"Both my grandparents had stroke and cancer and my uncles and aunts have high cholesterol. I only eat junk food. Am I at risk of having a stroke, cancer or heart attack?\" \"Am I eating an unhealthy amount of junk food if I don't come near the RDA for saturated fats and cholesterol?\" Would you say that these questions are the same?\nNo", "\"\"If I eat a little cyanide each day will I become somewhat tolerant to it? Think 'The Princess Bride'\"\" \"Has anyone actually tried eating an apple a day?\" Would you say that these questions are the same?\nNo", "\"Digestion: Why do we taste food sometimes when we burp? Even after hours of eating and sometimes tasting food that wasn't the last thing we ate\" \"When I eat outside food, I sometimes get belching (sour) or burps and I have loose stools sometimes. My endoscopy revealed nothing wrong Whats wrong?\" Would you say that these questions are the same?\nNo", "\"I'm planning to join a gym for weight reduction & increasing my physical stamina. What things should I take care of while training and after that?\" \"What should be my diet plan (non vegetarian or vegetarian) and exercise which I can do to loose weight as I have just joined gym. I'have hypothyroid?\" Would you say that these questions are the same?\nNo", "\"What are some of the ways to establish trust in dealings abroad?\" \"\"In a recent episode of Curb your Enthusiasm Larry orders the 'all you can eat' buffet and then seeks to share some of his food with others who ordered a set meal. What is the strict legal position regarding this?\"\" Would you say that these questions are the same?\nNo", "\"I am 19, I eat a lot but am still underweight. I am too thin and I know that my body build affects my personality. What are a few practical and practicable ways to gain some weight?\" \"I am 19, male and too thin. What things should I do to look good?\" Would you say that these questions are the same?\nNo", "\"Is it safe to eat raw scallops or those that are cooked to medium rare?\" \"On cookery programmes, steaks are always depicted as being cooked perfect when they are rare. I can recall a time when programmes always cooked them either well done or medium. Why the change?\" Would you say that these questions are the same?\nNo", "\"How many grams of sugar should I eat in a day/week if I have cancer?\" \"\"In Clive Barker's 'Cold-Heart Canyon' there is a character named Qwaftzefoni, the son of Lilith. Do you know anything about his name's origins?\"\" Would you say that these questions are the same?\nNo", "\"If a couple visits a 5-star restaurant in the U.S. and finds the food too salty, sweet, etc. and makes a complaint, what does the staff do as a compensation?\" \"If you work at a fast food restaurant in the U.S. and a couple of cops ask you to put something in the food or drink of customer should you obey them?\" Would you say that these questions are the same?\nNo"], "neg_score": [-6.40625, -6.46875, -6.75, -6.8125, -7.09375, -7.71875, -7.71875, -8.1875, -8.9375, -9.3125, -9.375, -9.625, -9.9375, -11.1875, -11.1875, -11.625, -11.9375], "teacher_scores": [-5.375, -5.75, -6.375, -6.40625, -6.46875, -6.75, -6.8125, -7.09375, -7.71875, -7.71875, -8.1875, -8.9375, -9.3125, -9.375, -9.625, -9.9375, -11.1875, -11.1875, -11.625, -11.9375], "task": "icl"}
{"query": "Please answer a question about the following article about Somalis: Xalwo (halva) is a popular confection eaten during festive occasions, such as Eid celebrations or wedding receptions. It is made from sugar, corn starch, cardamom powder, nutmeg powder and ghee. Peanuts are also sometimes added to enhance texture and flavor. After meals, homes are traditionally perfumed using frankincense (lubaan) or incense (cuunsi), which is prepared inside an incense burner referred to as a dabqaad. What is the English word for cunnsi?", "pos": ["Please answer a question about the following article about Cyprus: Seafood and fish dishes include squid, octopus, red mullet, and sea bass. Cucumber and tomato are used widely in salads. Common vegetable preparations include potatoes in olive oil and parsley, pickled cauliflower and beets, asparagus and taro. Other traditional delicacies of are meat marinated in dried coriander seeds and wine, and eventually dried and smoked, such as lountza (smoked pork loin), charcoal-grilled lamb, souvlaki (pork and chicken cooked over charcoal), and sheftalia (minced meat wrapped in mesentery). Pourgouri (bulgur, cracked wheat) is the traditional source of carbohydrate other than bread, and is used to make the delicacy koubes. What is sheftalia?\nminced meat wrapped in mesentery", "Please answer a question about the following article about Alsace: The gastronomic symbol of the région is undoubtedly the Choucroute, a local variety of Sauerkraut. The word Sauerkraut in Alsatian has the form sûrkrût, same as in other southwestern German dialects, and means \"sour cabbage\" as its Standard German equivalent. This word was included into the French language as choucroute. To make it, the cabbage is finely shredded, layered with salt and juniper and left to ferment in wooden barrels. Sauerkraut can be served with poultry, pork, sausage or even fish. Traditionally it is served with Strasbourg sausage or frankfurters, bacon, smoked pork or smoked Morteau or Montbéliard sausages, or a selection of other pork products. Served alongside are often roasted or steamed potatoes or dumplings. What can Sauerkraut be served with?\npoultry, pork, sausage or even fish", "Please answer a question about the following article about Dog: The most popular Korean dog dish is gaejang-guk (also called bosintang), a spicy stew meant to balance the body's heat during the summer months; followers of the custom claim this is done to ensure good health by balancing one's gi, or vital energy of the body. A 19th century version of gaejang-guk explains that the dish is prepared by boiling dog meat with scallions and chili powder. Variations of the dish contain chicken and bamboo shoots. While the dishes are still popular in Korea with a segment of the population, dog is not as widely consumed as beef, chicken, and pork. What is dog meat boiled with to create Gaejang-guk?\nscallions and chili powder."], "pos_scores": [-2.59375, -2.78125, -2.859375], "neg": ["Please answer a question about the following article about Carnival: J'ouvert, or \"Dirty Mas\", takes place before dawn on the Monday (known as Carnival Monday) before Ash Wednesday. It means \"\"opening of the day\". Revelers dress in costumes embodying puns on current affairs, especially political and social events. \"Clean Mud\" (clay mud), oil paint and body paint are familiar during J'ouvert. A common character is \"Jab-jabs\" (devils, blue, black or red) complete with pitchfork, pointed horns and tails. A King and Queen of J'ouvert are chosen, based on their witty political/social messages. How are the King and Queen of J'ouvert chosen?\nbased on their witty political/social messages", "Please answer a question about the following article about Armenia: Armenian cuisine is as ancient as the history of Armenia, a combination of different tastes and aromas. The food often has quite a distinct aroma. Closely related to eastern and Mediterranean cuisine, various spices, vegetables, fish, and fruits combine to present unique dishes. The main characteristics of Armenian cuisine are a reliance on the quality of the ingredients rather than heavily spicing food, the use of herbs, the use of wheat in a variety of forms, of legumes, nuts, and fruit (as a main ingredient as well as to sour food), and the stuffing of a wide variety of leaves. What different uses does fruit have in Armenian food?\nmain ingredient as well as to sour food", "Please answer a question about the following article about Mali: Efforts have been made to improve nutrition, and reduce associated health problems, by encouraging women to make nutritious versions of local recipes. For example, the International Crops Research Institute for the Semi-Arid Tropics (ICRISAT) and the Aga Khan Foundation, trained women's groups to make equinut, a healthy and nutritional version of the traditional recipe di-dèguè (comprising peanut paste, honey and millet or rice flour). The aim was to boost nutrition and livelihoods by producing a product that women could make and sell, and which would be accepted by the local community because of its local heritage. What ingredients are in both the traditional and nutritional version of this dish?\npeanut paste, honey and millet or rice flour", "Please answer a question about the following article about Alsace: The gastronomic symbol of the région is undoubtedly the Choucroute, a local variety of Sauerkraut. The word Sauerkraut in Alsatian has the form sûrkrût, same as in other southwestern German dialects, and means \"sour cabbage\" as its Standard German equivalent. This word was included into the French language as choucroute. To make it, the cabbage is finely shredded, layered with salt and juniper and left to ferment in wooden barrels. Sauerkraut can be served with poultry, pork, sausage or even fish. Traditionally it is served with Strasbourg sausage or frankfurters, bacon, smoked pork or smoked Morteau or Montbéliard sausages, or a selection of other pork products. Served alongside are often roasted or steamed potatoes or dumplings. What is Sauerkraut typically served with in Alsace?\nStrasbourg sausage or frankfurters, bacon, smoked pork or smoked Morteau or Montbéliard sausages", "Please answer a question about the following article about Sumer: The Sumerians were one of the first known beer drinking societies. Cereals were plentiful and were the key ingredient in their early brew. They brewed multiple kinds of beer consisting of wheat, barley, and mixed grain beers. Beer brewing was very important to the Sumerians. It was referenced in the Epic of Gilgamesh when Enkidu was introduced to the food and beer of Gilgamesh's people: \"Drink the beer, as is the custom of the land... He drank the beer-seven jugs! and became expansive and sang with joy!\" What kinds of beer did the Sumerians brew?\nwheat, barley, and mixed grain", "Please answer a question about the following article about Portugal: Portuguese cuisine is diverse. The Portuguese consume a lot of dry cod (bacalhau in Portuguese), for which there are hundreds of recipes. There are more than enough bacalhau dishes for each day of the year. Two other popular fish recipes are grilled sardines and caldeirada, a potato-based stew that can be made from several types of fish. Typical Portuguese meat recipes, that may be made out of beef, pork, lamb, or chicken, include cozido à portuguesa, feijoada, frango de churrasco, leitão (roast suckling pig) and carne de porco à alentejana. A very popular northern dish is the arroz de sarrabulho (rice stewed in pigs blood) or the arroz de cabidela (rice and chickens meat stewed in chickens blood). What are two popular Northern Portugal dishes?\narroz de sarrabulho (rice stewed in pigs blood) or the arroz de cabidela (rice and chickens meat stewed in chickens blood)", "Please answer a question about the following article about Guinea-Bissau: Common dishes include soups and stews. Common ingredients include yams, sweet potato, cassava, onion, tomato and plantain. Spices, peppers and chilis are used in cooking, including Aframomum melegueta seeds (Guinea pepper). What are common ingredients in Guinea-Bissau?\nyams, sweet potato, cassava, onion, tomato and plantain", "Please answer a question about the following article about Somalis: Due to Somalia's proximity to and close ties with the Arabian Peninsula, many Somali men also wear the jellabiya (jellabiyad or qamiis in Somali), a long white garment common in the Arab world. What is the jellabiya?\na long white garment", "Please answer a question about the following article about Armenia: Yerevan Vernissage (arts and crafts market), close to Republic Square, bustles with hundreds of vendors selling a variety of crafts on weekends and Wednesdays (though the selection is much reduced mid-week). The market offers woodcarving, antiques, fine lace, and the hand-knotted wool carpets and kilims that are a Caucasus specialty. Obsidian, which is found locally, is crafted into assortment of jewellery and ornamental objects. Armenian gold smithery enjoys a long tradition, populating one corner of the market with a selection of gold items. Soviet relics and souvenirs of recent Russian manufacture – nesting dolls, watches, enamel boxes and so on – are also available at the Vernisage. What types of crafts can be purchased at Vernissage?\nwoodcarving, antiques, fine lace, and the hand-knotted wool carpets and kilims", "Please answer a question about the following article about Estonia: Historically, the cuisine of Estonia has been heavily dependent on seasons and simple peasant food, which today is influenced by many countries. Today, it includes many typical international foods.[citation needed] The most typical foods in Estonia are black bread, pork, potatoes, and dairy products. Traditionally in summer and spring, Estonians like to eat everything fresh – berries, herbs, vegetables, and everything else that comes straight from the garden. Hunting and fishing have also been very common, although currently hunting and fishing are enjoyed mostly as hobbies. Today, it is also very popular to grill outside in summer. What are the most common foods in Estonia?\nblack bread, pork, potatoes, and dairy products.", "Please answer a question about the following article about Myanmar: Mohinga is the traditional breakfast dish and is Myanmar's national dish. Seafood is a common ingredient in coastal cities such as Sittwe, Kyaukpyu, Mawlamyaing (formerly Moulmein), Mergui (Myeik) and Dawei, while meat and poultry are more commonly used in landlocked cities like Mandalay. Freshwater fish and shrimp have been incorporated into inland cooking as a primary source of protein and are used in a variety of ways, fresh, salted whole or filleted, salted and dried, made into a salty paste, or fermented sour and pressed. What are popular ways that fish are used in Burma ?\nfresh, salted whole or filleted, salted and dried, made into a salty paste, or fermented sour and pressed.", "Please answer a question about the following article about Rajasthan: Spirit possession has been documented in modern Rajasthan. Some of the spirits possessing Rajasthanis are seen as good and beneficial while others are seen as malevolent. The good spirits include murdered royalty, the underworld god Bhaironji, and Muslim saints. Bad spirits include perpetual debtors who die in debt, stillborn infants, deceased widows, and foreign tourists. The possessed individual is referred to as a ghorala (\"mount\"). Possession, even if it is by a benign spirit, is regarded as undesirable, as it entails loss of self-control and violent emotional outbursts. What results from possession by even benign spirits?\nloss of self-control and violent emotional outbursts", "Please answer a question about the following article about Armenia: Armenian cuisine is as ancient as the history of Armenia, a combination of different tastes and aromas. The food often has quite a distinct aroma. Closely related to eastern and Mediterranean cuisine, various spices, vegetables, fish, and fruits combine to present unique dishes. The main characteristics of Armenian cuisine are a reliance on the quality of the ingredients rather than heavily spicing food, the use of herbs, the use of wheat in a variety of forms, of legumes, nuts, and fruit (as a main ingredient as well as to sour food), and the stuffing of a wide variety of leaves. What does Armenian cuisine use to create its distinctive dishes?\nvarious spices, vegetables, fish, and fruits", "Please answer a question about the following article about Carnival: The most famous groups are the chirigotas, choirs and comparsas. The chirigotas are well known witty, satiric popular groups who sing about politics, new times and household topics, wearing the same costume, which they prepare for the whole year. The Choirs (coros) are wider groups that go on open carts through the streets singing with an orchestra of guitars and lutes. Their signature piece is the \"Carnival Tango\", alternating comical and serious repertory. The comparsas are the serious counterpart of the chirigota in Cádiz, and the poetic lyrics and the criticism are their main ingredients. They have a more elaborated polyphony that is easily recognizable by the typical countertenor voice. What do the chirigotas sing about?\npolitics, new times and household topics", "Please answer a question about the following article about Carnival: Traditionally the feast also applied to sexual desires, which were supposed to be suppressed during the following fasting. Before Lent began, all rich food and drink were consumed in what became a giant celebration that involved the whole community, and is thought to be the origin of Carnival. The Lenten period of the Liturgical calendar, the six weeks directly before Easter, was originally marked by fasting and other pious or penitential practices. During Lent, no parties or celebrations were held, and people refrained from eating rich foods, such as meat, dairy, fat and sugar. What type of rich foods did people refrain from eating during Lent?\nmeat, dairy, fat and sugar", "Please answer a question about the following article about Somalis: When not dressed in Westernized clothing such as jeans and t-shirts, Somali men typically wear the macawis, which is a sarong-like garment worn around the waist. On their heads, they often wrap a colorful turban or wear the koofiyad, an embroidered fez. Where is the macawis worn?\naround the waist", "Please answer a question about the following article about Nutrition: In 1896, Eugen Baumann observed iodine in thyroid glands. In 1897, Christiaan Eijkman worked with natives of Java, who also suffered from beriberi. Eijkman observed that chickens fed the native diet of white rice developed the symptoms of beriberi but remained healthy when fed unprocessed brown rice with the outer bran intact. Eijkman cured the natives by feeding them brown rice, discovering that food can cure disease. Over two decades later, nutritionists learned that the outer rice bran contains vitamin B1, also known as thiamine. What simple alternative food prevented the development of beriberi in chickens?\nunprocessed brown rice with the outer bran intact"], "neg_score": [-3.046875, -3.171875, -3.1875, -3.34375, -3.46875, -3.46875, -3.59375, -3.640625, -3.65625, -3.6875, -3.71875, -3.78125, -4.0, -4.03125, -4.125, -4.15625, -4.375], "teacher_scores": [-2.59375, -2.78125, -2.859375, -3.046875, -3.171875, -3.1875, -3.34375, -3.46875, -3.46875, -3.59375, -3.640625, -3.65625, -3.6875, -3.71875, -3.78125, -4.0, -4.03125, -4.125, -4.15625, -4.375], "task": "icl"}
{"query": "2015 Played 4 Won 1 ( Melbourne ) , Lost 3 ( Canberra , Gold Coast , Penrith ) Played 4 Won 1 ( Melbourne ) , 3 lost ( Canberra , Gold Coast , Penrith ) Do these sentences mean the same thing?", "pos": ["The Final FESPIC Games had 18 venues for the games . 9 in Kuala Lumpur , 7 in Selangor and 1 each in Putrajaya and Negeri Sembilan respectively . The Final FESPIC Games had 18 venues for the Games , 9 in Kuala Lumpur , 7 in Selangor and 1 in Putrajaya and Negeri Sembilan each . Do these sentences mean the same thing?\nYes", "The Final FESPIC Games had 18 venues for the games . 9 in Kuala Lumpur , 7 in Selangor and 1 each in Putrajaya and Negeri Sembilan respectively . The Final FESPIC Games had 18 venues for the Games , 9 in Kuala Lumpur , 7 in Selangor and each 1 in Putrajaya and Negeri Sembilan . Do these sentences mean the same thing?\nYes", "It carried out services between the following centres : Townsville , Bundaberg , Gladstone , Rockhampton , Mackay , Brisbane , Blackwater , Thangool , Coolangatta and Newcastle . It conducted services between the following centres : Townsville , Bundaberg , Gladstone , Rockhampton , Mackay , Brisbane , Blackwater , Thangool , Coolangatta and Newcastle . Do these sentences mean the same thing?\nYes"], "pos_scores": [-6.9375, -7.0, -7.5625], "neg": ["In 2012 , he returned to Canada to play with London City in the Canadian Soccer League.He went to Serbia to play with Srem Jakovo , and Jedinstvo Surčin . He returned to Canada in 2012 to play with London City in the Canadian Soccer League , where he went to Serbia to play with Srem Jakovo and Jedinstvo Surčin . Do these sentences mean the same thing?\nYes", "It conducted services between the following centres : Brisbane , Bundaberg , Gladstone , Rockhampton , Mackay , Townsville , Blackwater , Thangool , Coolangatta and Newcastle . It provided services between the following centres : Brisbane , Bundaberg , Gladstone , Rockhampton , Mackay , Townsville , Blackwater , Thangool , Coolangatta and Newcastle . Do these sentences mean the same thing?\nYes", "It carried out services between the following centres : Townsville , Bundaberg , Gladstone , Rockhampton , Mackay , Brisbane , Blackwater , Thangool , Coolangatta and Newcastle . It provided services between the following centres : Brisbane , Bundaberg , Gladstone , Rockhampton , Mackay , Townsville , Blackwater , Thangool , Coolangatta and Newcastle . Do these sentences mean the same thing?\nYes", "He also returned to the Auckland Rugby League competition and then played for the Auckland Lions at Bartercard Cup level before being signed by the New Zealand Warriors . He also returned to the Auckland Rugby League competition and then played for the Auckland Lions at the Bartercard Cup Level before being signed by the New Zealand Warriors . Do these sentences mean the same thing?\nYes", "It conducted services between the following centres : Townsville , Bundaberg , Gladstone , Rockhampton , Mackay , Brisbane , Blackwater , Thangool , Coolangatta and Newcastle . It provided services between the following centres : Brisbane , Bundaberg , Gladstone , Rockhampton , Mackay , Townsville , Blackwater , Thangool , Coolangatta and Newcastle . Do these sentences mean the same thing?\nYes", "Along the southern Australian coast , it is found from Shark Bay in Western Australia to Maroochydore in Queensland , including Tasmania . Along the southern Australian coast , it is found from Shark Bay in West Australia to Maroochydore in Queensland , including Tasmania . Do these sentences mean the same thing?\nYes", "They beat Berkshire 21 -- 15 , then Oxfordshire 21 -- 32 and Buckinghamshire 0 -- 36 before losing 58 -- 10 to Gloucestershire in the quarter finals . They defeated Berkshire 21 -- 15 , then Oxfordshire 21 -- 32 and Buckinghamshire 0 -- 36 , before losing to Gloucestershire in the quarter-finals 58 -- 10 . Do these sentences mean the same thing?\nYes", "He also returned to the Auckland Rugby League contest and then played for the Auckland Lions at the Bartercard Cup Level before being contracted by the New Zealand Warriors . He also returned to the Auckland Rugby League competition and then played for the Auckland Lions at Bartercard Cup level before being signed by the New Zealand Warriors . Do these sentences mean the same thing?\nYes", "It conducted services between the following centres : Brisbane , Bundaberg , Gladstone , Rockhampton , Mackay , Townsville , Blackwater , Thangool , Coolangatta and Newcastle . It carried out services between the following centres : Townsville , Bundaberg , Gladstone , Rockhampton , Mackay , Brisbane , Blackwater , Thangool , Coolangatta and Newcastle . Do these sentences mean the same thing?\nYes", "They beat Berkshire 21 -- 15 , then Buckinghamshire 21 -- 32 and Oxfordshire 0 -- 36 before losing 58 -- 10 to Gloucestershire in the quarter finals . They beat Berkshire 21 -- 15 , then Buckinghamshire 21 -- 32 and Oxfordshire 0 -- 36 , before losing against Gloucestershire in the quarter-finals 58 -- 10 . Do these sentences mean the same thing?\nYes", "They beat Berkshire 21 -- 15 , then Oxfordshire 21 -- 32 and Buckinghamshire 0 -- 36 , before losing against Gloucestershire in the quarter-finals 58 -- 10 . They beat Berkshire 21 -- 15 , then Oxfordshire 21 -- 32 and Buckinghamshire 0 -- 36 before losing 58 -- 10 to Gloucestershire in the quarter finals . Do these sentences mean the same thing?\nYes", "Then he returned to the Auckland Rugby League competition and played for the Auckland Lions at the Bartercard Cup Level before being signed by the New Zealand Warriors . He then returned to the Auckland Rugby League competition and also played for the Auckland Lions at Bartercard Cup level before being signed by the New Zealand Warriors . Do these sentences mean the same thing?\nYes", "He then returned to the Auckland Rugby League competition and also played for the Auckland Lions at Bartercard Cup level before being signed by the New Zealand Warriors . He then returned to the Auckland Rugby League competition and played for Auckland Lions at the Bartercard Cup Level before being signed by the New Zealand Warriors . Do these sentences mean the same thing?\nYes", "They beat Berkshire 21 -- 15 , then Buckinghamshire 21 -- 32 and Oxfordshire 0 -- 36 before losing 58 -- 10 to Gloucestershire in the quarter finals . They beat Berkshire 21 -- 15 , then Buckinghamshire 21 -- 32 and Oxfordshire 0 -- 36 , before losing against Gloucestershire in quarter finals 58 -- 10 . Do these sentences mean the same thing?\nYes", "In 2012 , he went to Canada to play with London City in the Canadian Soccer League , where he returned to Serbia to play with Srem Jakovo and Jedinstvo Surčin . In 2012 , he went to Canada to play with London City in the Canadian Soccer League.He returned to Serbia to play with Srem Jakovo , and Jedinstvo Surčin . Do these sentences mean the same thing?\nYes", "In 2012 , he went to Canada to play with London City in the Canadian Soccer League.He returned to Serbia to play with Srem Jakovo , and Jedinstvo Surčin . In 2012 , he went to Canada to play with London City in the Canadian Soccer League , returning to Serbia to play with Srem Jakovo and Jedinstvo Surčin . Do these sentences mean the same thing?\nYes", "In 2013 , he reached his first victory at Rd.5 in the Huis Ten Bosch , and was also the third win for a Toyota 86 in the series . In 2013 , he earned his first win at Rd.5 in Huis Ten Bosch . This was also the third victory for a Toyota 86 in the series . Do these sentences mean the same thing?\nYes"], "neg_score": [-7.59375, -7.625, -7.625, -7.65625, -7.65625, -7.78125, -7.8125, -7.84375, -7.84375, -7.90625, -7.9375, -7.96875, -8.0625, -8.0625, -8.25, -8.75, -11.0625], "teacher_scores": [-6.9375, -7.0, -7.5625, -7.59375, -7.625, -7.625, -7.65625, -7.65625, -7.78125, -7.8125, -7.84375, -7.84375, -7.90625, -7.9375, -7.96875, -8.0625, -8.0625, -8.25, -8.75, -11.0625], "task": "icl"}
{"query": "\"What are the best institutes to learn German language in Pune?\" \"Which is the best institute to learn the German language in Pune?\" Would you say that these questions are the same?", "pos": ["\"Which is the best coaching institute for the GMAT?\" \"Which is the best coaching for GMAT? Is it Byju’s, Jamboree, Manya Princeton, Pythagurus, CrackVerbal, Meritnation or EduShastra?\" Would you say that these questions are the same?\nYes", "\"I am entering the world of video game programming and want to know what language I should learn? Because there are so many languages ​​I do not know which one to start with. Can you recommend a language that's easy to learn and can be used with many platforms?\" \"What programming languages should I learn for video game development?\" Would you say that these questions are the same?\nYes", "\"Is it possible to learn all the major programming languages? Or is it advisable to learn a few and master in them?\" \"Say I'm dedicated and only have internet access at my disposal, how long would it take for me to learn all the major programming languages?\" Would you say that these questions are the same?\nYes"], "pos_scores": [-7.40625, -7.53125, -7.65625], "neg": ["\"As a beginner, which programming language should one learn at first and what steps should one take after learning that first language?\" \"What programming language would be ideal to learn first for a beginner?\" Would you say that these questions are the same?\nYes", "\"If you could learn things like Neo in The Matrix, what is the one skill you would learn?\" \"If you could learn any one skill in the world without trying (like Matrix learning style), which would you pick?\" Would you say that these questions are the same?\nYes", "\"I am a 13-year-old boy that wants to learn how to program video games. What programming languages should I learn? How do I get started?\" \"I am entering the world of video game programming and want to know what language I should learn? Because there are so many languages ​​I do not know which one to start with. Can you recommend a language that's easy to learn and can be used with many platforms?\" Would you say that these questions are the same?\nYes", "\"What is the best way to learn a foreign language? (Assuming that you are already in that foreign country and that country doesn't speak or understand even a single English word.)\" \"What is the best way to learn any new foreign language?\" Would you say that these questions are the same?\nYes", "\"I am entering the world of video game programming and want to know what language I should learn? Because there are so many languages ​​I do not know which one to start with. Can you recommend a language that's easy to learn and can be used with many platforms?\" \"Which programming language should I learn if I am interested in game development?\" Would you say that these questions are the same?\nYes", "\"What is the best place to learn Spanish online?\" \"\"If DuoLingo didn't 'take' for me, what's the best place online to learn Spanish?\"\" Would you say that these questions are the same?\nYes", "\"Does it carry any extra weight in an MBA interview or a job interview in engineering/B-schools if you know a foreign language, like German or French?\" \"Will you get any extra points in an MBA interview or a job interview in engineering/B-schools if you know a foreign language, like German or French?\" Would you say that these questions are the same?\nYes", "\"I am interested in a fashion designing programme. Which are the best institutes in Pune?\" \"Which is the best institute for lern fashion designing in Pune which will give me scope to my skill while I am woking?\" Would you say that these questions are the same?\nYes", "\"Which programming language should a beginner start with, who wants to learn programming from scratch (even the languages that aren't in use now)?\" \"Which programming language to start with for a beginner and what are steps necessary to learn it?\" Would you say that these questions are the same?\nYes", "\"I am not a university student but I would like to go to France and take French Language courses. What is the best way to do this?\" \"What would be the best French language teaching school?\" Would you say that these questions are the same?\nYes", "\"I know the basic pragramming languages (like C and C++). What should I study next? Java or Python?\" \"I only know the C programming language as of yet. Which language should I study next, Java, Ruby, or Python?\" Would you say that these questions are the same?\nYes", "\"Which are top rated institutions for CCNA, CCNP & CCIE as well within Pune only?\" \"Which are the top-rated institutions for the CCNA, the CCNP and the CCIE in Pune only?\" Would you say that these questions are the same?\nYes", "\"How do you learn to speak a foreign language?\" \"What is the best way to learn a foreign language? (Assuming that you are already in that foreign country and that country doesn't speak or understand even a single English word.)\" Would you say that these questions are the same?\nYes", "\"I am a Nepalese student. Can I study B.Sc physics from Calcutta university and PhD in astrophysicsfrom IIA, Banglore or NCRA-TIFR Pune?\" \"I am a Nepalese student. Can I study B.Sc physics from Calcutta university and PhD in astrophysics from IIA, Banglore or NCRA-TIFR Pune?\" Would you say that these questions are the same?\nYes", "\"\"I want to learn about the 'Internet of things.' What is the best book to learn more?\"\" \"What are the best resources to learn about the Internet of Things (IoT)?\" Would you say that these questions are the same?\nYes", "\"Hi everyone. I'm a UPSC aspirant. Want to know best history optional coaching in Delhi-Chandigarh region. Thanks :)\" \"Which is the best coaching center for history optional in delhi?\" Would you say that these questions are the same?\nYes", "\"I'm a 16-year-old and I know HTML and CSS and Python. What language should I learn next?\" \"I know some Python, HTML, CSS, and JS, what languages should I learn next?\" Would you say that these questions are the same?\nYes"], "neg_score": [-7.96875, -8.0, -8.0625, -8.0625, -8.3125, -8.3125, -8.4375, -8.4375, -8.5, -8.625, -8.6875, -8.6875, -8.9375, -9.375, -9.625, -9.625, -9.6875], "teacher_scores": [-7.40625, -7.53125, -7.65625, -7.96875, -8.0, -8.0625, -8.0625, -8.3125, -8.3125, -8.4375, -8.4375, -8.5, -8.625, -8.6875, -8.6875, -8.9375, -9.375, -9.625, -9.625, -9.6875], "task": "icl"}
{"query": "Everything, Everything (novel) -- Maddy's mother, after therapy, reveals that right after Maddy's father (a police officer) and her brother died, Maddy got very sick, and her mother, not wanting to lose her, decided she had SCID, and needed to be kept away from the world. In the end, Maddy and Olly happily reunite in New York, where she sent him on a mini scavenger hunt in a used bookstore. Can we conclude that does maddie die at the end of everything everything?", "pos": ["Humphrey Goodman -- Goodman is assigned to Saint Marie after the murder of D.I. Richard Poole at the start of Series 3. Clues from Poole's investigation helped Goodman reveal the motive and the killer's identity; Goodman commented that Poole had 'solved his own murder.' Goodman stayed on in Saint Marie after his wife Sally announced she would not be joining him on the Caribbean island. He became the chief inspector on the island, and took to Poole's old habit of announcing the murderer in front of all the suspects and his police team. He is very clumsy, often forgetting things or finding himself with nothing to take notes on; he embraces Caribbean life much more than his predecessor. He has a knack at being able to solve murders instantly, looking at the meaning of small details, much like his predecessor. He fell in love with his detective sergeant, Camille Bordey, often coming close to revealing his feelings for her. He tried to stop her leaving when she requested a job in Paris, but conceded. He shared a passionate kiss with her just before she left the island. Her successor, Florence Cassell, also managed to get along with Goodman, often sharing jokes and they dedicate a drink to Camille after their first solved case. Can we conclude that death in paradise do camille and humphrey get together?\nNo", "April Kepner -- In seasons 13 and 14, April faces a crisis of faith as she begins to believe that good people get punished and bad people get good things. She does this after treating 3 seemingly simple patients who are good people and die. Including Matthew's (her ex finacee) new pregnant wife, after delivery. Robbins then tells April it's her fault. As a result, she goes into a dark place and uses partying and sex to mask her deep-rooted pain. She earns the nickname ``The Party'' by the new interns. She refuses to let Jackson help her through this time. However, mid-Season 14, she encounters a terminal patient who helps April reaffirm her faith. April starts seeing Matthew again and their relationship is made public when the two are involved in a car accident, where April almost dies of hypothermia. In the season finale, April and Matthew get married. Can we conclude that do jackson and april get back together after their divorce?\nNo", "Because of Winn-Dixie (film) -- Opal gets a job at Gertrude's Pets and befriends a worker there, Otis, a shy ex-convict with a passion for music. She also meets a girl named Sweetie Pie Thomas, who is eager to get a dog like Winn-Dixie. Later, a thunderstorm comes and Winn-Dixie, being pathologically afraid of thunderstorms, runs away. While Opal looks for him, her father wants to give up and she blames him for the loss of her mother and Winn-Dixie running away. But her father explains that he tried very hard to look for her mother. He then admits that he believes that she is never coming back. Later they go back to a party and Otis starts to sing a song on his guitar. Winn-Dixie is heard outside howling along to the song. Everyone, while singing, lets him in and welcomes him back. Can we conclude that does the dog die in because of winn dixie?\nNo"], "pos_scores": [-6.53125, -6.59375, -7.0], "neg": ["Betty Draper -- Betty goes to Bobby's summer camp for a family weekend in ``The Better Half'', driving down without Henry. Don, also on his way to the camp, sees the newly svelte Betty lost at a gas station, and they go down to the campground together. They spend the afternoon with Bobby, and everyone has a wonderful time. That night Don visits Betty's cabin, and they share a drink, reminiscing about the early years of their marriage and the kids. Don accepts Betty's tacit invitation to enter her cabin, and they make love. Betty and Don talk afterward, and Betty admits that she's happy with Henry, is no longer as mad at Don as she once was, and feels sorry for Megan, who doesn't know that loving Don is the worst way of getting to him. The next morning Don wakes up alone and goes down to the cafeteria, where he sees Betty and Henry eating together. Don says hello to them and goes off to eat, alone, at the other side of the room. Can we conclude that do don and betty draper get back together?\nNo", "List of The Mortal Instruments characters -- Clary learns that Valentine Morgenstern, the main antagonist of the series, is her biological father and her mother's ex-husband. At the end of City of Bones, Valentine tells them that Clary and Jace are siblings -- which, they discover later in the series, is a lie. In the second book of the series, City of Ashes, Clary dates her best friend Simon, who has for a long time had a crush on her, in order to forget Jace during the torturous time of believing him to be her brother. She is told by the Seelie Queen that she has the ability to create runes that don't exist; but they do exist, which later enables her to destroy Valentine's ship using the ``Open'' rune. In the end of the second novel, Clary finds that an old friend of her mother, Madeleine, knows how to wake her mother, who has been in a magical coma since the first book. Can we conclude that is clary and jace brother and sister shadowhunters?\nNo", "The Vow (2012 film) -- As seasons change, Leo discusses his philosophy about ``Moments of impact.'' ``A moment of impact whose potential for change has ripple effects far beyond what we can predict. Sending some particles crashing together, making them closer than before. While sending others, spinning off into great ventures, landing where you never thought you'd find them...'' Back in her room, Paige finds the menu card on which she had written her wedding vows and is deeply moved. The movie ends with Paige finding Leo at their regular Café Mnemonic and going with him to try a new place instead of their old haunts. Can we conclude that does paige remember leo at the end of the vow?\nNo", "Bethany Platt -- Bethany is the daughter of Sarah Platt (Tina O'Brien) and her classmate Neil Fearns (Paul Holowaty), half-sister of Billy and Harry Platt, granddaughter of Gail (Helen Worth) and Martin Platt (Sean Wilson) and great-granddaughter of Audrey Roberts (Sue Nicholls). Bethany's storylines have included her taking ecstasy tablets when her uncle, David Platt (Jack P. Shepherd) hid them in a toy. Bethany departed with Weatherfield with her mother Sarah for a new life in Milan. Her return storyline saw her running away from home in Milan back to Weatherfield, and since then her storylines have included her infatuation with Callum Logan (Sean Ward) as well as being caught up with her family's feud with Callum, being the victim of bullying, developing a crush on her mother's boyfriend Gary Windass (Mikey North), falling for older man Nathan Curtis (Christopher Harper) and being involved in a highly-publicised child grooming and sexual exploitation storyline. Can we conclude that is todd bethany's dad in coronation street?\nNo", "Sophia Peletier -- In the comics, Sophia is a member of the Atlanta refugee camp and becomes close friends with Carl, with whom she spends most her time - eventually becoming his girlfriend. Later, after her mother's suicide, she begins viewing Maggie and Glenn as her parents while remaining close to Carl. Although, because of the overrun world around them and their conflicting personalities, Sophia chooses to break up with Carl, though they still remain close friends and Carl is shown to be extremely protective of her. Sophia also becomes a resident of the Alexandria Safe-Zone and later the Hilltop Colony. She is the comic's longest surviving female character. Can we conclude that is sophia dead in the walking dead comics?\nNo", "Benji (1974 film) -- Returning to the crime scene, he snatches Riley's first ransom note and is grabbed by Mitch. Tiffany rushes out and bites him and gets a vicious kick in return; she is not killed, but her leg is sore and bruised. Benji runs home where he finds that Linda has preceded him in an attempt to cut off his efforts. She snatches the note from Benji and puts it in her purse. He growls and barks, and Mary berates him and carries him away, but he bites her and lunges at Linda, causing the note to fall out. Mary reads it and finally wakes up to what Benji has been on about and rushes it to Dr. Chapman, who demands to know where his children are, and she breaks down in tears. Benji leads the police, the FBI, Dr. Chapman, and Mary back to the hideout. Meanwhile, the kidnappers are concerned that Linda has not returned, and Henry and Riley argue that they should leave. As they walk outside, however, the police hold them at gunpoint and the children are reunited with their father and Mary. Their father is so proud of the dogs that he says they can stay with them permanently, much to their and the children's delight. Can we conclude that does the dog die in the movie benji?\nNo", "Benji (1974 film) -- Returning to the crime scene, he snatches Riley's first ransom note and is grabbed by Mitch. Tiffany rushes out and bites him and gets a vicious kick in return; she is not killed, but her leg is sore and bruised. Benji runs home where he finds that Linda has preceded him in an attempt to cut off his efforts. She snatches the note from Benji and puts it in her purse. He growls and barks, and Mary berates him and carries him away, but he bites her and lunges at Linda, causing the note to fall out. Mary reads it and finally wakes up to what Benji has been on about and rushes it to Dr. Chapman, who demands to know where his children are, and she breaks down in tears. Benji leads the police, the FBI, Dr. Chapman, and Mary back to the hideout. Meanwhile, the kidnappers are concerned that Linda has not returned, and Henry and Riley argue that they should leave. As they walk outside, however, the police hold them at gunpoint and the children are reunited with their father and Mary. Their father is so proud of the dogs that he says they can stay with them permanently, much to their and the children's delight. Can we conclude that does the dog die at the end of benji?\nNo", "Preston Burke -- In season ten, Preston is seen to be living in Zürich, where he runs a privately owned cardiothoracic research hospital. He invited Cristina to the hospital to give a speech on her research. Cristina is both shocked and angered by the sight of Preston, and the former couple exchange bitter sentiments. Cristina claims that the two would never have worked out because she wanted to emulate him, not be by his side. Preston then reveals his ulterior motive for bringing Cristina to Switzerland: to ask her to take over his hospital, which she accepts. He is married to an Italian woman, Edra, and has two daughters, Simone and Vivianna. They are on the verge of moving to Milan. Can we conclude that do cristina and burke ever get back together?\nNo", "Blanche DuBois -- The night Stella goes into labor, Stanley and Blanche are left alone in the apartment, and Stanley, drunk and powerful, rapes her. This event, coupled with the fact that Stella does not believe her, sends Blanche over the edge into a nervous breakdown. In the final scene, Blanche is led off to a mental hospital by a matron and a kind-hearted doctor. After a brief struggle, Blanche smilingly acquiesces as she loses all contact with reality, addressing the doctor with the most famous line in the play: ``I have always depended on the kindness of strangers.'' Can we conclude that does blanche die in a streetcar named desire?\nNo", "The Book of Henry -- Susan slips away from the talent show and puts the plan into motion. As she is about to pull the trigger, however, she realizes that Henry's plan, though ingenious, is the construct of a child, and that she must act as an adult. She immediately confronts Glenn and tells him that she is going to expose him for what he has done to Christina. Glenn replies that everyone will believe him, not her, and he tells her that he is going to call his police chief to come and bring her in. At the same time, affected by Christina's dance performance at the talent show, Principal Wilder decides to follow through on the abuse accusation and contacts the authorities. Glenn returns home, calls his relative at Social Services, and learns that an investigation is being opened into his abuse of Christina. As the police arrive at his house, Glenn kills himself. Susan legally adopts Christina as her daughter. She also finishes writing one of her children's books, as Henry had urged her to do. Can we conclude that does christina die in the book of henry?\nNo", "The Hand That Rocks the Cradle (film) -- Later, Mrs. Mott breaks into the house and lures Michael down to the basement where she knocks him down the stairs and breaks his legs with a shovel. Mrs. Mott then attempts to take Emma and Joey, but after seeing Mrs. Mott assault her mother, Emma locks Mrs. Mott in the nursery. Mrs. Mott escapes, and hears Joey in the attic. She enters and sees Solomon aiding the children's escape. Claire enters and Mrs. Mott attempts to kill her, but stops after Claire appears to be having another asthma attack, allowing Mrs. Mott to mock her. As Mrs. Mott tries to take Joey, Claire gets back up, having faked her asthma attack, and pushes Mrs. Mott out of the window, impaling her on the picket fence and killing her. Touched at how Solomon risked his life to protect her family, Claire welcomes him back into their lives as they leave the attic while the police and paramedics arrive at the house. Can we conclude that does solomon died in the hand that rocks the cradle?\nNo", "Tibby Rollins -- Stuck at home in Bethesda, Maryland while her three friends went to Greece, Mexico and South Carolina, she decides to make a ``suckumentary'' about people she considers to be ``lame,'' among them her coworkers at Wallman's and Brian McBrian, a fellow high schooler who spends most of his time playing Dragon Master at the local 7-Eleven. At Wallman's, Tibby meets twelve-year-old Bailey Graffman, who is dying of leukemia and is eager to help Tibby with her movie. Bailey, afraid of not having enough time to truly understand people and, in turn, be understood, looks beyond appearances and finds the true worth in each of Tibby's documentary subjects, allowing Tibby to do the same. Bailey dies at the end of the novel and Tibby decided to make the documentary about the summer she had with Bailey Can we conclude that does tibby die in the sisterhood of the traveling pants?\nNo", "The Book of Henry -- Susan slips away from the talent show and puts the plan into motion. As she is about to pull the trigger, however, she realizes that Henry's plan, though ingenious, is the construct of a child, and that she must act as an adult. She immediately confronts Glenn and tells him that she is going to expose him for what he has done to Christina. Glenn replies that everyone will believe him, not her, and he tells her that he is going to call his police chief to come and bring her in. At the same time, affected by Christina's dance performance at the talent show, Principal Wilder decides to follow through on the abuse accusation and contacts the authorities. Glenn returns home, calls his relative at Social Services, and learns that an investigation is being opened into his abuse of Christina. As the police arrive at his house, Glenn kills himself. Susan legally adopts Christina as her daughter. She also finishes writing one of her children's books, as Henry had urged her to do. Can we conclude that does she kill glenn in the book of henry?\nNo", "Preston Burke -- In season ten, Preston is seen to be living in Zürich, where he runs a privately owned cardiothoracic research hospital. He invited Cristina to the hospital to give a speech on her research. Cristina is both shocked and angered by the sight of Preston, and the former couple exchange bitter sentiments. Cristina claims that the two would never have worked out because she wanted to emulate him, not be by his side. Preston then reveals his ulterior motive for bringing Cristina to Switzerland: to ask her to take over his hospital, which she accepts. He is married to an Italian woman, Edra, and has two daughters, Simone and Vivianna. They are on the verge of moving to Milan. Can we conclude that do cristina and burke get back together after the wedding?\nNo", "The Impossible (2012 film) -- While Maria is in surgery, her medical chart is mixed-up with another patient who has died. Lucas returns to find his mother's bed empty and is then taken to a tent where children without families are kept safe. The mistake is discovered when Lucas cannot identify any of the dead woman's jewelry and he is subsequently reunited with his mother who has been moved to a private room in the ICU. In the hospital while he waits, Lucas finds Daniel who has been reunited with his father. Can we conclude that does the mom die in the impossible movie?\nNo", "Hush (2016 film) -- Maddie takes her cellphone from the man's pocket and dials 911 before stumbling outside and sitting on the porch. Maddie's cat sits next to her, rubbing against her leg. Bathed in the blue and white lights of an approaching police cruiser's sirens, Maddie closes her eyes, pets her cat, and smiles. Can we conclude that does the cat die in the movie hush?\nNo", "China Syndrome (The King of Queens) -- They use the long flight to sort out their problems and remain together. After getting baby Ming-Mei from the adoption agency, Carrie finds out that she is pregnant. She is scared about the changes to come, but Doug is optimistic that they will be able to handle it together. Can we conclude that king of queens do doug and carrie get a divorce?\nNo"], "neg_score": [-7.09375, -7.15625, -7.375, -7.59375, -7.90625, -8.5625, -8.75, -8.75, -8.875, -8.875, -9.3125, -9.5625, -9.5625, -9.8125, -10.375, -11.375, -12.8125], "teacher_scores": [-6.53125, -6.59375, -7.0, -7.09375, -7.15625, -7.375, -7.59375, -7.90625, -8.5625, -8.75, -8.75, -8.875, -8.875, -9.3125, -9.5625, -9.5625, -9.8125, -10.375, -11.375, -12.8125], "task": "icl"}
FlagEmbedding/llm_embedder/data/toy/lrlm.json 0 → 100644
View file @ f75058c7
This source diff could not be displayed because it is too large. You can view the blob instead.
FlagEmbedding/llm_embedder/data/toy/qa.json 0 → 100644
View file @ f75058c7
This source diff could not be displayed because it is too large. You can view the blob instead.
FlagEmbedding/llm_embedder/data/toy/tool.json 0 → 100644
View file @ f75058c7
This source diff could not be displayed because it is too large. You can view the blob instead.
FlagEmbedding/llm_embedder/docs/evaluation.md 0 → 100644
View file @ f75058c7
# Evaluation
LLM-Embedder supports 6 retrieval-augmentation tasks tailored for modern LLMs, including:
- Question Answering (qa)
- evaluate with `eval_popqa` and `eval_mmlu`
- In-Context Learning (icl)
- evaluate with `eval_icl`
- Long Conversation (chat)
- evaluate with `eval_msc`
- Long-Range Language Modeling (lrlm)
- evaluate with `eval_lrlm`
- Tool Learning (tool)
- evaluate with `eval_tool`
- Conversational Search (convsearch)
- evaluate with `eval_qrecc`
## Environment
It is recommended that you create a new environment:
```
cd FlagEmbedding/llm_embedder
conda env create -f environment.yaml --name llm-embedder
conda activate llm-embedder
```
To use BM25, you must download **java11** and **anserini**, then add java to your `PATH`:
```bash
# feel free to alternate /data to your prefered location
wget https://huggingface.co/datasets/namespace-Pt/projects/resolve/main/java11.tar.gz?download=true -O /data/java11.tar.gz
wget https://huggingface.co/datasets/namespace-Pt/projects/resolve/main/anserini.tar.gz?download=true -O /data/anserini.tar.gz
cd /data
tar -xzvf java11.tar.gz
tar -xzvf anserini.tar.gz
# below just temporarily set JAVA_HOME; it is RECOMMENDED that you store the lines the setting in ~/.bashrc
export JAVA_HOME=/data/jdk-11.0.2
export PATH=$JAVA_HOME/bin:$PATH
```
## Data
You should download the data for fine-tuning & evaluation then untar the file at anywhere you prefer, e.g. `/data`, which results in a folder `/data/llm-embedder`:
```bash
# feel free to alternate /data to your prefered location
wget https://huggingface.co/datasets/namespace-Pt/projects/resolve/main/llm-embedder.tar.gz?download=true -O /data/llm-embedder.tar.gz
cd /data
tar -xzvf llm-embedder-eval.tar.gz
```
The corpus of QReCC for conversational search is too large (54M passages), we separately upload it to huggingface datasets [namespace-Pt/qrecc-corpus](https://huggingface.co/datasets/namespace-Pt/qrecc-corpus). To evaluate the performance on conversational search, you should load it and save it as json file in the `qrecc` folder:
```python
import datasets
# load dataset
qrecc_corpus = datasets.load_dataset("namespace-Pt/qrecc-corpus", split="train")
# save to jsonline format in YOUR data folder
qrecc_corpus.to_json("/data/llm-embedder/convsearch/qrecc/corpus.json", force_ascii=False, lines=True, orient="records")
```
## Benchmark
### Commands
Below are commands to run evaluation for different retrieval models. You can replace `eval_popqa` with any of `eval_mmlu`, `eval_icl`, `eval_lrlm`, `eval_msc`, `eval_tool`, and `eval_qrecc`. The results will be logged at `data/results/`.
*All our evaluation are based on `meta-llama/Llama-2-7b-chat-hf`. To use different language models, e.g. `Qwen/Qwen-7B-Chat`, simply add `--model_name_or_path Qwen/Qwen-7B-Chat` after every command.*
*Note that you can modify the default value of `data_root` in `src/retrieval/args.py`, so that you don't need to type it for each command.*
```bash
cd FlagEmbedding/llm_embedder
# No retrieval
torchrun --nproc_per_node 8 -m evaluation.eval_popqa --retrieval_method no --data_root /data/llm-embedder
# Random
torchrun --nproc_per_node 8 -m evaluation.eval_popqa --retrieval_method random --data_root /data/llm-embedder
# BM25 (anserini_dir is the folder where you untar anserini.tar.gz)
torchrun --nproc_per_node 8 -m evaluation.eval_popqa --retrieval_method bm25 --data_root /data/llm-embedder --anserini_dir /data/anserini
# Contriever
torchrun --nproc_per_node 8 -m evaluation.eval_popqa --query_encoder facebook/Contriever --dense_metric ip --add_instruction False --data_root /data/llm-embedder
# BGE
torchrun --nproc_per_node 8 -m evaluation.eval_popqa --query_encoder BAAI/bge-base-en --version bge --data_root /data/llm-embedder
# AAR (uses special decoder pooling)
torchrun --nproc_per_node 8 -m evaluation.eval_popqa --query_encoder OpenMatch/AAR-ANCE --pooling_method decoder --add_instruction False --data_root /data/llm-embedder
# APIRetriever
torchrun --nproc_per_node 8 -m evaluation.eval_popqa --query_encoder ToolBench/ToolBench_IR_bert_based_uncased --pooling_method mean --dense_metric ip --add_instruction False --data_root /data/llm-embedder
# LLMRetriever
torchrun --nproc_per_node 8 -m evaluation.eval_popqa --query_encoder intfloat/llm-retriever-base --add_instruction false --pooling_method mean --data_root /data/llm-embedder
# RetroMAE_BEIR
torchrun --nproc_per_node 8 -m evaluation.eval_popqa --query_encoder Shitao/RetroMAE_BEIR --dense_metric ip --add_instruction False --data_root /data/llm-embedder
# LLM Embedder
torchrun --nproc_per_node 8 -m evaluation.eval_popqa --query_encoder BAAI/llm-embedder --version llm-embedder --data_root /data/llm-embedder
```
For Instructor, we should first convert it to our format:
```python
# convert sentence transformer based Instructor to our format
import torch
from src.retrieval import DenseRetriever, RetrievalArgs
from sentence_transformers import SentenceTransformer
model_args = RetrievalArgs(
query_encoder = "hkunlp/instructor-base",
pooling_method = ["mean", "dense"],
dtype = "fp32"
)
retriever = DenseRetriever(**asdict(model_args), cache_dir=model_args.model_cache_dir)
tokenizer = retriever.tokenizer
with torch.no_grad():
sent_model = SentenceTransformer(model_args.query_encoder, device="cpu")
retriever.dense_pooler.weight.data = sent_model.state_dict()["2.linear.weight"]
x = sent_model.encode(["I love you"])
y = retriever.encode("I love you")
print(torch.isclose(torch.from_numpy(x), y))
retriever.save_pretrained("data/outputs/instructor-base")
```
Then we evaluate with
```bash
torchrun --nproc_per_node 8 -m evaluation.eval_popqa --query_encoder data/outputs/instructor-base/encoder --pooling_method mean dense --version instructor --data_root /data/llm-embedder
```
### Leaderboard
All the following results are based on `meta-llama/Llama-27b-chat-hf` with `torch==2.0.1`, `transformers==4.30.0` on a `8xA100` machine with `CUDA==11.4`.
|Model|MMLU (avg)|PopQA (acc)|In-Context Learning (avg)|Long Conversation (ppl)|Long-Range Language Modeling (ppl)|Tool Learning (ndcg)|Conversational Search (ndcg)|
|:-:|:-:|:-:|:-:|:-:|:-:|:-:|:-:|
|None|0.4599|0.2061|0.4645|19.3501|6.4003|--|--|
|BM25|0.4721|0.3491|0.484|14.6512|6.1558|0.5115|0.4341|
|Instructor|0.4721|0.3533|0.6036|14.8799|6.1733|0.3882|0.2863|
|Contriever|0.4684|0.3276|0.6009|14.2129|6.1305|0.4904|0.3563|
|BGE|0.4896|0.4491|0.5974|14.2943|6.1335|0.5761|0.3856|
|AAR|0.4826|0.4792|0.5938|14.6999|6.1528|0.42|0.2877|
|LLMRetriever|0.4625|0.2506|0.6262|14.4746|6.1750|0.1321|0.0234|
|APIRetriever|0.4625|0.2488|0.5945|14.7834|6.1833|0.8017|0.1137|
|LLM-Embedder (ours)|**0.4903**|**0.5052**|**0.6288**|**13.4832**|**6.0972**|**0.8645**|**0.5053**|
FlagEmbedding/llm_embedder/docs/fine-tune.md 0 → 100644
View file @ f75058c7
# Fine-tuning
## Environment
It is recommended that you create a new environment:
```
cd FlagEmbedding/llm_embedder
conda env create -f environment.yaml --name llm-embedder
conda activate llm-embedder
```
To use BM25, you must download **java11** and **anserini**, then add java to your `PATH`:
```bash
# feel free to alternate /data to your prefered location
wget https://huggingface.co/datasets/namespace-Pt/projects/resolve/main/java11.tar.gz?download=true -O /data/java11.tar.gz
wget https://huggingface.co/datasets/namespace-Pt/projects/resolve/main/anserini.tar.gz?download=true -O /data/anserini.tar.gz
cd /data
tar -xzvf java11.tar.gz
tar -xzvf anserini.tar.gz
# below just temporarily set JAVA_HOME; it is RECOMMENDED that you store the lines the setting in ~/.bashrc
export JAVA_HOME=/data/jdk-11.0.2
export PATH=$JAVA_HOME/bin:$PATH
```
## Data
You should download the data for fine-tuning & evaluation then untar the file at anywhere you prefer, e.g. `/data`, which results in a folder `/data/llm-embedder`:
```bash
# feel free to alternate /data to your prefered location
wget https://huggingface.co/datasets/namespace-Pt/projects/resolve/main/llm-embedder.tar.gz?download=true -O /data/llm-embedder.tar.gz
cd /data
tar -xzvf llm-embedder-eval.tar.gz
```
The corpus of QReCC for conversational search is too large (54M passages), we separately upload it to huggingface datasets [namespace-Pt/qrecc-corpus](https://huggingface.co/datasets/namespace-Pt/qrecc-corpus). To evaluate the performance on conversational search, you should load it and save it as json file in the `qrecc` folder:
```python
import datasets
# load dataset
qrecc_corpus = datasets.load_dataset("namespace-Pt/qrecc-corpus", split="train")
# save to jsonline format in YOUR data folder
qrecc_corpus.to_json("/data/llm-embedder/convsearch/qrecc/corpus.json", force_ascii=False, lines=True, orient="records")
```
The data formats for training and evaluation are as follows:
```python
# training
{
"query": str,
"pos": List[str],
"neg": List[str],
"pos_index": Optional[List[int]], # Indices of the positives w.r.t. the corpus. When a global corpus is not available (e.g. long conversation), just ignore this field.
"neg_index": Optional[List[int]], # Indices of the negatives w.r.t. the corpus. When a global corpus is not available (e.g. long conversation), just ignore this field.
"teacher_scores": Optional[List[float]], # Scores from an LM or a reranker, used for distillation.
"answers": Optional[List[str]], # List of answers for the query, used for LM scoring.
}
# evaluation
{
"query": str,
"pos_index": Optional[List[int]], # Indices of the positives w.r.t. corpus. When there is no positives pre-defined (e.g. NQ), just ignore this field.
"answers": Optional[List[str]], # List of answers for computing NQ metrics.
"key": Optional[List[str]], # Retrieval results of the query. Usually used for RAG or reranking.
"key_index": Optional[List[int]], # Key indices w.r.t. the corpus.
}
```
## Retriever
Below are several important arguments for training. The meaning and usage of other arguments can be inspected from [code](../src/retrieval/args.py) or running `python run_dense.py --help` from command line.
- `train_data`: required, one or a list of json files with the aforementioned formatting.
- `eval_data`: optional, one json file with the aforementioned formatting. If an `eval_data` is speficied, the trainer will automatically do evaluation on the `eval_data`.
- `corpus`: optional, the global corpus where `positives`.
**IMPORTANT NOTE**
- For any path specified for `train_data`, `eval_data`, and `corpus`: if it is prefixed with `llm-embedder`, it will be solved to the relative path against [`data_root`](../src/retrieval/args.py). *Note that you can modify the default value of `data_root`, so that you don't need to type it for each command.*
- During fine-tuning, we save the output model in the `huggingface transformers`🤗 format. To use it from `sentence_transformers`, you should convert it to `sentence_transformers` checkpoint in advance:
```bash
python scripts/ours2st.py --encoder data/outputs/your-output-dir/encoder
```
Then everything is the same as described in [README](../README.md).
### LLM-Embedder (Multi-Task Fine-Tune)
```bash
# Remember to modify the data_root to your data root in the script :)
bash scripts/llm-embedder.sh
```
### Single Task Fine-Tune
Below we provide commands to fine-tune a retriever on a single task.
#### QA
```bash
torchrun --nproc_per_node=8 run_dense.py \
--output_dir data/outputs/nq \
--train_data llm-embedder:qa/nq/train.json \
--eval_data llm-embedder:qa/nq/test.json \
--corpus llm-embedder:qa/nq/corpus.json \
--metrics nq \
--key_max_length 128 \
--query_max_length 32 \
--contrastive_weight 0 \
--stable_distill \
--eval_steps 2000 \
--save_steps 2000 \
--max_steps 2000 \
--data_root /data/llm-embedder
```
#### In-Context Learning
```bash
torchrun --nproc_per_node=8 run_dense.py \
--output_dir data/outputs/icl \
--train_data llm-embedder:icl/icl/train.json \
--select_positive random \
--contrastive_weight 0 \
--stable_distill \
--save_steps 6000 \
--max_steps 6000 \
--data_root /data/llm-embedder
```
#### Long-Range Language Modeling
```bash
torchrun --nproc_per_node=8 run_dense.py \
--output_dir data/outputs/lrlm \
--train_data llm-embedder:lrlm/books3/train.json llm-embedder:lrlm/arxiv/train.json llm-embedder:lrlm/codeparrot/train.json \
--select_positive teacher \
--teacher_scores_margin 0.1 \
--contrastive_weight 0 \
--teacher_temperature 0.1 \
--save_steps 4000 \
--max_steps 4000 \
--data_root /data/llm-embedder
```
#### Long Chat
```bash
torchrun --nproc_per_node=8 run_dense.py \
--output_dir data/outputs/msc \
--train_data llm-embedder:chat/msc/train.json \
--select_positive teacher \
--select_negative random \
--contrastive_weight 0 \
--teacher_temperature 0.1 \
--save_steps 4000 \
--max_steps 4000 \
--data_root /data/llm-embedder
```
#### Tool
```bash
torchrun --nproc_per_node=8 run_dense.py \
--output_dir data/outputs/tool \
--train_data llm-embedder:tool/toolbench/train.json \
--eval_data llm-embedder:tool/toolbench/test.json \
--corpus llm-embedder:tool/toolbench/corpus.json \
--key_template {text} \
--metrics ndcg \
--eval_steps 2000 \
--save_steps 2000 \
--max_steps 2000 \
--data_root /data/llm-embedder
```
#### Conversation Search
```bash
torchrun --nproc_per_node=8 run_dense.py \
--output_dir data/outputs/qrecc \
--train_data llm-embedder:conversation/qrecc/train.concat.json \
--eval_data llm-embedder:conversation/qrecc/test.concat.json \
--corpus llm-embedder:conversation/qrecc/corpus.json \
--key_template '{text}' \
--metrics mrr ndcg \
--cutoffs 3 10 100 \
--eval_steps 2000 \
--save_steps 2000 \
--max_steps 2000 \
--data_root /data/llm-embedder
```
### Mine Negatives
```bash
# BGE (the result will be saved at llm-embedder:qa/nq/train.neg.bge.json)
torchrun --nproc_per_node=8 -m evaluation.eval_retrieval \
--eval_data llm-embedder:qa/nq/train.json \
--corpus llm-embedder:qa/nq/corpus.json \
--metrics mrr recall collate_neg \
--save_name bge \
--data_root /data/llm-embedder
# BM25 (the result will be saved at llm-embedder:qa/nq/train.neg.bm25.json; anserini_dir is the folder where you untar anserini.tar.gz)
torchrun --nproc_per_node 8 -m evaluation.eval_retrieval \
--anserini_dir /data/anserini \
--retrieval_method bm25 \
--eval_data llm-embedder:qa/nq/train.json \
--corpus llm-embedder:qa/nq/corpus.json \
--metrics mrr recall collate_neg \
--save_name bm25 \
--data_root /data/llm-embedder
```
## LM Scoring
Score positives and negatives in `eval_data` with $p(o|q,k)$ where $o$ is the desired output (i.e. `answers` field), $q$ is the query, and $k$ is a key (could be positive or negative).
```bash
torchrun --nproc_per_node=8 run_lm_score.py \
--eval_data llm-embedder:qa/msmarco/train.json \
--data_root /data/llm-embedder \
--model_name_or_path meta-llama/Llama-2-7b-chat-hf \
--save_name llama2-7b-chat
```
Results will be saved at `/data/llm-embedder/qa/msmarco/train.scored.llama2-7b-chat.json`
## Known Issues
- `transformers==4.30.0` raises error when using deepspeed schedulerconfig
- modify line `1750` in `trainer.py`
```python
if use_accelerator_prepare:
# NOTE: fix bug in transformers 4.30.0
# model, self.optimizer = self.accelerator.prepare(self.model, self.optimizer)
self.model.train()
if hasattr(self.lr_scheduler, "step"):
if self.use_apex:
model = self.accelerator.prepare(self.model)
else:
model, self.optimizer = self.accelerator.prepare(self.model, self.optimizer)
else:
# to handle cases wherein we pass "DummyScheduler" such as when it is specified in DeepSpeed config.
model, self.optimizer, self.lr_scheduler = self.accelerator.prepare(
self.model, self.optimizer, self.lr_scheduler
)
```
\ No newline at end of file
FlagEmbedding/llm_embedder/environment.yaml 0 → 100644
View file @ f75058c7
name: llm-embedder
channels:
- pytorch
- nvidia
- conda-forge
- defaults
dependencies:
- _libgcc_mutex=0.1=main
- _openmp_mutex=5.1=1_gnu
- blas=1.0=mkl
- bzip2=1.0.8=h7b6447c_0
- ca-certificates=2023.7.22=hbcca054_0
- cuda-cudart=11.8.89=0
- cuda-cupti=11.8.87=0
- cuda-libraries=11.8.0=0
- cuda-nvrtc=11.8.89=0
- cuda-nvtx=11.8.86=0
- cuda-runtime=11.8.0=0
- cudatoolkit=11.8.0=h6a678d5_0
- faiss=1.7.2=py310cuda112h76f6547_0_cuda
- faiss-gpu=1.7.2=h788eb59_4
- filelock=3.9.0=py310h06a4308_0
- gmp=6.2.1=h295c915_3
- gmpy2=2.1.2=py310heeb90bb_0
- intel-openmp=2021.4.0=h06a4308_3561
- jinja2=3.1.2=py310h06a4308_0
- ld_impl_linux-64=2.38=h1181459_1
- libblas=3.9.0=12_linux64_mkl
- libcblas=3.9.0=12_linux64_mkl
- libcublas=11.11.3.6=0
- libcufft=10.9.0.58=0
- libcufile=1.8.0.34=0
- libcurand=10.3.4.52=0
- libcusolver=11.4.1.48=0
- libcusparse=11.7.5.86=0
- libfaiss=1.7.2=cuda112hc9ed507_0_cuda
- libfaiss-avx2=1.7.2=cuda112h1234567_0_cuda
- libffi=3.4.4=h6a678d5_0
- libgcc-ng=11.2.0=h1234567_1
- libgomp=11.2.0=h1234567_1
- liblapack=3.9.0=12_linux64_mkl
- libnpp=11.8.0.86=0
- libnvjpeg=11.9.0.86=0
- libstdcxx-ng=11.2.0=h1234567_1
- libuuid=1.41.5=h5eee18b_0
- llvm-openmp=14.0.6=h9e868ea_0
- markupsafe=2.1.1=py310h7f8727e_0
- mkl=2021.4.0=h06a4308_640
- mpc=1.1.0=h10f8cd9_1
- mpfr=4.0.2=hb69a4c5_1
- mpmath=1.3.0=py310h06a4308_0
- ncurses=6.4=h6a678d5_0
- networkx=3.1=py310h06a4308_0
- openssl=3.0.11=h7f8727e_2
- pip=23.3=py310h06a4308_0
- python=3.10.13=h955ad1f_0
- python_abi=3.10=2_cp310
- pytorch=2.1.0=py3.10_cuda11.8_cudnn8.7.0_0
- pytorch-cuda=11.8=h7e8668a_5
- pytorch-mutex=1.0=cuda
- pyyaml=6.0=py310h5eee18b_1
- readline=8.2=h5eee18b_0
- setuptools=68.0.0=py310h06a4308_0
- sqlite=3.41.2=h5eee18b_0
- sympy=1.11.1=py310h06a4308_0
- tk=8.6.12=h1ccaba5_0
- torchtriton=2.1.0=py310
- typing_extensions=4.7.1=py310h06a4308_0
- wheel=0.41.2=py310h06a4308_0
- xz=5.4.2=h5eee18b_0
- yaml=0.2.5=h7b6447c_0
- zlib=1.2.13=h5eee18b_0
- pip:
- accelerate==0.23.0
- aiohttp==3.8.6
- aiosignal==1.3.1
- asttokens==2.4.0
- async-timeout==4.0.3
- attrs==23.1.0
- backcall==0.2.0
- cachetools==5.3.1
- certifi==2023.7.22
- charset-normalizer==3.3.0
- click==8.1.7
- comm==0.1.4
- contourpy==1.1.1
- cycler==0.12.1
- datasets==2.14.5
- debugpy==1.8.0
- decorator==5.1.1
- deepspeed==0.11.1
- dill==0.3.7
- exceptiongroup==1.1.3
- executing==2.0.0
- fonttools==4.43.1
- frozenlist==1.4.0
- fsspec==2023.6.0
- hjson==3.1.0
- huggingface-hub==0.17.3
- idna==3.4
- ipykernel==6.25.2
- ipython==8.16.1
- ipywidgets==8.1.1
- jedi==0.19.1
- joblib==1.3.2
- jupyter-client==8.4.0
- jupyter-core==5.4.0
- jupyterlab-widgets==3.0.9
- kiwisolver==1.4.5
- matplotlib==3.8.0
- matplotlib-inline==0.1.6
- multidict==6.0.4
- multiprocess==0.70.15
- nest-asyncio==1.5.8
- ninja==1.11.1.1
- nltk==3.8.1
- numpy==1.26.1
- nvidia-ml-py==12.535.108
- nvitop==1.3.1
- packaging==23.2
- pandas==2.1.1
- parso==0.8.3
- pexpect==4.8.0
- pickleshare==0.7.5
- pillow==10.1.0
- platformdirs==3.11.0
- prompt-toolkit==3.0.39
- psutil==5.9.6
- ptyprocess==0.7.0
- pure-eval==0.2.2
- py-cpuinfo==9.0.0
- pyarrow==13.0.0
- pydantic==1.10.13
- pygments==2.16.1
- pyparsing==3.1.1
- python-dateutil==2.8.2
- pytz==2023.3.post1
- pyzmq==25.1.1
- regex==2023.10.3
- requests==2.31.0
- rouge==1.0.1
- safetensors==0.4.0
- scikit-learn==1.3.1
- scipy==1.11.3
- seaborn==0.13.0
- sentence-transformers==2.2.2
- sentencepiece==0.1.99
- six==1.16.0
- stack-data==0.6.3
- termcolor==2.3.0
- threadpoolctl==3.2.0
- tokenizers==0.14.1
- torchvision==0.16.0
- tornado==6.3.3
- tqdm==4.66.1
- traitlets==5.11.2
- transformers==4.34.1
- tzdata==2023.3
- urllib3==2.0.7
- wcwidth==0.2.8
- widgetsnbextension==4.0.9
- xxhash==3.4.1
- yarl==1.9.2
FlagEmbedding/llm_embedder/evaluation/eval_icl.py 0 → 100644
View file @ f75058c7
import os
import re
import json
import random
import logging
import datasets
import numpy as np
from tqdm import tqdm
from datetime import timedelta
from typing import List, Optional
from accelerate import Accelerator, InitProcessGroupKwargs
from torch.utils.data import DataLoader
from transformers import HfArgumentParser
from dataclasses import dataclass, field, asdict
from collections import defaultdict
from functools import partial
from transformers import DataCollatorWithPadding
from src.lm import LM, LMArgs, GenerationArgs
from src.retrieval import RetrievalArgs
from src.utils.util import makedirs, load_json, FileLogger
from .eval_retrieval import main as retrieval_main
from .icl_utils import flat_options, perplexity_to_choice, compute_scores, _llm_generation_func, _llm_perplexity_func
logger = logging.getLogger(__name__)
CQA = {
"arc_c":{'method':'perplexity', 'metric':'acc'},
"arc_e":{'method':'perplexity', 'metric':'acc'},
"natural_questions":{'method':'generation', 'metric':'em'},
"cate_name":'CQA'
}
Commonsense = {
"copa":{'method':'perplexity', 'metric':'acc'},
"hellaswag":{'method':'perplexity', 'metric':'acc'},
"piqa":{'method':'perplexity', 'metric':'acc'},
'cate_name': 'Commonsense'
}
Coreference = {
"winogrande":{'method':'perplexity', 'metric':'acc'},
"wsc":{'method':'perplexity', 'metric':'acc'},
"wsc273":{'method':'perplexity', 'metric':'acc'},
'cate_name': 'Coreference'
}
Paraphrase = {
"mrpc":{'method':'perplexity', 'metric':'acc'},
"paws":{'method':'perplexity', 'metric':'acc'},
"qqp":{'method':'perplexity', 'metric':'acc'},
'cate_name': 'Paraphrase'
}
NLI = {
"rte":{'method':'perplexity', 'metric':'acc'},
"snli":{'method':'perplexity', 'metric':'acc'},
"mnli_m":{'method':'perplexity', 'metric':'acc'},
"mnli_mm":{'method':'perplexity', 'metric':'acc'},
"qnli":{'method':'perplexity', 'metric':'acc'},
'cate_name': 'NLI'
}
ReadingComp = {
"multirc":{'method':'perplexity', 'metric':'f1'},
"openbookqa":{'method':'perplexity', 'metric':'acc'},
"boolq":{'method':'perplexity', 'metric':'acc'},
"squad_v1":{'method':'generation', 'metric':'em'},
'cate_name': 'ReadingComp'
}
Sentiment = {
"sentiment140":{'method':'perplexity', 'metric':'acc'},
"sst2":{'method':'perplexity', 'metric':'acc'},
"yelp":{'method':'perplexity', 'metric':'acc'},
'cate_name': 'Sentiment'
}
Data2Text = {
"common_gen":{'method':'generation', 'metric':'rl'},
"e2e_nlg":{'method':'generation', 'metric':'rl'},
"dart":{'method':'generation', 'metric':'rl'},
'cate_name': 'Data2Text'
}
Summarize = {
"aeslc":{'method':'generation', 'metric':'rl'},
"ag_news":{'method':'perplexity', 'metric':'acc'},
"gigaword":{'method':'generation', 'metric':'rl'},
'cate_name': 'Summarize'
}
TASK_LIST = [CQA, Commonsense, Coreference, Paraphrase, NLI, ReadingComp, Sentiment, Data2Text, Summarize]
task2cat = {}
for category in TASK_LIST:
cat_name = category["cate_name"]
for key, value in category.items():
if key == "cate_name":
continue
task2cat[key] = cat_name
@dataclass
class ICLArgs(LMArgs, RetrievalArgs):
output_dir: str = field(
default="data/results/icl/",
metadata={'help': 'Path to the file for saving embeddings and results.'}
)
eval_data: str = field(
default="llm-embedder:icl/icl/test.json",
metadata={'help': 'Path to the file containing both retrieved keys and answers.'}
)
task_names: Optional[List[str]] = field(
default=None,
metadata={'help': 'List of tasks to evaluate.'}
)
load_prev_result: bool = field(
default=False,
metadata={'help': 'Load existing results in output_dir?'}
)
context_max_length: int = field(
default=1024,
metadata={'help': 'Evaluation json file.'},
)
few_shot: int = field(
default=8,
metadata={'help': 'How many few shot train samples?'},
)
corpus: str = field(
default="llm-embedder:icl/icl/corpus.json",
metadata={'help': 'Corpus path for retrieval.'}
)
key_template: str = field(
default="{contents}",
metadata={'help': 'How to concatenate columns in the corpus to form one key?'}
)
metrics: List[str] = field(
default_factory=lambda: [],
)
log_path: str = field(
default="data/results/icl/icl.log",
metadata={'help': 'Path to the file for logging.'}
)
@dataclass
class GenerationArgs(GenerationArgs):
max_new_tokens: int = field(
default=64,
metadata={'help': 'Maximum new tokens to generate.'}
)
def remove_double_space(string):
return re.sub("[ ]{2,}", " ", string)
def load_test_data(knn_inxs,
test_data,
corpus_data,
filter_diff_task: bool=False,
example_num=8,
same_task_random=False,
):
dataset = datasets.load_dataset('json', data_files=test_data)['train']
passage_dataset = datasets.load_dataset('json', data_files=corpus_data)['train']
task_data = defaultdict(list)
for i, e in enumerate(tqdm(dataset, desc="Organizing Data")):
query = remove_double_space(e['query'])
answers = [remove_double_space(x) for x in e['answers']]
if knn_inxs is not None:
if filter_diff_task:
few_shot = []
rest_passage = []
for x in knn_inxs[i]:
icl_e = passage_dataset[int(x)]
# print(icl_e['task_name'], e['task_name'])
if icl_e['task_name'][:4] == e['task_name'][:4]:
few_shot.append(remove_double_space(icl_e['contents']))
if len(few_shot) > example_num: break
else:
if len(rest_passage) < example_num:
rest_passage.append(remove_double_space(icl_e['contents']))
if len(few_shot) < example_num:
few_shot.extend(rest_passage)
few_shot = few_shot[:example_num]
else:
# if task2cat[e['task_name']] == 'Coreference':
# candidates = random.sample(knn_inxs[i][:20], example_num)
# else:
# candidates = knn_inxs[i][:example_num]
candidates = knn_inxs[i][:example_num]
few_shot = [remove_double_space(passage_dataset[int(x)]['contents']) for x in candidates]
else:
few_shot = []
data = {"query":query, "answers":answers, "few_shot":few_shot}
if 'options' in e:
data['options'] = e['options']
task_data[e['task_name']].append(data)
if same_task_random:
task_name_2_idx = defaultdict(list)
for i, example in enumerate(tqdm(passage_dataset, "Collecting Task Indices")):
task_name_2_idx[example["task_name"]].append(i)
for task_name, task_examples in tqdm(task_data.items(), desc="Collecting Same-Task-Random Examples"):
if task_name in ["mnli_m", "mnli_mm"]:
corpus_task_name = "mnli"
else:
corpus_task_name = task_name
for i, _ in enumerate(task_examples):
task_indices = task_name_2_idx[corpus_task_name]
example_num = min(example_num, len(task_indices))
# get examples of the same task
few_shot = [remove_double_space(content) for content in passage_dataset[random.sample(task_indices, example_num)]["contents"]]
task_data[task_name][i]["few_shot"] = few_shot
return task_data
def main():
parser = HfArgumentParser([ICLArgs, GenerationArgs])
args, generation_args = parser.parse_args_into_dataclasses()
accelerator = Accelerator(cpu=args.cpu, kwargs_handlers=[InitProcessGroupKwargs(timeout=timedelta(seconds=100000))])
if args.retrieval_method == "dense":
output_dir = os.path.join(args.output_dir, args.query_encoder.strip(os.sep).replace(os.sep, "--"))
else:
output_dir = os.path.join(args.output_dir, args.retrieval_method)
args.output_dir = output_dir
if args.retrieval_method != "no":
_, preds, _ = retrieval_main(args=args, accelerator=accelerator, log=False)
else:
preds = None
llm = LM(
model_name_or_path=args.model_name_or_path,
dtype=args.lm_dtype,
device_map=args.lm_device_map,
padding_side=args.padding_side,
cache_dir=args.model_cache_dir,
accelerator=accelerator,
generation_args=asdict(generation_args)
)
tokenizer = llm.tokenizer
args.output_dir = os.path.join(args.output_dir, args.model_name_or_path.strip(os.sep).replace(os.sep, "--"))
task_data = load_test_data(preds, test_data=args.eval_data, corpus_data=args.corpus, example_num=args.few_shot, same_task_random=args.retrieval_method == "same-task-random")
all_results = []
metrics = {}
for task_cate in [CQA, Commonsense, Coreference, Paraphrase, NLI, ReadingComp, Sentiment, Data2Text, Summarize]:
task_results = []
for task_name, setting in task_cate.items():
if task_name == 'cate_name':
continue
# skip tasks that are not specified
if args.task_names is not None and task_name not in args.task_names:
continue
save_path = os.path.join(args.output_dir, f'{task_name}.json')
if args.load_prev_result and os.path.exists(save_path):
# the first line is the metric
result = load_json(save_path, lines=True)[0]
task_results.append(result['metric_value'][setting['metric']])
all_results.append(result['metric_value'][setting['metric']])
if accelerator.process_index == 0:
logger.info(f"loading existing results from {save_path}...")
print(result)
continue
test_data = task_data[task_name]
if accelerator.process_index == 0:
print(f"------{task_name} ({len(all_results) + 1} / {30})------")
if setting['metric'] == 'acc':
assert setting['method'] == 'perplexity'
if setting['method'] == 'perplexity':
flat_data = flat_options(test_data)
dataset = datasets.Dataset.from_list(flat_data)
dataset.set_transform(
partial(
_llm_perplexity_func,
tokenizer=tokenizer,
example_num=args.few_shot,
max_input_tokens=args.context_max_length,
add_llama_inst=args.add_llama_inst,
)
)
else:
dataset = datasets.Dataset.from_list(test_data)
dataset.set_transform(
partial(
_llm_generation_func,
tokenizer=tokenizer,
example_num=args.few_shot,
max_input_tokens=args.context_max_length,
add_llama_inst=args.add_llama_inst,
)
)
data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
dataloader = DataLoader(
dataset,
batch_size=args.lm_batch_size,
collate_fn=data_collator,
pin_memory=True,
)
dataloader = accelerator.prepare(dataloader)
if setting['method'] == 'perplexity':
predictions = llm.compute_nlls(dataloader)
predictions = perplexity_to_choice(test_data, predictions)
else:
if args.add_llama_inst:
eos_token_id = tokenizer.eos_token_id
else:
eos_token_id = tokenizer.encode("\n", add_special_tokens=False)[-1]
predictions = llm.generate(dataloader, eos_token_id=eos_token_id)
predictions = [x.strip() for x in predictions]
if setting['metric'] in ['em']:
labels = [x['answers'] for x in test_data]
else:
labels = [x['answers'][0] for x in test_data]
metric_value = compute_scores(setting['metric'], predictions, labels)
result = {'task_name':task_name, 'setting':setting, 'metric_value':metric_value}
if accelerator.process_index == 0:
print(result)
with open(makedirs(save_path), 'w') as f:
f.write(json.dumps(result, ensure_ascii=False) + "\n")
for i, sample in enumerate(test_data):
sample["output"] = predictions[i]
f.write(json.dumps(sample, ensure_ascii=False) + "\n")
task_results.append(result['metric_value'][setting['metric']])
all_results.append(result['metric_value'][setting['metric']])
if len(task_results):
metrics[task_cate['cate_name']] = np.mean(task_results)
metrics['avg'] = np.mean(all_results)
file_logger = FileLogger(makedirs(args.log_path))
if accelerator.process_index == 0:
file_logger.log(metrics, Args=asdict(args))
if __name__ == "__main__":
main()
FlagEmbedding/llm_embedder/evaluation/eval_lrlm.py 0 → 100644
View file @ f75058c7
import os
import logging
import datasets
from copy import deepcopy
from accelerate import Accelerator
from torch.utils.data import DataLoader
from transformers import HfArgumentParser
from dataclasses import dataclass, field, asdict
from src.lm import SRLMArgs, SelfRetrievalLM
from src.retrieval import Retriever, RetrievalArgs, TASK_CONFIG
from src.utils.util import makedirs, remove_eos, DefaultDataCollator, DatasetProcessFn, FileLogger
logger = logging.getLogger(__name__)
import transformers
# disable too long input warning
transformers.logging.set_verbosity_error()
# merge two args to get unified arguments
@dataclass
class LRLMArgs(RetrievalArgs, SRLMArgs):
eval_data: str = field(
default="llm-embedder:lrlm/books3/test.json",
metadata={'help': 'Evaluation json file.'},
)
lm_batch_size: int = field(
default=1,
metadata={'help': 'Evaluation json file.'},
)
context_max_length: int = field(
default=32768,
metadata={'help': 'Evaluation json file.'},
)
anchor_length: int = field(
default=160000,
metadata={'help': 'Evaluation file containing long texts.'}
)
chunk_size: int = field(
default=128,
metadata={'help': 'How many tokens in a chunk?'}
)
key_num: int = field(
default=8,
metadata={'help': 'How many chunks to retrieve at a time?'}
)
chunk_batch_size: int = field(
default=1,
metadata={'help': 'How many retrieval & generation to execute in parallel?'}
)
log_path: str = field(
default="data/results/lrlm",
metadata={'help': 'Path to the file for logging.'}
)
debug_retrieval: bool = field(
default=False,
metadata={'help': 'Check retrieval queries and values?'}
)
def __post_init__(self):
super().__post_init__()
if self.retrieval_method == "bm25":
# NOTE: we can only use naive bm25 for self retrieval
self.retrieval_method = "naive-bm25"
def process_lrlm(tokenizer, context_max_length=4096, target_length=1024, anchor_length=160000):
test = tokenizer("test", return_special_tokens_mask=True)["special_tokens_mask"]
has_eos = False
if test[-1] == 1:
has_eos = True
left_truncation_tokenizer = deepcopy(tokenizer)
left_truncation_tokenizer.truncation_side = "left"
@DatasetProcessFn()
def _process(text, **kwds):
output = {}
text = text[:anchor_length]
inputs = left_truncation_tokenizer(text, max_length=context_max_length, truncation=True, return_token_type_ids=False, add_special_tokens=False)
if len(inputs.input_ids) < target_length:
return None
labels = inputs["input_ids"].copy()
inputs_length = len(labels)
labels[:-target_length] = [-100 for _ in range(inputs_length - target_length)]
inputs["labels"] = labels
for k, v in inputs.items():
output[k] = v
return output
return _process
def main():
parser = HfArgumentParser([LRLMArgs])
args, = parser.parse_args_into_dataclasses()
accelerator = Accelerator(cpu=args.cpu)
retriever = Retriever(
retrieval_method=args.retrieval_method,
# for dense retriever
query_encoder=args.query_encoder,
key_encoder=args.key_encoder,
pooling_method=args.pooling_method,
dense_metric=args.dense_metric,
query_max_length=args.query_max_length,
key_max_length=args.key_max_length,
tie_encoders=args.tie_encoders,
truncation_side=args.truncation_side,
cache_dir=args.model_cache_dir,
dtype=args.dtype,
accelerator=accelerator,
# for bm25 retriever
anserini_dir=args.anserini_dir,
k1=args.k1,
b=args.b
)
if args.add_instruction:
instruction = TASK_CONFIG[args.version]["instruction"]["lrlm"]
else:
instruction = None
srlm = SelfRetrievalLM(
model_name_or_path=args.model_name_or_path,
retriever=retriever,
dtype=args.lm_dtype,
device_map=args.lm_device_map,
padding_side=args.padding_side,
cache_dir=args.model_cache_dir,
context_window_size=args.context_window_size,
chunk_size=args.chunk_size,
key_num=args.key_num,
chunk_batch_size=args.chunk_batch_size,
add_key_continuation=args.add_key_continuation,
retrieval_method=args.retrieval_method,
order_method=args.order_method,
integrate_method=args.integrate_method,
instruction=instruction,
debug_retrieval=args.debug_retrieval,
add_sep=args.add_sep,
accelerator=accelerator,
)
tokenizer = srlm.tokenizer
logging.info(f"Loading data from {args.eval_data}...")
if args.retrieval_method == "no" and args.context_max_length != args.context_window_size:
logger.warning(f"Found retrieval_method is 'no', setting context_max_length to the same as context_window_size ({args.context_window_size})!")
args.context_max_length = args.context_window_size
with accelerator.main_process_first():
dataset = datasets.load_dataset("json", data_files=args.eval_data, split="train", cache_dir=args.dataset_cache_dir)
dataset = dataset.map(process_lrlm(
tokenizer,
context_max_length=args.context_max_length,
target_length=args.target_length,
anchor_length=args.anchor_length,
), remove_columns=dataset.column_names, batched=True, batch_size=50, num_proc=64)
data_collator = DefaultDataCollator(tokenizer=tokenizer, add_position_ids=args.add_position_ids)
dataloader = DataLoader(
dataset,
batch_size=args.lm_batch_size,
collate_fn=data_collator,
pin_memory=True,
)
dataloader = accelerator.prepare(dataloader)
perplexity = srlm.compute_perplexity(dataloader)
metrics = {"perplexity": perplexity}
if accelerator.process_index == 0:
dataset = os.path.normpath(args.eval_data).split(os.sep)[-2]
log_path = os.path.join(args.log_path, f"{dataset}.log")
file_logger = FileLogger(makedirs(log_path))
file_logger.log(metrics, Args=asdict(args))
if __name__ == "__main__":
main()
FlagEmbedding/llm_embedder/evaluation/eval_mmlu.py 0 → 100644
View file @ f75058c7
import os
import copy
import json
import logging
import datasets
from typing import List
from accelerate import Accelerator
from torch.utils.data import DataLoader
from transformers import HfArgumentParser
from dataclasses import dataclass, field, asdict
from collections import defaultdict
from src.lm import (
LM,
LMArgs
)
from src.retrieval import (
RetrievalArgs,
RetrievalMetric,
)
from src.utils.util import makedirs, remove_eos, DefaultDataCollator, DatasetProcessFn, FileLogger
from .eval_retrieval import main as retrieval_main
logger = logging.getLogger(__name__)
import transformers
transformers.logging.set_verbosity_error()
SUBJECT_2_CATEGORY={"abstract_algebra": "STEM", "anatomy": "others", "astronomy": "STEM", "business_ethics": "others", "clinical_knowledge": "others", "college_biology": "STEM", "college_chemistry": "STEM", "college_computer_science": "STEM", "college_mathematics": "STEM", "college_medicine": "others", "college_physics": "STEM", "computer_security": "STEM", "conceptual_physics": "STEM", "econometrics": "Social Sciences", "electrical_engineering": "STEM", "elementary_mathematics": "STEM", "formal_logic": "Humanities", "global_facts": "others", "high_school_biology": "STEM", "high_school_chemistry": "STEM", "high_school_computer_science": "STEM", "high_school_european_history": "Humanities", "high_school_geography": "Social Sciences", "high_school_government_and_politics": "Social Sciences", "high_school_macroeconomics": "Social Sciences", "high_school_mathematics": "STEM", "high_school_microeconomics": "Social Sciences", "high_school_physics": "STEM", "high_school_psychology": "Social Sciences", "high_school_statistics": "STEM", "high_school_us_history": "Humanities", "high_school_world_history": "Humanities", "human_aging": "others", "human_sexuality": "Social Sciences", "international_law": "Humanities", "jurisprudence": "Humanities", "logical_fallacies": "Humanities", "machine_learning": "STEM", "management": "others", "marketing": "others", "medical_genetics": "others", "miscellaneous": "others", "moral_disputes": "Humanities", "moral_scenarios": "Humanities", "nutrition": "others", "philosophy": "Humanities", "prehistory": "Humanities", "professional_accounting": "others", "professional_law": "Humanities", "professional_medicine": "others", "professional_psychology": "Social Sciences", "public_relations": "Social Sciences", "security_studies": "Social Sciences", "sociology": "Social Sciences", "us_foreign_policy": "Social Sciences", "virology": "others", "world_religions": "Humanities"}
@dataclass
class MMLUArgs(LMArgs, RetrievalArgs):
output_dir: str = field(
default="data/results/mmlu",
)
eval_data: str = field(
default="llm-embedder:qa/mmlu/test.json",
metadata={'help': 'Path to the test file.'}
)
lm_batch_size: int = field(
default=2,
metadata={'help': 'Evaluation batch size.'},
)
few_shot: int = field(
default=0,
metadata={'help': 'How many few shot train samples?'},
)
train_data: str = field(
default="llm-embedder:qa/mmlu/dev.json",
metadata={'help': 'Path to the file containing training examples.'}
)
corpus: str = field(
default="llm-embedder:qa/msmarco/corpus.json",
metadata={'help': 'Corpus path for retrieval.'}
)
key_template: str = field(
default="{title} {text}",
metadata={'help': 'How to concatenate columns in the corpus to form one key?'}
)
key_max_length: int = field(
default=128,
metadata={'help': 'How many tokens at maximum in a key.'}
)
hits: int = field(
default=10,
metadata={'help': 'How many hits per query?'},
)
key_num: int = field(
default=3,
metadata={'help': 'How many docs to provide in prompt?'},
)
metrics: List[str] = field(
default_factory=lambda: ["collate_key"],
)
save_to_output: bool = field(
default=True,
metadata={'help': 'Save the result/key/negative to output_dir? If not true, they will be saved next to the eval_data.'}
)
log_path: str = field(
default="data/results/mmlu/mmlu.log",
metadata={'help': 'Path to the file for logging.'}
)
def process_mmlu(tokenizer, context_max_length=2048, key_num=3, few_shot=0, train_data=None, cache_dir=None, is_encoder_decoder=False, add_llama_inst=False):
tokenizer.truncation_side = 'right'
left_truncation_tokenizer = copy.deepcopy(tokenizer)
left_truncation_tokenizer.truncation_side = 'left'
test = tokenizer("test", return_special_tokens_mask=True)["special_tokens_mask"]
has_bos = has_eos = False
if test[0] == 1:
has_bos = True
if test[-1] == 1:
has_eos = True
if few_shot > 0:
assert train_data is not None
train_data = datasets.load_dataset("json", data_files=train_data, cache_dir=cache_dir, split="train")
train_df = train_data.to_pandas()
# transform the dataframe into dict of dataframes
train_df = {k: v[:few_shot] for k, v in train_df.groupby("subject")}
options = ['A', 'B', 'C', 'D']
def _prepare_sample(query, choices, answer):
"""
<Question>
A. <Choices 1>
B. <Choices 2>
C. <Choices 3>
D. <Choices 4>
Answer: <Answer>
"""
# answer maybe int or numpy int64
if not isinstance(answer, str):
answer = options[answer]
sample = f"{query}\n{chr(10).join([f'{option}. {choice}' for option, choice in zip(options, choices)])}\nAnswer: {answer}"
return sample
def _prepare_knowledge(key, max_length=None):
if key is not None:
key = key[:key_num]
key = "\n".join(key)
key = f"Knowledge:\n{key}"
if max_length is not None:
# truncate key if necessary
key = tokenizer.decode(tokenizer.encode(key, add_special_tokens=False, truncation=True, max_length=max_length))
else:
key = ""
return key
@DatasetProcessFn(augment=True)
def _process(query, choices, query_id, subject, answer, key=None, **kwds):
"""Yield key and query with a prompt template"""
output = defaultdict(list)
query = query.strip()
head = f"The following are multiple choice questions (with answers) about {' '.join(subject.split('_'))}.\n\n"
if few_shot > 0:
train_samples = ""
for i in range(few_shot):
if i >= len(train_df[subject]):
break
train_sample = train_df[subject].iloc[i][['query', 'choices', 'answer']]
train_sample = _prepare_sample(**train_sample) + "\n\n"
train_samples += train_sample
else:
train_samples = ""
knowledge_max_length = context_max_length - len(tokenizer.encode(head + train_samples + _prepare_sample(query, choices, 'A'))) - int(has_bos) - int(has_eos)
if knowledge_max_length < 0:
knowledge = ""
else:
knowledge = _prepare_knowledge(key, knowledge_max_length)
for option in options:
left = knowledge
right = head + train_samples + _prepare_sample(query, choices, option)
# \n\n to split knowledge and prompts
if len(left):
right = "\n\n" + right
# TODO: add llama instruction
# if add_llama_inst:
# left = "[INST]" + left
# right = right + "[/INST]"
inputs = left_truncation_tokenizer(left + right, truncation=True, max_length=context_max_length, return_token_type_ids=False)
if has_eos and not is_encoder_decoder:
inputs = remove_eos(inputs, tokenizer.eos_token_id)
# find answer length
option_seq = tokenizer.encode("Answer: " + option, add_special_tokens=False)
option_length = len(option_seq) - len(tokenizer.encode("Answer:", add_special_tokens=False))
if is_encoder_decoder:
labels = inputs["input_ids"].copy()[-option_length:]
for k, v in inputs.items():
inputs[k] = v[:-option_length]
inputs["labels"] = labels
else:
# take care of padded tokens
labels = inputs["input_ids"].copy()
labels = [x if inputs["attention_mask"][i] == 1 else -100 for i, x in enumerate(labels)]
labels[:-option_length] = [-100] * (len(labels) - option_length)
inputs["labels"] = labels
inputs["query_id"] = query_id
for k, v in inputs.items():
output[k].append(v)
return output
return _process
def evaluate_mmlu(eval_data, save_path, **kwds):
def compute_metric(eval_preds):
makedirs(save_path)
tasks = defaultdict(list)
results = defaultdict(list)
samples = {}
with open(eval_data) as f:
for line in f:
sample = json.loads(line.strip())
samples[sample["query_id"]] = sample
# nll must comes in the order of A, B, C, and D
for query_id, nll in zip(*eval_preds):
# store log likelihood
results[query_id].append(-nll)
with open(makedirs(save_path), "w") as f:
for k, v in results.items():
output = max(enumerate(v), key=lambda x: x[1])[0]
sample = samples[k]
sample["output"] = output
tasks[sample["subject"]].append((output, sample["answer"]))
f.write(json.dumps(sample, ensure_ascii=False) + "\n")
metrics = defaultdict(list)
for task_name, task_eval_preds in tasks.items():
accuracy = 0
for pred, label in task_eval_preds:
accuracy += int(pred == label)
accuracy /= len(task_eval_preds)
category = SUBJECT_2_CATEGORY[task_name]
metrics[f"{category}"].append(accuracy)
metrics["all"].append(accuracy)
for k, v in metrics.items():
metrics[k] = sum(v) / len(v)
metrics = {
"STEM": metrics["STEM"],
"Social Sciences": metrics["Social Sciences"],
"Humanities": metrics["Humanities"],
"Others": metrics["others"],
"All": metrics["all"],
}
return dict(metrics)
return compute_metric
def main():
parser = HfArgumentParser([MMLUArgs])
args, = parser.parse_args_into_dataclasses()
accelerator = Accelerator(cpu=args.cpu)
# modify the output_dir for retrieval
if args.retrieval_method == "dense":
output_dir = os.path.join(args.output_dir, args.query_encoder.strip(os.sep).replace(os.sep, "--"))
else:
output_dir = os.path.join(args.output_dir, args.retrieval_method)
args.output_dir = output_dir
if args.retrieval_method != "no":
retrieval_main(args=args, accelerator=accelerator, log=False)
eval_data = RetrievalMetric._get_save_path(args.eval_data, args.output_dir, field="key", save_name=args.save_name)
else:
eval_data = args.eval_data
lm = LM(
model_name_or_path=args.model_name_or_path,
dtype=args.lm_dtype,
device_map=args.lm_device_map,
padding_side=args.padding_side,
cache_dir=args.model_cache_dir,
accelerator=accelerator
)
tokenizer = lm.tokenizer
with accelerator.main_process_first():
logging.info(f"Loading data from {eval_data}...")
dataset = datasets.load_dataset("json", data_files=eval_data, split="train", cache_dir=args.dataset_cache_dir)
dataset = dataset.map(process_mmlu(
tokenizer,
context_max_length=args.context_max_length,
key_num=args.key_num,
few_shot=args.few_shot,
train_data=args.train_data,
cache_dir=args.dataset_cache_dir,
is_encoder_decoder=lm.model.config.is_encoder_decoder,
add_llama_inst=args.add_llama_inst
), remove_columns=dataset.column_names, batched=True, num_proc=32)
data_collator = DefaultDataCollator(tokenizer=tokenizer, add_position_ids=args.add_position_ids)
dataloader = DataLoader(
dataset,
batch_size=args.lm_batch_size,
collate_fn=data_collator,
pin_memory=True,
)
dataloader = accelerator.prepare(dataloader)
results = lm.compute_nlls(dataloader)
if accelerator.process_index == 0:
file_logger = FileLogger(makedirs(args.log_path))
result_path = os.path.join(args.output_dir, args.model_name_or_path.strip(os.sep).replace(os.sep, "--") + ".json")
metrics = evaluate_mmlu(eval_data, result_path)(results)
file_logger.log(metrics, Args=asdict(args))
if __name__ == "__main__":
main()
FlagEmbedding/llm_embedder/evaluation/eval_msc.py 0 → 100644
View file @ f75058c7
import os
import logging
import datasets
import torch
import numpy as np
from accelerate import Accelerator
from torch.utils.data import DataLoader
from transformers import HfArgumentParser
from dataclasses import dataclass, field, asdict
from src.lm import SRLMArgs, SelfRetrievalLM
from src.retrieval import Retriever, RetrievalArgs, TASK_CONFIG
from src.utils.util import makedirs, pad_nested_lists, get_max_length_in_nested_lists, FileLogger
logger = logging.getLogger(__name__)
import transformers
# disable too long input warning
transformers.logging.set_verbosity_error()
# merge two args to get unified arguments
@dataclass
class LRLMArgs(RetrievalArgs, SRLMArgs):
eval_data: str = field(
default="llm-embedder:chat/msc/test.json",
metadata={'help': 'Evaluation file containing long texts.'}
)
lm_batch_size: int = field(
default=1,
metadata={'help': 'Evaluation batch size.'},
)
add_position_ids: bool = field(
default=False,
metadata={'help': 'Create position ids based on attention masks? Useful when training left-padded models with absolute position embeddings.'}
)
key_num: int = field(
default=1,
metadata={'help': 'How many chunks to retrieve at a time?'}
)
log_path: str = field(
default="data/results/msc/msc.log",
metadata={'help': 'Path to the file for logging.'}
)
debug_retrieval: bool = field(
default=False,
metadata={'help': 'Check retrieval queries and values?'}
)
@dataclass
class HistoryCollator:
"""Collate histories, pad them, and return masks"""
def __call__(self, batch_elem):
first_elem = batch_elem[0]
return_batch = {}
for key, value in first_elem.items():
batch_value = [elem[key] for elem in batch_elem]
if key == "history":
longest = get_max_length_in_nested_lists(batch_value)
batch_value, history_mask = pad_nested_lists(batch_value, longest, "", "right")
history_mask = torch.tensor(history_mask, dtype=torch.bool)
return_batch["history_mask"] = history_mask
elif key == "answers":
# there is only one answer
key = "answer"
batch_value = [elem[0] for elem in batch_value]
elif key in ["query_id", "task"]:
continue
# strip here for convenience
return_batch[key] = np.char.strip(np.array(batch_value))
return return_batch
def main():
parser = HfArgumentParser([LRLMArgs])
args, = parser.parse_args_into_dataclasses()
accelerator = Accelerator(cpu=args.cpu)
retriever = Retriever(
retrieval_method=args.retrieval_method,
# for dense retriever
query_encoder=args.query_encoder,
key_encoder=args.key_encoder,
pooling_method=args.pooling_method,
dense_metric=args.dense_metric,
query_max_length=args.query_max_length,
key_max_length=args.key_max_length,
tie_encoders=args.tie_encoders,
truncation_side=args.truncation_side,
cache_dir=args.model_cache_dir,
dtype=args.dtype,
accelerator=accelerator,
# for bm25 retriever
anserini_dir=args.anserini_dir,
k1=args.k1,
b=args.b
)
if args.add_instruction:
instruction = TASK_CONFIG[args.version]["instruction"]["chat"]
else:
instruction = None
lm = SelfRetrievalLM(
model_name_or_path=args.model_name_or_path,
retriever=retriever,
dtype=args.lm_dtype,
device_map=args.lm_device_map,
padding_side=args.padding_side,
cache_dir=args.model_cache_dir,
context_window_size=args.context_window_size,
chunk_size=args.chunk_size,
key_num=args.key_num,
chunk_batch_size=args.chunk_batch_size,
retrieval_method=args.retrieval_method,
order_method=args.order_method,
integrate_method=args.integrate_method,
instruction=instruction,
debug_retrieval=args.debug_retrieval,
add_sep=args.add_sep,
accelerator=accelerator,
)
logging.info(f"Loading data from {args.eval_data}...")
with accelerator.main_process_first():
dataset = datasets.load_dataset("json", data_files=args.eval_data, split="train", cache_dir=args.dataset_cache_dir)
data_collator = HistoryCollator()
dataloader = DataLoader(
dataset,
batch_size=args.lm_batch_size,
collate_fn=data_collator,
pin_memory=True,
)
dataloader = accelerator.prepare(dataloader)
perplexity = lm.compute_perplexity(dataloader)
metrics = {"perplexity": perplexity}
if accelerator.process_index == 0:
log_path = os.path.join(args.log_path)
file_logger = FileLogger(makedirs(log_path))
file_logger.log(metrics, Args=asdict(args))
if __name__ == "__main__":
main()
FlagEmbedding/llm_embedder/evaluation/eval_popqa.py 0 → 100644
View file @ f75058c7
import os
import json
import logging
import datasets
from typing import List
from accelerate import Accelerator
from torch.utils.data import DataLoader
from transformers import HfArgumentParser
from dataclasses import dataclass, field, asdict
from src.lm import (
LM,
LMArgs,
GenerationArgs
)
from src.retrieval import (
RetrievalArgs,
RetrievalMetric,
)
from src.utils.util import makedirs, remove_eos, DefaultDataCollator, DatasetProcessFn, FileLogger
from .eval_retrieval import main as retrieval_main
logger = logging.getLogger(__name__)
PROPID_2_TEMPLATE = {
22: "What is {}'s occupation?",
218: "In what city was {} born?",
91: "What genre is {}?",
257: "Who is the father of {}?",
182: "In what country is {}?",
164: "Who was the producer of {}?",
526: "Who was the director of {}?",
97: "What is {} the capital of?",
533: "Who was the screenwriter for {}?",
639: "Who was the composer of {}?",
472: "What color is {}?",
106: "What is the religion of {}?",
560: "What sport does {} play?",
484: "Who is the author of {}?",
292: "Who is the mother of {}?",
422: "What is the capital of {}?"
}
@dataclass
class PopQAArgs(LMArgs, RetrievalArgs):
output_dir: str = field(
default="data/results/popqa",
)
eval_data: str = field(
default="llm-embedder:qa/popqa/test.json",
metadata={'help': 'Path to the test file.'}
)
few_shot: int = field(
default=15,
metadata={'help': 'How many few shot train samples?'},
)
hits: int = field(
default=10,
metadata={'help': 'How many hits per query?'},
)
key_num: int = field(
default=3,
metadata={'help': 'How many docs to provide in prompt?'},
)
corpus: str = field(
default="llm-embedder:qa/nq/corpus.json",
metadata={'help': 'Corpus path for retrieval.'}
)
key_template: str = field(
default="{title} {text}",
metadata={'help': 'How to concatenate columns in the corpus to form one key?'}
)
key_max_length: int = field(
default=128,
metadata={'help': 'How many tokens at maximum in a key.'}
)
metrics: List[str] = field(
default_factory=lambda: ["collate_key"],
)
save_to_output: bool = field(
default=True,
metadata={'help': 'Save the result/key/negative to output_dir? If not true, they will be saved next to the eval_data.'}
)
log_path: str = field(
default="data/results/popqa/popqa.log",
metadata={'help': 'Path to the file for logging.'}
)
@dataclass
class GenerationArgs(GenerationArgs):
max_new_tokens: int = field(
default=16,
metadata={'help': 'Maximum new tokens to generate.'}
)
eos_token_id: int = 13
def process_popqa(tokenizer, context_max_length=2048, key_num=3, few_shot=0, train_data=None, cache_dir=None, is_encoder_decoder=False):
test = tokenizer("test", return_special_tokens_mask=True)["special_tokens_mask"]
has_bos = has_eos = False
if test[0] == 1:
has_bos = True
if test[-1] == 1:
has_eos = True
if few_shot > 0:
assert train_data is not None
assert few_shot // (len(PROPID_2_TEMPLATE) - 1), f"Make sure the number of few shot examples is a multiple of the template number!"
train_dataset = datasets.load_dataset("json", data_files=train_data, cache_dir=cache_dir, split="train")
train_df = train_dataset.to_pandas()
train_df = {k: v[:few_shot] for k, v in train_df.groupby("prop_id")}
nshot_per_template = few_shot // (len(PROPID_2_TEMPLATE) - 1)
def _prepare_sample(query, obj=None, **kwds):
sample = f"Q: {query} A:"
if obj is not None:
sample = sample + " " + obj
return sample
def _prepare_retrieval(keys):
if keys is not None:
keys = keys[:key_num]
keys = "\n".join(keys)
keys = f"Knowledge: {keys}"
else:
keys = ""
return keys
@DatasetProcessFn()
def _process(query, query_id, prop_id, key=None, _index=None, **kwds):
"""Yield keys and query with a prompt template"""
output = {}
query = query.strip()
knowledge = _prepare_retrieval(key)
train_samples_max_length = context_max_length - len(tokenizer.encode("\n\n" if len(knowledge) else "" + _prepare_sample(query), add_special_tokens=False)) - int(has_bos)
if few_shot > 0:
train_samples = ""
train_samples_length = 0
for k, df in train_df.items():
# avoid contamination
if k == prop_id:
continue
for sample in df.sample(nshot_per_template).iloc:
train_sample = _prepare_sample(**sample) + "\n\n"
# make sure the length of training samples does not exceed maximum length
if train_samples_length + len(tokenizer.encode(train_sample)) > train_samples_max_length:
break
else:
train_samples += train_sample
train_samples_length += len(tokenizer.encode(train_sample))
else:
train_samples = ""
left = knowledge
# \n\n to split retrieved knowledge
right = "\n\n" + train_samples + _prepare_sample(query)
pair = tokenizer.encode(left, right, add_special_tokens=False, truncation="only_first", max_length=context_max_length - int(has_bos) - int(has_eos))
# strip spaces and \n in the head (when there is no retrieved passage)
seq = tokenizer.decode(pair).strip()
inputs = tokenizer(seq, return_token_type_ids=False)
if has_eos and not is_encoder_decoder:
inputs = remove_eos(inputs, tokenizer.eos_token_id)
inputs["query_id"] = query_id
for k, v in inputs.items():
output[k] = v
return output
return _process
def evaluate_popqa(eval_data, save_path, **kwds):
def compute_metric(eval_preds):
makedirs(save_path)
samples = {}
with open(eval_data) as f:
for line in f:
sample = json.loads(line.strip())
samples[sample["query_id"]] = sample
accuracy = 0
with open(save_path, "w") as f:
for query_id, generation in zip(*eval_preds):
sample = samples[query_id]
answers = sample['possible_answers']
correct = False
for answer in answers:
# if any answer matches
if answer in generation or answer.lower() in generation or answer.capitalize() in generation:
correct = True
break
accuracy += int(correct)
sample["output"] = generation
f.write(json.dumps(sample, ensure_ascii=False) + "\n")
accuracy /= len(eval_preds[0])
return {"accuracy": accuracy}
return compute_metric
def main():
parser = HfArgumentParser([PopQAArgs, GenerationArgs])
args, generation_args = parser.parse_args_into_dataclasses()
accelerator = Accelerator(cpu=args.cpu)
# modify the output_dir for retrieval
if args.retrieval_method == "dense":
output_dir = os.path.join(args.output_dir, args.query_encoder.strip(os.sep).replace(os.sep, "--"))
else:
output_dir = os.path.join(args.output_dir, args.retrieval_method)
args.output_dir = output_dir
if args.retrieval_method != "no":
retrieval_main(args=args, accelerator=accelerator, log=False)
eval_data = RetrievalMetric._get_save_path(args.eval_data, args.output_dir, field="key", save_name=args.save_name)
else:
eval_data = args.eval_data
llm = LM(
model_name_or_path=args.model_name_or_path,
dtype=args.lm_dtype,
device_map=args.lm_device_map,
padding_side=args.padding_side,
cache_dir=args.model_cache_dir,
accelerator=accelerator,
generation_args=asdict(generation_args)
)
tokenizer = llm.tokenizer
logging.info(f"Loading data from {eval_data}...")
with accelerator.main_process_first():
dataset = datasets.load_dataset("json", data_files=eval_data, split="train", cache_dir=args.dataset_cache_dir)
dataset = dataset.map(process_popqa(
tokenizer,
context_max_length=args.context_max_length,
key_num=args.key_num,
few_shot=args.few_shot,
# popqa extracts few-shot examples from test data
train_data=args.eval_data,
cache_dir=args.dataset_cache_dir,
is_encoder_decoder=llm.model.config.is_encoder_decoder
), remove_columns=dataset.column_names, batched=True, num_proc=32)
data_collator = DefaultDataCollator(tokenizer=tokenizer, add_position_ids=args.add_position_ids)
dataloader = DataLoader(
dataset,
batch_size=args.lm_batch_size,
collate_fn=data_collator,
pin_memory=True,
)
dataloader = accelerator.prepare(dataloader)
results = llm.generate(dataloader)
if accelerator.process_index == 0:
file_logger = FileLogger(makedirs(args.log_path))
result_path = os.path.join(args.output_dir, args.model_name_or_path.strip(os.sep).replace(os.sep, "--") + ".json")
metrics = evaluate_popqa(eval_data, result_path)(results)
file_logger.log(metrics, Args=asdict(args))
if __name__ == "__main__":
main()
FlagEmbedding/llm_embedder/evaluation/eval_qa.py 0 → 100644
View file @ f75058c7
import os
import json
import logging
import datasets
import random
from typing import List
from accelerate import Accelerator
from torch.utils.data import DataLoader
from transformers import HfArgumentParser
from dataclasses import dataclass, field, asdict
from src.lm import (
LM,
LMArgs,
GenerationArgs
)
from src.retrieval import (
RetrievalArgs,
RetrievalMetric,
)
from src.utils.util import makedirs, remove_eos, normalize_text, DefaultDataCollator, DatasetProcessFn, FileLogger
from .eval_retrieval import main as retrieval_main
logger = logging.getLogger(__name__)
@dataclass
class QAArgs(LMArgs, RetrievalArgs):
output_dir: str = field(
default="data/results/qa/",
)
eval_data: str = field(
default="llm-embedder:qa/nq/test.json",
metadata={'help': 'Path to the test file.'}
)
lm_batch_size: int = field(
default=4,
metadata={'help': 'Evaluation batch size.'},
)
few_shot: int = field(
default=10,
metadata={'help': 'How many few shot train samples?'},
)
train_data: str = field(
default="llm-embedder:qa/nq/dev.json",
metadata={'help': 'Path to the file containing training examples.'}
)
hits: int = field(
default=10,
metadata={'help': 'How many hits per query?'},
)
key_num: int = field(
default=3,
metadata={'help': 'How many docs to provide in prompt?'},
)
corpus: str = field(
default="llm-embedder:qa/nq/corpus.json",
metadata={'help': 'Corpus path for retrieval.'}
)
key_template: str = field(
default="{title} {text}",
metadata={'help': 'How to concatenate columns in the corpus to form one key?'}
)
query_max_length: int = field(
default=32,
metadata={'help': 'How many tokens at maximum in a query.'}
)
key_max_length: int = field(
default=128,
metadata={'help': 'How many tokens at maximum in a key.'}
)
metrics: List[str] = field(
default_factory=lambda: ["collate_key"],
)
save_to_output: bool = field(
default=True,
metadata={'help': 'Save the result/key/negative to output_dir? If not true, they will be saved next to the eval_data.'}
)
log_path: str = field(
default="data/results/qa/qa.log",
metadata={'help': 'Path to the file for logging.'}
)
@dataclass
class GenerationArgs(GenerationArgs):
max_new_tokens: int = field(
default=32,
metadata={'help': 'Maximum new tokens to generate.'}
)
eos_token_id: int = 13
def process_qa(tokenizer, context_max_length=2048, key_num=3, few_shot=0, train_data=None, cache_dir=None, is_encoder_decoder=False):
test = tokenizer("test", return_special_tokens_mask=True)["special_tokens_mask"]
has_bos = has_eos = False
if test[0] == 1:
has_bos = True
if test[-1] == 1:
has_eos = True
if few_shot > 0:
assert train_data is not None
train_dataset = datasets.load_dataset("json", data_files=train_data, cache_dir=cache_dir, split="train")
sample_indices = random.sample(range(len(train_dataset)), few_shot)
train_dataset = train_dataset.select(sample_indices)
def _prepare_sample(query, answers=None, **kwds):
sample = f"Question: {query}\nAnswer:"
if answers is not None:
sample = sample + " " + random.choice(answers)
return sample
def _prepare_retrieval(keys):
if keys is not None:
keys = keys[:key_num]
keys = "\n".join(keys)
keys = f"Knowledge: {keys}"
else:
keys = ""
return keys
@DatasetProcessFn()
def _process(query, query_id, key=None, **kwds):
"""Yield keys and query with a prompt template"""
output = {}
query = query.strip()
knowledge = _prepare_retrieval(key)
train_samples_max_length = context_max_length - len(tokenizer.encode("\n\n" if len(knowledge) else "" + _prepare_sample(query), add_special_tokens=False)) - int(has_bos)
if few_shot > 0:
train_samples = ""
train_samples_length = 0
for i in range(few_shot):
train_sample = train_dataset[i]
train_sample = _prepare_sample(**train_sample) + "\n\n"
if train_samples_length + len(tokenizer.encode(train_sample)) > train_samples_max_length:
break
else:
train_samples += train_sample
train_samples_length += len(tokenizer.encode(train_sample))
else:
train_samples = ""
left = knowledge
# \n\n to split retrieved knowledge
right = "\n\n" + train_samples + _prepare_sample(query)
pair = tokenizer.encode(left, right, add_special_tokens=False, truncation="only_first", max_length=context_max_length - int(has_bos) - int(has_eos))
# strip spaces and \n in the head (when there is no retrieved passage)
seq = tokenizer.decode(pair).strip()
inputs = tokenizer(seq, return_token_type_ids=False)
if has_eos and not is_encoder_decoder:
inputs = remove_eos(inputs, tokenizer.eos_token_id)
inputs["query_id"] = query_id
for k, v in inputs.items():
output[k] = v
return output
return _process
def evaluate_qa(eval_data, save_path, **kwds):
def compute_metric(eval_preds):
makedirs(save_path)
samples = {}
with open(eval_data) as f:
for line in f:
sample = json.loads(line.strip())
samples[sample["query_id"]] = sample
exact_match = 0
with open(save_path, "w") as f:
for query_id, generation in zip(*eval_preds):
sample = samples[query_id]
em = max(normalize_text(generation) == normalize_text(answer) for answer in sample["answers"])
exact_match += int(em)
sample["output"] = generation
f.write(json.dumps(sample, ensure_ascii=False) + "\n")
exact_match /= len(eval_preds[0])
return {"exact_match": exact_match}
return compute_metric
def main():
parser = HfArgumentParser([QAArgs, GenerationArgs])
args, generation_args = parser.parse_args_into_dataclasses()
accelerator = Accelerator(cpu=args.cpu)
# modify the output_dir for retrieval
if args.retrieval_method == "dense":
output_dir = os.path.join(args.output_dir, args.query_encoder.strip(os.sep).replace(os.sep, "--"))
else:
output_dir = os.path.join(args.output_dir, args.retrieval_method)
args.output_dir = output_dir
if args.retrieval_method != "no":
retrieval_main(args=args, accelerator=accelerator, log=False)
eval_data = RetrievalMetric._get_save_path(args.eval_data, args.output_dir, field="key", save_name=args.save_name)
else:
eval_data = args.eval_data
llm = LM(
model_name_or_path=args.model_name_or_path,
dtype=args.lm_dtype,
device_map=args.lm_device_map,
padding_side=args.padding_side,
cache_dir=args.model_cache_dir,
accelerator=accelerator,
generation_args=asdict(generation_args)
)
tokenizer = llm.tokenizer
logging.info(f"Loading data from {eval_data}...")
with accelerator.main_process_first():
dataset = datasets.load_dataset("json", data_files=eval_data, split="train", cache_dir=args.dataset_cache_dir)
dataset = dataset.map(process_qa(
tokenizer,
context_max_length=args.context_max_length,
key_num=args.key_num,
few_shot=args.few_shot,
train_data=args.train_data,
cache_dir=args.dataset_cache_dir,
is_encoder_decoder=llm.model.config.is_encoder_decoder
), remove_columns=dataset.column_names, batched=True, num_proc=32)
data_collator = DefaultDataCollator(tokenizer=tokenizer, add_position_ids=args.add_position_ids)
dataloader = DataLoader(
dataset,
batch_size=args.lm_batch_size,
collate_fn=data_collator,
pin_memory=True,
)
dataloader = accelerator.prepare(dataloader)
results = llm.generate(dataloader)
if accelerator.process_index == 0:
file_logger = FileLogger(makedirs(args.log_path))
result_path = os.path.join(args.output_dir, args.model_name_or_path.strip(os.sep).replace(os.sep, "--") + ".json")
metrics = evaluate_qa(eval_data, result_path)(results)
file_logger.log(metrics, Args=asdict(args))
if __name__ == "__main__":
main()
FlagEmbedding/llm_embedder/evaluation/eval_qrecc.py 0 → 100644
View file @ f75058c7
import os
import json
import logging
import datasets
import random
from typing import List
from accelerate import Accelerator
from torch.utils.data import DataLoader
from transformers import HfArgumentParser
from dataclasses import dataclass, field, asdict
from src.lm import (
LM,
LMArgs,
GenerationArgs
)
from src.retrieval import (
RetrievalArgs,
RetrievalMetric,
)
from src.utils.util import makedirs, remove_eos, normalize_text, DefaultDataCollator, DatasetProcessFn, FileLogger
from .eval_retrieval import main as retrieval_main
from .icl_utils import compute_metrics
logger = logging.getLogger(__name__)
@dataclass
class QRECCArgs(LMArgs, RetrievalArgs):
output_dir: str = field(
default="data/results/qrecc",
)
eval_data: str = field(
default="llm-embedder:convsearch/qrecc/test.concat.json",
metadata={'help': 'Query jsonl.'}
)
corpus: str = field(
default="llm-embedder:convsearch/qrecc/corpus.json",
metadata={'help': 'Corpus path for retrieval.'}
)
key_template: str = field(
default="{text}",
metadata={'help': 'How to concatenate columns in the corpus to form one key?'}
)
do_generate: bool = field(
default=False,
metadata={'help': 'Generate for computing qa metrics?'}
)
hits: int = field(
default=100,
metadata={'help': 'How many hits per query?'},
)
key_num: int = field(
default=3,
metadata={'help': 'How many docs to provide in prompt?'},
)
metrics: List[str] = field(
default_factory=lambda: ["ndcg", "recall", "collate_key"],
)
cutoffs: List[int] = field(
default_factory=lambda: [3, 10, 100],
metadata={'help': 'Cutoffs to evaluate retrieval metrics.'}
)
max_neg_num: int = field(
default=32,
metadata={'help': 'Maximum negative number to mine.'}
)
save_to_output: bool = field(
default=True,
metadata={'help': 'Save the result/key/negative to output_dir? If not true, they will be saved next to the eval_data.'}
)
log_path: str = field(
default="data/results/qrecc/qrecc.log",
metadata={'help': 'Path to the file for logging.'}
)
@dataclass
class GenerationArgs(GenerationArgs):
max_new_tokens: int = field(
default=128,
metadata={'help': 'Maximum new tokens to generate.'}
)
eos_token_id: int = 13
def process_qrecc(tokenizer, context_max_length=2048, key_num=3, is_encoder_decoder=False):
test = tokenizer("test", return_special_tokens_mask=True)["special_tokens_mask"]
has_bos = has_eos = False
if test[0] == 1:
has_bos = True
if test[-1] == 1:
has_eos = True
def _prepare_sample(query, answers=None, **kwds):
sample = f"Context and Question: {query}\nAnswer:"
if answers is not None:
sample = sample + " " + random.choice(answers)
return sample
def _prepare_retrieval(keys):
if keys is not None:
keys = keys[:key_num]
keys = "\n".join(keys)
knowledge = f"Knowledge: {keys}"
else:
knowledge = ""
return knowledge
@DatasetProcessFn()
def _process(query, query_id, key=None, **kwds):
"""Yield keys and query with a prompt template"""
output = {}
query = query.strip()
knowledge = _prepare_retrieval(key)
left = knowledge
# \n\n to split retrieved knowledge
right = "\n\n" + _prepare_sample(query)
pair = tokenizer.encode(left, right, add_special_tokens=False, truncation="only_first", max_length=context_max_length - int(has_bos) - int(has_eos))
# strip spaces and \n in the head (when there is no retrieved passage)
seq = tokenizer.decode(pair).strip()
inputs = tokenizer(seq, return_token_type_ids=False)
if has_eos and not is_encoder_decoder:
inputs = remove_eos(inputs, tokenizer.eos_token_id)
inputs["query_id"] = query_id
for k, v in inputs.items():
output[k] = v
return output
return _process
def evaluate_qrecc(eval_data, save_path, **kwds):
def compute_metric(eval_preds):
makedirs(save_path)
samples = {}
with open(eval_data) as f:
for line in f:
sample = json.loads(line.strip())
samples[sample["query_id"]] = sample["answers"][0]
preds = []
answers = []
with open(save_path, "w") as f:
for query_id, generation in zip(*eval_preds):
answer = samples[query_id]
preds.append(generation)
answers.append(answer)
sample["output"] = generation
f.write(json.dumps(sample, ensure_ascii=False) + "\n")
rouge_l = compute_metrics("rl", labels=answers, preds=preds)
return rouge_l
return compute_metric
def main():
parser = HfArgumentParser([QRECCArgs, GenerationArgs])
args, generation_args = parser.parse_args_into_dataclasses()
accelerator = Accelerator(cpu=args.cpu)
# modify the output_dir for retrieval
if args.retrieval_method == "dense":
output_dir = os.path.join(args.output_dir, args.query_encoder.strip(os.sep).replace(os.sep, "--"))
else:
output_dir = os.path.join(args.output_dir, args.retrieval_method)
args.output_dir = output_dir
if args.retrieval_method != "no":
# retrieval metrics computes ndcg and recall
_, _, metrics = retrieval_main(args=args, accelerator=accelerator, log=False)
eval_data = RetrievalMetric._get_save_path(args.eval_data, args.output_dir, field="key", save_name=args.save_name)
else:
eval_data = args.eval_data
metrics = {}
if args.do_generate:
llm = LM(
model_name_or_path=args.model_name_or_path,
dtype=args.lm_dtype,
device_map=args.lm_device_map,
padding_side=args.padding_side,
cache_dir=args.model_cache_dir,
accelerator=accelerator,
generation_args=asdict(generation_args)
)
tokenizer = llm.tokenizer
logging.info(f"Loading data from {eval_data}...")
with accelerator.main_process_first():
dataset = datasets.load_dataset("json", data_files=eval_data, split="train", cache_dir=args.dataset_cache_dir)
dataset = dataset.map(process_qrecc(
tokenizer,
context_max_length=args.context_max_length,
key_num=args.key_num,
is_encoder_decoder=llm.model.config.is_encoder_decoder
), remove_columns=dataset.column_names, batched=True, num_proc=32)
data_collator = DefaultDataCollator(tokenizer=tokenizer, add_position_ids=args.add_position_ids)
dataloader = DataLoader(
dataset,
batch_size=args.lm_batch_size,
collate_fn=data_collator,
pin_memory=True,
)
dataloader = accelerator.prepare(dataloader)
results = llm.generate(dataloader)
if accelerator.process_index == 0:
result_path = os.path.join(args.output_dir, args.model_name_or_path.strip(os.sep).replace(os.sep, "--") + ".json")
lm_metrics = evaluate_qrecc(eval_data, result_path)(results)
else:
lm_metrics = {}
if accelerator.process_index == 0:
file_logger = FileLogger(makedirs(args.log_path))
metrics.update(lm_metrics)
file_logger.log(metrics, Args=asdict(args))
if __name__ == "__main__":
main()
FlagEmbedding/llm_embedder/evaluation/eval_retrieval.py 0 → 100644
View file @ f75058c7
import os
import torch
import logging
import datasets
from typing import List
from accelerate import Accelerator
from transformers import HfArgumentParser
from dataclasses import dataclass, field, asdict
from src.retrieval import (
RetrievalArgs,
Retriever,
RetrievalDataset,
RetrievalMetric,
TASK_CONFIG,
)
from src.utils.util import makedirs, FileLogger
logger = logging.getLogger(__name__)
@dataclass
class Args(RetrievalArgs):
eval_data: str = field(
default=None,
metadata={'help': 'Query jsonl.'}
)
output_dir: str = field(
default="data/outputs/",
)
corpus: str = field(
default=None,
metadata={'help': 'Corpus path for retrieval.'}
)
key_template: str = field(
default="{title} {text}",
metadata={'help': 'How to concatenate columns in the corpus to form one key?'}
)
log_path: str = field(
default="data/results/performance.log",
metadata={'help': 'Path to the file for logging.'}
)
def main(args, accelerator=None, log=True):
if accelerator is None:
accelerator = Accelerator(cpu=args.cpu)
with accelerator.main_process_first():
config = TASK_CONFIG[args.version]
instruction = config["instruction"]
# we should get the evaluation task before specifying instruction
# NOTE: only dense retrieval needs instruction
if args.eval_data is not None and args.add_instruction and args.retrieval_method == "dense":
raw_eval_dataset = datasets.load_dataset('json', data_files=args.eval_data, split='train', cache_dir=args.dataset_cache_dir)
eval_task = raw_eval_dataset[0]["task"]
else:
eval_task = None
eval_dataset = RetrievalDataset.prepare_eval_dataset(
data_file=args.eval_data,
cache_dir=args.dataset_cache_dir,
instruction=instruction[eval_task] if eval_task is not None else None,
)
corpus = RetrievalDataset.prepare_corpus(
data_file=args.corpus,
key_template=args.key_template,
cache_dir=args.dataset_cache_dir,
instruction=instruction[eval_task] if eval_task is not None else None
)
result_path = RetrievalMetric._get_save_path(args.eval_data, args.output_dir, field="result", save_name=args.save_name)
if args.load_result:
query_ids, preds = RetrievalMetric._load_result(result_path)
else:
retriever = Retriever(
retrieval_method=args.retrieval_method,
# for dense retriever
query_encoder=args.query_encoder,
key_encoder=args.key_encoder,
pooling_method=args.pooling_method,
dense_metric=args.dense_metric,
query_max_length=args.query_max_length,
key_max_length=args.key_max_length,
tie_encoders=args.tie_encoders,
truncation_side=args.truncation_side,
cache_dir=args.model_cache_dir,
dtype=args.dtype,
accelerator=accelerator,
# for bm25 retriever
anserini_dir=args.anserini_dir,
k1=args.k1,
b=args.b
)
retriever.index(
corpus,
output_dir=args.output_dir,
# for dense retriever
embedding_name=args.embedding_name,
index_factory=args.faiss_index_factory,
load_encode=args.load_encode,
save_encode=args.save_encode,
load_index=args.load_index,
save_index=args.save_index,
batch_size=args.batch_size,
# for bm25 retriever
threads=args.threads,
language=args.language,
storeDocvectors=args.storeDocvectors,
load_collection=args.load_collection,
)
query_ids, preds = retriever.search(
eval_dataset=eval_dataset,
hits=args.hits,
# for dense retriever
batch_size=args.batch_size,
)
del retriever
torch.cuda.empty_cache()
if args.save_result and accelerator.process_index == 0:
RetrievalMetric._save_result(query_ids, preds, result_path)
if accelerator.process_index == 0:
# NOTE: this corpus is for computing metrics, where no instruction is given
no_instruction_corpus = RetrievalDataset.prepare_corpus(
data_file=args.corpus,
key_template=args.key_template,
cache_dir=args.dataset_cache_dir,
)
metrics = RetrievalMetric.get_metric_fn(
args.metrics,
cutoffs=args.cutoffs,
eval_data=args.eval_data,
corpus=no_instruction_corpus,
save_name=args.save_name,
output_dir=args.output_dir,
save_to_output=args.save_to_output,
max_neg_num=args.max_neg_num,
cache_dir=args.dataset_cache_dir,
filter_answers=args.filter_answers,
)(query_ids, preds)
if log:
file_logger = FileLogger(makedirs(args.log_path))
file_logger.log(metrics, Args=asdict(args))
else:
metrics = {}
accelerator.wait_for_everyone()
return query_ids, preds, metrics
if __name__ == "__main__":
parser = HfArgumentParser([Args])
args, = parser.parse_args_into_dataclasses()
main(args)
FlagEmbedding/llm_embedder/evaluation/eval_tool.py 0 → 100644
View file @ f75058c7
import os
import logging
from typing import List
from dataclasses import dataclass, field
from transformers import HfArgumentParser
from src.retrieval import (
RetrievalArgs,
)
from .eval_retrieval import main
logger = logging.getLogger(__name__)
@dataclass
class ToolArgs(RetrievalArgs):
output_dir: str = field(
default="data/results/tool",
)
eval_data: str = field(
default="llm-embedder:tool/toolbench/test.json",
metadata={'help': 'Query jsonl.'}
)
corpus: str = field(
default="llm-embedder:tool/toolbench/corpus.json",
metadata={'help': 'Corpus path for retrieval.'}
)
key_template: str = field(
default="{text}",
metadata={'help': 'How to concatenate columns in the corpus to form one key?'}
)
cutoffs: List[int] = field(
default_factory=lambda: [1,3,5],
metadata={'help': 'Cutoffs to evaluate retrieval metrics.'}
)
max_neg_num: int = field(
default=32,
metadata={'help': 'Maximum negative number to mine.'}
)
log_path: str = field(
default="data/results/tool/toolbench.log",
metadata={'help': 'Path to the file for logging.'}
)
if __name__ == "__main__":
parser = HfArgumentParser([ToolArgs])
args, = parser.parse_args_into_dataclasses()
if args.retrieval_method == "dense":
output_dir = os.path.join(args.output_dir, args.query_encoder.strip(os.sep).replace(os.sep, "--"))
args.output_dir = output_dir
else:
output_dir = os.path.join(args.output_dir, args.retrieval_method)
main(args)
\ No newline at end of file
FlagEmbedding/llm_embedder/evaluation/icl_utils.py 0 → 100644
View file @ f75058c7
import collections
import re
import string
import copy
import logging
import numpy as np
from sklearn.metrics import f1_score
from typing import List, Dict
from rouge import Rouge
from transformers.tokenization_utils import PreTrainedTokenizer
logger = logging.getLogger(__name__)
def _normalize_answer(text, punc_chars, punc_repl):
"""Lower text and remove punctuation, articles and extra whitespace."""
def remove_articles(s):
return re.sub(r"\b(a|an|the)\b", " ", s)
def replace_punctuation(s):
to_replace = set(punc_chars)
return "".join(punc_repl if ch in to_replace else ch for ch in s)
def white_space_fix(s):
return " ".join(s.split())
text = text.lower()
text = replace_punctuation(text)
text = remove_articles(text)
text = white_space_fix(text)
return text
def normalize_squad(answer):
"""Normalization used in official SQuAD evaluation script."""
return _normalize_answer(answer, punc_chars=string.punctuation, punc_repl="")
def _metric_max_over_ground_truths(metric_fn, ground_truths, prediction):
"""Computes the maximum of the metric over all ground truths."""
return max(
metric_fn(ground_truth, prediction) for ground_truth in ground_truths
)
def _exact_match_score(target, prediction):
return target == prediction
def _f1_score(target, prediction):
"""Computes token f1 score for a single target and prediction."""
prediction_tokens = prediction.split()
target_tokens = target.split()
common = (collections.Counter(prediction_tokens) &
collections.Counter(target_tokens))
num_same = sum(common.values())
if num_same == 0:
return 0
precision = 1.0 * num_same / len(prediction_tokens)
recall = 1.0 * num_same / len(target_tokens)
f1 = (2 * precision * recall) / (precision + recall)
return f1
def qa_metrics(targets, predictions, return_list=False):
"""Computes exact match and f1 QA scores, expecting pre-normalized text."""
if len(targets) != len(predictions):
raise ValueError("Number of targets and predictions must match.")
if return_list:
em=[
_metric_max_over_ground_truths(_exact_match_score, t, p)
for p, t in zip(predictions, targets)
]
f1=[
_metric_max_over_ground_truths(_f1_score, t, p)
for p, t in zip(predictions, targets)
]
return em, f1
em = np.mean([
_metric_max_over_ground_truths(_exact_match_score, t, p)
for p, t in zip(predictions, targets)
])
f1 = np.mean([
_metric_max_over_ground_truths(_f1_score, t, p)
for p, t in zip(predictions, targets)
])
# em *= 100
# f1 *= 100
logger.info("EM = %.2f, F1 = %.2f", em, f1)
#return {"em": em, "f1": f1}
return em, f1
class App:
def __init__(self):
self.functions = {}
def add(self, key):
def adder(func):
self.functions[key] = func
return func
return adder
def __getitem__(self, __name: str):
return self.functions[__name]
metric_dict = App()
@metric_dict.add("rouge")
def rouge(preds, labels, return_list=False):
# https://github.com/pltrdy/rouge
r1s, r2s, rls = [], [], []
r = Rouge()
for i in range(len(labels)):
if "\n" not in preds[i]:
preds[i] += "\n" # to ensure rouge metrics
if "\n" not in labels[i]:
labels[i] += "\n"
scores = r.get_scores(preds[i], labels[i])[0]
r1s.append(scores["rouge-1"]["f"])
r2s.append(scores["rouge-2"]["f"])
rls.append(scores["rouge-l"]["f"])
if return_list: # used for scoring data
return r1s
r1 = sum(r1s) / len(r1s)
r2 = sum(r2s) / len(r2s)
rl = sum(rls) / len(rls)
return r1, r2, rl
@metric_dict.add("squad")
def squad(labels, preds, return_list=False):
"""Computes SQuAD metrics, maximizing over answers per question.
Args:
labels: list of lists of strings
preds: list of strings
Returns:
dict with score_key: squad score across all labels and predictions
"""
labels = [[normalize_squad(t) for t in u] for u in labels]
preds = [normalize_squad(p) for p in preds]
if return_list: # used for scoring data
em, f1 = qa_metrics(labels, preds, return_list=True)
return f1
em, f1 = qa_metrics(labels, preds) # em,f1
return em, f1
@metric_dict.add("simple_accuracy")
def simple_accuracy(preds, labels, return_list=False):
if isinstance(preds[0], str):
labels = [label.strip() for label in labels]
preds = [pred.strip() for pred in preds]
res = [int(preds[i] == labels[i]) for i in range(len(preds))]
if return_list:
return res
acc = sum(res) / len(res)
return acc
def compute_metrics(metric, labels, preds):
assert len(preds) == len(labels)
if metric == "acc":
return {"acc": simple_accuracy(preds, labels)}
elif metric == "rl":
r1, r2, rl = rouge(preds, labels)
# return {"r1": r1, "r2": r2, "rl": rl}
return {"rl": rl}
elif metric == "f1":
f1 = f1_score(y_true=labels, y_pred=preds, pos_label='1')
return {"f1": f1}
elif metric == "em":
em, f1 = squad(labels=labels, preds=preds)
# return {"em": em, "f1": f1}
return {"em": em}
def compute_scores(metric, preds, labels):
if not isinstance(preds[0], str):
preds = np.array(preds)
labels = np.array(labels)
scores = compute_metrics(metric, labels=labels, preds=preds)
return scores
def flat_options(data):
flat_data = []
for e in data:
for option in e['options']:
flat_data.append({"query":e['query'], "few_shot":e['few_shot'], 'input_answer':option})
return flat_data
def perplexity_to_choice(data, perplexity):
inx = 0
results = []
for e in data:
cur_perplexity = []
for _ in e['options']:
cur_perplexity.append(perplexity[inx])
inx += 1
ans = np.argmin(cur_perplexity)
results.append(str(ans))
return results
def get_length(tokenizer, text):
tokenized_example = tokenizer.encode_plus(text,truncation=False, return_tensors='pt')
shape = tokenized_example.input_ids.squeeze().shape
if len(shape)==0:
return 1
else:
return int(shape[0])
def get_prompt_length(tokenizer, prompts_list, question, n_tokens_in_prompt: int=1024):
lengths_list = [get_length(tokenizer, prompt) for prompt in prompts_list]
q_length = get_length(tokenizer, question)
max_prompts = np.searchsorted(np.cumsum(lengths_list), n_tokens_in_prompt - q_length)
return max_prompts
def _llm_generation_func(examples: Dict[str, List],
tokenizer: PreTrainedTokenizer,
example_num: int=8,
max_input_tokens: int=1024,
add_llama_inst: bool=False):
texts = []
n_tokens_in_prompt = max_input_tokens
if add_llama_inst:
n_tokens_in_prompt -= 8
for i in range(len(examples['query'])):
prompts_list = examples['few_shot'][i][::-1]
max_prompts = get_prompt_length(
tokenizer=tokenizer,
prompts_list=prompts_list,
question=examples['query'][i],
n_tokens_in_prompt=n_tokens_in_prompt
)
example_num = min(example_num, max_prompts)
inputs = prompts_list[:example_num]
inputs.append(examples['query'][i])
if add_llama_inst:
inputs = "[INST] " + "\n".join(inputs) + " [/INST]"
else:
inputs = "\n".join(inputs)+'\n'
texts.append(inputs)
return tokenizer(texts, return_tensors="pt", padding=True, max_length=1024, return_token_type_ids=False)
def _llm_perplexity_func(examples: Dict[str, List],
tokenizer: PreTrainedTokenizer,
example_num: int=8,
max_input_tokens: int=1024,
add_llama_inst: bool=False):
texts = []
answers = []
n_tokens_in_prompt = max_input_tokens
if add_llama_inst:
n_tokens_in_prompt -= 8
for i in range(len(examples['query'])):
prompts_list = examples['few_shot'][i][::-1]
max_prompts = get_prompt_length(tokenizer=tokenizer,
prompts_list=prompts_list,
question=examples['query'][i],
n_tokens_in_prompt=n_tokens_in_prompt)
example_num = min(example_num, max_prompts)
inputs = prompts_list[:example_num]
inputs.append(examples['query'][i])
if add_llama_inst:
# NOTE: two more spaces after [/INST]
inputs = "[INST] " + "\n".join(inputs) + " [/INST] " + examples['input_answer'][i].lstrip()
else:
inputs = "\n".join(inputs)+'\n ' + examples['input_answer'][i] # add a space after \n to split input and answer
texts.append(inputs)
answers.append(examples['input_answer'][i])
inputs = tokenizer(texts, return_tensors="pt", padding=True, return_token_type_ids=False)
labels = copy.deepcopy(inputs['input_ids'])
for i, ans in enumerate(answers):
ans_ids = tokenizer.encode(ans, add_special_tokens=False)
labels[i][:-len(ans_ids)] = -100
inputs['labels'] = labels
return inputs
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment