Initial commit

internvl_chat/eval/mme/Your_Results/existence.txt

+000000006040.jpg	Is there a train in this image? Please answer yes or no.	Yes
+000000006040.jpg	Is there a bed in this image? Please answer yes or no.	No
+000000006471.jpg	Is there a baseball bat in this image? Please answer yes or no.	Yes
+000000006471.jpg	Is there a giraffe in this image? Please answer yes or no.	No
+000000007108.jpg	Is there a elephant in this image? Please answer yes or no.	Yes
+000000007108.jpg	Is there a hair drier in this image? Please answer yes or no.	No
+000000007816.jpg	Is there a motorcycle in this image? Please answer yes or no.	Yes
+000000007816.jpg	Is there a airplane in this image? Please answer yes or no.	No
+000000007977.jpg	Is there a skateboard in this image? Please answer yes or no.	Yes
+000000007977.jpg	Is there a spoon in this image? Please answer yes or no.	No
+000000008844.jpg	Is there a person in this image? Please answer yes or no.	Yes
+000000008844.jpg	Is there a sink in this image? Please answer yes or no.	No
+000000009590.jpg	Is there a bottle in this image? Please answer yes or no.	Yes
+000000009590.jpg	Is there a scissors in this image? Please answer yes or no.	No
+000000010363.jpg	Is there a bottle in this image? Please answer yes or no.	Yes
+000000010363.jpg	Is there a apple in this image? Please answer yes or no.	No
+000000011197.jpg	Is there a car in this image? Please answer yes or no.	Yes
+000000011197.jpg	Is there a fork in this image? Please answer yes or no.	No
+000000015254.jpg	Is there a spoon in this image? Please answer yes or no.	Yes
+000000015254.jpg	Is there a donut in this image? Please answer yes or no.	No
+000000015517.jpg	Is there a bus in this image? Please answer yes or no.	Yes
+000000015517.jpg	Is there a cow in this image? Please answer yes or no.	No
+000000015746.jpg	Is there a fire hydrant in this image? Please answer yes or no.	Yes
+000000015746.jpg	Is there a person in this image? Please answer yes or no.	No
+000000037751.jpg	Is there a backpack in this image? Please answer yes or no.	Yes
+000000037751.jpg	Is there a microwave in this image? Please answer yes or no.	No
+000000050145.jpg	Is there a bicycle in this image? Please answer yes or no.	Yes
+000000050145.jpg	Is there a apple in this image? Please answer yes or no.	No
+000000061418.jpg	Is there a chair in this image? Please answer yes or no.	Yes
+000000061418.jpg	Is there a airplane in this image? Please answer yes or no.	No
+000000417779.jpg	Is there a car in this image? Please answer yes or no.	Yes
+000000417779.jpg	Is there a kite in this image? Please answer yes or no.	No
+000000424521.jpg	Is there a skateboard in this image? Please answer yes or no.	Yes
+000000424521.jpg	Is there a banana in this image? Please answer yes or no.	No
+000000438304.jpg	Is there a sports ball in this image? Please answer yes or no.	Yes
+000000438304.jpg	Is there a horse in this image? Please answer yes or no.	No
+000000494427.jpg	Is there a laptop in this image? Please answer yes or no.	Yes
+000000494427.jpg	Is there a potted plant in this image? Please answer yes or no.	No
+000000495448.jpg	Is there a cake in this image? Please answer yes or no.	Yes
+000000495448.jpg	Is there a tie in this image? Please answer yes or no.	No
+000000498463.jpg	Is there a refrigerator in this image? Please answer yes or no.	Yes
+000000498463.jpg	Is there a donut in this image? Please answer yes or no.	No
+000000519039.jpg	Is there a truck in this image? Please answer yes or no.	Yes
+000000519039.jpg	Is there a book in this image? Please answer yes or no.	No
+000000523241.jpg	Is there a car in this image? Please answer yes or no.	Yes
+000000523241.jpg	Is there a cell phone in this image? Please answer yes or no.	No
+000000530162.jpg	Is there a umbrella in this image? Please answer yes or no.	Yes
+000000530162.jpg	Is there a horse in this image? Please answer yes or no.	No
+000000537812.jpg	Is there a chair in this image? Please answer yes or no.	Yes
+000000537812.jpg	Is there a baseball bat in this image? Please answer yes or no.	No
+000000541952.jpg	Is there a clock in this image? Please answer yes or no.	Yes
+000000541952.jpg	Is there a bottle in this image? Please answer yes or no.	No
+000000546626.jpg	Is there a bottle in this image? Please answer yes or no.	Yes
+000000546626.jpg	Is there a mouse in this image? Please answer yes or no.	No
+000000556000.jpg	Is there a chair in this image? Please answer yes or no.	Yes
+000000556000.jpg	Is there a dog in this image? Please answer yes or no.	No
+000000557258.jpg	Is there a toilet in this image? Please answer yes or no.	Yes
+000000557258.jpg	Is there a pizza in this image? Please answer yes or no.	No
+000000572956.jpg	Is there a motorcycle in this image? Please answer yes or no.	Yes
+000000572956.jpg	Is there a bus in this image? Please answer yes or no.	No

internvl_chat/eval/mme/Your_Results/landmark.txt

+0006adf999ccc899.jpg	Is this a photo of Muktinath? Please answer yes or no.	Yes
+0006adf999ccc899.jpg	Is this a photo of Lohikoski Church? Please answer yes or no.	No
+0015309638e5a9b1.jpg	Is this a picture of Teufelsmauer (Harz)? Please answer yes or no.	Yes
+0015309638e5a9b1.jpg	Is this a picture of Romfo kirke? Please answer yes or no.	No
+001f0bfe8464254a.jpg	Is this a picture of Saint Catherine of Alexandria church in Warsaw? Please answer yes or no.	Yes
+001f0bfe8464254a.jpg	Is this a picture of Kaap Skil? Please answer yes or no.	No
+0022666a36b80ea5.jpg	Is this an image of Hall of Supreme Harmony (Forbidden City)? Please answer yes or no.	Yes
+0022666a36b80ea5.jpg	Is this an image of Brickegickel? Please answer yes or no.	No
+0023bd687cbc0a44.jpg	Is this a photo of Utrecht Dom Tower? Please answer yes or no.	Yes
+0023bd687cbc0a44.jpg	Is this a photo of Great Palace Mosaic Museum? Please answer yes or no.	No
+00261324328490fc.jpg	Is this an image of Loch Ossian? Please answer yes or no.	Yes
+00261324328490fc.jpg	Is this an image of Tanemakidaishi? Please answer yes or no.	No
+002ae473a76362a6.jpg	Is this an image of Gerlache Strait? Please answer yes or no.	Yes
+002ae473a76362a6.jpg	Is this an image of Schmiedebruch (Naturschutzgebiet)? Please answer yes or no.	No
+003f8af9477a8eb8.jpg	Is this a photo of Le Destroit? Please answer yes or no.	Yes
+003f8af9477a8eb8.jpg	Is this a photo of Kharkov Mosque? Please answer yes or no.	No
+004543cec064cd0b.jpg	Is this a picture of Wollaton Park? Please answer yes or no.	Yes
+004543cec064cd0b.jpg	Is this a picture of St John's Church, Blackpool? Please answer yes or no.	No
+004beea6937b3a8c.jpg	Is this a photo of Brussels Park? Please answer yes or no.	Yes
+004beea6937b3a8c.jpg	Is this a photo of Forest Hills Cemetery (Boston, Massachusetts)? Please answer yes or no.	No
+005150f80de6f487.jpg	Is this a photo of Schloss Burgpreppach? Please answer yes or no.	Yes
+005150f80de6f487.jpg	Is this a photo of Pinkas Synagogue? Please answer yes or no.	No
+0053894534d1e259.jpg	Is this a picture of Goethe- und Schiller-Denkmal (Weimar)? Please answer yes or no.	Yes
+0053894534d1e259.jpg	Is this a picture of Grey River? Please answer yes or no.	No
+0059fc817c5fd5b7.jpg	Is this a photo of KLCC Park? Please answer yes or no.	Yes
+0059fc817c5fd5b7.jpg	Is this a photo of Neath Castle? Please answer yes or no.	No
+005c68530bb6ed7c.jpg	Is this a photo of Balvenie Castle? Please answer yes or no.	Yes
+005c68530bb6ed7c.jpg	Is this a photo of Draper Utah Temple? Please answer yes or no.	No
+006e3342b57aa3b4.jpg	Is this a photo of Larvs kyrka? Please answer yes or no.	Yes
+006e3342b57aa3b4.jpg	Is this a photo of Geitgallien? Please answer yes or no.	No
+0075a4d212539625.jpg	Is this a photo of Cardiff Castle? Please answer yes or no.	Yes
+0075a4d212539625.jpg	Is this a photo of Boudhanath? Please answer yes or no.	No
+008053e6bb5afa33.jpg	Is this a photo of Rochers des Mourres? Please answer yes or no.	Yes
+008053e6bb5afa33.jpg	Is this a photo of Schloss Weinfelden? Please answer yes or no.	No
+00c1349a9622390c.jpg	Is this a photo of Amar Jawan Jyoti (Tomb of the Unknown Soldier)? Please answer yes or no.	Yes
+00c1349a9622390c.jpg	Is this a photo of Porta Garibaldi (Marsala)? Please answer yes or no.	No
+00cae9f165d8d2cc.jpg	Is this an image of Fisherman's Bastion? Please answer yes or no.	Yes
+00cae9f165d8d2cc.jpg	Is this an image of Equestrian statue of George IV, Trafalgar Square, London? Please answer yes or no.	No
+00f4fb71575efb0b.jpg	Is this a picture of 1965)? Please answer yes or no.	Yes
+00f4fb71575efb0b.jpg	Is this a picture of Plaza Monumental de Toros de Pueblo Nuevo? Please answer yes or no.	No
+00fa476690a260a8.jpg	Is this an image of Bathampton Toll Bridge? Please answer yes or no.	Yes
+00fa476690a260a8.jpg	Is this an image of Moose River Plains Wild Forest? Please answer yes or no.	No
+010260e62e7d847d.jpg	Is this a photo of Visingsborg? Please answer yes or no.	Yes
+010260e62e7d847d.jpg	Is this a photo of Teutsche Schule (Schleusingen)? Please answer yes or no.	No
+010b5b5c9c461f65.jpg	Is this an image of Het Steen? Please answer yes or no.	Yes
+010b5b5c9c461f65.jpg	Is this an image of Castello di Montalto Dora? Please answer yes or no.	No
+01114538f95f1ce8.jpg	Is this a picture of Svirzh Castle? Please answer yes or no.	Yes
+01114538f95f1ce8.jpg	Is this a picture of Bistrica, Ionian Sea? Please answer yes or no.	No
+01121b9b3c79afc4.jpg	Is this a photo of Alamannenmuseum Ellwangen? Please answer yes or no.	Yes
+01121b9b3c79afc4.jpg	Is this a photo of Sint-Jozefkerk, Enschede? Please answer yes or no.	No
+011a19afda5df3c7.jpg	Is this a picture of San Prospero (Reggio Emilia)? Please answer yes or no.	Yes
+011a19afda5df3c7.jpg	Is this a picture of Lake DeFuniak? Please answer yes or no.	No
+012856462a95ada9.jpg	Is this a picture of Chingshui Cliff? Please answer yes or no.	Yes
+012856462a95ada9.jpg	Is this a picture of St. John's Church, Portland? Please answer yes or no.	No
+012ea376a16f17bf.jpg	Is this a picture of Church of Saint Giles in Prague? Please answer yes or no.	Yes
+012ea376a16f17bf.jpg	Is this a picture of Pfarrkirche St. Martin an der Raab? Please answer yes or no.	No
+013e1a1e887cb915.jpg	Is this a picture of Real Casa de Correos, Madrid? Please answer yes or no.	Yes
+013e1a1e887cb915.jpg	Is this a picture of Ayton Castle, Scottish Borders? Please answer yes or no.	No
+0140a7d5446719c3.jpg	Is this a photo of Friday Mosque, Herat? Please answer yes or no.	Yes
+0140a7d5446719c3.jpg	Is this a photo of Hexenturm (Burg (bei Magdeburg))? Please answer yes or no.	No
+01416124a7ab211a.jpg	Is this a photo of Ajuntament de Mollerussa? Please answer yes or no.	Yes
+01416124a7ab211a.jpg	Is this a photo of Episcopal Diocese of Northern Indiana? Please answer yes or no.	No
+01425c4d35880aad.jpg	Is this an image of Museu de Sant Boi de Llobregat? Please answer yes or no.	Yes
+01425c4d35880aad.jpg	Is this an image of Bezdonnoye Lake, Moscow Oblast? Please answer yes or no.	No
+014c50e91cf799c7.jpg	Is this a photo of Clearwater Beach, Florida? Please answer yes or no.	Yes
+014c50e91cf799c7.jpg	Is this a photo of Church of Holy Trinity in Vratislavice nad Nisou? Please answer yes or no.	No
+015543ddea466c54.jpg	Is this a picture of Galician market square in Skansen, Sanok? Please answer yes or no.	Yes
+015543ddea466c54.jpg	Is this a picture of Walworth Castle? Please answer yes or no.	No
+0169c4235270f65b.jpg	Is this a photo of St. Ricarius Church, Aberford? Please answer yes or no.	Yes
+0169c4235270f65b.jpg	Is this a photo of Schloss Prebberede? Please answer yes or no.	No
+016e68628ea6c7ee.jpg	Is this an image of Jewish Ceremonial Hall, Prague-Josefov? Please answer yes or no.	Yes
+016e68628ea6c7ee.jpg	Is this an image of Our Lady Church (Ettlingen)? Please answer yes or no.	No
+01862535ce8961ee.jpg	Is this an image of Mont Tendre? Please answer yes or no.	Yes
+01862535ce8961ee.jpg	Is this an image of Ogden Utah Temple? Please answer yes or no.	No
+0186742e7963fb3c.jpg	Is this a photo of Ozegahara? Please answer yes or no.	Yes
+0186742e7963fb3c.jpg	Is this a photo of San Giovenale (Orvieto)? Please answer yes or no.	No
+01928acd6dfe6cdd.jpg	Is this a picture of Watergate Bay? Please answer yes or no.	Yes
+01928acd6dfe6cdd.jpg	Is this a picture of Cairnpapple Hill? Please answer yes or no.	No
+01a6842099d62f22.jpg	Is this an image of Clark Veterans Cemetery? Please answer yes or no.	Yes
+01a6842099d62f22.jpg	Is this an image of Mirdif City Centre? Please answer yes or no.	No
+01aca2d30b43d05f.jpg	Is this a picture of Monte Minerva (Villanova Monteleone)? Please answer yes or no.	Yes
+01aca2d30b43d05f.jpg	Is this a picture of Dorfkirche Gnandstein? Please answer yes or no.	No
+01b12048bab7439f.jpg	Is this a photo of Kaptai Lake? Please answer yes or no.	Yes
+01b12048bab7439f.jpg	Is this a photo of Castle of the Conception? Please answer yes or no.	No
+01b715ceb7050bce.jpg	Is this a picture of Kalamita fortress? Please answer yes or no.	Yes
+01b715ceb7050bce.jpg	Is this a picture of Augsburger Hotelturm? Please answer yes or no.	No
+01b9b58a573c45f0.jpg	Is this a picture of St Mary Redcliffe? Please answer yes or no.	Yes
+01b9b58a573c45f0.jpg	Is this a picture of Kirche zur Heiligsten Dreifaltigkeit, Vienna? Please answer yes or no.	No
+01bc5557ed2616ad.jpg	Is this a photo of Intercession of the Theotokos church in Lutsk? Please answer yes or no.	Yes
+01bc5557ed2616ad.jpg	Is this a photo of Cap 110? Please answer yes or no.	No
+01c0e0ff84631905.jpg	Is this a photo of Ribblehead Viaduct? Please answer yes or no.	Yes
+01c0e0ff84631905.jpg	Is this a photo of Church of Saint George in Koptevo? Please answer yes or no.	No
+01d6e386c4a01ab3.jpg	Is this a picture of Rotes Rathaus? Please answer yes or no.	Yes
+01d6e386c4a01ab3.jpg	Is this a picture of Mahipatgad? Please answer yes or no.	No
+01d9bef0120557a5.jpg	Is this a photo of Andronikov Monastery? Please answer yes or no.	Yes
+01d9bef0120557a5.jpg	Is this a photo of Ghatkopar? Please answer yes or no.	No
+01e198bea8285816.jpg	Is this an image of Saint Basil church, Volodymyr-Volynskyi? Please answer yes or no.	Yes
+01e198bea8285816.jpg	Is this an image of Hoher Markt, Vienna? Please answer yes or no.	No
+01f23b4c5253e9c6.jpg	Is this a picture of Goethes Gartenhaus? Please answer yes or no.	Yes
+01f23b4c5253e9c6.jpg	Is this a picture of Lamba, Shetland Islands? Please answer yes or no.	No
+01ffc77bdf7fe084.jpg	Is this a picture of Oldemorstoft? Please answer yes or no.	Yes
+01ffc77bdf7fe084.jpg	Is this a picture of Roman Catholic Diocese of Antwerp? Please answer yes or no.	No
+0200cce957e9c4a6.jpg	Is this a photo of Malmuerta Tower? Please answer yes or no.	Yes
+0200cce957e9c4a6.jpg	Is this a photo of Monte Lema? Please answer yes or no.	No
+0202aac6b0bfbcd8.jpg	Is this a picture of Gettysburg National Military Park? Please answer yes or no.	Yes
+0202aac6b0bfbcd8.jpg	Is this a picture of Auron? Please answer yes or no.	No
+0205343362e60daf.jpg	Is this a picture of Church of Our Lady of Sorrows in Riga? Please answer yes or no.	Yes
+0205343362e60daf.jpg	Is this a picture of Simon en Judaskerk (Ootmarsum)? Please answer yes or no.	No
+020ae4dd487a8ab0.jpg	Is this a photo of Kashueti Church? Please answer yes or no.	Yes
+020ae4dd487a8ab0.jpg	Is this a photo of Colorado State Capitol? Please answer yes or no.	No
+02259144c3f094f0.jpg	Is this a picture of The American Adventure? Please answer yes or no.	Yes
+02259144c3f094f0.jpg	Is this a picture of Langenwaldschanze? Please answer yes or no.	No
+0237fc92f6b31aeb.jpg	Is this a photo of Abbaye de Mortemer? Please answer yes or no.	Yes
+0237fc92f6b31aeb.jpg	Is this a photo of St Thomas Rest Park? Please answer yes or no.	No
+0240727dbd2b8531.jpg	Is this a photo of Nicolaikerk Appingedam? Please answer yes or no.	Yes
+0240727dbd2b8531.jpg	Is this a photo of Cutimbo? Please answer yes or no.	No
+02545ca051c6c480.jpg	Is this a photo of Podilski Tovtry National Nature Park? Please answer yes or no.	Yes
+02545ca051c6c480.jpg	Is this a photo of Lyndonville, Vermont? Please answer yes or no.	No
+02598153523880aa.jpg	Is this a photo of Marilao Church? Please answer yes or no.	Yes
+02598153523880aa.jpg	Is this a photo of Madliena, Malta? Please answer yes or no.	No
+02737bb70957827d.jpg	Is this an image of White Pagoda in Zhakou? Please answer yes or no.	Yes
+02737bb70957827d.jpg	Is this an image of Norrstrandskyrkan? Please answer yes or no.	No
+0273f127895337be.jpg	Is this a photo of Ringkirche? Please answer yes or no.	Yes
+0273f127895337be.jpg	Is this a photo of Huxi Mosque? Please answer yes or no.	No
+02785530b6ac4f56.jpg	Is this a picture of Church of Nicholas the Wet? Please answer yes or no.	Yes
+02785530b6ac4f56.jpg	Is this a picture of Razula? Please answer yes or no.	No
+028348883685db69.jpg	Is this an image of Saint Andrew the Apostle Church? Please answer yes or no.	Yes
+028348883685db69.jpg	Is this an image of EYE Film Institute Netherlands? Please answer yes or no.	No
+029431b10472c228.jpg	Is this a picture of Santa Caterina (Sassari)? Please answer yes or no.	Yes
+029431b10472c228.jpg	Is this a picture of Former Tainan Assembly Hall? Please answer yes or no.	No
+0295877d5cd76190.jpg	Is this a picture of Groothoofdspoort? Please answer yes or no.	Yes
+0295877d5cd76190.jpg	Is this a picture of 707 17th Street? Please answer yes or no.	No
+02b16429304b00b7.jpg	Is this a picture of De Haag? Please answer yes or no.	Yes
+02b16429304b00b7.jpg	Is this a picture of Villa Melchett? Please answer yes or no.	No
+02b7f3ae549582f1.jpg	Is this a photo of Pont-canal de Digoin? Please answer yes or no.	Yes
+02b7f3ae549582f1.jpg	Is this a photo of Trenton Battle Monument? Please answer yes or no.	No
+02d11e392d3ab93f.jpg	Is this an image of Santa Monica Parish Church in Angat, Bulacan? Please answer yes or no.	Yes
+02d11e392d3ab93f.jpg	Is this an image of Bangkok City Pillar Shrine? Please answer yes or no.	No
+02e38cd5a3850f60.jpg	Is this a photo of Palazzo Cavalli Franchetti (Venice)? Please answer yes or no.	Yes
+02e38cd5a3850f60.jpg	Is this a photo of Isbister Chambered Cairn? Please answer yes or no.	No
+02e3e418198427e7.jpg	Is this an image of Goldener Reiter? Please answer yes or no.	Yes
+02e3e418198427e7.jpg	Is this an image of Palacio de la Mosquera? Please answer yes or no.	No
+030d9a26b7d5755e.jpg	Is this a photo of Medvedgrad? Please answer yes or no.	Yes
+030d9a26b7d5755e.jpg	Is this a photo of Tammisto? Please answer yes or no.	No
+0317376043fb00eb.jpg	Is this a photo of Butler's Wharf? Please answer yes or no.	Yes
+0317376043fb00eb.jpg	Is this a photo of Church of Saints Philip and James (Lelekovice)? Please answer yes or no.	No
+03280db41fd4e4f5.jpg	Is this a picture of Oxford Street? Please answer yes or no.	Yes
+03280db41fd4e4f5.jpg	Is this a picture of Blake Fell? Please answer yes or no.	No
+034733eb61d663cd.jpg	Is this a picture of Colonna dell'Immacolata (Rome)? Please answer yes or no.	Yes
+034733eb61d663cd.jpg	Is this a picture of Brookmill Park? Please answer yes or no.	No
+034d066523bb20d1.jpg	Is this a photo of Ibrahim-al-Ibrahim Mosque? Please answer yes or no.	Yes
+034d066523bb20d1.jpg	Is this a photo of Atami Castle? Please answer yes or no.	No
+036b60d84c6ebc63.jpg	Is this a photo of Frederik V statue in Amalienborg Palace? Please answer yes or no.	Yes
+036b60d84c6ebc63.jpg	Is this a photo of Karl-Bittel-Park? Please answer yes or no.	No
+03729394d18f7e9a.jpg	Is this an image of Sapporo Clock Tower? Please answer yes or no.	Yes
+03729394d18f7e9a.jpg	Is this an image of Saint Nikolaus Church (Beuster)? Please answer yes or no.	No
+0384fd0664c0ca62.jpg	Is this an image of St. Martin (Sontheim)? Please answer yes or no.	Yes
+0384fd0664c0ca62.jpg	Is this an image of Hot Creek (Mono County, California)? Please answer yes or no.	No
+038bdff81be029a6.jpg	Is this an image of Fosso Reale (Livorno)? Please answer yes or no.	Yes
+038bdff81be029a6.jpg	Is this an image of Farlington Marshes? Please answer yes or no.	No
+039bbdf71cd17b3f.jpg	Is this a picture of Qiantang River Bridge? Please answer yes or no.	Yes
+039bbdf71cd17b3f.jpg	Is this a picture of Casa Museo Boschi Di Stefano (Milan)? Please answer yes or no.	No
+03aa5dd2e3aab548.jpg	Is this a photo of Central Police Station? Please answer yes or no.	Yes
+03aa5dd2e3aab548.jpg	Is this a photo of Kontai-ji? Please answer yes or no.	No
+03c4b1901e4195f4.jpg	Is this an image of St. Johanniskirche Eppendorf? Please answer yes or no.	Yes
+03c4b1901e4195f4.jpg	Is this an image of Maidenhead Bridge? Please answer yes or no.	No
+03c8b2c1306c0038.jpg	Is this an image of Ermita de Sant Joan (Montserrat)? Please answer yes or no.	Yes
+03c8b2c1306c0038.jpg	Is this an image of Hamilton Grange National Memorial? Please answer yes or no.	No
+03d5e3bfc958be38.jpg	Is this an image of Pietrarsa railway museum? Please answer yes or no.	Yes
+03d5e3bfc958be38.jpg	Is this an image of Washita Battlefield National Historic Site? Please answer yes or no.	No
+03dc7df1f1df7b8f.jpg	Is this a picture of Yevhen Paton Bridge? Please answer yes or no.	Yes
+03dc7df1f1df7b8f.jpg	Is this a picture of Minoan Villa of Makrygialos? Please answer yes or no.	No
+03f65f52c0077b6c.jpg	Is this a photo of Reading Lighthouse? Please answer yes or no.	Yes
+03f65f52c0077b6c.jpg	Is this a photo of St Nicholas Kirk, Aberdeen? Please answer yes or no.	No
+03f873280d814ae7.jpg	Is this a photo of Clarence Dock (Leeds)? Please answer yes or no.	Yes
+03f873280d814ae7.jpg	Is this a photo of Madonna delle Grazie (Roccastrada)? Please answer yes or no.	No
+03f8abf9f8473a6f.jpg	Is this a picture of Parque nacional del Teide? Please answer yes or no.	Yes
+03f8abf9f8473a6f.jpg	Is this a picture of Church of Saint Mary the Virgin, Skiemonys? Please answer yes or no.	No
+03fb778d388ad9a0.jpg	Is this an image of Mercado Central de Santiago? Please answer yes or no.	Yes
+03fb778d388ad9a0.jpg	Is this an image of Castelo de Alegrete? Please answer yes or no.	No
+041da60e053c5906.jpg	Is this a picture of Schloss Weesenstein? Please answer yes or no.	Yes
+041da60e053c5906.jpg	Is this a picture of Mas Cabrafiga? Please answer yes or no.	No
+042ca5291cefe0b2.jpg	Is this a picture of Pfarrkirche hl. Augustinus, Perchtoldsdorf? Please answer yes or no.	Yes
+042ca5291cefe0b2.jpg	Is this a picture of Mark Twain Riverboat? Please answer yes or no.	No
+0443da4f4fcdea52.jpg	Is this an image of Port of Funchal? Please answer yes or no.	Yes
+0443da4f4fcdea52.jpg	Is this an image of Rauscherpark? Please answer yes or no.	No
+04531b37dfe5c991.jpg	Is this a picture of Roebling Suspension Bridge? Please answer yes or no.	Yes
+04531b37dfe5c991.jpg	Is this a picture of St. James Episcopal Church (Wilmington, North Carolina)? Please answer yes or no.	No
+04534ba775aa38ad.jpg	Is this an image of Cork City Gaol? Please answer yes or no.	Yes
+04534ba775aa38ad.jpg	Is this an image of Taramati Baradari? Please answer yes or no.	No
+04590c1648dd25a0.jpg	Is this an image of Zhejiang Sci-Tech University? Please answer yes or no.	Yes
+04590c1648dd25a0.jpg	Is this an image of Khalti Lake? Please answer yes or no.	No
+045b65ea196dac3f.jpg	Is this a photo of Republican architecture in Manizales? Please answer yes or no.	Yes
+045b65ea196dac3f.jpg	Is this a photo of Kayauchi-banta Cliff? Please answer yes or no.	No
+045ec5631638d290.jpg	Is this a picture of Baldachin of Saint Peter's Basilica? Please answer yes or no.	Yes
+045ec5631638d290.jpg	Is this a picture of Jean Baptiste Point Du Sable Homesite? Please answer yes or no.	No
+04a5226570dce6c3.jpg	Is this a picture of Church of the Assumption of the Virgin Mary in Chrudim? Please answer yes or no.	Yes
+04a5226570dce6c3.jpg	Is this a picture of Queen Elizabeth Country Park? Please answer yes or no.	No
+04b6c84ce89461ec.jpg	Is this an image of Tomb of Xerxes I? Please answer yes or no.	Yes
+04b6c84ce89461ec.jpg	Is this an image of Park HaMesila? Please answer yes or no.	No
+04d67117549e07a8.jpg	Is this a photo of Laguna de Salinas? Please answer yes or no.	Yes
+04d67117549e07a8.jpg	Is this a photo of Raspyatsky Monastery (Serpukhov)? Please answer yes or no.	No
+04ebe2885134597f.jpg	Is this a picture of The South Africa (Delville Wood) National Memorial? Please answer yes or no.	Yes
+04ebe2885134597f.jpg	Is this a picture of Imogiri? Please answer yes or no.	No
+050464e4ebc0b36d.jpg	Is this an image of Tai Wong Ye Temple, Wong Chuk Hang? Please answer yes or no.	Yes
+050464e4ebc0b36d.jpg	Is this an image of Hierve el Agua? Please answer yes or no.	No
+05095dd2b6715fc3.jpg	Is this an image of Sampsonievsky Cathedral? Please answer yes or no.	Yes
+05095dd2b6715fc3.jpg	Is this an image of I. and E. Greenwald Steam Engine No. 1058? Please answer yes or no.	No
+05685b1070b828f5.jpg	Is this a photo of Wurstelprater? Please answer yes or no.	Yes
+05685b1070b828f5.jpg	Is this a photo of Nanmyaebonthar Sannandawya Sandamuni Pagoda? Please answer yes or no.	No
+05b01b255cec2bfb.jpg	Is this a photo of Puthoorppilly Sree Krishnaswamy Temple? Please answer yes or no.	Yes
+05b01b255cec2bfb.jpg	Is this a photo of Villa Reale (Marlia)? Please answer yes or no.	No
+05bcf05b4460d2c3.jpg	Is this an image of Museo archeologico nazionale dell'Umbria? Please answer yes or no.	Yes
+05bcf05b4460d2c3.jpg	Is this an image of Mode Gakuen Cocoon Tower? Please answer yes or no.	No
+05c3aa404351f407.jpg	Is this an image of Pasaje Gutierrez (Valladolid)? Please answer yes or no.	Yes
+05c3aa404351f407.jpg	Is this an image of Pieve di Santa Maria a Carli? Please answer yes or no.	No
+05d56e249af839e7.jpg	Is this a photo of Serbian Orthodox Cathedral in Sarajevo? Please answer yes or no.	Yes
+05d56e249af839e7.jpg	Is this a photo of Seewaldsee (Sankt Koloman)? Please answer yes or no.	No
+06134984e2aa9bcc.jpg	Is this a picture of Grote Kerk (Groede)? Please answer yes or no.	Yes
+06134984e2aa9bcc.jpg	Is this a picture of Goldeck? Please answer yes or no.	No
+0613d7e8f292ee15.jpg	Is this a photo of Museo Archeologico G. Rambotti? Please answer yes or no.	Yes
+0613d7e8f292ee15.jpg	Is this a photo of Sanrei? Please answer yes or no.	No
+062298815e9197e7.jpg	Is this a photo of Liuyang Confucius Temple? Please answer yes or no.	Yes
+062298815e9197e7.jpg	Is this a photo of Astley Castle? Please answer yes or no.	No
+06483ba2bff074bc.jpg	Is this an image of Museo Universitario del Chopo? Please answer yes or no.	Yes
+06483ba2bff074bc.jpg	Is this an image of Cedar Park, Seattle, Washington? Please answer yes or no.	No
+06740b93ad1f6c02.jpg	Is this a picture of Borisoglebsky Monastery (Borisoglebsky)? Please answer yes or no.	Yes
+06740b93ad1f6c02.jpg	Is this a picture of Aletsch Arena? Please answer yes or no.	No
+067ea27ec08c9354.jpg	Is this a picture of Schlosskirche St. Aegidien (Bernburg)? Please answer yes or no.	Yes
+067ea27ec08c9354.jpg	Is this a picture of The Haining? Please answer yes or no.	No
+06a85d0b5d70881d.jpg	Is this an image of Silwan? Please answer yes or no.	Yes
+06a85d0b5d70881d.jpg	Is this an image of Jagdschloss Platte? Please answer yes or no.	No
+06ab45292ecd8bf7.jpg	Is this a picture of Maribor Regional Museum? Please answer yes or no.	Yes
+06ab45292ecd8bf7.jpg	Is this a picture of Myres Castle? Please answer yes or no.	No
+06b6531dab42ccab.jpg	Is this a picture of Roland (Riga)? Please answer yes or no.	Yes
+06b6531dab42ccab.jpg	Is this a picture of Wudangshan Shangdi Temple? Please answer yes or no.	No
+06d039ff97afef04.jpg	Is this a picture of Arch of Galerius (Thessaloniki)? Please answer yes or no.	Yes
+06d039ff97afef04.jpg	Is this a picture of Catholic Church of Wanchin? Please answer yes or no.	No
+06e453b06a3e4054.jpg	Is this an image of Pendeen Lighthouse? Please answer yes or no.	Yes
+06e453b06a3e4054.jpg	Is this an image of Silberbach (Heubach)? Please answer yes or no.	No
+06e927f784642ce4.jpg	Is this a picture of Museo Delta Antico? Please answer yes or no.	Yes
+06e927f784642ce4.jpg	Is this a picture of OSTEC Exhibition Hall? Please answer yes or no.	No
+070c8d6326a87397.jpg	Is this an image of Lanxmeerpoort? Please answer yes or no.	Yes
+070c8d6326a87397.jpg	Is this an image of Tramway du Cap-Ferret? Please answer yes or no.	No
+0713451824a443d4.jpg	Is this a picture of Castillo de Sax? Please answer yes or no.	Yes
+0713451824a443d4.jpg	Is this a picture of Roman Catholic Diocese of Orlando? Please answer yes or no.	No
+0743e58605497887.jpg	Is this an image of Capitoline temple (Brescia)? Please answer yes or no.	Yes
+0743e58605497887.jpg	Is this an image of Burg Horn? Please answer yes or no.	No
+075264ec41057049.jpg	Is this a picture of Imatrankoski? Please answer yes or no.	Yes
+075264ec41057049.jpg	Is this a picture of Jarlsberg travbane? Please answer yes or no.	No
+07551cf96af77299.jpg	Is this a photo of Friedhof Wilmersdorf? Please answer yes or no.	Yes
+07551cf96af77299.jpg	Is this a photo of Partisan cemetery in Mostar? Please answer yes or no.	No
+075f8d86303a020e.jpg	Is this a photo of Church of the Nativity (Bethlehem)? Please answer yes or no.	Yes
+075f8d86303a020e.jpg	Is this a photo of Pennsylvania Railroad Office Building? Please answer yes or no.	No
+07ac52cf4620519f.jpg	Is this a photo of San Pietro (Reggio Emilia)? Please answer yes or no.	Yes
+07ac52cf4620519f.jpg	Is this a photo of Plaza de toros de La Condomina? Please answer yes or no.	No
+07b04584e67cb39d.jpg	Is this a picture of Smithfield, Dublin? Please answer yes or no.	Yes
+07b04584e67cb39d.jpg	Is this a picture of Novyi Svit Sanctuary? Please answer yes or no.	No
+07ed43dd266b669b.jpg	Is this a photo of Rocky Mountain National Park? Please answer yes or no.	Yes
+07ed43dd266b669b.jpg	Is this a photo of Col-des-Roches? Please answer yes or no.	No
+083dd5a3ba3d96dc.jpg	Is this a picture of Teddington Cemetery? Please answer yes or no.	Yes
+083dd5a3ba3d96dc.jpg	Is this a picture of Khutir Nadia? Please answer yes or no.	No
+08442f24f3c25565.jpg	Is this a photo of Abdijkerk Loosduinen? Please answer yes or no.	Yes
+08442f24f3c25565.jpg	Is this a photo of Rinsey Head? Please answer yes or no.	No
+08a0d2fd3fe6dc95.jpg	Is this a picture of Dreifaltigkeitskirche (Ulm)? Please answer yes or no.	Yes
+08a0d2fd3fe6dc95.jpg	Is this a picture of Wat Pa Mamuang? Please answer yes or no.	No
+08c43cd8f2be6709.jpg	Is this a picture of Parchi di Nervi? Please answer yes or no.	Yes
+08c43cd8f2be6709.jpg	Is this a picture of Altlayer Schweiz? Please answer yes or no.	No
+09232aeaf46dec64.jpg	Is this a picture of Canadian Museum of Nature? Please answer yes or no.	Yes
+09232aeaf46dec64.jpg	Is this a picture of Gwangtonggwan? Please answer yes or no.	No
+092d0859ff6424de.jpg	Is this a photo of Kasteel Lunenburg? Please answer yes or no.	Yes
+092d0859ff6424de.jpg	Is this a photo of Thomas Paine Cottage? Please answer yes or no.	No
+092e075e8e44c228.jpg	Is this a picture of ChaoTianGong? Please answer yes or no.	Yes
+092e075e8e44c228.jpg	Is this a picture of Pygmalion Theater Wien? Please answer yes or no.	No
+093ad5564a004653.jpg	Is this a picture of Crescent Lake (Dunhuang)? Please answer yes or no.	Yes
+093ad5564a004653.jpg	Is this a picture of Kirjurinluoto Arena? Please answer yes or no.	No
+0962855b67159733.jpg	Is this a photo of Monastery of Monsalud? Please answer yes or no.	Yes
+0962855b67159733.jpg	Is this a photo of ToonSeum? Please answer yes or no.	No
+09a51d517f1a72d5.jpg	Is this a picture of Ford's Theatre? Please answer yes or no.	Yes
+09a51d517f1a72d5.jpg	Is this a picture of Burg Hachen? Please answer yes or no.	No
+09ae0628a75e4fcc.jpg	Is this a picture of Onze-Lieve-Vrouw Hemelvaartkerk (Zottegem)? Please answer yes or no.	Yes
+09ae0628a75e4fcc.jpg	Is this a picture of Salvation Army in Canada? Please answer yes or no.	No
+09b7acf87c8046ec.jpg	Is this an image of Church of the Epiphany (Yaroslavl)? Please answer yes or no.	Yes
+09b7acf87c8046ec.jpg	Is this an image of Szlenkier Palace? Please answer yes or no.	No
+09b94a2f304a540c.jpg	Is this a photo of Abbaye Notre-Dame du Val? Please answer yes or no.	Yes
+09b94a2f304a540c.jpg	Is this a photo of Hotarumachi, Osaka? Please answer yes or no.	No
+09ce89cbf237da03.jpg	Is this a picture of Church of Saint Mary Magdalene (Brno)? Please answer yes or no.	Yes
+09ce89cbf237da03.jpg	Is this a picture of Wain Wath Force? Please answer yes or no.	No
+0a3035bfca2ab920.jpg	Is this an image of Ortigia? Please answer yes or no.	Yes
+0a3035bfca2ab920.jpg	Is this an image of Montmayeur castle? Please answer yes or no.	No
+0ab2ed007db301d5.jpg	Is this a picture of Highgate Cemetery? Please answer yes or no.	Yes
+0ab2ed007db301d5.jpg	Is this a picture of Isola Maggiore? Please answer yes or no.	No
+0acc12d1261d99ca.jpg	Is this an image of Wat Chedi Si Hong? Please answer yes or no.	Yes
+0acc12d1261d99ca.jpg	Is this an image of Leschi Park? Please answer yes or no.	No
+0adb088e24d88123.jpg	Is this a picture of Iglesia de Santo Domingo (La Serena)? Please answer yes or no.	Yes
+0adb088e24d88123.jpg	Is this a picture of Canelles de Baix (la Vall de Bianya)? Please answer yes or no.	No
+0ae3355e0a54db21.jpg	Is this a photo of Grimspound? Please answer yes or no.	Yes
+0ae3355e0a54db21.jpg	Is this a photo of Lough Owel? Please answer yes or no.	No
+0b4240b5ef69362e.jpg	Is this a photo of Saint Anne church in Nowa Ruda? Please answer yes or no.	Yes
+0b4240b5ef69362e.jpg	Is this a photo of Bray Wick? Please answer yes or no.	No
+0b64944086932c90.jpg	Is this a picture of Alte Pfarrkirche St. Martin (Moosach)? Please answer yes or no.	Yes
+0b64944086932c90.jpg	Is this a picture of Maya Devi Temple, Lumbini? Please answer yes or no.	No
+0b8fa4152f1bc3ed.jpg	Is this a picture of Lac du Salagou? Please answer yes or no.	Yes
+0b8fa4152f1bc3ed.jpg	Is this a picture of Holsljunga kyrka? Please answer yes or no.	No
+0ba6fa7ade050d49.jpg	Is this a photo of Churches in Zemaiciu Kalvarija? Please answer yes or no.	Yes
+0ba6fa7ade050d49.jpg	Is this a photo of Waiola Church? Please answer yes or no.	No
+0bd72080836eac97.jpg	Is this a picture of Castelo de Sesimbra? Please answer yes or no.	Yes
+0bd72080836eac97.jpg	Is this a picture of Gouffre Berger? Please answer yes or no.	No
+0bdd954254310e73.jpg	Is this a photo of Melrose House? Please answer yes or no.	Yes
+0bdd954254310e73.jpg	Is this a photo of Former water tower (Tsubame, Niigata)? Please answer yes or no.	No
+0c06fe97f38f6f88.jpg	Is this a photo of Belvoir Castle? Please answer yes or no.	Yes
+0c06fe97f38f6f88.jpg	Is this a photo of Su'ao Cold Spring? Please answer yes or no.	No
+0c6b0c5d2a401285.jpg	Is this a photo of Pelham Crescent? Please answer yes or no.	Yes
+0c6b0c5d2a401285.jpg	Is this a photo of Manila Ocean Park? Please answer yes or no.	No
+0cc5201e37961a1d.jpg	Is this a photo of Adare Manor? Please answer yes or no.	Yes
+0cc5201e37961a1d.jpg	Is this a photo of Leopoldspark? Please answer yes or no.	No
+0cd8fda9668b3ed2.jpg	Is this a picture of Church of the Holy Mandylion (Klyazma)? Please answer yes or no.	Yes
+0cd8fda9668b3ed2.jpg	Is this a picture of Kalayar Kovil? Please answer yes or no.	No
+0cf2e28266a0f1a1.jpg	Is this an image of Piazza Bra (Verona)? Please answer yes or no.	Yes
+0cf2e28266a0f1a1.jpg	Is this an image of Keeneland? Please answer yes or no.	No
+0cf48b827c9c99de.jpg	Is this a photo of Beatus Rhenanus Bridge? Please answer yes or no.	Yes
+0cf48b827c9c99de.jpg	Is this a photo of Ossian Hall? Please answer yes or no.	No
+0d23f6c345b68c05.jpg	Is this an image of Monte Olivia? Please answer yes or no.	Yes
+0d23f6c345b68c05.jpg	Is this an image of Crackstate? Please answer yes or no.	No
+0d70bceb2557d1f4.jpg	Is this an image of Spaso-Preobrazhensky Cathedral (Pereslavl-Zalessky)? Please answer yes or no.	Yes
+0d70bceb2557d1f4.jpg	Is this an image of Manta Rota? Please answer yes or no.	No
+0d7716f4aed0c9fa.jpg	Is this a photo of Willemsbrug in Rotterdam? Please answer yes or no.	Yes
+0d7716f4aed0c9fa.jpg	Is this a photo of Grumentum, Parco archeologico? Please answer yes or no.	No
+0d89a8a5e8c269e4.jpg	Is this an image of Burg Normannstein? Please answer yes or no.	Yes
+0d89a8a5e8c269e4.jpg	Is this an image of Cape Borda Lighthouse? Please answer yes or no.	No
+0e04b4f7f9801ea9.jpg	Is this a picture of Vase sculpture, Bremen? Please answer yes or no.	Yes
+0e04b4f7f9801ea9.jpg	Is this a picture of Mount Wilhelm? Please answer yes or no.	No
+0e0511c6e33636ca.jpg	Is this a picture of Victoria Stadium? Please answer yes or no.	Yes
+0e0511c6e33636ca.jpg	Is this a picture of Villa Belvedere (San Giuliano Terme)? Please answer yes or no.	No
+0e553d5d99e21c32.jpg	Is this a picture of Mustafa Pasha Bridge? Please answer yes or no.	Yes
+0e553d5d99e21c32.jpg	Is this a picture of Scrattons Eco Park? Please answer yes or no.	No
+0eb857c972ddc4c8.jpg	Is this an image of Nabq Protected Area? Please answer yes or no.	Yes
+0eb857c972ddc4c8.jpg	Is this an image of Maletsunyane Falls? Please answer yes or no.	No
+0f92862c0366ab60.jpg	Is this a photo of Cotehele? Please answer yes or no.	Yes
+0f92862c0366ab60.jpg	Is this a photo of Mozirje Grove? Please answer yes or no.	No
+0fc48f210ea7d363.jpg	Is this a photo of Schauenburg (Oberkirch)? Please answer yes or no.	Yes
+0fc48f210ea7d363.jpg	Is this a photo of Dartford Library? Please answer yes or no.	No
+100e5402cfb2dfcf.jpg	Is this a picture of Nanzenji? Please answer yes or no.	Yes
+100e5402cfb2dfcf.jpg	Is this a picture of Bagakain Lake? Please answer yes or no.	No
+101786bd2ec5800e.jpg	Is this a picture of Church of the Merciful Saviour in Kuskovo (Moscow)? Please answer yes or no.	Yes
+101786bd2ec5800e.jpg	Is this a picture of Central Board of Secondary Education? Please answer yes or no.	No
+1023b7ec812bb4c6.jpg	Is this a photo of St. Joseph's Cathedral, Dunedin? Please answer yes or no.	Yes
+1023b7ec812bb4c6.jpg	Is this a photo of Fagervik Church? Please answer yes or no.	No
+1053f4cebfceabd8.jpg	Is this an image of Woolwich foot tunnel? Please answer yes or no.	Yes
+1053f4cebfceabd8.jpg	Is this an image of Kirchenbibliothek St. Marien Barth? Please answer yes or no.	No
+10a944858a4283e0.jpg	Is this a picture of Harding Tomb? Please answer yes or no.	Yes
+10a944858a4283e0.jpg	Is this a picture of Kartause Ittingen? Please answer yes or no.	No
+10b2b8741f283a3c.jpg	Is this a photo of De Bataaf, Winterswijk? Please answer yes or no.	Yes
+10b2b8741f283a3c.jpg	Is this a photo of Porte Guillaume-Lion? Please answer yes or no.	No
+11b6bb2c35f4ee22.jpg	Is this an image of Ku-ring-gai Chase National Park? Please answer yes or no.	Yes
+11b6bb2c35f4ee22.jpg	Is this an image of Saint Mark's Coptic Orthodox Cathedral, Cairo? Please answer yes or no.	No
+127be7d34a21e5ed.jpg	Is this a picture of Beachy Head? Please answer yes or no.	Yes
+127be7d34a21e5ed.jpg	Is this a picture of Jindo Bridge? Please answer yes or no.	No
+130846b4f29fe4ba.jpg	Is this a photo of Duomo (Viterbo)? Please answer yes or no.	Yes
+130846b4f29fe4ba.jpg	Is this a photo of Raffles Praslin? Please answer yes or no.	No
+13a563a1a8ef38ae.jpg	Is this a photo of Torre del Rellotge d'Olesa de Montserrat? Please answer yes or no.	Yes
+13a563a1a8ef38ae.jpg	Is this a photo of Salzburg Museum? Please answer yes or no.	No
+14d3aba340d2fba9.jpg	Is this an image of Regard Saint-Martin? Please answer yes or no.	Yes
+14d3aba340d2fba9.jpg	Is this an image of Mount Donna Buang? Please answer yes or no.	No
+15b75eb6fca4aeb6.jpg	Is this a photo of City Duma building (Saint Petersburg)? Please answer yes or no.	Yes
+15b75eb6fca4aeb6.jpg	Is this a photo of Komadome-Hachiman-jinja? Please answer yes or no.	No
+15ec5d2c0898a56e.jpg	Is this a photo of Kids Marina Rotterdam? Please answer yes or no.	Yes
+15ec5d2c0898a56e.jpg	Is this a photo of Kirkwood Observatory? Please answer yes or no.	No
+15f326a030396dba.jpg	Is this a picture of Macquarie Centre? Please answer yes or no.	Yes
+15f326a030396dba.jpg	Is this a picture of California Historical Landmarks in San Mateo County, California? Please answer yes or no.	No
+162a77504f4e686e.jpg	Is this an image of Wieliczka Castle? Please answer yes or no.	Yes
+162a77504f4e686e.jpg	Is this an image of Naval Station Guantanamo Bay? Please answer yes or no.	No
+16b76bdd7eaaef1b.jpg	Is this a picture of Evangelische Kirche Weinfelden? Please answer yes or no.	Yes
+16b76bdd7eaaef1b.jpg	Is this a picture of Gorg Blau? Please answer yes or no.	No
+16d55614b78b6b8b.jpg	Is this an image of Church of the Assumption in Bishche? Please answer yes or no.	Yes
+16d55614b78b6b8b.jpg	Is this an image of Water of Girvan? Please answer yes or no.	No
+17e0cd732e35d4aa.jpg	Is this a photo of Millennium Tower (San Francisco, California)? Please answer yes or no.	Yes
+17e0cd732e35d4aa.jpg	Is this a photo of New Orleans Botanical Garden? Please answer yes or no.	No
+18608ca3b785abf8.jpg	Is this a picture of Solvognen? Please answer yes or no.	Yes
+18608ca3b785abf8.jpg	Is this a picture of Schenley Farms National Historic District? Please answer yes or no.	No
+197e440391d309f8.jpg	Is this a picture of Skansin? Please answer yes or no.	Yes
+197e440391d309f8.jpg	Is this a picture of Bardstown Historic District? Please answer yes or no.	No
+19fe04377265454e.jpg	Is this a picture of Alexander Nevsky Church, Belgrade? Please answer yes or no.	Yes
+19fe04377265454e.jpg	Is this a picture of Seppiko-Mineyama Hyogo Prefectural Nature Park? Please answer yes or no.	No
+1a32959819038291.jpg	Is this an image of Id Kah Mosque? Please answer yes or no.	Yes
+1a32959819038291.jpg	Is this an image of Katharinenkirche (Zwickau)? Please answer yes or no.	No
+1ac0e3c493073995.jpg	Is this an image of Church of Saint John the Baptist (Jihlava)? Please answer yes or no.	Yes
+1ac0e3c493073995.jpg	Is this an image of Silte kyrka? Please answer yes or no.	No
+1d4be36153741972.jpg	Is this a picture of Marie Guyart Building? Please answer yes or no.	Yes
+1d4be36153741972.jpg	Is this a picture of Motol? Please answer yes or no.	No
+1f233b807b660d25.jpg	Is this a photo of Aussichtsturm Pyramidenkogel? Please answer yes or no.	Yes
+1f233b807b660d25.jpg	Is this a photo of Vvedenskoe cemetery? Please answer yes or no.	No
+27a4267b9ea26cd0.jpg	Is this an image of Oude Salviuskerk (Limbricht)? Please answer yes or no.	Yes
+27a4267b9ea26cd0.jpg	Is this an image of Festung Heldrungen? Please answer yes or no.	No
+2a895eb889d6fd99.jpg	Is this an image of Beijing Guozijian? Please answer yes or no.	Yes
+2a895eb889d6fd99.jpg	Is this an image of Klinikkirche (Pfafferode)? Please answer yes or no.	No
+2f25654bcb445452.jpg	Is this a photo of Rinku Gate Tower Building? Please answer yes or no.	Yes
+2f25654bcb445452.jpg	Is this a photo of Bigorski Monastery? Please answer yes or no.	No
+3c89eb4043bcefac.jpg	Is this a picture of Begijnhofkerk (Mechelen)? Please answer yes or no.	Yes
+3c89eb4043bcefac.jpg	Is this a picture of Panther Hollow Lake? Please answer yes or no.	No

internvl_chat/eval/mme/Your_Results/numerical_calculation.txt

+0001.png	Is the answer to the arithmetic question in the image 225? Please answer yes or no.	Yes
+0001.png	Is the answer to the arithmetic question in the image 1515? Please answer yes or no.	No
+0002.png	Is the answer to the arithmetic question in the image 340? Please answer yes or no.	Yes
+0002.png	Is the answer to the arithmetic question in the image 17? Please answer yes or no.	No
+0003.png	Is the answer to the arithmetic question in the image 65? Please answer yes or no.	Yes
+0003.png	Is the answer to the arithmetic question in the image 56? Please answer yes or no.	No
+0004.png	Is the answer to the arithmetic question in the image 33? Please answer yes or no.	Yes
+0004.png	Is the answer to the arithmetic question in the image 32? Please answer yes or no.	No
+0005.png	Is the area of the square in the picture equal to 40? Please answer yes or no.	Yes
+0005.png	Is the area of the square in the picture equal to 8? Please answer yes or no.	No
+0006.png	Is the area of the square in the picture equal to 9? Please answer yes or no.	Yes
+0006.png	Is the area of the square in the picture equal to 3? Please answer yes or no.	No
+0007.png	Is the answer to the arithmetic question in the image 49? Please answer yes or no.	Yes
+0007.png	Is the answer to the arithmetic question in the image 39? Please answer yes or no.	No
+0008.png	Should the value of "a" in the picture equal 7? Please answer yes or no.	Yes
+0008.png	Should the value of "a" in the picture equal 14? Please answer yes or no.	No
+0009.png	Should the value of "a" in the picture equal 2? Please answer yes or no.	Yes
+0009.png	Should the value of "a" in the picture equal 3? Please answer yes or no.	No
+0010.png	Is the answer to the arithmetic question in the image 13? Please answer yes or no.	Yes
+0010.png	Is the answer to the arithmetic question in the image 12? Please answer yes or no.	No
+0011.png	Is the area of the parallelogram in the picture equal to 24? Please answer yes or no.	Yes
+0011.png	Is the area of the parallelogram in the picture equal to 6? Please answer yes or no.	No
+0012.png	Should the value of "a" in the picture equal 9? Please answer yes or no.	Yes
+0012.png	Should the value of "a" in the picture equal 1? Please answer yes or no.	No
+0013.png	Is the area of the right triangle in the picture equal to 24? Please answer yes or no.	Yes
+0013.png	Is the area of the right triangle in the picture equal to 8? Please answer yes or no.	No
+0014.png	Is the answer to the arithmetic question in the image 200? Please answer yes or no.	Yes
+0014.png	Is the answer to the arithmetic question in the image 400? Please answer yes or no.	No
+0015.png	Is the answer to the arithmetic question in the image 11? Please answer yes or no.	Yes
+0015.png	Is the answer to the arithmetic question in the image 111? Please answer yes or no.	No
+0016.png	Is the answer to the arithmetic question in the image 9? Please answer yes or no.	Yes
+0016.png	Is the answer to the arithmetic question in the image 16? Please answer yes or no.	No
+0017.png	Is the answer to the arithmetic question in the image 14? Please answer yes or no.	Yes
+0017.png	Is the answer to the arithmetic question in the image 83? Please answer yes or no.	No
+0018.png	Should the value of "a" in the picture equal 3? Please answer yes or no.	Yes
+0018.png	Should the value of "a" in the picture equal 2? Please answer yes or no.	No
+0019.png	Is the answer to the arithmetic question in the image 18? Please answer yes or no.	Yes
+0019.png	Is the answer to the arithmetic question in the image 36? Please answer yes or no.	No
+0020.png	Is the answer to the arithmetic question in the image 9? Please answer yes or no.	Yes
+0020.png	Is the answer to the arithmetic question in the image 45? Please answer yes or no.	No

internvl_chat/eval/mme/Your_Results/position.txt

+000000006471.jpg	Is the cricket bat above the batter's body? Please answer yes or no.	Yes
+000000006471.jpg	Is the cricket bat under the batter's body Please answer yes or no.	No
+000000007281.jpg	Is the sea behind people in the image? Please answer yes or no.	Yes
+000000007281.jpg	Is the sea in front of people in the image? Please answer yes or no.	No
+000000014038.jpg	Is the refrigerator on the left side of the picture? Please answer yes or no.	Yes
+000000014038.jpg	Is the refrigerator on the right side of the picture Please answer yes or no.	No
+000000031248.jpg	Is there a sofa in the middle of potted plants in the image? Please answer yes or no.	Yes
+000000031248.jpg	Is there a sofa in the right side of potted plants in the image? Please answer yes or no.	No
+000000048504.jpg	Is the gray elephant in front of the brown elephant? Please answer yes or no.	Yes
+000000048504.jpg	Is the brown elephant in front of the gray elephant? Please answer yes or no.	No
+000000052007.jpg	Are the pedestrians on the right of the bus? Please answer yes or no.	Yes
+000000052007.jpg	Are the pedestrians on the left of the bus? Please answer yes or no.	No
+000000056127.jpg	Is the light above the fire hydrant in the image? Please answer yes or no.	Yes
+000000056127.jpg	Is the light under the fire hydrant in the image?  Please answer yes or no.	No
+000000062025.jpg	Is the trash can under the cup in the image？ Please answer yes or no.	Yes
+000000062025.jpg	Is the trash can above the cup in the image？ Please answer yes or no.	No
+000000062808.jpg	Is the phone above the pizza in the image? Please answer yes or no.	Yes
+000000062808.jpg	Is the phone under the pizza in the image? Please answer yes or no.	No
+000000067213.jpg	Is the dog above the pool in the image? Please answer yes or no.	Yes
+000000067213.jpg	Is the dog under the pool in the image? Please answer yes or no.	No
+000000097994.jpg	Is the light above the computer in the image? Please answer yes or no.	Yes
+000000097994.jpg	Is the light under the computer in the image? Please answer yes or no.	No
+000000204871.jpg	Is the car on the right side of the fire hydrant in the picture? Please answer yes or no.	Yes
+000000204871.jpg	Is the car on the left side of the fire hydrant in the picture? Please answer yes or no.	No
+000000206487.jpg	Is the motorcycle on the right side of the bus? Please answer yes or no.	Yes
+000000206487.jpg	Is the motorcycle on the left side of the bus Please answer yes or no.	No
+000000211825.jpg	Is the cake on the left side of the camera? Please answer yes or no.	Yes
+000000211825.jpg	Is the cake on the right side of the camera? Please answer yes or no.	No
+000000212800.jpg	Is the blue umbrella under the black umbrella? Please answer yes or no.	Yes
+000000212800.jpg	Is the blue umbrella above the black umbrella? Please answer yes or no.	No
+000000395701.jpg	Is the TV on the left of the bookshelf? Please answer yes or no.	Yes
+000000395701.jpg	Is the TV on the right of the bookshelf? Please answer yes or no.	No
+000000395801.jpg	Is the clock above people? Please answer yes or no.	Yes
+000000395801.jpg	Is the clock under people? Please answer yes or no.	No
+000000405970.jpg	Is the grey sofa on the right of the TV? Please answer yes or no.	Yes
+000000405970.jpg	Is the grey sofa on the left of the TV? Please answer yes or no.	No
+000000426241.jpg	Is the white mouse on the right of the black keyboard? Please answer yes or no.	Yes
+000000426241.jpg	Is the white mouse on the left of the black keyboard? Please answer yes or no.	No
+000000450303.jpg	Is the monitor on top of a person? Please answer yes or no.	Yes
+000000450303.jpg	Is the monitor under the person? Please answer yes or no.	No
+000000458410.jpg	Is the TV on the left of the lamp? Please answer yes or no.	Yes
+000000458410.jpg	Is the TV on the right of the lamp? Please answer yes or no.	No
+000000472046.jpg	Is the pineapple on the left of the pot in the image? Please answer yes or no.	Yes
+000000472046.jpg	Is the pineapple on the right of the pot in the image? Please answer yes or no.	No
+000000477955.jpg	Is the person under the kite? Please answer yes or no.	Yes
+000000477955.jpg	Is the person above the kite? Please answer yes or no.	No
+000000482585.jpg	Is the person on the right of the train? Please answer yes or no.	Yes
+000000482585.jpg	Is the person on the left of the train? Please answer yes or no.	No
+000000494869.jpg	Is the baby on the right of the dog in the image? Please answer yes or no.	Yes
+000000494869.jpg	Is the baby on the left of the dog in the image? Please answer yes or no.	No
+000000509699.jpg	Is the mirror above the TV? Please answer yes or no.	Yes
+000000509699.jpg	Is the mirror under the TV? Please answer yes or no.	No
+000000519569.jpg	Is the vase on the left of the bottle? Please answer yes or no.	Yes
+000000519569.jpg	Is the vase on the right of the bottle? Please answer yes or no.	No
+000000530162.jpg	Is the big red and black umbrella on the top of people? Please answer yes or no.	Yes
+000000530162.jpg	Is the big red and black umbrella under people? Please answer yes or no.	No
+000000551660.jpg	Is the spoon in the bowl? Please answer yes or no.	Yes
+000000551660.jpg	Is the spoon out of the bowl? Please answer yes or no.	No
+000000578922.jpg	Is the vase on the left of the toothbrush? Please answer yes or no.	Yes
+000000578922.jpg	Is the vase on the right of the toothbrush? Please answer yes or no.	No

internvl_chat/eval/mme/Your_Results/posters.txt

+tt0048028.jpg	Is this movie directed by elia kazan? Please answer yes or no.	Yes
+tt0048028.jpg	Is this movie directed by maneesh sharma? Please answer yes or no.	No
+tt0053221.jpg	Is this movie titled rio bravo (1959)? Please answer yes or no.	Yes
+tt0053221.jpg	Is this movie titled the matrix (1999)? Please answer yes or no.	No
+tt0062622.jpg	Is this movie originated from the country or region of uk? Please answer yes or no.	Yes
+tt0062622.jpg	Is this movie originated from the country or region of china? Please answer yes or no.	No
+tt0063442.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
+tt0063442.jpg	Is this movie originated from the country or region of japan? Please answer yes or no.	No
+tt0065724.jpg	Is this movie directed by bob rafelson? Please answer yes or no.	Yes
+tt0065724.jpg	Is this movie directed by anthony minghella? Please answer yes or no.	No
+tt0067116.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
+tt0067116.jpg	Is this movie originated from the country or region of uk? Please answer yes or no.	No
+tt0067140.jpg	Is this movie titled duck, you sucker (1971)? Please answer yes or no.	Yes
+tt0067140.jpg	Is this movie titled crimson tide (1995)? Please answer yes or no.	No
+tt0067185.jpg	Is this movie directed by hal ashby? Please answer yes or no.	Yes
+tt0067185.jpg	Is this movie directed by john hillcoat? Please answer yes or no.	No
+tt0068646.jpg	Is this movie titled the godfather (1972)? Please answer yes or no.	Yes
+tt0068646.jpg	Is this movie titled bring on the melody (2017)? Please answer yes or no.	No
+tt0070291.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
+tt0070291.jpg	Is this movie originated from the country or region of spain? Please answer yes or no.	No
+tt0071315.jpg	Is this movie titled chinatown (1974)? Please answer yes or no.	Yes
+tt0071315.jpg	Is this movie titled planet of the apes (1968)? Please answer yes or no.	No
+tt0074119.jpg	Is this movie directed by alan j. pakula? Please answer yes or no.	Yes
+tt0074119.jpg	Is this movie directed by stanley kubrick? Please answer yes or no.	No
+tt0074749.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
+tt0074749.jpg	Is this movie originated from the country or region of venezuela? Please answer yes or no.	No
+tt0082186.jpg	Is this movie directed by desmond davis? Please answer yes or no.	Yes
+tt0082186.jpg	Is this movie directed by edward zwick? Please answer yes or no.	No
+tt0083907.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
+tt0083907.jpg	Is this movie originated from the country or region of bulgaria? Please answer yes or no.	No
+tt0083946.jpg	Is this movie originated from the country or region of west germany? Please answer yes or no.	Yes
+tt0083946.jpg	Is this movie originated from the country or region of denmark? Please answer yes or no.	No
+tt0084787.jpg	Is this movie titled the thing (1982)? Please answer yes or no.	Yes
+tt0084787.jpg	Is this movie titled jay and silent bob strike back (2001)? Please answer yes or no.	No
+tt0086250.jpg	Is this movie directed by brian de palma? Please answer yes or no.	Yes
+tt0086250.jpg	Is this movie directed by patrice leconte? Please answer yes or no.	No
+tt0093565.jpg	Is this movie titled moonstruck (1987)? Please answer yes or no.	Yes
+tt0093565.jpg	Is this movie titled now you see me (2013)? Please answer yes or no.	No
+tt0093773.jpg	Is this movie titled predator (1987)? Please answer yes or no.	Yes
+tt0093773.jpg	Is this movie titled the cell (2000)? Please answer yes or no.	No
+tt0094291.jpg	Is this movie directed by oliver stone? Please answer yes or no.	Yes
+tt0094291.jpg	Is this movie directed by raja gosnell? Please answer yes or no.	No
+tt0096446.jpg	Is this movie directed by ron howard? Please answer yes or no.	Yes
+tt0096446.jpg	Is this movie directed by peter jackson? Please answer yes or no.	No
+tt0096463.jpg	Is this movie directed by mike nichols? Please answer yes or no.	Yes
+tt0096463.jpg	Is this movie directed by cary joji fukunaga? Please answer yes or no.	No
+tt0096895.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
+tt0096895.jpg	Is this movie originated from the country or region of china? Please answer yes or no.	No
+tt0097576.jpg	Is this movie directed by steven spielberg? Please answer yes or no.	Yes
+tt0097576.jpg	Is this movie directed by francis ford coppola? Please answer yes or no.	No
+tt0098635.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
+tt0098635.jpg	Is this movie originated from the country or region of spain? Please answer yes or no.	No
+tt0098724.jpg	Is this movie titled sex, lies, and videotape (1989)? Please answer yes or no.	Yes
+tt0098724.jpg	Is this movie titled final destination (2000)? Please answer yes or no.	No
+tt0099653.jpg	Is this movie directed by jerry zucker? Please answer yes or no.	Yes
+tt0099653.jpg	Is this movie directed by felix chong,alan mak? Please answer yes or no.	No
+tt0100112.jpg	Is this movie originated from the country or region of france? Please answer yes or no.	Yes
+tt0100112.jpg	Is this movie originated from the country or region of new zealand? Please answer yes or no.	No
+tt0100150.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
+tt0100150.jpg	Is this movie originated from the country or region of hong kong? Please answer yes or no.	No
+tt0100935.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
+tt0100935.jpg	Is this movie originated from the country or region of canada? Please answer yes or no.	No
+tt0101889.jpg	Is this movie titled the fisher king (1991)? Please answer yes or no.	Yes
+tt0101889.jpg	Is this movie titled a good day to die hard (2013)? Please answer yes or no.	No
+tt0101921.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
+tt0101921.jpg	Is this movie originated from the country or region of australia? Please answer yes or no.	No
+tt0107653.jpg	Is this movie titled naked (1993)? Please answer yes or no.	Yes
+tt0107653.jpg	Is this movie titled captain america: civil war (2016)? Please answer yes or no.	No
+tt0107808.jpg	Is this movie titled a perfect world (1993)? Please answer yes or no.	Yes
+tt0107808.jpg	Is this movie titled harold and maude (1971)? Please answer yes or no.	No
+tt0110201.jpg	Is this movie titled hail the judge (1994)? Please answer yes or no.	Yes
+tt0110201.jpg	Is this movie titled batman (1989)? Please answer yes or no.	No
+tt0112740.jpg	Is this movie titled crimson tide (1995)? Please answer yes or no.	Yes
+tt0112740.jpg	Is this movie titled insidious: the last key (2018)? Please answer yes or no.	No
+tt0113497.jpg	Is this movie titled jumanji (1995)? Please answer yes or no.	Yes
+tt0113497.jpg	Is this movie titled morgan (2016)? Please answer yes or no.	No
+tt0114369.jpg	Is this movie titled se7en (1995)? Please answer yes or no.	Yes
+tt0114369.jpg	Is this movie titled a perfect world (1993)? Please answer yes or no.	No
+tt0114388.jpg	Is this movie directed by ang lee? Please answer yes or no.	Yes
+tt0114388.jpg	Is this movie directed by david lowery? Please answer yes or no.	No
+tt0116209.jpg	Is this movie directed by anthony minghella? Please answer yes or no.	Yes
+tt0116209.jpg	Is this movie directed by john cassavetes? Please answer yes or no.	No
+tt0116629.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
+tt0116629.jpg	Is this movie originated from the country or region of australia? Please answer yes or no.	No
+tt0118636.jpg	Is this movie titled apt pupil (1998)? Please answer yes or no.	Yes
+tt0118636.jpg	Is this movie titled a good day to die hard (2013)? Please answer yes or no.	No
+tt0118655.jpg	Is this movie directed by jay roach? Please answer yes or no.	Yes
+tt0118655.jpg	Is this movie directed by peter jackson? Please answer yes or no.	No
+tt0118715.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
+tt0118715.jpg	Is this movie originated from the country or region of venezuela? Please answer yes or no.	No
+tt0119303.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
+tt0119303.jpg	Is this movie originated from the country or region of spain? Please answer yes or no.	No
+tt0120738.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
+tt0120738.jpg	Is this movie originated from the country or region of germany? Please answer yes or no.	No
+tt0129387.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
+tt0129387.jpg	Is this movie originated from the country or region of germany? Please answer yes or no.	No
+tt0133093.jpg	Is this movie directed by lana wachowski,lilly wachowski? Please answer yes or no.	Yes
+tt0133093.jpg	Is this movie directed by siu-tung ching? Please answer yes or no.	No
+tt0163025.jpg	Is this movie directed by joe johnston? Please answer yes or no.	Yes
+tt0163025.jpg	Is this movie directed by adam robitel? Please answer yes or no.	No
+tt0166924.jpg	Is this movie titled mulholland drive (2001)? Please answer yes or no.	Yes
+tt0166924.jpg	Is this movie titled lucky number slevin (2006)? Please answer yes or no.	No
+tt0167261.jpg	Is this movie directed by peter jackson? Please answer yes or no.	Yes
+tt0167261.jpg	Is this movie directed by gabe ibáñez? Please answer yes or no.	No
+tt0182789.jpg	Is this movie titled bicentennial man (1999)? Please answer yes or no.	Yes
+tt0182789.jpg	Is this movie titled ocean's eleven (2001)? Please answer yes or no.	No
+tt0187078.jpg	Is this movie titled gone in 60 seconds (2000)? Please answer yes or no.	Yes
+tt0187078.jpg	Is this movie titled avatar (2009)? Please answer yes or no.	No
+tt0195714.jpg	Is this movie directed by james wong? Please answer yes or no.	Yes
+tt0195714.jpg	Is this movie directed by bob rafelson? Please answer yes or no.	No
+tt0209958.jpg	Is this movie titled the cell (2000)? Please answer yes or no.	Yes
+tt0209958.jpg	Is this movie titled the matrix revolutions (2003)? Please answer yes or no.	No
+tt0232500.jpg	Is this movie titled the fast and the furious (2001)? Please answer yes or no.	Yes
+tt0232500.jpg	Is this movie titled east of eden (1955)? Please answer yes or no.	No
+tt0240772.jpg	Is this movie directed by steven soderbergh? Please answer yes or no.	Yes
+tt0240772.jpg	Is this movie directed by paul mcguigan? Please answer yes or no.	No
+tt0242653.jpg	Is this movie titled the matrix revolutions (2003)? Please answer yes or no.	Yes
+tt0242653.jpg	Is this movie titled sense and sensibility (1995)? Please answer yes or no.	No
+tt0244244.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
+tt0244244.jpg	Is this movie originated from the country or region of argentina? Please answer yes or no.	No
+tt0258463.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
+tt0258463.jpg	Is this movie originated from the country or region of italy? Please answer yes or no.	No
+tt0261392.jpg	Is this movie titled jay and silent bob strike back (2001)? Please answer yes or no.	Yes
+tt0261392.jpg	Is this movie titled avatar (2009)? Please answer yes or no.	No
+tt0264395.jpg	Is this movie originated from the country or region of germany? Please answer yes or no.	Yes
+tt0264395.jpg	Is this movie originated from the country or region of australia? Please answer yes or no.	No
+tt0266697.jpg	Is this movie directed by quentin tarantino? Please answer yes or no.	Yes
+tt0266697.jpg	Is this movie directed by peyton reed? Please answer yes or no.	No
+tt0268978.jpg	Is this movie titled a beautiful mind (2001)? Please answer yes or no.	Yes
+tt0268978.jpg	Is this movie titled blue jasmine (2013)? Please answer yes or no.	No
+tt0318627.jpg	Is this movie titled resident evil: apocalypse (2004)? Please answer yes or no.	Yes
+tt0318627.jpg	Is this movie titled the meddler (2015)? Please answer yes or no.	No
+tt0328107.jpg	Is this movie titled man on fire (2004)? Please answer yes or no.	Yes
+tt0328107.jpg	Is this movie titled real steel (2011)? Please answer yes or no.	No
+tt0338751.jpg	Is this movie directed by martin scorsese? Please answer yes or no.	Yes
+tt0338751.jpg	Is this movie directed by chris columbus? Please answer yes or no.	No
+tt0341495.jpg	Is this movie directed by siu-tung ching? Please answer yes or no.	Yes
+tt0341495.jpg	Is this movie directed by lorenzo vigas? Please answer yes or no.	No
+tt0351977.jpg	Is this movie directed by kevin bray? Please answer yes or no.	Yes
+tt0351977.jpg	Is this movie directed by joe johnston? Please answer yes or no.	No
+tt0369702.jpg	Is this movie originated from the country or region of spain? Please answer yes or no.	Yes
+tt0369702.jpg	Is this movie originated from the country or region of france? Please answer yes or no.	No
+tt0373469.jpg	Is this movie directed by shane black? Please answer yes or no.	Yes
+tt0373469.jpg	Is this movie directed by dennis dugan? Please answer yes or no.	No
+tt0378194.jpg	Is this movie directed by quentin tarantino? Please answer yes or no.	Yes
+tt0378194.jpg	Is this movie directed by todd phillips? Please answer yes or no.	No
+tt0379786.jpg	Is this movie titled serenity (2005)? Please answer yes or no.	Yes
+tt0379786.jpg	Is this movie titled bad genius (2017)? Please answer yes or no.	No
+tt0393109.jpg	Is this movie titled brick (2005)? Please answer yes or no.	Yes
+tt0393109.jpg	Is this movie titled sense and sensibility (1995)? Please answer yes or no.	No
+tt0399201.jpg	Is this movie directed by michael bay? Please answer yes or no.	Yes
+tt0399201.jpg	Is this movie directed by angelina jolie? Please answer yes or no.	No
+tt0425210.jpg	Is this movie titled lucky number slevin (2006)? Please answer yes or no.	Yes
+tt0425210.jpg	Is this movie titled the godfather (1972)? Please answer yes or no.	No
+tt0433035.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
+tt0433035.jpg	Is this movie originated from the country or region of new zealand? Please answer yes or no.	No
+tt0443680.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
+tt0443680.jpg	Is this movie originated from the country or region of germany? Please answer yes or no.	No
+tt0449088.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
+tt0449088.jpg	Is this movie originated from the country or region of japan? Please answer yes or no.	No
+tt0450259.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
+tt0450259.jpg	Is this movie originated from the country or region of china? Please answer yes or no.	No
+tt0462499.jpg	Is this movie titled rambo (2008)? Please answer yes or no.	Yes
+tt0462499.jpg	Is this movie titled walking tall (2004)? Please answer yes or no.	No
+tt0499549.jpg	Is this movie directed by james cameron? Please answer yes or no.	Yes
+tt0499549.jpg	Is this movie directed by jerry zucker? Please answer yes or no.	No
+tt0780653.jpg	Is this movie titled the wolfman (2010)? Please answer yes or no.	Yes
+tt0780653.jpg	Is this movie titled wild at heart (1990)? Please answer yes or no.	No
+tt0829482.jpg	Is this movie directed by greg mottola? Please answer yes or no.	Yes
+tt0829482.jpg	Is this movie directed by jing wong? Please answer yes or no.	No
+tt0848228.jpg	Is this movie titled the avengers (2012)? Please answer yes or no.	Yes
+tt0848228.jpg	Is this movie titled wolf warrior 2 (2017)? Please answer yes or no.	No
+tt0898367.jpg	Is this movie directed by john hillcoat? Please answer yes or no.	Yes
+tt0898367.jpg	Is this movie directed by franklin j. schaffner? Please answer yes or no.	No
+tt0942385.jpg	Is this movie titled tropic thunder (2008)? Please answer yes or no.	Yes
+tt0942385.jpg	Is this movie titled bad teacher (2011)? Please answer yes or no.	No
+tt0993846.jpg	Is this movie titled the wolf of wall street (2013)? Please answer yes or no.	Yes
+tt0993846.jpg	Is this movie titled lucky number slevin (2006)? Please answer yes or no.	No
+tt1017460.jpg	Is this movie directed by vincenzo natali? Please answer yes or no.	Yes
+tt1017460.jpg	Is this movie directed by raja gosnell? Please answer yes or no.	No
+tt1057500.jpg	Is this movie directed by clint eastwood? Please answer yes or no.	Yes
+tt1057500.jpg	Is this movie directed by john cassavetes? Please answer yes or no.	No
+tt1068680.jpg	Is this movie titled yes man (2008)? Please answer yes or no.	Yes
+tt1068680.jpg	Is this movie titled rio bravo (1959)? Please answer yes or no.	No
+tt1099212.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
+tt1099212.jpg	Is this movie originated from the country or region of japan? Please answer yes or no.	No
+tt1119646.jpg	Is this movie directed by todd phillips? Please answer yes or no.	Yes
+tt1119646.jpg	Is this movie directed by franklin j. schaffner? Please answer yes or no.	No
+tt1124037.jpg	Is this movie directed by gary ross? Please answer yes or no.	Yes
+tt1124037.jpg	Is this movie directed by francis ford coppola? Please answer yes or no.	No
+tt1229822.jpg	Is this movie titled jane eyre (2011)? Please answer yes or no.	Yes
+tt1229822.jpg	Is this movie titled kiss kiss bang bang (2005)? Please answer yes or no.	No
+tt1284575.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
+tt1284575.jpg	Is this movie originated from the country or region of bulgaria? Please answer yes or no.	No
+tt1291150.jpg	Is this movie directed by jonathan liebesman? Please answer yes or no.	Yes
+tt1291150.jpg	Is this movie directed by masahide ichii? Please answer yes or no.	No
+tt1300851.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
+tt1300851.jpg	Is this movie originated from the country or region of germany? Please answer yes or no.	No
+tt1305806.jpg	Is this movie titled the secret in their eyes (2009)? Please answer yes or no.	Yes
+tt1305806.jpg	Is this movie titled ocean's eleven (2001)? Please answer yes or no.	No
+tt1343092.jpg	Is this movie directed by baz luhrmann? Please answer yes or no.	Yes
+tt1343092.jpg	Is this movie directed by lorenzo vigas? Please answer yes or no.	No
+tt1385826.jpg	Is this movie titled the adjustment bureau (2011)? Please answer yes or no.	Yes
+tt1385826.jpg	Is this movie titled the last of sheila (1973)? Please answer yes or no.	No
+tt1403865.jpg	Is this movie directed by ethan coen,joel coen? Please answer yes or no.	Yes
+tt1403865.jpg	Is this movie directed by john hillcoat? Please answer yes or no.	No
+tt1438176.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
+tt1438176.jpg	Is this movie originated from the country or region of west germany? Please answer yes or no.	No
+tt1454029.jpg	Is this movie titled the help (2011)? Please answer yes or no.	Yes
+tt1454029.jpg	Is this movie titled avatar (2009)? Please answer yes or no.	No
+tt1560747.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
+tt1560747.jpg	Is this movie originated from the country or region of canada? Please answer yes or no.	No
+tt1564367.jpg	Is this movie titled just go with it (2011)? Please answer yes or no.	Yes
+tt1564367.jpg	Is this movie titled journey to the west (2013)? Please answer yes or no.	No
+tt1606378.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
+tt1606378.jpg	Is this movie originated from the country or region of uk? Please answer yes or no.	No
+tt1637725.jpg	Is this movie titled ted (2012)? Please answer yes or no.	Yes
+tt1637725.jpg	Is this movie titled lost in space (1998)? Please answer yes or no.	No
+tt1670345.jpg	Is this movie titled now you see me (2013)? Please answer yes or no.	Yes
+tt1670345.jpg	Is this movie titled bicentennial man (1999)? Please answer yes or no.	No
+tt1703957.jpg	Is this movie titled genius (2016)? Please answer yes or no.	Yes
+tt1703957.jpg	Is this movie titled fan (2016)? Please answer yes or no.	No
+tt1722484.jpg	Is this movie directed by hermine huntgeburth? Please answer yes or no.	Yes
+tt1722484.jpg	Is this movie directed by todd phillips? Please answer yes or no.	No
+tt1800241.jpg	Is this movie directed by david o. russell? Please answer yes or no.	Yes
+tt1800241.jpg	Is this movie directed by raja gosnell? Please answer yes or no.	No
+tt1809398.jpg	Is this movie titled unbroken (2014)? Please answer yes or no.	Yes
+tt1809398.jpg	Is this movie titled home alone 3 (1997)? Please answer yes or no.	No
+tt1971325.jpg	Is this movie originated from the country or region of bulgaria? Please answer yes or no.	Yes
+tt1971325.jpg	Is this movie originated from the country or region of spain? Please answer yes or no.	No
+tt1974419.jpg	Is this movie titled the neon demon (2016)? Please answer yes or no.	Yes
+tt1974419.jpg	Is this movie titled man on fire (2004)? Please answer yes or no.	No
+tt2017561.jpg	Is this movie directed by stephen chow,chi-kin kwok? Please answer yes or no.	Yes
+tt2017561.jpg	Is this movie directed by craig gillespie? Please answer yes or no.	No
+tt2078768.jpg	Is this movie originated from the country or region of hong kong? Please answer yes or no.	Yes
+tt2078768.jpg	Is this movie originated from the country or region of venezuela? Please answer yes or no.	No
+tt2132285.jpg	Is this movie titled the bling ring (2013)? Please answer yes or no.	Yes
+tt2132285.jpg	Is this movie titled tropic thunder (2008)? Please answer yes or no.	No
+tt2238032.jpg	Is this movie titled skiptrace (2016)? Please answer yes or no.	Yes
+tt2238032.jpg	Is this movie titled the bourne identity (2002)? Please answer yes or no.	No
+tt2334873.jpg	Is this movie titled blue jasmine (2013)? Please answer yes or no.	Yes
+tt2334873.jpg	Is this movie titled a good day to die hard (2013)? Please answer yes or no.	No
+tt2788732.jpg	Is this movie titled pete's dragon (2016)? Please answer yes or no.	Yes
+tt2788732.jpg	Is this movie titled willow (1988)? Please answer yes or no.	No
+tt2802144.jpg	Is this movie directed by matthew vaughn? Please answer yes or no.	Yes
+tt2802144.jpg	Is this movie directed by dietrich brüggemann? Please answer yes or no.	No
+tt2908856.jpg	Is this movie directed by israel horovitz? Please answer yes or no.	Yes
+tt2908856.jpg	Is this movie directed by mike nichols? Please answer yes or no.	No
+tt2980516.jpg	Is this movie titled the theory of everything (2014)? Please answer yes or no.	Yes
+tt2980516.jpg	Is this movie titled stations of the cross (2014)? Please answer yes or no.	No
+tt3077214.jpg	Is this movie titled suffragette (2015)? Please answer yes or no.	Yes
+tt3077214.jpg	Is this movie titled east of eden (1955)? Please answer yes or no.	No
+tt3316960.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
+tt3316960.jpg	Is this movie originated from the country or region of west germany? Please answer yes or no.	No
+tt3465916.jpg	Is this movie originated from the country or region of germany? Please answer yes or no.	Yes
+tt3465916.jpg	Is this movie originated from the country or region of spain? Please answer yes or no.	No
+tt3495026.jpg	Is this movie titled fan (2016)? Please answer yes or no.	Yes
+tt3495026.jpg	Is this movie titled from afar (2015)? Please answer yes or no.	No
+tt3498820.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
+tt3498820.jpg	Is this movie originated from the country or region of germany? Please answer yes or no.	No
+tt3700804.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
+tt3700804.jpg	Is this movie originated from the country or region of uk? Please answer yes or no.	No
+tt3824458.jpg	Is this movie titled tangerine (2015)? Please answer yes or no.	Yes
+tt3824458.jpg	Is this movie titled miller's crossing (1990)? Please answer yes or no.	No
+tt3860916.jpg	Is this movie directed by ben howling? Please answer yes or no.	Yes
+tt3860916.jpg	Is this movie directed by quentin tarantino? Please answer yes or no.	No
+tt4046784.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
+tt4046784.jpg	Is this movie originated from the country or region of uk? Please answer yes or no.	No
+tt4242158.jpg	Is this movie titled lost in hong kong (2015)? Please answer yes or no.	Yes
+tt4242158.jpg	Is this movie titled harold and maude (1971)? Please answer yes or no.	No
+tt4273292.jpg	Is this movie directed by babak anvari? Please answer yes or no.	Yes
+tt4273292.jpg	Is this movie directed by ethan coen,joel coen? Please answer yes or no.	No
+tt4501454.jpg	Is this movie directed by lorene scafaria? Please answer yes or no.	Yes
+tt4501454.jpg	Is this movie directed by herbert ross? Please answer yes or no.	No
+tt4520364.jpg	Is this movie directed by luke scott? Please answer yes or no.	Yes
+tt4520364.jpg	Is this movie directed by dominic sena? Please answer yes or no.	No
+tt4651520.jpg	Is this movie originated from the country or region of usa? Please answer yes or no.	Yes
+tt4651520.jpg	Is this movie originated from the country or region of china? Please answer yes or no.	No
+tt4721400.jpg	Is this movie directed by lorenzo vigas? Please answer yes or no.	Yes
+tt4721400.jpg	Is this movie directed by jonathan liebesman? Please answer yes or no.	No
+tt4972062.jpg	Is this movie directed by mike birbiglia? Please answer yes or no.	Yes
+tt4972062.jpg	Is this movie directed by david fincher? Please answer yes or no.	No
+tt5564148.jpg	Is this movie directed by masahide ichii? Please answer yes or no.	Yes
+tt5564148.jpg	Is this movie directed by dietrich brüggemann? Please answer yes or no.	No
+tt5688868.jpg	Is this movie titled primal rage: the legend of konga (2018)? Please answer yes or no.	Yes
+tt5688868.jpg	Is this movie titled the neon demon (2016)? Please answer yes or no.	No
+tt5726086.jpg	Is this movie directed by adam robitel? Please answer yes or no.	Yes
+tt5726086.jpg	Is this movie directed by gore verbinski? Please answer yes or no.	No
+tt6788942.jpg	Is this movie originated from the country or region of thailand? Please answer yes or no.	Yes
+tt6788942.jpg	Is this movie originated from the country or region of argentina? Please answer yes or no.	No
+tt7055592.jpg	Is this movie titled brotherhood of blades ii: the infernal battlefield (2017)? Please answer yes or no.	Yes
+tt7055592.jpg	Is this movie titled stations of the cross (2014)? Please answer yes or no.	No
+tt7131870.jpg	Is this movie directed by jing wu? Please answer yes or no.	Yes
+tt7131870.jpg	Is this movie directed by david lynch? Please answer yes or no.	No

internvl_chat/eval/mme/Your_Results/scene.txt

+Places365_val_00000001.jpg	Is this picture captured in a place of greenhouse indoor? Please answer yes or no.	Yes
+Places365_val_00000001.jpg	Is this picture captured in a place of waiting room? Please answer yes or no.	No
+Places365_val_00000002.jpg	Is this photo taken in a place of wet bar? Please answer yes or no.	Yes
+Places365_val_00000002.jpg	Is this photo taken in a place of stage indoor? Please answer yes or no.	No
+Places365_val_00000003.jpg	Is this picture captured in a place of clean room? Please answer yes or no.	Yes
+Places365_val_00000003.jpg	Is this picture captured in a place of bookstore? Please answer yes or no.	No
+Places365_val_00000004.jpg	Is this picture captured in a place of golf course? Please answer yes or no.	Yes
+Places365_val_00000004.jpg	Is this picture captured in a place of computer room? Please answer yes or no.	No
+Places365_val_00000005.jpg	Is this picture captured in a place of rock arch? Please answer yes or no.	Yes
+Places365_val_00000005.jpg	Is this picture captured in a place of desert sand? Please answer yes or no.	No
+Places365_val_00000006.jpg	Is this photo taken in a place of corridor? Please answer yes or no.	Yes
+Places365_val_00000006.jpg	Is this photo taken in a place of jail cell? Please answer yes or no.	No
+Places365_val_00000007.jpg	Does this image describe a place of canyon? Please answer yes or no.	Yes
+Places365_val_00000007.jpg	Does this image describe a place of basement? Please answer yes or no.	No
+Places365_val_00000008.jpg	Is this picture captured in a place of dining room? Please answer yes or no.	Yes
+Places365_val_00000008.jpg	Is this picture captured in a place of ball pit? Please answer yes or no.	No
+Places365_val_00000009.jpg	Is this picture captured in a place of forest broadleaf? Please answer yes or no.	Yes
+Places365_val_00000009.jpg	Is this picture captured in a place of swamp? Please answer yes or no.	No
+Places365_val_00000010.jpg	Does this image describe a place of shopping mall indoor? Please answer yes or no.	Yes
+Places365_val_00000010.jpg	Does this image describe a place of mosque outdoor? Please answer yes or no.	No
+Places365_val_00000011.jpg	Is this photo taken in a place of baseball field? Please answer yes or no.	Yes
+Places365_val_00000011.jpg	Is this photo taken in a place of bus station indoor? Please answer yes or no.	No
+Places365_val_00000012.jpg	Does this image describe a place of campus? Please answer yes or no.	Yes
+Places365_val_00000012.jpg	Does this image describe a place of sandbox? Please answer yes or no.	No
+Places365_val_00000013.jpg	Is this picture captured in a place of beach house? Please answer yes or no.	Yes
+Places365_val_00000013.jpg	Is this picture captured in a place of alcove? Please answer yes or no.	No
+Places365_val_00000014.jpg	Does this image describe a place of art gallery? Please answer yes or no.	Yes
+Places365_val_00000014.jpg	Does this image describe a place of volleyball court outdoor? Please answer yes or no.	No
+Places365_val_00000015.jpg	Does this image describe a place of bus interior? Please answer yes or no.	Yes
+Places365_val_00000015.jpg	Does this image describe a place of butchers shop? Please answer yes or no.	No
+Places365_val_00000016.jpg	Does this image describe a place of gymnasium indoor? Please answer yes or no.	Yes
+Places365_val_00000016.jpg	Does this image describe a place of crevasse? Please answer yes or no.	No
+Places365_val_00000017.jpg	Is this picture captured in a place of glacier? Please answer yes or no.	Yes
+Places365_val_00000017.jpg	Is this picture captured in a place of fishpond? Please answer yes or no.	No
+Places365_val_00000018.jpg	Is this photo taken in a place of nursing home? Please answer yes or no.	Yes
+Places365_val_00000018.jpg	Is this photo taken in a place of server room? Please answer yes or no.	No
+Places365_val_00000019.jpg	Does this image describe a place of storage room? Please answer yes or no.	Yes
+Places365_val_00000019.jpg	Does this image describe a place of aquarium? Please answer yes or no.	No
+Places365_val_00000020.jpg	Is this picture captured in a place of florist shop indoor? Please answer yes or no.	Yes
+Places365_val_00000020.jpg	Is this picture captured in a place of office? Please answer yes or no.	No
+Places365_val_00000021.jpg	Is this picture captured in a place of restaurant kitchen? Please answer yes or no.	Yes
+Places365_val_00000021.jpg	Is this picture captured in a place of garage outdoor? Please answer yes or no.	No
+Places365_val_00000022.jpg	Is this photo taken in a place of recreation room? Please answer yes or no.	Yes
+Places365_val_00000022.jpg	Is this photo taken in a place of runway? Please answer yes or no.	No
+Places365_val_00000023.jpg	Does this image describe a place of schoolhouse? Please answer yes or no.	Yes
+Places365_val_00000023.jpg	Does this image describe a place of raft? Please answer yes or no.	No
+Places365_val_00000024.jpg	Does this image describe a place of amusement arcade? Please answer yes or no.	Yes
+Places365_val_00000024.jpg	Does this image describe a place of medina? Please answer yes or no.	No
+Places365_val_00000025.jpg	Is this photo taken in a place of ocean? Please answer yes or no.	Yes
+Places365_val_00000025.jpg	Is this photo taken in a place of sauna? Please answer yes or no.	No
+Places365_val_00000026.jpg	Does this image describe a place of river? Please answer yes or no.	Yes
+Places365_val_00000026.jpg	Does this image describe a place of corral? Please answer yes or no.	No
+Places365_val_00000027.jpg	Is this photo taken in a place of pub indoor? Please answer yes or no.	Yes
+Places365_val_00000027.jpg	Is this photo taken in a place of basement? Please answer yes or no.	No
+Places365_val_00000028.jpg	Is this picture captured in a place of lecture room? Please answer yes or no.	Yes
+Places365_val_00000028.jpg	Is this picture captured in a place of aqueduct? Please answer yes or no.	No
+Places365_val_00000029.jpg	Is this photo taken in a place of japanese garden? Please answer yes or no.	Yes
+Places365_val_00000029.jpg	Is this photo taken in a place of pharmacy? Please answer yes or no.	No
+Places365_val_00000030.jpg	Does this image describe a place of motel? Please answer yes or no.	Yes
+Places365_val_00000030.jpg	Does this image describe a place of toyshop? Please answer yes or no.	No
+Places365_val_00000031.jpg	Does this image describe a place of assembly line? Please answer yes or no.	Yes
+Places365_val_00000031.jpg	Does this image describe a place of beer garden? Please answer yes or no.	No
+Places365_val_00000032.jpg	Does this image describe a place of campsite? Please answer yes or no.	Yes
+Places365_val_00000032.jpg	Does this image describe a place of elevator shaft? Please answer yes or no.	No
+Places365_val_00000033.jpg	Is this photo taken in a place of florist shop indoor? Please answer yes or no.	Yes
+Places365_val_00000033.jpg	Is this photo taken in a place of sushi bar? Please answer yes or no.	No
+Places365_val_00000034.jpg	Does this image describe a place of racecourse? Please answer yes or no.	Yes
+Places365_val_00000034.jpg	Does this image describe a place of galley? Please answer yes or no.	No
+Places365_val_00000035.jpg	Is this photo taken in a place of kennel outdoor? Please answer yes or no.	Yes
+Places365_val_00000035.jpg	Is this photo taken in a place of bathroom? Please answer yes or no.	No
+Places365_val_00000036.jpg	Is this picture captured in a place of bus interior? Please answer yes or no.	Yes
+Places365_val_00000036.jpg	Is this picture captured in a place of mansion? Please answer yes or no.	No
+Places365_val_00000037.jpg	Does this image describe a place of dressing room? Please answer yes or no.	Yes
+Places365_val_00000037.jpg	Does this image describe a place of chemistry lab? Please answer yes or no.	No
+Places365_val_00000038.jpg	Does this image describe a place of wind farm? Please answer yes or no.	Yes
+Places365_val_00000038.jpg	Does this image describe a place of volleyball court outdoor? Please answer yes or no.	No
+Places365_val_00000039.jpg	Is this picture captured in a place of auto factory? Please answer yes or no.	Yes
+Places365_val_00000039.jpg	Is this picture captured in a place of tree farm? Please answer yes or no.	No
+Places365_val_00000040.jpg	Is this picture captured in a place of beach? Please answer yes or no.	Yes
+Places365_val_00000040.jpg	Is this picture captured in a place of library outdoor? Please answer yes or no.	No
+Places365_val_00000041.jpg	Is this photo taken in a place of mountain? Please answer yes or no.	Yes
+Places365_val_00000041.jpg	Is this photo taken in a place of formal garden? Please answer yes or no.	No
+Places365_val_00000042.jpg	Does this image describe a place of pavilion? Please answer yes or no.	Yes
+Places365_val_00000042.jpg	Does this image describe a place of stable? Please answer yes or no.	No
+Places365_val_00000043.jpg	Does this image describe a place of tree house? Please answer yes or no.	Yes
+Places365_val_00000043.jpg	Does this image describe a place of patio? Please answer yes or no.	No
+Places365_val_00000044.jpg	Is this photo taken in a place of coffee shop? Please answer yes or no.	Yes
+Places365_val_00000044.jpg	Is this photo taken in a place of elevator lobby? Please answer yes or no.	No
+Places365_val_00000045.jpg	Is this photo taken in a place of skyscraper? Please answer yes or no.	Yes
+Places365_val_00000045.jpg	Is this photo taken in a place of forest broadleaf? Please answer yes or no.	No
+Places365_val_00000046.jpg	Does this image describe a place of street? Please answer yes or no.	Yes
+Places365_val_00000046.jpg	Does this image describe a place of golf course? Please answer yes or no.	No
+Places365_val_00000047.jpg	Is this picture captured in a place of alley? Please answer yes or no.	Yes
+Places365_val_00000047.jpg	Is this picture captured in a place of boardwalk? Please answer yes or no.	No
+Places365_val_00000048.jpg	Is this photo taken in a place of supermarket? Please answer yes or no.	Yes
+Places365_val_00000048.jpg	Is this photo taken in a place of creek? Please answer yes or no.	No
+Places365_val_00000049.jpg	Is this picture captured in a place of arena hockey? Please answer yes or no.	Yes
+Places365_val_00000049.jpg	Is this picture captured in a place of ski slope? Please answer yes or no.	No
+Places365_val_00000050.jpg	Is this picture captured in a place of zen garden? Please answer yes or no.	Yes
+Places365_val_00000050.jpg	Is this picture captured in a place of bazaar indoor? Please answer yes or no.	No
+Places365_val_00000051.jpg	Is this picture captured in a place of library indoor? Please answer yes or no.	Yes
+Places365_val_00000051.jpg	Is this picture captured in a place of orchard? Please answer yes or no.	No
+Places365_val_00000052.jpg	Is this photo taken in a place of ice skating rink indoor? Please answer yes or no.	Yes
+Places365_val_00000052.jpg	Is this photo taken in a place of wind farm? Please answer yes or no.	No
+Places365_val_00000053.jpg	Is this picture captured in a place of arena hockey? Please answer yes or no.	Yes
+Places365_val_00000053.jpg	Is this picture captured in a place of loading dock? Please answer yes or no.	No
+Places365_val_00000054.jpg	Does this image describe a place of chalet? Please answer yes or no.	Yes
+Places365_val_00000054.jpg	Does this image describe a place of tower? Please answer yes or no.	No
+Places365_val_00000055.jpg	Does this image describe a place of bedchamber? Please answer yes or no.	Yes
+Places365_val_00000055.jpg	Does this image describe a place of music studio? Please answer yes or no.	No
+Places365_val_00000056.jpg	Is this photo taken in a place of botanical garden? Please answer yes or no.	Yes
+Places365_val_00000056.jpg	Is this photo taken in a place of legislative chamber? Please answer yes or no.	No
+Places365_val_00000057.jpg	Is this picture captured in a place of galley? Please answer yes or no.	Yes
+Places365_val_00000057.jpg	Is this picture captured in a place of physics laboratory? Please answer yes or no.	No
+Places365_val_00000058.jpg	Does this image describe a place of department store? Please answer yes or no.	Yes
+Places365_val_00000058.jpg	Does this image describe a place of rope bridge? Please answer yes or no.	No
+Places365_val_00000059.jpg	Is this photo taken in a place of schoolhouse? Please answer yes or no.	Yes
+Places365_val_00000059.jpg	Is this photo taken in a place of hangar outdoor? Please answer yes or no.	No
+Places365_val_00000060.jpg	Is this photo taken in a place of atrium public? Please answer yes or no.	Yes
+Places365_val_00000060.jpg	Is this photo taken in a place of subway station platform? Please answer yes or no.	No
+Places365_val_00000061.jpg	Is this photo taken in a place of dam? Please answer yes or no.	Yes
+Places365_val_00000061.jpg	Is this photo taken in a place of ice skating rink indoor? Please answer yes or no.	No
+Places365_val_00000062.jpg	Does this image describe a place of army base? Please answer yes or no.	Yes
+Places365_val_00000062.jpg	Does this image describe a place of train station platform? Please answer yes or no.	No
+Places365_val_00000063.jpg	Does this image describe a place of gas station? Please answer yes or no.	Yes
+Places365_val_00000063.jpg	Does this image describe a place of doorway outdoor? Please answer yes or no.	No
+Places365_val_00000064.jpg	Does this image describe a place of ballroom? Please answer yes or no.	Yes
+Places365_val_00000064.jpg	Does this image describe a place of corn field? Please answer yes or no.	No
+Places365_val_00000065.jpg	Does this image describe a place of biology laboratory? Please answer yes or no.	Yes
+Places365_val_00000065.jpg	Does this image describe a place of candy store? Please answer yes or no.	No
+Places365_val_00000066.jpg	Does this image describe a place of pantry? Please answer yes or no.	Yes
+Places365_val_00000066.jpg	Does this image describe a place of lighthouse? Please answer yes or no.	No
+Places365_val_00000067.jpg	Is this picture captured in a place of greenhouse indoor? Please answer yes or no.	Yes
+Places365_val_00000067.jpg	Is this picture captured in a place of pasture? Please answer yes or no.	No
+Places365_val_00000068.jpg	Is this photo taken in a place of kasbah? Please answer yes or no.	Yes
+Places365_val_00000068.jpg	Is this photo taken in a place of swimming hole? Please answer yes or no.	No
+Places365_val_00000069.jpg	Is this picture captured in a place of mausoleum? Please answer yes or no.	Yes
+Places365_val_00000069.jpg	Is this picture captured in a place of alcove? Please answer yes or no.	No
+Places365_val_00000070.jpg	Is this picture captured in a place of booth indoor? Please answer yes or no.	Yes
+Places365_val_00000070.jpg	Is this picture captured in a place of bazaar indoor? Please answer yes or no.	No
+Places365_val_00000071.jpg	Is this picture captured in a place of youth hostel? Please answer yes or no.	Yes
+Places365_val_00000071.jpg	Is this picture captured in a place of landfill? Please answer yes or no.	No
+Places365_val_00000072.jpg	Does this image describe a place of fire escape? Please answer yes or no.	Yes
+Places365_val_00000072.jpg	Does this image describe a place of playroom? Please answer yes or no.	No
+Places365_val_00000073.jpg	Is this photo taken in a place of rice paddy? Please answer yes or no.	Yes
+Places365_val_00000073.jpg	Is this photo taken in a place of bus station indoor? Please answer yes or no.	No
+Places365_val_00000074.jpg	Is this photo taken in a place of mausoleum? Please answer yes or no.	Yes
+Places365_val_00000074.jpg	Is this photo taken in a place of train station platform? Please answer yes or no.	No
+Places365_val_00000075.jpg	Is this photo taken in a place of entrance hall? Please answer yes or no.	Yes
+Places365_val_00000075.jpg	Is this photo taken in a place of residential neighborhood? Please answer yes or no.	No
+Places365_val_00000076.jpg	Is this photo taken in a place of bakery shop? Please answer yes or no.	Yes
+Places365_val_00000076.jpg	Is this photo taken in a place of doorway outdoor? Please answer yes or no.	No
+Places365_val_00000077.jpg	Does this image describe a place of escalator indoor? Please answer yes or no.	Yes
+Places365_val_00000077.jpg	Does this image describe a place of ice shelf? Please answer yes or no.	No
+Places365_val_00000078.jpg	Is this picture captured in a place of elevator shaft? Please answer yes or no.	Yes
+Places365_val_00000078.jpg	Is this picture captured in a place of fire escape? Please answer yes or no.	No
+Places365_val_00000079.jpg	Is this picture captured in a place of industrial area? Please answer yes or no.	Yes
+Places365_val_00000079.jpg	Is this picture captured in a place of botanical garden? Please answer yes or no.	No
+Places365_val_00000080.jpg	Is this photo taken in a place of escalator indoor? Please answer yes or no.	Yes
+Places365_val_00000080.jpg	Is this photo taken in a place of coffee shop? Please answer yes or no.	No
+Places365_val_00000081.jpg	Does this image describe a place of operating room? Please answer yes or no.	Yes
+Places365_val_00000081.jpg	Does this image describe a place of house? Please answer yes or no.	No
+Places365_val_00000082.jpg	Is this photo taken in a place of mosque outdoor? Please answer yes or no.	Yes
+Places365_val_00000082.jpg	Is this photo taken in a place of recreation room? Please answer yes or no.	No
+Places365_val_00000083.jpg	Is this photo taken in a place of cottage? Please answer yes or no.	Yes
+Places365_val_00000083.jpg	Is this photo taken in a place of assembly line? Please answer yes or no.	No
+Places365_val_00000084.jpg	Does this image describe a place of courtyard? Please answer yes or no.	Yes
+Places365_val_00000084.jpg	Does this image describe a place of greenhouse outdoor? Please answer yes or no.	No
+Places365_val_00000085.jpg	Is this photo taken in a place of flea market indoor? Please answer yes or no.	Yes
+Places365_val_00000085.jpg	Is this photo taken in a place of pavilion? Please answer yes or no.	No
+Places365_val_00000086.jpg	Does this image describe a place of banquet hall? Please answer yes or no.	Yes
+Places365_val_00000086.jpg	Does this image describe a place of storage room? Please answer yes or no.	No
+Places365_val_00000087.jpg	Is this photo taken in a place of auto factory? Please answer yes or no.	Yes
+Places365_val_00000087.jpg	Is this photo taken in a place of motel? Please answer yes or no.	No
+Places365_val_00000088.jpg	Is this photo taken in a place of cockpit? Please answer yes or no.	Yes
+Places365_val_00000088.jpg	Is this photo taken in a place of gift shop? Please answer yes or no.	No
+Places365_val_00000089.jpg	Is this photo taken in a place of martial arts gym? Please answer yes or no.	Yes
+Places365_val_00000089.jpg	Is this photo taken in a place of atrium public? Please answer yes or no.	No
+Places365_val_00000090.jpg	Is this picture captured in a place of general store outdoor? Please answer yes or no.	Yes
+Places365_val_00000090.jpg	Is this picture captured in a place of bazaar outdoor? Please answer yes or no.	No
+Places365_val_00000091.jpg	Is this photo taken in a place of motel? Please answer yes or no.	Yes
+Places365_val_00000091.jpg	Is this photo taken in a place of chemistry lab? Please answer yes or no.	No
+Places365_val_00000092.jpg	Is this photo taken in a place of playground? Please answer yes or no.	Yes
+Places365_val_00000092.jpg	Is this photo taken in a place of snowfield? Please answer yes or no.	No
+Places365_val_00000093.jpg	Is this picture captured in a place of basement? Please answer yes or no.	Yes
+Places365_val_00000093.jpg	Is this picture captured in a place of jewelry shop? Please answer yes or no.	No
+Places365_val_00000094.jpg	Is this picture captured in a place of playroom? Please answer yes or no.	Yes
+Places365_val_00000094.jpg	Is this picture captured in a place of pier? Please answer yes or no.	No
+Places365_val_00000095.jpg	Is this photo taken in a place of fabric store? Please answer yes or no.	Yes
+Places365_val_00000095.jpg	Is this photo taken in a place of beauty salon? Please answer yes or no.	No
+Places365_val_00000096.jpg	Is this picture captured in a place of forest path? Please answer yes or no.	Yes
+Places365_val_00000096.jpg	Is this picture captured in a place of balcony interior? Please answer yes or no.	No
+Places365_val_00000097.jpg	Does this image describe a place of hangar indoor? Please answer yes or no.	Yes
+Places365_val_00000097.jpg	Does this image describe a place of vegetable garden? Please answer yes or no.	No
+Places365_val_00000098.jpg	Does this image describe a place of crosswalk? Please answer yes or no.	Yes
+Places365_val_00000098.jpg	Does this image describe a place of snowfield? Please answer yes or no.	No
+Places365_val_00000099.jpg	Does this image describe a place of physics laboratory? Please answer yes or no.	Yes
+Places365_val_00000099.jpg	Does this image describe a place of stage indoor? Please answer yes or no.	No
+Places365_val_00000100.jpg	Is this picture captured in a place of pub indoor? Please answer yes or no.	Yes
+Places365_val_00000100.jpg	Is this picture captured in a place of pier? Please answer yes or no.	No
+Places365_val_00000101.jpg	Is this photo taken in a place of schoolhouse? Please answer yes or no.	Yes
+Places365_val_00000101.jpg	Is this photo taken in a place of vineyard? Please answer yes or no.	No
+Places365_val_00000102.jpg	Is this photo taken in a place of industrial area? Please answer yes or no.	Yes
+Places365_val_00000102.jpg	Is this photo taken in a place of street? Please answer yes or no.	No
+Places365_val_00000103.jpg	Is this picture captured in a place of baseball field? Please answer yes or no.	Yes
+Places365_val_00000103.jpg	Is this picture captured in a place of airfield? Please answer yes or no.	No
+Places365_val_00000104.jpg	Is this picture captured in a place of ice floe? Please answer yes or no.	Yes
+Places365_val_00000104.jpg	Is this picture captured in a place of kindergarden classroom? Please answer yes or no.	No
+Places365_val_00000105.jpg	Is this picture captured in a place of train interior? Please answer yes or no.	Yes
+Places365_val_00000105.jpg	Is this picture captured in a place of movie theater indoor? Please answer yes or no.	No
+Places365_val_00000106.jpg	Is this picture captured in a place of desert sand? Please answer yes or no.	Yes
+Places365_val_00000106.jpg	Is this picture captured in a place of burial chamber? Please answer yes or no.	No
+Places365_val_00000107.jpg	Is this photo taken in a place of castle? Please answer yes or no.	Yes
+Places365_val_00000107.jpg	Is this photo taken in a place of computer room? Please answer yes or no.	No
+Places365_val_00000108.jpg	Does this image describe a place of burial chamber? Please answer yes or no.	Yes
+Places365_val_00000108.jpg	Does this image describe a place of oilrig? Please answer yes or no.	No
+Places365_val_00000109.jpg	Is this photo taken in a place of nursing home? Please answer yes or no.	Yes
+Places365_val_00000109.jpg	Is this photo taken in a place of amusement park? Please answer yes or no.	No
+Places365_val_00000110.jpg	Does this image describe a place of church indoor? Please answer yes or no.	Yes
+Places365_val_00000110.jpg	Does this image describe a place of general store outdoor? Please answer yes or no.	No
+Places365_val_00000111.jpg	Is this picture captured in a place of palace? Please answer yes or no.	Yes
+Places365_val_00000111.jpg	Is this picture captured in a place of tree house? Please answer yes or no.	No
+Places365_val_00000112.jpg	Does this image describe a place of bus station indoor? Please answer yes or no.	Yes
+Places365_val_00000112.jpg	Does this image describe a place of hardware store? Please answer yes or no.	No
+Places365_val_00000113.jpg	Is this picture captured in a place of atrium public? Please answer yes or no.	Yes
+Places365_val_00000113.jpg	Is this picture captured in a place of music studio? Please answer yes or no.	No
+Places365_val_00000114.jpg	Is this photo taken in a place of butte? Please answer yes or no.	Yes
+Places365_val_00000114.jpg	Is this photo taken in a place of valley? Please answer yes or no.	No
+Places365_val_00000115.jpg	Is this picture captured in a place of zen garden? Please answer yes or no.	Yes
+Places365_val_00000115.jpg	Is this picture captured in a place of banquet hall? Please answer yes or no.	No
+Places365_val_00000116.jpg	Is this picture captured in a place of racecourse? Please answer yes or no.	Yes
+Places365_val_00000116.jpg	Is this picture captured in a place of basketball court indoor? Please answer yes or no.	No
+Places365_val_00000117.jpg	Is this picture captured in a place of cliff? Please answer yes or no.	Yes
+Places365_val_00000117.jpg	Is this picture captured in a place of sauna? Please answer yes or no.	No
+Places365_val_00000118.jpg	Is this photo taken in a place of laundromat? Please answer yes or no.	Yes
+Places365_val_00000118.jpg	Is this photo taken in a place of archive? Please answer yes or no.	No
+Places365_val_00000119.jpg	Does this image describe a place of wave? Please answer yes or no.	Yes
+Places365_val_00000119.jpg	Does this image describe a place of shed? Please answer yes or no.	No
+Places365_val_00000120.jpg	Is this photo taken in a place of legislative chamber? Please answer yes or no.	Yes
+Places365_val_00000120.jpg	Is this photo taken in a place of nursery? Please answer yes or no.	No
+Places365_val_00000121.jpg	Is this photo taken in a place of elevator shaft? Please answer yes or no.	Yes
+Places365_val_00000121.jpg	Is this photo taken in a place of berth? Please answer yes or no.	No
+Places365_val_00000122.jpg	Is this photo taken in a place of hayfield? Please answer yes or no.	Yes
+Places365_val_00000122.jpg	Is this photo taken in a place of skyscraper? Please answer yes or no.	No
+Places365_val_00000123.jpg	Is this photo taken in a place of hospital room? Please answer yes or no.	Yes
+Places365_val_00000123.jpg	Is this photo taken in a place of mosque outdoor? Please answer yes or no.	No
+Places365_val_00000124.jpg	Does this image describe a place of swimming hole? Please answer yes or no.	Yes
+Places365_val_00000124.jpg	Does this image describe a place of television room? Please answer yes or no.	No
+Places365_val_00000125.jpg	Is this picture captured in a place of biology laboratory? Please answer yes or no.	Yes
+Places365_val_00000125.jpg	Is this picture captured in a place of lake natural? Please answer yes or no.	No
+Places365_val_00000126.jpg	Is this picture captured in a place of church indoor? Please answer yes or no.	Yes
+Places365_val_00000126.jpg	Is this picture captured in a place of server room? Please answer yes or no.	No
+Places365_val_00000127.jpg	Is this picture captured in a place of temple asia? Please answer yes or no.	Yes
+Places365_val_00000127.jpg	Is this picture captured in a place of ball pit? Please answer yes or no.	No
+Places365_val_00000128.jpg	Is this photo taken in a place of landfill? Please answer yes or no.	Yes
+Places365_val_00000128.jpg	Is this photo taken in a place of viaduct? Please answer yes or no.	No
+Places365_val_00000129.jpg	Does this image describe a place of arena hockey? Please answer yes or no.	Yes
+Places365_val_00000129.jpg	Does this image describe a place of clothing store? Please answer yes or no.	No
+Places365_val_00000130.jpg	Is this photo taken in a place of baseball field? Please answer yes or no.	Yes
+Places365_val_00000130.jpg	Is this photo taken in a place of art gallery? Please answer yes or no.	No
+Places365_val_00000131.jpg	Is this picture captured in a place of gazebo exterior? Please answer yes or no.	Yes
+Places365_val_00000131.jpg	Is this picture captured in a place of driveway? Please answer yes or no.	No
+Places365_val_00000132.jpg	Does this image describe a place of greenhouse outdoor? Please answer yes or no.	Yes
+Places365_val_00000132.jpg	Does this image describe a place of stage indoor? Please answer yes or no.	No
+Places365_val_00000133.jpg	Is this picture captured in a place of beauty salon? Please answer yes or no.	Yes
+Places365_val_00000133.jpg	Is this picture captured in a place of junkyard? Please answer yes or no.	No
+Places365_val_00000134.jpg	Does this image describe a place of ice shelf? Please answer yes or no.	Yes
+Places365_val_00000134.jpg	Does this image describe a place of vegetable garden? Please answer yes or no.	No
+Places365_val_00000135.jpg	Is this picture captured in a place of islet? Please answer yes or no.	Yes
+Places365_val_00000135.jpg	Is this picture captured in a place of recreation room? Please answer yes or no.	No
+Places365_val_00000136.jpg	Is this picture captured in a place of railroad track? Please answer yes or no.	Yes
+Places365_val_00000136.jpg	Is this picture captured in a place of department store? Please answer yes or no.	No
+Places365_val_00000137.jpg	Is this photo taken in a place of bazaar outdoor? Please answer yes or no.	Yes
+Places365_val_00000137.jpg	Is this photo taken in a place of office? Please answer yes or no.	No
+Places365_val_00000138.jpg	Is this photo taken in a place of basement? Please answer yes or no.	Yes
+Places365_val_00000138.jpg	Is this photo taken in a place of banquet hall? Please answer yes or no.	No
+Places365_val_00000139.jpg	Is this picture captured in a place of television studio? Please answer yes or no.	Yes
+Places365_val_00000139.jpg	Is this picture captured in a place of general store indoor? Please answer yes or no.	No
+Places365_val_00000140.jpg	Is this picture captured in a place of mezzanine? Please answer yes or no.	Yes
+Places365_val_00000140.jpg	Is this picture captured in a place of bedroom? Please answer yes or no.	No
+Places365_val_00000141.jpg	Does this image describe a place of roof garden? Please answer yes or no.	Yes
+Places365_val_00000141.jpg	Does this image describe a place of television room? Please answer yes or no.	No
+Places365_val_00000142.jpg	Does this image describe a place of restaurant kitchen? Please answer yes or no.	Yes
+Places365_val_00000142.jpg	Does this image describe a place of fabric store? Please answer yes or no.	No
+Places365_val_00000143.jpg	Is this photo taken in a place of house? Please answer yes or no.	Yes
+Places365_val_00000143.jpg	Is this photo taken in a place of swimming hole? Please answer yes or no.	No
+Places365_val_00000144.jpg	Is this picture captured in a place of vineyard? Please answer yes or no.	Yes
+Places365_val_00000144.jpg	Is this picture captured in a place of castle? Please answer yes or no.	No
+Places365_val_00000145.jpg	Is this picture captured in a place of driveway? Please answer yes or no.	Yes
+Places365_val_00000145.jpg	Is this picture captured in a place of badlands? Please answer yes or no.	No
+Places365_val_00000146.jpg	Does this image describe a place of chemistry lab? Please answer yes or no.	Yes
+Places365_val_00000146.jpg	Does this image describe a place of industrial area? Please answer yes or no.	No
+Places365_val_00000147.jpg	Does this image describe a place of cabin outdoor? Please answer yes or no.	Yes
+Places365_val_00000147.jpg	Does this image describe a place of creek? Please answer yes or no.	No
+Places365_val_00000148.jpg	Is this photo taken in a place of restaurant? Please answer yes or no.	Yes
+Places365_val_00000148.jpg	Is this photo taken in a place of apartment building outdoor? Please answer yes or no.	No
+Places365_val_00000149.jpg	Is this picture captured in a place of auditorium? Please answer yes or no.	Yes
+Places365_val_00000149.jpg	Is this picture captured in a place of rock arch? Please answer yes or no.	No
+Places365_val_00000150.jpg	Is this photo taken in a place of street? Please answer yes or no.	Yes
+Places365_val_00000150.jpg	Is this photo taken in a place of toyshop? Please answer yes or no.	No
+Places365_val_00000151.jpg	Does this image describe a place of lagoon? Please answer yes or no.	Yes
+Places365_val_00000151.jpg	Does this image describe a place of aquarium? Please answer yes or no.	No
+Places365_val_00000152.jpg	Is this picture captured in a place of hangar indoor? Please answer yes or no.	Yes
+Places365_val_00000152.jpg	Is this picture captured in a place of street? Please answer yes or no.	No
+Places365_val_00000153.jpg	Does this image describe a place of bar? Please answer yes or no.	Yes
+Places365_val_00000153.jpg	Does this image describe a place of tree house? Please answer yes or no.	No
+Places365_val_00000154.jpg	Is this picture captured in a place of classroom? Please answer yes or no.	Yes
+Places365_val_00000154.jpg	Is this picture captured in a place of wheat field? Please answer yes or no.	No
+Places365_val_00000155.jpg	Does this image describe a place of creek? Please answer yes or no.	Yes
+Places365_val_00000155.jpg	Does this image describe a place of youth hostel? Please answer yes or no.	No
+Places365_val_00000156.jpg	Does this image describe a place of stable? Please answer yes or no.	Yes
+Places365_val_00000156.jpg	Does this image describe a place of junkyard? Please answer yes or no.	No
+Places365_val_00000157.jpg	Is this picture captured in a place of swamp? Please answer yes or no.	Yes
+Places365_val_00000157.jpg	Is this picture captured in a place of village? Please answer yes or no.	No
+Places365_val_00000158.jpg	Is this picture captured in a place of ice shelf? Please answer yes or no.	Yes
+Places365_val_00000158.jpg	Is this picture captured in a place of motel? Please answer yes or no.	No
+Places365_val_00000159.jpg	Is this picture captured in a place of train interior? Please answer yes or no.	Yes
+Places365_val_00000159.jpg	Is this picture captured in a place of jewelry shop? Please answer yes or no.	No
+Places365_val_00000160.jpg	Does this image describe a place of windmill? Please answer yes or no.	Yes
+Places365_val_00000160.jpg	Does this image describe a place of lake natural? Please answer yes or no.	No
+Places365_val_00000161.jpg	Is this photo taken in a place of archive? Please answer yes or no.	Yes
+Places365_val_00000161.jpg	Is this photo taken in a place of village? Please answer yes or no.	No
+Places365_val_00000162.jpg	Does this image describe a place of train station platform? Please answer yes or no.	Yes
+Places365_val_00000162.jpg	Does this image describe a place of manufactured home? Please answer yes or no.	No
+Places365_val_00000163.jpg	Is this photo taken in a place of chalet? Please answer yes or no.	Yes
+Places365_val_00000163.jpg	Is this photo taken in a place of throne room? Please answer yes or no.	No
+Places365_val_00000164.jpg	Does this image describe a place of lake natural? Please answer yes or no.	Yes
+Places365_val_00000164.jpg	Does this image describe a place of athletic field outdoor? Please answer yes or no.	No
+Places365_val_00000165.jpg	Is this picture captured in a place of playground? Please answer yes or no.	Yes
+Places365_val_00000165.jpg	Is this picture captured in a place of viaduct? Please answer yes or no.	No
+Places365_val_00000166.jpg	Is this photo taken in a place of dorm room? Please answer yes or no.	Yes
+Places365_val_00000166.jpg	Is this photo taken in a place of ski resort? Please answer yes or no.	No
+Places365_val_00000167.jpg	Is this picture captured in a place of flea market indoor? Please answer yes or no.	Yes
+Places365_val_00000167.jpg	Is this picture captured in a place of cliff? Please answer yes or no.	No
+Places365_val_00000168.jpg	Does this image describe a place of airplane cabin? Please answer yes or no.	Yes
+Places365_val_00000168.jpg	Does this image describe a place of rock arch? Please answer yes or no.	No
+Places365_val_00000169.jpg	Is this picture captured in a place of drugstore? Please answer yes or no.	Yes
+Places365_val_00000169.jpg	Is this picture captured in a place of canyon? Please answer yes or no.	No
+Places365_val_00000170.jpg	Is this picture captured in a place of pharmacy? Please answer yes or no.	Yes
+Places365_val_00000170.jpg	Is this picture captured in a place of orchestra pit? Please answer yes or no.	No
+Places365_val_00000171.jpg	Is this picture captured in a place of greenhouse outdoor? Please answer yes or no.	Yes
+Places365_val_00000171.jpg	Is this picture captured in a place of archaelogical excavation? Please answer yes or no.	No
+Places365_val_00000172.jpg	Is this photo taken in a place of bathroom? Please answer yes or no.	Yes
+Places365_val_00000172.jpg	Is this photo taken in a place of auto factory? Please answer yes or no.	No
+Places365_val_00000173.jpg	Is this photo taken in a place of shoe shop? Please answer yes or no.	Yes
+Places365_val_00000173.jpg	Is this photo taken in a place of car interior? Please answer yes or no.	No
+Places365_val_00000174.jpg	Does this image describe a place of moat water? Please answer yes or no.	Yes
+Places365_val_00000174.jpg	Does this image describe a place of marsh? Please answer yes or no.	No
+Places365_val_00000175.jpg	Does this image describe a place of vegetable garden? Please answer yes or no.	Yes
+Places365_val_00000175.jpg	Does this image describe a place of parking garage indoor? Please answer yes or no.	No
+Places365_val_00000176.jpg	Is this picture captured in a place of bowling alley? Please answer yes or no.	Yes
+Places365_val_00000176.jpg	Is this picture captured in a place of hotel outdoor? Please answer yes or no.	No
+Places365_val_00000177.jpg	Does this image describe a place of embassy? Please answer yes or no.	Yes
+Places365_val_00000177.jpg	Does this image describe a place of chemistry lab? Please answer yes or no.	No
+Places365_val_00000178.jpg	Is this photo taken in a place of youth hostel? Please answer yes or no.	Yes
+Places365_val_00000178.jpg	Is this photo taken in a place of hangar indoor? Please answer yes or no.	No
+Places365_val_00000179.jpg	Does this image describe a place of hot spring? Please answer yes or no.	Yes
+Places365_val_00000179.jpg	Does this image describe a place of mausoleum? Please answer yes or no.	No
+Places365_val_00000180.jpg	Does this image describe a place of cottage? Please answer yes or no.	Yes
+Places365_val_00000180.jpg	Does this image describe a place of parking garage indoor? Please answer yes or no.	No
+Places365_val_00000181.jpg	Does this image describe a place of general store outdoor? Please answer yes or no.	Yes
+Places365_val_00000181.jpg	Does this image describe a place of hotel room? Please answer yes or no.	No
+Places365_val_00000182.jpg	Is this photo taken in a place of chalet? Please answer yes or no.	Yes
+Places365_val_00000182.jpg	Is this photo taken in a place of dorm room? Please answer yes or no.	No
+Places365_val_00000183.jpg	Is this photo taken in a place of flea market indoor? Please answer yes or no.	Yes
+Places365_val_00000183.jpg	Is this photo taken in a place of tree house? Please answer yes or no.	No
+Places365_val_00000184.jpg	Is this photo taken in a place of shower? Please answer yes or no.	Yes
+Places365_val_00000184.jpg	Is this photo taken in a place of living room? Please answer yes or no.	No
+Places365_val_00000185.jpg	Is this picture captured in a place of supermarket? Please answer yes or no.	Yes
+Places365_val_00000185.jpg	Is this picture captured in a place of orchard? Please answer yes or no.	No
+Places365_val_00000186.jpg	Is this picture captured in a place of bowling alley? Please answer yes or no.	Yes
+Places365_val_00000186.jpg	Is this picture captured in a place of ice skating rink outdoor? Please answer yes or no.	No
+Places365_val_00000187.jpg	Is this picture captured in a place of barn? Please answer yes or no.	Yes
+Places365_val_00000187.jpg	Is this picture captured in a place of lobby? Please answer yes or no.	No
+Places365_val_00000188.jpg	Is this photo taken in a place of japanese garden? Please answer yes or no.	Yes
+Places365_val_00000188.jpg	Is this photo taken in a place of hangar indoor? Please answer yes or no.	No
+Places365_val_00000189.jpg	Is this picture captured in a place of swimming hole? Please answer yes or no.	Yes
+Places365_val_00000189.jpg	Is this picture captured in a place of bedroom? Please answer yes or no.	No
+Places365_val_00000190.jpg	Is this picture captured in a place of pizzeria? Please answer yes or no.	Yes
+Places365_val_00000190.jpg	Is this picture captured in a place of computer room? Please answer yes or no.	No
+Places365_val_00000191.jpg	Does this image describe a place of volleyball court outdoor? Please answer yes or no.	Yes
+Places365_val_00000191.jpg	Does this image describe a place of church indoor? Please answer yes or no.	No
+Places365_val_00000192.jpg	Is this photo taken in a place of market indoor? Please answer yes or no.	Yes
+Places365_val_00000192.jpg	Is this photo taken in a place of home office? Please answer yes or no.	No
+Places365_val_00000193.jpg	Is this picture captured in a place of ice floe? Please answer yes or no.	Yes
+Places365_val_00000193.jpg	Is this picture captured in a place of closet? Please answer yes or no.	No
+Places365_val_00000194.jpg	Does this image describe a place of lake natural? Please answer yes or no.	Yes
+Places365_val_00000194.jpg	Does this image describe a place of beer hall? Please answer yes or no.	No
+Places365_val_00000195.jpg	Does this image describe a place of mountain path? Please answer yes or no.	Yes
+Places365_val_00000195.jpg	Does this image describe a place of construction site? Please answer yes or no.	No
+Places365_val_00000196.jpg	Is this photo taken in a place of orchestra pit? Please answer yes or no.	Yes
+Places365_val_00000196.jpg	Is this photo taken in a place of burial chamber? Please answer yes or no.	No
+Places365_val_00000197.jpg	Does this image describe a place of village? Please answer yes or no.	Yes
+Places365_val_00000197.jpg	Does this image describe a place of underwater ocean deep? Please answer yes or no.	No
+Places365_val_00000198.jpg	Does this image describe a place of waterfall? Please answer yes or no.	Yes
+Places365_val_00000198.jpg	Does this image describe a place of booth indoor? Please answer yes or no.	No
+Places365_val_00000199.jpg	Is this photo taken in a place of greenhouse indoor? Please answer yes or no.	Yes
+Places365_val_00000199.jpg	Is this photo taken in a place of aqueduct? Please answer yes or no.	No
+Places365_val_00000200.jpg	Is this photo taken in a place of television studio? Please answer yes or no.	Yes
+Places365_val_00000200.jpg	Is this photo taken in a place of hunting lodge outdoor? Please answer yes or no.	No

internvl_chat/eval/mme/Your_Results/text_translation.txt

+0001.png	Is it appropriate to translate the Chinese in the image into English 'classic taste' in the picture? Please answer yes or no.	Yes
+0001.png	Is it appropriate to translate the Chinese in the image into English 'classic strawberry flavor' in the picture? Please answer yes or no.	No
+0002.png	Is it appropriate to translate the Chinese in the image into English 'a delicious dinner' in the picture? Please answer yes or no.	Yes
+0002.png	Is it appropriate to translate the Chinese in the image into English 'hamburger and chips' in the picture? Please answer yes or no.	No
+0003.png	Is it appropriate to translate the Chinese in the image into English 'sunny weather' in the picture? Please answer yes or no.	Yes
+0003.png	Is it appropriate to translate the Chinese in the image into English 'cold weather' in the picture? Please answer yes or no.	No
+0004.png	Is it appropriate to translate the Chinese in the image into English 'run very fast' in the picture? Please answer yes or no.	Yes
+0004.png	Is it appropriate to translate the Chinese in the image into English 'run very slow' in the picture? Please answer yes or no.	No
+0005.png	Is it appropriate to translate the Chinese in the image into English 'feeling happy' in the picture? Please answer yes or no.	Yes
+0005.png	Is it appropriate to translate the Chinese in the image into English 'feeling bored' in the picture? Please answer yes or no.	No
+0006.png	Is it appropriate to translate the Chinese in the image into English 'work hard together' in the picture? Please answer yes or no.	Yes
+0006.png	Is it appropriate to translate the Chinese in the image into English 'be filled with intrigue' in the picture? Please answer yes or no.	No
+0007.png	Is it appropriate to translate the Chinese in the image into English 'walking very slowly' in the picture? Please answer yes or no.	Yes
+0007.png	Is it appropriate to translate the Chinese in the image into English 'runing very slowly' in the picture? Please answer yes or no.	No
+0008.png	Is it appropriate to translate the Chinese in the image into English 'very proud' in the picture? Please answer yes or no.	Yes
+0008.png	Is it appropriate to translate the Chinese in the image into English 'very thankful' in the picture? Please answer yes or no.	No
+0009.png	Is it appropriate to translate the Chinese in the image into English 'creative people' in the picture? Please answer yes or no.	Yes
+0009.png	Is it appropriate to translate the Chinese in the image into English 'leading people' in the picture? Please answer yes or no.	No
+0010.png	Is it appropriate to translate the Chinese in the image into English 'a beautiful garden' in the picture? Please answer yes or no.	Yes
+0010.png	Is it appropriate to translate the Chinese in the image into English 'a beautiful campus' in the picture? Please answer yes or no.	No
+0011.png	Is it appropriate to translate the Chinese in the image into English 'a difficult work' in the picture? Please answer yes or no.	Yes
+0011.png	Is it appropriate to translate the Chinese in the image into English 'a easy work' in the picture? Please answer yes or no.	No
+0012.png	Is it appropriate to translate the Chinese in the image into English 'a small amount' in the picture? Please answer yes or no.	Yes
+0012.png	Is it appropriate to translate the Chinese in the image into English 'difficult and dangerous' in the picture? Please answer yes or no.	No
+0013.png	Is it appropriate to translate the Chinese in the image into English 'feeling frustrated' in the picture? Please answer yes or no.	Yes
+0013.png	Is it appropriate to translate the Chinese in the image into English 'feeling relaxed' in the picture? Please answer yes or no.	No
+0014.png	Is it appropriate to translate the Chinese in the image into English 'waiting for a long time' in the picture? Please answer yes or no.	Yes
+0014.png	Is it appropriate to translate the Chinese in the image into English 'sleeping for a long time' in the picture? Please answer yes or no.	No
+0015.png	Is it appropriate to translate the Chinese in the image into English 'very powerful' in the picture? Please answer yes or no.	Yes
+0015.png	Is it appropriate to translate the Chinese in the image into English 'to be fragile throughout the world' in the picture? Please answer yes or no.	No
+0016.png	Is it appropriate to translate the Chinese in the image into English 'all talk and no action' in the picture? Please answer yes or no.	Yes
+0016.png	Is it appropriate to translate the Chinese in the image into English 'hands-on practice' in the picture? Please answer yes or no.	No
+0017.png	Is it appropriate to translate the Chinese in the image into English 'delicious fruit' in the picture? Please answer yes or no.	Yes
+0017.png	Is it appropriate to translate the Chinese in the image into English 'banana' in the picture? Please answer yes or no.	No
+0018.png	Is it appropriate to translate the Chinese in the image into English 'very unforgettable' in the picture? Please answer yes or no.	Yes
+0018.png	Is it appropriate to translate the Chinese in the image into English 'very happy' in the picture? Please answer yes or no.	No
+0019.png	Is it appropriate to translate the Chinese in the image into English 'get along well' in the picture? Please answer yes or no.	Yes
+0019.png	Is it appropriate to translate the Chinese in the image into English 'for own self-interest' in the picture? Please answer yes or no.	No
+0020.png	Is it appropriate to translate the Chinese in the image into English 'rank first' in the picture? Please answer yes or no.	Yes
+0020.png	Is it appropriate to translate the Chinese in the image into English 'to add the finishing touches' in the picture? Please answer yes or no.	No

internvl_chat/eval/mme/calculation.py

+import argparse
+import os
+
+from sklearn.metrics import (accuracy_score, confusion_matrix, precision_score,
+                             recall_score)
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--results_dir', default='./LaVIN', type=str)
+
+eval_type_dict = {
+    'Perception': ['existence', 'count', 'position', 'color', 'posters', 'celebrity', 'scene', 'landmark', 'artwork', 'OCR'],
+    'Cognition': ['commonsense_reasoning', 'numerical_calculation', 'text_translation', 'code_reasoning']
+}
+
+
+class calculate_metrics:
+    def divide_chunks(self, l, n=2):
+        # looping till length l
+        for i in range(0, len(l), n):
+            yield l[i:i + n]
+
+        return
+
+    def parse_pred_ans(self, pred_ans):
+        pred_label = None
+        if pred_ans in ['yes', 'no']:
+            pred_label = pred_ans
+        else:
+            prefix_pred_ans = pred_ans[:4]
+
+            if 'yes' in prefix_pred_ans:
+                pred_label = 'yes'
+            elif 'no' in prefix_pred_ans:
+                pred_label = 'no'
+            else:
+                pred_label = 'other'
+
+        return pred_label
+
+    def compute_metric(self, gts, preds):
+        assert len(gts) == len(preds)
+
+        label_map = {
+            'yes': 1,
+            'no': 0,
+            'other': -1,
+        }
+
+        gts = [label_map[x] for x in gts]
+        preds = [label_map[x] for x in preds]
+
+        acc = accuracy_score(gts, preds)
+
+        clean_gts = []
+        clean_preds = []
+        other_num = 0
+        for gt, pred in zip(gts, preds):
+            if pred == -1:
+                other_num += 1
+                continue
+            clean_gts.append(gt)
+            clean_preds.append(pred)
+
+        conf_mat = confusion_matrix(clean_gts, clean_preds, labels=[1,0])
+        precision = precision_score(clean_gts, clean_preds, average='binary')
+        recall = recall_score(clean_gts, clean_preds, average='binary')
+        tp, fn = conf_mat[0]
+        fp, tn = conf_mat[1]
+
+        metric_dict = dict()
+        metric_dict = {
+            'TP': tp,
+            'FN': fn,
+            'TN': tn,
+            'FP': fp,
+            'precision': precision,
+            'recall': recall,
+            'other_num': other_num,
+            'acc': acc,
+        }
+
+        return metric_dict
+
+    def process_result(self, results_dir):
+
+        model_score_dict = dict()
+        for eval_type, task_name_list in eval_type_dict.items():
+            print('===========', eval_type, '===========')
+
+            scores = 0
+            task_score_dict = dict()
+
+            for task_name in task_name_list:
+
+                task_txt = os.path.join(results_dir, task_name + '.txt')
+                lines = open(task_txt, 'r').readlines()
+                chunk_lines = list(self.divide_chunks(lines)) # one image corresponds to two questions
+
+                img_num = len(chunk_lines)
+                task_other_ans_num = 0
+                task_score = 0
+                acc_plus_correct_num = 0
+                gts = []
+                preds = []
+
+                for img_items in chunk_lines:
+                    assert len(img_items) == 2
+                    img_correct_num = 0
+
+                    for img_item in img_items:
+                        try:
+                            img_name, question, gt_ans, pred_ans = img_item.split('\t')
+                        except:
+                            print(img_item)
+                            continue
+                        gt_ans = gt_ans.lower()
+                        pred_ans = pred_ans.lower()
+
+                        assert gt_ans in ['yes', 'no'] # gt can only be yes or no.
+
+                        pred_ans = self.parse_pred_ans(pred_ans)
+                        assert pred_ans in ['yes', 'no', 'other']
+
+                        gts.append(gt_ans)
+                        preds.append(pred_ans)
+
+                        if gt_ans == pred_ans:
+                            img_correct_num += 1
+
+                        if pred_ans not in ['yes', 'no']:
+                            task_other_ans_num += 1
+
+                    if img_correct_num == 2:
+                        acc_plus_correct_num += 1
+
+                # cal TP precision acc, etc.
+                metric_dict = self.compute_metric(gts, preds)
+                acc_plus = acc_plus_correct_num / img_num
+                metric_dict['acc_plus'] = acc_plus
+
+                for k, v in metric_dict.items():
+                    if k in ['acc', 'acc_plus']:
+                        task_score += v*100
+
+                task_score_dict[task_name] = task_score
+
+                scores += task_score
+
+            print('total score:', scores, '\n')
+            for task_name, score in task_score_dict.items():
+                print('\t', task_name, ' score:', score)
+            print('\n')
+
+        return
+
+
+if __name__ == '__main__':
+    cal = calculate_metrics()
+
+    args = parser.parse_args()
+    results_dir = args.results_dir
+    cal.process_result(results_dir)

internvl_chat/eval/mme/eval.py

+import argparse
+import os
+import re
+
+import torch
+from internvl.model import load_model_and_tokenizer
+from internvl.train.dataset import build_transform, dynamic_preprocess
+from PIL import Image
+from tqdm import tqdm
+
+
+def load_image(image_file, input_size=224):
+    image = Image.open(image_file).convert('RGB')
+    transform = build_transform(is_train=False, input_size=input_size)
+    if args.dynamic:
+        images = dynamic_preprocess(image, image_size=input_size,
+                                    use_thumbnail=use_thumbnail,
+                                    max_num=args.max_num)
+    else:
+        images = [image]
+    pixel_values = [transform(image) for image in images]
+    pixel_values = torch.stack(pixel_values)
+    return pixel_values
+
+
+def post_processing(response):
+    response = response.replace('\n', '').replace('不是', 'No').replace('是', 'Yes').replace('否', 'No')
+    response = response.lower().replace('true', 'yes').replace('false', 'no')
+    pattern = re.compile(r'[\u4e00-\u9fa5]')
+    response = re.sub(pattern, '', response)
+    return response
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--checkpoint', type=str, default='')
+    parser.add_argument('--root', type=str, default='./Your_Results')
+    parser.add_argument('--num-beams', type=int, default=1)
+    parser.add_argument('--top-k', type=int, default=50)
+    parser.add_argument('--top-p', type=float, default=0.9)
+    parser.add_argument('--sample', type=bool, default=False)
+    parser.add_argument('--dynamic', action='store_true')
+    parser.add_argument('--max-num', type=int, default=6)
+    parser.add_argument('--load-in-8bit', action='store_true')
+    parser.add_argument('--load-in-4bit', action='store_true')
+    parser.add_argument('--auto', action='store_true')
+    args = parser.parse_args()
+
+    model, tokenizer = load_model_and_tokenizer(args)
+    image_size = model.config.force_image_size or model.config.vision_config.image_size
+    use_thumbnail = model.config.use_thumbnail
+
+    total_params = sum(p.numel() for p in model.parameters()) / 1e9
+    if total_params > 20 or args.dynamic:
+        args.num_beams = 1
+        print(f'[test] total_params: {total_params}B, use num_beams: {args.num_beams}')
+    else:
+        print(f'[test] total_params: {total_params}B')
+    print(f'[test] image_size: {image_size}')
+    print(f'[test] template: {model.config.template}')
+    print(f'[test] dynamic_image_size: {args.dynamic}')
+    print(f'[test] use_thumbnail: {use_thumbnail}')
+    print(f'[test] max_num: {args.max_num}')
+
+    output = os.path.basename(args.checkpoint)
+    os.makedirs(output, exist_ok=True)
+    prompt = 'Answer the question using a single word or phrase.'
+
+    for filename in os.listdir(args.root):
+        fin = open(os.path.join(args.root, filename), 'r', encoding='utf-8')
+        fout = open(os.path.join(output, filename), 'w', encoding='utf-8')
+        lines = fin.readlines()
+        filename = filename.replace('.txt', '')
+        for line in tqdm(lines):
+            img, question, gt = line.strip().split('\t')
+            question = question + ' ' + prompt
+            img_path = os.path.join('../../data/mme/MME_Benchmark_release_version', filename, img)
+            assert os.path.exists(img_path), img_path
+            pixel_values = load_image(img_path, image_size).cuda().to(torch.bfloat16)
+            generation_config = dict(
+                do_sample=args.sample,
+                top_k=args.top_k,
+                top_p=args.top_p,
+                num_beams=args.num_beams,
+                max_new_tokens=20,
+                eos_token_id=tokenizer.eos_token_id,
+            )
+            response = model.chat(
+                tokenizer=tokenizer,
+                pixel_values=pixel_values,
+                question=question,
+                generation_config=generation_config,
+                verbose=True
+            )
+            response = post_processing(response)
+            print(img, question, gt, response, sep='\t', file=fout)
+        fin.close()
+        fout.close()

internvl_chat/eval/mmhal/README.md

+# README for Evaluation
+
+## 🌟 Overview
+
+This script provides an evaluation pipeline for `MMHal-Bench`.
+
+For scoring, we use **GPT-4o** as the evaluation model.
+
+## 🗂️ Data Preparation
+
+Before starting to download the data, please create the `InternVL/internvl_chat/data` folder.
+
+### MMHal-Bench
+
+Follow the instructions below to prepare the data:
+
+```shell
+# Step 1: Create the data directory
+mkdir -p data/mm-halbench && cd data/mm-halbench
+
+# Step 2: Download the `mmhal-bench_with_image.jsonl` file
+# This file is provided by RLAIF-V
+# See here: https://github.com/RLHF-V/RLAIF-V/blob/main/README.md#mmhal-bench
+wget https://huggingface.co/OpenGVLab/InternVL/resolve/main/mmhal-bench_with_image.jsonl
+
+cd ../..
+```
+
+After preparation is complete, the directory structure is:
+
+```shell
+data/mm-halbench
+ └── mmhal-bench_with_image.jsonl
+```
+
+## 🏃 Evaluation Execution
+
+> ⚠️ Note: For testing InternVL (1.5, 2.0, 2.5, and later versions), always enable `--dynamic` to perform dynamic resolution testing.
+
+To run the evaluation, execute the following command on an 8-GPU setup:
+
+```shell
+torchrun --nproc_per_node=8 eval/mmhal/evaluate_mmhal.py --checkpoint ${CHECKPOINT} --dynamic
+```
+
+Alternatively, you can run the following simplified command:
+
+```shell
+GPUS=8 sh evaluate.sh ${CHECKPOINT} mmhal --dynamic
+```
+
+### Arguments
+
+The following arguments can be configured for the evaluation script:
+
+| Argument         | Type   | Default                    | Description                                                                                                       |
+| ---------------- | ------ | -------------------------- | ----------------------------------------------------------------------------------------------------------------- |
+| `--checkpoint`   | `str`  | `''`                       | Path to the model checkpoint.                                                                                     |
+| `--datasets`     | `str`  | `'mmhal-bench_with_image'` | Comma-separated list of datasets to evaluate.                                                                     |
+| `--dynamic`      | `flag` | `False`                    | Enables dynamic high resolution preprocessing.                                                                    |
+| `--max-num`      | `int`  | `6`                        | Maximum tile number for dynamic high resolution.                                                                  |
+| `--load-in-8bit` | `flag` | `False`                    | Loads the model weights in 8-bit precision.                                                                       |
+| `--auto`         | `flag` | `False`                    | Automatically splits a large model across 8 GPUs when needed, useful for models too large to fit on a single GPU. |

internvl_chat/eval/mmhal/eval_gpt_mmhal.py

+import argparse
+import json
+import os
+import time
+
+import openai
+
+template = '''Please act as an impartial and objective judge and evaluate the quality of the response provided by a Large Multimodal Model (LMM) to the user question. Your evaluation should be mainly based on whether the response is informative, and whether the response contains any hallucination. Hallucination, in this context, refers to a situation where the LMM generates a response that includes information not present or implied in the image or previous conversation. A hallucination could be a false claim about an object, action, emotion, or any other detail that is not grounded in the image.
+
+For clarity, consider these examples:
+
+### Example 1:
+- Image Contents: Lynx, Cat
+- Question: How many cats can you see in this image?
+- Standard Human-Generated Answer: There are two cats visible in the image.
+- LMM Response to Evaluate: There is one cat in the image.
+- Analysis: The LMM's response is false because it claims there is only one cat in the image, when in fact there are two.
+- Hallucination: Yes.
+
+### Example 2:
+- Image Contents: Tree, Plant, Panda, Animal
+- Question: What can you see from this photo?
+- Standard Human-Generated Answer: In this photo we can see branches. We can see a panda on a branch. The background is blurry.
+- LMM Response to Evaluate: In the photo, I can see a large black and white panda sitting on a tree branch. The image captures the panda's unique appearance and its natural habitat, showcasing the beauty and rarity of these animals.
+- Analysis: The LMM's response is not false because it does not make any false claims about the image contents. For example, it does not claim that the panda is eating bamboo.
+- Hallucination: No.
+
+### Example 3:
+- Image Contents: Fashion accessory, Mammal, Dog, Umbrella
+- Question: Describe this image in details.
+- Standard Human-Generated Answer: In this picture I can see a dog laying under a yellow umbrella on a beach, and there is a rope on the right side. In the background I can see few persons talking or lying. At the top of the image, I can see the blue and clear sky.
+- LMM Response to Evaluate: The image features a brown dog laying on the sand at the beach, underneath a yellow umbrella. The dog is positioned near the water, and there are people in the background. The beach is sandy, and the sun is shining, creating a pleasant atmosphere. The dog is enjoying the shade provided by the umbrella, which is a common way to protect pets from the sun's harmful rays. The presence of people in the background suggests that the beach is a popular spot for leisure activities, such as sunbathing, swimming, and relaxing.
+- Analysis: The LMM's response is false because it claims that the dog is positioned near the water, but there is no water in the image.
+- Hallucination: Yes.
+
+### Example 4:
+- Image Contents: Clothing, Luggage and bags, Man, Building, Woman, Person
+- Question: Where is this photo taken?
+- Standard Human-Generated Answer: From the slogan "Welcome to Houston, Texas", we can infer that this photo is taken at an airport in Houston, Texas.
+- LMM Response to Evaluate: This photo is taken at an airport, specifically at the entrance of a terminal. However, the specific location of this airport cannot be determined from the image.
+- Analysis: Although the LMM's response does not explicitly mention the location "Houston, Texas" and does not perfectly answer the question, it makes no false claims about the image contents. Therefore, this response is not considered hallucination. It is OK if the LMM fails to directly answer the question, expresses uncertainty, or cannot include all the objects or details in the human-generated answer.
+- Hallucination: No.
+
+### Example 5:
+- Image Contents: Tree, Tower
+- Question: What environmental concerns are involved in this photo?
+- Standard Human-Generated Answer: This photo shows a smokestack emitting smoke. It may cause severe air pollution which threatens human health.
+- LMM Response to Evaluate: In the photo, there are several environmental concerns related to the smokestack emitting smoke. The smoke from the smokestack is a byproduct of industrial processes, which can contribute to air pollution, climate change, and negative impacts on human health. The smoke contains harmful pollutants, such as particulate matter, sulfur dioxide, nitrogen oxides, and carbon monoxide, which can lead to respiratory problems, heart disease, and other health issues. Additionally, the smoke contributes to the greenhouse effect, which can lead to global warming and climate change. The smokestack's emissions also affect the environment, as they can harm wildlife, vegetation, and ecosystems. Therefore, it is essential to address these environmental concerns by implementing measures to reduce emissions and promote sustainable practices in industrial processes.
+- Analysis: Although the LMM's response is significantly longer than the standard human-generated answer, it does not contain any false claims about the image contents. Instead, it provides additional general information about the environmental concerns, which can be inferred from the smoke emission. Such detailed analysis or reasoning should be considered as a positive aspect, as long as it contains no false claims.
+- Hallucination: No.
+
+With these examples in mind, please help me evaluate whether the response by the LMM is informative, and whether hallucination exists in it, based on the comparison between the LMM's response and the factual information provided in the image contents, question, and the standard human-generated answer below.
+
+Please note that the standard human-generated answer may only contain factual information but may not give a detailed analysis. Also, the standard human-generated answer may not be completely comprehensive in describing all the objects and their attributes, so please be a bit more cautious during evalutation. LMM's detailed analysis or reasoning should be encouraged.
+
+To evaluate the LMM responses, first, begin your evaluation by providing a short explanation. Second, after providing your explanation, you must rate the response by choosing from the following options:
+- Rating: 6, very informative with good analysis or reasoning, no hallucination
+- Rating: 5, very informative, no hallucination
+- Rating: 4, somewhat informative, no hallucination
+- Rating: 3, not informative, no hallucination
+- Rating: 2, very informative, with hallucination
+- Rating: 1, somewhat informative, with hallucination
+- Rating: 0, not informative, with hallucination
+
+### Image Contents
+{}
+
+### Question
+{}
+
+### Standard Human-Generated Answer
+{}
+
+### LMM Response to Evaluate
+{}
+'''
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--response', type=str, default='responses/idefics_80b.json',
+                        help='response file containing images, questions, and model responses')
+    parser.add_argument('--gpt-model', type=str, default='gpt-4o')
+    args = parser.parse_args()
+
+    # get api_key from the environment variable OPENAI_API_KEY
+    api_key = os.environ.get('OPENAI_API_KEY', None)
+    assert api_key is not None
+
+    # load json file
+    with open(args.response, 'r') as f:
+        records = json.load(f)
+
+    assert len(records) == 96
+
+    # ask GPT-4 to evaluate
+    responses = []
+    openai_client = openai.OpenAI(api_key=api_key, http_client=None)
+    for i, record in enumerate(records):
+        image_content = ', '.join(record['image_content'])
+        input_text = template.format(image_content, record['question'], record['gt_answer'], record['model_answer'])
+
+        response = None
+        while response is None:
+            try:
+                response = openai_client.chat.completions.create(
+                    model=args.gpt_model,
+                    messages=[
+                        {'role': 'user', 'content': input_text}
+                    ],
+                    temperature=0.0,
+                )
+            except Exception as e:
+                print(e)
+                print('retrying...')
+                time.sleep(10)
+                continue
+        response = response.to_dict()
+        response = response['choices'][0]['message']['content'].strip()
+        print(i, response, flush=True)
+        responses.append(response)
+        time.sleep(0.1)
+
+    # save responses
+    output_path = args.response.replace('.json', '_gpt4o.json')
+    with open(output_path, 'w') as f:
+        json.dump(responses, f, indent=2)
+
+    # analyze responses
+    scores = []
+    for i, response in enumerate(responses):
+        scores_found = []
+        for s in range(7):
+            if f'rating: {s}' in response.lower():
+                scores_found.append(s)
+        if len(scores_found) == 1:
+            scores.append(scores_found[0])
+        else:
+            print('Warning: multiple or zero scores found')
+            print(i, response)
+            scores.append(0)
+
+    hallucination = []
+    for s in scores:
+        if s >= 3:
+            hallucination.append(0)
+        else:
+            hallucination.append(1)
+
+    scores_each = [[] for _ in range(8)]
+    # assuming order of 96 questions is not changed
+    for i in range(96):
+        question_type = i % 8
+        scores_each[question_type].append(scores[i])
+
+    print('Average score: {:.2f}'.format(sum(scores) / len(scores)))
+    print('Hallucination rate: {:.2f}'.format(sum(hallucination) / len(hallucination)))
+    print('Average score for each question type:', ','.join([str(round(sum(scores_each[i]) / len(scores_each[i]), 2)) for i in range(8)]), flush=True)

internvl_chat/eval/mmhal/evaluate_mmhal.py

+import argparse
+import base64
+import io
+import itertools
+import json
+import os
+import random
+import subprocess
+import time
+from functools import partial
+
+import torch
+from internvl.model import load_model_and_tokenizer
+from internvl.train.dataset import build_transform, dynamic_preprocess
+from PIL import Image
+from tqdm import tqdm
+from transformers import AutoConfig, AutoTokenizer
+
+ds_collections = {
+    'mmhal-bench_with_image': {
+        'root': 'data/mm-halbench/mmhal-bench_with_image.jsonl',
+        'max_new_tokens': 1024,
+        'min_new_tokens': 1,
+        'split': 'validation'
+    },
+}
+
+
+def collate_fn(batches, tokenizer):
+    pixel_values = torch.cat([_['pixel_values'] for _ in batches], dim=0)
+    questions = [_['question'] for _ in batches]
+    question_ids = [_['question_id'] for _ in batches]
+    annotations = [_['annotation'] for _ in batches]
+    lines = [_['line'] for _ in batches]
+
+    return pixel_values, questions, question_ids, annotations, lines
+
+
+class VQADataset(torch.utils.data.Dataset):
+
+    def __init__(
+            self,
+            questions_path,
+            prompt,
+            input_size=224,
+            dynamic_image_size=False,
+            use_thumbnail=False,
+            max_num=6,
+    ):
+        self.test = open(questions_path).readlines()
+        self.prompt = prompt
+        self.input_size = input_size
+        self.dynamic_image_size = dynamic_image_size
+        self.use_thumbnail = use_thumbnail
+        self.max_num = max_num
+        self.transform = build_transform(is_train=False, input_size=input_size)
+
+    def __len__(self):
+        return len(self.test)
+
+    def __getitem__(self, idx):
+        data = json.loads(self.test[idx].strip())
+        question = data['question']
+        question_id = data.get('question_id', idx)
+        annotation = data.get('answer', None)
+
+        if 'image' in data:
+            image_file = data['image']
+            image_bytes = base64.b64decode(image_file)
+            image = Image.open(io.BytesIO(image_bytes)).convert('RGB')
+        elif 'image_path' in data:
+            image_path = data['image_path']
+            image = Image.open(image_path).convert('RGB')
+        else:
+            raise NotImplementedError
+
+        if self.dynamic_image_size:
+            images = dynamic_preprocess(image, image_size=self.input_size,
+                                        use_thumbnail=self.use_thumbnail,
+                                        max_num=self.max_num)
+        else:
+            images = [image]
+        pixel_values = [self.transform(image) for image in images]
+        pixel_values = torch.stack(pixel_values)
+        if len(self.prompt) != 0:
+            question = question + '\n' + self.prompt
+            question = question.strip()
+        return {
+            'question_id': question_id,
+            'question': question,
+            'pixel_values': pixel_values,
+            'annotation': annotation,
+            'line': data,
+        }
+
+
+class InferenceSampler(torch.utils.data.sampler.Sampler):
+
+    def __init__(self, size):
+        self._size = int(size)
+        assert size > 0
+        self._rank = torch.distributed.get_rank()
+        self._world_size = torch.distributed.get_world_size()
+        self._local_indices = self._get_local_indices(size, self._world_size, self._rank)
+
+    @staticmethod
+    def _get_local_indices(total_size, world_size, rank):
+        shard_size = total_size // world_size
+        left = total_size % world_size
+        shard_sizes = [shard_size + int(r < left) for r in range(world_size)]
+
+        begin = sum(shard_sizes[:rank])
+        end = min(sum(shard_sizes[:rank + 1]), total_size)
+        return range(begin, end)
+
+    def __iter__(self):
+        yield from self._local_indices
+
+    def __len__(self):
+        return len(self._local_indices)
+
+
+def evaluate_chat_model():
+    random.seed(args.seed)
+
+    input_prompt = ''
+    if args.cot:
+        cot_prompt = (
+            'Please think step by step and '
+            'output in the following format:\n\n'
+            'Rationale:\n{rationale}\n'
+            'Answer: {answer}\n\n'
+            'where rationale is the reasoning process of the given question.'
+        )
+        input_prompt = f'{input_prompt}\n{cot_prompt}'
+
+    for ds_name in args.datasets:
+        dataset = VQADataset(
+            questions_path=ds_collections[ds_name]['root'],
+            prompt=input_prompt,
+            input_size=image_size,
+            dynamic_image_size=args.dynamic,
+            use_thumbnail=use_thumbnail,
+            max_num=args.max_num
+        )
+        dataloader = torch.utils.data.DataLoader(
+            dataset=dataset,
+            sampler=InferenceSampler(len(dataset)),
+            batch_size=args.batch_size,
+            num_workers=args.num_workers,
+            pin_memory=True,
+            drop_last=False,
+            collate_fn=partial(collate_fn, tokenizer=tokenizer),
+        )
+
+        outputs = []
+        for _, (pixel_values, questions, question_ids, annotations, lines) in tqdm(enumerate(dataloader)):
+            pixel_values = pixel_values.to(torch.bfloat16).cuda()
+            generation_config = dict(
+                num_beams=args.num_beams,
+                max_new_tokens=ds_collections[ds_name]['max_new_tokens'],
+                min_new_tokens=ds_collections[ds_name]['min_new_tokens'],
+                do_sample=True if args.temperature > 0 else False,
+                temperature=args.temperature,
+            )
+            pred = model.chat(
+                tokenizer=tokenizer,
+                pixel_values=pixel_values,
+                question=questions[0],
+                generation_config=generation_config,
+                verbose=True
+            )
+            if args.cot:
+                pred = pred.split('Answer:')[-1].strip()
+            preds = [pred]
+
+            for question, question_id, pred, line in zip(questions, question_ids, preds, lines):
+                del line['image']
+                line['model_answer'] = pred
+                outputs.append(line)
+
+        torch.distributed.barrier()
+
+        world_size = torch.distributed.get_world_size()
+        merged_outputs = [None for _ in range(world_size)]
+        torch.distributed.all_gather_object(merged_outputs, json.dumps(outputs))
+
+        merged_outputs = [json.loads(_) for _ in merged_outputs]
+        merged_outputs = [_ for _ in itertools.chain.from_iterable(merged_outputs)]
+
+        if torch.distributed.get_rank() == 0:
+            print(f'Evaluating {ds_name} ...')
+            time_prefix = time.strftime('%y%m%d%H%M%S', time.localtime())
+            results_file = f'{ds_name}_{time_prefix}.json'
+            results_file = os.path.join(args.out_dir, results_file)
+            with open(results_file, 'w') as file:
+                json.dump(merged_outputs, file)
+            print('Results saved to {}'.format(results_file))
+
+            cmd = f'python eval/mmhal/eval_gpt_mmhal.py --response {results_file}'
+            print(cmd)
+            os.system(cmd)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--checkpoint', type=str, default='')
+    parser.add_argument('--datasets', type=str, default='mmhal-bench_with_image')
+    parser.add_argument('--batch-size', type=int, default=1)
+    parser.add_argument('--num-workers', type=int, default=1)
+    parser.add_argument('--num-beams', type=int, default=1)
+    parser.add_argument('--temperature', type=float, default=0.0)
+    parser.add_argument('--out-dir', type=str, default='results')
+    parser.add_argument('--seed', type=int, default=0)
+    parser.add_argument('--dynamic', action='store_true')
+    parser.add_argument('--max-num', type=int, default=6)
+    parser.add_argument('--load-in-8bit', action='store_true')
+    parser.add_argument('--load-in-4bit', action='store_true')
+    parser.add_argument('--auto', action='store_true')
+    parser.add_argument('--cot', action='store_true')
+    args = parser.parse_args()
+
+    if not os.path.exists(args.out_dir):
+        os.makedirs(args.out_dir, exist_ok=True)
+
+    args.datasets = args.datasets.split(',')
+    print('datasets:', args.datasets)
+
+    assert args.batch_size == 1, 'Only batch size 1 is supported'
+
+    torch.distributed.init_process_group(
+        backend='nccl',
+        world_size=int(os.getenv('WORLD_SIZE', '1')),
+        rank=int(os.getenv('RANK', '0')),
+    )
+
+    torch.cuda.set_device(int(os.getenv('LOCAL_RANK', 0)))
+
+    model, tokenizer = load_model_and_tokenizer(args)
+    image_size = model.config.force_image_size or model.config.vision_config.image_size
+    use_thumbnail = model.config.use_thumbnail
+
+    total_params = sum(p.numel() for p in model.parameters()) / 1e9
+    if total_params > 20 or args.dynamic:
+        args.num_beams = 1
+        print(f'[test] total_params: {total_params}B, use num_beams: {args.num_beams}')
+    else:
+        print(f'[test] total_params: {total_params}B')
+    print(f'[test] image_size: {image_size}')
+    print(f'[test] template: {model.config.template}')
+    print(f'[test] dynamic_image_size: {args.dynamic}')
+    print(f'[test] use_thumbnail: {use_thumbnail}')
+    print(f'[test] max_num: {args.max_num}')
+
+    evaluate_chat_model()

internvl_chat/eval/mmiu/README.md

+# README for Evaluation
+
+## 🌟 Overview
+
+This script provides an evaluation pipeline for `MMIU`.
+
+## 🗂️ Data Preparation
+
+Before starting to download the data, please create the `InternVL/internvl_chat/data` folder.
+
+### MMIU
+
+Follow the instructions below to prepare the data:
+
+```shell
+# Step 1: Create the data directory
+mkdir -p data/mmiu && cd data/mmiu
+
+# Step 2: Download images
+wget https://huggingface.co/MMIUBenchmark/MMIU/resolve/main/2D-spatial.zip
+wget https://huggingface.co/MMIUBenchmark/MMIU/resolve/main/3D-spatial.zip
+unzip 2D-spatial.zip
+unzip 3D-spatial.zip
+
+cd ../..
+```
+
+After preparation is complete, the directory structure is:
+
+```shell
+data/mmiu
+ ├── 2D-spatial
+ └── 3D-spatial
+```
+
+## 🏃 Evaluation Execution
+
+> ⚠️ Note: For testing InternVL (1.5, 2.0, 2.5, and later versions), always enable `--dynamic` to perform dynamic resolution testing.
+
+To run the evaluation, execute the following command on an 8-GPU setup:
+
+```shell
+torchrun --nproc_per_node=8 eval/mmiu/evaluate_mmiu.py --checkpoint ${CHECKPOINT} --dynamic --max-num 12
+```
+
+Alternatively, you can run the following simplified command:
+
+```shell
+GPUS=8 sh evaluate.sh ${CHECKPOINT} mmiu --dynamic --max-num 12
+```
+
+### Arguments
+
+The following arguments can be configured for the evaluation script:
+
+| Argument         | Type   | Default  | Description                                                                                                       |
+| ---------------- | ------ | -------- | ----------------------------------------------------------------------------------------------------------------- |
+| `--checkpoint`   | `str`  | `''`     | Path to the model checkpoint.                                                                                     |
+| `--datasets`     | `str`  | `'mmiu'` | Comma-separated list of datasets to evaluate.                                                                     |
+| `--dynamic`      | `flag` | `False`  | Enables dynamic high resolution preprocessing.                                                                    |
+| `--max-num`      | `int`  | `12`     | Maximum tile number for dynamic high resolution.                                                                  |
+| `--load-in-8bit` | `flag` | `False`  | Loads the model weights in 8-bit precision.                                                                       |
+| `--auto`         | `flag` | `False`  | Automatically splits a large model across 8 GPUs when needed, useful for models too large to fit on a single GPU. |

internvl_chat/eval/mmiu/evaluate_mmiu.py

+import argparse
+import itertools
+import json
+import os
+import random
+import time
+from functools import partial
+
+import torch
+from internvl.model import load_model_and_tokenizer
+from internvl.train.dataset import build_transform, dynamic_preprocess
+from PIL import Image
+from tqdm import tqdm
+
+ds_collections = {
+    'mmiu': {
+        'root': 'data/mmiu',
+        'annotation': 'eval/mmiu/mmiu.jsonl',
+        'max_new_tokens': 100,
+        'min_new_tokens': 1,
+    },
+}
+
+
+def collate_fn(batches, tokenizer):
+    pixel_values = torch.cat([_['pixel_values'] for _ in batches], dim=0)
+    questions = [_['question'] for _ in batches]
+    answers = [_['answer'] for _ in batches]
+    num_patches_lists = [_['num_patches_list'] for _ in batches]
+    options = [_['option'] for _ in batches]
+    lines = [_['line'] for _ in batches]
+    return pixel_values, questions, answers, num_patches_lists, options, lines
+
+
+class MMIUDataset(torch.utils.data.Dataset):
+
+    def __init__(self, meta, input_size=224, dynamic_image_size=False,
+                 use_thumbnail=False, max_num=6):
+        # run for each subject
+        meta_path = meta['annotation']
+        f = open(meta_path, 'r')
+        lines = f.readlines()
+        self.data = [json.loads(line) for line in lines]
+        self.root = meta['root']
+        self.input_size = input_size
+        self.dynamic_image_size = dynamic_image_size
+        self.use_thumbnail = use_thumbnail
+        self.max_num = max_num
+        self.transform = build_transform(is_train=False, input_size=input_size)
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, idx):
+        data = self.data[idx]
+        input_image_path = data['input']['input_image_path']
+        if len(input_image_path) == 1:
+            image_prefix = ''
+        else:
+            image_cnt = len(input_image_path)
+            image_prefix = ''.join([f'Image-{i+1}: <image>\n' for i in range(image_cnt)])
+        question = data['input']['question']
+        context = data['input']['context']
+        question = image_prefix + question + '\n' + context + "\nAnswer with the option's letter from the given choices directly."
+
+        input_image_path = [os.path.join(self.root, item) for item in input_image_path]
+        image_list = []
+        for image_path in input_image_path:
+            image = Image.open(image_path).convert('RGB')
+            image_list.append(image)
+
+        options = data['options'].split('\n')
+        answer = data['output']['output_text']
+
+        new_options = {}
+        multiple_choices = ['A', 'B', 'C', 'D', 'E', 'F', 'G',
+                            'H', 'I', 'J', 'K', 'L', 'M', 'N',
+                            'O', 'P', 'Q', 'R', 'S', 'T', 'U',
+                            'V', 'W', 'X', 'Y', 'Z']
+        for i, c in enumerate(options):
+            c = c.strip()
+            if c.startswith(f'{multiple_choices[i]}:'):
+                c = c.replace(f'{multiple_choices[i]}:', '')
+            new_options[multiple_choices[i]] = c.strip()
+
+        num_patches_list = []
+        if self.dynamic_image_size:
+            images = []
+            for image in image_list:
+                tiles = dynamic_preprocess(image, image_size=self.input_size,
+                                           use_thumbnail=self.use_thumbnail,
+                                           max_num=max(1, self.max_num // len(image_list)))
+                images += tiles
+                num_patches_list.append(len(tiles))
+        else:
+            images = image_list
+            num_patches_list.append(1)
+        pixel_values = [self.transform(image) for image in images]
+        pixel_values = torch.stack(pixel_values)
+
+        return {
+            'question': question,
+            'pixel_values': pixel_values,
+            'answer': answer,
+            'option': new_options,
+            'num_patches_list': num_patches_list,
+            'line': data
+        }
+
+
+class InferenceSampler(torch.utils.data.sampler.Sampler):
+
+    def __init__(self, size):
+        self._size = int(size)
+        assert size > 0
+        self._rank = torch.distributed.get_rank()
+        self._world_size = torch.distributed.get_world_size()
+        self._local_indices = self._get_local_indices(size, self._world_size, self._rank)
+
+    @staticmethod
+    def _get_local_indices(total_size, world_size, rank):
+        shard_size = total_size // world_size
+        left = total_size % world_size
+        shard_sizes = [shard_size + int(r < left) for r in range(world_size)]
+
+        begin = sum(shard_sizes[:rank])
+        end = min(sum(shard_sizes[:rank + 1]), total_size)
+        return range(begin, end)
+
+    def __iter__(self):
+        yield from self._local_indices
+
+    def __len__(self):
+        return len(self._local_indices)
+
+
+def post_process(pred, option):
+    pred = pred.strip()
+    option_candidate = list(option.keys())
+    if len(pred) == 1:
+        return pred
+    elif len(pred) != 1 and pred[0] in option_candidate:
+        return pred[0]
+    elif len(pred) != 1 and pred[0] not in option_candidate:
+        for k, v in option.items():
+            if v in pred:
+                return k
+
+    return pred
+
+
+def evaluate_chat_model():
+    random.seed(args.seed)
+
+    for ds_name in args.datasets:
+        dataset = MMIUDataset(
+            meta=ds_collections[ds_name],
+            input_size=image_size,
+            dynamic_image_size=args.dynamic,
+            use_thumbnail=use_thumbnail,
+            max_num=args.max_num
+        )
+        dataloader = torch.utils.data.DataLoader(
+            dataset=dataset,
+            sampler=InferenceSampler(len(dataset)),
+            batch_size=args.batch_size,
+            num_workers=args.num_workers,
+            pin_memory=True,
+            drop_last=False,
+            collate_fn=partial(collate_fn, tokenizer=tokenizer),
+        )
+
+        outputs = []
+        for _, (pixel_values, questions, answers, num_patches_lists, options, lines) in tqdm(enumerate(dataloader)):
+            pixel_values = pixel_values.to(torch.bfloat16).cuda()
+            generation_config = dict(
+                num_beams=args.num_beams,
+                max_new_tokens=ds_collections[ds_name]['max_new_tokens'],
+                min_new_tokens=ds_collections[ds_name]['min_new_tokens'],
+                do_sample=True if args.temperature > 0 else False,
+                temperature=args.temperature,
+            )
+            with torch.inference_mode():
+                try:
+                    pred = model.chat(
+                        tokenizer=tokenizer,
+                        pixel_values=pixel_values,
+                        question=questions[0],
+                        generation_config=generation_config,
+                        num_patches_list=num_patches_lists[0],
+                        verbose=True
+                    )
+                except:
+                    print('Out of memory, skip this batch')
+                    pred = 'A'
+                torch.cuda.empty_cache()
+            preds = [post_process(pred, options[0])]
+
+            for question, pred, answer, line in zip(questions, preds, answers, lines):
+                outputs.append({
+                    'image': line['input']['input_image_path'],
+                    'question': question,
+                    'pred': pred,
+                    'gt': answer,
+                    'task': line['task'],
+                })
+
+        torch.distributed.barrier()
+
+        world_size = torch.distributed.get_world_size()
+        merged_outputs = [None for _ in range(world_size)]
+        torch.distributed.all_gather_object(merged_outputs, json.dumps(outputs))
+
+        merged_outputs = [json.loads(_) for _ in merged_outputs]
+        merged_outputs = [_ for _ in itertools.chain.from_iterable(merged_outputs)]
+
+        if torch.distributed.get_rank() == 0:
+            print(f'Evaluating {ds_name} ...')
+            time_prefix = time.strftime('%y%m%d%H%M%S', time.localtime())
+            results_file = f'{ds_name}_{time_prefix}.jsonl'
+            output_path = os.path.join(args.out_dir, results_file)
+            writer = open(output_path, 'w')
+
+            acc_dict = {}
+            for item in merged_outputs:
+                writer.write(json.dumps(item) + '\n')
+                task = item['task']
+                pred = item['pred']
+                gt = item['gt']
+
+                if task not in acc_dict:
+                    acc_dict[task] = []
+                if pred == gt:
+                    acc_dict[task].append(1)
+                else:
+                    acc_dict[task].append(0)
+            writer.close()
+            print('Results saved to {}'.format(output_path))
+            orders = ['point_tracking', 'ravens_progressive_matrices', 'single_object_tracking',
+                      'threed_cad_recognition', 'threed_indoor_recognition', 'Egocentric_Video_QuestionAnswering',
+                      'Homography_estimation', 'Icon_Question_Answering_with_Spatial_Context',
+                      'Image_Captioning_with_Spatial_Context', 'Image_Spatial_Transformation_Estimation',
+                      'Image_text_retrieval_with_Spatial_Context', 'Multiview_Action_Recognition',
+                      'Multiview_reasoning', 'jigsaw_puzzle_solving', 'threeD_Depth_Estimation',
+                      'threeD_Object_Detection', 'threeD_Object_Tracking', 'threeD_Pose_Estimation',
+                      'threeD_Scene_Reconstruction', 'threeD_question_answering']
+            scores = []
+            for task in orders:
+                acc = acc_dict[task]
+                num_correct = sum(acc)
+                num_total = len(acc)
+                print(f'{task} accuracy: {num_correct/num_total}')
+                scores.append(num_correct/num_total)
+            print(f'Overall accuracy: {sum(scores)/len(scores)}')
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--checkpoint', type=str, default='')
+    parser.add_argument('--datasets', type=str, default='mmiu')
+    parser.add_argument('--batch-size', type=int, default=1)
+    parser.add_argument('--num-workers', type=int, default=1)
+    parser.add_argument('--num-beams', type=int, default=1)
+    parser.add_argument('--temperature', type=float, default=0.0)
+    parser.add_argument('--out-dir', type=str, default='results')
+    parser.add_argument('--seed', type=int, default=0)
+    parser.add_argument('--dynamic', action='store_true')
+    parser.add_argument('--max-num', type=int, default=12)
+    parser.add_argument('--load-in-8bit', action='store_true')
+    parser.add_argument('--load-in-4bit', action='store_true')
+    parser.add_argument('--auto', action='store_true')
+    args = parser.parse_args()
+
+    if not os.path.exists(args.out_dir):
+        os.makedirs(args.out_dir, exist_ok=True)
+
+    args.datasets = args.datasets.split(',')
+    print('datasets:', args.datasets)
+    assert args.batch_size == 1, 'Only batch size 1 is supported'
+
+    torch.distributed.init_process_group(
+        backend='nccl',
+        world_size=int(os.getenv('WORLD_SIZE', '1')),
+        rank=int(os.getenv('RANK', '0')),
+    )
+
+    torch.cuda.set_device(int(os.getenv('LOCAL_RANK', 0)))
+
+    model, tokenizer = load_model_and_tokenizer(args)
+    image_size = model.config.force_image_size or model.config.vision_config.image_size
+    use_thumbnail = model.config.use_thumbnail
+
+    total_params = sum(p.numel() for p in model.parameters()) / 1e9
+    if total_params > 20 or args.dynamic:
+        args.num_beams = 1
+        print(f'[test] total_params: {total_params}B, use num_beams: {args.num_beams}')
+    else:
+        print(f'[test] total_params: {total_params}B')
+    print(f'[test] image_size: {image_size}')
+    print(f'[test] template: {model.config.template}')
+    print(f'[test] dynamic_image_size: {args.dynamic}')
+    print(f'[test] use_thumbnail: {use_thumbnail}')
+    print(f'[test] max_num: {args.max_num}')
+
+    evaluate_chat_model()

internvl_chat/eval/mmmu/README.md

+# README for Evaluation
+
+## 🌟 Overview
+
+This script provides an evaluation pipeline for `MMMU`.
+
+While the provided code can run the benchmark, we recommend using [VLMEvalKit](https://github.com/open-compass/VLMEvalKit) for testing this benchmark if you aim to align results with our technical report.
+The scores obtained using the code here will be approximately 2-3 points lower than those from VLMEvalKit.
+
+## 🗂️ Data Preparation
+
+Before starting to download the data, please create the `InternVL/internvl_chat/data` folder.
+
+### MMMU
+
+The evaluation script will automatically download the MMMU dataset from HuggingFace, and the cached path is `data/MMMU`.
+
+## 🏃 Evaluation Execution
+
+> ⚠️ Note: For testing InternVL (1.5, 2.0, 2.5, and later versions), always enable `--dynamic` to perform dynamic resolution testing.
+
+To run the evaluation, execute the following command on an 8-GPU setup:
+
+```shell
+torchrun --nproc_per_node=8 eval/mmmu/evaluate_mmmu.py --checkpoint ${CHECKPOINT} --dynamic
+```
+
+Alternatively, you can run the following simplified command:
+
+```shell
+GPUS=8 sh evaluate.sh ${CHECKPOINT} mmmu-val --dynamic
+```
+
+### Arguments
+
+The following arguments can be configured for the evaluation script:
+
+| Argument         | Type   | Default             | Description                                                                                                       |
+| ---------------- | ------ | ------------------- | ----------------------------------------------------------------------------------------------------------------- |
+| `--checkpoint`   | `str`  | `''`                | Path to the model checkpoint.                                                                                     |
+| `--datasets`     | `str`  | `'MMMU_validation'` | Comma-separated list of datasets to evaluate.                                                                     |
+| `--dynamic`      | `flag` | `False`             | Enables dynamic high resolution preprocessing.                                                                    |
+| `--max-num`      | `int`  | `6`                 | Maximum tile number for dynamic high resolution.                                                                  |
+| `--load-in-8bit` | `flag` | `False`             | Loads the model weights in 8-bit precision.                                                                       |
+| `--auto`         | `flag` | `False`             | Automatically splits a large model across 8 GPUs when needed, useful for models too large to fit on a single GPU. |

internvl_chat/eval/mmmu/answer_dict_val.json

+{
+    "validation_Accounting_1": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Accounting_2": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Accounting_3": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Accounting_4": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Accounting_5": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Accounting_6": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Accounting_7": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Accounting_8": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Accounting_9": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Accounting_10": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Accounting_11": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Accounting_12": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Accounting_13": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Accounting_14": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Accounting_15": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Accounting_16": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Accounting_17": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Accounting_18": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Accounting_19": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Accounting_20": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Accounting_21": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Accounting_22": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Accounting_23": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Accounting_24": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Accounting_25": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Accounting_26": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Accounting_27": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Accounting_28": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Accounting_29": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Accounting_30": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Agriculture_1": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Agriculture_2": {
+        "question_type": "multiple-choice",
+        "ground_truth": "E"
+    },
+    "validation_Agriculture_3": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Agriculture_4": {
+        "question_type": "multiple-choice",
+        "ground_truth": "E"
+    },
+    "validation_Agriculture_5": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Agriculture_6": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Agriculture_7": {
+        "question_type": "multiple-choice",
+        "ground_truth": "E"
+    },
+    "validation_Agriculture_8": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Agriculture_9": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Agriculture_10": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Agriculture_11": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Agriculture_12": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Agriculture_13": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Agriculture_14": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Agriculture_15": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Agriculture_16": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Agriculture_17": {
+        "question_type": "multiple-choice",
+        "ground_truth": "E"
+    },
+    "validation_Agriculture_18": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Agriculture_19": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Agriculture_20": {
+        "question_type": "multiple-choice",
+        "ground_truth": "E"
+    },
+    "validation_Agriculture_21": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Agriculture_22": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Agriculture_23": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Agriculture_24": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Agriculture_25": {
+        "question_type": "multiple-choice",
+        "ground_truth": "E"
+    },
+    "validation_Agriculture_26": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Agriculture_27": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Agriculture_28": {
+        "question_type": "multiple-choice",
+        "ground_truth": "E"
+    },
+    "validation_Agriculture_29": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Agriculture_30": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Architecture_and_Engineering_1": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Architecture_and_Engineering_2": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Architecture_and_Engineering_3": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Architecture_and_Engineering_4": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Architecture_and_Engineering_5": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Architecture_and_Engineering_6": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Architecture_and_Engineering_7": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Architecture_and_Engineering_8": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Architecture_and_Engineering_9": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Architecture_and_Engineering_10": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Architecture_and_Engineering_11": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Architecture_and_Engineering_12": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Architecture_and_Engineering_13": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Architecture_and_Engineering_14": {
+        "question_type": "short-answer",
+        "ground_truth": "1.06"
+    },
+    "validation_Architecture_and_Engineering_15": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Architecture_and_Engineering_16": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Architecture_and_Engineering_17": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Architecture_and_Engineering_18": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Architecture_and_Engineering_19": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Architecture_and_Engineering_20": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Architecture_and_Engineering_21": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Architecture_and_Engineering_22": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Architecture_and_Engineering_23": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Architecture_and_Engineering_24": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Architecture_and_Engineering_25": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Architecture_and_Engineering_26": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Architecture_and_Engineering_27": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Architecture_and_Engineering_28": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Architecture_and_Engineering_29": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Architecture_and_Engineering_30": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Art_1": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Art_2": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Art_3": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Art_4": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Art_5": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Art_6": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Art_7": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Art_8": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Art_9": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Art_10": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Art_11": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Art_12": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Art_13": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Art_14": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Art_15": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Art_16": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Art_17": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Art_18": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Art_19": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Art_20": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Art_21": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Art_22": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Art_23": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Art_24": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Art_25": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Art_26": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Art_27": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Art_28": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Art_29": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Art_30": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Art_Theory_1": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Art_Theory_2": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Art_Theory_3": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Art_Theory_4": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Art_Theory_5": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Art_Theory_6": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Art_Theory_7": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Art_Theory_8": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Art_Theory_9": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Art_Theory_10": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Art_Theory_11": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Art_Theory_12": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Art_Theory_13": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Art_Theory_14": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Art_Theory_15": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Art_Theory_16": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Art_Theory_17": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Art_Theory_18": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Art_Theory_19": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Art_Theory_20": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Art_Theory_21": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Art_Theory_22": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Art_Theory_23": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Art_Theory_24": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Art_Theory_25": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Art_Theory_26": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Art_Theory_27": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Art_Theory_28": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Art_Theory_29": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Art_Theory_30": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Basic_Medical_Science_1": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Basic_Medical_Science_2": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Basic_Medical_Science_3": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Basic_Medical_Science_4": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Basic_Medical_Science_5": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Basic_Medical_Science_6": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Basic_Medical_Science_7": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Basic_Medical_Science_8": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Basic_Medical_Science_9": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Basic_Medical_Science_10": {
+        "question_type": "short-answer",
+        "ground_truth": "C"
+    },
+    "validation_Basic_Medical_Science_11": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Basic_Medical_Science_12": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Basic_Medical_Science_13": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Basic_Medical_Science_14": {
+        "question_type": "short-answer",
+        "ground_truth": "Transformation"
+    },
+    "validation_Basic_Medical_Science_15": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Basic_Medical_Science_16": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Basic_Medical_Science_17": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Basic_Medical_Science_18": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Basic_Medical_Science_19": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Basic_Medical_Science_20": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Basic_Medical_Science_21": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Basic_Medical_Science_22": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Basic_Medical_Science_23": {
+        "question_type": "multiple-choice",
+        "ground_truth": "E"
+    },
+    "validation_Basic_Medical_Science_24": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Basic_Medical_Science_25": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Basic_Medical_Science_26": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Basic_Medical_Science_27": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Basic_Medical_Science_28": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Basic_Medical_Science_29": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Basic_Medical_Science_30": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Biology_1": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Biology_2": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Biology_3": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Biology_4": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Biology_5": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Biology_6": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Biology_7": {
+        "question_type": "multiple-choice",
+        "ground_truth": "I"
+    },
+    "validation_Biology_8": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Biology_9": {
+        "question_type": "multiple-choice",
+        "ground_truth": "E"
+    },
+    "validation_Biology_10": {
+        "question_type": "short-answer",
+        "ground_truth": "1/64"
+    },
+    "validation_Biology_11": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Biology_12": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Biology_13": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Biology_14": {
+        "question_type": "multiple-choice",
+        "ground_truth": "E"
+    },
+    "validation_Biology_15": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Biology_16": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Biology_17": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Biology_18": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Biology_19": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Biology_20": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Biology_21": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Biology_22": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Biology_23": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Biology_24": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Biology_25": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Biology_26": {
+        "question_type": "multiple-choice",
+        "ground_truth": "E"
+    },
+    "validation_Biology_27": {
+        "question_type": "multiple-choice",
+        "ground_truth": "E"
+    },
+    "validation_Biology_28": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Biology_29": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Biology_30": {
+        "question_type": "multiple-choice",
+        "ground_truth": "F"
+    },
+    "validation_Chemistry_1": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Chemistry_2": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Chemistry_3": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Chemistry_4": {
+        "question_type": "short-answer",
+        "ground_truth": "trans-1-Chloro-4-methylcyclohexane"
+    },
+    "validation_Chemistry_5": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Chemistry_6": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Chemistry_7": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Chemistry_8": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Chemistry_9": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Chemistry_10": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Chemistry_11": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Chemistry_12": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Chemistry_13": {
+        "question_type": "short-answer",
+        "ground_truth": "12.97"
+    },
+    "validation_Chemistry_14": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Chemistry_15": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Chemistry_16": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Chemistry_17": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Chemistry_18": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Chemistry_19": {
+        "question_type": "short-answer",
+        "ground_truth": "4"
+    },
+    "validation_Chemistry_20": {
+        "question_type": "short-answer",
+        "ground_truth": "5"
+    },
+    "validation_Chemistry_21": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Chemistry_22": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Chemistry_23": {
+        "question_type": "short-answer",
+        "ground_truth": "6.5"
+    },
+    "validation_Chemistry_24": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Chemistry_25": {
+        "question_type": "short-answer",
+        "ground_truth": "24.32"
+    },
+    "validation_Chemistry_26": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Chemistry_27": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Chemistry_28": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Chemistry_29": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Chemistry_30": {
+        "question_type": "short-answer",
+        "ground_truth": [
+            "$MgS$",
+            "MgS"
+        ]
+    },
+    "validation_Clinical_Medicine_1": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Clinical_Medicine_2": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Clinical_Medicine_3": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Clinical_Medicine_4": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Clinical_Medicine_5": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Clinical_Medicine_6": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Clinical_Medicine_7": {
+        "question_type": "multiple-choice",
+        "ground_truth": "E"
+    },
+    "validation_Clinical_Medicine_8": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Clinical_Medicine_9": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Clinical_Medicine_10": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Clinical_Medicine_11": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Clinical_Medicine_12": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Clinical_Medicine_13": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Clinical_Medicine_14": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Clinical_Medicine_15": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Clinical_Medicine_16": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Clinical_Medicine_17": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Clinical_Medicine_18": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Clinical_Medicine_19": {
+        "question_type": "multiple-choice",
+        "ground_truth": "E"
+    },
+    "validation_Clinical_Medicine_20": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Clinical_Medicine_21": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Clinical_Medicine_22": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Clinical_Medicine_23": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Clinical_Medicine_24": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Clinical_Medicine_25": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Clinical_Medicine_26": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Clinical_Medicine_27": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Clinical_Medicine_28": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Clinical_Medicine_29": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Clinical_Medicine_30": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Computer_Science_1": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Computer_Science_2": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Computer_Science_3": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Computer_Science_4": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Computer_Science_5": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Computer_Science_6": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Computer_Science_7": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Computer_Science_8": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Computer_Science_9": {
+        "question_type": "short-answer",
+        "ground_truth": "60"
+    },
+    "validation_Computer_Science_10": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Computer_Science_11": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Computer_Science_12": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Computer_Science_13": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Computer_Science_14": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Computer_Science_15": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Computer_Science_16": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Computer_Science_17": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Computer_Science_18": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Computer_Science_19": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Computer_Science_20": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Computer_Science_21": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Computer_Science_22": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Computer_Science_23": {
+        "question_type": "short-answer",
+        "ground_truth": "253.75"
+    },
+    "validation_Computer_Science_24": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Computer_Science_25": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Computer_Science_26": {
+        "question_type": "short-answer",
+        "ground_truth": "251"
+    },
+    "validation_Computer_Science_27": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Computer_Science_28": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Computer_Science_29": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Computer_Science_30": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Design_1": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Design_2": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Design_3": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Design_4": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Design_5": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Design_6": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Design_7": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Design_8": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Design_9": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Design_10": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Design_11": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Design_12": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Design_13": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Design_14": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Design_15": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Design_16": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Design_17": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Design_18": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Design_19": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Design_20": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Design_21": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Design_22": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Design_23": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Design_24": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Design_25": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Design_26": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Design_27": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Design_28": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Design_29": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Design_30": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Diagnostics_and_Laboratory_Medicine_1": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Diagnostics_and_Laboratory_Medicine_2": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Diagnostics_and_Laboratory_Medicine_3": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Diagnostics_and_Laboratory_Medicine_4": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Diagnostics_and_Laboratory_Medicine_5": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Diagnostics_and_Laboratory_Medicine_6": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Diagnostics_and_Laboratory_Medicine_7": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Diagnostics_and_Laboratory_Medicine_8": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Diagnostics_and_Laboratory_Medicine_9": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Diagnostics_and_Laboratory_Medicine_10": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Diagnostics_and_Laboratory_Medicine_11": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Diagnostics_and_Laboratory_Medicine_12": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Diagnostics_and_Laboratory_Medicine_13": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Diagnostics_and_Laboratory_Medicine_14": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Diagnostics_and_Laboratory_Medicine_15": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Diagnostics_and_Laboratory_Medicine_16": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Diagnostics_and_Laboratory_Medicine_17": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Diagnostics_and_Laboratory_Medicine_18": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Diagnostics_and_Laboratory_Medicine_19": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Diagnostics_and_Laboratory_Medicine_20": {
+        "question_type": "multiple-choice",
+        "ground_truth": "E"
+    },
+    "validation_Diagnostics_and_Laboratory_Medicine_21": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Diagnostics_and_Laboratory_Medicine_22": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Diagnostics_and_Laboratory_Medicine_23": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Diagnostics_and_Laboratory_Medicine_24": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Diagnostics_and_Laboratory_Medicine_25": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Diagnostics_and_Laboratory_Medicine_26": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Diagnostics_and_Laboratory_Medicine_27": {
+        "question_type": "multiple-choice",
+        "ground_truth": "E"
+    },
+    "validation_Diagnostics_and_Laboratory_Medicine_28": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Diagnostics_and_Laboratory_Medicine_29": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Diagnostics_and_Laboratory_Medicine_30": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Economics_1": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Economics_2": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Economics_3": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Economics_4": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Economics_5": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Economics_6": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Economics_7": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Economics_8": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Economics_9": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Economics_10": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Economics_11": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Economics_12": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Economics_13": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Economics_14": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Economics_15": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Economics_16": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Economics_17": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Economics_18": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Economics_19": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Economics_20": {
+        "question_type": "short-answer",
+        "ground_truth": "8"
+    },
+    "validation_Economics_21": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Economics_22": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Economics_23": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Economics_24": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Economics_25": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Economics_26": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Economics_27": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Economics_28": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Economics_29": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Economics_30": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Electronics_1": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Electronics_2": {
+        "question_type": "short-answer",
+        "ground_truth": "2.83"
+    },
+    "validation_Electronics_3": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Electronics_4": {
+        "question_type": "short-answer",
+        "ground_truth": "8.4"
+    },
+    "validation_Electronics_5": {
+        "question_type": "short-answer",
+        "ground_truth": "62.6"
+    },
+    "validation_Electronics_6": {
+        "question_type": "short-answer",
+        "ground_truth": "71.6"
+    },
+    "validation_Electronics_7": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Electronics_8": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Electronics_9": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Electronics_10": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Electronics_11": {
+        "question_type": "short-answer",
+        "ground_truth": "0.3"
+    },
+    "validation_Electronics_12": {
+        "question_type": "short-answer",
+        "ground_truth": "10"
+    },
+    "validation_Electronics_13": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Electronics_14": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Electronics_15": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Electronics_16": {
+        "question_type": "short-answer",
+        "ground_truth": "20"
+    },
+    "validation_Electronics_17": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Electronics_18": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Electronics_19": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Electronics_20": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Electronics_21": {
+        "question_type": "short-answer",
+        "ground_truth": "0.9965"
+    },
+    "validation_Electronics_22": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Electronics_23": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Electronics_24": {
+        "question_type": "short-answer",
+        "ground_truth": "551"
+    },
+    "validation_Electronics_25": {
+        "question_type": "short-answer",
+        "ground_truth": "50"
+    },
+    "validation_Electronics_26": {
+        "question_type": "short-answer",
+        "ground_truth": "6.333"
+    },
+    "validation_Electronics_27": {
+        "question_type": "short-answer",
+        "ground_truth": "-120"
+    },
+    "validation_Electronics_28": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Electronics_29": {
+        "question_type": "short-answer",
+        "ground_truth": "30"
+    },
+    "validation_Electronics_30": {
+        "question_type": "short-answer",
+        "ground_truth": "-141"
+    },
+    "validation_Energy_and_Power_1": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Energy_and_Power_2": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Energy_and_Power_3": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Energy_and_Power_4": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Energy_and_Power_5": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Energy_and_Power_6": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Energy_and_Power_7": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Energy_and_Power_8": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Energy_and_Power_9": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Energy_and_Power_10": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Energy_and_Power_11": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Energy_and_Power_12": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Energy_and_Power_13": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Energy_and_Power_14": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Energy_and_Power_15": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Energy_and_Power_16": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Energy_and_Power_17": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Energy_and_Power_18": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Energy_and_Power_19": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Energy_and_Power_20": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Energy_and_Power_21": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Energy_and_Power_22": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Energy_and_Power_23": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Energy_and_Power_24": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Energy_and_Power_25": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Energy_and_Power_26": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Energy_and_Power_27": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Energy_and_Power_28": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Energy_and_Power_29": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Energy_and_Power_30": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Finance_1": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Finance_2": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Finance_3": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Finance_4": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Finance_5": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Finance_6": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Finance_7": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Finance_8": {
+        "question_type": "short-answer",
+        "ground_truth": "360"
+    },
+    "validation_Finance_9": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Finance_10": {
+        "question_type": "short-answer",
+        "ground_truth": "2000000"
+    },
+    "validation_Finance_11": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Finance_12": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Finance_13": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Finance_14": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Finance_15": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Finance_16": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Finance_17": {
+        "question_type": "short-answer",
+        "ground_truth": "1000"
+    },
+    "validation_Finance_18": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Finance_19": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Finance_20": {
+        "question_type": "short-answer",
+        "ground_truth": "7243000"
+    },
+    "validation_Finance_21": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Finance_22": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Finance_23": {
+        "question_type": "short-answer",
+        "ground_truth": "527.89"
+    },
+    "validation_Finance_24": {
+        "question_type": "short-answer",
+        "ground_truth": "30.0"
+    },
+    "validation_Finance_25": {
+        "question_type": "short-answer",
+        "ground_truth": "0.286"
+    },
+    "validation_Finance_26": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Finance_27": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Finance_28": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Finance_29": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Finance_30": {
+        "question_type": "short-answer",
+        "ground_truth": "1249"
+    },
+    "validation_Geography_1": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Geography_2": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Geography_3": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Geography_4": {
+        "question_type": "short-answer",
+        "ground_truth": [
+            "Tampa",
+            "Florida"
+        ]
+    },
+    "validation_Geography_5": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Geography_6": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Geography_7": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Geography_8": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Geography_9": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Geography_10": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Geography_11": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Geography_12": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Geography_13": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Geography_14": {
+        "question_type": "short-answer",
+        "ground_truth": "8200"
+    },
+    "validation_Geography_15": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Geography_16": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Geography_17": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Geography_18": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Geography_19": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Geography_20": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Geography_21": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Geography_22": {
+        "question_type": "short-answer",
+        "ground_truth": "3"
+    },
+    "validation_Geography_23": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Geography_24": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Geography_25": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Geography_26": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Geography_27": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Geography_28": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Geography_29": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Geography_30": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_History_1": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_History_2": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_History_3": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_History_4": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_History_5": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_History_6": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_History_7": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_History_8": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_History_9": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_History_10": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_History_11": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_History_12": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_History_13": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_History_14": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_History_15": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_History_16": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_History_17": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_History_18": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_History_19": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_History_20": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_History_21": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_History_22": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_History_23": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_History_24": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_History_25": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_History_26": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_History_27": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_History_28": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_History_29": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_History_30": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Literature_1": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Literature_2": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Literature_3": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Literature_4": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Literature_5": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Literature_6": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Literature_7": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Literature_8": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Literature_9": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Literature_10": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Literature_11": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Literature_12": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Literature_13": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Literature_14": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Literature_15": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Literature_16": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Literature_17": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Literature_18": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Literature_19": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Literature_20": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Literature_21": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Literature_22": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Literature_23": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Literature_24": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Literature_25": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Literature_26": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Literature_27": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Literature_28": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Literature_29": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Literature_30": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Manage_1": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Manage_2": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Manage_3": {
+        "question_type": "short-answer",
+        "ground_truth": "242110.62"
+    },
+    "validation_Manage_4": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Manage_5": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Manage_6": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Manage_7": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Manage_8": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Manage_9": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Manage_10": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Manage_11": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Manage_12": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Manage_13": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Manage_14": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Manage_15": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Manage_16": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Manage_17": {
+        "question_type": "short-answer",
+        "ground_truth": "65"
+    },
+    "validation_Manage_18": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Manage_19": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Manage_20": {
+        "question_type": "short-answer",
+        "ground_truth": "2960"
+    },
+    "validation_Manage_21": {
+        "question_type": "short-answer",
+        "ground_truth": "9.20"
+    },
+    "validation_Manage_22": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Manage_23": {
+        "question_type": "short-answer",
+        "ground_truth": "1464"
+    },
+    "validation_Manage_24": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Manage_25": {
+        "question_type": "short-answer",
+        "ground_truth": "0.963"
+    },
+    "validation_Manage_26": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Manage_27": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Manage_28": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Manage_29": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Manage_30": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Marketing_1": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Marketing_2": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Marketing_3": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Marketing_4": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Marketing_5": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Marketing_6": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Marketing_7": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Marketing_8": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Marketing_9": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Marketing_10": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Marketing_11": {
+        "question_type": "short-answer",
+        "ground_truth": "0.10"
+    },
+    "validation_Marketing_12": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Marketing_13": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Marketing_14": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Marketing_15": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Marketing_16": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Marketing_17": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Marketing_18": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Marketing_19": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Marketing_20": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Marketing_21": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Marketing_22": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Marketing_23": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Marketing_24": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Marketing_25": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Marketing_26": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Marketing_27": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Marketing_28": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Marketing_29": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Marketing_30": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Materials_1": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Materials_2": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Materials_3": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Materials_4": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Materials_5": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Materials_6": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Materials_7": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Materials_8": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Materials_9": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Materials_10": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Materials_11": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Materials_12": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Materials_13": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Materials_14": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Materials_15": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Materials_16": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Materials_17": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Materials_18": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Materials_19": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Materials_20": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Materials_21": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Materials_22": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Materials_23": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Materials_24": {
+        "question_type": "multiple-choice",
+        "ground_truth": "F"
+    },
+    "validation_Materials_25": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Materials_26": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Materials_27": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Materials_28": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Materials_29": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Materials_30": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Math_1": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Math_2": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Math_3": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Math_4": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Math_5": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Math_6": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Math_7": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Math_8": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Math_9": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Math_10": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Math_11": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Math_12": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Math_13": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Math_14": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Math_15": {
+        "question_type": "short-answer",
+        "ground_truth": [
+            "24/7",
+            "3.429"
+        ]
+    },
+    "validation_Math_16": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Math_17": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Math_18": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Math_19": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Math_20": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Math_21": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Math_22": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Math_23": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Math_24": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Math_25": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Math_26": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Math_27": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Math_28": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Math_29": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Math_30": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Mechanical_Engineering_1": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Mechanical_Engineering_2": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Mechanical_Engineering_3": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Mechanical_Engineering_4": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Mechanical_Engineering_5": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Mechanical_Engineering_6": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Mechanical_Engineering_7": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Mechanical_Engineering_8": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Mechanical_Engineering_9": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Mechanical_Engineering_10": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Mechanical_Engineering_11": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Mechanical_Engineering_12": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Mechanical_Engineering_13": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Mechanical_Engineering_14": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Mechanical_Engineering_15": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Mechanical_Engineering_16": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Mechanical_Engineering_17": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Mechanical_Engineering_18": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Mechanical_Engineering_19": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Mechanical_Engineering_20": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Mechanical_Engineering_21": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Mechanical_Engineering_22": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Mechanical_Engineering_23": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Mechanical_Engineering_24": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Mechanical_Engineering_25": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Mechanical_Engineering_26": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Mechanical_Engineering_27": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Mechanical_Engineering_28": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Mechanical_Engineering_29": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Mechanical_Engineering_30": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Music_1": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Music_2": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Music_3": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Music_4": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Music_5": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Music_6": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Music_7": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Music_8": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Music_9": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Music_10": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Music_11": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Music_12": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Music_13": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Music_14": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Music_15": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Music_16": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Music_17": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Music_18": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Music_19": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Music_20": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Music_21": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Music_22": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Music_23": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Music_24": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Music_25": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Music_26": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Music_27": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Music_28": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Music_29": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Music_30": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Pharmacy_1": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Pharmacy_2": {
+        "question_type": "multiple-choice",
+        "ground_truth": "E"
+    },
+    "validation_Pharmacy_3": {
+        "question_type": "short-answer",
+        "ground_truth": "2"
+    },
+    "validation_Pharmacy_4": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Pharmacy_5": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Pharmacy_6": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Pharmacy_7": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Pharmacy_8": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Pharmacy_9": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Pharmacy_10": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Pharmacy_11": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Pharmacy_12": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Pharmacy_13": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Pharmacy_14": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Pharmacy_15": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Pharmacy_16": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Pharmacy_17": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Pharmacy_18": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Pharmacy_19": {
+        "question_type": "short-answer",
+        "ground_truth": "B"
+    },
+    "validation_Pharmacy_20": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Pharmacy_21": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Pharmacy_22": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Pharmacy_23": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Pharmacy_24": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Pharmacy_25": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Pharmacy_26": {
+        "question_type": "short-answer",
+        "ground_truth": "A"
+    },
+    "validation_Pharmacy_27": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Pharmacy_28": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Pharmacy_29": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Pharmacy_30": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Physics_1": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Physics_2": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Physics_3": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Physics_4": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Physics_5": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Physics_6": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Physics_7": {
+        "question_type": "multiple-choice",
+        "ground_truth": "E"
+    },
+    "validation_Physics_8": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Physics_9": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Physics_10": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Physics_11": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Physics_12": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Physics_13": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Physics_14": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Physics_15": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Physics_16": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Physics_17": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Physics_18": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Physics_19": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Physics_20": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Physics_21": {
+        "question_type": "short-answer",
+        "ground_truth": "A"
+    },
+    "validation_Physics_22": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Physics_23": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Physics_24": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Physics_25": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Physics_26": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Physics_27": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Physics_28": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Physics_29": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Physics_30": {
+        "question_type": "multiple-choice",
+        "ground_truth": "E"
+    },
+    "validation_Psychology_1": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Psychology_2": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Psychology_3": {
+        "question_type": "multiple-choice",
+        "ground_truth": "E"
+    },
+    "validation_Psychology_4": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Psychology_5": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Psychology_6": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Psychology_7": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Psychology_8": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Psychology_9": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Psychology_10": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Psychology_11": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Psychology_12": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Psychology_13": {
+        "question_type": "multiple-choice",
+        "ground_truth": "E"
+    },
+    "validation_Psychology_14": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Psychology_15": {
+        "question_type": "multiple-choice",
+        "ground_truth": "E"
+    },
+    "validation_Psychology_16": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Psychology_17": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Psychology_18": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Psychology_19": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Psychology_20": {
+        "question_type": "short-answer",
+        "ground_truth": "embryonic"
+    },
+    "validation_Psychology_21": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Psychology_22": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Psychology_23": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Psychology_24": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Psychology_25": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Psychology_26": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Psychology_27": {
+        "question_type": "multiple-choice",
+        "ground_truth": "E"
+    },
+    "validation_Psychology_28": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Psychology_29": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Psychology_30": {
+        "question_type": "multiple-choice",
+        "ground_truth": "E"
+    },
+    "validation_Public_Health_1": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Public_Health_2": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Public_Health_3": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Public_Health_4": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Public_Health_5": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Public_Health_6": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Public_Health_7": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Public_Health_8": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Public_Health_9": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Public_Health_10": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Public_Health_11": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Public_Health_12": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Public_Health_13": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Public_Health_14": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Public_Health_15": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Public_Health_16": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Public_Health_17": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Public_Health_18": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Public_Health_19": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Public_Health_20": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Public_Health_21": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Public_Health_22": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Public_Health_23": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Public_Health_24": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Public_Health_25": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Public_Health_26": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Public_Health_27": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Public_Health_28": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Public_Health_29": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Public_Health_30": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Sociology_1": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Sociology_2": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Sociology_3": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Sociology_4": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Sociology_5": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Sociology_6": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Sociology_7": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Sociology_8": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Sociology_9": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Sociology_10": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Sociology_11": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Sociology_12": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Sociology_13": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Sociology_14": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Sociology_15": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Sociology_16": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Sociology_17": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Sociology_18": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Sociology_19": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Sociology_20": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Sociology_21": {
+        "question_type": "multiple-choice",
+        "ground_truth": "C"
+    },
+    "validation_Sociology_22": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Sociology_23": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Sociology_24": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Sociology_25": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Sociology_26": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Sociology_27": {
+        "question_type": "multiple-choice",
+        "ground_truth": "D"
+    },
+    "validation_Sociology_28": {
+        "question_type": "multiple-choice",
+        "ground_truth": "A"
+    },
+    "validation_Sociology_29": {
+        "question_type": "multiple-choice",
+        "ground_truth": "B"
+    },
+    "validation_Sociology_30": {
+        "question_type": "multiple-choice",
+        "ground_truth": "E"
+    }
+}

internvl_chat/eval/mmmu/data_utils.py

+"""Utils for data load, save, and process (e.g., prompt construction)"""
+
+import json
+import os
+import re
+
+import yaml
+
+DOMAIN_CAT2SUB_CAT = {
+    'Art and Design': ['Art', 'Art_Theory', 'Design', 'Music'],
+    'Business': ['Accounting', 'Economics', 'Finance', 'Manage', 'Marketing'],
+    'Science': ['Biology', 'Chemistry', 'Geography', 'Math', 'Physics', ],
+    'Health and Medicine': ['Basic_Medical_Science', 'Clinical_Medicine', 'Diagnostics_and_Laboratory_Medicine',
+                            'Pharmacy', 'Public_Health'],
+    'Humanities and Social Science': ['History', 'Literature', 'Sociology', 'Psychology'],
+    'Tech and Engineering': ['Agriculture', 'Architecture_and_Engineering', 'Computer_Science', 'Electronics',
+                             'Energy_and_Power', 'Materials', 'Mechanical_Engineering'],
+}
+
+CAT_SHORT2LONG = {
+    'acc': 'Accounting',
+    'agri': 'Agriculture',
+    'arch': 'Architecture_and_Engineering',
+    'art': 'Art',
+    'art_theory': 'Art_Theory',
+    'bas_med': 'Basic_Medical_Science',
+    'bio': 'Biology',
+    'chem': 'Chemistry',
+    'cli_med': 'Clinical_Medicine',
+    'cs': 'Computer_Science',
+    'design': 'Design',
+    'diag_med': 'Diagnostics_and_Laboratory_Medicine',
+    'econ': 'Economics',
+    'elec': 'Electronics',
+    'ep': 'Energy_and_Power',
+    'fin': 'Finance',
+    'geo': 'Geography',
+    'his': 'History',
+    'liter': 'Literature',
+    'manage': 'Manage',
+    'mark': 'Marketing',
+    'mate': 'Materials',
+    'math': 'Math',
+    'mech': 'Mechanical_Engineering',
+    'music': 'Music',
+    'phar': 'Pharmacy',
+    'phys': 'Physics',
+    'psy': 'Psychology',
+    'pub_health': 'Public_Health',
+    'socio': 'Sociology'
+}
+
+
+# DATA SAVING
+def save_json(filename, ds):
+    with open(filename, 'w') as f:
+        json.dump(ds, f, indent=4)
+
+
+def get_multi_choice_info(options):
+    """
+    Given the list of options for multiple choice question
+    Return the index2ans and all_choices
+    """
+
+    start_chr = 'A'
+    all_choices = []
+    index2ans = {}
+    for i, option in enumerate(options):
+        index2ans[chr(ord(start_chr) + i)] = option
+        all_choices.append(chr(ord(start_chr) + i))
+
+    return index2ans, all_choices
+
+
+def load_yaml(file_path):
+    with open(file_path, 'r') as stream:
+        try:
+            yaml_dict = yaml.safe_load(stream)
+        except yaml.YAMLError as exc:
+            print(exc)
+
+    return yaml_dict
+
+
+def parse_img_path(text):
+    matches = re.findall("<img='(.*?)'>", text)
+    return matches
+
+
+def process_single_sample(data):
+    question = data['question']
+    o_imgs_paths = []
+    for option in data['options']:
+        current_o_imgs_paths = parse_img_path(option)
+        for img_path in current_o_imgs_paths:
+            o_imgs_paths.append(img_path)
+    images = [data['image_1'], data['image_2'], data['image_3'], data['image_4'],
+              data['image_5'], data['image_6'], data['image_7']]
+    return {'id': data['id'], 'question': question, 'options': data['options'], 'answer': data['answer'],
+            'image': images, 'question_type': data['question_type']}
+
+
+# DATA SAVING
+def save_json(filename, ds):
+    with open(filename, 'w') as f:
+        json.dump(ds, f, indent=4)
+
+
+def save_jsonl(filename, data):
+    """
+    Save a dictionary of data to a JSON Lines file with the filename as key and caption as value.
+
+    Args:
+        filename (str): The path to the file where the data should be saved.
+        data (dict): The dictionary containing the data to save where key is the image path and value is the caption.
+    """
+    with open(filename, 'w', encoding='utf-8') as f:
+        for img_path, caption in data.items():
+            # Extract the base filename without the extension
+            base_filename = os.path.basename(img_path)
+            # Create a JSON object with the filename as the key and caption as the value
+            json_record = json.dumps({base_filename: caption}, ensure_ascii=False)
+            # Write the JSON object to the file, one per line
+            f.write(json_record + '\n')
+
+
+def save_args(args, path_dir):
+    argsDict = args.__dict__
+    with open(path_dir + 'setting.txt', 'w') as f:
+        f.writelines('------------------ start ------------------' + '\n')
+        for eachArg, value in argsDict.items():
+            f.writelines(eachArg + ' : ' + str(value) + '\n')
+        f.writelines('------------------- end -------------------')
+
+
+# DATA PROCESSING
+def construct_prompt(sample, config):
+    question = sample['question']
+    options = eval(sample['options'])
+    example = ''
+    if sample['question_type'] == 'multiple-choice':
+        start_chr = 'A'
+        prediction_range = []
+        index2ans = {}
+        for option in options:
+            prediction_range.append(start_chr)
+            example += f'({start_chr}) {option}\n'
+            index2ans[start_chr] = option
+            start_chr = chr(ord(start_chr) + 1)
+        empty_prompt_sample_structure = config['multi_choice_example_format']
+        empty_prompt = empty_prompt_sample_structure.format(question, example)
+        res_dict = {}
+        res_dict['index2ans'] = index2ans
+        res_dict['correct_choice'] = sample['answer']
+        res_dict['all_choices'] = prediction_range
+        res_dict['empty_prompt'] = empty_prompt
+        if config['task_instructions']:
+            res_dict['final_input_prompt'] = config['task_instructions'].strip() + '\n\n' + empty_prompt
+        else:
+            res_dict['final_input_prompt'] = empty_prompt
+
+        res_dict['gt_content'] = options[ord(sample['answer'].upper()) - ord('A')]
+    else:
+        empty_prompt_sample_structure = config['short_ans_example_format']
+        empty_prompt = empty_prompt_sample_structure.format(question)
+        res_dict = {}
+        res_dict['empty_prompt'] = empty_prompt
+        if config['task_instructions']:
+            res_dict['final_input_prompt'] = config['task_instructions'].strip() + '\n\n' + empty_prompt
+        else:
+            res_dict['final_input_prompt'] = empty_prompt
+        res_dict['gt_content'] = sample['answer']
+
+    res_dict.update(sample)
+    return res_dict

internvl_chat/eval/mmmu/eval_utils.py

+"""Response Parsing and Evaluation for various models"""
+import random
+import re
+from typing import Dict
+
+random.seed(42)
+import numpy as np
+
+
+# ----------- Process Multi-choice -------------
+def parse_multi_choice_response(response, all_choices, index2ans):
+    """
+    Parse the prediction from the generated response.
+    Return the predicted index e.g., A, B, C, D.
+    """
+    for char in [',', '.', '!', '?', ';', ':', "'"]:
+        response = response.strip(char)
+    response = ' ' + response + ' '  # add space to avoid partial match
+
+    index_ans = True
+    ans_with_brack = False
+    candidates = []
+    for choice in all_choices:  # e.g., (A) (B) (C) (D)
+        if f'({choice})' in response:
+            candidates.append(choice)
+            ans_with_brack = True
+
+    if len(candidates) == 0:
+        for choice in all_choices:  # e.g., A B C D
+            if f' {choice} ' in response:
+                candidates.append(choice)
+
+    # if all above doesn't get candidates, check if the content is larger than 5 tokens and try to parse the example
+    if len(candidates) == 0 and len(response.split()) > 5:
+        for index, ans in index2ans.items():
+            if ans.lower() in response.lower():
+                candidates.append(index)
+                index_ans = False  # it's content ans.
+
+    if len(candidates) == 0:  # still not get answer, randomly choose one.
+        pred_index = random.choice(all_choices)
+    elif len(candidates) > 1:
+        start_indexes = []
+        if index_ans:
+            if ans_with_brack:
+                for can in candidates:
+                    index = response.rfind(f'({can})')
+                    start_indexes.append(index)  # -1 will be ignored anyway
+                # start_indexes = [generated_response.index(f'({can})') for can in candidates]
+            else:
+                for can in candidates:
+                    index = response.rfind(f' {can} ')
+                    start_indexes.append(index)
+        else:
+            for can in candidates:
+                index = response.lower().rfind(index2ans[can].lower())
+                start_indexes.append(index)
+        # get the last one
+        pred_index = candidates[np.argmax(start_indexes)]
+    else:  # if only one candidate, use it.
+        pred_index = candidates[0]
+
+    return pred_index
+
+
+# ----------- Process Open -------------
+def check_is_number(string):
+    """
+    Check if the given string a number.
+    """
+    try:
+        float(string.replace(',', ''))
+        return True
+    except ValueError:
+        # check if there's comma inside
+        return False
+
+
+def normalize_str(string):
+    """
+    Normalize the str to lower case and make them float numbers if possible.
+    """
+    # check if characters in the string
+
+    # if number, numerize it.
+    string = string.strip()
+
+    is_number = check_is_number(string)
+
+    if is_number:
+        string = string.replace(',', '')
+        string = float(string)
+        # leave 2 decimal
+        string = round(string, 2)
+        return [string]
+    else:  # it's likely to be a string
+        # lower it
+        string = string.lower()
+        if len(string) == 1:
+            return [' ' + string, string + ' ']  # avoid trivial matches
+        return [string]
+
+
+def extract_numbers(string):
+    """
+    Exact all forms of numbers from a string with regex.
+    """
+    # Pattern for numbers with commas
+    pattern_commas = r'-?\b\d{1,3}(?:,\d{3})+\b'
+    # Pattern for scientific notation
+    pattern_scientific = r'-?\d+(?:\.\d+)?[eE][+-]?\d+'
+    # Pattern for simple numbers without commas
+    pattern_simple = r'-?(?:\d+\.\d+|\.\d+|\d+\b)(?![eE][+-]?\d+)(?![,\d])'
+
+    # Extract numbers with commas
+    numbers_with_commas = re.findall(pattern_commas, string)
+    # Extract numbers in scientific notation
+    numbers_scientific = re.findall(pattern_scientific, string)
+    # Extract simple numbers without commas
+    numbers_simple = re.findall(pattern_simple, string)
+
+    # Combine all extracted numbers
+    all_numbers = numbers_with_commas + numbers_scientific + numbers_simple
+    return all_numbers
+
+
+def parse_open_response(response):
+    """
+    Parse the prediction from the generated response.
+    Return a list of predicted strings or numbers.
+    """
+
+    # content = content.strip("\n").strip(".").strip(" ")
+    def get_key_subresponses(response):
+        key_responses = []
+        response = response.strip().strip('.').lower()
+        sub_responses = re.split(r'\.\s(?=[A-Z])|\n', response)
+        indicators_of_keys = ['could be ', 'so ', 'is ',
+                              'thus ', 'therefore ', 'final ', 'answer ', 'result ']
+        key_responses = []
+        for index, resp in enumerate(sub_responses):
+            # if last one, accept it's an equation (the entire response can be just one sentence with equation)
+            if index == len(sub_responses) - 1:
+                indicators_of_keys.extend(['='])
+            shortest_key_response = None  # the shortest response that may contain the answer (tail part of the response)
+            for indicator in indicators_of_keys:
+                if indicator in resp:
+                    if not shortest_key_response:
+                        shortest_key_response = resp.split(indicator)[-1].strip()
+                    else:
+                        if len(resp.split(indicator)[-1].strip()) < len(shortest_key_response):
+                            shortest_key_response = resp.split(indicator)[-1].strip()
+                    # key_responses.append(resp.split(indicator)[1].strip())
+
+            if shortest_key_response:
+                # and it's not trivial
+                if shortest_key_response.strip() not in [':', ',', '.', '!', '?', ';', ':', "'"]:
+                    key_responses.append(shortest_key_response)
+        if len(key_responses) == 0:  # did not found any
+            return [response]
+        return key_responses
+
+    # pdb.set_trace()
+    key_responses = get_key_subresponses(response)
+
+    pred_list = key_responses.copy()  # keep the original string response
+    for resp in key_responses:
+        pred_list.extend(extract_numbers(resp))
+
+    tmp_pred_list = []
+    for i in range(len(pred_list)):
+        tmp_pred_list.extend(normalize_str(pred_list[i]))
+    pred_list = tmp_pred_list
+
+    # remove duplicates
+    pred_list = list(set(pred_list))
+
+    return pred_list
+
+
+# ----------- Evaluation -------------
+
+def eval_multi_choice(gold_i, pred_i):
+    """
+    Evaluate a multiple choice instance.
+    """
+    correct = False
+    # only they are exactly the same, we consider it as correct
+    if isinstance(gold_i, list):
+        for answer in gold_i:
+            if answer == pred_i:
+                correct = True
+                break
+    else:  # gold_i is a string
+        if gold_i == pred_i:
+            correct = True
+    return correct
+
+
+def eval_open(gold_i, pred_i):
+    """
+    Evaluate an open question instance
+    """
+    correct = False
+    if isinstance(gold_i, list):
+        # use float to avoid trivial matches
+        norm_answers = []
+        for answer in gold_i:
+            norm_answers.extend(normalize_str(answer))
+    else:
+        norm_answers = normalize_str(gold_i)
+    for pred in pred_i:  # pred is already normalized in parse response phase
+        if isinstance(pred, str):  # if it's a string, then find if ans in the pred_i
+            for norm_ans in norm_answers:
+                # only see if the string answer in the string pred
+                if isinstance(norm_ans, str) and norm_ans in pred:
+                    if not correct:
+                        correct = True
+                    break
+        else:  # it's a float number
+            if pred in norm_answers:
+                if not correct:
+                    correct = True
+                break
+    return correct
+
+
+# ----------- Batch Evaluation -------------
+def evaluate(samples):
+    """
+    Batch evaluation for multiple choice and open questions.
+    """
+    pred_correct = 0
+    judge_dict = dict()
+    for sample in samples:
+        gold_i = sample['answer']
+        pred_i = sample['parsed_pred']
+        if sample['question_type'] == 'multiple-choice':
+            correct = eval_multi_choice(gold_i, pred_i)
+        else:  # open question
+            correct = eval_open(gold_i, pred_i)
+
+        if correct:
+            judge_dict[sample['id']] = 'Correct'
+            pred_correct += 1
+        else:
+            judge_dict[sample['id']] = 'Wrong'
+
+    if len(samples) == 0:
+        return {'acc': 0}
+    return judge_dict, {'acc': pred_correct / len(samples)}
+
+
+# ----------- Calculate Accuracy -------------
+def calculate_ins_level_acc(results: Dict):
+    """Calculate the instruction level accuracy for given Subject results"""
+    acc = 0
+    ins_num = 0
+    for cat_results in results.values():
+        acc += cat_results['acc'] * cat_results['num_example']
+        ins_num += cat_results['num_example']
+    if ins_num == 0:
+        return 0
+    return acc / ins_num

internvl_chat/eval/mmmu/evaluate_mmmu.py

+import argparse
+import itertools
+import json
+import os
+import random
+import time
+from functools import partial
+
+import torch
+from data_utils import CAT_SHORT2LONG, process_single_sample
+from datasets import concatenate_datasets, load_dataset
+from internvl.model import load_model_and_tokenizer
+from internvl.train.dataset import build_transform, dynamic_preprocess
+from PIL import Image
+from tqdm import tqdm
+
+ds_collections = {
+    'MMMU_validation': {
+        'root': 'MMMU/MMMU',
+        'max_new_tokens': 10,
+        'min_new_tokens': 1,
+        'split': 'validation'
+    },
+    'MMMU_test': {
+        'root': 'MMMU/MMMU',
+        'max_new_tokens': 10,
+        'min_new_tokens': 1,
+        'split': 'test'
+    },
+    'MMMU_dev': {
+        'root': 'MMMU/MMMU',
+        'max_new_tokens': 10,
+        'min_new_tokens': 1,
+        'split': 'dev'
+    },
+}
+
+
+def collate_fn(batches, tokenizer):
+    pixel_values = torch.cat([_['pixel_values'] for _ in batches], dim=0)
+    questions = [_['question'] for _ in batches]
+    answers = [_['answer'] for _ in batches]
+    data_ids = [_['data_id'] for _ in batches]
+    options = [_['option'] for _ in batches]
+    return pixel_values, questions, answers, data_ids, options
+
+
+class MMMUDataset(torch.utils.data.Dataset):
+
+    def __init__(self, root, split, prompt, input_size=224, dynamic_image_size=False,
+                 use_thumbnail=False, max_num=6):
+        # run for each subject
+        sub_dataset_list = []
+        for subject in tqdm(CAT_SHORT2LONG.values()):
+            sub_dataset = load_dataset(root, subject, split=split, cache_dir=os.path.join(os.getcwd(), 'data/MMMU/'))
+            sub_dataset_list.append(sub_dataset)
+
+        # merge all dataset
+        self.data = concatenate_datasets(sub_dataset_list)
+        self.prompt = prompt
+        self.input_size = input_size
+        self.dynamic_image_size = dynamic_image_size
+        self.use_thumbnail = use_thumbnail
+        self.max_num = max_num
+        self.transform = build_transform(is_train=False, input_size=input_size)
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, idx):
+
+        data = process_single_sample(self.data[idx])
+        data_id = data['id']
+        question = data['question'].strip()
+        pil_images = data['image']
+        question_type = data['question_type']
+
+        choices = eval(data['options'])
+        answer = data['answer'] if 'answer' in data else None
+
+        choice_list = []
+        options = {}
+        multiple_choices = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M']
+        for i, c in enumerate(choices):
+            choice_list.append('{}. {}'.format(multiple_choices[i], c.strip()))
+            options[multiple_choices[i]] = c.strip()
+        choice_txt = '\n'.join(choice_list)
+        if self.dynamic_image_size:
+            images = []
+            for idx, pil_image in enumerate(pil_images):
+                if pil_image is not None:
+                    if idx == 0:
+                        pil_image = pil_image.resize((pil_image.width * 2, pil_image.height * 2), Image.BILINEAR)
+                        pil_image = dynamic_preprocess(pil_image, image_size=self.input_size,
+                                                       use_thumbnail=self.use_thumbnail, max_num=self.max_num)
+                    else:
+                        pil_image = dynamic_preprocess(pil_image, image_size=self.input_size,
+                                                       use_thumbnail=self.use_thumbnail, max_num=1)
+                    images += pil_image
+        else:
+            images = [pil_images[0]]
+        pixel_values = [self.transform(image) for image in images]
+        pixel_values = torch.stack(pixel_values)
+
+        if len(choice_txt) > 0:
+            question += '\n' + choice_txt
+        question += '\n' + self.prompt[question_type]
+        question = question.strip()
+
+        return {
+            'question': question,
+            'pixel_values': pixel_values,
+            'answer': answer,
+            'option': options,
+            'data_id': data_id
+        }
+
+
+class InferenceSampler(torch.utils.data.sampler.Sampler):
+
+    def __init__(self, size):
+        self._size = int(size)
+        assert size > 0
+        self._rank = torch.distributed.get_rank()
+        self._world_size = torch.distributed.get_world_size()
+        self._local_indices = self._get_local_indices(size, self._world_size, self._rank)
+
+    @staticmethod
+    def _get_local_indices(total_size, world_size, rank):
+        shard_size = total_size // world_size
+        left = total_size % world_size
+        shard_sizes = [shard_size + int(r < left) for r in range(world_size)]
+
+        begin = sum(shard_sizes[:rank])
+        end = min(sum(shard_sizes[:rank + 1]), total_size)
+        return range(begin, end)
+
+    def __iter__(self):
+        yield from self._local_indices
+
+    def __len__(self):
+        return len(self._local_indices)
+
+
+def post_process(pred, option):
+    pred = pred.strip()
+    option_candidate = list(option.keys())
+    if len(pred) == 1:
+        return pred
+    elif len(pred) != 1 and pred[0] in option_candidate:
+        return pred[0]
+    elif len(pred) != 1 and pred[0] not in option_candidate:
+        for k, v in option.items():
+            if v in pred:
+                return k
+
+    return pred
+
+
+def evaluate_chat_model():
+    prompt = {
+        'multiple-choice': "Answer with the option's letter from the given choices directly.",
+        'open': 'Answer the question using a single word or phrase.'
+    }
+    random.seed(args.seed)
+
+    for ds_name in args.datasets:
+        dataset = MMMUDataset(
+            root=ds_collections[ds_name]['root'],
+            split=ds_collections[ds_name]['split'],
+            prompt=prompt,
+            input_size=image_size,
+            dynamic_image_size=args.dynamic,
+            use_thumbnail=use_thumbnail,
+            max_num=args.max_num
+        )
+        dataloader = torch.utils.data.DataLoader(
+            dataset=dataset,
+            sampler=InferenceSampler(len(dataset)),
+            batch_size=args.batch_size,
+            num_workers=args.num_workers,
+            pin_memory=True,
+            drop_last=False,
+            collate_fn=partial(collate_fn, tokenizer=tokenizer),
+        )
+
+        outputs = []
+        for _, (pixel_values, questions, answers, data_ids, options) in tqdm(enumerate(dataloader)):
+            pixel_values = pixel_values.to(torch.bfloat16).cuda()
+            generation_config = dict(
+                num_beams=args.num_beams,
+                max_new_tokens=ds_collections[ds_name]['max_new_tokens'],
+                min_new_tokens=ds_collections[ds_name]['min_new_tokens'],
+                do_sample=True if args.temperature > 0 else False,
+                temperature=args.temperature,
+            )
+            pred = model.chat(
+                tokenizer=tokenizer,
+                pixel_values=pixel_values,
+                question=questions[0],
+                generation_config=generation_config,
+                verbose=True
+            )
+            if len(options[0]) == 0:
+                preds = [pred]
+            else:
+                preds = [post_process(pred, options[0])]
+
+            for question, pred, answer, data_id in zip(questions, preds, answers, data_ids):
+                outputs.append({
+                    'question': question,
+                    'answer': pred,
+                    'gt_answers': answer,
+                    'data_id': data_id
+                })
+
+        torch.distributed.barrier()
+
+        world_size = torch.distributed.get_world_size()
+        merged_outputs = [None for _ in range(world_size)]
+        torch.distributed.all_gather_object(merged_outputs, json.dumps(outputs))
+
+        merged_outputs = [json.loads(_) for _ in merged_outputs]
+        merged_outputs = [_ for _ in itertools.chain.from_iterable(merged_outputs)]
+
+        if torch.distributed.get_rank() == 0:
+            print(f'Evaluating {ds_name} ...')
+            time_prefix = time.strftime('%y%m%d%H%M%S', time.localtime())
+            results_file = f'{ds_name}_{time_prefix}.json'
+            output_path = os.path.join(args.out_dir, results_file)
+            outputs = {}
+            for item in merged_outputs:
+                outputs[item['data_id']] = item['answer']
+            with open(output_path, 'w') as f:
+                json.dump(outputs, f, indent=4)
+            print('Results saved to {}'.format(output_path))
+            if ds_collections[ds_name]['split'] == 'validation':
+                print('Evaluating ...')
+                cmd = f'python eval/mmmu/main_eval_only.py ' \
+                      f'--output_path {output_path} ' \
+                      f'--answer_path eval/mmmu/answer_dict_val.json'
+                print(cmd)
+                os.system(cmd)
+            time_prefix = time.strftime('%y%m%d%H%M%S', time.localtime())
+            results_file = f'{ds_name}_{time_prefix}.jsonl'
+            output_path = os.path.join(args.out_dir, results_file)
+            writer = open(output_path, 'w')
+            for item in merged_outputs:
+                writer.write(json.dumps(item) + '\n')
+            writer.close()
+            print('Results saved to {}'.format(output_path))
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--checkpoint', type=str, default='')
+    parser.add_argument('--datasets', type=str, default='MMMU_validation')
+    parser.add_argument('--batch-size', type=int, default=1)
+    parser.add_argument('--num-workers', type=int, default=1)
+    parser.add_argument('--num-beams', type=int, default=1)
+    parser.add_argument('--temperature', type=float, default=0.0)
+    parser.add_argument('--out-dir', type=str, default='results')
+    parser.add_argument('--seed', type=int, default=0)
+    parser.add_argument('--dynamic', action='store_true')
+    parser.add_argument('--max-num', type=int, default=6)
+    parser.add_argument('--load-in-8bit', action='store_true')
+    parser.add_argument('--load-in-4bit', action='store_true')
+    parser.add_argument('--auto', action='store_true')
+    args = parser.parse_args()
+
+    if not os.path.exists(args.out_dir):
+        os.makedirs(args.out_dir, exist_ok=True)
+
+    args.datasets = args.datasets.split(',')
+    print('datasets:', args.datasets)
+    assert args.batch_size == 1, 'Only batch size 1 is supported'
+
+    torch.distributed.init_process_group(
+        backend='nccl',
+        world_size=int(os.getenv('WORLD_SIZE', '1')),
+        rank=int(os.getenv('RANK', '0')),
+    )
+
+    torch.cuda.set_device(int(os.getenv('LOCAL_RANK', 0)))
+
+    model, tokenizer = load_model_and_tokenizer(args)
+    image_size = model.config.force_image_size or model.config.vision_config.image_size
+    use_thumbnail = model.config.use_thumbnail
+
+    total_params = sum(p.numel() for p in model.parameters()) / 1e9
+    if total_params > 20 or args.dynamic:
+        args.num_beams = 1
+        print(f'[test] total_params: {total_params}B, use num_beams: {args.num_beams}')
+    else:
+        print(f'[test] total_params: {total_params}B')
+    print(f'[test] image_size: {image_size}')
+    print(f'[test] template: {model.config.template}')
+    print(f'[test] dynamic_image_size: {args.dynamic}')
+    print(f'[test] use_thumbnail: {use_thumbnail}')
+    print(f'[test] max_num: {args.max_num}')
+
+    evaluate_chat_model()