import os
# 设置环境变量（仅在当前会话有效）
os.environ["HF_ENDPOINT"] = "https://hf-mirror.com"  

!huggingface-cli login --token hf_HqxmRaSxadGZynzH*****

from datasets import load_dataset

ds = load_dataset("gaia-benchmark/GAIA", '2023_all', cache_dir="cache")

# 查看训练集样本数
print("Train samples:", len(ds['test']))
print("Validation samples:", len(ds['validation']))

from pprint import pprint

# 查看数据集的特征
pprint(ds['validation'].features)

# 查看第一条数据的元数据
sample = ds['validation'][0]

pprint(sample)

A paper about AI regulation that was originally submitted to arXiv.org in June 2022 shows a figure with three axes, where each axis has a label word at both ends. Which of these words is used to describe a type of society in a Physics and Society article submitted to arXiv.org on August 11, 2016?

翻译：一篇最初于2022年6月提交到arXiv.org的关于AI监管的论文展示了一个包含三个坐标轴的图表，每个轴的两端都带有标签词。在2016年8月11日提交给arXiv.org的《物理与社会》文章中，这些词中哪个被用来描述一种社会类型

egalitarian
翻译：平等主义

I’m researching species that became invasive after people who kept them as pets released them. There’s a certain species of fish that was popularized as a pet by being the main character of the movie Finding Nemo. According to the USGS, where was this fish found as a nonnative species, before the year 2020? I need the answer formatted as the five-digit zip codes of the places the species was found, separated by commas if there is more than one place.

调查因宠物放生导致入侵的生物。某鱼类因《海底总动员》主角（尼莫）走红，随后被放生至非原生地。根据USGS数据，2020年前此鱼作为外来物种被发现于何处？需以5位邮编（多个则逗号分隔）呈现答案。

34689

The attached spreadsheet shows the inventory for a movie and video game rental store in Seattle, Washington. What is the title of the oldest Blu-Ray recorded in this spreadsheet? Return it as appearing in the spreadsheet.

所附电子表格是美国华盛顿州西雅图市某影碟游戏租赁店的库存清单，需回答：表格中记录的年代最早的蓝光光碟（Blu-Ray）的标题是什么？答案必须严格按表格内原文格式返回。

Time-Parking 2: Parallel Universe

Review the chess position provided in the image. It is black's turn. Provide the correct next move for black which guarantees a win. Please provide your response in algebraic notation.

查看图片中给出的棋局情况。轮到黑方走棋了。给出黑方能确保获胜的正确下一步走法。请以国际象棋的代数表示法给出您的回答。

Rd5

import json
from os import environ
import os

from datasets import Dataset

from opencompass.registry import LOAD_DATASET
from opencompass.utils import get_data_path

from .base import BaseDataset

@LOAD_DATASET.register_module()
class GAIADataset(BaseDataset):

    @staticmethod
    def load(path, local_mode: bool = False):        
        from datasets import load_dataset
        try:
            # 因为ModelScope的GAIA数据集读取存在问题，所以从huggingface读取
            ds = load_dataset("gaia-benchmark/GAIA", '2023_all', split='validation')
            rows = []
            for item in ds:
                rows.append({
                    'question': item['Question'],
                    'answerKey': item['Final answer'],
                    'file_path': item['file_path'],
                    'file_name': item['file_name'],
                    'level': item['Level']
                })
        except Exception as e:
            print(f"Error loading local file: {e}")

        return Dataset.from_list(rows)

from opencompass.openicl.icl_prompt_template import PromptTemplate
from opencompass.openicl.icl_retriever import ZeroRetriever
from opencompass.openicl.icl_inferencer import GenInferencer
from opencompass.openicl.icl_evaluator import AccEvaluator
from opencompass.datasets import GAIADataset
from opencompass.utils.text_postprocessors import first_capital_postprocess

gaia_reader_cfg = dict(
    input_columns='question',
    output_column='answerKey',
    test_split='test')

gaia_infer_cfg = dict(
    prompt_template=dict(
        type=PromptTemplate,
        template=dict(round=[
            dict(
                role='HUMAN',
                prompt=
                '请根据问题：{question}\n给出答案。答：'
            ),
        ]),
    ),
    retriever=dict(type=ZeroRetriever),
    inferencer=dict(type=GenInferencer),
)
gaia_eval_cfg = dict(
    evaluator=dict(type=AccEvaluator),
    pred_role='BOT',
    pred_postprocessor=dict(type=first_capital_postprocess),
)

gaia_datasets = [
    dict(
        abbr='gaia-validation',
        type=GAIADataset,
        path='opencompass/gaia',
        local_mode=False,
        reader_cfg=gaia_reader_cfg,
        infer_cfg=gaia_infer_cfg,
        eval_cfg=gaia_eval_cfg,
    )
]

DATASETS_MAPPING = {
    # GAIA Datasets
    "opencompass/gaia": {
        "ms_id": None,
        "hf_id": "gaia-benchmark/GAIA",
        "local": "./data/gaia/",
    },
    # 以下内容省略

- gaia:
    name: GAIA
    category: Reasoning
    paper: https://arxiv.org/abs/2311.12983
    configpath: opencompass/configs/datasets/GAIA/gaia_gen.py
    configpath_llmjudge: ''

from .gaia import *  # noqa: F401, F403

{
    "version": "0.2.0",
    "configurations": [
        {
            "name": "OpenCompass",
            "type": "python",
            "request": "launch",
            "module": "opencompass.cli.main",
            "cwd": "${workspaceFolder}/libs/OpenCompass",
            "python": "${command:python.interpreterPath}",
            "env": {
                "MODEL": "deepseek-ai/DeepSeek-V3",
                "API_KEY": "sk-pboelsoxvgeapocquovvdkvv******",
                "API_URL": "https://api.siliconflow.cn/v1/"  
            },
            "args": [
                "--models", "custom_api", 
                "--datasets", "gaia_gen", 
                "--debug", "-m", "all"]
        }
    ]
} 

INSERT INTO datasets (
    name, 
    description, 
    category,
    type,
    file_path, 
    configuration, 
    user_id, 
    is_active
) VALUES 
(
    'gaia_gen', 
    'GAIA数据集，这是一种严苛的评估Agent通用能力评测的数据集，其中包含165个任务，每个任务都需要agent借助外部工具来完成。', 
    '智能体', 
    'benchmark', 
    '/data/gaia', 
    '{"format": "chat"}', 
    1, 
    1
);