Skip to content

GitLab

Switch branch/tag
Find file Normal viewHistoryPermalink
training_example.json 2.52 KB
Newer Older
mashun1's avatar
uitars  
mashun1 committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72 
[
    {
        "role": "system",
        "content": "You are a helpful assistant.",
        "loss_mask": 0
    },
    {
        "role": "user",
        "content": "You are a GUI agent. You are given a task and your action history, with screenshots. You need to perform the next action to complete the task. \n\n## Output Format\n\nAction: ...\n\n\n## Action Space\nclick(start_box='<|box_start|>(x1,y1)<|box_end|>')\nselect(start_box='<|box_start|>(x1,y1)<|box_end|>') # To expand a dropdown menu or select an item from it.\ntype(content='', start_box='<|box_start|>(x1,y1)<|box_end|>') # First click on the textbox, and then type the content.\n\n## User Instruction\nBuy a pop rock album CD from the United Kingdom that was released in 2016, is between £15 and £20 and in perfect condition.\n\n ",
        "loss_mask": 0
    },
    {
        "role": "assistant",
        "content": "Action: click(start_box='<|box_start|>(495,30)<|box_end|>')",
        "loss_mask": 0
    },
    {
        "role": "assistant",
        "content": "Action: click(start_box='<|box_start|>(381,153)<|box_end|>')",
        "loss_mask": 0
    },
    {
        "role": "assistant",
        "content": "Action: click(start_box='<|box_start|>(66,263)<|box_end|>')",
        "loss_mask": 0
    },
    {
        "role": "assistant",
        "content": "Action: click(start_box='<|box_start|>(66,851)<|box_end|>')",
        "loss_mask": 0
    },
    {
        "role": "user",
        "content": "<|vision_start|><|image_pad|>...<|image_pad|><|vision_end|>",
        "loss_mask": 0
    },
    {
        "role": "assistant",
        "content": "Action: click(start_box='<|box_start|>(66,638)<|box_end|>')",
        "loss_mask": 0
    },
    {
        "role": "user",
        "content": "<|vision_start|><|image_pad|>...<|image_pad|><|vision_end|>",
        "loss_mask": 0
    },
    {
        "role": "assistant",
        "content": "Action: click(start_box='<|box_start|>(66,86)<|box_end|>')",
        "loss_mask": 0
    },
    {
        "role": "user",
        "content": "<|vision_start|><|image_pad|>...<|image_pad|><|vision_end|>",
        "loss_mask": 0
    },
    {
        "role": "assistant",
        "content": "Action: click(start_box='<|box_start|>(66,5)<|box_end|>')",
        "loss_mask": 0
    },
    {
        "role": "user",
        "content": "<|vision_start|><|image_pad|>...<|image_pad|><|vision_end|>",
        "loss_mask": 0
    },
    {
        "role": "assistant",
        "content": "Action: click(start_box='<|box_start|>(293,218)<|box_end|>')",
        "loss_mask": 1
    }
]