sampling_ext.cpp 2.85 KB
Newer Older
1
2
3
// TODO: this is a temporary wrapper to allow calling C++ code from CGo
#include "sampling.h"
#include "sampling_ext.h"
4
#include "json-schema-to-grammar.h"
5
6
7
#include "llama.h"
#include "llama-model.h"
#include "llama-model-loader.h"
8

9
10
11
struct common_sampler *common_sampler_cinit(const struct llama_model *model, struct common_sampler_cparams *params) {
    try {
        common_params_sampling sparams;
Jesse Gross's avatar
Jesse Gross committed
12
13
14
15
16
17
18
19
20
21
22
23
24
25
        sparams.top_k = params->top_k;
        sparams.top_p = params->top_p;
        sparams.min_p = params->min_p;
        sparams.typ_p = params->typical_p;
        sparams.temp = params->temp;
        sparams.penalty_last_n = params->penalty_last_n;
        sparams.penalty_repeat = params->penalty_repeat;
        sparams.penalty_freq = params->penalty_freq;
        sparams.penalty_present = params->penalty_present;
        sparams.mirostat = params->mirostat;
        sparams.mirostat_tau = params->mirostat_tau;
        sparams.mirostat_eta = params->mirostat_eta;
        sparams.seed = params->seed;
        sparams.grammar = params->grammar;
26
27
28
29
        sparams.xtc_probability = 0.0;
        sparams.xtc_threshold = 0.5;
        return common_sampler_init(model, sparams);
    } catch (const std::exception &err) {
Jesse Gross's avatar
Jesse Gross committed
30
31
        return nullptr;
    }
32
33
}

34
35
void common_sampler_cfree(struct common_sampler *sampler) {
    common_sampler_free(sampler);
36
37
}

38
39
void common_sampler_creset(struct common_sampler *sampler) {
    common_sampler_reset(sampler);
40
41
}

42
43
void common_sampler_caccept(struct common_sampler *sampler, llama_token id, bool apply_grammar) {
    common_sampler_accept(sampler, id, apply_grammar);
44
45
}

46
47
llama_token common_sampler_csample(struct common_sampler *sampler, struct llama_context *ctx, int idx) {
    return common_sampler_sample(sampler, ctx, idx);
48
}
49
50
51
52
53

int schema_to_grammar(const char *json_schema, char *grammar, size_t max_len)
{
    try
    {
54
        nlohmann::ordered_json schema = nlohmann::ordered_json::parse(json_schema);
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
        std::string grammar_str = json_schema_to_grammar(schema);
        size_t len = grammar_str.length();
        if (len >= max_len)
        {
            len = max_len - 1;
        }
        strncpy(grammar, grammar_str.c_str(), len);
        return len;
    }
    catch (const std::exception &e)
    {
        strncpy(grammar, "", max_len - 1);
        return 0;
    }
}
70
71
72
73
74
75

struct llama_vocab * llama_load_vocab_from_file(const char * fname) {
    llama_vocab * vocab = new llama_vocab();
    try {
        const auto kv = LLM_KV(LLM_ARCH_UNKNOWN);
        std::vector<std::string> splits = {};
76
        llama_model_loader ml(std::string(fname), splits, false, false, nullptr, nullptr);
77
78
79
80
81
82
83
84
85
86
87
88
        vocab->load(ml, kv);
    } catch (const std::exception & err) {
        LLAMA_LOG_ERROR("%s: error loading model: %s\n", __func__, err.what());
        return nullptr;
    }

    return vocab;
}

void llama_free_vocab(struct llama_vocab * vocab) {
    delete vocab;
}