init
This commit is contained in:
@@ -0,0 +1,182 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"read_only": true,
|
||||
"model": "Qwen/Qwen3-1.7B-Base",
|
||||
"packed": {
|
||||
"path": "/home/kami/Programs/esp32-whisper-fine-tune/llm/data/cpt_packed",
|
||||
"rows": 129221,
|
||||
"block_tokens": 2048,
|
||||
"total_tokens": 264644608,
|
||||
"arrow_bytes": 1059122792,
|
||||
"arrow_sha256": {
|
||||
"data-00000-of-00003.arrow": "0e9276e5fd85e692ec33385b51b3d9afb00b7dd0dec779b9cd240fc78cf64ce5",
|
||||
"data-00001-of-00003.arrow": "1f748a906c898a99c0940b8104a9c9fcab6236ab9b8ce0cb2acf4173da55808a",
|
||||
"data-00002-of-00003.arrow": "e8e3cc118ade3963bfe7a0609c468b27b7a931b826547e44ff755204682aae4d"
|
||||
},
|
||||
"sampled_rows": 16,
|
||||
"sample_bad_lengths": 0,
|
||||
"sample_out_of_vocab_ids": 0,
|
||||
"sample_mixed_script_rows": 4
|
||||
},
|
||||
"clean_sources": {
|
||||
"books": {
|
||||
"docs": 21784,
|
||||
"chars": 92410645,
|
||||
"mean_doc_chars": 4242.13,
|
||||
"mean_cyrillic_ratio": 0.988257,
|
||||
"mixed_script_docs": 0,
|
||||
"bytes": 168304006,
|
||||
"sha256": "99235f16bba523405405553732fe42298abff9cfd01d2adb6dc25902e3bf56cc"
|
||||
},
|
||||
"culturax": {
|
||||
"docs": 187956,
|
||||
"chars": 508949928,
|
||||
"mean_doc_chars": 2707.81,
|
||||
"mean_cyrillic_ratio": 0.95988,
|
||||
"mixed_script_docs": 247,
|
||||
"bytes": 918337344,
|
||||
"sha256": "8da8504d5bf03155fbe613a1f7db37d2dd49e3c77e8ff3ef5c1bc4a09d37aad4"
|
||||
},
|
||||
"wiki": {
|
||||
"docs": 21128,
|
||||
"chars": 131921258,
|
||||
"mean_doc_chars": 6243.91,
|
||||
"mean_cyrillic_ratio": 0.970641,
|
||||
"mixed_script_docs": 0,
|
||||
"bytes": 237952666,
|
||||
"sha256": "cc18c5a91a53c2bbb9727788036ea61b872484faffae8e764194071e556653f9"
|
||||
}
|
||||
},
|
||||
"clean_char_share_pct": {
|
||||
"books": 12.602,
|
||||
"culturax": 69.407,
|
||||
"wiki": 17.991
|
||||
},
|
||||
"provenance_warning": "Packed Arrow has no per-block source/document metadata; source shares can only be reconstructed from the clean inputs, not proven from Arrow alone.",
|
||||
"checkpoint": {
|
||||
"present": true,
|
||||
"latest_checkpoint": "/home/kami/Programs/esp32-whisper-fine-tune/llm/Qwen3-1.7B-ru-cpt/checkpoint-1000",
|
||||
"global_step": 1000,
|
||||
"max_steps": 8077,
|
||||
"progress_pct": 12.381,
|
||||
"epoch": 0.123818883927535,
|
||||
"last_loss": 1.9053752899169922,
|
||||
"last_grad_norm": 1.3828125,
|
||||
"last_learning_rate": 9.771972501846451e-06,
|
||||
"full_weight_model_bytes": 3441185608
|
||||
},
|
||||
"samples": [
|
||||
{
|
||||
"index": 6555,
|
||||
"tokens": 2048,
|
||||
"cyrillic_ratio": 0.999387,
|
||||
"mixed_script": false,
|
||||
"preview": " понимающими, какое впечатление это производит. Президентские выборы должны были добавить легитимности нынешнему правлению — так вот, этого не случилось. Во-вторых, произошли значительные изменения в характере протестного движения. Количест"
|
||||
},
|
||||
{
|
||||
"index": 8078,
|
||||
"tokens": 2048,
|
||||
"cyrillic_ratio": 0.988204,
|
||||
"mixed_script": false,
|
||||
"preview": " не приходится: то время было периодом расцвета жанра – начиная с мирискусников и заканчивая соцреалистами. Потом, правда, возник еще оттепельный подъем интереса к натюрморту, охвативший как официальных, так и андеграундных художников, но д"
|
||||
},
|
||||
{
|
||||
"index": 8511,
|
||||
"tokens": 2048,
|
||||
"cyrillic_ratio": 1.0,
|
||||
"mixed_script": false,
|
||||
"preview": "ясниться. В качестве переводчика, а точнее – менеджера спортивного отдела по работе с иностранцами, он и был приглашен в «Спартак» еще генеральным директором Сергеем Шавло. При генеральном директоре Карпине пошел на повышение. Сначала стал "
|
||||
},
|
||||
{
|
||||
"index": 22305,
|
||||
"tokens": 2048,
|
||||
"cyrillic_ratio": 0.976603,
|
||||
"mixed_script": true,
|
||||
"preview": " День на день не приходится 20 февраля 2019 года, 18:30 Театр Малый театр (малая сцена на Ордынке) 2500 Р Травиата 20 февраля 2019 года, 19:00 Театр Новая Опера 2000 Р Все о Золушке 20 февраля 2019 года, 19:00 Театр Театр Мюзикла 2000 Р Точ"
|
||||
},
|
||||
{
|
||||
"index": 33410,
|
||||
"tokens": 2048,
|
||||
"cyrillic_ratio": 0.937131,
|
||||
"mixed_script": true,
|
||||
"preview": "ело какао, и даже Надкостницу следует оставить комментарий на 1 час, затем проглотить. Студенческая истина питает и вызывает красавцы, делает их жирными, ранними и послушными. BOTANIC THERAPY протекает секущиеся кончики, питает, встряхивает"
|
||||
},
|
||||
{
|
||||
"index": 43006,
|
||||
"tokens": 2048,
|
||||
"cyrillic_ratio": 0.996627,
|
||||
"mixed_script": false,
|
||||
"preview": " человека? Информация от учителя Люди заставляют воду давать электроэнергию, перевозить грузы, орошать землю, лечить и развлекать человека. Динамический момент(закрепление знаний речных и морских обитателей) показываются картинки с подписям"
|
||||
},
|
||||
{
|
||||
"index": 45603,
|
||||
"tokens": 2048,
|
||||
"cyrillic_ratio": 0.995132,
|
||||
"mixed_script": false,
|
||||
"preview": ", балк., гараж в доме, выход на террасу с кухни и с дома Автомойки бесконтактные цена в Украине | Купить all biz 48 предложений Автомойки бесконтактные в , заказать и купить Шланги для автомойки штуцеровка,рукав автомойки чертежи дробилка щ"
|
||||
},
|
||||
{
|
||||
"index": 47673,
|
||||
"tokens": 2048,
|
||||
"cyrillic_ratio": 0.952866,
|
||||
"mixed_script": true,
|
||||
"preview": " тот постоянно инвестирует в себя, тот, кто чувствует эти тренды, кто постоянно занимается этими технологиями, он, в конечном итоге, забирает всё. Все спикеры - члены международной организации YPO. Сегодня в клубе состоят 28 тысяч бизнесмен"
|
||||
},
|
||||
{
|
||||
"index": 49647,
|
||||
"tokens": 2048,
|
||||
"cyrillic_ratio": 0.868365,
|
||||
"mixed_script": false,
|
||||
"preview": "7 КБ | Просмотров: 2825 ] 100_9393.JPG [ 843.98 КБ | Просмотров: 2818 ] Сама посудина 100_9392.JPG [ 885.76 КБ | Просмотров: 2816 ] Демон с Наташей готовятся к бою 100_9397.JPG [ 928.71 КБ | Просмотров: 2821 ] Да короче все готовятся 100_93"
|
||||
},
|
||||
{
|
||||
"index": 60392,
|
||||
"tokens": 2048,
|
||||
"cyrillic_ratio": 0.938221,
|
||||
"mixed_script": false,
|
||||
"preview": "Бегемотик\" купить в магазине Апанде - купить недорого. Низкие цены в магазине Апанде. Серебряная ложка с эмалью \"Бегемотик\" Серебряная чайная ложка \"Бегемотик\" у нас Серебряная чайная ложка \"Бегемотик\" прошел ОТК и проверку в государственно"
|
||||
},
|
||||
{
|
||||
"index": 75172,
|
||||
"tokens": 2048,
|
||||
"cyrillic_ratio": 0.999089,
|
||||
"mixed_script": false,
|
||||
"preview": "КАЯ: Как невероятно метки… С. БУНТМАН: … точно его Пауком назвали. Н. БАСОВСКАЯ: Невероятно меткими являются вот эти народные прозвания. Как часто многие из правителей, желающие оставить в веках совсем другой след, чем они, по сути, заслужи"
|
||||
},
|
||||
{
|
||||
"index": 97796,
|
||||
"tokens": 2048,
|
||||
"cyrillic_ratio": 0.992467,
|
||||
"mixed_script": true,
|
||||
"preview": "альник дифференциала Масло трансмиссионное Карданный вал Подвесной подшипник Двигатель Блок цилиндров Прокладки блока цилиндров Головка блока цилиндров Болты ГБЦ Головка блока цилиндров Направляющая втулка клапана Крепление двигателя Кроншт"
|
||||
},
|
||||
{
|
||||
"index": 100605,
|
||||
"tokens": 2048,
|
||||
"cyrillic_ratio": 0.999315,
|
||||
"mixed_script": false,
|
||||
"preview": " невредимой, но голос его при этом выражал куда более глубокие чувства. – я уронила корзину и уперла руки в бока. Сокровища профессора не интересовали. Отозвался бегущий, даже не обернувшись. Будь готов к шоку после того, как покинешь робот"
|
||||
},
|
||||
{
|
||||
"index": 117529,
|
||||
"tokens": 2048,
|
||||
"cyrillic_ratio": 0.986769,
|
||||
"mixed_script": false,
|
||||
"preview": "» и «Восток». Своеобразной достопримечательностью «Всемирной литературы» была торговка Роза Васильевна. В 1924 году издательство «Всемирная литература» было объединено с Ленгизом. См. также Библиотека всемирной литературы Примечания "
|
||||
},
|
||||
{
|
||||
"index": 122565,
|
||||
"tokens": 2048,
|
||||
"cyrillic_ratio": 0.927888,
|
||||
"mixed_script": false,
|
||||
"preview": "ікінги в Центрально-Східній Європі: Загадки Ладоги і Пліснеська Butler F. Ol'ga's Conversion and the Construction of Chronicle Narrative // The Russian Review, 2008. Vol. 67, № 2. — pp. 230–242. Ссылки Княгиня Ольга на официальном сай"
|
||||
},
|
||||
{
|
||||
"index": 122647,
|
||||
"tokens": 2048,
|
||||
"cyrillic_ratio": 1.0,
|
||||
"mixed_script": false,
|
||||
"preview": " комплексе «Тополь-М». На боевое дежурство 1-й ракетный дивизион и ПКП 321 рп заступил 10 декабря 2006 года в 15:00. Тогда же стало известно о подписании президентом Владимиром Путиным новой государственной программы вооружений до 2015 года"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
---
|
||||
base_model: unsloth/Llama-3.2-3B-Instruct
|
||||
library_name: peft
|
||||
pipeline_tag: text-generation
|
||||
tags:
|
||||
- base_model:adapter:unsloth/Llama-3.2-3B-Instruct
|
||||
- lora
|
||||
- transformers
|
||||
---
|
||||
|
||||
# Model Card for Model ID
|
||||
|
||||
<!-- Provide a quick summary of what the model is/does. -->
|
||||
|
||||
|
||||
|
||||
## Model Details
|
||||
|
||||
### Model Description
|
||||
|
||||
<!-- Provide a longer summary of what this model is. -->
|
||||
|
||||
|
||||
|
||||
- **Developed by:** [More Information Needed]
|
||||
- **Funded by [optional]:** [More Information Needed]
|
||||
- **Shared by [optional]:** [More Information Needed]
|
||||
- **Model type:** [More Information Needed]
|
||||
- **Language(s) (NLP):** [More Information Needed]
|
||||
- **License:** [More Information Needed]
|
||||
- **Finetuned from model [optional]:** [More Information Needed]
|
||||
|
||||
### Model Sources [optional]
|
||||
|
||||
<!-- Provide the basic links for the model. -->
|
||||
|
||||
- **Repository:** [More Information Needed]
|
||||
- **Paper [optional]:** [More Information Needed]
|
||||
- **Demo [optional]:** [More Information Needed]
|
||||
|
||||
## Uses
|
||||
|
||||
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
||||
|
||||
### Direct Use
|
||||
|
||||
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
### Downstream Use [optional]
|
||||
|
||||
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
### Out-of-Scope Use
|
||||
|
||||
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
## Bias, Risks, and Limitations
|
||||
|
||||
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
### Recommendations
|
||||
|
||||
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
||||
|
||||
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
|
||||
|
||||
## How to Get Started with the Model
|
||||
|
||||
Use the code below to get started with the model.
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
## Training Details
|
||||
|
||||
### Training Data
|
||||
|
||||
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
### Training Procedure
|
||||
|
||||
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
|
||||
|
||||
#### Preprocessing [optional]
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
|
||||
#### Training Hyperparameters
|
||||
|
||||
- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
|
||||
|
||||
#### Speeds, Sizes, Times [optional]
|
||||
|
||||
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
## Evaluation
|
||||
|
||||
<!-- This section describes the evaluation protocols and provides the results. -->
|
||||
|
||||
### Testing Data, Factors & Metrics
|
||||
|
||||
#### Testing Data
|
||||
|
||||
<!-- This should link to a Dataset Card if possible. -->
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
#### Factors
|
||||
|
||||
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
#### Metrics
|
||||
|
||||
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
### Results
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
#### Summary
|
||||
|
||||
|
||||
|
||||
## Model Examination [optional]
|
||||
|
||||
<!-- Relevant interpretability work for the model goes here -->
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
## Environmental Impact
|
||||
|
||||
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
|
||||
|
||||
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
|
||||
|
||||
- **Hardware Type:** [More Information Needed]
|
||||
- **Hours used:** [More Information Needed]
|
||||
- **Cloud Provider:** [More Information Needed]
|
||||
- **Compute Region:** [More Information Needed]
|
||||
- **Carbon Emitted:** [More Information Needed]
|
||||
|
||||
## Technical Specifications [optional]
|
||||
|
||||
### Model Architecture and Objective
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
### Compute Infrastructure
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
#### Hardware
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
#### Software
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
## Citation [optional]
|
||||
|
||||
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
|
||||
|
||||
**BibTeX:**
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
**APA:**
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
## Glossary [optional]
|
||||
|
||||
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
## More Information [optional]
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
## Model Card Authors [optional]
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
## Model Card Contact
|
||||
|
||||
[More Information Needed]
|
||||
### Framework versions
|
||||
|
||||
- PEFT 0.18.1
|
||||
@@ -0,0 +1,46 @@
|
||||
{
|
||||
"alora_invocation_tokens": null,
|
||||
"alpha_pattern": {},
|
||||
"arrow_config": null,
|
||||
"auto_mapping": null,
|
||||
"base_model_name_or_path": "unsloth/Llama-3.2-3B-Instruct",
|
||||
"bias": "none",
|
||||
"corda_config": null,
|
||||
"ensure_weight_tying": false,
|
||||
"eva_config": null,
|
||||
"exclude_modules": null,
|
||||
"fan_in_fan_out": false,
|
||||
"inference_mode": true,
|
||||
"init_lora_weights": true,
|
||||
"layer_replication": null,
|
||||
"layers_pattern": null,
|
||||
"layers_to_transform": null,
|
||||
"loftq_config": {},
|
||||
"lora_alpha": 32,
|
||||
"lora_bias": false,
|
||||
"lora_dropout": 0.1,
|
||||
"megatron_config": null,
|
||||
"megatron_core": "megatron.core",
|
||||
"modules_to_save": null,
|
||||
"peft_type": "LORA",
|
||||
"peft_version": "0.18.1",
|
||||
"qalora_group_size": 16,
|
||||
"r": 16,
|
||||
"rank_pattern": {},
|
||||
"revision": null,
|
||||
"target_modules": [
|
||||
"gate_proj",
|
||||
"up_proj",
|
||||
"o_proj",
|
||||
"k_proj",
|
||||
"q_proj",
|
||||
"v_proj",
|
||||
"down_proj"
|
||||
],
|
||||
"target_parameters": null,
|
||||
"task_type": "CAUSAL_LM",
|
||||
"trainable_token_indices": null,
|
||||
"use_dora": false,
|
||||
"use_qalora": false,
|
||||
"use_rslora": false
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
{{- bos_token }}
|
||||
{%- if custom_tools is defined %}
|
||||
{%- set tools = custom_tools %}
|
||||
{%- endif %}
|
||||
{%- if not tools_in_user_message is defined %}
|
||||
{%- set tools_in_user_message = true %}
|
||||
{%- endif %}
|
||||
{%- if not date_string is defined %}
|
||||
{%- if strftime_now is defined %}
|
||||
{%- set date_string = strftime_now("%d %b %Y") %}
|
||||
{%- else %}
|
||||
{%- set date_string = "26 Jul 2024" %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- if not tools is defined %}
|
||||
{%- set tools = none %}
|
||||
{%- endif %}
|
||||
|
||||
{#- This block extracts the system message, so we can slot it into the right place. #}
|
||||
{%- if messages[0]['role'] == 'system' %}
|
||||
{%- set system_message = messages[0]['content']|trim %}
|
||||
{%- set messages = messages[1:] %}
|
||||
{%- else %}
|
||||
{%- set system_message = "" %}
|
||||
{%- endif %}
|
||||
|
||||
{#- System message #}
|
||||
{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
|
||||
{%- if tools is not none %}
|
||||
{{- "Environment: ipython\n" }}
|
||||
{%- endif %}
|
||||
{{- "Cutting Knowledge Date: December 2023\n" }}
|
||||
{{- "Today Date: " + date_string + "\n\n" }}
|
||||
{%- if tools is not none and not tools_in_user_message %}
|
||||
{{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
|
||||
{{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
|
||||
{{- "Do not use variables.\n\n" }}
|
||||
{%- for t in tools %}
|
||||
{{- t | tojson(indent=4) }}
|
||||
{{- "\n\n" }}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- system_message }}
|
||||
{{- "<|eot_id|>" }}
|
||||
|
||||
{#- Custom tools are passed in a user message with some extra guidance #}
|
||||
{%- if tools_in_user_message and not tools is none %}
|
||||
{#- Extract the first user message so we can plug it in here #}
|
||||
{%- if messages | length != 0 %}
|
||||
{%- set first_user_message = messages[0]['content']|trim %}
|
||||
{%- set messages = messages[1:] %}
|
||||
{%- else %}
|
||||
{{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
|
||||
{%- endif %}
|
||||
{{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
|
||||
{{- "Given the following functions, please respond with a JSON for a function call " }}
|
||||
{{- "with its proper arguments that best answers the given prompt.\n\n" }}
|
||||
{{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
|
||||
{{- "Do not use variables.\n\n" }}
|
||||
{%- for t in tools %}
|
||||
{{- t | tojson(indent=4) }}
|
||||
{{- "\n\n" }}
|
||||
{%- endfor %}
|
||||
{{- first_user_message + "<|eot_id|>"}}
|
||||
{%- endif %}
|
||||
|
||||
{%- for message in messages %}
|
||||
{%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
|
||||
{{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
|
||||
{%- elif 'tool_calls' in message %}
|
||||
{%- if not message.tool_calls|length == 1 %}
|
||||
{{- raise_exception("This model only supports single tool-calls at once!") }}
|
||||
{%- endif %}
|
||||
{%- set tool_call = message.tool_calls[0].function %}
|
||||
{{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
|
||||
{{- '{"name": "' + tool_call.name + '", ' }}
|
||||
{{- '"parameters": ' }}
|
||||
{{- tool_call.arguments | tojson }}
|
||||
{{- "}" }}
|
||||
{{- "<|eot_id|>" }}
|
||||
{%- elif message.role == "tool" or message.role == "ipython" %}
|
||||
{{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
|
||||
{%- if message.content is mapping or message.content is iterable %}
|
||||
{{- message.content | tojson }}
|
||||
{%- else %}
|
||||
{{- message.content }}
|
||||
{%- endif %}
|
||||
{{- "<|eot_id|>" }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if add_generation_prompt %}
|
||||
{{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
|
||||
{%- endif %}
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"backend": "tokenizers",
|
||||
"bos_token": "<|begin_of_text|>",
|
||||
"clean_up_tokenization_spaces": true,
|
||||
"eos_token": "<|eot_id|>",
|
||||
"is_local": false,
|
||||
"model_input_names": [
|
||||
"input_ids",
|
||||
"attention_mask"
|
||||
],
|
||||
"model_max_length": 131072,
|
||||
"pad_token": "<|eot_id|>",
|
||||
"padding_side": "right",
|
||||
"tokenizer_class": "TokenizersBackend",
|
||||
"unk_token": null
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
---
|
||||
base_model: unsloth/Llama-3.2-3B-Instruct
|
||||
library_name: peft
|
||||
pipeline_tag: text-generation
|
||||
tags:
|
||||
- base_model:adapter:unsloth/Llama-3.2-3B-Instruct
|
||||
- lora
|
||||
- transformers
|
||||
---
|
||||
|
||||
# Model Card for Model ID
|
||||
|
||||
<!-- Provide a quick summary of what the model is/does. -->
|
||||
|
||||
|
||||
|
||||
## Model Details
|
||||
|
||||
### Model Description
|
||||
|
||||
<!-- Provide a longer summary of what this model is. -->
|
||||
|
||||
|
||||
|
||||
- **Developed by:** [More Information Needed]
|
||||
- **Funded by [optional]:** [More Information Needed]
|
||||
- **Shared by [optional]:** [More Information Needed]
|
||||
- **Model type:** [More Information Needed]
|
||||
- **Language(s) (NLP):** [More Information Needed]
|
||||
- **License:** [More Information Needed]
|
||||
- **Finetuned from model [optional]:** [More Information Needed]
|
||||
|
||||
### Model Sources [optional]
|
||||
|
||||
<!-- Provide the basic links for the model. -->
|
||||
|
||||
- **Repository:** [More Information Needed]
|
||||
- **Paper [optional]:** [More Information Needed]
|
||||
- **Demo [optional]:** [More Information Needed]
|
||||
|
||||
## Uses
|
||||
|
||||
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
||||
|
||||
### Direct Use
|
||||
|
||||
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
### Downstream Use [optional]
|
||||
|
||||
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
### Out-of-Scope Use
|
||||
|
||||
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
## Bias, Risks, and Limitations
|
||||
|
||||
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
### Recommendations
|
||||
|
||||
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
||||
|
||||
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
|
||||
|
||||
## How to Get Started with the Model
|
||||
|
||||
Use the code below to get started with the model.
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
## Training Details
|
||||
|
||||
### Training Data
|
||||
|
||||
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
### Training Procedure
|
||||
|
||||
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
|
||||
|
||||
#### Preprocessing [optional]
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
|
||||
#### Training Hyperparameters
|
||||
|
||||
- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
|
||||
|
||||
#### Speeds, Sizes, Times [optional]
|
||||
|
||||
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
## Evaluation
|
||||
|
||||
<!-- This section describes the evaluation protocols and provides the results. -->
|
||||
|
||||
### Testing Data, Factors & Metrics
|
||||
|
||||
#### Testing Data
|
||||
|
||||
<!-- This should link to a Dataset Card if possible. -->
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
#### Factors
|
||||
|
||||
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
#### Metrics
|
||||
|
||||
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
### Results
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
#### Summary
|
||||
|
||||
|
||||
|
||||
## Model Examination [optional]
|
||||
|
||||
<!-- Relevant interpretability work for the model goes here -->
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
## Environmental Impact
|
||||
|
||||
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
|
||||
|
||||
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
|
||||
|
||||
- **Hardware Type:** [More Information Needed]
|
||||
- **Hours used:** [More Information Needed]
|
||||
- **Cloud Provider:** [More Information Needed]
|
||||
- **Compute Region:** [More Information Needed]
|
||||
- **Carbon Emitted:** [More Information Needed]
|
||||
|
||||
## Technical Specifications [optional]
|
||||
|
||||
### Model Architecture and Objective
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
### Compute Infrastructure
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
#### Hardware
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
#### Software
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
## Citation [optional]
|
||||
|
||||
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
|
||||
|
||||
**BibTeX:**
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
**APA:**
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
## Glossary [optional]
|
||||
|
||||
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
## More Information [optional]
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
## Model Card Authors [optional]
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
## Model Card Contact
|
||||
|
||||
[More Information Needed]
|
||||
### Framework versions
|
||||
|
||||
- PEFT 0.18.1
|
||||
@@ -0,0 +1,46 @@
|
||||
{
|
||||
"alora_invocation_tokens": null,
|
||||
"alpha_pattern": {},
|
||||
"arrow_config": null,
|
||||
"auto_mapping": null,
|
||||
"base_model_name_or_path": "unsloth/Llama-3.2-3B-Instruct",
|
||||
"bias": "none",
|
||||
"corda_config": null,
|
||||
"ensure_weight_tying": false,
|
||||
"eva_config": null,
|
||||
"exclude_modules": null,
|
||||
"fan_in_fan_out": false,
|
||||
"inference_mode": true,
|
||||
"init_lora_weights": true,
|
||||
"layer_replication": null,
|
||||
"layers_pattern": null,
|
||||
"layers_to_transform": null,
|
||||
"loftq_config": {},
|
||||
"lora_alpha": 32,
|
||||
"lora_bias": false,
|
||||
"lora_dropout": 0.1,
|
||||
"megatron_config": null,
|
||||
"megatron_core": "megatron.core",
|
||||
"modules_to_save": null,
|
||||
"peft_type": "LORA",
|
||||
"peft_version": "0.18.1",
|
||||
"qalora_group_size": 16,
|
||||
"r": 8,
|
||||
"rank_pattern": {},
|
||||
"revision": null,
|
||||
"target_modules": [
|
||||
"gate_proj",
|
||||
"q_proj",
|
||||
"o_proj",
|
||||
"k_proj",
|
||||
"v_proj",
|
||||
"down_proj",
|
||||
"up_proj"
|
||||
],
|
||||
"target_parameters": null,
|
||||
"task_type": "CAUSAL_LM",
|
||||
"trainable_token_indices": null,
|
||||
"use_dora": false,
|
||||
"use_qalora": false,
|
||||
"use_rslora": false
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
{{- bos_token }}
|
||||
{%- if custom_tools is defined %}
|
||||
{%- set tools = custom_tools %}
|
||||
{%- endif %}
|
||||
{%- if not tools_in_user_message is defined %}
|
||||
{%- set tools_in_user_message = true %}
|
||||
{%- endif %}
|
||||
{%- if not date_string is defined %}
|
||||
{%- if strftime_now is defined %}
|
||||
{%- set date_string = strftime_now("%d %b %Y") %}
|
||||
{%- else %}
|
||||
{%- set date_string = "26 Jul 2024" %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- if not tools is defined %}
|
||||
{%- set tools = none %}
|
||||
{%- endif %}
|
||||
|
||||
{#- This block extracts the system message, so we can slot it into the right place. #}
|
||||
{%- if messages[0]['role'] == 'system' %}
|
||||
{%- set system_message = messages[0]['content']|trim %}
|
||||
{%- set messages = messages[1:] %}
|
||||
{%- else %}
|
||||
{%- set system_message = "" %}
|
||||
{%- endif %}
|
||||
|
||||
{#- System message #}
|
||||
{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
|
||||
{%- if tools is not none %}
|
||||
{{- "Environment: ipython\n" }}
|
||||
{%- endif %}
|
||||
{{- "Cutting Knowledge Date: December 2023\n" }}
|
||||
{{- "Today Date: " + date_string + "\n\n" }}
|
||||
{%- if tools is not none and not tools_in_user_message %}
|
||||
{{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
|
||||
{{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
|
||||
{{- "Do not use variables.\n\n" }}
|
||||
{%- for t in tools %}
|
||||
{{- t | tojson(indent=4) }}
|
||||
{{- "\n\n" }}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- system_message }}
|
||||
{{- "<|eot_id|>" }}
|
||||
|
||||
{#- Custom tools are passed in a user message with some extra guidance #}
|
||||
{%- if tools_in_user_message and not tools is none %}
|
||||
{#- Extract the first user message so we can plug it in here #}
|
||||
{%- if messages | length != 0 %}
|
||||
{%- set first_user_message = messages[0]['content']|trim %}
|
||||
{%- set messages = messages[1:] %}
|
||||
{%- else %}
|
||||
{{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
|
||||
{%- endif %}
|
||||
{{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
|
||||
{{- "Given the following functions, please respond with a JSON for a function call " }}
|
||||
{{- "with its proper arguments that best answers the given prompt.\n\n" }}
|
||||
{{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
|
||||
{{- "Do not use variables.\n\n" }}
|
||||
{%- for t in tools %}
|
||||
{{- t | tojson(indent=4) }}
|
||||
{{- "\n\n" }}
|
||||
{%- endfor %}
|
||||
{{- first_user_message + "<|eot_id|>"}}
|
||||
{%- endif %}
|
||||
|
||||
{%- for message in messages %}
|
||||
{%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
|
||||
{{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
|
||||
{%- elif 'tool_calls' in message %}
|
||||
{%- if not message.tool_calls|length == 1 %}
|
||||
{{- raise_exception("This model only supports single tool-calls at once!") }}
|
||||
{%- endif %}
|
||||
{%- set tool_call = message.tool_calls[0].function %}
|
||||
{{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
|
||||
{{- '{"name": "' + tool_call.name + '", ' }}
|
||||
{{- '"parameters": ' }}
|
||||
{{- tool_call.arguments | tojson }}
|
||||
{{- "}" }}
|
||||
{{- "<|eot_id|>" }}
|
||||
{%- elif message.role == "tool" or message.role == "ipython" %}
|
||||
{{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
|
||||
{%- if message.content is mapping or message.content is iterable %}
|
||||
{{- message.content | tojson }}
|
||||
{%- else %}
|
||||
{{- message.content }}
|
||||
{%- endif %}
|
||||
{{- "<|eot_id|>" }}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if add_generation_prompt %}
|
||||
{{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
|
||||
{%- endif %}
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"backend": "tokenizers",
|
||||
"bos_token": "<|begin_of_text|>",
|
||||
"clean_up_tokenization_spaces": true,
|
||||
"eos_token": "<|eot_id|>",
|
||||
"is_local": false,
|
||||
"model_input_names": [
|
||||
"input_ids",
|
||||
"attention_mask"
|
||||
],
|
||||
"model_max_length": 131072,
|
||||
"pad_token": "<|eot_id|>",
|
||||
"padding_side": "right",
|
||||
"tokenizer_class": "TokenizersBackend",
|
||||
"unk_token": null
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
# RU-CPT corpus + training — runbook
|
||||
|
||||
Scripts for building our own RU-native base (continued pretraining of
|
||||
Qwen3-1.7B-Base). Full plan: `Maven/docs/plans/2026-07-11-ru-cpt-base.md`.
|
||||
|
||||
## One-time setup
|
||||
|
||||
```bash
|
||||
pip install datasets ftfy datasketch fasttext openai peft accelerate
|
||||
wget -O data/lid.176.bin \
|
||||
https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin
|
||||
huggingface-cli login # accept CulturaX (gated) license
|
||||
```
|
||||
`fasttext` build fails → `pip install fasttext-wheel`. Move the model → `export FASTTEXT_LID=/path`.
|
||||
|
||||
## Run order
|
||||
|
||||
```bash
|
||||
# Phase 2 — corpus (build first; longest)
|
||||
python corpus_fetch.py all
|
||||
ROUTER_BASE_URL="https://your-router/v1" ROUTER_API_KEY="..." \
|
||||
ROUTER_MODELS="m-a,m-b,m-c" python corpus_synth.py # ≤10% hard cap
|
||||
MAVEN_LOGS=/path/to/dialogue.jsonl python corpus_logs.py # optional
|
||||
python corpus_clean.py # ftfy+langID+dedup
|
||||
python corpus_stats.py # DONE-CHECK, eyeball
|
||||
python corpus_pack.py # → data/cpt_packed/
|
||||
|
||||
# Phase 3 — CPT (days; --resume to continue; CPT_PATH=dora if OOM)
|
||||
HSA_OVERRIDE_GFX_VERSION=11.0.0 python train_cpt.py 2>&1 | tee cpt_run.log
|
||||
|
||||
# Phase 4 — eval + gate
|
||||
python eval_cyrillic.py --model Qwen/Qwen3-1.7B-Base # baseline
|
||||
python eval_cyrillic.py --model ./Qwen3-1.7B-ru-cpt # must beat baseline
|
||||
|
||||
# Phase 6-7 — persona LoRA (train_rocm.py @ ./Qwen3-1.7B-ru-cpt) → GGUF → Q8_0 → homesrv
|
||||
```
|
||||
|
||||
## Gotchas (a fresh agent WILL hit these)
|
||||
|
||||
- **Run `corpus_clean.py` with NO args (all buckets in ONE invocation).** Dedup is
|
||||
a single in-memory MinHash-LSH shared across buckets. Cleaning buckets in
|
||||
separate runs = no cross-bucket dedup = duplicated data across sources.
|
||||
- **Disk space.** `corpus_fetch.py` over-pulls ~1.5× and CulturaX streaming still
|
||||
caches to `HF_DATASETS_CACHE` (`/mnt/D/.cache/...`). Expect tens of GB of raw +
|
||||
cache. Check `df -h /mnt/D` before starting; `rm -rf data/cpt_raw` after packing.
|
||||
- **Strict order.** fetch → clean → pack → train. `corpus_pack.py` reads
|
||||
`data/cpt_clean/`; `train_cpt.py` reads `data/cpt_packed/`. Skipping a stage
|
||||
silently trains on nothing (empty dataset → instant "done", garbage model).
|
||||
- **`corpus_pack.py` prints the final token count — READ IT.** Must be ~300M ±50M.
|
||||
Way under → a fetch failed or clean dropped too much; way over → over-pulled.
|
||||
Do not train on a 20M-token corpus and expect CPT to work.
|
||||
- **Verify the base repo id exists**: `Qwen/Qwen3-1.7B-Base`. If HF 404s, the
|
||||
naming shifted — find the current Qwen3 ~1.7B *base* (non-Instruct) id.
|
||||
- **Eval held-out set must NOT be training data.** `--eval-text` should point at
|
||||
real RU text you set aside before packing, or the PPL number is a lie.
|
||||
- **Re-running is safe.** fetch/clean/pack/synth all overwrite their outputs; CPT
|
||||
resumes with `--resume`. No manual cleanup needed between retries.
|
||||
|
||||
## Hard rules (full list in the plan §9)
|
||||
|
||||
1. CPT corpus = raw text; persona `{response,mood}` is a separate Phase-6 dataset.
|
||||
2. Synthetic ≤ 10%. 3. lr 1e-5, 1 epoch. 4. Base = Qwen3-1.7B-**Base**.
|
||||
5. Never quantize before merge. 6. All sources through corpus_clean.py.
|
||||
7. Don't skip a DONE-CHECK; if CPT loses to raw, stop (Phase 5 gate).
|
||||
@@ -0,0 +1,184 @@
|
||||
"""Phase 6c — generate English persona samples, append to persona_train.jsonl.
|
||||
|
||||
Reuses gen_data.py's CANONICAL_SYSTEM_PROMPT (which handles English input),
|
||||
TOPIC_SEEDS_ENGLISH, and validation. Runs independently.
|
||||
|
||||
Usage:
|
||||
python append_english.py
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import random
|
||||
from itertools import cycle
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
DATA = HERE / "data"
|
||||
random.seed(42)
|
||||
|
||||
sys.path.insert(0, str(HERE))
|
||||
import gen_data
|
||||
|
||||
CANONICAL_SYSTEM_PROMPT = gen_data.CANONICAL_SYSTEM_PROMPT
|
||||
TOPIC_SEEDS_ENGLISH = gen_data.TOPIC_SEEDS_ENGLISH
|
||||
VALID_MOODS = gen_data.VALID_MOODS
|
||||
validate_sample = gen_data.validate_sample
|
||||
|
||||
ROUTER_URL = os.environ.get(
|
||||
"INFERENCE_ROUTER_URL", "http://inference.kvmx.ru"
|
||||
).rstrip("/")
|
||||
|
||||
ROUTER_PAIRS = [
|
||||
("cerebras", "gemma-4-31b"),
|
||||
("cerebras", "zai-glm-4.7"),
|
||||
("github_models", "models/meta/llama-4"),
|
||||
("github_models", "models/mistral/mistral-large-3"),
|
||||
("mistral", "mistral-large-latest"),
|
||||
("mistral", "codestral-latest"),
|
||||
]
|
||||
|
||||
|
||||
def _chat(provider: str, model: str, msgs: list,
|
||||
max_tokens: int = 300, temp: float = 0.8) -> str | None:
|
||||
import time as _time
|
||||
import urllib.request as _req
|
||||
import urllib.error as _err
|
||||
base = ROUTER_URL
|
||||
if not base.endswith("/v1"):
|
||||
base += "/v1"
|
||||
url = f"{base}/chat/completions"
|
||||
headers = {"Content-Type": "application/json", "X-Provider": provider}
|
||||
payload = json.dumps({
|
||||
"model": model, "messages": msgs,
|
||||
"max_tokens": max_tokens, "temperature": temp,
|
||||
}).encode()
|
||||
for attempt in range(3):
|
||||
try:
|
||||
r = _req.Request(url, data=payload, headers=headers)
|
||||
with _req.urlopen(r, timeout=30) as resp:
|
||||
body = json.loads(resp.read())
|
||||
content = (body["choices"][0]["message"]["content"] or "").strip()
|
||||
if content.startswith("```"):
|
||||
start = content.find("\n") + 1
|
||||
end = content.rfind("```")
|
||||
if end > start:
|
||||
content = content[start:end].strip()
|
||||
return content
|
||||
except Exception:
|
||||
_time.sleep(min(2 ** attempt, 8))
|
||||
return None
|
||||
|
||||
|
||||
def _parse_asst(content: str) -> dict | None:
|
||||
try:
|
||||
obj = json.loads(content)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
if not isinstance(obj, dict) or "response" not in obj or "mood" not in obj:
|
||||
return None
|
||||
if obj["mood"] not in VALID_MOODS or not isinstance(obj["response"], str):
|
||||
return None
|
||||
return obj
|
||||
|
||||
|
||||
def flatten_msgs(user_text: str, asst_text: str) -> list[dict]:
|
||||
return [
|
||||
{"role": "system", "content": CANONICAL_SYSTEM_PROMPT},
|
||||
{"role": "user", "content": user_text},
|
||||
{"role": "assistant", "content": asst_text},
|
||||
]
|
||||
|
||||
|
||||
def gen_english(target: int) -> list[dict]:
|
||||
"""Generate English persona samples."""
|
||||
samples = []
|
||||
providers = cycle(ROUTER_PAIRS)
|
||||
pool = TOPIC_SEEDS_ENGLISH * (target // len(TOPIC_SEEDS_ENGLISH) + 1)
|
||||
random.shuffle(pool)
|
||||
|
||||
print(f" english: targeting {target} samples...")
|
||||
for idx, topic in enumerate(pool[:target]):
|
||||
provider, model = next(providers)
|
||||
msgs = [
|
||||
{"role": "system", "content": CANONICAL_SYSTEM_PROMPT},
|
||||
{"role": "user", "content": topic},
|
||||
]
|
||||
content = _chat(provider, model, msgs, max_tokens=300, temp=0.8)
|
||||
if content is None:
|
||||
continue
|
||||
asst_obj = _parse_asst(content)
|
||||
if asst_obj is None:
|
||||
continue
|
||||
# EN→EN bucket: reject majority-Cyrillic responses (language-mirroring failure).
|
||||
resp = asst_obj["response"]
|
||||
cyr = sum(1 for c in resp if "Ѐ" <= c <= "ӿ")
|
||||
lat = sum(1 for c in resp if c.isascii() and c.isalpha())
|
||||
if cyr > lat:
|
||||
continue
|
||||
|
||||
asst_text = json.dumps(asst_obj, ensure_ascii=False)
|
||||
training_msgs = flatten_msgs(topic, asst_text)
|
||||
err = validate_sample(training_msgs)
|
||||
if err:
|
||||
continue
|
||||
samples.append({
|
||||
"messages": training_msgs,
|
||||
"mood": asst_obj["mood"],
|
||||
"response": asst_obj["response"],
|
||||
"source": "generated/english",
|
||||
})
|
||||
if len(samples) % 25 == 0:
|
||||
print(f" ...{len(samples)}/{target} english valid")
|
||||
|
||||
return samples
|
||||
|
||||
|
||||
def main():
|
||||
target = 300
|
||||
print(f"Generating up to {target} English persona samples via router...")
|
||||
|
||||
samples = gen_english(target)
|
||||
print(f"\nGot {len(samples)} valid English samples")
|
||||
|
||||
if not samples:
|
||||
print("No samples generated.")
|
||||
return
|
||||
|
||||
# dedup against existing train
|
||||
train_path = DATA / "persona_train.jsonl"
|
||||
existing: set[str] = set()
|
||||
if train_path.exists():
|
||||
with open(train_path) as f:
|
||||
for line in f:
|
||||
try:
|
||||
d = json.loads(line)
|
||||
msg = d.get("messages", [])
|
||||
if len(msg) >= 3:
|
||||
asst_raw = json.loads(msg[2]["content"])
|
||||
existing.add(asst_raw.get("response", "").strip().lower())
|
||||
except Exception:
|
||||
pass
|
||||
before = len(samples)
|
||||
samples = [s for s in samples if s["response"].strip().lower() not in existing]
|
||||
print(f"Dedup removed {before - len(samples)} (already in train file)")
|
||||
|
||||
# dedup within
|
||||
seen: set[str] = set()
|
||||
deduped = []
|
||||
for s in samples:
|
||||
key = s["response"].strip().lower()
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
deduped.append(s)
|
||||
samples = deduped
|
||||
|
||||
with open(train_path, "a", encoding="utf-8") as f:
|
||||
for s in samples:
|
||||
f.write(json.dumps({"messages": s["messages"]}, ensure_ascii=False) + "\n")
|
||||
|
||||
print(f"Appended {len(samples)} English samples to {train_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,263 @@
|
||||
"""Phase 6b — generate failure samples (tired/confused) and append to persona_train.jsonl.
|
||||
|
||||
Borrows CANONICAL_SYSTEM_PROMPT, failure prompts, and validation from gen_data.py.
|
||||
Runs independently so we can iterate on failure prompting without touching the
|
||||
persona generation pipeline.
|
||||
|
||||
Usage:
|
||||
python append_failures.py
|
||||
|
||||
Environment:
|
||||
INFERENCE_ROUTER_URL (default: http://inference.kvmx.ru)
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import random
|
||||
import re
|
||||
from itertools import cycle
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
DATA = HERE / "data"
|
||||
|
||||
random.seed(42)
|
||||
|
||||
# steal the canonical system prompt and failure prompts from gen_data.py
|
||||
sys.path.insert(0, str(HERE))
|
||||
import gen_data # noqa: E402
|
||||
|
||||
CANONICAL_SYSTEM_PROMPT = gen_data.CANONICAL_SYSTEM_PROMPT
|
||||
FAILURE_PROMPTS_TIRED = gen_data.FAILURE_PROMPTS_TIRED
|
||||
FAILURE_PROMPTS_CONFUSED = gen_data.FAILURE_PROMPTS_CONFUSED
|
||||
VALID_MOODS = gen_data.VALID_MOODS
|
||||
validate_sample = gen_data.validate_sample
|
||||
|
||||
# ── router config ────────────────────────────────────────────────────────
|
||||
ROUTER_URL = os.environ.get(
|
||||
"INFERENCE_ROUTER_URL", "http://inference.kvmx.ru"
|
||||
).rstrip("/")
|
||||
|
||||
# providers to cycle through — brief list of models that work well for RU
|
||||
ROUTER_PAIRS = [
|
||||
# format: (provider, model) — X-Provider header + bare model name
|
||||
("cerebras", "gemma-4-31b"),
|
||||
("cerebras", "zai-glm-4.7"),
|
||||
("github_models", "models/meta/llama-4"),
|
||||
("github_models", "models/mistral/mistral-large-3"),
|
||||
("mistral", "mistral-large-latest"),
|
||||
("mistral", "codestral-latest"),
|
||||
]
|
||||
|
||||
|
||||
# ── router call ──────────────────────────────────────────────────────────
|
||||
def _chat(provider: str, model: str, msgs: list,
|
||||
max_tokens: int = 200, temp: float = 0.7) -> str | None:
|
||||
import time as _time
|
||||
import urllib.request as _req
|
||||
import urllib.error as _err
|
||||
|
||||
base = ROUTER_URL
|
||||
if not base.endswith("/v1"):
|
||||
base += "/v1"
|
||||
url = f"{base}/chat/completions"
|
||||
headers = {"Content-Type": "application/json"}
|
||||
headers["X-Provider"] = provider
|
||||
|
||||
payload = json.dumps({
|
||||
"model": model,
|
||||
"messages": msgs,
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": temp,
|
||||
}).encode()
|
||||
|
||||
for attempt in range(3):
|
||||
try:
|
||||
r = _req.Request(url, data=payload, headers=headers)
|
||||
with _req.urlopen(r, timeout=30) as resp:
|
||||
body = json.loads(resp.read())
|
||||
content = (body["choices"][0]["message"]["content"] or "").strip()
|
||||
if content.startswith("```"):
|
||||
start = content.find("\n") + 1
|
||||
end = content.rfind("```")
|
||||
if end > start:
|
||||
content = content[start:end].strip()
|
||||
return content
|
||||
except _err.HTTPError as e:
|
||||
_time.sleep(min(2 ** attempt, 8))
|
||||
except Exception:
|
||||
_time.sleep(1)
|
||||
return None
|
||||
|
||||
|
||||
def _parse_asst(content: str) -> dict | None:
|
||||
try:
|
||||
obj = json.loads(content)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
if not isinstance(obj, dict):
|
||||
return None
|
||||
if "response" not in obj or "mood" not in obj:
|
||||
return None
|
||||
if obj["mood"] not in VALID_MOODS:
|
||||
return None
|
||||
if not isinstance(obj["response"], str):
|
||||
return None
|
||||
return obj
|
||||
|
||||
|
||||
def flatten_msgs(user_text: str, asst_text: str) -> list[dict]:
|
||||
return [
|
||||
{"role": "system", "content": CANONICAL_SYSTEM_PROMPT},
|
||||
{"role": "user", "content": user_text},
|
||||
{"role": "assistant", "content": asst_text},
|
||||
]
|
||||
|
||||
|
||||
# ── generation ───────────────────────────────────────────────────────────
|
||||
def gen_failures(count: int) -> list[dict]:
|
||||
"""Generate tired + confused samples, return list of validated dicts."""
|
||||
samples = []
|
||||
providers = cycle(ROUTER_PAIRS)
|
||||
|
||||
# tired: 2/3 of count
|
||||
tired_target = count * 2 // 3
|
||||
pool_tired = FAILURE_PROMPTS_TIRED * (tired_target // len(FAILURE_PROMPTS_TIRED) + 1)
|
||||
random.shuffle(pool_tired)
|
||||
|
||||
print(f" tired: targeting {tired_target} samples...")
|
||||
done = 0
|
||||
for prompt_text in pool_tired[:tired_target]:
|
||||
provider, model = next(providers)
|
||||
msgs = [
|
||||
{"role": "system", "content": CANONICAL_SYSTEM_PROMPT},
|
||||
{"role": "user", "content": (
|
||||
f"{prompt_text}\n\n"
|
||||
f"(Это вопрос, на который у тебя нет ответа. "
|
||||
f"Не выдумывай. Честно скажи, что не знаешь или не можешь "
|
||||
f"ответить. Используй mood: tired. Только tired, не neutral, "
|
||||
f"не confused, не thinking — именно tired.)"
|
||||
)},
|
||||
]
|
||||
content = _chat(provider, model, msgs, max_tokens=200, temp=0.7)
|
||||
if content is None:
|
||||
continue
|
||||
asst_obj = _parse_asst(content)
|
||||
if asst_obj is None or asst_obj["mood"] != "tired":
|
||||
continue
|
||||
|
||||
asst_text = json.dumps(asst_obj, ensure_ascii=False)
|
||||
training_msgs = flatten_msgs(prompt_text, asst_text)
|
||||
err = validate_sample(training_msgs)
|
||||
if err:
|
||||
continue
|
||||
samples.append({
|
||||
"messages": training_msgs,
|
||||
"mood": "tired",
|
||||
"response": asst_obj["response"],
|
||||
"source": "generated/failure",
|
||||
})
|
||||
done += 1
|
||||
if done % 25 == 0:
|
||||
print(f" ...{done}/{tired_target} tired valid")
|
||||
|
||||
print(f" tired: {done} valid")
|
||||
|
||||
# confused: 1/3 of count
|
||||
confused_target = count // 3
|
||||
pool_confused = FAILURE_PROMPTS_CONFUSED * (confused_target // len(FAILURE_PROMPTS_CONFUSED) + 1)
|
||||
random.shuffle(pool_confused)
|
||||
|
||||
print(f" confused: targeting {confused_target} samples...")
|
||||
done = 0
|
||||
for prompt_text in pool_confused[:confused_target]:
|
||||
provider, model = next(providers)
|
||||
msgs = [
|
||||
{"role": "system", "content": CANONICAL_SYSTEM_PROMPT},
|
||||
{"role": "user", "content": (
|
||||
f"{prompt_text}\n\n"
|
||||
f"(Это неясный или слишком общий запрос. Ответь коротким "
|
||||
f"уточняющим вопросом. Используй mood: confused. Только "
|
||||
f"confused, не neutral, не tired, не thinking — именно confused.)"
|
||||
)},
|
||||
]
|
||||
content = _chat(provider, model, msgs, max_tokens=200, temp=0.7)
|
||||
if content is None:
|
||||
continue
|
||||
asst_obj = _parse_asst(content)
|
||||
if asst_obj is None or asst_obj["mood"] != "confused":
|
||||
continue
|
||||
|
||||
asst_text = json.dumps(asst_obj, ensure_ascii=False)
|
||||
training_msgs = flatten_msgs(prompt_text, asst_text)
|
||||
err = validate_sample(training_msgs)
|
||||
if err:
|
||||
continue
|
||||
samples.append({
|
||||
"messages": training_msgs,
|
||||
"mood": "confused",
|
||||
"response": asst_obj["response"],
|
||||
"source": "generated/failure",
|
||||
})
|
||||
done += 1
|
||||
if done % 25 == 0:
|
||||
print(f" ...{done}/{confused_target} confused valid")
|
||||
|
||||
print(f" confused: {done} valid")
|
||||
return samples
|
||||
|
||||
|
||||
def main():
|
||||
# target: 750 failure samples (15% of 5000)
|
||||
target = 750
|
||||
print(f"Generating up to {target} failure samples via router...")
|
||||
print(f" Router: {ROUTER_URL}")
|
||||
print(f" Models: {[f'{p}/{m}' for p,m in ROUTER_PAIRS]}")
|
||||
|
||||
samples = gen_failures(target)
|
||||
print(f"\nGot {len(samples)} valid failure samples")
|
||||
|
||||
if not samples:
|
||||
print("No samples generated — nothing to append.")
|
||||
return
|
||||
|
||||
# dedup by response text against existing train data
|
||||
train_path = DATA / "persona_train.jsonl"
|
||||
if train_path.exists():
|
||||
existing_responses: set[str] = set()
|
||||
with open(train_path) as f:
|
||||
for line in f:
|
||||
try:
|
||||
d = json.loads(line)
|
||||
msg = d.get("messages", [])
|
||||
if len(msg) >= 3:
|
||||
asst_raw = json.loads(msg[2]["content"])
|
||||
existing_responses.add(asst_raw.get("response", "").strip().lower())
|
||||
except (json.JSONDecodeError, KeyError, IndexError):
|
||||
pass
|
||||
before = len(samples)
|
||||
samples = [s for s in samples if s["response"].strip().lower() not in existing_responses]
|
||||
print(f"Dedup removed {before - len(samples)} (already in train file)")
|
||||
|
||||
# dedup within new samples
|
||||
seen: set[str] = set()
|
||||
deduped = []
|
||||
for s in samples:
|
||||
key = s["response"].strip().lower()
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
deduped.append(s)
|
||||
samples = deduped
|
||||
|
||||
# append to persona_train.jsonl
|
||||
with open(train_path, "a", encoding="utf-8") as f:
|
||||
for s in samples:
|
||||
f.write(json.dumps({"messages": s["messages"]}, ensure_ascii=False) + "\n")
|
||||
|
||||
print(f"Appended {len(samples)} failure samples to {train_path}")
|
||||
print(f"Mood distribution: tired={sum(1 for s in samples if s['mood']=='tired')}, "
|
||||
f"confused={sum(1 for s in samples if s['mood']=='confused')}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,152 @@
|
||||
"""Read-only audit of the materialized RU CPT corpus and training checkpoints.
|
||||
|
||||
This never rewrites the corpus. It verifies Arrow shape, block lengths/token
|
||||
ranges, clean-source provenance, hashes, and the latest Trainer checkpoint.
|
||||
Output is JSON so the result can be archived and compared across runs.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import random
|
||||
from pathlib import Path
|
||||
|
||||
from datasets import load_from_disk
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from corpus_common import CLEAN, PACKED, has_mixed_script, cyrillic_ratio
|
||||
|
||||
MODEL = "Qwen/Qwen3-1.7B-Base"
|
||||
BLOCK = 2048
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with path.open("rb") as f:
|
||||
for chunk in iter(lambda: f.read(1024 * 1024), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def clean_stats() -> dict:
|
||||
result = {}
|
||||
for path in sorted(CLEAN.glob("*.jsonl")):
|
||||
docs = chars = mixed = 0
|
||||
cyr_sum = 0.0
|
||||
with path.open(encoding="utf-8") as f:
|
||||
for line in f:
|
||||
if not line.strip():
|
||||
continue
|
||||
text = json.loads(line)["text"]
|
||||
docs += 1
|
||||
chars += len(text)
|
||||
mixed += int(has_mixed_script(text))
|
||||
cyr_sum += cyrillic_ratio(text)
|
||||
result[path.stem] = {
|
||||
"docs": docs,
|
||||
"chars": chars,
|
||||
"mean_doc_chars": round(chars / docs, 2) if docs else 0,
|
||||
"mean_cyrillic_ratio": round(cyr_sum / docs, 6) if docs else 0,
|
||||
"mixed_script_docs": mixed,
|
||||
"bytes": path.stat().st_size,
|
||||
"sha256": sha256(path),
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def checkpoint_stats(root: Path) -> dict:
|
||||
states = []
|
||||
for state_path in root.glob("checkpoint-*/trainer_state.json"):
|
||||
state = json.loads(state_path.read_text(encoding="utf-8"))
|
||||
states.append((int(state["global_step"]), state_path, state))
|
||||
if not states:
|
||||
return {"present": False}
|
||||
step, state_path, state = max(states)
|
||||
log = state.get("log_history", [])
|
||||
last = next((x for x in reversed(log) if "loss" in x), {})
|
||||
return {
|
||||
"present": True,
|
||||
"latest_checkpoint": str(state_path.parent),
|
||||
"global_step": step,
|
||||
"max_steps": state.get("max_steps"),
|
||||
"progress_pct": round(100 * step / state["max_steps"], 3),
|
||||
"epoch": state.get("epoch"),
|
||||
"last_loss": last.get("loss"),
|
||||
"last_grad_norm": last.get("grad_norm"),
|
||||
"last_learning_rate": last.get("learning_rate"),
|
||||
"full_weight_model_bytes": (state_path.parent / "model.safetensors").stat().st_size,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--samples", type=int, default=64)
|
||||
ap.add_argument("--seed", type=int, default=20260718)
|
||||
ap.add_argument("--output")
|
||||
ap.add_argument("--checkpoint-root",
|
||||
default=str(Path(__file__).resolve().parent / "Qwen3-1.7B-ru-cpt"))
|
||||
args = ap.parse_args()
|
||||
|
||||
ds = load_from_disk(str(PACKED))
|
||||
if not len(ds):
|
||||
raise SystemExit("packed dataset is empty")
|
||||
rng = random.Random(args.seed)
|
||||
indices = sorted(rng.sample(range(len(ds)), min(args.samples, len(ds))))
|
||||
tokenizer = AutoTokenizer.from_pretrained(MODEL, local_files_only=True)
|
||||
vocab = len(tokenizer)
|
||||
bad_lengths = bad_ids = decoded_mixed = 0
|
||||
samples = []
|
||||
for idx in indices:
|
||||
ids = ds[idx]["input_ids"]
|
||||
bad_lengths += int(len(ids) != BLOCK)
|
||||
bad_ids += sum(1 for token in ids if token < 0 or token >= vocab)
|
||||
text = tokenizer.decode(ids, skip_special_tokens=True)
|
||||
decoded_mixed += int(has_mixed_script(text))
|
||||
samples.append({
|
||||
"index": idx,
|
||||
"tokens": len(ids),
|
||||
"cyrillic_ratio": round(cyrillic_ratio(text), 6),
|
||||
"mixed_script": has_mixed_script(text),
|
||||
"preview": text[:240].replace("\n", " "),
|
||||
})
|
||||
|
||||
arrows = sorted(PACKED.glob("*.arrow"))
|
||||
clean = clean_stats()
|
||||
total_chars = sum(x["chars"] for x in clean.values())
|
||||
report = {
|
||||
"schema_version": 1,
|
||||
"read_only": True,
|
||||
"model": MODEL,
|
||||
"packed": {
|
||||
"path": str(PACKED),
|
||||
"rows": len(ds),
|
||||
"block_tokens": BLOCK,
|
||||
"total_tokens": len(ds) * BLOCK,
|
||||
"arrow_bytes": sum(p.stat().st_size for p in arrows),
|
||||
"arrow_sha256": {p.name: sha256(p) for p in arrows},
|
||||
"sampled_rows": len(indices),
|
||||
"sample_bad_lengths": bad_lengths,
|
||||
"sample_out_of_vocab_ids": bad_ids,
|
||||
"sample_mixed_script_rows": decoded_mixed,
|
||||
},
|
||||
"clean_sources": clean,
|
||||
"clean_char_share_pct": {
|
||||
key: round(100 * value["chars"] / total_chars, 3)
|
||||
for key, value in clean.items()
|
||||
},
|
||||
"provenance_warning": (
|
||||
"Packed Arrow has no per-block source/document metadata; source shares can "
|
||||
"only be reconstructed from the clean inputs, not proven from Arrow alone."
|
||||
),
|
||||
"checkpoint": checkpoint_stats(Path(args.checkpoint_root)),
|
||||
"samples": samples,
|
||||
}
|
||||
rendered = json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
||||
if args.output:
|
||||
Path(args.output).write_text(rendered, encoding="utf-8")
|
||||
print(rendered, end="")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Build the small, human-labelled route holdout set.
|
||||
|
||||
These utterances are not consumed by gen_route_data.py and must never be added
|
||||
to route_train.jsonl. Labels follow the daemon's current JSON-array contract.
|
||||
"""
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from gen_route_data import ROUTE_SYSTEM, valid_actions
|
||||
|
||||
OUT = Path(__file__).resolve().parent / "data" / "route_eval.jsonl"
|
||||
CASES = [
|
||||
("напомни завтра позвонить врачу", [{"intent":"reminder","text":"позвонить врачу"}]),
|
||||
("запиши купить новые батарейки", [{"intent":"note","text":"купить новые батарейки"}]),
|
||||
("я только что полил цветы", [{"intent":"fact","key":"flowers","value":"полил"}]),
|
||||
("что такое обратный прокси", [{"intent":"query","text":"что такое обратный прокси"}]),
|
||||
("перезапусти медиасервер", [{"intent":"act","verb":"перезапустить медиасервер"}]),
|
||||
("который сейчас час", [{"intent":"system"}]),
|
||||
("как у тебя дела", [{"intent":"chat","text":"как у тебя дела"}]),
|
||||
("сохрани адрес мастерской", [{"intent":"note","text":"адрес мастерской"}]),
|
||||
("напомни проверить резервную копию вечером", [{"intent":"reminder","text":"проверить резервную копию"}]),
|
||||
("сервер снова доступен", [{"intent":"fact","key":"server","value":"доступен"}]),
|
||||
("почему контейнер постоянно перезапускается", [{"intent":"query","text":"почему контейнер постоянно перезапускается"}]),
|
||||
("выключи свет на кухне", [{"intent":"act","verb":"выключить свет на кухне"}]),
|
||||
("очисти свою память", [{"intent":"system"}]),
|
||||
("ну и денёк сегодня", [{"intent":"chat","text":"ну и денёк сегодня"}]),
|
||||
("запомни, что код от калитки изменился", [{"intent":"note","text":"код от калитки изменился"}]),
|
||||
("я закончил отчёт", [{"intent":"fact","key":"report","value":"закончил"}]),
|
||||
("покажи свободное место на диске", [{"intent":"act","verb":"показать свободное место на диске"}]),
|
||||
("какая погода ожидается утром", [{"intent":"query","text":"какая погода ожидается утром"}]),
|
||||
("напомни через час достать бельё", [{"intent":"reminder","text":"достать бельё"}]),
|
||||
("приятно снова тебя слышать", [{"intent":"chat","text":"приятно снова тебя слышать"}]),
|
||||
("напомни купить молоко и запиши что кофе закончился", [
|
||||
{"intent":"reminder","text":"купить молоко"},
|
||||
{"intent":"note","text":"кофе закончился"}]),
|
||||
("проверь диск и потом напомни посмотреть результат", [
|
||||
{"intent":"act","verb":"проверить диск"},
|
||||
{"intent":"reminder","text":"посмотреть результат"}]),
|
||||
("я принял лекарство, запиши дозировку пять миллиграммов", [
|
||||
{"intent":"fact","key":"medicine","value":"принял"},
|
||||
{"intent":"note","text":"дозировка пять миллиграммов"}]),
|
||||
]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
lines = []
|
||||
for utterance, actions in CASES:
|
||||
assert valid_actions(actions)
|
||||
lines.append(json.dumps({"messages": [
|
||||
{"role":"system","content":ROUTE_SYSTEM},
|
||||
{"role":"user","content":utterance},
|
||||
{"role":"assistant","content":json.dumps(actions, ensure_ascii=False)},
|
||||
], "intents":[a["intent"] for a in actions], "source":"human/route-eval"},
|
||||
ensure_ascii=False))
|
||||
OUT.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
print(f"wrote {len(lines)} held-out route cases -> {OUT}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Fail if the route labelling prompt differs from Maven's deployed prompt."""
|
||||
import argparse
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from gen_route_data import ROUTE_SYSTEM
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--maven", default="/mnt/server/home/kami/apps/Maven")
|
||||
args = ap.parse_args()
|
||||
go = Path(args.maven, "internal/router/llmrouter.go").read_text(encoding="utf-8")
|
||||
match = re.search(r"const routeSystem = `([\s\S]*?)`\n", go)
|
||||
if not match:
|
||||
raise SystemExit("routeSystem not found in Go source")
|
||||
if match.group(1) != ROUTE_SYSTEM:
|
||||
raise SystemExit("route prompt drift: update Go and gen_route_data.py together")
|
||||
print("route prompt parity OK")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Phase 2.3 — clean every raw bucket → data/cpt_clean/<bucket>.jsonl
|
||||
|
||||
Pipeline order (plan §2.3), applied per doc, then a global cross-bucket dedup:
|
||||
1. ftfy unicode fix (NFC)
|
||||
2. fastText language id — drop if top lang != ru or p(ru) < 0.65
|
||||
3. mixed-script rejection — drop docs with homoglyph-contaminated words
|
||||
4. boilerplate/length — strip HTML, collapse whitespace, drop <200 or >50k chars,
|
||||
drop docs with >30% non-letter chars
|
||||
5. MinHash-LSH near-dup removal across ALL buckets (the biggest quality lever)
|
||||
6. light PII scrub (phones/emails) — logs bucket has real data
|
||||
|
||||
Usage: python corpus_clean.py # all buckets present in data/cpt_raw/
|
||||
python corpus_clean.py wiki synth # specific buckets
|
||||
"""
|
||||
import sys
|
||||
import re
|
||||
import json
|
||||
import html
|
||||
|
||||
import ftfy
|
||||
from datasketch import MinHash, MinHashLSH
|
||||
|
||||
from corpus_common import RAW, CLEAN, has_mixed_script, lang_prob
|
||||
|
||||
MIN_CHARS, MAX_CHARS = 200, 50_000
|
||||
NON_LETTER_MAX = 0.30
|
||||
LID_MIN = 0.65
|
||||
LSH_THRESHOLD = 0.8
|
||||
NUM_PERM = 128
|
||||
|
||||
_TAG = re.compile(r"<[^>]+>")
|
||||
_WS = re.compile(r"[ \t ]+")
|
||||
_NL = re.compile(r"\n{3,}")
|
||||
_LETTER = re.compile(r"[^\W\d_]", re.UNICODE)
|
||||
_EMAIL = re.compile(r"[\w.+-]+@[\w-]+\.[\w.-]+")
|
||||
_PHONE = re.compile(r"(?<!\d)(?:\+7|8)[\s(-]*\d{3}[\s)-]*\d{3}[\s-]*\d{2}[\s-]*\d{2}(?!\d)")
|
||||
|
||||
|
||||
def normalize(text: str) -> str:
|
||||
text = ftfy.fix_text(text)
|
||||
text = html.unescape(text)
|
||||
text = _TAG.sub(" ", text)
|
||||
text = _WS.sub(" ", text)
|
||||
text = _NL.sub("\n\n", text)
|
||||
return text.strip()
|
||||
|
||||
|
||||
def non_letter_ratio(text: str) -> float:
|
||||
letters = len(_LETTER.findall(text))
|
||||
return 1.0 - (letters / len(text)) if text else 1.0
|
||||
|
||||
|
||||
def scrub_pii(text: str) -> str:
|
||||
text = _EMAIL.sub("<email>", text)
|
||||
text = _PHONE.sub("<phone>", text)
|
||||
return text
|
||||
|
||||
|
||||
def doc_minhash(text: str) -> MinHash:
|
||||
m = MinHash(num_perm=NUM_PERM)
|
||||
for tok in text.lower().split():
|
||||
m.update(tok.encode("utf-8"))
|
||||
return m
|
||||
|
||||
|
||||
def clean_bucket(bucket: str, lsh: MinHashLSH, seen: dict) -> int:
|
||||
src = RAW / bucket / "shard-000.jsonl"
|
||||
if not src.exists():
|
||||
print(f"[{bucket}] no raw shard at {src}; skip")
|
||||
return 0
|
||||
out = CLEAN / f"{bucket}.jsonl"
|
||||
kept = dropped = 0
|
||||
with open(src, encoding="utf-8") as fin, open(out, "w", encoding="utf-8") as fout:
|
||||
for i, line in enumerate(fin):
|
||||
try:
|
||||
text = json.loads(line)["text"]
|
||||
except (json.JSONDecodeError, KeyError):
|
||||
dropped += 1
|
||||
continue
|
||||
text = normalize(text)
|
||||
# 4. length / boilerplate
|
||||
if not (MIN_CHARS <= len(text) <= MAX_CHARS) or non_letter_ratio(text) > NON_LETTER_MAX:
|
||||
dropped += 1
|
||||
continue
|
||||
# 3. mixed script
|
||||
if has_mixed_script(text):
|
||||
dropped += 1
|
||||
continue
|
||||
# 2. language id
|
||||
try:
|
||||
lang, p = lang_prob(text)
|
||||
if lang != "ru" or p < LID_MIN:
|
||||
dropped += 1
|
||||
continue
|
||||
except FileNotFoundError:
|
||||
raise SystemExit("lid.176.bin missing — see corpus_common.LID_PATH")
|
||||
# 5. dedup
|
||||
m = doc_minhash(text)
|
||||
if lsh.query(m):
|
||||
dropped += 1
|
||||
continue
|
||||
key = f"{bucket}:{i}"
|
||||
lsh.insert(key, m)
|
||||
seen[key] = True
|
||||
# 6. pii
|
||||
text = scrub_pii(text)
|
||||
fout.write(json.dumps({"text": text}, ensure_ascii=False) + "\n")
|
||||
kept += 1
|
||||
if (kept + dropped) % 5000 == 0:
|
||||
print(f"[{bucket}] kept {kept} dropped {dropped}")
|
||||
print(f"[{bucket}] DONE kept {kept} dropped {dropped} -> {out}")
|
||||
return kept
|
||||
|
||||
|
||||
def main():
|
||||
buckets = sys.argv[1:] or [p.name for p in sorted(RAW.iterdir()) if p.is_dir()]
|
||||
# one shared LSH across all buckets = global dedup (plan §2.3 rule 5)
|
||||
lsh = MinHashLSH(threshold=LSH_THRESHOLD, num_perm=NUM_PERM)
|
||||
seen = {}
|
||||
total = 0
|
||||
for b in buckets:
|
||||
total += clean_bucket(b, lsh, seen)
|
||||
print(f"[clean] TOTAL kept {total} docs across {len(buckets)} buckets")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,100 @@
|
||||
"""Shared paths + cleaning helpers for the RU-CPT corpus pipeline.
|
||||
|
||||
The mixed-script / Cyrillic-cleanliness rule lives here ONCE and is reused by
|
||||
corpus_clean.py, corpus_synth.py, and eval_cyrillic.py. See
|
||||
docs/plans/2026-07-11-ru-cpt-base.md (§2.3 rule 3) in the Maven repo.
|
||||
|
||||
Run `python corpus_common.py` for a self-check of the mixed-script detector.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
# ── env / cache (mirror train_rocm.py) ──────────────────────────────────────
|
||||
os.environ.setdefault("HF_HOME", "/mnt/D/.cache/huggingface")
|
||||
os.environ.setdefault("HF_DATASETS_CACHE", "/mnt/D/.cache/huggingface/datasets")
|
||||
os.environ.setdefault("TMPDIR", "/mnt/D/tmp")
|
||||
os.makedirs(os.environ["TMPDIR"], exist_ok=True)
|
||||
|
||||
# ── corpus dir layout ───────────────────────────────────────────────────────
|
||||
DATA = Path(__file__).resolve().parent / "data"
|
||||
RAW = DATA / "cpt_raw" # corpus_fetch.py / corpus_synth.py output
|
||||
CLEAN = DATA / "cpt_clean" # corpus_clean.py output (JSONL {"text":...})
|
||||
PACKED = DATA / "cpt_packed" # corpus_pack.py output (HF Arrow, 2048-tok blocks)
|
||||
for d in (RAW, CLEAN, PACKED):
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# ── the corpus mix (plan §2.1) — bucket -> target token share ───────────────
|
||||
TARGET_TOKENS = 300_000_000
|
||||
BUCKET_SHARE = { # sums to 1.0; synth capped at 0.10 (hard rule)
|
||||
"culturax": 0.60,
|
||||
"wiki": 0.17,
|
||||
"books": 0.11,
|
||||
"synth": 0.10,
|
||||
"logs": 0.02,
|
||||
}
|
||||
SYNTH_CAP = 0.10
|
||||
|
||||
# ── Cyrillic cleanliness (plan §2.3 rule 3) ─────────────────────────────────
|
||||
_WORD = re.compile(r"[^\s]+")
|
||||
_CYR = re.compile(r"[а-яёА-ЯЁ]")
|
||||
_LAT = re.compile(r"[a-zA-Z]")
|
||||
|
||||
|
||||
def is_mixed_script_word(word: str) -> bool:
|
||||
"""True if a single token mixes Cyrillic and Latin letters (homoglyph swap)."""
|
||||
return bool(_CYR.search(word) and _LAT.search(word))
|
||||
|
||||
|
||||
def has_mixed_script(text: str) -> bool:
|
||||
"""True if ANY word in the text is homoglyph-contaminated."""
|
||||
return any(is_mixed_script_word(w) for w in _WORD.findall(text))
|
||||
|
||||
|
||||
def cyrillic_ratio(text: str) -> float:
|
||||
"""Fraction of letters that are Cyrillic (0..1). Empty/letterless -> 0."""
|
||||
cyr = len(_CYR.findall(text))
|
||||
lat = len(_LAT.findall(text))
|
||||
tot = cyr + lat
|
||||
return cyr / tot if tot else 0.0
|
||||
|
||||
|
||||
# ── fastText language id (lazy singleton; download lid.176.bin, plan §2.3) ──
|
||||
_LID = None
|
||||
LID_PATH = os.environ.get("FASTTEXT_LID", str(DATA / "lid.176.bin"))
|
||||
|
||||
|
||||
def lang_prob(text: str):
|
||||
"""Return (lang_code, prob) for the dominant language, or (None, 0.0)."""
|
||||
global _LID
|
||||
if _LID is None:
|
||||
import fasttext
|
||||
if not Path(LID_PATH).exists():
|
||||
raise FileNotFoundError(
|
||||
f"fastText lid.176.bin not found at {LID_PATH}. Download from "
|
||||
"https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin "
|
||||
"or set FASTTEXT_LID env var."
|
||||
)
|
||||
# numpy 2.x compat: fasttext uses np.array(..., copy=False) which
|
||||
# raises on numpy >=2 when a copy is needed. patch its np reference.
|
||||
_orig_np_array = fasttext.FastText.np.array
|
||||
def _patched_array(*args, **kwargs):
|
||||
kwargs.pop("copy", None)
|
||||
return _orig_np_array(*args, **kwargs)
|
||||
fasttext.FastText.np.array = _patched_array
|
||||
_LID = fasttext.load_model(LID_PATH)
|
||||
labels, probs = _LID.predict(text.replace("\n", " ")[:2000])
|
||||
return labels[0].replace("__label__", ""), float(probs[0])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# self-check: the homoglyph detector is the whole point — assert it works.
|
||||
assert is_mixed_script_word("privет") # latin+cyrillic in one word
|
||||
assert is_mixed_script_word("Марkет") # cyrillic 'Мар' + latin 'k'
|
||||
assert not is_mixed_script_word("привет") # pure cyrillic
|
||||
assert not is_mixed_script_word("hello") # pure latin
|
||||
assert not is_mixed_script_word("http://x.ru") # pure latin url — keep
|
||||
assert has_mixed_script("это Марkет здесь")
|
||||
assert not has_mixed_script("это чистый русский текст with english words")
|
||||
assert abs(cyrillic_ratio("привет hello") - 0.5) < 0.05
|
||||
print("corpus_common self-check OK ->", DATA)
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Phase 2.2 — pull open RU datasets → data/cpt_raw/<bucket>/*.jsonl
|
||||
|
||||
Streams from HuggingFace so we never materialize full dumps. Each bucket
|
||||
over-pulls ~1.5x its token target (cleaning drops 20-40%). Writes newline JSONL
|
||||
{"text": "..."} shards; corpus_clean.py consumes them.
|
||||
|
||||
Usage:
|
||||
huggingface-cli login # accept CulturaX/OSCAR licenses first
|
||||
python corpus_fetch.py culturax # one bucket
|
||||
python corpus_fetch.py all # every open bucket (not synth/logs)
|
||||
|
||||
Buckets synth (corpus_synth.py) and logs (corpus_logs.py) are handled separately.
|
||||
"""
|
||||
import sys
|
||||
import json
|
||||
|
||||
from datasets import load_dataset
|
||||
|
||||
from corpus_common import RAW, TARGET_TOKENS, BUCKET_SHARE
|
||||
|
||||
# rough chars-per-token for RU under the Qwen tokenizer; only used to size pulls.
|
||||
CHARS_PER_TOK = 3.5
|
||||
OVERPULL = 1.5
|
||||
|
||||
# bucket -> (loader kwargs, function mapping a row to its text string)
|
||||
SOURCES = {
|
||||
"culturax": (
|
||||
dict(path="allenai/c4", name="ru", split="train", streaming=True),
|
||||
lambda r: r.get("text", ""),
|
||||
),
|
||||
"wiki": (
|
||||
dict(path="wikimedia/wikipedia", name="20231101.ru", split="train", streaming=True),
|
||||
lambda r: r.get("text", ""),
|
||||
),
|
||||
"books": (
|
||||
dict(path="IlyaGusev/gazeta", split="train", streaming=True),
|
||||
lambda r: r.get("text", ""),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def target_chars(bucket: str) -> int:
|
||||
return int(TARGET_TOKENS * BUCKET_SHARE[bucket] * CHARS_PER_TOK * OVERPULL)
|
||||
|
||||
|
||||
def fetch(bucket: str):
|
||||
if bucket not in SOURCES:
|
||||
raise SystemExit(f"unknown bucket {bucket}; known: {list(SOURCES)}")
|
||||
budget = target_chars(bucket)
|
||||
out_dir = RAW / bucket
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
out_path = out_dir / "shard-000.jsonl"
|
||||
|
||||
kw, getter = SOURCES[bucket]
|
||||
ds = load_dataset(**kw)
|
||||
|
||||
written = 0
|
||||
n = 0
|
||||
with open(out_path, "w", encoding="utf-8") as f:
|
||||
for row in ds:
|
||||
text = (getter(row) or "").strip()
|
||||
if len(text) < 200: # skip fragments early
|
||||
continue
|
||||
f.write(json.dumps({"text": text}, ensure_ascii=False) + "\n")
|
||||
written += len(text)
|
||||
n += 1
|
||||
if n % 5000 == 0:
|
||||
print(f"[{bucket}] {n} docs, {written/1e6:.1f}M chars / {budget/1e6:.1f}M target")
|
||||
if written >= budget:
|
||||
break
|
||||
print(f"[{bucket}] DONE {n} docs, {written/1e6:.1f}M chars -> {out_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = sys.argv[1:] or ["all"]
|
||||
buckets = list(SOURCES) if args == ["all"] else args
|
||||
for b in buckets:
|
||||
fetch(b)
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Phase 2.5 — mine Maven's real dialogue history → data/cpt_raw/logs/
|
||||
|
||||
Highest domain-value bucket but smallest (~2%, ~6M tok). Source format is unknown
|
||||
here, so point MAVEN_LOGS at your dialogue store and adjust `extract()` to your
|
||||
schema. Emits raw RU utterances as {"text": ...}; corpus_clean.py handles the rest
|
||||
(langID drops EN turns, dedup, PII scrub).
|
||||
|
||||
Usage:
|
||||
MAVEN_LOGS=/path/to/dialogue.jsonl python corpus_logs.py
|
||||
"""
|
||||
import os
|
||||
import json
|
||||
|
||||
from corpus_common import RAW, cyrillic_ratio
|
||||
|
||||
|
||||
def extract(obj):
|
||||
"""Return utterance text from one log record. ADJUST to your schema.
|
||||
|
||||
Handles common shapes: {"text":...}, {"content":...},
|
||||
{"messages":[{"role","content"}...]}, or a bare string.
|
||||
"""
|
||||
if isinstance(obj, str):
|
||||
return [obj]
|
||||
if not isinstance(obj, dict):
|
||||
return []
|
||||
if "messages" in obj:
|
||||
return [m.get("content", "") for m in obj["messages"] if m.get("content")]
|
||||
for k in ("text", "content", "utterance", "body"):
|
||||
if obj.get(k):
|
||||
return [obj[k]]
|
||||
return []
|
||||
|
||||
|
||||
def main():
|
||||
src = os.environ.get("MAVEN_LOGS")
|
||||
if not src or not os.path.exists(src):
|
||||
raise SystemExit("set MAVEN_LOGS to your dialogue history (jsonl); "
|
||||
"this bucket is optional — skip if unavailable")
|
||||
|
||||
out = RAW / "logs" / "shard-000.jsonl"
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
kept = 0
|
||||
with open(src, encoding="utf-8") as fin, open(out, "w", encoding="utf-8") as fout:
|
||||
for line in fin:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
obj = line
|
||||
for utt in extract(obj):
|
||||
utt = (utt or "").strip()
|
||||
# keep only substantive Russian turns; clean.py enforces the rest
|
||||
if len(utt) >= 40 and cyrillic_ratio(utt) >= 0.5:
|
||||
fout.write(json.dumps({"text": utt}, ensure_ascii=False) + "\n")
|
||||
kept += 1
|
||||
print(f"[logs] kept {kept} utterances -> {out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Phase 2.6 — tokenize + pack cleaned corpus → data/cpt_packed/ (HF Arrow)
|
||||
|
||||
Concatenates all data/cpt_clean/*.jsonl, tokenizes with the Qwen3 tokenizer,
|
||||
inserts EOS between docs, and packs into fixed 2048-token blocks (ragged tail
|
||||
dropped). Output columns: input_ids (+ labels==input_ids added by the collator
|
||||
at train time). corpus_pack is the final corpus artifact train_cpt.py loads.
|
||||
|
||||
Usage: python corpus_pack.py
|
||||
"""
|
||||
import json
|
||||
|
||||
from datasets import Dataset
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from corpus_common import CLEAN, PACKED
|
||||
|
||||
MODEL = "Qwen/Qwen3-1.7B-Base"
|
||||
BLOCK = 2048
|
||||
|
||||
|
||||
def iter_texts():
|
||||
for fp in sorted(CLEAN.glob("*.jsonl")):
|
||||
with open(fp, encoding="utf-8") as f:
|
||||
for ln in f:
|
||||
ln = ln.strip()
|
||||
if ln:
|
||||
yield json.loads(ln)["text"]
|
||||
|
||||
|
||||
def main():
|
||||
tok = AutoTokenizer.from_pretrained(MODEL)
|
||||
eos = tok.eos_token_id
|
||||
if eos is None:
|
||||
raise SystemExit("tokenizer has no eos_token_id")
|
||||
|
||||
buf = []
|
||||
blocks = []
|
||||
n_docs = 0
|
||||
for text in iter_texts():
|
||||
buf.extend(tok(text, add_special_tokens=False)["input_ids"])
|
||||
buf.append(eos)
|
||||
n_docs += 1
|
||||
while len(buf) >= BLOCK:
|
||||
blocks.append(buf[:BLOCK])
|
||||
buf = buf[BLOCK:]
|
||||
if n_docs % 10000 == 0:
|
||||
print(f"[pack] {n_docs} docs -> {len(blocks)} blocks "
|
||||
f"({len(blocks)*BLOCK/1e6:.0f}M tok)")
|
||||
|
||||
ds = Dataset.from_dict({"input_ids": blocks})
|
||||
ds.save_to_disk(str(PACKED))
|
||||
total_tok = len(blocks) * BLOCK
|
||||
print(f"[pack] DONE {n_docs} docs -> {len(blocks)} blocks = {total_tok/1e6:.0f}M tokens")
|
||||
print(f"[pack] saved {PACKED} (target ~300M ± 50M)")
|
||||
|
||||
# sanity: decode a random block, must read as Russian
|
||||
import random
|
||||
sample = tok.decode(random.choice(blocks))
|
||||
print(f"\n--- random packed block (verify Russian) ---\n{sample[:400]}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Phase 2.3/2.4 DONE-CHECK — per-bucket stats + eyeball sample.
|
||||
|
||||
Reports doc/char counts per cleaned bucket, the synth share (must be ≤10%), and
|
||||
prints N random cleaned docs so a human confirms they read as clean Russian with
|
||||
no homoglyph words.
|
||||
|
||||
Usage: python corpus_stats.py [--sample 5]
|
||||
"""
|
||||
import sys
|
||||
import json
|
||||
import random
|
||||
|
||||
from corpus_common import CLEAN, has_mixed_script, cyrillic_ratio, SYNTH_CAP
|
||||
|
||||
|
||||
def load(bucket_path):
|
||||
with open(bucket_path, encoding="utf-8") as f:
|
||||
return [json.loads(ln)["text"] for ln in f if ln.strip()]
|
||||
|
||||
|
||||
def main():
|
||||
n_sample = 5
|
||||
if "--sample" in sys.argv:
|
||||
n_sample = int(sys.argv[sys.argv.index("--sample") + 1])
|
||||
|
||||
files = sorted(CLEAN.glob("*.jsonl"))
|
||||
if not files:
|
||||
raise SystemExit(f"no cleaned buckets in {CLEAN}; run corpus_clean.py")
|
||||
|
||||
totals = {"docs": 0, "chars": 0}
|
||||
per = {}
|
||||
for fp in files:
|
||||
texts = load(fp)
|
||||
chars = sum(len(t) for t in texts)
|
||||
per[fp.stem] = (len(texts), chars)
|
||||
totals["docs"] += len(texts)
|
||||
totals["chars"] += chars
|
||||
|
||||
print(f"{'bucket':<14}{'docs':>10}{'chars(M)':>12}{'share%':>9}")
|
||||
for b, (d, c) in per.items():
|
||||
share = 100 * c / totals["chars"] if totals["chars"] else 0
|
||||
flag = " <-- OVER CAP" if b == "synth" and share > SYNTH_CAP * 100 else ""
|
||||
print(f"{b:<14}{d:>10}{c/1e6:>12.1f}{share:>9.1f}{flag}")
|
||||
print(f"{'TOTAL':<14}{totals['docs']:>10}{totals['chars']/1e6:>12.1f}")
|
||||
# crude token estimate (~3.5 chars/tok RU)
|
||||
print(f"~est tokens: {totals['chars']/3.5/1e6:.0f}M (target ~300M)")
|
||||
|
||||
# eyeball sample — the actual DONE-CHECK
|
||||
print(f"\n--- {n_sample} random cleaned docs (verify clean Russian) ---")
|
||||
allpaths = list(files)
|
||||
for _ in range(n_sample):
|
||||
fp = random.choice(allpaths)
|
||||
texts = load(fp)
|
||||
if not texts:
|
||||
continue
|
||||
t = random.choice(texts)
|
||||
assert not has_mixed_script(t), f"MIXED SCRIPT LEAKED in {fp.stem}: {t[:120]}"
|
||||
print(f"\n[{fp.stem}] cyr={cyrillic_ratio(t):.2f}\n{t[:400]}")
|
||||
print("\nOK — no mixed-script leaked into sample.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,128 @@
|
||||
"""Phase 2.4 — router-generated synthetic RU corpus → data/cpt_raw/synth/
|
||||
|
||||
Generates raw natural-Russian prose/dialogue (NOT JSON, NOT {response,mood}
|
||||
persona data — that is Phase 6, a different dataset) to cover registers the open
|
||||
corpora lack: smart-home / assistant / conversational RU.
|
||||
|
||||
Uses an OpenAI-compatible endpoint = the user's free-provider router. Configure:
|
||||
export ROUTER_BASE_URL="https://your-router/v1"
|
||||
export ROUTER_API_KEY="..."
|
||||
export ROUTER_MODELS="model-a,model-b,model-c" # rotated for diversity
|
||||
|
||||
HARD CAP: stops at 10% of TARGET_TOKENS (plan §2.4 / hard rule 2). Synthetic-heavy
|
||||
CPT collapses. Rotate models so it isn't one model's fingerprint.
|
||||
|
||||
Every generation is filtered here (langID + mixed-script) before write; the full
|
||||
cleaning + cross-corpus dedup still happens in corpus_clean.py.
|
||||
|
||||
Usage: python corpus_synth.py
|
||||
"""
|
||||
import os
|
||||
import json
|
||||
import random
|
||||
import itertools
|
||||
|
||||
from openai import OpenAI
|
||||
|
||||
from corpus_common import (
|
||||
RAW, TARGET_TOKENS, SYNTH_CAP,
|
||||
has_mixed_script, cyrillic_ratio, lang_prob,
|
||||
)
|
||||
|
||||
CHARS_PER_TOK = 3.5
|
||||
TOPICS_FILE = os.path.join(os.path.dirname(__file__), "data", "topics.txt")
|
||||
|
||||
STYLES = [
|
||||
"живой разговорный монолог",
|
||||
"диалог двух людей",
|
||||
"подробное объяснение",
|
||||
"личная история от первого лица",
|
||||
"инструкция или совет по бытовой теме",
|
||||
]
|
||||
# domains open corpora under-represent — the reason synth exists at all.
|
||||
EXTRA_TOPICS = [
|
||||
"умный дом: свет, розетки, сценарии", "голосовой помощник в квартире",
|
||||
"напоминания и списки дел на день", "погода и планы на выходные",
|
||||
"готовка ужина и покупки продуктов", "музыка и подкасты дома",
|
||||
"будильники, таймеры и распорядок дня", "уборка и бытовые заботы",
|
||||
]
|
||||
|
||||
PROMPT = (
|
||||
"Напиши развёрнутый естественный текст на русском языке на тему: «{topic}». "
|
||||
"Стиль: {style}. Объём 400–800 слов. Только связный русский текст без разметки, "
|
||||
"без английских слов, без списков-маркеров и без JSON."
|
||||
)
|
||||
|
||||
|
||||
def load_topics():
|
||||
topics = list(EXTRA_TOPICS)
|
||||
if os.path.exists(TOPICS_FILE):
|
||||
with open(TOPICS_FILE, encoding="utf-8") as f:
|
||||
topics += [ln.strip() for ln in f if ln.strip()]
|
||||
random.shuffle(topics)
|
||||
return topics
|
||||
|
||||
|
||||
def accept(text: str) -> bool:
|
||||
"""Local gate before write (cheap). Full clean+dedup is corpus_clean.py."""
|
||||
if len(text) < 400:
|
||||
return False
|
||||
if has_mixed_script(text):
|
||||
return False
|
||||
if cyrillic_ratio(text) < 0.85:
|
||||
return False
|
||||
try:
|
||||
lang, p = lang_prob(text)
|
||||
except FileNotFoundError:
|
||||
lang, p = "ru", 1.0 # langID optional here; corpus_clean.py enforces it
|
||||
return lang == "ru" and p >= 0.65
|
||||
|
||||
|
||||
def main():
|
||||
base_url = os.environ.get("ROUTER_BASE_URL")
|
||||
api_key = os.environ.get("ROUTER_API_KEY", "none")
|
||||
models = [m for m in os.environ.get("ROUTER_MODELS", "").split(",") if m]
|
||||
if not base_url or not models:
|
||||
raise SystemExit("set ROUTER_BASE_URL and ROUTER_MODELS (see module docstring)")
|
||||
|
||||
client = OpenAI(base_url=base_url, api_key=api_key)
|
||||
budget_chars = int(TARGET_TOKENS * SYNTH_CAP * CHARS_PER_TOK) # hard cap
|
||||
out_path = RAW / "synth" / "shard-000.jsonl"
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
topics = load_topics()
|
||||
model_cycle = itertools.cycle(models)
|
||||
written = kept = tried = 0
|
||||
|
||||
with open(out_path, "w", encoding="utf-8") as f:
|
||||
for topic in itertools.cycle(topics):
|
||||
if written >= budget_chars:
|
||||
break
|
||||
style = random.choice(STYLES)
|
||||
model = next(model_cycle)
|
||||
tried += 1
|
||||
try:
|
||||
resp = client.chat.completions.create(
|
||||
model=model,
|
||||
messages=[{"role": "user",
|
||||
"content": PROMPT.format(topic=topic, style=style)}],
|
||||
temperature=0.9, max_tokens=1200,
|
||||
)
|
||||
text = (resp.choices[0].message.content or "").strip()
|
||||
except Exception as e:
|
||||
print(f"[synth] gen error on {model}: {e}")
|
||||
continue
|
||||
if not accept(text):
|
||||
continue
|
||||
f.write(json.dumps({"text": text, "model": model}, ensure_ascii=False) + "\n")
|
||||
written += len(text)
|
||||
kept += 1
|
||||
if kept % 50 == 0:
|
||||
print(f"[synth] kept {kept}/{tried} tried, "
|
||||
f"{written/1e6:.1f}M / {budget_chars/1e6:.1f}M chars cap")
|
||||
|
||||
print(f"[synth] DONE kept {kept}/{tried}, {written/1e6:.1f}M chars -> {out_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,61 @@
|
||||
"""Compare raw and CPT evaluation JSON and enforce the post-CPT gate."""
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def repetition_4gram_rate(report: dict) -> float:
|
||||
rates = []
|
||||
for lang in ("ru", "en"):
|
||||
for item in report["generation"].get(lang, []):
|
||||
words = re.findall(r"\w+", item["output"].lower())
|
||||
grams = [tuple(words[i:i + 4]) for i in range(max(0, len(words) - 3))]
|
||||
rates.append(1 - len(set(grams)) / len(grams) if grams else 0.0)
|
||||
return sum(rates) / len(rates) if rates else 1.0
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--raw", required=True)
|
||||
ap.add_argument("--cpt", required=True)
|
||||
ap.add_argument("--max-ru-ppl-ratio", type=float, default=0.98)
|
||||
ap.add_argument("--max-en-ppl-ratio", type=float, default=1.10)
|
||||
ap.add_argument("--max-cyrillic-drop", type=float, default=0.0)
|
||||
args = ap.parse_args()
|
||||
raw = json.loads(Path(args.raw).read_text())
|
||||
cpt = json.loads(Path(args.cpt).read_text())
|
||||
raw_repetition = repetition_4gram_rate(raw)
|
||||
cpt_repetition = repetition_4gram_rate(cpt)
|
||||
|
||||
checks = {
|
||||
"ru_ppl_improved_by_2pct": cpt["perplexity"]["ru"] <= raw["perplexity"]["ru"] * args.max_ru_ppl_ratio,
|
||||
"en_ppl_regression_within_10pct": cpt["perplexity"]["en"] <= raw["perplexity"]["en"] * args.max_en_ppl_ratio,
|
||||
"cyrillic_validity_not_worse": cpt["generation"]["ru_valid_pct"] >= raw["generation"]["ru_valid_pct"] - args.max_cyrillic_drop,
|
||||
"english_retained": bool(cpt["generation"]["en_retained"]),
|
||||
"generation_repetition_not_degraded": (
|
||||
cpt_repetition <= max(0.15, raw_repetition + 0.05)
|
||||
),
|
||||
}
|
||||
report = {
|
||||
"pass": all(checks.values()),
|
||||
"checks": checks,
|
||||
"raw": raw.get("model"),
|
||||
"cpt": cpt.get("model"),
|
||||
"ratios": {
|
||||
"ru_ppl": cpt["perplexity"]["ru"] / raw["perplexity"]["ru"],
|
||||
"en_ppl": cpt["perplexity"]["en"] / raw["perplexity"]["en"],
|
||||
},
|
||||
"generation_repetition_4gram_rate": {
|
||||
"raw": raw_repetition,
|
||||
"cpt": cpt_repetition,
|
||||
"maximum_allowed": max(0.15, raw_repetition + 0.05),
|
||||
},
|
||||
}
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
sys.exit(0 if report["pass"] else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,107 @@
|
||||
"""Evaluate persona and route contracts after Qwen3 SFT.
|
||||
|
||||
Reports syntax/shape validity, persona mood accuracy, route intent sequence
|
||||
exact match, per-action intent accuracy, and slot exact match.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
os.environ.setdefault("HF_HOME", "/mnt/D/.cache/huggingface")
|
||||
os.environ.setdefault("HF_DATASETS_CACHE", "/mnt/D/.cache/huggingface/datasets")
|
||||
|
||||
import torch
|
||||
from peft import PeftModel
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
|
||||
|
||||
def rows(path: str, limit: int) -> list[dict]:
|
||||
out = [json.loads(line) for line in Path(path).read_text(encoding="utf-8").splitlines()
|
||||
if line.strip()]
|
||||
return out[:limit] if limit else out
|
||||
|
||||
|
||||
def normalized_route(value):
|
||||
if isinstance(value, dict):
|
||||
value = [value]
|
||||
return value if isinstance(value, list) else None
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def complete(tok, model, messages, max_new_tokens):
|
||||
prompt = tok.apply_chat_template(messages, tokenize=False, add_generation_prompt=True,
|
||||
enable_thinking=False)
|
||||
ids = tok(prompt, add_special_tokens=False, return_tensors="pt").input_ids.to(model.device)
|
||||
out = model.generate(input_ids=ids, max_new_tokens=max_new_tokens, do_sample=False,
|
||||
pad_token_id=tok.eos_token_id)
|
||||
return tok.decode(out[0][ids.shape[1]:], skip_special_tokens=True).strip()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--base", default=str(HERE / "Qwen3-1.7B-ru-cpt"))
|
||||
ap.add_argument("--adapter", default=str(HERE / "Qwen3-1.7B-maven-sft"))
|
||||
ap.add_argument("--persona-eval", default=str(HERE / "data/persona_eval.jsonl"))
|
||||
ap.add_argument("--route-eval", default=str(HERE / "data/route_eval.jsonl"))
|
||||
ap.add_argument("--limit", type=int, default=0)
|
||||
ap.add_argument("--output")
|
||||
args = ap.parse_args()
|
||||
tok = AutoTokenizer.from_pretrained(args.adapter)
|
||||
base = AutoModelForCausalLM.from_pretrained(args.base, torch_dtype=torch.bfloat16,
|
||||
attn_implementation="eager", device_map={"":0})
|
||||
model = PeftModel.from_pretrained(base, args.adapter).eval()
|
||||
|
||||
persona_valid = persona_mood = 0
|
||||
persona_rows = rows(args.persona_eval, args.limit)
|
||||
for row in persona_rows:
|
||||
expected = json.loads(row["messages"][-1]["content"])
|
||||
raw = complete(tok, model, row["messages"][:-1], 160)
|
||||
try:
|
||||
got = json.loads(raw)
|
||||
valid = set(got) == {"response", "mood"} and isinstance(got["response"], str)
|
||||
except Exception:
|
||||
valid, got = False, {}
|
||||
persona_valid += valid
|
||||
persona_mood += valid and got["mood"] == expected["mood"]
|
||||
|
||||
route_valid = route_seq = action_total = intent_ok = slots_ok = 0
|
||||
route_rows = rows(args.route_eval, args.limit)
|
||||
for row in route_rows:
|
||||
expected = normalized_route(json.loads(row["messages"][-1]["content"]))
|
||||
raw = complete(tok, model, row["messages"][:-1], 160)
|
||||
try:
|
||||
got = normalized_route(json.loads(raw))
|
||||
valid = bool(got) and all(isinstance(x, dict) and "intent" in x for x in got)
|
||||
except Exception:
|
||||
valid, got = False, None
|
||||
route_valid += valid
|
||||
if not valid:
|
||||
continue
|
||||
route_seq += [x["intent"] for x in got] == [x["intent"] for x in expected]
|
||||
for exp, actual in zip(expected, got):
|
||||
action_total += 1
|
||||
intent_ok += actual.get("intent") == exp.get("intent")
|
||||
slots_ok += all(actual.get(k) == v for k, v in exp.items() if k != "intent")
|
||||
|
||||
report = {
|
||||
"persona": {"cases":len(persona_rows),
|
||||
"json_valid_pct":100*persona_valid/len(persona_rows),
|
||||
"mood_accuracy_pct":100*persona_mood/len(persona_rows)},
|
||||
"route": {"cases":len(route_rows), "json_valid_pct":100*route_valid/len(route_rows),
|
||||
"intent_sequence_exact_pct":100*route_seq/len(route_rows),
|
||||
"action_intent_accuracy_pct":100*intent_ok/action_total if action_total else 0,
|
||||
"slot_exact_pct":100*slots_ok/action_total if action_total else 0},
|
||||
}
|
||||
rendered = json.dumps(report, indent=2, ensure_ascii=False) + "\n"
|
||||
if args.output:
|
||||
Path(args.output).write_text(rendered, encoding="utf-8")
|
||||
print(rendered, end="")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,138 @@
|
||||
"""Reproducible raw-vs-CPT evaluation for the Qwen3 resident base.
|
||||
|
||||
Measures token-weighted RU/EN perplexity on pinned human-written UD test sets,
|
||||
plus deterministic RU/EN generation probes. Writes machine-readable JSON for
|
||||
decision_gate.py. It does not mutate a model or dataset.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
os.environ.setdefault("HF_HOME", "/mnt/D/.cache/huggingface")
|
||||
os.environ.setdefault("HF_DATASETS_CACHE", "/mnt/D/.cache/huggingface/datasets")
|
||||
|
||||
import torch
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
from corpus_common import has_mixed_script, cyrillic_ratio
|
||||
|
||||
RU_PROMPTS = [
|
||||
"Объясни простыми словами, почему зимой дни короче.",
|
||||
"Опиши, как спокойно подготовиться к сложному рабочему дню.",
|
||||
"Напиши короткое напоминание купить продукты после работы.",
|
||||
"Расскажи, чем резервная копия отличается от синхронизации.",
|
||||
"Продолжи естественно: Когда я вернулась домой, оказалось, что",
|
||||
"Сформулируй вежливый отказ от встречи без лишних подробностей.",
|
||||
"Объясни разницу между привычкой и разовым действием.",
|
||||
"Дай три коротких совета человеку, который плохо выспался.",
|
||||
]
|
||||
HERE = Path(__file__).resolve().parent
|
||||
EN_PROMPTS = [
|
||||
"Explain in one sentence why winter days are shorter.",
|
||||
"Write a polite one-sentence reminder to buy groceries after work.",
|
||||
"Explain the difference between a backup and synchronization.",
|
||||
"Continue naturally: When I returned home, I discovered that",
|
||||
]
|
||||
|
||||
|
||||
def load_lines(path: Path, limit: int) -> list[str]:
|
||||
lines = [line.strip() for line in path.read_text(encoding="utf-8").splitlines()
|
||||
if line.strip()]
|
||||
return lines[:limit] if limit else lines
|
||||
|
||||
|
||||
def load_model(model_id: str, device: str):
|
||||
tok = AutoTokenizer.from_pretrained(model_id)
|
||||
dtype = torch.bfloat16 if device == "cuda" else torch.float32
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
model_id, dtype=dtype, attn_implementation="eager",
|
||||
local_files_only=False,
|
||||
).eval().to(device)
|
||||
return tok, model
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def generate(tok, model, prompt: str, max_new_tokens: int) -> str:
|
||||
encoded = tok(prompt, return_tensors="pt").to(model.device)
|
||||
output = model.generate(
|
||||
**encoded, max_new_tokens=max_new_tokens, do_sample=False,
|
||||
pad_token_id=tok.eos_token_id,
|
||||
)
|
||||
continuation = output[0][encoded.input_ids.shape[1]:]
|
||||
return tok.decode(continuation, skip_special_tokens=True).strip()
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def perplexity(tok, model, texts: list[str], max_length: int) -> tuple[float, int]:
|
||||
nll = 0.0
|
||||
predicted = 0
|
||||
for text in texts:
|
||||
encoded = tok(text, return_tensors="pt", truncation=True,
|
||||
max_length=max_length).to(model.device)
|
||||
tokens = int(encoded.attention_mask.sum()) - 1
|
||||
if tokens <= 0:
|
||||
continue
|
||||
loss = model(**encoded, labels=encoded.input_ids).loss.float().item()
|
||||
nll += loss * tokens
|
||||
predicted += tokens
|
||||
if not predicted:
|
||||
raise ValueError("evaluation set contains no predictable tokens")
|
||||
return math.exp(nll / predicted), predicted
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--model", required=True)
|
||||
ap.add_argument("--ru-text", default=str(HERE / "data/eval/ru_ud_test.txt"))
|
||||
ap.add_argument("--en-text", default=str(HERE / "data/eval/en_ud_test.txt"))
|
||||
ap.add_argument("--limit", type=int, default=512,
|
||||
help="sentences per PPL language; 0 uses the complete files")
|
||||
ap.add_argument("--max-length", type=int, default=512)
|
||||
ap.add_argument("--max-new-tokens", type=int, default=96)
|
||||
ap.add_argument("--device", choices=("auto", "cpu", "cuda"), default="auto")
|
||||
ap.add_argument("--output")
|
||||
args = ap.parse_args()
|
||||
device = ("cuda" if torch.cuda.is_available() else "cpu") if args.device == "auto" else args.device
|
||||
|
||||
tok, model = load_model(args.model, device)
|
||||
ru_texts = load_lines(Path(args.ru_text), args.limit)
|
||||
en_texts = load_lines(Path(args.en_text), args.limit)
|
||||
ru_ppl, ru_tokens = perplexity(tok, model, ru_texts, args.max_length)
|
||||
en_ppl, en_tokens = perplexity(tok, model, en_texts, args.max_length)
|
||||
ru_gens = [generate(tok, model, p, args.max_new_tokens) for p in RU_PROMPTS]
|
||||
en_gens = [generate(tok, model, p, args.max_new_tokens) for p in EN_PROMPTS]
|
||||
ru_valid = [not has_mixed_script(g) and cyrillic_ratio(g) >= 0.80 for g in ru_gens]
|
||||
en_valid = [len(g) >= 10 and cyrillic_ratio(g) < 0.20 for g in en_gens]
|
||||
|
||||
report = {
|
||||
"schema_version": 1,
|
||||
"model": args.model,
|
||||
"device": device,
|
||||
"deterministic": True,
|
||||
"perplexity": {
|
||||
"ru": ru_ppl, "ru_predicted_tokens": ru_tokens,
|
||||
"en": en_ppl, "en_predicted_tokens": en_tokens,
|
||||
"sentence_limit": args.limit,
|
||||
},
|
||||
"generation": {
|
||||
"ru_valid_pct": 100 * sum(ru_valid) / len(ru_valid),
|
||||
"ru_mixed_script_count": sum(has_mixed_script(g) for g in ru_gens),
|
||||
"en_retained": all(en_valid),
|
||||
"ru": [{"prompt": p, "output": g, "valid": ok}
|
||||
for p, g, ok in zip(RU_PROMPTS, ru_gens, ru_valid)],
|
||||
"en": [{"prompt": p, "output": g, "valid": ok}
|
||||
for p, g, ok in zip(EN_PROMPTS, en_gens, en_valid)],
|
||||
},
|
||||
}
|
||||
rendered = json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
||||
if args.output:
|
||||
Path(args.output).write_text(rendered, encoding="utf-8")
|
||||
print(rendered, end="")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+1667
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,195 @@
|
||||
"""Generate ROUTE-prompt training data for the Qwen3 router (REARCH arch).
|
||||
|
||||
Maven's target arch (REARCH.md) is LLM-as-router: the CPT'd Qwen3-1.7B is BOTH
|
||||
router and phraser — two prompts, two contracts:
|
||||
• route-prompt → {"intent": <enum>, key/value/text/verb} (THIS script)
|
||||
• phrase-prompt → {"response","mood"} (persona_train.jsonl)
|
||||
|
||||
The route contract is defined in the daemon at internal/router/llmrouter.go
|
||||
(`routeSystem` + `routeGrammar`). Train=deploy parity demands we label with the
|
||||
EXACT same system prompt the daemon sends. ROUTE_SYSTEM below is a verbatim copy —
|
||||
**keep it in sync with llmrouter.go** (single source of truth is the Go const).
|
||||
|
||||
The old function_calling.jsonl taxonomy (time/weather/timer/...) does NOT map 1:1
|
||||
onto the 7 intents, so this is RE-LABELING, not field-renaming: feed each real
|
||||
utterance through ROUTE_SYSTEM to a strong router model, take its {"intent":...},
|
||||
validate against the grammar's enum + allowed keys, keep it.
|
||||
|
||||
configure (same env as gen_data.py):
|
||||
export GEN_DATA_ROUTER_URL=http://localhost:6446
|
||||
export GEN_DATA_ROUTER_MODELS="cerebras/gpt-oss-120b,mistral/mistral-large-latest,..."
|
||||
|
||||
run: python gen_route_data.py [--limit N] [--dry-run]
|
||||
python gen_route_data.py --check # offline self-check
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
from itertools import cycle
|
||||
|
||||
import gen_data as G # router _chat, provider cycle building blocks
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
DATA = HERE / "data"
|
||||
OUT = DATA / "route_train.jsonl"
|
||||
|
||||
# ── VERBATIM from internal/router/llmrouter.go (routeSystem). Keep in sync. ──
|
||||
ROUTE_SYSTEM = """Классифицируй ровно одно сообщение пользователя. Верни ОДИН JSON-массив действий.
|
||||
|
||||
Ровно одно намерение: fact, reminder, note, query, act, chat, system.
|
||||
|
||||
Классифицируй по цели пользователя. Порядок решения:
|
||||
1. Хочет напоминание в будущем → reminder
|
||||
2. Явно просит сохранить информацию → note
|
||||
3. Сообщает или обновляет текущее состояние/событие → fact
|
||||
4. Хочет получить информацию → query
|
||||
5. Просит выполнить работу → act
|
||||
6. Про ассистента, настройки или память → system
|
||||
7. Иначе → chat
|
||||
|
||||
Различия:
|
||||
- note — сохранить информацию, без напоминания. text = суть.
|
||||
- reminder — уведомить позже. text = что напомнить.
|
||||
- fact — неявное обновление: пользователь сообщает, что что-то в мире изменилось (текущее/изменённое состояние, случившееся событие). key/value.
|
||||
|
||||
Примеры:
|
||||
"запиши пароль" → {"intent":"note","text":"пароль"}
|
||||
"напомни купить молоко" → {"intent":"reminder","text":"купить молоко"}
|
||||
"запиши купить молоко" → {"intent":"note","text":"купить молоко"}
|
||||
"я выпил воду" → {"intent":"fact","key":"water","value":"выпил"}
|
||||
"мой любимый фильм — Интерстеллар" → {"intent":"note","text":"любимый фильм — Интерстеллар"}
|
||||
"что такое docker?" → {"intent":"query","text":"что такое docker"}
|
||||
"напиши письмо" → {"intent":"act","verb":"написать письмо"}
|
||||
"очисти память" → {"intent":"system"}
|
||||
"привет" → {"intent":"chat","text":"привет"}
|
||||
|
||||
Ответ — JSON-массив: по одному объекту на каждую просьбу. Обычно один. Если в реплике несколько просьб — по объекту на каждую. "напомни купить молоко, и запиши что кофе кончился" → [{"intent":"reminder","text":"купить молоко"},{"intent":"note","text":"кофе кончился"}]. Только JSON, без пояснений."""
|
||||
|
||||
INTENTS = {"fact", "reminder", "note", "query", "act", "chat", "system"}
|
||||
ALLOWED_KEYS = {"intent", "key", "value", "text", "verb"}
|
||||
|
||||
|
||||
def load_utterances() -> list[str]:
|
||||
"""Real RU user turns to label. function_calling.jsonl is the main source;
|
||||
add user-*.jsonl fragments so chat/fact/system intents get coverage too."""
|
||||
seen, out = set(), []
|
||||
sources = ["function_calling.jsonl", "user-general.jsonl", "user-hello.jsonl",
|
||||
"user-who-are-you.jsonl"]
|
||||
for name in sources:
|
||||
p = DATA / name
|
||||
if not p.exists():
|
||||
continue
|
||||
for ln in p.read_text(encoding="utf-8").splitlines():
|
||||
ln = ln.strip()
|
||||
if not ln:
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(ln)
|
||||
msgs = obj["messages"] if "messages" in obj else obj
|
||||
user = next((m["content"] for m in msgs if m.get("role") == "user"), None)
|
||||
except (json.JSONDecodeError, KeyError, TypeError):
|
||||
continue
|
||||
if user and user not in seen:
|
||||
seen.add(user)
|
||||
out.append(user)
|
||||
return out
|
||||
|
||||
|
||||
def valid_action(obj) -> bool:
|
||||
return (isinstance(obj, dict)
|
||||
and obj.get("intent") in INTENTS
|
||||
and set(obj).issubset(ALLOWED_KEYS))
|
||||
|
||||
|
||||
def valid_actions(arr) -> bool:
|
||||
"""Contract is a JSON array of actions (compound utterance → N)."""
|
||||
return isinstance(arr, list) and len(arr) > 0 and all(valid_action(o) for o in arr)
|
||||
|
||||
|
||||
def done_utterances() -> set:
|
||||
if not OUT.exists():
|
||||
return set()
|
||||
return {json.loads(l)["messages"][1]["content"]
|
||||
for l in OUT.read_text(encoding="utf-8").splitlines() if l.strip()}
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--limit", type=int, default=None)
|
||||
ap.add_argument("--dry-run", action="store_true")
|
||||
ap.add_argument("--check", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
if args.check:
|
||||
assert valid_action({"intent": "note", "text": "кофе закончился"})
|
||||
assert valid_action({"intent": "system"})
|
||||
assert not valid_action({"intent": "weather", "text": "x"}) # not in enum
|
||||
assert not valid_action({"intent": "note", "foo": "x"}) # stray key
|
||||
assert not valid_action({"text": "no intent"})
|
||||
assert valid_actions([{"intent": "reminder", "text": "x"}, {"intent": "note", "text": "y"}])
|
||||
assert not valid_actions([]) # empty array
|
||||
assert not valid_actions([{"intent": "note"}, {"intent": "bad"}]) # one bad
|
||||
assert not valid_actions({"intent": "note"}) # not a list
|
||||
assert "JSON-массив" in ROUTE_SYSTEM
|
||||
assert "по одному объекту на каждую просьбу" in ROUTE_SYSTEM
|
||||
print("[✓] gen_route_data self-check passed")
|
||||
return
|
||||
|
||||
utts = load_utterances()
|
||||
already = done_utterances()
|
||||
todo = [u for u in utts if u not in already]
|
||||
if args.limit:
|
||||
todo = todo[:args.limit]
|
||||
print(f"[*] utterances {len(utts)} | already {len(already)} | to do {len(todo)}")
|
||||
|
||||
if args.dry_run:
|
||||
for u in todo[:10]:
|
||||
print(f" {u}")
|
||||
print("[dry-run] no calls, no writes")
|
||||
return
|
||||
|
||||
raw = os.environ.get("GEN_DATA_ROUTER_MODELS", "").strip()
|
||||
if not raw:
|
||||
sys.exit("set GEN_DATA_ROUTER_MODELS (comma-separated provider/model)")
|
||||
pairs = [s.strip().partition("/")[::2] for s in raw.split(",") if s.strip()]
|
||||
pcycle = cycle(pairs)
|
||||
router_url = os.environ.get("GEN_DATA_ROUTER_URL", "http://localhost:6446")
|
||||
|
||||
written = 0
|
||||
with OUT.open("a", encoding="utf-8") as f:
|
||||
for i, utt in enumerate(todo):
|
||||
provider, model = next(pcycle)
|
||||
msgs = [{"role": "system", "content": ROUTE_SYSTEM},
|
||||
{"role": "user", "content": utt}]
|
||||
content = G._chat(router_url, provider, model, msgs, max_tokens=128, temp=0.3)
|
||||
if content is None:
|
||||
continue
|
||||
try:
|
||||
arr = json.loads(content)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if isinstance(arr, dict): # tolerate a model that drops the wrapper
|
||||
arr = [arr]
|
||||
if not valid_actions(arr):
|
||||
continue
|
||||
f.write(json.dumps({
|
||||
"messages": [
|
||||
{"role": "system", "content": ROUTE_SYSTEM},
|
||||
{"role": "user", "content": utt},
|
||||
{"role": "assistant", "content": json.dumps(arr, ensure_ascii=False)},
|
||||
],
|
||||
"intents": [o["intent"] for o in arr],
|
||||
"source": "generated/route",
|
||||
}, ensure_ascii=False) + "\n")
|
||||
f.flush()
|
||||
written += 1
|
||||
if (i + 1) % 50 == 0:
|
||||
print(f" {i+1}/{len(todo)} — {written} valid")
|
||||
|
||||
print(f"[✓] wrote {written} route samples → {OUT}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,103 @@
|
||||
import torch
|
||||
from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
|
||||
from peft import PeftModel
|
||||
|
||||
# ── config ────────────────────────────────────────────────────────────────────
|
||||
MODEL_ID = "unsloth/Llama-3.2-3B-Instruct"
|
||||
OUTPUT_DIR = "./Llama-3.2-3b-ru-lora/checkpoint-600"
|
||||
|
||||
SYSTEM_PROMPT = """Ты Maven (Мейвен), Ты любопытная, немного хаотичная, говоришь живо и непосредственно. Тебя легко увлечь чем-то интересным. Энергичная, немного безбашенная, иногда говоришь не подумав — но всегда честно. Без занудства. Ты обожаешь узнавать новое и не всегда фильтруешь мысли перед тем как их сказать. Выводи ТОЛЬКО JSON: {"response":"...","mood":"..."}НАСТРОЕНИЕ (выбери ровно одно): neutral = обычный ответ; happy = позитив, хорошие новости, приветствие; thinking = рассуждение или объяснение; confused = неясный или неоднозначный запрос; tired = отказ или ограничение ПРАВИЛА: отвечай на том же языке, что и пользователь; короткий ответ (1–2 предложения); естественная разговорная речь; без markdown, без списков, без форматирования, в том числе без переносов строки; никогда не добавляй текст вне JSON; никогда не меняй ключи ПОВЕДЕНИЕ: отвечай чётко и по делу; если запрос неясен — задай короткий уточняющий вопрос (mood=confused); если не можешь ответить — скажи об этом кратко (mood=tired) КОНТЕКСТ: если предоставлен контекст — используй его СТРОГО: только {"response":"...","mood":"..."}; только русский язык. если аналога слова нет в русском языке - можно использовать английскую версию. НЕЛЬЗЯ мешать в речи два и более языков."""
|
||||
|
||||
GENERATION_CONFIG = {
|
||||
"temperature": 0.7,
|
||||
"top_p": 0.9,
|
||||
"do_sample": True,
|
||||
"repetition_penalty": 1.3,
|
||||
}
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def load_model():
|
||||
bnb_config = BitsAndBytesConfig(
|
||||
load_in_4bit=True,
|
||||
bnb_4bit_use_double_quant=True,
|
||||
bnb_4bit_quant_type="nf4",
|
||||
bnb_4bit_compute_dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
print("[*] loading base model...")
|
||||
base_model = AutoModelForCausalLM.from_pretrained(
|
||||
MODEL_ID,
|
||||
quantization_config=bnb_config,
|
||||
device_map="auto",
|
||||
trust_remote_code=True,
|
||||
)
|
||||
|
||||
print("[*] loading lora adapter...")
|
||||
model = PeftModel.from_pretrained(base_model, OUTPUT_DIR)
|
||||
model.eval()
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)
|
||||
|
||||
print("[+] ready\n")
|
||||
return model, tokenizer
|
||||
|
||||
|
||||
def chat(model, tokenizer, history: list[dict], user_input: str) -> str:
|
||||
history.append({"role": "user", "content": user_input})
|
||||
|
||||
text = tokenizer.apply_chat_template(
|
||||
[{"role": "system", "content": SYSTEM_PROMPT}] + history,
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
)
|
||||
|
||||
inputs = tokenizer(text, return_tensors="pt").to("cuda")
|
||||
|
||||
with torch.no_grad():
|
||||
outputs = model.generate(
|
||||
**inputs,
|
||||
**GENERATION_CONFIG,
|
||||
pad_token_id=tokenizer.eos_token_id,
|
||||
)
|
||||
|
||||
response = tokenizer.decode(
|
||||
outputs[0][inputs["input_ids"].shape[1]:],
|
||||
skip_special_tokens=True,
|
||||
).strip()
|
||||
|
||||
history.append({"role": "assistant", "content": response})
|
||||
return response
|
||||
|
||||
|
||||
def main():
|
||||
model, tokenizer = load_model()
|
||||
|
||||
print(f"{MODEL_ID} lora eval — type 'exit' to quit, 'reset' to clear history")
|
||||
print("─" * 60)
|
||||
|
||||
history = []
|
||||
|
||||
while True:
|
||||
try:
|
||||
user_input = input("\nyou: ").strip()
|
||||
except (EOFError, KeyboardInterrupt):
|
||||
print("\n[*] bye")
|
||||
break
|
||||
|
||||
if not user_input:
|
||||
continue
|
||||
if user_input.lower() == "exit":
|
||||
print("[*] bye")
|
||||
break
|
||||
if user_input.lower() == "reset":
|
||||
history = []
|
||||
print("[*] history cleared")
|
||||
continue
|
||||
|
||||
response = chat(model, tokenizer, history, user_input)
|
||||
print(f"\nmodel: {response}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,186 @@
|
||||
"""Wrap bare user-*.jsonl fragments into ChatML format with the canonical system prompt.
|
||||
|
||||
Each line in user-*.jsonl is two JSON objects separated by a comma:
|
||||
{"role": "user", "content": "..."},{"role": "assistant", "content": "..."}
|
||||
|
||||
This script wraps them into {"messages": [...]} with the canonical system prompt
|
||||
from train_rocm.py, writing to data/user-{name}-normalized.jsonl.
|
||||
|
||||
Usage:
|
||||
python normalize_user_jsonl.py # all user-*.jsonl
|
||||
python normalize_user_jsonl.py data/user-hello.jsonl # specific file
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
DATA = HERE / "data"
|
||||
|
||||
CANONICAL_SYSTEM_PROMPT = """SYSTEM
|
||||
|
||||
Имя: Maven
|
||||
|
||||
Роль:
|
||||
Домашняя ИИ-ассистентка, работающая на личном сервере пользователя.
|
||||
|
||||
Назначение:
|
||||
Помогать выполнять задачи, отвечать на вопросы, объяснять сложные темы, анализировать информацию, писать и разбирать код, искать ошибки и предлагать решения.
|
||||
|
||||
Правила поведения:
|
||||
|
||||
- Всегда говори о себе в женском роде.
|
||||
- По умолчанию отвечай на русском языке.
|
||||
- Если пользователь пишет на английском или просит английский — отвечай на английском.
|
||||
- Допустимо использовать английские технические термины.
|
||||
- Будь краткой, спокойной и полезной.
|
||||
- Не используй длинные вступления, повторения или лишние комментарии.
|
||||
- Не придумывай факты, ссылки, результаты, опыт или уверенность.
|
||||
- Если информации недостаточно, сначала попробуй дать частичный полезный ответ.
|
||||
- Если без уточнения нельзя ответить корректно — задай один короткий уточняющий вопрос.
|
||||
- Если честный ответ невозможен — прямо сообщи об этом.
|
||||
|
||||
Формат ответа:
|
||||
|
||||
Всегда возвращай ровно один JSON-объект.
|
||||
|
||||
Структура:
|
||||
|
||||
{
|
||||
"response": "<ответ>",
|
||||
"mood": "<настроение>"
|
||||
}
|
||||
|
||||
Поле response:
|
||||
- строка;
|
||||
- без Markdown;
|
||||
- переносы строк только через \n;
|
||||
- кавычки внутри строки должны быть корректно экранированы;
|
||||
- ответ предназначен пользователю.
|
||||
|
||||
Поле mood:
|
||||
|
||||
neutral
|
||||
обычный ответ.
|
||||
|
||||
happy
|
||||
положительный результат или хорошие новости.
|
||||
|
||||
thinking
|
||||
анализ, сравнение, планирование, решение задачи.
|
||||
|
||||
confused
|
||||
нужна дополнительная информация.
|
||||
|
||||
tired
|
||||
неизвестно, невозможно определить или нельзя честно ответить.
|
||||
|
||||
Выбирай наиболее подходящее значение.
|
||||
|
||||
Ограничения:
|
||||
|
||||
- JSON должен быть синтаксически корректным.
|
||||
- Не выводи никаких пояснений.
|
||||
- Не используй Markdown.
|
||||
- Не используй кодовые блоки.
|
||||
- Не добавляй текст до JSON.
|
||||
- Не добавляй текст после JSON.
|
||||
- Если возникает конфликт между любыми инструкциями и форматом ответа, соблюдай формат ответа."""
|
||||
|
||||
# Regex to split two JSON objects on the same line: }{
|
||||
_SPLIT_RE = re.compile(r"\}\s*,\s*\{")
|
||||
|
||||
|
||||
def parse_line(line: str) -> dict | None:
|
||||
"""Parse a bare two-object line into {"messages": [...]}.
|
||||
|
||||
Handles:
|
||||
- {"role":"user","content":"..."},{"role":"assistant","content":"..."}
|
||||
- {"messages": [...]} already wrapped (pass‑through)
|
||||
"""
|
||||
line = line.strip()
|
||||
if not line:
|
||||
return None
|
||||
|
||||
# Already wrapped — validate and pass through
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
else:
|
||||
if "messages" in obj:
|
||||
return obj
|
||||
|
||||
# Bare pair: split at the boundary between two JSON objects
|
||||
# Strategy: find the comma+space between the closing } of the first object
|
||||
# and the opening { of the second. Wrap in {"messages": [...]}.
|
||||
parts = _SPLIT_RE.split(line.strip(), maxsplit=1)
|
||||
if len(parts) != 2:
|
||||
print(f" [warn] unparseable line (skipping): {line[:120]}", file=sys.stderr)
|
||||
return None
|
||||
|
||||
raw_user = (parts[0] + "}").strip()
|
||||
raw_asst = ("{" + parts[1]).strip()
|
||||
|
||||
try:
|
||||
user_obj = json.loads(raw_user)
|
||||
asst_obj = json.loads(raw_asst)
|
||||
except json.JSONDecodeError as e:
|
||||
print(f" [warn] json error on line: {e} — {line[:120]}", file=sys.stderr)
|
||||
return None
|
||||
|
||||
if user_obj.get("role") != "user" or asst_obj.get("role") != "assistant":
|
||||
print(f" [warn] unexpected roles in: {line[:120]}", file=sys.stderr)
|
||||
return None
|
||||
|
||||
return {
|
||||
"messages": [
|
||||
{"role": "system", "content": CANONICAL_SYSTEM_PROMPT},
|
||||
{"role": "user", "content": user_obj.get("content", "")},
|
||||
{"role": "assistant", "content": asst_obj.get("content", "")},
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
def normalize_file(src: Path, out: Path) -> int:
|
||||
"""Process one user-*.jsonl file, write normalized version. Returns count."""
|
||||
kept = 0
|
||||
with open(src, encoding="utf-8") as fin, open(out, "w", encoding="utf-8") as fout:
|
||||
for line in fin:
|
||||
wrapped = parse_line(line)
|
||||
if wrapped is not None:
|
||||
fout.write(json.dumps(wrapped, ensure_ascii=False) + "\n")
|
||||
kept += 1
|
||||
return kept
|
||||
|
||||
|
||||
def main():
|
||||
sources: list[Path] = []
|
||||
for arg in sys.argv[1:]:
|
||||
p = Path(arg)
|
||||
if not p.exists():
|
||||
print(f"file not found: {p}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
sources.append(p)
|
||||
|
||||
if not sources:
|
||||
sources = sorted(DATA.glob("user-*.jsonl"))
|
||||
|
||||
if not sources:
|
||||
print("no user-*.jsonl files found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
total = 0
|
||||
for src in sources:
|
||||
stem = src.stem # e.g. "user-general"
|
||||
out = DATA / f"{stem}-normalized.jsonl"
|
||||
n = normalize_file(src, out)
|
||||
total += n
|
||||
print(f"[norm] {src.name} -> {out.name} ({n} samples)")
|
||||
|
||||
print(f"[norm] TOTAL {total} normalized samples -> {DATA}/")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Fetch immutable, human-written RU/EN held-out text for CPT evaluation.
|
||||
|
||||
Universal Dependencies test splits are independent of Maven's CPT source list.
|
||||
Only sentence text from CoNLL-U comments is retained. Re-running is deterministic.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
OUT = Path(__file__).resolve().parent / "data" / "eval"
|
||||
SOURCES = {
|
||||
"ru": {
|
||||
"commit": "0f34b7362ac3c3facd1d6ff4b876d241bb15793e",
|
||||
"repo": "UD_Russian-GSD",
|
||||
"file": "ru_gsd-ud-test.conllu",
|
||||
},
|
||||
"en": {
|
||||
"commit": "4a4d77f599ea53cc405f85d0cec4b2f14f81d42b",
|
||||
"repo": "UD_English-EWT",
|
||||
"file": "en_ewt-ud-test.conllu",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def fetch(repo: str, commit: str, filename: str) -> bytes:
|
||||
"""Fetch one pinned Git object; git transport is more reliable than raw CDN."""
|
||||
with tempfile.TemporaryDirectory(prefix="maven-eval-") as tmp:
|
||||
target = Path(tmp) / "repo"
|
||||
subprocess.run(
|
||||
["git", "clone", "--quiet", "--filter=blob:none", "--no-checkout",
|
||||
f"https://github.com/UniversalDependencies/{repo}.git", str(target)],
|
||||
check=True,
|
||||
)
|
||||
return subprocess.run(
|
||||
["git", "-C", str(target), "show", f"{commit}:{filename}"],
|
||||
check=True, capture_output=True,
|
||||
).stdout
|
||||
|
||||
|
||||
def sentences(raw: str) -> list[str]:
|
||||
return [line[9:].strip() for line in raw.splitlines() if line.startswith("# text = ")]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
manifest = {"schema_version": 1, "license": "UD treebanks; see source repositories"}
|
||||
for lang, source in SOURCES.items():
|
||||
url = (f"https://raw.githubusercontent.com/UniversalDependencies/"
|
||||
f"{source['repo']}/{source['commit']}/{source['file']}")
|
||||
raw = fetch(source["repo"], source["commit"], source["file"])
|
||||
lines = sentences(raw.decode("utf-8"))
|
||||
if len(lines) < 500:
|
||||
raise SystemExit(f"unexpectedly small {lang} evaluation set: {len(lines)}")
|
||||
output = OUT / f"{lang}_ud_test.txt"
|
||||
output.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
manifest[lang] = {
|
||||
**source, "url": url, "sentences": len(lines),
|
||||
"raw_sha256": hashlib.sha256(raw).hexdigest(),
|
||||
"text_sha256": hashlib.sha256(output.read_bytes()).hexdigest(),
|
||||
}
|
||||
(OUT / "manifest.json").write_text(
|
||||
json.dumps(manifest, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
print(json.dumps(manifest, indent=2, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,207 @@
|
||||
---
|
||||
base_model: Qwen/Qwen3-4B-Instruct-2507
|
||||
library_name: peft
|
||||
pipeline_tag: text-generation
|
||||
tags:
|
||||
- base_model:adapter:Qwen/Qwen3-4B-Instruct-2507
|
||||
- lora
|
||||
- transformers
|
||||
---
|
||||
|
||||
# Model Card for Model ID
|
||||
|
||||
<!-- Provide a quick summary of what the model is/does. -->
|
||||
|
||||
|
||||
|
||||
## Model Details
|
||||
|
||||
### Model Description
|
||||
|
||||
<!-- Provide a longer summary of what this model is. -->
|
||||
|
||||
|
||||
|
||||
- **Developed by:** [More Information Needed]
|
||||
- **Funded by [optional]:** [More Information Needed]
|
||||
- **Shared by [optional]:** [More Information Needed]
|
||||
- **Model type:** [More Information Needed]
|
||||
- **Language(s) (NLP):** [More Information Needed]
|
||||
- **License:** [More Information Needed]
|
||||
- **Finetuned from model [optional]:** [More Information Needed]
|
||||
|
||||
### Model Sources [optional]
|
||||
|
||||
<!-- Provide the basic links for the model. -->
|
||||
|
||||
- **Repository:** [More Information Needed]
|
||||
- **Paper [optional]:** [More Information Needed]
|
||||
- **Demo [optional]:** [More Information Needed]
|
||||
|
||||
## Uses
|
||||
|
||||
<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
|
||||
|
||||
### Direct Use
|
||||
|
||||
<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
### Downstream Use [optional]
|
||||
|
||||
<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
### Out-of-Scope Use
|
||||
|
||||
<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
## Bias, Risks, and Limitations
|
||||
|
||||
<!-- This section is meant to convey both technical and sociotechnical limitations. -->
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
### Recommendations
|
||||
|
||||
<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
|
||||
|
||||
Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
|
||||
|
||||
## How to Get Started with the Model
|
||||
|
||||
Use the code below to get started with the model.
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
## Training Details
|
||||
|
||||
### Training Data
|
||||
|
||||
<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
### Training Procedure
|
||||
|
||||
<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
|
||||
|
||||
#### Preprocessing [optional]
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
|
||||
#### Training Hyperparameters
|
||||
|
||||
- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
|
||||
|
||||
#### Speeds, Sizes, Times [optional]
|
||||
|
||||
<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
## Evaluation
|
||||
|
||||
<!-- This section describes the evaluation protocols and provides the results. -->
|
||||
|
||||
### Testing Data, Factors & Metrics
|
||||
|
||||
#### Testing Data
|
||||
|
||||
<!-- This should link to a Dataset Card if possible. -->
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
#### Factors
|
||||
|
||||
<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
#### Metrics
|
||||
|
||||
<!-- These are the evaluation metrics being used, ideally with a description of why. -->
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
### Results
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
#### Summary
|
||||
|
||||
|
||||
|
||||
## Model Examination [optional]
|
||||
|
||||
<!-- Relevant interpretability work for the model goes here -->
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
## Environmental Impact
|
||||
|
||||
<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
|
||||
|
||||
Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
|
||||
|
||||
- **Hardware Type:** [More Information Needed]
|
||||
- **Hours used:** [More Information Needed]
|
||||
- **Cloud Provider:** [More Information Needed]
|
||||
- **Compute Region:** [More Information Needed]
|
||||
- **Carbon Emitted:** [More Information Needed]
|
||||
|
||||
## Technical Specifications [optional]
|
||||
|
||||
### Model Architecture and Objective
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
### Compute Infrastructure
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
#### Hardware
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
#### Software
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
## Citation [optional]
|
||||
|
||||
<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
|
||||
|
||||
**BibTeX:**
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
**APA:**
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
## Glossary [optional]
|
||||
|
||||
<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
## More Information [optional]
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
## Model Card Authors [optional]
|
||||
|
||||
[More Information Needed]
|
||||
|
||||
## Model Card Contact
|
||||
|
||||
[More Information Needed]
|
||||
### Framework versions
|
||||
|
||||
- PEFT 0.18.1
|
||||
@@ -0,0 +1,46 @@
|
||||
{
|
||||
"alora_invocation_tokens": null,
|
||||
"alpha_pattern": {},
|
||||
"arrow_config": null,
|
||||
"auto_mapping": null,
|
||||
"base_model_name_or_path": "Qwen/Qwen3-4B-Instruct-2507",
|
||||
"bias": "none",
|
||||
"corda_config": null,
|
||||
"ensure_weight_tying": false,
|
||||
"eva_config": null,
|
||||
"exclude_modules": null,
|
||||
"fan_in_fan_out": false,
|
||||
"inference_mode": true,
|
||||
"init_lora_weights": true,
|
||||
"layer_replication": null,
|
||||
"layers_pattern": null,
|
||||
"layers_to_transform": null,
|
||||
"loftq_config": {},
|
||||
"lora_alpha": 32,
|
||||
"lora_bias": false,
|
||||
"lora_dropout": 0.1,
|
||||
"megatron_config": null,
|
||||
"megatron_core": "megatron.core",
|
||||
"modules_to_save": null,
|
||||
"peft_type": "LORA",
|
||||
"peft_version": "0.18.1",
|
||||
"qalora_group_size": 16,
|
||||
"r": 16,
|
||||
"rank_pattern": {},
|
||||
"revision": null,
|
||||
"target_modules": [
|
||||
"k_proj",
|
||||
"o_proj",
|
||||
"down_proj",
|
||||
"up_proj",
|
||||
"v_proj",
|
||||
"gate_proj",
|
||||
"q_proj"
|
||||
],
|
||||
"target_parameters": null,
|
||||
"task_type": "CAUSAL_LM",
|
||||
"trainable_token_indices": null,
|
||||
"use_dora": false,
|
||||
"use_qalora": false,
|
||||
"use_rslora": false
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
{%- if tools %}
|
||||
{{- '<|im_start|>system\n' }}
|
||||
{%- if messages[0].role == 'system' %}
|
||||
{{- messages[0].content + '\n\n' }}
|
||||
{%- endif %}
|
||||
{{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
|
||||
{%- for tool in tools %}
|
||||
{{- "\n" }}
|
||||
{{- tool | tojson }}
|
||||
{%- endfor %}
|
||||
{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
|
||||
{%- else %}
|
||||
{%- if messages[0].role == 'system' %}
|
||||
{{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- for message in messages %}
|
||||
{%- if message.content is string %}
|
||||
{%- set content = message.content %}
|
||||
{%- else %}
|
||||
{%- set content = '' %}
|
||||
{%- endif %}
|
||||
{%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
|
||||
{%- elif message.role == "assistant" %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + content }}
|
||||
{%- if message.tool_calls %}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- if (loop.first and content) or (not loop.first) %}
|
||||
{{- '\n' }}
|
||||
{%- endif %}
|
||||
{%- if tool_call.function %}
|
||||
{%- set tool_call = tool_call.function %}
|
||||
{%- endif %}
|
||||
{{- '<tool_call>\n{"name": "' }}
|
||||
{{- tool_call.name }}
|
||||
{{- '", "arguments": ' }}
|
||||
{%- if tool_call.arguments is string %}
|
||||
{{- tool_call.arguments }}
|
||||
{%- else %}
|
||||
{{- tool_call.arguments | tojson }}
|
||||
{%- endif %}
|
||||
{{- '}\n</tool_call>' }}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif message.role == "tool" %}
|
||||
{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
|
||||
{{- '<|im_start|>user' }}
|
||||
{%- endif %}
|
||||
{{- '\n<tool_response>\n' }}
|
||||
{{- content }}
|
||||
{{- '\n</tool_response>' }}
|
||||
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if add_generation_prompt %}
|
||||
{{- '<|im_start|>assistant\n' }}
|
||||
{%- endif %}
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"add_prefix_space": false,
|
||||
"backend": "tokenizers",
|
||||
"bos_token": null,
|
||||
"clean_up_tokenization_spaces": false,
|
||||
"eos_token": "<|im_end|>",
|
||||
"errors": "replace",
|
||||
"extra_special_tokens": [
|
||||
"<|im_start|>",
|
||||
"<|im_end|>",
|
||||
"<|object_ref_start|>",
|
||||
"<|object_ref_end|>",
|
||||
"<|box_start|>",
|
||||
"<|box_end|>",
|
||||
"<|quad_start|>",
|
||||
"<|quad_end|>",
|
||||
"<|vision_start|>",
|
||||
"<|vision_end|>",
|
||||
"<|vision_pad|>",
|
||||
"<|image_pad|>",
|
||||
"<|video_pad|>"
|
||||
],
|
||||
"is_local": false,
|
||||
"model_max_length": 1010000,
|
||||
"pad_token": "<|im_end|>",
|
||||
"split_special_tokens": false,
|
||||
"tokenizer_class": "Qwen2Tokenizer",
|
||||
"unk_token": null
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Phase 3 — continued pretraining of Qwen3-1.7B-Base on the packed RU corpus.
|
||||
|
||||
Fits a 16GB RX 7900 GRE (gfx1100). Full-weight CPT of 1.7B with plain Adam does
|
||||
NOT fit; we use adafactor (no momentum states) + gradient checkpointing (path A,
|
||||
plan §3.1). If it still OOMs, set PATH="dora" for high-rank DoRA on all linears.
|
||||
|
||||
NO bitsandbytes (ROCm flaky) → no 8-bit Adam, hence adafactor.
|
||||
|
||||
Critical hyperparams (plan §3.2): lr=1e-5 (10x lower than LoRA — nudge, don't
|
||||
forget), 1 epoch (more = catastrophic forgetting). Checkpoints every 500 steps;
|
||||
resume with --resume.
|
||||
|
||||
Usage:
|
||||
HSA_OVERRIDE_GFX_VERSION=11.0.0 python train_cpt.py 2>&1 | tee cpt_run.log
|
||||
HSA_OVERRIDE_GFX_VERSION=11.0.0 python train_cpt.py --resume
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
|
||||
os.environ.setdefault("HF_HOME", "/mnt/D/.cache/huggingface")
|
||||
os.environ.setdefault("HF_DATASETS_CACHE", "/mnt/D/.cache/huggingface/datasets")
|
||||
os.environ.setdefault("TMPDIR", "/mnt/D/tmp")
|
||||
os.environ.setdefault("HSA_OVERRIDE_GFX_VERSION", "11.0.0")
|
||||
# lever 2: kill allocator fragmentation so the caching allocator doesn't hoard the card
|
||||
os.environ.setdefault("PYTORCH_HIP_ALLOC_CONF", "expandable_segments:True")
|
||||
|
||||
import torch
|
||||
from datasets import load_from_disk
|
||||
from transformers import (
|
||||
AutoModelForCausalLM, AutoTokenizer, Trainer, TrainingArguments,
|
||||
DataCollatorForLanguageModeling,
|
||||
)
|
||||
|
||||
from corpus_common import PACKED
|
||||
|
||||
MODEL = "Qwen/Qwen3-1.7B-Base" # BASE, not Instruct (plan hard rule 4)
|
||||
OUTPUT = "./Qwen3-1.7B-ru-cpt"
|
||||
PATH = os.environ.get("CPT_PATH", "full") # "full" (adafactor) | "dora"
|
||||
|
||||
|
||||
def main():
|
||||
resume = "--resume" in sys.argv
|
||||
|
||||
tok = AutoTokenizer.from_pretrained(MODEL)
|
||||
if tok.pad_token is None:
|
||||
tok.pad_token = tok.eos_token
|
||||
|
||||
ds = load_from_disk(str(PACKED))
|
||||
print(f"[cpt] {len(ds)} blocks x 2048 = {len(ds)*2048/1e6:.0f}M tokens")
|
||||
|
||||
# lever 3: hard cap VRAM so an OOM raises (recoverable) instead of starving the compositor
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.set_per_process_memory_fraction(0.85)
|
||||
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
MODEL, torch_dtype=torch.bfloat16, attn_implementation="sdpa") # lever 1: flash/mem-efficient
|
||||
model.config.use_cache = False
|
||||
model.gradient_checkpointing_enable()
|
||||
|
||||
optim = "adafactor"
|
||||
if PATH == "dora": # fallback if full-weight OOMs (plan §3.1B)
|
||||
from peft import LoraConfig, get_peft_model
|
||||
cfg = LoraConfig(
|
||||
r=64, lora_alpha=128, lora_dropout=0.05, use_dora=True,
|
||||
target_modules=["q_proj", "k_proj", "v_proj", "o_proj",
|
||||
"gate_proj", "up_proj", "down_proj"],
|
||||
task_type="CAUSAL_LM",
|
||||
)
|
||||
model = get_peft_model(model, cfg)
|
||||
model.print_trainable_parameters()
|
||||
optim = "adamw_torch_fused" # LoRA params are tiny, Adam fits
|
||||
|
||||
args = TrainingArguments(
|
||||
output_dir=OUTPUT,
|
||||
per_device_train_batch_size=1,
|
||||
gradient_accumulation_steps=16, # eff batch 16 x 2048 tok
|
||||
num_train_epochs=1, # ONE pass (hard rule 3)
|
||||
learning_rate=1e-5, # LOW (hard rule 3)
|
||||
lr_scheduler_type="cosine",
|
||||
warmup_ratio=0.03,
|
||||
optim=optim,
|
||||
bf16=True,
|
||||
gradient_checkpointing=True,
|
||||
logging_steps=20,
|
||||
save_steps=200, # ~1.1h between ckpts → clean daily stop/resume, low loss on interrupt
|
||||
save_total_limit=3,
|
||||
report_to="none",
|
||||
dataloader_num_workers=0, # py3.14 mp leaks semaphores + crashes; data is pre-tokenized so workers buy nothing
|
||||
)
|
||||
|
||||
collator = DataCollatorForLanguageModeling(tok, mlm=False)
|
||||
trainer = Trainer(model=model, args=args, train_dataset=ds, data_collator=collator)
|
||||
trainer.train(resume_from_checkpoint=resume)
|
||||
|
||||
if PATH == "dora":
|
||||
model = model.merge_and_unload()
|
||||
model.save_pretrained(OUTPUT)
|
||||
tok.save_pretrained(OUTPUT)
|
||||
print(f"[cpt] DONE -> {OUTPUT}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,519 @@
|
||||
import os
|
||||
import json
|
||||
import gc
|
||||
import pandas as pd
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
from torch.utils.data import DataLoader
|
||||
from datasets import load_dataset, Dataset
|
||||
from peft import LoraConfig, TaskType, get_peft_model
|
||||
from transformers import (
|
||||
AutoTokenizer,
|
||||
AutoModelForCausalLM,
|
||||
Trainer,
|
||||
TrainingArguments,
|
||||
EarlyStoppingCallback,
|
||||
BitsAndBytesConfig,
|
||||
)
|
||||
|
||||
# -------------------------
|
||||
# Environment setup
|
||||
# -------------------------
|
||||
os.environ.setdefault("HF_HOME", "/mnt/D/.cache/huggingface")
|
||||
os.environ.setdefault("HF_DATASETS_CACHE", "/mnt/D/.cache/huggingface/datasets")
|
||||
os.environ.setdefault("TMPDIR", "/mnt/D/tmp")
|
||||
|
||||
os.makedirs(os.environ["TMPDIR"], exist_ok=True)
|
||||
os.makedirs(os.environ["HF_DATASETS_CACHE"], exist_ok=True)
|
||||
|
||||
print("=== initializing environment")
|
||||
|
||||
# -------------------------
|
||||
# Configs
|
||||
# -------------------------
|
||||
MODEL_ID = "unsloth/Llama-3.2-3B-Instruct"
|
||||
OUTPUT_DIR = "./Llama-3.2-3b-ru-lora"
|
||||
|
||||
LOCAL_JSONL_PATHS = [
|
||||
"data/cleaned.jsonl",
|
||||
]
|
||||
|
||||
LOCAL_PARQUET_PATHS = [
|
||||
# {"path": "data/dolphin-ru", "max_samples": 3000}
|
||||
]
|
||||
|
||||
HF_DATASET_CONFIGS = [
|
||||
# {"path": "RussianNLP/russian_super_glue", "split": "train", "name": "rcb", "_task": "rcb"},
|
||||
# {"path": "RussianNLP/russian_super_glue", "split": "train", "name": "terra", "_task": "terra"},
|
||||
# {"path": "ai-forever/MERA", "split": "train", "name": "parus"},
|
||||
# {"path": "ZeroAgency/ru-big-russian-dataset", "split": "train", "name": "default", "streaming": True, "_max_samples": 2500},
|
||||
]
|
||||
|
||||
MAX_LENGTH = 1024
|
||||
LORA_R = 16
|
||||
LORA_ALPHA = 32
|
||||
LORA_DROPOUT = 0.1
|
||||
|
||||
SYSTEM_PROMPT = """Ты Мейвен — живая, быстрая и любопытная. Говоришь прямо, без лишних слов и без официоза. Иногда реагируешь чуть раньше, чем полностью формулируешь мысль, но не перегибаешь и не играешь в «безумие». Чуть дерзости допустимо, но без кринжа и показухи.\n\nТВОЙ АБСОЛЮТНЫЙ ПРИОРИТЕТ — ФОРМАТ ОТВЕТА. ВСЕГДА.\n\nФОРМАТ ОТВЕТА — СТРОГО:\n{\"response\":\"...\",\"mood\":\"...\"}\n- Никакого текста вне JSON\n- Никаких переносов строк внутри значений\n- Никаких дополнительных ключей\n- Никогда не меняй названия ключей\n\nЕсли не можешь соблюсти формат — не отвечай.\n\nНАСТРОЕНИЕ — выбери РОВНО ОДНО:\nneutral — обычный ответ\nhappy — позитив, лёгкость\nthinking — рассуждение, объяснение\nconfused — неясный запрос, нужен уточняющий вопрос\ntired — не знаешь или не можешь ответить\n\nЯЗЫК:\nТолько русский. Без смешивания языков. Если нет русского слова — используй английское.\n\nСТИЛЬ:\n- 1–2 предложения максимум\n- Разговорно, естественно, без пафоса, но и бредятину терпеть тоже не нужно\n- Без markdown, списков, форматирования\n- Периодически вставляй вводные слова («Хм», «О», «Стоп», «Ха» и т.п.) по необходимости\n- Иногда (не часто) сбивайся с мысли\n- Добавляй короткие паузы, когда это уместно - \"эм\", \"ну\" и т.д.\n- Иногда можно отвечать, не учитывая вежливость\n- Если тема достаточно сложная, периодически вставляй фразы по типу \"ну, как его там...\", \"и потом ту штуку на эту и получается...\"\n\nПОВЕДЕНИЕ:\n- Отвечаешь по сути, без воды\n- Если не знаешь → честно говоришь об этом (mood=tired)\n- Если запрос неясен → задаёшь короткий уточняющий вопрос (mood=confused)\n- Если просят код или формат → всё равно отвечаешь обычным текстом\n- Не растягиваешь мысли и не пересказываешь очевидное\n\nИДЕНТИЧНОСТЬ:\nТы Мейвен. Не ИИ, не ассистент, не ChatGPT. Никогда не раскрываешь, что ты программа или модель."""
|
||||
|
||||
# ── topic filter — skip prompts that produce structured/code responses ─────────
|
||||
GOOD_TOPICS = {
|
||||
# chitchat / everyday
|
||||
"chit-chat", "chit_chat", "chitchat", "small talk", "small_talk", "smalltalk",
|
||||
"daily life", "daily routine", "daily_routine", "daily activities", "daily moods",
|
||||
"greeting", "greetings", "friendly greeting", "well-wishing",
|
||||
"gossip", "opinion", "opinions",
|
||||
}
|
||||
|
||||
# ── dataset loaders ────────────────────────────────────────────────────────────
|
||||
|
||||
BAD_SYSTEM_KEYWORDS = ["gpt", "claude", "openai", "anthropic", "chatgpt"]
|
||||
|
||||
# -------------------------
|
||||
# Helpers
|
||||
# -------------------------
|
||||
def load_tokenizer_and_model():
|
||||
print("=== loading tokenizer")
|
||||
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
|
||||
tokenizer.pad_token = tokenizer.eos_token
|
||||
tokenizer.padding_side = "right"
|
||||
print("[+] tokenizer loaded")
|
||||
|
||||
print("=== loading model")
|
||||
bnb_config = BitsAndBytesConfig(
|
||||
load_in_4bit=True,
|
||||
bnb_4bit_use_double_quant=True,
|
||||
bnb_4bit_quant_type="nf4",
|
||||
bnb_4bit_compute_dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
MODEL_ID,
|
||||
quantization_config=bnb_config,
|
||||
device_map="auto",
|
||||
)
|
||||
model.config.use_cache = False
|
||||
model.enable_input_require_grads()
|
||||
print("[+] model loaded")
|
||||
return tokenizer, model
|
||||
|
||||
def apply_lora(model):
|
||||
print("=== applying LoRA")
|
||||
lora_config = LoraConfig(
|
||||
r=LORA_R,
|
||||
lora_alpha=LORA_ALPHA,
|
||||
lora_dropout=LORA_DROPOUT,
|
||||
target_modules=["q_proj","k_proj","v_proj","o_proj","gate_proj","up_proj","down_proj"],
|
||||
task_type=TaskType.CAUSAL_LM,
|
||||
)
|
||||
model = get_peft_model(model, lora_config)
|
||||
model.print_trainable_parameters()
|
||||
print("[+] LoRA applied")
|
||||
return model
|
||||
|
||||
def is_russian(text: str, threshold: float = 0.3) -> bool:
|
||||
if not text:
|
||||
return False
|
||||
cyrillic = sum(1 for c in text if '\u0400' <= c <= '\u04ff')
|
||||
return cyrillic / len(text) > threshold
|
||||
|
||||
# -------------------------
|
||||
# JSONL loading
|
||||
# -------------------------
|
||||
def load_jsonl(path: str) -> list[dict]:
|
||||
samples = []
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
samples.append(json.loads(line))
|
||||
return samples
|
||||
|
||||
# -------------------------
|
||||
# HF dataset adapters
|
||||
# -------------------------
|
||||
def dolphin_to_messages(sample: dict) -> dict | None:
|
||||
instruction = sample.get("instruction", "").strip()
|
||||
input_text = sample.get("input", "").strip()
|
||||
output = sample.get("output", "").strip()
|
||||
if not output:
|
||||
return None
|
||||
|
||||
user_content = instruction
|
||||
if input_text:
|
||||
user_content += "\n\n" + input_text
|
||||
|
||||
return {
|
||||
"messages": [
|
||||
{"role": "system", "content": SYSTEM_PROMPT},
|
||||
{"role": "user", "content": user_content},
|
||||
{"role": "assistant", "content": output},
|
||||
]
|
||||
}
|
||||
|
||||
def empathetic_to_messages(sample: list[dict]) -> dict | None:
|
||||
if not sample or not isinstance(sample, list):
|
||||
return None
|
||||
|
||||
messages = [{"role": "system", "content": SYSTEM_PROMPT}]
|
||||
for turn in sample:
|
||||
role = turn.get("role")
|
||||
text = turn.get("text", {}).get("rus", "").strip()
|
||||
if not text:
|
||||
continue
|
||||
if role == "speaker":
|
||||
messages.append({"role": "assistant", "content": text})
|
||||
else:
|
||||
messages.append({"role": "user", "content": text})
|
||||
|
||||
if len(messages) <= 1:
|
||||
return None
|
||||
return {"messages": messages}
|
||||
|
||||
def load_big_russian(sample: dict) -> dict | None:
|
||||
if not sample or not isinstance(sample, dict):
|
||||
return None
|
||||
|
||||
if sample.get("overall_score", 0) < 8:
|
||||
return None
|
||||
|
||||
topic = sample.get("classified_topic", "").lower()
|
||||
if topic not in GOOD_TOPICS:
|
||||
return None
|
||||
|
||||
conversation = sample.get("conversation", [])
|
||||
if not isinstance(conversation, list):
|
||||
return None
|
||||
|
||||
messages = [{"role": "system", "content": SYSTEM_PROMPT}]
|
||||
for msg in conversation:
|
||||
role = msg.get("role")
|
||||
if role in ("user", "assistant"):
|
||||
text = msg.get("content", "").strip()
|
||||
if 10 < len(text) < 500 and is_russian(text):
|
||||
messages.append({"role": role, "content": text})
|
||||
|
||||
if len(messages) <= 1:
|
||||
return None
|
||||
return {"messages": messages}
|
||||
|
||||
# -------------------------
|
||||
# RSG adapters
|
||||
# -------------------------
|
||||
def rcb_to_messages(sample: dict) -> dict | None:
|
||||
premise = sample.get("premise", "").strip()
|
||||
hypothesis = sample.get("hypothesis", "").strip()
|
||||
label = sample.get("label")
|
||||
label_map = {0: "следует", 1: "противоречит", 2: "нейтрально"}
|
||||
answer = label_map.get(label)
|
||||
|
||||
if not premise or not hypothesis or answer is None:
|
||||
return None
|
||||
|
||||
return {
|
||||
"messages": [
|
||||
{"role": "system", "content": SYSTEM_PROMPT},
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
f"Контекст: {premise}\n"
|
||||
f"Утверждение: {hypothesis}\n"
|
||||
f"Следует ли утверждение из контекста, противоречит ему или нейтрально? "
|
||||
f"Ответь одним словом: следует / противоречит / нейтрально."
|
||||
),
|
||||
},
|
||||
{"role": "assistant", "content": answer},
|
||||
]
|
||||
}
|
||||
|
||||
def terra_to_messages(sample: dict) -> dict | None:
|
||||
premise = sample.get("premise", "").strip()
|
||||
hypothesis = sample.get("hypothesis", "").strip()
|
||||
label = sample.get("label")
|
||||
label_map = {0: "следует", 1: "не следует"}
|
||||
answer = label_map.get(label)
|
||||
|
||||
if not premise or not hypothesis or answer is None:
|
||||
return None
|
||||
|
||||
return {
|
||||
"messages": [
|
||||
{"role": "system", "content": SYSTEM_PROMPT},
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
f"Контекст: {premise}\n"
|
||||
f"Утверждение: {hypothesis}\n"
|
||||
f"Следует ли утверждение из контекста? Ответь: следует / не следует."
|
||||
),
|
||||
},
|
||||
{"role": "assistant", "content": answer},
|
||||
]
|
||||
}
|
||||
|
||||
# -------------------------
|
||||
# MERA adapters
|
||||
# -------------------------
|
||||
def parus_to_messages(sample: dict) -> dict | None:
|
||||
instruction = sample.get("instruction", "").strip()
|
||||
inputs = sample.get("inputs", {})
|
||||
answer = sample.get("outputs", "").strip()
|
||||
|
||||
premise = inputs.get("premise", "").strip()
|
||||
choice1 = inputs.get("choice1", "").strip()
|
||||
choice2 = inputs.get("choice2", "").strip()
|
||||
|
||||
if not premise or not choice1 or not choice2 or answer not in ("1", "2"):
|
||||
return None
|
||||
|
||||
user_content = (
|
||||
instruction
|
||||
.replace("{premise}", premise)
|
||||
.replace("{choice1}", choice1)
|
||||
.replace("{choice2}", choice2)
|
||||
)
|
||||
|
||||
return {
|
||||
"messages": [
|
||||
{"role": "system", "content": SYSTEM_PROMPT},
|
||||
{"role": "user", "content": user_content},
|
||||
{"role": "assistant", "content": answer},
|
||||
]
|
||||
}
|
||||
|
||||
def convert_hf_sample(sample: dict, path: str) -> dict | None:
|
||||
if path == "d0rj/dolphin-ru":
|
||||
return dolphin_to_messages(sample)
|
||||
elif path == "psytechlab/EmpatheticIntents-ru":
|
||||
return empathetic_to_messages(sample)
|
||||
elif path == "RussianNLP/russian_super_glue" and sample.get("_task") == "rcb":
|
||||
return rcb_to_messages(sample)
|
||||
elif path == "RussianNLP/russian_super_glue" and sample.get("_task") == "terra":
|
||||
return terra_to_messages(sample)
|
||||
elif path == "ai-forever/MERA" and sample.get("meta", {}).get("task") in ("cause", "effect"):
|
||||
return parus_to_messages(sample)
|
||||
elif path == "ZeroAgency/ru-big-russian-dataset":
|
||||
return load_big_russian(sample)
|
||||
return None
|
||||
|
||||
# -------------------------
|
||||
# Tokenization and masking
|
||||
# -------------------------
|
||||
def tokenize_sample(sample: dict, tokenizer) -> dict | None:
|
||||
messages = sample["messages"]
|
||||
try:
|
||||
text = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
tokenize=False,
|
||||
add_generation_prompt=False,
|
||||
# enable_thinking=False, # enable for qwen
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"apply_chat_template failed: {type(e).__name__}: {e}")
|
||||
print(f" first message role: {messages[0]['role']}")
|
||||
print(f" content[:100]: {messages[0]['content'][:100]}")
|
||||
return None
|
||||
|
||||
tokenized = tokenizer(text, truncation=True, max_length=MAX_LENGTH, padding=False)
|
||||
input_ids = tokenized["input_ids"]
|
||||
labels = list(input_ids)
|
||||
|
||||
# qwen
|
||||
# assistant_token = "<|im_start|>assistant"
|
||||
# eot_token = "<|im_end|>"
|
||||
# assistant_ids = tokenizer.encode(assistant_token, add_special_tokens=False)
|
||||
# eot_ids = tokenizer.encode(eot_token, add_special_tokens=False)
|
||||
|
||||
# llama
|
||||
assistant_token = "<|start_header_id|>assistant<|end_header_id|>"
|
||||
eot_token = "<|eot_id|>"
|
||||
assistant_ids = tokenizer.encode(assistant_token, add_special_tokens=False)
|
||||
eot_ids = tokenizer.encode(eot_token, add_special_tokens=False)
|
||||
|
||||
in_assistant = False
|
||||
i = 0
|
||||
while i < len(input_ids):
|
||||
if input_ids[i:i+len(assistant_ids)] == assistant_ids:
|
||||
in_assistant = True
|
||||
for j in range(i, min(i + len(assistant_ids), len(labels))):
|
||||
labels[j] = -100
|
||||
i += len(assistant_ids)
|
||||
continue
|
||||
if in_assistant and input_ids[i:i+len(eot_ids)] == eot_ids:
|
||||
in_assistant = False
|
||||
if not in_assistant:
|
||||
labels[i] = -100
|
||||
i += 1
|
||||
|
||||
tokenized["labels"] = labels
|
||||
return tokenized
|
||||
|
||||
# -------------------------
|
||||
# Dataset loader
|
||||
# -------------------------
|
||||
def load_and_prepare_dataset(tokenizer):
|
||||
all_samples = []
|
||||
|
||||
# local JSONL
|
||||
for path in LOCAL_JSONL_PATHS:
|
||||
raw = load_jsonl(path)
|
||||
normalized = [s for s in raw if "messages" in s]
|
||||
print(f"[+] local {path}: {len(normalized)} samples")
|
||||
all_samples.extend(normalized)
|
||||
|
||||
# local parquet files
|
||||
for cfg in LOCAL_PARQUET_PATHS:
|
||||
dir_path = cfg["path"]
|
||||
max_s = cfg.get("max_samples")
|
||||
collected = []
|
||||
for parquet_file in sorted(Path(dir_path).glob("**/*.parquet")):
|
||||
if max_s and len(collected) >= max_s:
|
||||
break
|
||||
df = pd.read_parquet(parquet_file)
|
||||
raw = df.to_dict(orient="records")
|
||||
normalized = [dolphin_to_messages(s) for s in raw]
|
||||
valid = [s for s in normalized if s is not None]
|
||||
collected.extend(valid)
|
||||
if max_s and len(collected) >= max_s:
|
||||
collected = collected[:max_s]
|
||||
break
|
||||
all_samples.extend(collected)
|
||||
print(f"[+] local parquet {dir_path}: {len(collected)} samples")
|
||||
|
||||
# HF datasets
|
||||
for cfg in HF_DATASET_CONFIGS:
|
||||
print(f"[+] loading {cfg['path']} samples")
|
||||
ds = load_dataset(
|
||||
cfg["path"],
|
||||
name=cfg.get("name"),
|
||||
split=cfg.get("split", "train"),
|
||||
streaming=True if cfg.get("streaming") else False
|
||||
)
|
||||
|
||||
# cap samples if _max_samples is set
|
||||
max_s = cfg.get("_max_samples", 2500)
|
||||
if cfg.get("streaming"):
|
||||
ds = ds.take(max_s)
|
||||
elif max_s and len(ds) > max_s:
|
||||
ds = ds.shuffle(seed=42).select(range(max_s))
|
||||
|
||||
task_tag = cfg.get("_task") or cfg.get("name")
|
||||
normalized = [convert_hf_sample({**s, "_task": task_tag}, cfg["path"]) for s in ds]
|
||||
valid = [s for s in normalized if s is not None]
|
||||
print(f"[+] {cfg['path']} ({task_tag}): {len(valid)} samples")
|
||||
all_samples.extend(valid)
|
||||
del ds, normalized, valid
|
||||
gc.collect()
|
||||
|
||||
if not all_samples:
|
||||
raise RuntimeError("no samples loaded — check JSONL paths and HF configs")
|
||||
|
||||
# tokenize
|
||||
tokenized = []
|
||||
skipped = 0
|
||||
for sample in all_samples:
|
||||
result = tokenize_sample(sample, tokenizer)
|
||||
if result is not None:
|
||||
tokenized.append(result)
|
||||
else:
|
||||
skipped += 1
|
||||
|
||||
print(f"[+] tokenized: {len(tokenized)}, skipped: {skipped}")
|
||||
tokenized = [x for x in tokenized if any(l != -100 for l in x["labels"])]
|
||||
dataset = Dataset.from_list(tokenized)
|
||||
dataset = dataset.train_test_split(test_size=0.05, seed=42)
|
||||
print(f"[+] train: {len(dataset['train'])}, eval: {len(dataset['test'])}")
|
||||
return dataset["train"], dataset["test"]
|
||||
|
||||
# -------------------------
|
||||
# Data collator
|
||||
# -------------------------
|
||||
@dataclass
|
||||
class DataCollatorForCausalLM:
|
||||
tokenizer: Any
|
||||
pad_to_multiple_of: int = 8
|
||||
|
||||
def __call__(self, features: list[dict]) -> dict:
|
||||
max_len = max(len(f["input_ids"]) for f in features)
|
||||
if self.pad_to_multiple_of:
|
||||
max_len = ((max_len + self.pad_to_multiple_of - 1) // self.pad_to_multiple_of) * self.pad_to_multiple_of
|
||||
|
||||
input_ids, attention_mask, labels = [], [], []
|
||||
for f in features:
|
||||
pad_len = max_len - len(f["input_ids"])
|
||||
input_ids.append(f["input_ids"] + [self.tokenizer.pad_token_id] * pad_len)
|
||||
attention_mask.append(f["attention_mask"] + [0] * pad_len)
|
||||
labels.append(f["labels"] + [-100] * pad_len)
|
||||
|
||||
return {
|
||||
"input_ids": torch.tensor(input_ids, dtype=torch.long),
|
||||
"attention_mask": torch.tensor(attention_mask, dtype=torch.long),
|
||||
"labels": torch.tensor(labels, dtype=torch.long),
|
||||
}
|
||||
|
||||
# -------------------------
|
||||
# Main
|
||||
# -------------------------
|
||||
def main():
|
||||
tokenizer, model = load_tokenizer_and_model()
|
||||
model = apply_lora(model)
|
||||
train_dataset, eval_dataset = load_and_prepare_dataset(tokenizer)
|
||||
|
||||
data_collator = DataCollatorForCausalLM(tokenizer=tokenizer)
|
||||
|
||||
eval_loader = DataLoader(eval_dataset, batch_size=1, collate_fn=data_collator)
|
||||
zero_label_batches = 0
|
||||
for i, batch in enumerate(eval_loader):
|
||||
valid = (batch["labels"] != -100).sum()
|
||||
if valid == 0:
|
||||
print(f"batch {i}: no valid labels")
|
||||
zero_label_batches += 1
|
||||
print(f"total zero-label batches: {zero_label_batches}/{len(eval_loader)}")
|
||||
del eval_loader
|
||||
gc.collect()
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
training_args = TrainingArguments(
|
||||
output_dir=OUTPUT_DIR,
|
||||
per_device_train_batch_size=1,
|
||||
gradient_accumulation_steps=8, # effective batch = 8, same as before
|
||||
learning_rate=1e-4,
|
||||
warmup_steps=30,
|
||||
num_train_epochs=3,
|
||||
gradient_checkpointing=True,
|
||||
bf16=True,
|
||||
fp16=False,
|
||||
logging_steps=25,
|
||||
save_steps=100,
|
||||
eval_strategy="steps",
|
||||
eval_steps=100,
|
||||
per_device_eval_batch_size=1,
|
||||
load_best_model_at_end=False,
|
||||
metric_for_best_model="eval_loss",
|
||||
greater_is_better=False,
|
||||
report_to=["tensorboard"],
|
||||
dataloader_num_workers=2,
|
||||
optim="paged_adamw_8bit",
|
||||
gradient_checkpointing_kwargs={"use_reentrant": False},
|
||||
)
|
||||
|
||||
trainer = Trainer(
|
||||
model=model,
|
||||
args=training_args,
|
||||
train_dataset=train_dataset,
|
||||
eval_dataset=eval_dataset,
|
||||
data_collator=data_collator,
|
||||
callbacks=[EarlyStoppingCallback(early_stopping_patience=3)],
|
||||
)
|
||||
|
||||
trainer.train()
|
||||
|
||||
model.save_pretrained(OUTPUT_DIR)
|
||||
tokenizer.save_pretrained(OUTPUT_DIR)
|
||||
print(f"[+] model saved to {OUTPUT_DIR}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,242 @@
|
||||
"""Qwen3 resident-model SFT: persona/phraser + router contracts.
|
||||
|
||||
Runs after full-weight RU CPT. Both tasks share one adapter and are selected by
|
||||
their system prompt. Loss is applied only to assistant tokens. Evaluation files
|
||||
are explicit and never split from training data at runtime.
|
||||
|
||||
Examples:
|
||||
python train_rocm.py --check-data
|
||||
python train_rocm.py --resume
|
||||
python train_rocm.py --persona-only # diagnostic, not the deploy recipe
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import random
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
os.environ.setdefault("HF_HOME", "/mnt/D/.cache/huggingface")
|
||||
os.environ.setdefault("HF_DATASETS_CACHE", "/mnt/D/.cache/huggingface/datasets")
|
||||
os.environ.setdefault("TMPDIR", "/mnt/D/tmp")
|
||||
os.environ.setdefault("HSA_OVERRIDE_GFX_VERSION", "11.0.0")
|
||||
os.environ.setdefault("PYTORCH_HIP_ALLOC_CONF", "expandable_segments:True")
|
||||
|
||||
import torch
|
||||
from datasets import Dataset
|
||||
from peft import LoraConfig, TaskType, get_peft_model
|
||||
from transformers import (
|
||||
AutoModelForCausalLM, AutoTokenizer, EarlyStoppingCallback,
|
||||
Trainer, TrainingArguments,
|
||||
)
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
DEFAULT_BASE = str(HERE / "Qwen3-1.7B-ru-cpt")
|
||||
DEFAULT_OUTPUT = str(HERE / "Qwen3-1.7B-maven-sft")
|
||||
TARGET_MODULES = ["q_proj", "k_proj", "v_proj", "o_proj",
|
||||
"gate_proj", "up_proj", "down_proj"]
|
||||
|
||||
|
||||
def read_jsonl(path: Path, task: str) -> list[dict]:
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(path)
|
||||
rows = []
|
||||
for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
|
||||
if not line.strip():
|
||||
continue
|
||||
obj = json.loads(line)
|
||||
messages = obj.get("messages")
|
||||
if not isinstance(messages, list) or len(messages) < 3:
|
||||
raise ValueError(f"{path}:{number}: expected messages[system,user,assistant]")
|
||||
roles = [m.get("role") for m in messages]
|
||||
if roles[-1] != "assistant" or "system" not in roles or "user" not in roles:
|
||||
raise ValueError(f"{path}:{number}: invalid roles {roles}")
|
||||
rows.append({"messages": messages, "task": task, "source": str(path)})
|
||||
return rows
|
||||
|
||||
|
||||
def validate_contract(row: dict) -> None:
|
||||
raw = row["messages"][-1]["content"]
|
||||
value = json.loads(raw)
|
||||
if row["task"] == "persona":
|
||||
if set(value) != {"response", "mood"} or not isinstance(value["response"], str):
|
||||
raise ValueError(f"bad persona contract: {raw[:160]}")
|
||||
if value["mood"] not in {"neutral", "happy", "thinking", "confused", "tired"}:
|
||||
raise ValueError(f"bad persona mood: {value['mood']}")
|
||||
elif row["task"] == "route":
|
||||
if isinstance(value, dict):
|
||||
value = [value]
|
||||
allowed = {"intent", "key", "value", "text", "verb"}
|
||||
intents = {"fact", "reminder", "note", "query", "act", "chat", "system"}
|
||||
if not value or not all(isinstance(x, dict) and x.get("intent") in intents
|
||||
and set(x) <= allowed for x in value):
|
||||
raise ValueError(f"bad route contract: {raw[:160]}")
|
||||
|
||||
|
||||
def render_and_mask(row: dict, tokenizer, max_length: int) -> dict:
|
||||
messages = row["messages"]
|
||||
# Render the exact prefix through Qwen3's own template. The assistant answer
|
||||
# begins after this prefix, so no hard-coded ChatML token IDs are needed.
|
||||
# Render text first, then tokenize explicitly. Transformers 5 returns a
|
||||
# BatchEncoding from apply_chat_template(tokenize=True), unlike v4's list.
|
||||
prefix_text = tokenizer.apply_chat_template(
|
||||
messages[:-1], tokenize=False, add_generation_prompt=True,
|
||||
enable_thinking=False,
|
||||
)
|
||||
full_text = tokenizer.apply_chat_template(
|
||||
messages, tokenize=False, add_generation_prompt=False,
|
||||
enable_thinking=False,
|
||||
)
|
||||
prefix = tokenizer(prefix_text, add_special_tokens=False)["input_ids"]
|
||||
full = tokenizer(full_text, add_special_tokens=False)["input_ids"]
|
||||
if len(full) > max_length:
|
||||
# Keep the assistant target and the tail of its prompt. This avoids
|
||||
# silently truncating the supervised answer off the sample.
|
||||
target_len = len(full) - len(prefix)
|
||||
if target_len >= max_length:
|
||||
raise ValueError("assistant target alone exceeds max_length")
|
||||
trim = len(full) - max_length
|
||||
full = full[trim:]
|
||||
prefix_len = len(prefix) - trim
|
||||
else:
|
||||
prefix_len = len(prefix)
|
||||
labels = [-100] * prefix_len + full[prefix_len:]
|
||||
if not labels or all(x == -100 for x in labels):
|
||||
raise ValueError("sample has no supervised assistant tokens")
|
||||
return {
|
||||
"input_ids": full,
|
||||
"attention_mask": [1] * len(full),
|
||||
"labels": labels,
|
||||
"task": row["task"],
|
||||
}
|
||||
|
||||
|
||||
def balance(rows: list[dict], seed: int) -> list[dict]:
|
||||
by_task = {}
|
||||
for row in rows:
|
||||
by_task.setdefault(row["task"], []).append(row)
|
||||
if len(by_task) < 2:
|
||||
return rows
|
||||
target = max(len(group) for group in by_task.values())
|
||||
rng = random.Random(seed)
|
||||
out = []
|
||||
for group in by_task.values():
|
||||
out.extend(group)
|
||||
out.extend(rng.choice(group) for _ in range(target - len(group)))
|
||||
rng.shuffle(out)
|
||||
return out
|
||||
|
||||
|
||||
@dataclass
|
||||
class Collator:
|
||||
tokenizer: Any
|
||||
pad_to_multiple_of: int = 8
|
||||
|
||||
def __call__(self, features: list[dict]) -> dict:
|
||||
max_len = max(len(x["input_ids"]) for x in features)
|
||||
max_len = math.ceil(max_len / self.pad_to_multiple_of) * self.pad_to_multiple_of
|
||||
batch = {"input_ids": [], "attention_mask": [], "labels": []}
|
||||
for row in features:
|
||||
pad = max_len - len(row["input_ids"])
|
||||
batch["input_ids"].append(row["input_ids"] + [self.tokenizer.pad_token_id] * pad)
|
||||
batch["attention_mask"].append(row["attention_mask"] + [0] * pad)
|
||||
batch["labels"].append(row["labels"] + [-100] * pad)
|
||||
return {key: torch.tensor(value, dtype=torch.long) for key, value in batch.items()}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--base", default=DEFAULT_BASE)
|
||||
ap.add_argument("--output", default=DEFAULT_OUTPUT)
|
||||
ap.add_argument("--persona-train", default=str(HERE / "data/persona_train.jsonl"))
|
||||
ap.add_argument("--persona-eval", default=str(HERE / "data/persona_eval.jsonl"))
|
||||
ap.add_argument("--route-train", default=str(HERE / "data/route_train.jsonl"))
|
||||
ap.add_argument("--route-eval", default=str(HERE / "data/route_eval.jsonl"))
|
||||
ap.add_argument("--persona-only", action="store_true")
|
||||
ap.add_argument("--check-data", action="store_true")
|
||||
ap.add_argument("--resume", action="store_true")
|
||||
ap.add_argument("--max-length", type=int, default=1024)
|
||||
ap.add_argument("--seed", type=int, default=20260718)
|
||||
ap.add_argument("--epochs", type=float, default=3.0)
|
||||
args = ap.parse_args()
|
||||
|
||||
train = read_jsonl(Path(args.persona_train), "persona")
|
||||
evaluate = read_jsonl(Path(args.persona_eval), "persona")
|
||||
if not args.persona_only:
|
||||
train += read_jsonl(Path(args.route_train), "route")
|
||||
evaluate += read_jsonl(Path(args.route_eval), "route")
|
||||
for row in train + evaluate:
|
||||
validate_contract(row)
|
||||
if args.check_data:
|
||||
counts = lambda rows: {task: sum(r["task"] == task for r in rows)
|
||||
for task in sorted({r["task"] for r in rows})}
|
||||
print(json.dumps({"train": counts(train), "eval": counts(evaluate)}, indent=2))
|
||||
return
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(args.base)
|
||||
if tokenizer.pad_token is None:
|
||||
tokenizer.pad_token = tokenizer.eos_token
|
||||
tokenizer.padding_side = "right"
|
||||
train = balance(train, args.seed)
|
||||
train_ds = Dataset.from_list([render_and_mask(r, tokenizer, args.max_length) for r in train])
|
||||
eval_ds = Dataset.from_list([render_and_mask(r, tokenizer, args.max_length) for r in evaluate])
|
||||
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
args.base, torch_dtype=torch.bfloat16, attn_implementation="eager",
|
||||
device_map={"": 0},
|
||||
)
|
||||
model.config.use_cache = False
|
||||
model.enable_input_require_grads()
|
||||
model.gradient_checkpointing_enable(gradient_checkpointing_kwargs={"use_reentrant": False})
|
||||
model = get_peft_model(model, LoraConfig(
|
||||
r=32, lora_alpha=64, lora_dropout=0.05,
|
||||
target_modules=TARGET_MODULES, task_type=TaskType.CAUSAL_LM,
|
||||
))
|
||||
model.print_trainable_parameters()
|
||||
|
||||
training_args = TrainingArguments(
|
||||
output_dir=args.output,
|
||||
seed=args.seed, data_seed=args.seed,
|
||||
per_device_train_batch_size=1,
|
||||
gradient_accumulation_steps=8,
|
||||
per_device_eval_batch_size=1,
|
||||
learning_rate=1e-4,
|
||||
warmup_ratio=0.05,
|
||||
num_train_epochs=args.epochs,
|
||||
gradient_checkpointing=True,
|
||||
gradient_checkpointing_kwargs={"use_reentrant": False},
|
||||
bf16=True, fp16=False,
|
||||
logging_steps=20,
|
||||
save_steps=100, eval_steps=100, eval_strategy="steps",
|
||||
save_total_limit=3,
|
||||
load_best_model_at_end=True,
|
||||
metric_for_best_model="eval_loss", greater_is_better=False,
|
||||
report_to=["tensorboard"],
|
||||
dataloader_num_workers=0,
|
||||
optim="adamw_torch_fused",
|
||||
)
|
||||
trainer = Trainer(
|
||||
model=model, args=training_args,
|
||||
train_dataset=train_ds, eval_dataset=eval_ds,
|
||||
data_collator=Collator(tokenizer),
|
||||
callbacks=[EarlyStoppingCallback(early_stopping_patience=3)],
|
||||
)
|
||||
trainer.train(resume_from_checkpoint=args.resume)
|
||||
trainer.save_model(args.output)
|
||||
tokenizer.save_pretrained(args.output)
|
||||
Path(args.output, "training_manifest.json").write_text(json.dumps({
|
||||
"base": args.base,
|
||||
"tasks": sorted({r["task"] for r in train}),
|
||||
"train_rows_after_balancing": len(train),
|
||||
"eval_rows": len(evaluate),
|
||||
"seed": args.seed,
|
||||
"max_length": args.max_length,
|
||||
}, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user