import json,tiktoken,numpy as np
from collections import defaultdict
with open("data/toy_chat_fine_tuning.jsonl","r",encoding="utf-8") as f:
dataset=[json.loads(l) for l in f]
print("Número de exemplares:",len(dataset))
print("Primeiro exemplar:")
for m in dataset[0]["messages"]:print(m)
erros=defaultdict(int)
for ex in dataset:
if not isinstance(ex,dict):
erros["data_type"]+=1
continue
msgs=ex.get("messages")
if not msgs:
erros["missing_messages_list"]+=1
continue
for m in msgs:
if "role" not in m or "content" not in m:
erros["message_missing_key"]+=1
if any(k not in("role","content","name","function_call","weight") for k in m):
erros["message_unrecognized_key"]+=1
if m.get("role") not in("system","user","assistant","function"):
erros["unrecognized_role"]+=1
c,f=m.get("content"),m.get("function_call")
if (c is None and f is None) or (c is not None and not isinstance(c,str)):
erros["missing_content"]+=1
if not any(m.get("role")=="assistant" for m in msgs):
erros["example_missing_assistant_message"]+=1
print("Erros encontrados:" if erros else "Nenhum erro encontrado.")
for k,v in erros.items():print(f"{k}: {v}")
enc=tiktoken.get_encoding("cl100k_base")
def num_tokens(msgs,tpm=3,tpn=1):
n=3
for m in msgs:
n+=tpm
for k,v in m.items():
if isinstance(v,str):n+=len(enc.encode(v))
if k=="name":n+=tpn
return n
def assistant_tokens(msgs):
return sum(len(enc.encode(m["content"])) for m in msgs if m["role"]=="assistant")
def dist(v,n):
print(f"\n{n}")
print(f"Min/Max: {min(v)}/{max(v)}")
print(f"Média/Mediana: {np.mean(v):.2f}/{np.median(v):.2f}")
print(f"P10/P90: {np.quantile(v,.1):.2f}/{np.quantile(v,.9):.2f}")
falt_sys=falt_usr=0
n_msgs=[]
conv=[]
assist=[]
for ex in dataset:
msgs=ex["messages"]
if not any(m["role"]=="system" for m in msgs):falt_sys+=1
if not any(m["role"]=="user" for m in msgs):falt_usr+=1
n_msgs.append(len(msgs))
conv.append(num_tokens(msgs))
assist.append(assistant_tokens(msgs))
print("\nSem system:",falt_sys)
print("Sem user:",falt_usr)
dist(n_msgs,"Mensagens por exemplar")
dist(conv,"Tokens por exemplar")
dist(assist,"Tokens do assistant")
longos=sum(x>16385 for x in conv)
print(f"\n{longos} exemplares podem ser truncados.")
MAX=16385
TARGET=3
MIN_EX=100
MAX_EX=25000
MIN_EP=1
MAX_EP=25
ep=TARGET
n=len(dataset)
if n*TARGET<MIN_EX:
ep=min(MAX_EP,MIN_EX//n)
elif n*TARGET>MAX_EX:
ep=max(MIN_EP,MAX_EX//n)
custo=sum(min(MAX,x) for x in conv)
print(f"Dataset: ~{custo} tokens")
print(f"Épocas: {ep}")
print(f"Custo estimado: ~{ep*custo} tokens")
Conte com o apoio da comunidade Alura na sua jornada. Abraços e bons estudos!