|
|
|
@ -221,8 +221,6 @@ def perplexity(model, data, batch_size=32):
|
|
|
|
# Progress update
|
|
|
|
# Progress update
|
|
|
|
processed = min(i + batch_size, total_sequences)
|
|
|
|
processed = min(i + batch_size, total_sequences)
|
|
|
|
print(f"\rppl {processed}/{total_sequences} ({processed/total_sequences*100:.1f}%)", end="", flush=True)
|
|
|
|
print(f"\rppl {processed}/{total_sequences} ({processed/total_sequences*100:.1f}%)", end="", flush=True)
|
|
|
|
|
|
|
|
|
|
|
|
print() # Final newline
|
|
|
|
|
|
|
|
return np.exp(total_loss_sum / total_tokens_count)
|
|
|
|
return np.exp(total_loss_sum / total_tokens_count)
|
|
|
|
|
|
|
|
|
|
|
|
#################################### Model #########################################mo
|
|
|
|
#################################### Model #########################################mo
|
|
|
|
|