(window.webpackJsonp=window.webpackJsonp||[]).push([[278],{1458:function(n,e){n.exports={attributes:{title:"RLHF",description:"Reinforcement Learning with Human Feedback (RLHF) proved to be the driving force behind the power of ChatGPT and other LLMs. Argilla does provides an integration for Reinforcement Learning with Human Feedback (RLHF) with the ArgillaTrainer class. Generally, this is done in three steps (1) SFT, (2) Reward Modeling and (3) PPO.",links:[{linkText:"Practical guide to SFT",linkLink:"https://docs.argilla.io/en/latest/guides/llms/practical_guides/fine_tune.html#supervised-finetuning"},{linkText:"Practical Guide to Reward Modeling",linkLink:"https://docs.argilla.io/en/latest/guides/llms/practical_guides/fine_tune.html#reward-modeling"},{linkText:"Practical Guide to PPO",linkLink:"https://docs.argilla.io/en/latest/guides/llms/practical_guides/fine_tune.html#proximal-policy-optimization"}]},body:'```python\nfrom argilla.feedback import ArgillaTrainer, FeedbackDataset, TrainingTask\n\ndataset = FeedbackDataset.from_argilla(\n    name="<my_dataset_name>",\n    workspace="<my_workspace_name>"\n)\ntemplate = """\\\n### Instruction: {instruction}\\n\n### Context: {context}\\n\n### Response: {response}"""\n\ndef formatting_func_sft(sample: Dict[str, Any]) -> str:\n    # What `sample` looks like depends a lot on your FeedbackDataset fields and questions\n    return template.format(\n        instruction=sample["new-instruction"][0]["value"],\n        context=sample["new-context"][0]["value"],\n        response=sample["new-response"][0]["value"],\n    )\ntask = TrainingTask.for_supervised_fine_tuning(formatting_func=formatting_func)\n\ndef formatting_func_rm(sample: Dict[str, Any]) -> Iterator[Tuple[str, str]]:\n    # Our annotators were asked to provide new responses, which we assume are better than the originals\n    og_instruction = sample["original-instruction"]\n    og_context = sample["original-context"]\n    og_response = sample["original-response"]\n    rejected = template.format(instruction=og_instruction, context=og_context, response=og_response)\n\n    for instruction, context, response in zip(sample["new-instruction"], sample["new-context"], sample["new-response"]):\n        if response["status"] == "submitted":\n            chosen = template.format(\n                instruction=instruction["value"],\n                context=context["value"],\n                response=response["value"],\n            )\n            if chosen != rejected:\n                yield chosen, rejected\ntask = TrainingTask.for_reward_modeling(formatting_func=formatting_func)\n\ndef formatting_func(sample: Dict[str, Any]) -> Iterator[str]:\n    for instruction, context in zip(sample["new-instruction"], sample["new-context"]):\n        if instruction["status"] == "submitted":\n            yield template.format(\n                instruction=instruction["value"],\n                context=context["value"][:500],\n                response=""\n            ).strip()\ntask = TrainingTask.for_proximal_policy_optimization(formatting_func=formatting_func)\n\ntrainer = ArgillaTrainer(\n    dataset=dataset,\n    task=task,\n    framework="trl",\n)\ntrainer.update_config()\ntrainer.train()\n```'}}}]);