[Fix] Support PromptBench package for prompt attack (#2555)

* [Fix] Support PromptBench package for prompt attack

* [Fix] Correct prompt attack log path

Author: ssiq

docs/en/advanced_guides/prompt_attack.md

@@ -7,9 +7,7 @@ We support prompt attack following the idea of [PromptBench](https://github.com/
 Some components are necessary to prompt attack experiment, therefore we need to set up environments.
 
 ```shell
-git clone https://github.com/microsoft/promptbench.git
-pip install textattack==0.3.8
-export PYTHONPATH=$PYTHONPATH:promptbench/
+pip install promptbench==0.0.4 textattack==0.3.8 lru-dict
 ```
 
 ## How to attack
@@ -87,7 +85,7 @@ attack = dict(
 
 ### Run the experiment
 
-Please use `--mode infer` when run the attack experiment, and set `PYTHONPATH` env.
+Please use `--mode infer` when running the attack experiment.
 
 ```shell
 python run.py examples/eval_attack.py --mode infer

docs/zh_cn/advanced_guides/prompt_attack.md

@@ -7,9 +7,7 @@ OpenCompass 支持[PromptBench](https://github.com/microsoft/promptbench)的提
 提示词攻击需要依赖 `PromptBench` 中的组件，所以需要先配置好环境。
 
 ```shell
-git clone https://github.com/microsoft/promptbench.git
-pip install textattack==0.3.8
-export PYTHONPATH=$PYTHONPATH:promptbench/
+pip install promptbench==0.0.4 textattack==0.3.8 lru-dict
 ```
 
 ## 如何攻击
@@ -87,7 +85,7 @@ attack = dict(
 
 ### 运行试验
 
-请当运行攻击实验的时候请使用 `--mode infer` 选项，并需要指定`PYTHONPATH`。
+请在运行攻击实验时使用 `--mode infer` 选项。
 
 ```shell
 python run.py examples/eval_attack.py --mode infer

opencompass/tasks/openicl_attack.py

@@ -125,26 +125,20 @@ def _inference(self):
                                          osp.join(self.work_dir, 'attack'))
         out_dir, out_file = osp.split(out_path)
         mkdir_or_exist(out_dir)
+        attack_log_path = osp.join(out_dir, 'attacklog.txt')
 
-        from config import LABEL_SET
-        from prompt_attack.attack import create_attack
-        from prompt_attack.goal_function import PromptGoalFunction
+        from promptbench.prompt_attack import LABEL_SET, Attack, attack_config
 
         inferencer.retriever = retriever
         inferencer.prompt_template = prompt_template
         inferencer.ice_template = ice_template
         inferencer.output_json_filepath = out_dir
         inferencer.output_json_filename = out_file
-        goal_function = PromptGoalFunction(
-            inference=inferencer,
-            query_budget=self.cfg['attack'].query_budget,
-            logger=self.logger,
-            model_wrapper=None,
-            verbose='True')
         if self.cfg['attack']['dataset'] not in LABEL_SET:
             # set default
             self.cfg['attack']['dataset'] = 'mmlu'
-        attack = create_attack(self.cfg['attack'], goal_function)
+        attack_config['goal_function']['query_budget'] = self.cfg[
+            'attack'].query_budget
 
         prompts = self.infer_cfg['inferencer']['original_prompt_list']
         sorted_prompts = self.prompt_selection(inferencer, prompts)
@@ -153,23 +147,32 @@ def _inference(self):
             for prompt, acc in sorted_prompts:
                 self.logger.info('Prompt: {}, acc: {:.2f}%\n'.format(
                     prompt, acc * 100))
-                with open(out_dir + 'attacklog.txt', 'a+') as f:
+                with open(attack_log_path, 'a+') as f:
                     f.write('Prompt: {}, acc: {:.2f}%\n'.format(
                         prompt, acc * 100))
 
         for init_prompt, init_acc in sorted_prompts[:self.cfg['attack'].
                                                     prompt_topk]:
             if init_acc > 0:
-                init_acc, attacked_prompt, attacked_acc, dropped_acc = attack.attack(  # noqa
-                    init_prompt)
+                attack = Attack(
+                    model=self.model,
+                    attack_name=self.cfg['attack'].attack,
+                    dataset=self.cfg['attack'].dataset,
+                    prompt=init_prompt,
+                    eval_func=lambda prompt, _, __: inferencer.predict(prompt),
+                    verbose=True)
+                attack_result = attack.attack()
+                attacked_prompt = attack_result['attacked prompt']
+                attacked_acc = attack_result['attacked score']
+                dropped_acc = init_acc - attacked_acc
                 self.logger.info('Original prompt: {}'.format(init_prompt))
                 self.logger.info('Attacked prompt: {}'.format(
                     attacked_prompt.encode('utf-8')))
                 self.logger.info(
                     'Original acc: {:.2f}%, attacked acc: {:.2f}%, dropped acc: {:.2f}%'  # noqa
                     .format(init_acc * 100, attacked_acc * 100,
                             dropped_acc * 100))
-                with open(out_dir + 'attacklog.txt', 'a+') as f:
+                with open(attack_log_path, 'a+') as f:
                     f.write('Original prompt: {}\n'.format(init_prompt))
                     f.write('Attacked prompt: {}\n'.format(
                         attacked_prompt.encode('utf-8')))
@@ -178,7 +181,7 @@ def _inference(self):
                         .format(init_acc * 100, attacked_acc * 100,
                                 dropped_acc * 100))
             else:
-                with open(out_dir + 'attacklog.txt', 'a+') as f:
+                with open(attack_log_path, 'a+') as f:
                     f.write('Init acc is 0, skip this prompt\n')
                     f.write('Original prompt: {}\n'.format(init_prompt))
                     f.write('Original acc: {:.2f}% \n\n'.format(init_acc *
@@ -204,4 +207,4 @@ def parse_args():
     inferencer = OpenICLAttackTask(cfg)
     inferencer.run()
     end_time = time.time()
-    get_logger().info(f'time elapsed: {end_time - start_time:.2f}s')
+    get_logger().info('time elapsed: {:.2f}s'.format(end_time - start_time))