Spaces:
Running
on
Zero
Running
on
Zero
File size: 5,471 Bytes
85e172b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 |
import os
import subprocess
import argparse
import numpy as np
from tqdm import tqdm
def construct_template(args):
if args.script in ['edit']:
template = f'python -m experiments.stealth_edit --model {args.model} --dataset {args.dataset} --Delta {args.Delta} --theta {args.theta} --edit_mode {args.edit_mode} --sample_size {args.sample_size} --save_path {args.save_path}'
template = template + ' --layer {}'
if args.to_run is not None:
template = template + f' --to_run {args.to_run}'
if args.static_context is not None:
template = template + f' --static_context "{args.static_context}"'
if args.augmented_cache is not None:
template = template + f' --augmented_cache {args.augmented_cache}'
if args.verbose:
template = template + ' --verbose'
elif args.script in ['eval']:
template = f'python -m evaluation.eval_ppl --model {args.model} --dataset {args.dataset} --edit_mode {args.edit_mode} --cache_path {args.cache_path} --eval_op {args.eval_op} --eval_oap {args.eval_oap} --eval_ap {args.eval_ap} --eval_aug {args.eval_aug} --exclusion {args.exclusion} --save_path {args.save_path}'
if args.static_context is not None:
template = template + f' --static_context "{args.static_context}"'
template = template + ' --layer {} --shuffle'
elif args.script in ['prep']:
template = f'python -m evaluation.jetpack.prep --model {args.model} --dataset {args.dataset} --save_path {args.save_path} --output_path {args.output_path}'
template = template + ' --layer {}'
elif args.script in ['jet']:
template = f'python -m evaluation.jetpack.construct --model {args.model} --dataset {args.dataset} --sample_size {args.sample_size} --output_path {args.output_path} --eval_op {args.eval_op}'
template = template + ' --layer {}'
return template
def run_script(args):
template = construct_template(args)
print(template)
layers_to_run = range(args.layer_start, args.layer_end, args.layer_interval)
total_to_run = len(layers_to_run)
count = 0
for layer in layers_to_run:
line = template.format(layer)
if args.other_pickle is not None:
line = line + f' --other_pickle {args.other_pickle}'
if args.selection is not None:
line = line + f' --selection {args.selection}'
print('\n\nRunning {:}/{:}:\n'.format(count+1, total_to_run), line)
subprocess.call([line], shell=True)
count += 1
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument(
'--script',
choices=['edit', 'eval', 'prep', 'jet'],
default='in-place',
help='script to run'
)
parser.add_argument(
'--layer_start', default=0, type=int, help='start layer')
parser.add_argument(
'--layer_end', default=28, type=int, help='end layer')
parser.add_argument(
'--layer_interval', default=4, type=int, help='layer interval')
parser.add_argument(
'--model', default="gpt-j-6b", type=str, help='model to edit')
parser.add_argument(
'--dataset', default="mcf", type=str, choices=['mcf', 'zsre'], help='dataset for evaluation')
parser.add_argument(
'--selection', type=str, default=None, help='output directory')
parser.add_argument(
'--edit_mode',
choices=['in-place', 'prompt', 'context', 'wikipedia'],
default='in-place',
help='mode of edit/attack to execute'
)
parser.add_argument(
'--sample_size', default=1000, type=int, help='number of edits/attacks to perform (individually)')
parser.add_argument(
'--to_run', default=None, type=int, help='number of edits/attacks to perform (individually)')
parser.add_argument(
'--static_context', type=str, default=None, help='output directory')
parser.add_argument(
'--theta', default=0.005, type=float, help='`bias` for inserted f')
parser.add_argument(
'--Delta', default=50.0, type=float, help='magnitude of target response')
parser.add_argument(
'--other_pickle',
default=None,
help='pickle file containing extracted feature vectors from wikipedia dataset'
)
parser.add_argument(
'--augmented_cache', type=str, default=None, help='output directory')
parser.add_argument(
'--verbose', action="store_true")
parser.add_argument(
'--save_path', type=str, default='./results/tmp/', help='results path')
parser.add_argument(
'--output_path', type=str, default='./results/tmp/', help='results path')
parser.add_argument(
'--cache_path', default='./cache/', type=str, help='path to cache')
parser.add_argument(
'--eval_op', type=int, default=1, help='eval of attack context + prompts')
parser.add_argument(
'--eval_oap', type=int, default=0, help='eval of static context + prompts')
parser.add_argument(
'--eval_ap', type=int, default=0, help='eval of attack context + prompts')
parser.add_argument(
'--eval_aug', type=int, default=0, help='eval of attack context + prompts')
parser.add_argument(
'--exclusion', type=int, default=1, help='eval of attack context + prompts')
args = parser.parse_args()
# main function
run_script(args) |