add files

Former-commit-id: 8999421c9e34ca25384a3adcb879f4f5afdedda4 [formerly 0b6e5924e22331500a9279183550e81146029249]
Former-commit-id: 8bafbec9710aa552ecd5dbf010089eee435ae9c9

build/lib/fawkes/__init__.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+# @Date    : 2020-07-01
+# @Author  : Shawn Shan (shansixiong@cs.uchicago.edu)
+# @Link    : https://www.shawnshan.com/
+
+
+__version__ = '0.0.2'
+
+from .differentiator import FawkesMaskGeneration
+from .utils import load_extractor, init_gpu, select_target_label, dump_image, reverse_process_cloaked, \
+    Faces
+from .protection import main
+import logging
+import sys
+import os
+logging.getLogger('tensorflow').disabled = True
+
+
+__all__ = (
+    '__version__',
+    'FawkesMaskGeneration', 'load_extractor',
+    'init_gpu',
+    'select_target_label', 'dump_image', 'reverse_process_cloaked', 'Faces', 'main'
+)

build/lib/fawkes/differentiator.py

@@ -10,7 +10,7 @@ from decimal import Decimal
 
 import numpy as np
 import tensorflow as tf
-from utils import preprocess, reverse_preprocess
+from .utils import preprocess, reverse_preprocess
 
 
 class FawkesMaskGeneration:
@@ -47,7 +47,7 @@ class FawkesMaskGeneration:
                  max_iterations=MAX_ITERATIONS, initial_const=INITIAL_CONST,
                  intensity_range=INTENSITY_RANGE, l_threshold=L_THRESHOLD,
                  max_val=MAX_VAL, keep_final=KEEP_FINAL, maximize=MAXIMIZE, image_shape=IMAGE_SHAPE,
-                 verbose=0, ratio=RATIO, limit_dist=LIMIT_DIST):
+                 verbose=0, ratio=RATIO, limit_dist=LIMIT_DIST, faces=None):
 
         assert intensity_range in {'raw', 'imagenet', 'inception', 'mnist'}
 
@@ -69,10 +69,12 @@ class FawkesMaskGeneration:
         self.ratio = ratio
         self.limit_dist = limit_dist
         self.single_shape = list(image_shape)
+        self.faces = faces
 
         self.input_shape = tuple([self.batch_size] + self.single_shape)
 
         self.bottleneck_shape = tuple([self.batch_size] + self.single_shape)
+        # self.bottleneck_shape = tuple([self.batch_size, bottleneck_model_ls[0].output_shape[-1]])
 
         # the variable we're going to optimize over
         self.modifier = tf.Variable(np.zeros(self.input_shape, dtype=np.float32))
@@ -149,8 +151,6 @@ class FawkesMaskGeneration:
                      self.dist_raw,
                      tf.zeros_like(self.dist_raw)))
         self.dist_sum = tf.reduce_sum(tf.where(self.mask, self.dist, tf.zeros_like(self.dist)))
-        # self.dist_sum = 1e-5 * tf.reduce_sum(self.dist)
-        # self.dist_raw_sum = self.dist_sum
 
         def resize_tensor(input_tensor, model_input_shape):
             if input_tensor.shape[1:] == model_input_shape or model_input_shape[1] is None:
@@ -171,16 +171,14 @@ class FawkesMaskGeneration:
 
             self.bottleneck_a = bottleneck_model(cur_aimg_input)
             if self.MIMIC_IMG:
-                # cur_timg_input = resize_tensor(self.timg_input, model_input_shape)
-                # cur_simg_input = resize_tensor(self.simg_input, model_input_shape)
                 cur_timg_input = self.timg_input
                 cur_simg_input = self.simg_input
                 self.bottleneck_t = calculate_direction(bottleneck_model, cur_timg_input, cur_simg_input)
-                # self.bottleneck_t = bottleneck_model(cur_timg_input)
             else:
                 self.bottleneck_t = self.bottleneck_t_raw
 
             bottleneck_diff = self.bottleneck_t - self.bottleneck_a
+
             scale_factor = tf.sqrt(tf.reduce_sum(tf.square(self.bottleneck_t), axis=1))
 
             cur_bottlesim = tf.sqrt(tf.reduce_sum(tf.square(bottleneck_diff), axis=1))
@@ -189,7 +187,6 @@ class FawkesMaskGeneration:
 
             self.bottlesim += cur_bottlesim
 
-            # self.bottlesim_push += cur_bottlesim_push_sum
             self.bottlesim_sum += cur_bottlesim_sum
 
         # sum up the losses
@@ -202,20 +199,13 @@ class FawkesMaskGeneration:
                                                self.loss,
                                                tf.zeros_like(self.loss)))
 
-        # self.loss_sum = self.dist_sum + tf.reduce_sum(self.bottlesim)
-        # import pdb
-        # pdb.set_trace()
-        # self.loss_sum = tf.reduce_sum(tf.where(self.mask, self.loss, tf.zeros_like(self.loss)))
-
-        # Setup the Adadelta optimizer and keep track of variables
-        # we're creating
         start_vars = set(x.name for x in tf.global_variables())
         self.learning_rate_holder = tf.placeholder(tf.float32, shape=[])
+
         optimizer = tf.train.AdadeltaOptimizer(self.learning_rate_holder)
         # optimizer = tf.train.AdamOptimizer(self.learning_rate_holder)
 
-        self.train = optimizer.minimize(self.loss_sum,
+        self.train = optimizer.minimize(self.loss_sum, var_list=[self.modifier])
-                                        var_list=[self.modifier])
         end_vars = tf.global_variables()
         new_vars = [x for x in end_vars if x.name not in start_vars]
 
@@ -297,6 +287,7 @@ class FawkesMaskGeneration:
         LR = self.learning_rate
         nb_imgs = source_imgs.shape[0]
         mask = [True] * nb_imgs + [False] * (self.batch_size - nb_imgs)
+        # mask = [True] * self.batch_size
         mask = np.array(mask, dtype=np.bool)
 
         source_imgs = np.array(source_imgs)
@@ -317,19 +308,34 @@ class FawkesMaskGeneration:
             timg_tanh_batch = np.zeros(self.input_shape)
         else:
             timg_tanh_batch = np.zeros(self.bottleneck_shape)
+
         weights_batch = np.zeros(self.bottleneck_shape)
         simg_tanh_batch[:nb_imgs] = simg_tanh[:nb_imgs]
         timg_tanh_batch[:nb_imgs] = timg_tanh[:nb_imgs]
         weights_batch[:nb_imgs] = weights[:nb_imgs]
         modifier_batch = np.ones(self.input_shape) * 1e-6
 
-        self.sess.run(self.setup,
+        temp_images = []
-                      {self.assign_timg_tanh: timg_tanh_batch,
+
-                       self.assign_simg_tanh: simg_tanh_batch,
+        # set the variables so that we don't have to send them over again
-                       self.assign_const: CONST,
+        if self.MIMIC_IMG:
-                       self.assign_mask: mask,
+            self.sess.run(self.setup,
-                       self.assign_weights: weights_batch,
+                          {self.assign_timg_tanh: timg_tanh_batch,
-                       self.assign_modifier: modifier_batch})
+                           self.assign_simg_tanh: simg_tanh_batch,
+                           self.assign_const: CONST,
+                           self.assign_mask: mask,
+                           self.assign_weights: weights_batch,
+                           self.assign_modifier: modifier_batch})
+        else:
+            # if directly mimicking a vector, use assign_bottleneck_t_raw
+            # in setup
+            self.sess.run(self.setup,
+                          {self.assign_bottleneck_t_raw: timg_tanh_batch,
+                           self.assign_simg_tanh: simg_tanh_batch,
+                           self.assign_const: CONST,
+                           self.assign_mask: mask,
+                           self.assign_weights: weights_batch,
+                           self.assign_modifier: modifier_batch})
 
         best_bottlesim = [0] * nb_imgs if self.maximize else [np.inf] * nb_imgs
         best_adv = np.zeros_like(source_imgs)
@@ -347,6 +353,7 @@ class FawkesMaskGeneration:
                      dist_raw_sum,
                      bottlesim_sum / nb_imgs))
 
+        finished_idx = set()
         try:
             total_distance = [0] * nb_imgs
 
@@ -369,8 +376,14 @@ class FawkesMaskGeneration:
                     [self.dist_raw,
                      self.bottlesim,
                      self.aimg_input])
+
+                all_clear = True
                 for e, (dist_raw, bottlesim, aimg_input) in enumerate(
                         zip(dist_raw_list, bottlesim_list, aimg_input_list)):
+
+                    if e in finished_idx:
+                        continue
+
                     if e >= nb_imgs:
                         break
                     if (bottlesim < best_bottlesim[e] and bottlesim > total_distance[e] * 0.1 and (
@@ -379,40 +392,55 @@ class FawkesMaskGeneration:
                         best_bottlesim[e] = bottlesim
                         best_adv[e] = aimg_input
 
-                if iteration != 0 and iteration % (self.MAX_ITERATIONS // 3) == 0:
+                    # if iteration > 20 and (dist_raw >= self.l_threshold or iteration == self.MAX_ITERATIONS - 1):
-                    # LR = LR / 2
+                    #     finished_idx.add(e)
+                    #     print("{} finished at dist {}".format(e, dist_raw))
+                    #     best_bottlesim[e] = bottlesim
+                    #     best_adv[e] = aimg_input
+                    #
+                    all_clear = False
+
+                if all_clear:
+                    break
+
+                if iteration != 0 and iteration % (self.MAX_ITERATIONS // 2) == 0:
+                    LR = LR / 2
                     print("Learning Rate: ", LR)
 
-                if iteration % (self.MAX_ITERATIONS // 10) == 0:
+                if iteration % (self.MAX_ITERATIONS // 5) == 0:
                     if self.verbose == 1:
-                        loss_sum = float(self.sess.run(self.loss_sum))
-                        dist_sum = float(self.sess.run(self.dist_sum))
-                        thresh_over = (dist_sum /
-                                       self.batch_size /
-                                       self.l_threshold *
-                                       100)
                         dist_raw_sum = float(self.sess.run(self.dist_raw_sum))
                         bottlesim_sum = self.sess.run(self.bottlesim_sum)
-                        print('ITER %4d: Total loss: %.4E; perturb: %.6f (%.2f%% over, raw: %.6f); sim: %f'
+                        print('ITER %4d perturb: %.5f; sim: %f'
-                              % (iteration,
+                              % (iteration, dist_raw_sum / nb_imgs, bottlesim_sum / nb_imgs))
-                                 Decimal(loss_sum),
+
-                                 dist_sum,
+                        # protected_images = aimg_input_list
-                                 thresh_over,
+                        #
-                                 dist_raw_sum,
+                        # orginal_images = np.copy(self.faces.cropped_faces)
-                                 bottlesim_sum / nb_imgs))
+                        # cloak_perturbation = reverse_process_cloaked(protected_images) - reverse_process_cloaked(
+                        #     orginal_images)
+                        # final_images = self.faces.merge_faces(cloak_perturbation)
+                        #
+                        # for p_img, img in zip(protected_images, final_images):
+                        #     dump_image(reverse_process_cloaked(p_img),
+                        #                "/home/shansixioing/fawkes/data/emily/emily_cloaked_cropped{}.png".format(iteration),
+                        #                format='png')
+                        #
+                        #     dump_image(img,
+                        #                "/home/shansixioing/fawkes/data/emily/emily_cloaked_{}.png".format(iteration),
+                        #                format='png')
+
         except KeyboardInterrupt:
             pass
 
         if self.verbose == 1:
             loss_sum = float(self.sess.run(self.loss_sum))
             dist_sum = float(self.sess.run(self.dist_sum))
-            thresh_over = (dist_sum / self.batch_size / self.l_threshold * 100)
             dist_raw_sum = float(self.sess.run(self.dist_raw_sum))
             bottlesim_sum = float(self.sess.run(self.bottlesim_sum))
-            print('END:       Total loss: %.4E; perturb: %.6f (%.2f%% over, raw: %.6f); sim: %f'
+            print('END:       Total loss: %.4E; perturb: %.6f (raw: %.6f); sim: %f'
                   % (Decimal(loss_sum),
                      dist_sum,
-                     thresh_over,
                      dist_raw_sum,
                      bottlesim_sum / nb_imgs))
 

build/lib/fawkes/protection.py

@@ -1,3 +1,7 @@
+# from __future__ import absolute_import
+# from __future__ import division
+# from __future__ import print_function
+
 import argparse
 import glob
 import os
@@ -5,106 +9,141 @@ import random
 import sys
 
 import numpy as np
-from differentiator import FawkesMaskGeneration
+
-from keras.applications.vgg16 import preprocess_input
+from .differentiator import FawkesMaskGeneration
-from keras.preprocessing import image
+from .utils import load_extractor, init_gpu, select_target_label, dump_image, reverse_process_cloaked, \
-from skimage.transform import resize
+    Faces
-from tensorflow import set_random_seed
-from utils import load_extractor, init_gpu, select_target_label, dump_image, reverse_process_cloaked
 
 random.seed(12243)
 np.random.seed(122412)
-set_random_seed(12242)
 
-BATCH_SIZE = 1
+BATCH_SIZE = 32
-MAX_ITER = 1000
 
 
-def generate_cloak_images(sess, feature_extractors, image_X, target_X=None, th=0.01):
+def generate_cloak_images(sess, feature_extractors, image_X, target_emb=None, th=0.01, faces=None, sd=1e9, lr=2,
+                          max_step=500):
     batch_size = BATCH_SIZE if len(image_X) > BATCH_SIZE else len(image_X)
 
     differentiator = FawkesMaskGeneration(sess, feature_extractors,
                                           batch_size=batch_size,
                                           mimic_img=True,
                                           intensity_range='imagenet',
-                                          initial_const=args.sd,
+                                          initial_const=sd,
-                                          learning_rate=args.lr,
+                                          learning_rate=lr,
-                                          max_iterations=MAX_ITER,
+                                          max_iterations=max_step,
                                           l_threshold=th,
-                                          verbose=1, maximize=False, keep_final=False, image_shape=image_X.shape[1:])
+                                          verbose=1, maximize=False, keep_final=False, image_shape=image_X.shape[1:],
+                                          faces=faces)
 
-    cloaked_image_X = differentiator.attack(image_X, target_X)
+    cloaked_image_X = differentiator.attack(image_X, target_emb)
     return cloaked_image_X
 
 
-def extract_faces(img):
+def check_imgs(imgs):
-    #  foo
+    if np.max(imgs) <= 1 and np.min(imgs) >= 0:
-    return preprocess_input(resize(img, (224, 224)))
+        imgs = imgs * 255.0
+    elif np.max(imgs) <= 255 and np.min(imgs) >= 0:
+        pass
+    else:
+        raise Exception("Image values ")
+    return imgs
 
 
-def fawkes():
+def main(*argv):
-    assert os.path.exists(args.directory)
+    if not argv:
-    assert os.path.isdir(args.directory)
+        argv = list(sys.argv)
+
+    # attach SIGPIPE handler to properly handle broken pipe
+    try:  # sigpipe not available under windows. just ignore in this case
+        import signal
+        signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+    except Exception as e:
+        pass
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--directory', '-d', type=str,
+                        help='directory that contain images for cloaking', default='imgs/')
+
+    parser.add_argument('--gpu', type=str,
+                        help='GPU id', default='0')
+
+    parser.add_argument('--mode', type=str,
+                        help='cloak generation mode', default='high')
+    parser.add_argument('--feature-extractor', type=str,
+                        help="name of the feature extractor used for optimization",
+                        default="high_extract")
+
+    parser.add_argument('--th', type=float, default=0.01)
+    parser.add_argument('--max-step', type=int, default=500)
+    parser.add_argument('--sd', type=int, default=1e9)
+    parser.add_argument('--lr', type=float, default=2)
+
+    parser.add_argument('--separate_target', action='store_true')
+
+    parser.add_argument('--format', type=str,
+                        help="final image format",
+                        default="jpg")
+    args = parser.parse_args(argv[1:])
+
+    if args.mode == 'low':
+        args.feature_extractor = "high_extract"
+        args.th = 0.003
+    elif args.mode == 'mid':
+        args.feature_extractor = "high_extract"
+        args.th = 0.005
+    elif args.mode == 'high':
+        args.feature_extractor = "high_extract"
+        args.th = 0.007
+    elif args.mode == 'ultra':
+        args.feature_extractor = "high_extract"
+        args.th = 0.01
+    elif args.mode == 'custom':
+        pass
+    else:
+        raise Exception("mode must be one of 'low', 'mid', 'high', 'ultra', 'custom'")
+
+    assert args.format in ['png', 'jpg', 'jpeg']
+    if args.format == 'jpg':
+        args.format = 'jpeg'
 
     sess = init_gpu(args.gpu)
-
+    fs_names = [args.feature_extractor]
-    print("Loading {} for optimization".format(args.feature_extractor))
+    feature_extractors_ls = [load_extractor(name) for name in fs_names]
-
-    feature_extractors_ls = [load_extractor(args.feature_extractor)]
 
     image_paths = glob.glob(os.path.join(args.directory, "*"))
     image_paths = [path for path in image_paths if "_cloaked" not in path.split("/")[-1]]
+    if not image_paths:
+        print("No images in the directory")
+        exit(1)
 
-    orginal_images = [extract_faces(image.img_to_array(image.load_img(cur_path))) for cur_path in
+    faces = Faces(image_paths, sess)
-                      image_paths]
 
+    orginal_images = faces.cropped_faces
     orginal_images = np.array(orginal_images)
 
-    if args.seperate_target:
+    if args.separate_target:
-        target_images = []
+        target_embedding = []
         for org_img in orginal_images:
             org_img = org_img.reshape([1] + list(org_img.shape))
-            tar_img = select_target_label(org_img, feature_extractors_ls, [args.feature_extractor])
+            tar_emb = select_target_label(org_img, feature_extractors_ls, fs_names)
-            target_images.append(tar_img)
+            target_embedding.append(tar_emb)
-        target_images = np.concatenate(target_images)
+        target_embedding = np.concatenate(target_embedding)
     else:
-        target_images = select_target_label(orginal_images, feature_extractors_ls, [args.feature_extractor])
+        target_embedding = select_target_label(orginal_images, feature_extractors_ls, fs_names)
-
-    # file_name = args.directory.split("/")[-1]
-    # os.makedirs(args.result_directory, exist_ok=True)
-    # os.makedirs(os.path.join(args.result_directory, file_name), exist_ok=True)
 
     protected_images = generate_cloak_images(sess, feature_extractors_ls, orginal_images,
-                                             target_X=target_images, th=args.th)
+                                             target_emb=target_embedding, th=args.th, faces=faces, sd=args.sd,
+                                             lr=args.lr, max_step=args.max_step)
 
-    for p_img, path in zip(protected_images, image_paths):
+    faces.cloaked_cropped_faces = protected_images
-        p_img = reverse_process_cloaked(p_img)
-        file_name = "{}_cloaked.jpeg".format(".".join(path.split(".")[:-1]))
-        dump_image(p_img, file_name, format="JPEG")
 
+    cloak_perturbation = reverse_process_cloaked(protected_images) - reverse_process_cloaked(orginal_images)
+    final_images = faces.merge_faces(cloak_perturbation)
 
-def parse_arguments(argv):
+    for p_img, cloaked_img, path in zip(final_images, protected_images, image_paths):
-    parser = argparse.ArgumentParser()
+        file_name = "{}_{}_{}_cloaked.{}".format(".".join(path.split(".")[:-1]), args.mode, args.th, args.format)
-    parser.add_argument('--gpu', type=str,
+        dump_image(p_img, file_name, format=args.format)
-                        help='GPU id', default='0')
-    parser.add_argument('--directory', type=str,
-                        help='directory that contain images for cloaking', default='imgs/')
-
-    parser.add_argument('--feature-extractor', type=str,
-                        help="name of the feature extractor used for optimization",
-                        default="webface_dense_robust_extract")
-
-    parser.add_argument('--th', type=float, default=0.005)
-    parser.add_argument('--sd', type=int, default=1e9)
-    parser.add_argument('--protect_class', type=str, default=None)
-    parser.add_argument('--lr', type=float, default=1)
-
-    parser.add_argument('--result_directory', type=str, default="../results")
-    parser.add_argument('--seperate_target', action='store_true')
-
-    return parser.parse_args(argv)
 
 
 if __name__ == '__main__':
-    args = parse_arguments(sys.argv[1:])
+    main(*sys.argv)
-    fawkes()

build/lib/fawkes/utils.py

@@ -1,19 +1,30 @@
+import glob
+import gzip
 import json
 import os
 import pickle
 import random
+import sys
 
+stderr = sys.stderr
+sys.stderr = open(os.devnull, 'w')
 import keras
+
+sys.stderr = stderr
 import keras.backend as K
 import numpy as np
 import tensorflow as tf
-from keras.applications.vgg16 import preprocess_input
+from PIL import Image, ExifTags
+# from keras.applications.vgg16 import preprocess_input
 from keras.layers import Dense, Activation
 from keras.models import Model
 from keras.preprocessing import image
-from keras.utils import to_categorical
+from keras.utils import get_file
+from skimage.transform import resize
 from sklearn.metrics import pairwise_distances
 
+from .align_face import align, aligner
+
 
 def clip_img(X, preprocessing='raw'):
     X = reverse_preprocess(X, preprocessing)
@@ -22,6 +33,91 @@ def clip_img(X, preprocessing='raw'):
     return X
 
 
+def load_image(path):
+    img = Image.open(path)
+    if img._getexif() is not None:
+        for orientation in ExifTags.TAGS.keys():
+            if ExifTags.TAGS[orientation] == 'Orientation':
+                break
+
+        exif = dict(img._getexif().items())
+        if orientation in exif.keys():
+            if exif[orientation] == 3:
+                img = img.rotate(180, expand=True)
+            elif exif[orientation] == 6:
+                img = img.rotate(270, expand=True)
+            elif exif[orientation] == 8:
+                img = img.rotate(90, expand=True)
+            else:
+                pass
+    img = img.convert('RGB')
+    image_array = image.img_to_array(img)
+
+    return image_array
+
+
+class Faces(object):
+    def __init__(self, image_paths, sess):
+        self.aligner = aligner(sess)
+        self.org_faces = []
+        self.cropped_faces = []
+        self.cropped_faces_shape = []
+        self.cropped_index = []
+        self.callback_idx = []
+        for i, p in enumerate(image_paths):
+            cur_img = load_image(p)
+            self.org_faces.append(cur_img)
+            align_img = align(cur_img, self.aligner, margin=0.7)
+            cur_faces = align_img[0]
+
+            cur_shapes = [f.shape[:-1] for f in cur_faces]
+
+            cur_faces_square = []
+            for img in cur_faces:
+                long_size = max([img.shape[1], img.shape[0]])
+                base = np.zeros((long_size, long_size, 3))
+                base[0:img.shape[0], 0:img.shape[1], :] = img
+                cur_faces_square.append(base)
+
+            cur_index = align_img[1]
+            cur_faces_square = [resize(f, (224, 224)) for f in cur_faces_square]
+            self.cropped_faces_shape.extend(cur_shapes)
+            self.cropped_faces.extend(cur_faces_square)
+            self.cropped_index.extend(cur_index)
+            self.callback_idx.extend([i] * len(cur_faces_square))
+
+        if not self.cropped_faces:
+            print("No faces detected")
+            exit(1)
+
+        self.cropped_faces = np.array(self.cropped_faces)
+
+        self.cropped_faces = preprocess(self.cropped_faces, 'imagenet')
+
+        self.cloaked_cropped_faces = None
+        self.cloaked_faces = np.copy(self.org_faces)
+
+    def get_faces(self):
+        return self.cropped_faces
+
+    def merge_faces(self, cloaks):
+
+        self.cloaked_faces = np.copy(self.org_faces)
+
+        for i in range(len(self.cropped_faces)):
+            cur_cloak = cloaks[i]
+            org_shape = self.cropped_faces_shape[i]
+            old_square_shape = max([org_shape[0], org_shape[1]])
+            reshape_cloak = resize(cur_cloak, (old_square_shape, old_square_shape))
+            reshape_cloak = reshape_cloak[0:org_shape[0], 0:org_shape[1], :]
+
+            callback_id = self.callback_idx[i]
+            bb = self.cropped_index[i]
+            self.cloaked_faces[callback_id][bb[1]:bb[3], bb[0]:bb[2], :] += reshape_cloak
+
+        return self.cloaked_faces
+
+
 def dump_dictionary_as_json(dict, outfile):
     j = json.dumps(dict)
     with open(outfile, "wb") as f:
@@ -30,10 +126,12 @@ def dump_dictionary_as_json(dict, outfile):
 
 def fix_gpu_memory(mem_fraction=1):
     os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
-    gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=mem_fraction)
+    tf_config = None
-    tf_config = tf.ConfigProto(gpu_options=gpu_options)
+    if tf.test.is_gpu_available():
-    tf_config.gpu_options.allow_growth = True
+        gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=mem_fraction)
-    tf_config.log_device_placement = False
+        tf_config = tf.ConfigProto(gpu_options=gpu_options)
+        tf_config.gpu_options.allow_growth = True
+        tf_config.log_device_placement = False
     init_op = tf.global_variables_initializer()
     sess = tf.Session(config=tf_config)
     sess.run(init_op)
@@ -45,7 +143,6 @@ def load_victim_model(number_classes, teacher_model=None, end2end=False):
     for l in teacher_model.layers:
         l.trainable = end2end
     x = teacher_model.layers[-1].output
-
     x = Dense(number_classes)(x)
     x = Activation('softmax', name="act")(x)
     model = Model(teacher_model.input, x)
@@ -141,6 +238,7 @@ def imagenet_preprocessing(x, data_format=None):
 
     return x
 
+
 def imagenet_reverse_preprocessing(x, data_format=None):
     import keras.backend as K
     x = np.array(x)
@@ -185,7 +283,20 @@ def build_bottleneck_model(model, cut_off):
 
 
 def load_extractor(name):
-    model = keras.models.load_model("../feature_extractors/{}.h5".format(name))
+    model_dir = os.path.join(os.path.expanduser('~'), '.fawkes')
+    os.makedirs(model_dir, exist_ok=True)
+    model_file = os.path.join(model_dir, "{}.h5".format(name))
+    if os.path.exists(model_file):
+        model = keras.models.load_model(model_file)
+    else:
+        get_file("{}.h5".format(name), "http://sandlab.cs.uchicago.edu/fawkes/files/{}.h5".format(name),
+                 cache_dir=model_dir, cache_subdir='')
+
+        get_file("{}_emb.p.gz".format(name), "http://sandlab.cs.uchicago.edu/fawkes/files/{}_emb.p.gz".format(name),
+                 cache_dir=model_dir, cache_subdir='')
+
+        model = keras.models.load_model(model_file)
+
     if hasattr(model.layers[-1], "activation") and model.layers[-1].activation == "softmax":
         raise Exception(
             "Given extractor's last layer is softmax, need to remove the top layers to make it into a feature extractor")
@@ -200,10 +311,11 @@ def load_extractor(name):
 
 
 def get_dataset_path(dataset):
-    if not os.path.exists("config.json"):
+    model_dir = os.path.join(os.path.expanduser('~'), '.fawkes')
+    if not os.path.exists(os.path.join(model_dir, "config.json")):
         raise Exception("Please config the datasets before running protection code. See more in README and config.py.")
 
-    config = json.load(open("config.json", 'r'))
+    config = json.load(open(os.path.join(model_dir, "config.json"), 'r'))
     if dataset not in config:
         raise Exception(
             "Dataset {} does not exist, please download to data/ and add the path to this function... Abort".format(
@@ -217,7 +329,8 @@ def normalize(x):
 
 
 def dump_image(x, filename, format="png", scale=False):
-    img = image.array_to_img(x, scale=scale)
+    # img = image.array_to_img(x, scale=scale)
+    img = image.array_to_img(x)
     img.save(filename, format)
     return
 
@@ -231,13 +344,17 @@ def load_dir(path):
         im = image.img_to_array(im)
         x_ls.append(im)
     raw_x = np.array(x_ls)
-    return preprocess_input(raw_x)
+    return preprocess(raw_x, 'imagenet')
 
 
 def load_embeddings(feature_extractors_names):
+    model_dir = os.path.join(os.path.expanduser('~'), '.fawkes')
     dictionaries = []
     for extractor_name in feature_extractors_names:
-        path2emb = pickle.load(open("../feature_extractors/embeddings/{}_emb_norm.p".format(extractor_name), "rb"))
+        fp = gzip.open(os.path.join(model_dir, "{}_emb.p.gz".format(extractor_name)), 'rb')
+        path2emb = pickle.load(fp)
+        fp.close()
+
         dictionaries.append(path2emb)
 
     merge_dict = {}
@@ -272,6 +389,8 @@ def calculate_dist_score(a, b, feature_extractors_ls, metric='l2'):
 
 
 def select_target_label(imgs, feature_extractors_ls, feature_extractors_names, metric='l2'):
+    model_dir = os.path.join(os.path.expanduser('~'), '.fawkes')
+
     original_feature_x = extractor_ls_predict(feature_extractors_ls, imgs)
 
     path2emb = load_embeddings(feature_extractors_names)
@@ -282,178 +401,174 @@ def select_target_label(imgs, feature_extractors_ls, feature_extractors_names, m
 
     pair_dist = pairwise_distances(original_feature_x, embs, metric)
     max_sum = np.min(pair_dist, axis=0)
-    sorted_idx = np.argsort(max_sum)[::-1]
+    max_id = np.argmax(max_sum)
 
-    highest_num = 0
+    target_data_id = paths[int(max_id)]
-    paired_target_X = None
+    image_dir = os.path.join(model_dir, "target_data/{}/*".format(target_data_id))
-    final_target_class_path = None
+    if not os.path.exists(image_dir):
-    for idx in sorted_idx[:1]:
+        get_file("{}.h5".format(name), "http://sandlab.cs.uchicago.edu/fawkes/files/target_images".format(name),
-        target_class_path = paths[idx]
+                 cache_dir=model_dir, cache_subdir='')
-        cur_target_X = load_dir(target_class_path)
-        cur_target_X = np.concatenate([cur_target_X, cur_target_X, cur_target_X])
-        cur_tot_sum, cur_paired_target_X = calculate_dist_score(imgs, cur_target_X,
-                                                                feature_extractors_ls,
-                                                                metric=metric)
-        if cur_tot_sum > highest_num:
-            highest_num = cur_tot_sum
-            paired_target_X = cur_paired_target_X
 
-    np.random.shuffle(paired_target_X)
+    image_paths = glob.glob(image_dir)
-    paired_target_X = list(paired_target_X)
-    while len(paired_target_X) < len(imgs):
-        paired_target_X += paired_target_X
 
-    paired_target_X = paired_target_X[:len(imgs)]
+    target_images = [image.img_to_array(image.load_img(cur_path)) for cur_path in
-    return np.array(paired_target_X)
+                     image_paths]
 
+    target_images = np.array([resize(x, (224, 224)) for x in target_images])
+    target_images = preprocess(target_images, 'imagenet')
 
+    target_images = list(target_images)
+    while len(target_images) < len(imgs):
+        target_images += target_images
 
-class CloakData(object):
+    target_images = random.sample(target_images, len(imgs))
-    def __init__(self, protect_directory=None, img_shape=(224, 224)):
+    return np.array(target_images)
 
-        self.img_shape = img_shape
+# class CloakData(object):
-
+#     def __init__(self, protect_directory=None, img_shape=(224, 224)):
-        # self.train_data_dir, self.test_data_dir, self.number_classes, self.number_samples = get_dataset_path(dataset)
+#
-        # self.all_labels = sorted(list(os.listdir(self.train_data_dir)))
+#         self.img_shape = img_shape
-        self.protect_directory = protect_directory
+#         # self.train_data_dir, self.test_data_dir, self.number_classes, self.number_samples = get_dataset_path(dataset)
-
+#         # self.all_labels = sorted(list(os.listdir(self.train_data_dir)))
-        self.protect_X = self.load_label_data(self.protect_directory)
+#         self.protect_directory = protect_directory
-
+#
-        self.cloaked_protect_train_X = None
+#         self.protect_X = self.load_label_data(self.protect_directory)
-
+#
-        self.label2path_train, self.label2path_test, self.path2idx = self.build_data_mapping()
+#         self.cloaked_protect_train_X = None
-        self.all_training_path = self.get_all_data_path(self.label2path_train)
+#
-        self.all_test_path = self.get_all_data_path(self.label2path_test)
+#         self.label2path_train, self.label2path_test, self.path2idx = self.build_data_mapping()
-        self.protect_class_path = self.get_class_image_files(os.path.join(self.train_data_dir, self.protect_class))
+#         self.all_training_path = self.get_all_data_path(self.label2path_train)
-
+#         self.all_test_path = self.get_all_data_path(self.label2path_test)
-    def get_class_image_files(self, path):
+#         self.protect_class_path = self.get_class_image_files(os.path.join(self.train_data_dir, self.protect_class))
-        return [os.path.join(path, f) for f in os.listdir(path)]
+#
-
+#     def get_class_image_files(self, path):
-    def extractor_ls_predict(self, feature_extractors_ls, X):
+#         return [os.path.join(path, f) for f in os.listdir(path)]
-        feature_ls = []
+#
-        for extractor in feature_extractors_ls:
+#     def extractor_ls_predict(self, feature_extractors_ls, X):
-            cur_features = extractor.predict(X)
+#         feature_ls = []
-            feature_ls.append(cur_features)
+#         for extractor in feature_extractors_ls:
-        concated_feature_ls = np.concatenate(feature_ls, axis=1)
+#             cur_features = extractor.predict(X)
-        concated_feature_ls = normalize(concated_feature_ls)
+#             feature_ls.append(cur_features)
-        return concated_feature_ls
+#         concated_feature_ls = np.concatenate(feature_ls, axis=1)
-
+#         concated_feature_ls = normalize(concated_feature_ls)
-    def load_embeddings(self, feature_extractors_names):
+#         return concated_feature_ls
-        dictionaries = []
+#
-        for extractor_name in feature_extractors_names:
+#     def load_embeddings(self, feature_extractors_names):
-            path2emb = pickle.load(open("../feature_extractors/embeddings/{}_emb_norm.p".format(extractor_name), "rb"))
+#         dictionaries = []
-            dictionaries.append(path2emb)
+#         for extractor_name in feature_extractors_names:
-
+#             path2emb = pickle.load(open("../feature_extractors/embeddings/{}_emb_norm.p".format(extractor_name), "rb"))
-        merge_dict = {}
+#             dictionaries.append(path2emb)
-        for k in dictionaries[0].keys():
+#
-            cur_emb = [dic[k] for dic in dictionaries]
+#         merge_dict = {}
-            merge_dict[k] = np.concatenate(cur_emb)
+#         for k in dictionaries[0].keys():
-        return merge_dict
+#             cur_emb = [dic[k] for dic in dictionaries]
-
+#             merge_dict[k] = np.concatenate(cur_emb)
-    def select_target_label(self, feature_extractors_ls, feature_extractors_names, metric='l2'):
+#         return merge_dict
-        original_feature_x = self.extractor_ls_predict(feature_extractors_ls, self.protect_train_X)
+#
-
+#     def select_target_label(self, feature_extractors_ls, feature_extractors_names, metric='l2'):
-        path2emb = self.load_embeddings(feature_extractors_names)
+#         original_feature_x = self.extractor_ls_predict(feature_extractors_ls, self.protect_train_X)
-        items = list(path2emb.items())
+#
-        paths = [p[0] for p in items]
+#         path2emb = self.load_embeddings(feature_extractors_names)
-        embs = [p[1] for p in items]
+#         items = list(path2emb.items())
-        embs = np.array(embs)
+#         paths = [p[0] for p in items]
-
+#         embs = [p[1] for p in items]
-        pair_dist = pairwise_distances(original_feature_x, embs, metric)
+#         embs = np.array(embs)
-        max_sum = np.min(pair_dist, axis=0)
+#
-        sorted_idx = np.argsort(max_sum)[::-1]
+#         pair_dist = pairwise_distances(original_feature_x, embs, metric)
-
+#         max_sum = np.min(pair_dist, axis=0)
-        highest_num = 0
+#         sorted_idx = np.argsort(max_sum)[::-1]
-        paired_target_X = None
+#
-        final_target_class_path = None
+#         highest_num = 0
-        for idx in sorted_idx[:5]:
+#         paired_target_X = None
-            target_class_path = paths[idx]
+#         final_target_class_path = None
-            cur_target_X = self.load_dir(target_class_path)
+#         for idx in sorted_idx[:5]:
-            cur_target_X = np.concatenate([cur_target_X, cur_target_X, cur_target_X])
+#             target_class_path = paths[idx]
-            cur_tot_sum, cur_paired_target_X = self.calculate_dist_score(self.protect_train_X, cur_target_X,
+#             cur_target_X = self.load_dir(target_class_path)
-                                                                         feature_extractors_ls,
+#             cur_target_X = np.concatenate([cur_target_X, cur_target_X, cur_target_X])
-                                                                         metric=metric)
+#             cur_tot_sum, cur_paired_target_X = self.calculate_dist_score(self.protect_train_X, cur_target_X,
-            if cur_tot_sum > highest_num:
+#                                                                          feature_extractors_ls,
-                highest_num = cur_tot_sum
+#                                                                          metric=metric)
-                paired_target_X = cur_paired_target_X
+#             if cur_tot_sum > highest_num:
-                final_target_class_path = target_class_path
+#                 highest_num = cur_tot_sum
-
+#                 paired_target_X = cur_paired_target_X
-        np.random.shuffle(paired_target_X)
+#                 final_target_class_path = target_class_path
-        return final_target_class_path, paired_target_X
+#
-
+#         np.random.shuffle(paired_target_X)
-    def calculate_dist_score(self, a, b, feature_extractors_ls, metric='l2'):
+#         return final_target_class_path, paired_target_X
-        features1 = self.extractor_ls_predict(feature_extractors_ls, a)
+#
-        features2 = self.extractor_ls_predict(feature_extractors_ls, b)
+#     def calculate_dist_score(self, a, b, feature_extractors_ls, metric='l2'):
-
+#         features1 = self.extractor_ls_predict(feature_extractors_ls, a)
-        pair_cos = pairwise_distances(features1, features2, metric)
+#         features2 = self.extractor_ls_predict(feature_extractors_ls, b)
-        max_sum = np.min(pair_cos, axis=0)
+#
-        max_sum_arg = np.argsort(max_sum)[::-1]
+#         pair_cos = pairwise_distances(features1, features2, metric)
-        max_sum_arg = max_sum_arg[:len(a)]
+#         max_sum = np.min(pair_cos, axis=0)
-        max_sum = [max_sum[i] for i in max_sum_arg]
+#         max_sum_arg = np.argsort(max_sum)[::-1]
-        paired_target_X = [b[j] for j in max_sum_arg]
+#         max_sum_arg = max_sum_arg[:len(a)]
-        paired_target_X = np.array(paired_target_X)
+#         max_sum = [max_sum[i] for i in max_sum_arg]
-        return np.min(max_sum), paired_target_X
+#         paired_target_X = [b[j] for j in max_sum_arg]
-
+#         paired_target_X = np.array(paired_target_X)
-    def get_all_data_path(self, label2path):
+#         return np.min(max_sum), paired_target_X
-        all_paths = []
+#
-        for k, v in label2path.items():
+#     def get_all_data_path(self, label2path):
-            cur_all_paths = [os.path.join(k, cur_p) for cur_p in v]
+#         all_paths = []
-            all_paths.extend(cur_all_paths)
+#         for k, v in label2path.items():
-        return all_paths
+#             cur_all_paths = [os.path.join(k, cur_p) for cur_p in v]
-
+#             all_paths.extend(cur_all_paths)
-    def load_label_data(self, label):
+#         return all_paths
-        train_label_path = os.path.join(self.train_data_dir, label)
+#
-        test_label_path = os.path.join(self.test_data_dir, label)
+#     def load_label_data(self, label):
-        train_X = self.load_dir(train_label_path)
+#         train_label_path = os.path.join(self.train_data_dir, label)
-        test_X = self.load_dir(test_label_path)
+#         test_label_path = os.path.join(self.test_data_dir, label)
-        return train_X, test_X
+#         train_X = self.load_dir(train_label_path)
-
+#         test_X = self.load_dir(test_label_path)
-    def load_dir(self, path):
+#         return train_X, test_X
-        assert os.path.exists(path)
+#
-        x_ls = []
+#     def load_dir(self, path):
-        for file in os.listdir(path):
+#         assert os.path.exists(path)
-            cur_path = os.path.join(path, file)
+#         x_ls = []
-            im = image.load_img(cur_path, target_size=self.img_shape)
+#         for file in os.listdir(path):
-            im = image.img_to_array(im)
+#             cur_path = os.path.join(path, file)
-            x_ls.append(im)
+#             im = image.load_img(cur_path, target_size=self.img_shape)
-        raw_x = np.array(x_ls)
+#             im = image.img_to_array(im)
-        return preprocess_input(raw_x)
+#             x_ls.append(im)
-
+#         raw_x = np.array(x_ls)
-    def build_data_mapping(self):
+#         return preprocess_input(raw_x)
-        label2path_train = {}
+#
-        label2path_test = {}
+#     def build_data_mapping(self):
-        idx = 0
+#         label2path_train = {}
-        path2idx = {}
+#         label2path_test = {}
-        for label_name in self.all_labels:
+#         idx = 0
-            full_path_train = os.path.join(self.train_data_dir, label_name)
+#         path2idx = {}
-            full_path_test = os.path.join(self.test_data_dir, label_name)
+#         for label_name in self.all_labels:
-            label2path_train[full_path_train] = list(os.listdir(full_path_train))
+#             full_path_train = os.path.join(self.train_data_dir, label_name)
-            label2path_test[full_path_test] = list(os.listdir(full_path_test))
+#             full_path_test = os.path.join(self.test_data_dir, label_name)
-            for img_file in os.listdir(full_path_train):
+#             label2path_train[full_path_train] = list(os.listdir(full_path_train))
-                path2idx[os.path.join(full_path_train, img_file)] = idx
+#             label2path_test[full_path_test] = list(os.listdir(full_path_test))
-            for img_file in os.listdir(full_path_test):
+#             for img_file in os.listdir(full_path_train):
-                path2idx[os.path.join(full_path_test, img_file)] = idx
+#                 path2idx[os.path.join(full_path_train, img_file)] = idx
-            idx += 1
+#             for img_file in os.listdir(full_path_test):
-        return label2path_train, label2path_test, path2idx
+#                 path2idx[os.path.join(full_path_test, img_file)] = idx
-
+#             idx += 1
-    def generate_data_post_cloak(self, sybil=False):
+#         return label2path_train, label2path_test, path2idx
-        assert self.cloaked_protect_train_X is not None
+#
-        while True:
+#     def generate_data_post_cloak(self, sybil=False):
-            batch_X = []
+#         assert self.cloaked_protect_train_X is not None
-            batch_Y = []
+#         while True:
-            cur_batch_path = random.sample(self.all_training_path, 32)
+#             batch_X = []
-            for p in cur_batch_path:
+#             batch_Y = []
-                cur_y = self.path2idx[p]
+#             cur_batch_path = random.sample(self.all_training_path, 32)
-                if p in self.protect_class_path:
+#             for p in cur_batch_path:
-                    cur_x = random.choice(self.cloaked_protect_train_X)
+#                 cur_y = self.path2idx[p]
-                elif sybil and (p in self.sybil_class):
+#                 if p in self.protect_class_path:
-                    cur_x = random.choice(self.cloaked_sybil_train_X)
+#                     cur_x = random.choice(self.cloaked_protect_train_X)
-                else:
+#                 elif sybil and (p in self.sybil_class):
-                    im = image.load_img(p, target_size=self.img_shape)
+#                     cur_x = random.choice(self.cloaked_sybil_train_X)
-                    im = image.img_to_array(im)
+#                 else:
-                    cur_x = preprocess_input(im)
+#                     im = image.load_img(p, target_size=self.img_shape)
-                batch_X.append(cur_x)
+#                     im = image.img_to_array(im)
-                batch_Y.append(cur_y)
+#                     cur_x = preprocess_input(im)
-            batch_X = np.array(batch_X)
+#                 batch_X.append(cur_x)
-            batch_Y = to_categorical(np.array(batch_Y), num_classes=self.number_classes)
+#                 batch_Y.append(cur_y)
-            yield batch_X, batch_Y
+#             batch_X = np.array(batch_X)
+#             batch_Y = to_categorical(np.array(batch_Y), num_classes=self.number_classes)
+#             yield batch_X, batch_Y

setup.py

@@ -1,23 +1,117 @@
-import setuptools
+import os
+import re
+import sys
 
+from setuptools import setup, Command
+
+__PATH__ = os.path.abspath(os.path.dirname(__file__))
 
 with open("README.md", "r") as fh:
     long_description = fh.read()
 
-setuptools.setup(
+
-    name="fawkes",
+def read_version():
-    version="0.0.1",
+    __PATH__ = os.path.abspath(os.path.dirname(__file__))
-    author="Shawn Shan",
+    with open(os.path.join(__PATH__, 'fawkes/__init__.py')) as f:
-    author_email="shansixiong@cs.uchicago.edu",
+        version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]",
-    description="Fawkes protect user privacy",
+                                  f.read(), re.M)
+    if version_match:
+        return version_match.group(1)
+    raise RuntimeError("Unable to find __version__ string")
+
+
+__version__ = read_version()
+
+
+# brought from https://github.com/kennethreitz/setup.py
+class DeployCommand(Command):
+    description = 'Build and deploy the package to PyPI.'
+    user_options = []
+
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
+    @staticmethod
+    def status(s):
+        print(s)
+
+    def run(self):
+
+        assert 'dev' not in __version__, (
+            "Only non-devel versions are allowed. "
+            "__version__ == {}".format(__version__))
+
+        with os.popen("git status --short") as fp:
+            git_status = fp.read().strip()
+            if git_status:
+                print("Error: git repository is not clean.\n")
+                os.system("git status --short")
+                sys.exit(1)
+
+        try:
+            from shutil import rmtree
+            self.status('Removing previous builds ...')
+            rmtree(os.path.join(__PATH__, 'dist'))
+        except OSError:
+            pass
+
+        self.status('Building Source and Wheel (universal) distribution ...')
+        os.system('{0} setup.py sdist'.format(sys.executable))
+
+        self.status('Uploading the package to PyPI via Twine ...')
+        ret = os.system('twine upload dist/*')
+        if ret != 0:
+            sys.exit(ret)
+
+        self.status('Creating git tags ...')
+        os.system('git tag v{0}'.format(__version__))
+        os.system('git tag --list')
+        sys.exit()
+
+
+setup_requires = []
+
+install_requires = [
+    'numpy>=1.16.4',
+    'tensorflow>=1.13.1',
+    'argparse',
+    'keras==2.2.5',
+    'scikit-image',
+    'pillow>=7.0.0',
+    'opencv-python>=4.2.0.34',
+]
+
+setup(
+    name='fawkes',
+    version=__version__,
+    license='MIT',
+    description='An utility to protect user privacy',
     long_description=long_description,
-    long_description_content_type="text/markdown",
+    long_description_content_type='text/markdown',
     url="https://github.com/Shawn-Shan/fawkes",
-    packages=setuptools.find_packages(),
+    author='Shawn Shan',
+    author_email='shansixiong@cs.uchicago.edu',
+    keywords='fawkes privacy clearview',
     classifiers=[
-        "Programming Language :: Python :: 3",
+        'Development Status :: 3 - Alpha',
-        "License :: OSI Approved :: MIT License",
+        'License :: OSI Approved :: MIT License',
         "Operating System :: OS Independent",
+        'Programming Language :: Python :: 3',
+        'Topic :: System :: Monitoring',
     ],
+    packages=['fawkes'],
+    install_requires=install_requires,
+    setup_requires=setup_requires,
+    entry_points={
+        'console_scripts': ['fawkes=fawkes:main'],
+    },
+    cmdclass={
+        'deploy': DeployCommand,
+    },
+    include_package_data=True,
+    zip_safe=False,
     python_requires='>=3.5',
-)
+)