روش های تولید دیپ فیک (جعل عمیق) و ساختار آنها — راهنمای جامع

1import numpy as np
2import sys
3import tensorflow as tf
4
5import math
6import struct
7import argparse
8import time
9import os
10import cPickle
11import random
12import platform
13import glob
14
15plat = platform.dist()[0]
16if plat == "Ubuntu":
17  base = "/home/supasorn/"
18else:
19  base = "/projects/grail/supasorn2nb/"
20
21def readSingleInt(path):
22  with open(path) as f:
23    return int(f.readline())
24
25def readCVFloatMat(fl):
26  f = open(fl)
27  t = struct.unpack('B', f.read(1))[0]
28  if t != 5:
29    return 0
30  h = struct.unpack('i', f.read(4))[0]
31  w = struct.unpack('i', f.read(4))[0]
32  return np.reshape(np.array(struct.unpack('%df' % (h * w), f.read(4 * h * w)), float), (h, w))
33
34def _str_to_bool(s):
35  if s.lower() not in ['true', 'false']:
36      raise ValueError('Need bool; got %r' % s)
37  return s.lower() == 'true'
38
39def add_boolean_argument(parser, name, default=False):
40  group = parser.add_mutually_exclusive_group()
41  group.add_argument(
42      '--' + name, nargs='?', default=default, const=True, type=_str_to_bool)
43  group.add_argument('--no' + name, dest=name, action='store_false')
44
45def normalizeData(lst, savedir, name, varnames, normalize=True):
46  allstrokes = np.concatenate(lst)
47  mean = np.mean(allstrokes, 0)
48  std = np.std(allstrokes, 0) 
49
50  f = open(savedir + "/" + name + ".txt", "w")
51  minv = np.min(allstrokes, 0)
52  maxv = np.max(allstrokes, 0)
53
54  if not isinstance(normalize, list):
55    normalize = [normalize] * len(mean)
56
57  for i, n in enumerate(varnames):
58    if normalize[i]:
59      f.write(n + "\n  mean: %f\n  std :%f\n  min :%f\n  max :%f\n\n" % (mean[i], std[i], minv[i], maxv[i]))
60    else:
61      f.write(n + "\n  mean: %f (-> 0)\n  std :%f (-> 1)\n  min :%f\n  max :%f\n\n" % (mean[i], std[i], minv[i], maxv[i]))
62      mean[i] = 0
63      std[i] = 1
64
65  np.save(savedir + '/' + name + '.npy', {'min': minv, 'max': maxv, 'mean': mean, 'std': std})
66  for i in range(len(lst)):
67    lst[i] = (lst[i] - mean) / std
68
69  f.close()
70  return mean, std
71
72
73class TFBase(object):
74  def __init__(self):
75    np.random.seed(42)
76    random.seed(42)
77    self.parser = argparse.ArgumentParser()
78    self.addDefaultParameters()
79
80  def addDefaultParameters(self):
81    self.parser.add_argument('--num_epochs', type=int, default=300,
82                       help='number of epochs')
83    self.parser.add_argument('--save_every', type=int, default=10,
84                       help='save frequency')
85    self.parser.add_argument('--grad_clip', type=float, default=10.,
86                       help='clip gradients at this value')
87    self.parser.add_argument('--learning_rate', type=float, default=0.001,
88                       help='learning rate')
89    self.parser.add_argument('--decay_rate', type=float, default=1,
90                       help='decay rate for rmsprop')
91    self.parser.add_argument('--keep_prob', type=float, default=1,
92                       help='dropout keep probability')
93
94    self.parser.add_argument('--save_dir', type=str, default='',
95                       help='save directory')
96    self.parser.add_argument('--usetrainingof', type=str, default='',
97                       help='trainingset')
98
99    add_boolean_argument(self.parser, "reprocess")
100    add_boolean_argument(self.parser, "normalizeinput", default=True)
101
102  def normalize(self, inps, outps):
103    meani, stdi = normalizeData(inps["training"], "save/" + self.args.save_dir, "statinput", ["fea%02d" % x for x in range(inps["training"][0].shape[1])], normalize=self.args.normalizeinput)
104    meano, stdo = normalizeData(outps["training"], "save/" + self.args.save_dir, "statoutput", ["fea%02d" % x for x in range(outps["training"][0].shape[1])], normalize=self.args.normalizeoutput)
105
106    for i in range(len(inps["validation"])):
107      inps["validation"][i] = (inps["validation"][i] - meani) / stdi;
108
109    for i in range(len(outps["validation"])):
110      outps["validation"][i] = (outps["validation"][i] - meano) / stdo;
111
112    return meani, stdi, meano, stdo
113
114  def loadData(self):
115    if not os.path.exists("save/"):
116      os.mkdir("save/")
117    if not os.path.exists("save/" + self.args.save_dir):
118      os.mkdir("save/" + self.args.save_dir)
119
120    if len(self.args.usetrainingof):
121      data_file = "data/training_" + self.args.usetrainingof + ".cpkl"
122    else:
123      data_file = "data/training_" + self.args.save_dir + ".cpkl"
124
125    if not (os.path.exists(data_file)) or self.args.reprocess:
126      print "creating training data cpkl file from raw source"
127      inps, outps = self.preprocess(data_file)
128
129      meani, stdi, meano, stdo = self.normalize(inps, outps)
130
131      if not os.path.exists(os.path.dirname(data_file)):
132        os.mkdir(os.path.dirname(data_file))
133      f = open(data_file, "wb")
134      cPickle.dump({"input": inps["training"], "inputmean": meani, "inputstd": stdi, "output": outps["training"], "outputmean":meano, "outputstd": stdo, "vinput": inps["validation"], "voutput": outps["validation"]}, f, protocol=2) 
135      f.close() 
136
137
138    f = open(data_file,"rb")
139    data = cPickle.load(f)
140    inps = {"training": data["input"], "validation": data["vinput"]} 
141    outps = {"training": data["output"], "validation": data["voutput"]} 
142    f.close()
143
144    self.dimin = inps["training"][0].shape[1]
145    self.dimout = outps["training"][0].shape[1]
146
147    self.inps, self.outps = self.load_preprocessed(inps, outps)
148    self.num_batches = {}
149    self.pointer = {}
150    for key in self.inps:
151      self.num_batches[key] = 0
152      for inp in self.inps[key]:
153        self.num_batches[key] += int(math.ceil((len(inp) - 2) / self.args.seq_length))
154      self.num_batches[key] = int(self.num_batches[key] / self.args.batch_size)
155      self.reset_batch_pointer(key)
156
157  def preprocess(self):
158    raise NotImplementedError()
159
160  def next_batch(self, key="training"):
161    # returns a randomised, seq_length sized portion of the training data
162    x_batch = []
163    y_batch = []
164    for i in xrange(self.args.batch_size):
165      inp = self.inps[key][self.pointer[key]]
166      outp = self.outps[key][self.pointer[key]]
167
168      n_batch = int(math.ceil((len(inp) - 2) / self.args.seq_length)) 
169
170      idx = random.randint(1, len(inp) - self.args.seq_length - 1)
171      x_batch.append(np.copy(inp[idx:idx+self.args.seq_length]))
172      y_batch.append(np.copy(outp[idx:idx+self.args.seq_length]))
173
174      if random.random() < 1.0 / float(n_batch): 
175        self.tick_batch_pointer(key)
176    return x_batch, y_batch
177
178  def tick_batch_pointer(self, key):
179    self.pointer[key] += 1
180    if self.pointer[key] >= len(self.inps[key]):
181      self.pointer[key] = 0
182
183  def reset_batch_pointer(self, key):
184    self.pointer[key] = 0
185
186
187  def test(self):
188    # only use save_dir from args
189    save_dir = self.args.save_dir
190
191    with open(os.path.join("save/" + save_dir, 'config.pkl')) as f:
192      saved_args = cPickle.load(f)
193
194    if len(saved_args.usetrainingof):
195      pt = saved_args.usetrainingof
196    else:
197      pt = save_dir
198
199    with open("./data/training_" + pt + ".cpkl", "rb") as f:
200      raw = cPickle.load(f)
201
202    model = self.model(saved_args, True)
203    sess = tf.InteractiveSession()
204    saver = tf.train.Saver()
205
206    ckpt = tf.train.get_checkpoint_state("save/" + save_dir)
207    saver.restore(sess, ckpt.model_checkpoint_path)
208    print "loading model: ", ckpt.model_checkpoint_path
209
210    saved_args.input = self.args.input
211
212    self.sample(sess, saved_args, raw, pt)
213
214  def train(self):
215    with open(os.path.join("save/" + self.args.save_dir, 'config.pkl'), 'w') as f:
216      cPickle.dump(self.args, f)
217
218    with tf.Session() as sess:
219      model = self.model(self.args)
220
221      tf.initialize_all_variables().run()
222      ts = TrainingStatus(sess, self.args.num_epochs, self.num_batches["training"], save_interval = self.args.save_every, graph = sess.graph, save_dir = "save/" + self.args.save_dir)
223
224      print "training batches: ", self.num_batches["training"]
225      for e in xrange(ts.startEpoch, self.args.num_epochs):
226        sess.run(tf.assign(self.lr, self.args.learning_rate * (self.args.decay_rate ** e)))
227        self.reset_batch_pointer("training")
228        self.reset_batch_pointer("validation")
229
230
231        state = []
232        for c, m in self.initial_state: 
233          state.append((c.eval(), m.eval()))
234
235        fetches = []
236        fetches.append(self.cost)
237        fetches.append(self.train_op)
238
239        feed_dict = {}
240        for i, (c, m) in enumerate(self.initial_state):
241          feed_dict[c], feed_dict[m] = state[i]
242
243        for b in xrange(self.num_batches["training"]):
244          ts.tic()
245          x, y = self.next_batch()
246
247          feed_dict[self.input_data] = x
248          feed_dict[self.target_data] = y
249
250          res = sess.run(fetches, feed_dict)
251          train_loss = res[0]
252
253          print ts.tocBatch(e, b, train_loss)
254
255        validLoss = 0
256        if self.num_batches["validation"] > 0:
257          fetches = []
258          fetches.append(self.cost)
259          for b in xrange(self.num_batches["validation"]):
260            x, y = self.next_batch("validation")
261
262            feed_dict[self.input_data] = x
263            feed_dict[self.target_data] = y
264
265            loss = sess.run(fetches, feed_dict)
266            validLoss += loss[0]
267          validLoss /= self.num_batches["validation"]
268          
269        ts.tocEpoch(sess, e, validLoss)
270
271
272class TrainingStatus:
273  def __init__(self, sess, num_epochs, num_batches, logwrite_interval = 25, eta_interval = 25, save_interval = 100, save_dir = "save", graph = None):
274    if not os.path.exists(save_dir):
275      os.mkdir(save_dir)
276    #if graph is not None:
277      #self.writer = tf.train.SummaryWriter(save_dir, graph)
278    #else:
279      #self.writer = tf.train.SummaryWriter(save_dir)
280
281    self.save_dir = save_dir
282    self.model_dir = os.path.join(save_dir, 'model.ckpt')
283    #self.saver = tf.train.Saver(tf.all_variables(), max_to_keep = 0)
284    self.saver = tf.train.Saver(tf.all_variables())
285
286    lastCheckpoint = tf.train.latest_checkpoint(save_dir) 
287    if lastCheckpoint is None:
288      self.startEpoch = 0
289    else:
290      print "Last checkpoint :", lastCheckpoint
291      self.startEpoch = int(lastCheckpoint.split("-")[-1])
292      self.saver.restore(sess, lastCheckpoint)
293
294    print "startEpoch = ", self.startEpoch
295
296    self.logwrite_interval = logwrite_interval
297    self.eta_interval = eta_interval
298    self.totalTask = num_epochs * num_batches
299    self.num_epochs = num_epochs
300    self.num_batches = num_batches
301    self.save_interval = save_interval
302
303    self.etaCount = 0
304    self.etaStart = time.time()
305    self.duration = 0
306
307    self.avgloss = 0
308    self.avgcount = 0
309
310  def tic(self):
311      self.start = time.time()
312
313  def tocBatch(self, e, b, loss):
314      self.end = time.time()
315      taskNum = (e * self.num_batches + b)
316
317      self.etaCount += 1
318      if self.etaCount % self.eta_interval == 0:
319        self.duration = time.time() - self.etaStart
320        self.etaStart = time.time()
321
322      etaTime = float(self.totalTask - (taskNum + 1)) / self.eta_interval * self.duration
323      m, s = divmod(etaTime, 60)
324      h, m = divmod(m, 60)
325      etaString = "%d:%02d:%02d" % (h, m, s)
326      self.avgloss += loss
327      self.avgcount += 1
328
329      if taskNum == 0:
330        with open(self.save_dir + "/avgloss.txt", "w") as f:
331          f.write("0 %f %f\n" % (loss, loss))
332
333      return "%.2f%% (%d/%d): %.3f  t %.3f  @ %s (%s)" % (taskNum * 100.0 / self.totalTask, e, self.num_epochs, loss, self.end - self.start, time.strftime("%a %d %H:%M:%S", time.localtime(time.time() + etaTime)), etaString)
334
335  def tocEpoch(self, sess, e, validLoss=0):
336    if (e + 1) % self.save_interval == 0 or e == self.num_epochs - 1:
337      self.saver.save(sess, self.model_dir, global_step = e + 1)
338      print "model saved to {}".format(self.model_dir)
339
340    
341    lines = open(self.save_dir + "/avgloss.txt", "r").readlines()
342    with open(self.save_dir + "/avgloss.txt", "w") as f:
343      for line in lines:
344        if int(line.split(" ")[0]) >= e + 1:
345          break
346        f.write(line)
347      f.write("%d %f %f\n" % (e+1, self.avgloss / self.avgcount, validLoss))
348
349    self.avgcount = 0
350    self.avgloss = 0;

1import sys
2
3from util import *
4
5import json
6import copy
7import random
8import platform
9import bisect
10import numpy as np
11
12
13class Speech(TFBase):
14  def __init__(self):
15    super(Speech, self).__init__()
16    self.parser.add_argument('--timedelay', type=int, default=20,
17                       help='time delay between output and input')
18    self.parser.add_argument('--rnn_size', type=int, default=60,
19                     help='size of RNN hidden state')
20    self.parser.add_argument('--num_layers', type=int, default=1,
21                     help='number of layers in the RNN')
22    self.parser.add_argument('--batch_size', type=int, default=100,
23                     help='minibatch size')
24    self.parser.add_argument('--seq_length', type=int, default=100,
25                     help='RNN sequence length')
26    self.parser.add_argument('--input', type=str, default='',
27                       help='input for generation')
28    self.parser.add_argument('--input2', type=str, default='',
29                       help='input for any mfcc wav file')
30    self.parser.add_argument('--guy', type=str, default='Obama2',
31                       help='dataset')
32    self.parser.add_argument('--normalizeoutput', action='store_true')
33
34    self.args = self.parser.parse_args()
35    if self.args.save_dir == "":
36      raise ValueError('Missing save_dir')
37
38    # self.training_dir = base + "/face-singleview/data/" + self.args.guy + "/"
39    self.training_dir = "obama_data/"
40
41    self.fps = 29.97
42    self.loadData()
43    self.model = self.standardL2Model
44
45    self.audioinput = len(self.args.input2)
46    if (self.audioinput):
47      self.args.input = self.args.input2
48
49    if len(self.args.input):
50      self.test()
51    else:
52      self.train()
53
54  def createInputFeature(self, audio, audiodiff, timestamps, startframe, nframe):
55    startAudio = bisect.bisect_left(timestamps, (startframe - 1) / self.fps)
56    endAudio = bisect.bisect_right(timestamps, (startframe + nframe - 2) / self.fps)
57
58    inp = np.concatenate((audio[startAudio:endAudio, :-1], audiodiff[startAudio:endAudio, :]), axis=1)
59    return startAudio, endAudio, inp 
60
61
62  def preprocess(self, save_dir):
63    files = [x.split("\t")[0].strip() for x in open(self.training_dir + "processed_fps.txt", "r").readlines()]
64
65    inps = {"training": [], "validation": []}
66    outps = {"training": [], "validation": []}
67
68    # validation = 0.2
69    validation = 0
70    for i in range(len(files)):
71      tp = "training" if random.random() > validation else "validation"
72
73      dnums = sorted([os.path.basename(x) for x in glob.glob(self.training_dir + files[i] + "}}*")])
74
75      audio = np.load(self.training_dir + "/audio/normalized-cep13/" + files[i] + ".wav.npy") 
76      audiodiff = audio[1:,:-1] - audio[:-1, :-1]
77
78      print files[i], audio.shape, tp
79      timestamps = audio[:, -1]
80
81      for dnum in dnums:
82        print dnum 
83        fids = readCVFloatMat(self.training_dir + dnum + "/frontalfidsCoeff_unrefined.bin")
84        if not os.path.exists(self.training_dir + dnum + "/startframe.txt"):
85          startframe = 1
86        else:
87          startframe = readSingleInt(self.training_dir + dnum + "/startframe.txt")
88        nframe = readSingleInt(self.training_dir + dnum + "/nframe.txt")
89
90        startAudio, endAudio, inp = self.createInputFeature(audio, audiodiff, timestamps, startframe, nframe)
91
92        outp = np.zeros((endAudio - startAudio, fids.shape[1]), dtype=np.float32)
93        leftmark = 0
94        for aud in range(startAudio, endAudio):
95          audiotime = audio[aud, -1]
96          while audiotime >= (startframe - 1 + leftmark + 1) / self.fps:
97            leftmark += 1
98          t = (audiotime - (startframe - 1 + leftmark) / self.fps) * self.fps;
99          outp[aud - startAudio, :] = fids[leftmark, :] * (1 - t) + fids[min(len(fids) - 1, leftmark + 1), :] * t;
100            
101        inps[tp].append(inp)
102        outps[tp].append(outp)
103
104    return (inps, outps)
105
106  def standardL2Model(self, args, infer=False):
107    if infer:
108      args.batch_size = 1
109      args.seq_length = 1
110
111    cell_fn = tf.nn.rnn_cell.LSTMCell
112    cell = cell_fn(args.rnn_size, state_is_tuple=True)
113
114    if infer == False and args.keep_prob < 1: # training mode
115      cell0 = tf.nn.rnn_cell.DropoutWrapper(cell, input_keep_prob = args.keep_prob)
116      cell1 = tf.nn.rnn_cell.DropoutWrapper(cell, input_keep_prob = args.keep_prob, output_keep_prob = args.keep_prob)
117      self.network = tf.nn.rnn_cell.MultiRNNCell([cell0] * (args.num_layers -1) + [cell1], state_is_tuple=True)
118    else:
119      self.network = tf.nn.rnn_cell.MultiRNNCell([cell] * args.num_layers, state_is_tuple=True)
120
121
122    self.input_data = tf.placeholder(dtype=tf.float32, shape=[None, args.seq_length, self.dimin])
123    self.target_data = tf.placeholder(dtype=tf.float32, shape=[None, args.seq_length, self.dimout])
124    self.initial_state = self.network.zero_state(batch_size=args.batch_size, dtype=tf.float32)
125
126    with tf.variable_scope('rnnlm'):
127      output_w = tf.get_variable("output_w", [args.rnn_size, self.dimout])
128      output_b = tf.get_variable("output_b", [self.dimout])
129
130    inputs = tf.split(1, args.seq_length, self.input_data)
131    inputs = [tf.squeeze(input_, [1]) for input_ in inputs]
132
133    outputs, states = tf.nn.seq2seq.rnn_decoder(inputs, self.initial_state, self.network, loop_function=None, scope='rnnlm')
134
135    output = tf.reshape(tf.concat(1, outputs), [-1, args.rnn_size])
136    output = tf.nn.xw_plus_b(output, output_w, output_b)
137    self.final_state = states
138    self.output = output
139
140    flat_target_data = tf.reshape(self.target_data,[-1, self.dimout])
141        
142    lossfunc = tf.reduce_sum(tf.squared_difference(flat_target_data, output))
143    #lossfunc = tf.reduce_sum(tf.abs(flat_target_data - output))
144    self.cost = lossfunc / (args.batch_size * args.seq_length * self.dimout)
145
146    self.lr = tf.Variable(0.0, trainable=False)
147    tvars = tf.trainable_variables()
148    grads, _ = tf.clip_by_global_norm(tf.gradients(self.cost, tvars), args.grad_clip)
149    optimizer = tf.train.AdamOptimizer(self.lr)
150    self.train_op = optimizer.apply_gradients(zip(grads, tvars))
151
152  def load_preprocessed(self, inps, outps):
153    newinps = {"training": [], "validation": []}
154    newoutps = {"training": [], "validation": []}
155    for key in newinps:
156      for i in range(len(inps[key])):
157        if len(inps[key][i]) - self.args.timedelay >= (self.args.seq_length+2):
158          if self.args.timedelay > 0:
159            newinps[key].append(inps[key][i][self.args.timedelay:])
160            newoutps[key].append(outps[key][i][:-self.args.timedelay])
161          else:
162            newinps[key].append(inps[key][i])
163            newoutps[key].append(outps[key][i])
164    print "load preprocessed", len(newinps), len(newoutps)
165    return newinps, newoutps
166
167
168  def sample(self, sess, args, data, pt):
169    if self.audioinput:
170      self.sample_audioinput(sess, args, data, pt)
171    else:
172      self.sample_videoinput(sess, args, data, pt)
173
174  def sample_audioinput(self, sess, args, data, pt):
175    meani, stdi, meano, stdo = data["inputmean"], data["inputstd"], data["outputmean"], data["outputstd"]
176    audio = np.load(self.training_dir + "/audio/normalized-cep13/" + self.args.input2 + ".wav.npy") 
177
178    audiodiff = audio[1:,:-1] - audio[:-1, :-1]
179    timestamps = audio[:, -1]
180
181    times = audio[:, -1]
182    inp = np.concatenate((audio[:-1, :-1], audiodiff[:, :]), axis=1)
183
184    state = []
185    for c, m in self.initial_state: # initial_state: ((c1, m1), (c2, m2))
186      state.append((c.eval(), m.eval()))
187
188    if not os.path.exists("results/"):
189      os.mkdir("results/")
190
191    f = open("results/" + self.args.input2 + "_" + args.save_dir + ".txt", "w")
192    print "output to results/" + self.args.input2 + "_" + args.save_dir + ".txt"
193    f.write("%d %d\n" % (len(inp), self.dimout + 1))
194    fetches = []
195    fetches.append(self.output)
196    for c, m in self.final_state: # final_state: ((c1, m1), (c2, m2))
197      fetches.append(c)
198      fetches.append(m)
199
200    feed_dict = {}
201    for i in range(len(inp)):
202      for j, (c, m) in enumerate(self.initial_state):
203        feed_dict[c], feed_dict[m] = state[j]
204
205      input = (inp[i] - meani) / stdi
206      feed_dict[self.input_data] = [[input]]
207      res = sess.run(fetches, feed_dict)
208      output = res[0] * stdo + meano
209
210      if i >= args.timedelay:
211        shifttime = times[i - args.timedelay]
212      else:
213        shifttime = times[0]
214      f.write(("%f " % shifttime) + " ".join(["%f" % x for x in output[0]]) + "\n")
215
216      state_flat = res[1:]
217      state = [state_flat[i:i+2] for i in range(0, len(state_flat), 2)] 
218    f.close()
219
220
221def main():
222  s = Speech()
223
224if __name__ == '__main__':
225  main()