Flake8 / Pep8 fixes

Author: dasguptar

.gitignore

@@ -9,4 +9,3 @@ job.py
 sftp-config.json
 .ftpconfig
 .ftpignore
-.vscode

LICENSE

@@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+SOFTWARE.

config.py

@@ -2,8 +2,9 @@
 
 
 def parse_args():
-    parser = argparse.ArgumentParser(description='PyTorch TreeLSTM for Sentence Similarity on Dependency Trees')
-    #
+    parser = argparse.ArgumentParser(
+        description='PyTorch TreeLSTM for Sentence Similarity on Dependency Trees')
+    # data arguments
     parser.add_argument('--data', default='data/sick/',
                         help='path to dataset')
     parser.add_argument('--glove', default='data/glove/',

fetch_and_preprocess.sh

@@ -4,4 +4,4 @@ python2.7 scripts/download.py
 
 CLASSPATH="lib:lib/stanford-parser/stanford-parser.jar:lib/stanford-parser/stanford-parser-3.5.1-models.jar"
 javac -cp $CLASSPATH lib/*.java
-python2.7 scripts/preprocess-sick.py
+python2.7 scripts/preprocess-sick.py

main.py

@@ -73,7 +73,9 @@ def main():
         build_vocab(token_files, sick_vocab_file)
 
     # get vocab object from vocab file previously written
-    vocab = Vocab(filename=sick_vocab_file, data=[Constants.PAD_WORD, Constants.UNK_WORD, Constants.BOS_WORD, Constants.EOS_WORD])
+    vocab = Vocab(filename=sick_vocab_file,
+                  data=[Constants.PAD_WORD, Constants.UNK_WORD,
+                        Constants.BOS_WORD, Constants.EOS_WORD])
     logger.debug('==> SICK vocabulary size : %d ' % vocab.size())
 
     # load SICK dataset splits
@@ -101,22 +103,25 @@ def main():
 
     # initialize model, criterion/loss_function, optimizer
     model = SimilarityTreeLSTM(
-                vocab.size(),
-                args.input_dim,
-                args.mem_dim,
-                args.hidden_dim,
-                args.num_classes,
-                args.sparse,
-                args.freeze_embed)
+        vocab.size(),
+        args.input_dim,
+        args.mem_dim,
+        args.hidden_dim,
+        args.num_classes,
+        args.sparse,
+        args.freeze_embed)
     criterion = nn.KLDivLoss()
     if args.cuda:
         model.cuda(), criterion.cuda()
     if args.optim == 'adam':
-        optimizer = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=args.lr, weight_decay=args.wd)
+        optimizer = optim.Adam(filter(lambda p: p.requires_grad,
+                                      model.parameters()), lr=args.lr, weight_decay=args.wd)
     elif args.optim == 'adagrad':
-        optimizer = optim.Adagrad(filter(lambda p: p.requires_grad, model.parameters()), lr=args.lr, weight_decay=args.wd)
+        optimizer = optim.Adagrad(filter(lambda p: p.requires_grad,
+                                         model.parameters()), lr=args.lr, weight_decay=args.wd)
     elif args.optim == 'sgd':
-        optimizer = optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), lr=args.lr, weight_decay=args.wd)
+        optimizer = optim.SGD(filter(lambda p: p.requires_grad,
+                                     model.parameters()), lr=args.lr, weight_decay=args.wd)
     metrics = Metrics(args.num_classes)
 
     # for words common to dataset vocab and GLOVE, use GLOVE vectors
@@ -130,7 +135,8 @@ def main():
         logger.debug('==> GLOVE vocabulary size: %d ' % glove_vocab.size())
         emb = torch.Tensor(vocab.size(), glove_emb.size(1)).normal_(-0.05, 0.05)
         # zero out the embeddings for padding and other special words if they are absent in vocab
-        for idx, item in enumerate([Constants.PAD_WORD, Constants.UNK_WORD, Constants.BOS_WORD, Constants.EOS_WORD]):
+        for idx, item in enumerate([Constants.PAD_WORD, Constants.UNK_WORD,
+                                    Constants.BOS_WORD, Constants.EOS_WORD]):
             emb[idx].zero_()
         for word in vocab.labelToIdx.keys():
             if glove_vocab.getIndex(word):
@@ -146,29 +152,32 @@ def main():
 
     best = -float('inf')
     for epoch in range(args.epochs):
-        train_loss             = trainer.train(train_dataset)
+        train_loss = trainer.train(train_dataset)
         train_loss, train_pred = trainer.test(train_dataset)
-        dev_loss, dev_pred     = trainer.test(dev_dataset)
-        test_loss, test_pred   = trainer.test(test_dataset)
+        dev_loss, dev_pred = trainer.test(dev_dataset)
+        test_loss, test_pred = trainer.test(test_dataset)
 
         train_pearson = metrics.pearson(train_pred, train_dataset.labels)
         train_mse = metrics.mse(train_pred, train_dataset.labels)
-        logger.info('==> Epoch {}, Train \tLoss: {}\tPearson: {}\tMSE: {}'.format(epoch, train_loss, train_pearson, train_mse))
+        logger.info('==> Epoch {}, Train \tLoss: {}\tPearson: {}\tMSE: {}'.format(
+            epoch, train_loss, train_pearson, train_mse))
         dev_pearson = metrics.pearson(dev_pred, dev_dataset.labels)
         dev_mse = metrics.mse(dev_pred, dev_dataset.labels)
-        logger.info('==> Epoch {}, Dev \tLoss: {}\tPearson: {}\tMSE: {}'.format(epoch, dev_loss, dev_pearson, dev_mse))
+        logger.info('==> Epoch {}, Dev \tLoss: {}\tPearson: {}\tMSE: {}'.format(
+            epoch, dev_loss, dev_pearson, dev_mse))
         test_pearson = metrics.pearson(test_pred, test_dataset.labels)
         test_mse = metrics.mse(test_pred, test_dataset.labels)
-        logger.info('==> Epoch {}, Test \tLoss: {}\tPearson: {}\tMSE: {}'.format(epoch, test_loss, test_pearson, test_mse))
+        logger.info('==> Epoch {}, Test \tLoss: {}\tPearson: {}\tMSE: {}'.format(
+            epoch, test_loss, test_pearson, test_mse))
 
         if best < test_pearson:
             best = test_pearson
             checkpoint = {
-                'model': trainer.model.state_dict(), 
+                'model': trainer.model.state_dict(),
                 'optim': trainer.optimizer,
                 'pearson': test_pearson, 'mse': test_mse,
                 'args': args, 'epoch': epoch
-                }
+            }
             logger.debug('==> New optimum found, checkpointing everything now...')
             torch.save(checkpoint, '%s.pt' % os.path.join(args.save, args.expname))
 

model.py

@@ -25,17 +25,18 @@ def node_forward(self, inputs, child_c, child_h):
         i, o, u = F.sigmoid(i), F.sigmoid(o), F.tanh(u)
 
         f = F.sigmoid(
-                self.fh(child_h) +
-                self.fx(inputs).repeat(len(child_h), 1)
-            )
+            self.fh(child_h) +
+            self.fx(inputs).repeat(len(child_h), 1)
+        )
         fc = torch.mul(f, child_c)
 
         c = torch.mul(i, u) + torch.sum(fc, dim=0, keepdim=True)
         h = torch.mul(o, F.tanh(c))
         return c, h
 
     def forward(self, tree, inputs):
-        _ = [self.forward(tree.children[idx], inputs) for idx in range(tree.num_children)]
+        for idx in range(tree.num_children):
+            self.forward(tree.children[idx], inputs)
 
         if tree.num_children == 0:
             child_c = Var(inputs[0].data.new(1, self.mem_dim).fill_(0.))

scripts/download.py

@@ -10,27 +10,26 @@
 import urllib2
 import sys
 import os
-import shutil
 import zipfile
-import gzip
+
 
 def download(url, dirpath):
     filename = url.split('/')[-1]
     filepath = os.path.join(dirpath, filename)
     try:
         u = urllib2.urlopen(url)
-    except:
-        print("URL %s failed to open" %url)
+    except Exception as e:
+        print("URL %s failed to open" % url)
         raise Exception
     try:
         f = open(filepath, 'wb')
-    except:
-        print("Cannot write %s" %filepath)
+    except Exception as e:
+        print("Cannot write %s" % filepath)
         raise Exception
     try:
         filesize = int(u.info().getheaders("Content-Length")[0])
-    except:
-        print("URL %s failed to report length" %url)
+    except Exception as e:
+        print("URL %s failed to report length" % url)
         raise Exception
     print("Downloading: %s Bytes: %s" % (filename, filesize))
 
@@ -47,19 +46,22 @@ def download(url, dirpath):
         downloaded += len(buf)
         f.write(buf)
         status = (("[%-" + str(status_width + 1) + "s] %3.2f%%") %
-            ('=' * int(float(downloaded) / filesize * status_width) + '>', downloaded * 100. / filesize))
+                  ('=' * int(downloaded / filesize * status_width) + '>',
+                   downloaded * 100. / filesize))
         print(status, end='')
         sys.stdout.flush()
     f.close()
     return filepath
 
+
 def unzip(filepath):
     print("Extracting: " + filepath)
     dirpath = os.path.dirname(filepath)
     with zipfile.ZipFile(filepath) as zf:
         zf.extractall(dirpath)
     os.remove(filepath)
 
+
 def download_tagger(dirpath):
     tagger_dir = 'stanford-tagger'
     if os.path.exists(os.path.join(dirpath, tagger_dir)):
@@ -74,6 +76,7 @@ def download_tagger(dirpath):
     os.remove(filepath)
     os.rename(os.path.join(dirpath, zip_dir), os.path.join(dirpath, tagger_dir))
 
+
 def download_parser(dirpath):
     parser_dir = 'stanford-parser'
     if os.path.exists(os.path.join(dirpath, parser_dir)):
@@ -88,6 +91,7 @@ def download_parser(dirpath):
     os.remove(filepath)
     os.rename(os.path.join(dirpath, zip_dir), os.path.join(dirpath, parser_dir))
 
+
 def download_wordvecs(dirpath):
     if os.path.exists(dirpath):
         print('Found Glove vectors - skip')
@@ -97,6 +101,7 @@ def download_wordvecs(dirpath):
     url = 'http://www-nlp.stanford.edu/data/glove.840B.300d.zip'
     unzip(download(url, dirpath))
 
+
 def download_sick(dirpath):
     if os.path.exists(dirpath):
         print('Found SICK dataset - skip')
@@ -110,6 +115,7 @@ def download_sick(dirpath):
     unzip(download(trial_url, dirpath))
     unzip(download(test_url, dirpath))
 
+
 if __name__ == '__main__':
     base_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 

scripts/preprocess-sick.py

@@ -6,33 +6,37 @@
 import os
 import glob
 
+
 def make_dirs(dirs):
     for d in dirs:
         if not os.path.exists(d):
             os.makedirs(d)
 
+
 def dependency_parse(filepath, cp='', tokenize=True):
     print('\nDependency parsing ' + filepath)
     dirpath = os.path.dirname(filepath)
     filepre = os.path.splitext(os.path.basename(filepath))[0]
     tokpath = os.path.join(dirpath, filepre + '.toks')
     parentpath = os.path.join(dirpath, filepre + '.parents')
-    relpath =  os.path.join(dirpath, filepre + '.rels')
+    relpath = os.path.join(dirpath, filepre + '.rels')
     tokenize_flag = '-tokenize - ' if tokenize else ''
     cmd = ('java -cp %s DependencyParse -tokpath %s -parentpath %s -relpath %s %s < %s'
-        % (cp, tokpath, parentpath, relpath, tokenize_flag, filepath))
+           % (cp, tokpath, parentpath, relpath, tokenize_flag, filepath))
     os.system(cmd)
 
+
 def constituency_parse(filepath, cp='', tokenize=True):
     dirpath = os.path.dirname(filepath)
     filepre = os.path.splitext(os.path.basename(filepath))[0]
     tokpath = os.path.join(dirpath, filepre + '.toks')
     parentpath = os.path.join(dirpath, filepre + '.cparents')
     tokenize_flag = '-tokenize - ' if tokenize else ''
     cmd = ('java -cp %s ConstituencyParse -tokpath %s -parentpath %s %s < %s'
-        % (cp, tokpath, parentpath, tokenize_flag, filepath))
+           % (cp, tokpath, parentpath, tokenize_flag, filepath))
     os.system(cmd)
 
+
 def build_vocab(filepaths, dst_path, lowercase=True):
     vocab = set()
     for filepath in filepaths:
@@ -45,26 +49,29 @@ def build_vocab(filepaths, dst_path, lowercase=True):
         for w in sorted(vocab):
             f.write(w + '\n')
 
+
 def split(filepath, dst_dir):
     with open(filepath) as datafile, \
-         open(os.path.join(dst_dir, 'a.txt'), 'w') as afile, \
-         open(os.path.join(dst_dir, 'b.txt'), 'w') as bfile,  \
-         open(os.path.join(dst_dir, 'id.txt'), 'w') as idfile, \
-         open(os.path.join(dst_dir, 'sim.txt'), 'w') as simfile:
-            datafile.readline()
-            for line in datafile:
-                i, a, b, sim, ent = line.strip().split('\t')
-                idfile.write(i + '\n')
-                afile.write(a + '\n')
-                bfile.write(b + '\n')
-                simfile.write(sim + '\n')
+            open(os.path.join(dst_dir, 'a.txt'), 'w') as afile, \
+            open(os.path.join(dst_dir, 'b.txt'), 'w') as bfile,  \
+            open(os.path.join(dst_dir, 'id.txt'), 'w') as idfile, \
+            open(os.path.join(dst_dir, 'sim.txt'), 'w') as simfile:
+        datafile.readline()
+        for line in datafile:
+            i, a, b, sim, ent = line.strip().split('\t')
+            idfile.write(i + '\n')
+            afile.write(a + '\n')
+            bfile.write(b + '\n')
+            simfile.write(sim + '\n')
+
 
 def parse(dirpath, cp=''):
     dependency_parse(os.path.join(dirpath, 'a.txt'), cp=cp, tokenize=True)
     dependency_parse(os.path.join(dirpath, 'b.txt'), cp=cp, tokenize=True)
     constituency_parse(os.path.join(dirpath, 'a.txt'), cp=cp, tokenize=True)
     constituency_parse(os.path.join(dirpath, 'b.txt'), cp=cp, tokenize=True)
 
+
 if __name__ == '__main__':
     print('=' * 80)
     print('Preprocessing SICK dataset')

trainer.py

@@ -9,19 +9,19 @@
 class Trainer(object):
     def __init__(self, args, model, criterion, optimizer):
         super(Trainer, self).__init__()
-        self.args       = args
-        self.model      = model
-        self.criterion  = criterion
-        self.optimizer  = optimizer
-        self.epoch      = 0
+        self.args = args
+        self.model = model
+        self.criterion = criterion
+        self.optimizer = optimizer
+        self.epoch = 0
 
     # helper function for training
     def train(self, dataset):
         self.model.train()
         self.optimizer.zero_grad()
         total_loss = 0.0
         indices = torch.randperm(len(dataset))
-        for idx in tqdm(range(len(dataset)),desc='Training epoch ' + str(self.epoch + 1) + ''):
+        for idx in tqdm(range(len(dataset)), desc='Training epoch ' + str(self.epoch + 1) + ''):
             ltree, lsent, rtree, rsent, label = dataset[indices[idx]]
             linput, rinput = Var(lsent), Var(rsent)
             target = Var(map_label_to_target(label, dataset.num_classes))
@@ -44,7 +44,7 @@ def test(self, dataset):
         total_loss = 0
         predictions = torch.zeros(len(dataset))
         indices = torch.arange(1, dataset.num_classes + 1)
-        for idx in tqdm(range(len(dataset)),desc='Testing epoch  ' + str(self.epoch) + ''):
+        for idx in tqdm(range(len(dataset)), desc='Testing epoch  ' + str(self.epoch) + ''):
             ltree, lsent, rtree, rsent, label = dataset[idx]
             linput, rinput = Var(lsent, volatile=True), Var(rsent, volatile=True)
             target = Var(map_label_to_target(label, dataset.num_classes), volatile=True)

utils.py

@@ -54,12 +54,13 @@ def build_vocab(filenames, vocabfile):
         for token in sorted(vocab):
             f.write(token + '\n')
 
+
 # mapping from scalar to vector
-def map_label_to_target(label,num_classes):
-    target = torch.zeros(1,num_classes)
+def map_label_to_target(label, num_classes):
+    target = torch.zeros(1, num_classes)
     ceil = int(math.ceil(label))
     floor = int(math.floor(label))
-    if ceil==floor:
+    if ceil == floor:
         target[0][floor-1] = 1
     else:
         target[0][floor-1] = ceil - label