Pycharm debugger internal error

system: Ubuntu, pycharm 2019.3.2, pytorch 1.5.1, python 3.7

I'm facing really strange bug. When debugging the debugger hangs when going step by step (F8) I have noticed that most of the time it happened when pytorch operations take place. 

Debugging through breakpoints (using F9) the debugger works as expected.

If I try to debug the same code the debugger hangs in different places (this seems like an internal problem). When trying to collect the logs I can't find my pycharm's log dir under .cache, I only can find `PyCharm2020,1` there (I have two versions installed on my system).

When I use Evaluate Expression during debugging process I'm getting the following error:

Traceback (most recent call last):
File "/cortex/users/aviv/softwares/pycharm-2019.3/plugins/python/helpers/pydev/_pydevd_bundle/pydevd_comm.py", line 1767, in pydevd_find_thread_by_id
pydevd_log(1, "Available: %s\n" % [get_thread_id(t) for t in threads] % thread_id)
TypeError: not all arguments converted during string formatting
Traceback (most recent call last):
File "/cortex/users/aviv/softwares/pycharm-2019.3/plugins/python/helpers/pydev/_pydevd_bundle/pydevd_comm.py", line 1767, in pydevd_find_thread_by_id
pydevd_log(1, "Available: %s\n" % [get_thread_id(t) for t in threads] % thread_id)
TypeError: not all arguments converted during string formatting
Traceback (most recent call last):
File "/cortex/users/aviv/softwares/pycharm-2019.3/plugins/python/helpers/pydev/_pydevd_bundle/pydevd_comm.py", line 1767, in pydevd_find_thread_by_id
pydevd_log(1, "Available: %s\n" % [get_thread_id(t) for t in threads] % thread_id)
TypeError: not all arguments converted during string formatting
Traceback (most recent call last):
File "/cortex/users/aviv/softwares/pycharm-2019.3/plugins/python/helpers/pydev/_pydevd_bundle/pydevd_comm.py", line 1767, in pydevd_find_thread_by_id
pydevd_log(1, "Available: %s\n" % [get_thread_id(t) for t in threads] % thread_id)
TypeError: not all arguments converted during string formatting
Traceback (most recent call last):
File "/cortex/users/aviv/softwares/pycharm-2019.3/plugins/python/helpers/pydev/_pydevd_bundle/pydevd_comm.py", line 1767, in pydevd_find_thread_by_id
pydevd_log(1, "Available: %s\n" % [get_thread_id(t) for t in threads] % thread_id)
TypeError: not all arguments converted during string formatting
Traceback (most recent call last):
File "/cortex/users/aviv/softwares/pycharm-2019.3/plugins/python/helpers/pydev/_pydevd_bundle/pydevd_comm.py", line 1767, in pydevd_find_thread_by_id
pydevd_log(1, "Available: %s\n" % [get_thread_id(t) for t in threads] % thread_id)
TypeError: not all arguments converted during string formatting
Traceback (most recent call last):
File "/cortex/users/aviv/softwares/pycharm-2019.3/plugins/python/helpers/pydev/_pydevd_bundle/pydevd_comm.py", line 1767, in pydevd_find_thread_by_id
pydevd_log(1, "Available: %s\n" % [get_thread_id(t) for t in threads] % thread_id)
TypeError: not all arguments converted during string formatting
Traceback (most recent call last):
File "/cortex/users/aviv/softwares/pycharm-2019.3/plugins/python/helpers/pydev/_pydevd_bundle/pydevd_comm.py", line 1767, in pydevd_find_thread_by_id
pydevd_log(1, "Available: %s\n" % [get_thread_id(t) for t in threads] % thread_id)
TypeError: not all arguments converted during string formatting
Traceback (most recent call last):
File "/cortex/users/aviv/softwares/pycharm-2019.3/plugins/python/helpers/pydev/_pydevd_bundle/pydevd_comm.py", line 1767, in pydevd_find_thread_by_id
pydevd_log(1, "Available: %s\n" % [get_thread_id(t) for t in threads] % thread_id)
TypeError: not all arguments converted during string formatting
Traceback (most recent call last):
File "/cortex/users/aviv/softwares/pycharm-2019.3/plugins/python/helpers/pydev/_pydevd_bundle/pydevd_comm.py", line 1767, in pydevd_find_thread_by_id
pydevd_log(1, "Available: %s\n" % [get_thread_id(t) for t in threads] % thread_id)
TypeError: not all arguments converted during string formatting
Traceback (most recent call last):
File "/cortex/users/aviv/softwares/pycharm-2019.3/plugins/python/helpers/pydev/_pydevd_bundle/pydevd_comm.py", line 1767, in pydevd_find_thread_by_id
pydevd_log(1, "Available: %s\n" % [get_thread_id(t) for t in threads] % thread_id)
TypeError: not all arguments converted during string formatting
Traceback (most recent call last):
File "/cortex/users/aviv/softwares/pycharm-2019.3/plugins/python/helpers/pydev/_pydevd_bundle/pydevd_comm.py", line 1767, in pydevd_find_thread_by_id
pydevd_log(1, "Available: %s\n" % [get_thread_id(t) for t in threads] % thread_id)
TypeError: not all arguments converted during string formatting

here is code snip which reproduce the problem on my setup:

import os
import random
import fnmatch

import matplotlib.pyplot as plt
import torch
import numpy as np
from torch.utils.data import Dataset
from torchvision.transforms import transforms
from PIL import Image

from experiments.pascal_voc import custom_transforms as tr

class VOCSegmentation(Dataset):
"""
PascalVoc dataset
"""
NUM_CLASSES = 21

def __init__(self,
base_size=513,
crop_size=512,
base_dir: str = "/cortex/data/images/paretoHN/data/pascal_voc/VOCdevkit/VOC2012",
split='train',
):
"""
:param base_dir: path to VOC dataset directory
:param split: train/val
"""
super().__init__()
self._base_dir = base_dir
self._image_dir = os.path.join(self._base_dir, 'JPEGImages')
self._cat_dir = os.path.join(self._base_dir, 'SegmentationClass')

if isinstance(split, str):
self.split = [split]
else:
split.sort()
self.split = split

self.base_size = base_size
self.crop_size = crop_size

_splits_dir = os.path.join(self._base_dir, 'ImageSets', 'Segmentation')

self.im_ids = []
self.images = []
self.categories = []

for splt in self.split:
with open(os.path.join(os.path.join(_splits_dir, splt + '.txt')), "r") as f:
lines = f.read().splitlines()

for ii, line in enumerate(lines):
_image = os.path.join(self._image_dir, line + ".jpg")
_cat = os.path.join(self._cat_dir, line + ".png")
assert os.path.isfile(_image)
assert os.path.isfile(_cat)
self.im_ids.append(line)
self.images.append(_image)
self.categories.append(_cat)

assert (len(self.images) == len(self.categories))

# Display stats
print('Number of images in {}: {:d}'.format(split, len(self.images)))
print('#' * 8 + " Remember to add ignore_index=255 in your loss " + '#' * 8)

def __len__(self):
return len(self.images)

def __getitem__(self, index):
_img, _target = self._make_img_gt_point_pair(index)
sample = {'image': _img, 'label': _target}

for split in self.split:
if split == "train":
sample = self.transform_tr(sample)
elif split == 'val':
sample = self.transform_val(sample)
return sample['image'], sample['label'].type(torch.LongTensor)

def _make_img_gt_point_pair(self, index):
_img = Image.open(self.images[index]).convert('RGB')
_target = Image.open(self.categories[index])

return _img, _target

def transform_tr(self, sample):
composed_transforms = transforms.Compose([
tr.RandomScaleCrop(base_size=self.base_size, crop_size=self.crop_size),
tr.ToTensor()])
return composed_transforms(sample)

def transform_val(self, sample):

composed_transforms = transforms.Compose([
tr.FixScaleCrop(crop_size=self.crop_size),
tr.ToTensor()])
return composed_transforms(sample)

def __str__(self):
return 'VOC2012(split=' + str(self.split) + ')'

if __name__ == '__main__':
from torch.utils.data import DataLoader
import matplotlib.pyplot as plt

voc_train = VOCSegmentation()

batch_size = 8
dataloader = DataLoader(voc_train, batch_size=batch_size, shuffle=True, num_workers=0)
img, label = next(iter(dataloader))

idx = random.randint(0, 8)
plt.subplot(121)
img_p = img[idx].type(torch.uint8)
plt.imshow(transforms.ToPILImage()(img_p))
plt.subplot(122)
plt.imshow(label[idx].numpy())
plt.show()

print("debug")

class CityScape(Dataset):
"""Modification of: https://github.com/lorenmt/mtan/blob/master/im2im_pred/create_dataset.py
The (pre-processed) data is available here: https://www.dropbox.com/s/q2333k4eyrnezbh/cityscapes.zip?dl=0
"""
def __init__(self, root, n_classes, train=True, mode="segmentation"):
self.train = train
self.root = os.path.expanduser(root)
self.n_classes = n_classes
self.mode = mode

# read the data file
if train:
self.data_path = root + '/train'
else:
self.data_path = root + '/val'

# calculate data length
self.data_len = len(fnmatch.filter(os.listdir(self.data_path + '/image'), '*.npy'))

def _get_seg_label(self, n_classes, index):
if n_classes == 19:
semantic = torch.from_numpy(np.load(self.data_path + f'/label_19/{index}.npy'))
elif n_classes == 7:
semantic = torch.from_numpy(np.load(self.data_path + f'/label_7/{index}.npy'))
elif n_classes == 2:
semantic = torch.from_numpy(np.load(self.data_path + f'/label_2/{index}.npy'))
else:
raise NotImplementedError("Currently only {19,7,2} classes are supported")
return semantic

def __getitem__(self, index):
image = torch.from_numpy(np.moveaxis(np.load(self.data_path + f'/image/{index}.npy'), -1, 0))

if self.mode == "segmentation":
semantic = []
for c in self.n_classes:

semantic.append(self._get_seg_label(c, index))

return (
image.float(),
semantic[0].type(torch.LongTensor),
semantic[1].type(torch.LongTensor)
)

elif self.mode == "depth":
assert isinstance(self.n_classes, int), "n_classes must be a scalar!"
# get image name from the pandas df

semantic = self._get_seg_label(self.n_classes, index)

try:
depth = torch.from_numpy(np.moveaxis(np.load(self.data_path + f'/depth/{index}.npy'), -1, 0))

except:
print(f'depth of index = {index} is bad...')
depth = torch.zeros((1, 128, 256))

return (
image.float(),
semantic.type(torch.LongTensor),
depth.float()
)

elif self.mode == "single_seg":
assert isinstance(self.n_classes, int), "n_classes must be a scalar!"

semantic = self._get_seg_label(self.n_classes, index)

return (
image.float(),
semantic.type(torch.LongTensor),
)

else:
raise NotImplementedError("This mode is not supported use {segmentation, depth}")

def __len__(self):
return self.data_len

task1_train_set = CityScape("/cortex/data/images/cityscapes_mtan", train=True, n_classes=19, mode="single_seg")
task1_test_set = CityScape("/cortex/data/images/cityscapes_mtan", train=False, n_classes=19, mode="single_seg")
task2_train_set = VOCSegmentation(split="train", crop_size=256)
task2_test_set = VOCSegmentation(split="val", crop_size=256)

task1_train_loader = torch.utils.data.DataLoader(
dataset=task1_train_set,
batch_size=8,
shuffle=True,
num_workers=12
)
task1_test_loader = torch.utils.data.DataLoader(
dataset=task1_test_set,
batch_size=8,
shuffle=False,
num_workers=12
)
task2_train_loader = torch.utils.data.DataLoader(
dataset=task2_train_set,
batch_size=8,
shuffle=True,
num_workers=12
)
task2_test_loader = torch.utils.data.DataLoader(
dataset=task2_test_set,
batch_size=8,
shuffle=False,
num_workers=12
)
for i in range(10):
idx = random.randint(0, 7)
img1, seg1 = next(iter(task1_train_loader))
img2, seg2 = next(iter(task2_train_loader))
plt.subplot(221)
plt.imshow(img1[idx].premute(1,2,0).cpu().numpy().astype(np.uint8))
plt.subplot(222)
plt.imshow(seg1[idx].cpu().numpy().astype(np.uint8))
plt.subplot(223)
plt.imshow(img2[idx].premute(1,2,0).cpu().numpy().astype(np.uint8))
plt.subplot(224)
plt.imshow(seg2[idx].cpu().numpy().astype(np.uint8))
plt.show()
plt.close()