diff --git a/3d_classification/torch/densenet_evaluation_array.py b/3d_classification/torch/densenet_evaluation_array.py index 43428ba4d5..07f4650120 100644 --- a/3d_classification/torch/densenet_evaluation_array.py +++ b/3d_classification/torch/densenet_evaluation_array.py @@ -12,7 +12,7 @@ import logging import os import sys - +import json import numpy as np import torch from torch.utils.data import DataLoader @@ -27,21 +27,17 @@ def main(): logging.basicConfig(stream=sys.stdout, level=logging.INFO) # IXI dataset as a demo, downloadable from https://brain-development.org/ixi-dataset/ - images = [ - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI607-Guys-1097-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI175-HH-1570-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI385-HH-2078-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI344-Guys-0905-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI409-Guys-0960-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI584-Guys-1129-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI253-HH-1694-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI092-HH-1436-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI574-IOP-1156-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI585-Guys-1130-T1.nii.gz"]), - ] - + # here we load part of the datalist from FHIR format config file + with open("ixi_datalist.json") as ixi_datalist: + datalist = json.load(ixi_datalist) + dirpath = os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1"]) + images = list() + for i in range(21, 30): + filename = datalist["entry"][i]["resource"]["content"]["url"].split("//")[-1] + images.append(os.path.join(dirpath, filename)) # 2 binary labels for gender classification: man and woman - labels = np.array([0, 0, 1, 0, 1, 0, 1, 0, 1, 0], dtype=np.int64) + labels = [0 if datalist["entry"][i]["resource"]["note"]["text"] == "man" else 1 for i in range(21, 30)] + labels = np.array(labels, dtype=np.int64) # Define transforms for image val_transforms = Compose([ScaleIntensity(), AddChannel(), Resize((96, 96, 96)), ToTensor()]) diff --git a/3d_classification/torch/densenet_evaluation_dict.py b/3d_classification/torch/densenet_evaluation_dict.py index 0d6b2420fb..e00b85305a 100644 --- a/3d_classification/torch/densenet_evaluation_dict.py +++ b/3d_classification/torch/densenet_evaluation_dict.py @@ -12,7 +12,7 @@ import logging import os import sys - +import json import numpy as np import torch from torch.utils.data import DataLoader @@ -27,21 +27,18 @@ def main(): logging.basicConfig(stream=sys.stdout, level=logging.INFO) # IXI dataset as a demo, downloadable from https://brain-development.org/ixi-dataset/ - images = [ - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI607-Guys-1097-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI175-HH-1570-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI385-HH-2078-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI344-Guys-0905-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI409-Guys-0960-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI584-Guys-1129-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI253-HH-1694-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI092-HH-1436-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI574-IOP-1156-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI585-Guys-1130-T1.nii.gz"]), - ] - + # here we load part of the datalist from FHIR format config file + with open("ixi_datalist.json") as ixi_datalist: + datalist = json.load(ixi_datalist) + dirpath = os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1"]) + images = list() + for i in range(21, 30): + filename = datalist["entry"][i]["resource"]["content"]["url"].split("//")[-1] + images.append(os.path.join(dirpath, filename)) # 2 binary labels for gender classification: man and woman - labels = np.array([0, 0, 1, 0, 1, 0, 1, 0, 1, 0], dtype=np.int64) + labels = [0 if datalist["entry"][i]["resource"]["note"]["text"] == "man" else 1 for i in range(21, 30)] + labels = np.array(labels, dtype=np.int64) + val_files = [{"img": img, "label": label} for img, label in zip(images, labels)] # Define transforms for image diff --git a/3d_classification/torch/densenet_training_array.py b/3d_classification/torch/densenet_training_array.py index a0712a4a57..46e6483ff6 100644 --- a/3d_classification/torch/densenet_training_array.py +++ b/3d_classification/torch/densenet_training_array.py @@ -12,7 +12,7 @@ import logging import os import sys - +import json import numpy as np import torch from torch.utils.data import DataLoader @@ -28,31 +28,17 @@ def main(): logging.basicConfig(stream=sys.stdout, level=logging.INFO) # IXI dataset as a demo, downloadable from https://brain-development.org/ixi-dataset/ - images = [ - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI314-IOP-0889-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI249-Guys-1072-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI609-HH-2600-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI173-HH-1590-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI020-Guys-0700-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI342-Guys-0909-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI134-Guys-0780-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI577-HH-2661-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI066-Guys-0731-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI130-HH-1528-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI607-Guys-1097-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI175-HH-1570-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI385-HH-2078-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI344-Guys-0905-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI409-Guys-0960-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI584-Guys-1129-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI253-HH-1694-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI092-HH-1436-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI574-IOP-1156-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI585-Guys-1130-T1.nii.gz"]), - ] - + # here we load part of the datalist from FHIR format config file + with open("ixi_datalist.json") as ixi_datalist: + datalist = json.load(ixi_datalist) + dirpath = os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1"]) + images = list() + for i in range(0, 20): + filename = datalist["entry"][i]["resource"]["content"]["url"].split("//")[-1] + images.append(os.path.join(dirpath, filename)) # 2 binary labels for gender classification: man and woman - labels = np.array([0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0], dtype=np.int64) + labels = [0 if datalist["entry"][i]["resource"]["note"]["text"] == "man" else 1 for i in range(0, 20)] + labels = np.array(labels, dtype=np.int64) # Define transforms train_transforms = Compose([ScaleIntensity(), AddChannel(), Resize((96, 96, 96)), RandRotate90(), ToTensor()]) diff --git a/3d_classification/torch/densenet_training_dict.py b/3d_classification/torch/densenet_training_dict.py index 6d4a590e87..8cabfff777 100644 --- a/3d_classification/torch/densenet_training_dict.py +++ b/3d_classification/torch/densenet_training_dict.py @@ -12,7 +12,7 @@ import logging import os import sys - +import json import numpy as np import torch from torch.utils.data import DataLoader @@ -28,31 +28,18 @@ def main(): logging.basicConfig(stream=sys.stdout, level=logging.INFO) # IXI dataset as a demo, downloadable from https://brain-development.org/ixi-dataset/ - images = [ - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI314-IOP-0889-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI249-Guys-1072-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI609-HH-2600-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI173-HH-1590-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI020-Guys-0700-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI342-Guys-0909-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI134-Guys-0780-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI577-HH-2661-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI066-Guys-0731-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI130-HH-1528-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI607-Guys-1097-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI175-HH-1570-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI385-HH-2078-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI344-Guys-0905-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI409-Guys-0960-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI584-Guys-1129-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI253-HH-1694-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI092-HH-1436-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI574-IOP-1156-T1.nii.gz"]), - os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1", "IXI585-Guys-1130-T1.nii.gz"]), - ] - + # here we load part of the datalist from FHIR format config file + with open("ixi_datalist.json") as ixi_datalist: + datalist = json.load(ixi_datalist) + dirpath = os.sep.join(["workspace", "data", "medical", "ixi", "IXI-T1"]) + images = list() + for i in range(0, 20): + filename = datalist["entry"][i]["resource"]["content"]["url"].split("//")[-1] + images.append(os.path.join(dirpath, filename)) # 2 binary labels for gender classification: man and woman - labels = np.array([0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0], dtype=np.int64) + labels = [0 if datalist["entry"][i]["resource"]["note"]["text"] == "man" else 1 for i in range(0, 20)] + labels = np.array(labels, dtype=np.int64) + train_files = [{"img": img, "label": label} for img, label in zip(images[:10], labels[:10])] val_files = [{"img": img, "label": label} for img, label in zip(images[-10:], labels[-10:])] diff --git a/3d_classification/torch/ixi_datalist.json b/3d_classification/torch/ixi_datalist.json new file mode 100644 index 0000000000..3cadfe54fa --- /dev/null +++ b/3d_classification/torch/ixi_datalist.json @@ -0,0 +1,460 @@ +{ + "resourceType": "Bundle", + "id": "ixi-datalist", + "meta": { + "lastUpdated": "2020-09-25T01:00:00Z" + }, + "type": "collection", + "entry": [ + { + "resource": { + "resourceType": "Media", + "id": "1", + "status": "unknown", + "type": "image", + "content": { + "contentType": "application/x-gzip", + "url": "file://IXI314-IOP-0889-T1.nii.gz" + }, + "note": { + "text": "man" + } + } + }, + { + "resource": { + "resourceType": "Media", + "id": "2", + "status": "unknown", + "type": "image", + "content": { + "contentType": "application/x-gzip", + "url": "file://IXI249-Guys-1072-T1.nii.gz" + }, + "note": { + "text": "man" + } + } + }, + { + "resource": { + "resourceType": "Media", + "id": "3", + "status": "unknown", + "type": "image", + "content": { + "contentType": "application/x-gzip", + "url": "file://IXI609-HH-2600-T1.nii.gz" + }, + "note": { + "text": "man" + } + } + }, + { + "resource": { + "resourceType": "Media", + "id": "4", + "status": "unknown", + "type": "image", + "content": { + "contentType": "application/x-gzip", + "url": "file://IXI173-HH-1590-T1.nii.gz" + }, + "note": { + "text": "woman" + } + } + }, + { + "resource": { + "resourceType": "Media", + "id": "5", + "status": "unknown", + "type": "image", + "content": { + "contentType": "application/x-gzip", + "url": "file://IXI020-Guys-0700-T1.nii.gz" + }, + "note": { + "text": "man" + } + } + }, + { + "resource": { + "resourceType": "Media", + "id": "6", + "status": "unknown", + "type": "image", + "content": { + "contentType": "application/x-gzip", + "url": "file://IXI342-Guys-0909-T1.nii.gz" + }, + "note": { + "text": "man" + } + } + }, + { + "resource": { + "resourceType": "Media", + "id": "7", + "status": "unknown", + "type": "image", + "content": { + "contentType": "application/x-gzip", + "url": "file://IXI134-Guys-0780-T1.nii.gz" + }, + "note": { + "text": "man" + } + } + }, + { + "resource": { + "resourceType": "Media", + "id": "8", + "status": "unknown", + "type": "image", + "content": { + "contentType": "application/x-gzip", + "url": "file://IXI577-HH-2661-T1.nii.gz" + }, + "note": { + "text": "woman" + } + } + }, + { + "resource": { + "resourceType": "Media", + "id": "9", + "status": "unknown", + "type": "image", + "content": { + "contentType": "application/x-gzip", + "url": "file://IXI066-Guys-0731-T1.nii.gz" + }, + "note": { + "text": "woman" + } + } + }, + { + "resource": { + "resourceType": "Media", + "id": "10", + "status": "unknown", + "type": "image", + "content": { + "contentType": "application/x-gzip", + "url": "file://IXI130-HH-1528-T1.nii.gz" + }, + "note": { + "text": "man" + } + } + }, + { + "resource": { + "resourceType": "Media", + "id": "11", + "status": "unknown", + "type": "image", + "content": { + "contentType": "application/x-gzip", + "url": "file://IXI607-Guys-1097-T1.nii.gz" + }, + "note": { + "text": "man" + } + } + }, + { + "resource": { + "resourceType": "Media", + "id": "12", + "status": "unknown", + "type": "image", + "content": { + "contentType": "application/x-gzip", + "url": "file://IXI175-HH-1570-T1.nii.gz" + }, + "note": { + "text": "man" + } + } + }, + { + "resource": { + "resourceType": "Media", + "id": "13", + "status": "unknown", + "type": "image", + "content": { + "contentType": "application/x-gzip", + "url": "file://IXI385-HH-2078-T1.nii.gz" + }, + "note": { + "text": "woman" + } + } + }, + { + "resource": { + "resourceType": "Media", + "id": "14", + "status": "unknown", + "type": "image", + "content": { + "contentType": "application/x-gzip", + "url": "file://IXI344-Guys-0905-T1.nii.gz" + }, + "note": { + "text": "man" + } + } + }, + { + "resource": { + "resourceType": "Media", + "id": "15", + "status": "unknown", + "type": "image", + "content": { + "contentType": "application/x-gzip", + "url": "file://IXI409-Guys-0960-T1.nii.gz" + }, + "note": { + "text": "woman" + } + } + }, + { + "resource": { + "resourceType": "Media", + "id": "16", + "status": "unknown", + "type": "image", + "content": { + "contentType": "application/x-gzip", + "url": "file://IXI584-Guys-1129-T1.nii.gz" + }, + "note": { + "text": "man" + } + } + }, + { + "resource": { + "resourceType": "Media", + "id": "17", + "status": "unknown", + "type": "image", + "content": { + "contentType": "application/x-gzip", + "url": "file://IXI253-HH-1694-T1.nii.gz" + }, + "note": { + "text": "woman" + } + } + }, + { + "resource": { + "resourceType": "Media", + "id": "18", + "status": "unknown", + "type": "image", + "content": { + "contentType": "application/x-gzip", + "url": "file://IXI092-HH-1436-T1.nii.gz" + }, + "note": { + "text": "man" + } + } + }, + { + "resource": { + "resourceType": "Media", + "id": "19", + "status": "unknown", + "type": "image", + "content": { + "contentType": "application/x-gzip", + "url": "file://IXI574-IOP-1156-T1.nii.gz" + }, + "note": { + "text": "woman" + } + } + }, + { + "resource": { + "resourceType": "Media", + "id": "20", + "status": "unknown", + "type": "image", + "content": { + "contentType": "application/x-gzip", + "url": "file://IXI585-Guys-1130-T1.nii.gz" + }, + "note": { + "text": "man" + } + } + }, + { + "resource": { + "resourceType": "Media", + "id": "21", + "status": "unknown", + "type": "image", + "content": { + "contentType": "application/x-gzip", + "url": "file://IXI607-Guys-1097-T1.nii.gz" + }, + "note": { + "text": "man" + } + } + }, + { + "resource": { + "resourceType": "Media", + "id": "22", + "status": "unknown", + "type": "image", + "content": { + "contentType": "application/x-gzip", + "url": "file://IXI175-HH-1570-T1.nii.gz" + }, + "note": { + "text": "man" + } + } + }, + { + "resource": { + "resourceType": "Media", + "id": "23", + "status": "unknown", + "type": "image", + "content": { + "contentType": "application/x-gzip", + "url": "file://IXI385-HH-2078-T1.nii.gz" + }, + "note": { + "text": "woman" + } + } + }, + { + "resource": { + "resourceType": "Media", + "id": "24", + "status": "unknown", + "type": "image", + "content": { + "contentType": "application/x-gzip", + "url": "file://IXI344-Guys-0905-T1.nii.gz" + }, + "note": { + "text": "man" + } + } + }, + { + "resource": { + "resourceType": "Media", + "id": "25", + "status": "unknown", + "type": "image", + "content": { + "contentType": "application/x-gzip", + "url": "file://IXI409-Guys-0960-T1.nii.gz" + }, + "note": { + "text": "woman" + } + } + }, + { + "resource": { + "resourceType": "Media", + "id": "26", + "status": "unknown", + "type": "image", + "content": { + "contentType": "application/x-gzip", + "url": "file://IXI584-Guys-1129-T1.nii.gz" + }, + "note": { + "text": "man" + } + } + }, + { + "resource": { + "resourceType": "Media", + "id": "27", + "status": "unknown", + "type": "image", + "content": { + "contentType": "application/x-gzip", + "url": "file://IXI253-HH-1694-T1.nii.gz" + }, + "note": { + "text": "woman" + } + } + }, + { + "resource": { + "resourceType": "Media", + "id": "28", + "status": "unknown", + "type": "image", + "content": { + "contentType": "application/x-gzip", + "url": "file://IXI092-HH-1436-T1.nii.gz" + }, + "note": { + "text": "man" + } + } + }, + { + "resource": { + "resourceType": "Media", + "id": "29", + "status": "unknown", + "type": "image", + "content": { + "contentType": "application/x-gzip", + "url": "file://IXI574-IOP-1156-T1.nii.gz" + }, + "note": { + "text": "woman" + } + } + }, + { + "resource": { + "resourceType": "Media", + "id": "30", + "status": "unknown", + "type": "image", + "content": { + "contentType": "application/x-gzip", + "url": "file://IXI585-Guys-1130-T1.nii.gz" + }, + "note": { + "text": "man" + } + } + } + ] +}