新冠肺炎CT辅助诊断文献实战-02

孟想成真

接着上一讲的内容，上一讲已经把第一个模型训练完毕。
该模型的主要作用就是将CT图像分为三种类型

• NiCT：无用CT图片；
  
• pCT：潜在的可能与cov19相关的图片；
  
• nCT：与cov19无关的图片;
  

然后我们现在就要将这个模型运用在cohort1和cohort2的模型中

1. # load model
3. # the model and initial setting save in VGG_Simple.py
4. from VGG_Simple import *
5. %matplotlib inline
6. %config InlineBackend.figure_format = 'svg'
8. # load model parameter
9. version = 'version_14'  # choose best model path
10. path = os.path.join(os.getcwd(), 'check_point', version)
11. ckpt = str(os.listdir(path)[2])  # choose best model
12. print(ckpt)
14. # Create model
15. ckpt = os.path.join(path, ckpt)
16. class_weight = torch.FloatTensor([2, 5, 2]).cuda()
17. myloss = nn.CrossEntropyLoss(weight=class_weight)
18. model = VGG_Simple(myloss=myloss)
19. epoch=30-val_acc=0.99097-val_loss=0.03236.ckpt

复制代码

这里要注意一下，因为我们是将数据批量的下载到patientCT这个文件夹中，所以首先我们需要将 cohort1 和 cohort2 的数据分离开来。
cohort1 patient 1-1170
chohrt2 patient 1171 - 1521

1. batch_size = 64
2. # get patient CT-file path
3. cohort1 = ['Patient {}'.format(i) for i in range(0,1171)]
4. cohort2 = ['Patient {}'.format(i) for i in range(1171,1522)]
5. patient_list = os.listdir('../patientCT/')
7. # attention! not all samples have CT-file
8. cohort1 = set(cohort1) & set(patient_list)
9. cohort2 = set(cohort2) & set(patient_list)
11. cohort1 = [os.path.join('..','patientCT',cohort,cohort) for cohort in cohort1]
12. cohort2 = [os.path.join('..','patientCT',cohort,cohort) for cohort in cohort2]
14. cohort1 = sorted(cohort1)
15. cohort2 = sorted(cohort2)
17. # make an empty dataframe
18. result = pd.DataFrame(columns=['NiCT', 'nCT', 'pCT', 'patientId', 'label'])

复制代码

导入模型（一）

1. for i in range(len(cohort1)):
2.     print(i)
3.     cohort_sample_path = cohort1[i] # get patient CT-file path
4.     cohort_sample = cohort_sample_path.split('\\')[3] # get patientId
5.     # import patient one by one
6.     cohort = torchvision.datasets.ImageFolder(cohort_sample_path,
7.                                             transform=torchvision.transforms.Compose([
8.                                                 torchvision.transforms.Grayscale(num_output_channels=1),
9.                                                 torchvision.transforms.Scale(256),
10.                                                 torchvision.transforms.CenterCrop(200),
11.                                                 torchvision.transforms.ToTensor()])
12.                                             )
13.     # load all ct file
14.     cohort = torch.utils.data.DataLoader(cohort, batch_size=batch_size,shuffle=False,num_workers =24)
16.     XRawData = []
17.     YRawData = []
18.     for i,(x,y) in enumerate(cohort):
19.         XRawData.append(x)
20.         YRawData.append(y)
22.     XRawData = torch.cat(XRawData)
23.     YRawData = torch.cat(YRawData)
25.     cohort_iter = makeDataiter(XRawData, YRawData, batch_size=batch_size,shuffle=False)
27.     model.test_predict = []
28.     model.test_sample_label = []
29.     model.test_decoder = []
30.     model.test_matrix = []
31.     model.test_conv = []
32.     model.test_primary = []
33.     model.test_digitcaps = []
35.     trainer = pl.Trainer(resume_from_checkpoint=ckpt, gpus=-1)
37.     # predict patient ct
38.     trainer.test(model, cohort_iter)
39.     predict = torch.cat(model.test_predict)
40.     predict_label = toLabel(predict).cpu().numpy()
42.     # get each picture id
43.     sample = [cohort_sample +' ' + cohort.dataset.samples[i][0].split(
44.         '\\')[5] for i in range(len(cohort.dataset.samples))]
45.     predict = pd.DataFrame(predict.numpy())
46.     predict.columns = ['NiCT','nCT',  'pCT']
47.     predict.index = sample
48.     predict['patientId'] = cohort_sample
49.     predict['label'] = predict_label
50.     predict['label'] = predict['label'].replace(
51.        [0,1,2],  # 构建 label
52.        ['NiCT','nCT',  'pCT']).astype(str)
54.     # get top 10 pct
56.     predict = predict.sort_values(by='pCT',ascending = False)
57.     predict = predict.iloc[0:10,]
59.     result = pd.concat([result,predict])

复制代码

分别对两套数据进行预测，并将结果保存输出

1. # result2 = pd.DataFrame(columns=['NiCT', 'nCT', 'pCT', 'patientId', 'label'])
2. #
3. # for i in range(len(cohort2)):
4. #     print(i)
5. #     cohort_sample_path = cohort2[i]
6. #     cohort_sample = cohort_sample_path.split('\\')[3]
7. #     # 依次导入数据
8. #     cohort = torchvision.datasets.ImageFolder(cohort_sample_path,
9. #                                             transform=torchvision.transforms.Compose([
10. #                                                 torchvision.transforms.Grayscale(num_output_channels=1),
11. #                                                 torchvision.transforms.Scale(256),
12. #                                                 torchvision.transforms.CenterCrop(200),
13. #                                                 torchvision.transforms.ToTensor()])
14. #                                             )
15. #     cohort = torch.utils.data.DataLoader(cohort, batch_size=batch_size,shuffle=False,num_workers =24)
16. #
17. #     # 数据类型转换
18. #     XRawData = []
19. #     YRawData = []
20. #     for i,(x,y) in enumerate(cohort):
21. #         XRawData.append(x)
22. #         YRawData.append(y)
23. #         
24. #     XRawData = torch.cat(XRawData)
25. #     YRawData = torch.cat(YRawData)
26. #     
27. #     cohort_iter = makeDataiter(XRawData, YRawData, batch_size=batch_size,shuffle=False)
28. #
29. #     model.test_predict = []
30. #     model.test_sample_label = []
31. #     model.test_decoder = []
32. #     model.test_matrix = []
33. #     model.test_conv = []
34. #     model.test_primary = []
35. #     model.test_digitcaps = []
36. #     
37. #     trainer = pl.Trainer(resume_from_checkpoint=ckpt, gpus=-1)
38. #     
39. #     # predict patient ct
40. #     trainer.test(model, cohort_iter)
41. #     predict = torch.cat(model.test_predict)
42. #     predict_label = toLabel(predict).cpu().numpy()
43. #     
44. #     # get each picture id
45. #     sample = [cohort_sample +' ' + cohort.dataset.samples[i][0].split(
46. #         '\\')[5] for i in range(len(cohort.dataset.samples))]
47. #     predict = pd.DataFrame(predict.numpy())
48. #     predict.columns = ['NiCT','nCT',  'pCT']
49. #     predict.index = sample
50. #     predict['patientId'] = cohort_sample
51. #     predict['label'] = predict_label
52. #     predict['label'] = predict['label'].replace(
53. #        [0,1,2],  # 构建 label
54. #        ['NiCT','nCT',  'pCT']).astype(str)
55. #     
56. #     # get top 10 pct
57. #     
58. #     predict = predict.sort_values(by='pCT',ascending = False)
59. #     predict = predict.iloc[0:10,]
60. #
61. #     result2 = pd.concat([result2,predict])
62. # save result
63. # result.to_csv('cohort1_pCT_detection.csv')
64. # result2.to_csv('cohort2_pCT_detection.csv')

复制代码

导入数据，进行模型可靠性验证

1. cohort1_pCT = pd.read_csv('cohort1_pCT_detection.csv',index_col=0)
2. cohort2_pCT = pd.read_csv('cohort2_pCT_detection.csv',index_col=0)
3. meta = pd.read_csv('metadata.csv',index_col=0)

复制代码

这里我们要稍微对meta稍微处理一下，因为并不是每一个样本都有CT

1. cohort1 = set(cohort1_pCT.index.map(lambda x: x.split(' IMG')[0]))
2. cohort2 = set(cohort2_pCT.index.map(lambda x: x.split(' IMG')[0]))
3. patientIds = list(cohort1)+list(cohort2)
4. meta = meta.loc[patientIds]
5. # cohort1 control pictures
6. cohort1_predict_control = set(cohort1_pCT[cohort1_pCT['label'] != 'pCT'].index.map(
7.     lambda x: x.split(' IMG')[0]))
9. # cohort2 control pictures
10. cohort2_predict_control = set(cohort2_pCT[cohort2_pCT['label'] != 'pCT'].index.map(
11.     lambda x: x.split(' IMG')[0]))
13. # cohort1 pCT pictures
14. cohort1_predict_pCT = set(cohort1_pCT[cohort1_pCT['label'] == 'pCT'].index.map(
15.     lambda x: x.split(' IMG')[0]))
17. # cohort2 pCT pictures
18. cohort2_predict_pCT = set(cohort2_pCT[cohort2_pCT['label'] == 'pCT'].index.map(
19.     lambda x: x.split(' IMG')[0]))
21. cohort1_ambiguous = cohort1_predict_control & cohort1_predict_pCT
22. cohort2_ambiguous = cohort2_predict_control & cohort2_predict_pCT
24. cohort1_control = cohort1_predict_control - cohort1_ambiguous
25. cohort2_control = cohort2_predict_control - cohort2_ambiguous
27. cohort1_pCT = cohort1_predict_pCT - cohort1_ambiguous
28. cohort2_pCT = cohort2_predict_pCT - cohort2_ambiguous
30. true_control = set(meta[meta['Morbidity outcome'] == 'Control'].index)

复制代码

这里我们注意到，原始数据中控制组分为两部分，其中一部分为Community-acquired pneumonia即普通肺炎，另一部分为Control,我们就暂且当作完全健康的CT
那么根据文章所给出的定义，按道理来说pCT应该在完全健康组中不存在

1. Counter(meta['Morbidity outcome'])
2. Counter({'Regular': 542,
3.          'Severe': 170,
4.          'Suspected': 259,
5.          'Control': 96,
6.          'Control (Community-acquired pneumonia)': 218,
7.          'Mild': 22,
8.          'Critically ill': 35})
9. len(set(meta[meta['Morbidity outcome'] == 'Control'].index) & cohort1_pCT)
10. len(set(meta[meta['Morbidity outcome'] == 'Control'].index) & cohort1_ambiguous)
11. len(set(meta[meta['Morbidity outcome'] == 'Control'].index) & cohort1_control)
13. len(set(meta[meta['Morbidity outcome'] == 'Control'].index) & cohort2_pCT)
14. len(set(meta[meta['Morbidity outcome'] == 'Control'].index) & cohort2_ambiguous)
15. len(set(meta[meta['Morbidity outcome'] == 'Control'].index) & cohort2_control)

复制代码

  11
15
70
0
0
0
  在健康对照中根据输出的结果，有大约10%左右的假阳性
另外，我们还得关心一下阳性率的情况

1. disease = list(
2.     set(meta[(meta['Morbidity outcome'] != 'Control') & (
3.         meta['Morbidity outcome'] != 'Control (Community-acquired pneumonia)') & (
4.         meta['Morbidity outcome'] != 'Suspected')
5.     ].index))
6. len(set(disease) & cohort1_control)
7. len(set(disease) & cohort2_control)

复制代码

  212 102
211+102 / 769
  从上面的输出结果来看，阳性率大约为60%左右，还是可以接受的
因此，可以开始训练第二个模型

1. Best Regards,  
2. Yuan.SH  
3. ---------------------------------------
4. School of Basic Medical Sciences,  
5. Fujian Medical University,  
6. Fuzhou, Fujian, China.  
7. please contact with me via the following ways:  
8. (a) e-mail :yuansh3354@163.com

复制代码

来源：https://blog.csdn.net/qq_40966210/article/details/114021462
免责声明：如果侵犯了您的权益，请联系站长，我们会及时删除侵权内容，谢谢合作！