from fastai.vision.all import *
from sklearn.model_selection import StratifiedKFold

import os, cv2
from albumentations import (
    Compose,GaussianBlur,HorizontalFlip,MedianBlur,MotionBlur,OneOf,
    RandomBrightness,RandomContrast,Resize,ShiftScaleRotate,VerticalFlip
    )
path = Path('./data/images')

class AlbumentationsTransform(RandTransform):
    "A transform handler for multiple `Albumentation` transforms"
    split_idx,order=None,2
    def __init__(self, train_aug, valid_aug): store_attr()
    
    def before_call(self, b, split_idx):
        self.idx = split_idx
    
    def encodes(self, img: PILImage):
        if self.idx == 0:
            aug_img = self.train_aug(image=np.array(img))['image']
        else:
            aug_img = self.valid_aug(image=np.array(img))['image']

        return PILImage.create(aug_img)

We can now define our transforms as normal. These are from the code base I am working from. I move Normalize to batch transforms and use the fastai normalize instead, which does it in batches on the GPU in fastai. It's a small performance boost. With a dataset this small it probably just wasn't worth the time to set that up on the GPU using pytorch lightning, but it's free in fastai so may as well.

image_size = [480, 768]
def get_train_aug(image_size): return Compose(
    [
        Resize(height=image_size[0], width=image_size[1]),
        OneOf([RandomBrightness(limit=0.1, p=1), RandomContrast(limit=0.1, p=1)]), #fastai has
        OneOf([MotionBlur(blur_limit=3), MedianBlur(blur_limit=3), GaussianBlur(blur_limit=3)], p=0.5),
        VerticalFlip(p=0.5),#Dihedral
        HorizontalFlip(p=0.5),
        ShiftScaleRotate(
            shift_limit=0.2,
            scale_limit=0.2,
            rotate_limit=20,
            interpolation=cv2.INTER_LINEAR,
            border_mode=cv2.BORDER_REFLECT_101,
            p=1,
        ),
    ])

def get_valid_aug(image_size): return  Compose(
    [
        Resize(height=image_size[0], width=image_size[1]),
    ])

item_tfms = [AlbumentationsTransform(get_train_aug(image_size), get_valid_aug(image_size))]
batch_tfms = [Normalize.from_stats(*imagenet_stats)]

Data

def process_df(path):
    df = pd.read_csv(path)
    df['image_id'] = 'data/images/' + df.image_id + '.jpg'
    df['label'] = df.apply(lambda x: " ".join([cat for cat in df.columns if x[cat] == 1]), axis=1)
    return df

train = process_df('data/train.csv')
train.sample(n=5)

for img in train.image_id:
    img_loaded= Image.open(img)
    if img_loaded.shape == (1365, 2048): continue
    print(img,img_loaded.shape)
    img_loaded.transpose(Image.TRANSPOSE).save(img)

Now, there is nothing special about the data so we can use the high level API to create a dataloaders object. We can take a look at a few images with fastai's built in visualizations. They look good - so we can feel good that we are looking at the data in the right way and that our transforms aren't doing something super weird.

dls = ImageDataLoaders.from_df(train,bs=16,seed=2020,item_tfms=item_tfms, batch_tfms=batch_tfms,label_col=5)
dls.show_batch(max_n=16)

class CrossEntropyLossOneHot(nn.Module):
    def __init__(self):
        super(CrossEntropyLossOneHot, self).__init__()
        self.log_softmax = nn.LogSoftmax(dim=-1)

    def forward(self, preds, labels):
        return torch.mean(torch.sum(-labels * self.log_softmax(preds), -1))

import pretrainedmodels

def l2_norm(input, axis=1):
    norm = torch.norm(input, 2, axis, True)
    output = torch.div(input, norm)
    return output


class BinaryHead(nn.Module):
    def __init__(self, num_class=4, emb_size=2048, s=16.0):
        super(BinaryHead, self).__init__()
        self.s = s
        self.fc = nn.Sequential(nn.Linear(emb_size, num_class))

    def forward(self, fea):
        fea = l2_norm(fea)
        logit = self.fc(fea) * self.s
        return logit


class se_resnext50_32x4d(nn.Module):
    def __init__(self):
        super(se_resnext50_32x4d, self).__init__()

        self.model_ft = nn.Sequential(
            *list(pretrainedmodels.__dict__["se_resnext50_32x4d"](num_classes=1000, pretrained="imagenet").children())[
                :-2
            ]
        )
        self.avg_pool = nn.AdaptiveAvgPool2d((1, 1))
        self.model_ft.last_linear = None
        self.fea_bn = nn.BatchNorm1d(2048)
        self.fea_bn.bias.requires_grad_(False)
        self.binary_head = BinaryHead(4, emb_size=2048, s=1)
        self.dropout = nn.Dropout(p=0.2)

    def forward(self, x):

        img_feature = self.model_ft(x)
        img_feature = self.avg_pool(img_feature)
        img_feature = img_feature.view(img_feature.size(0), -1)
        fea = self.fea_bn(img_feature)
        # fea = self.dropout(fea)
        output = self.binary_head(fea)

        return output

class SoftLabelCB(Callback):
    
    def before_train(self):
        self.imgs_list = L(o for o in self.dl.items.iloc[:,0].values) # get list of images in the order they are drawn this epoch
        self.df = self.dl.items.set_index('image_id')
    
    def before_validate(self):
        self.imgs_list = L(o for o in self.dl.items.iloc[:,0].values) # get list of images in the order they are drawn this epoch
        self.df = self.dl.items.set_index('image_id')

    def before_batch(self):
        df = self.df
        imgs = self.imgs_list[self.dl._DataLoader__idxs[self.iter*self.dl.bs:self.iter*self.dl.bs+self.dl.bs]]
        one_hot_yb = df.loc[imgs,df.columns[:-1]].values
        self.learn.yb = (Tensor(one_hot_yb).cuda(),)

When I train I will be doing folds to generate new labels for the training set for knowledge distillation. I might as well set up inference and make predictions for each of the folds. I'm also adding 4 fold TTA - it's no effort for me to do it in fastai (learn.get_preds vs learn.tta.)

Inference

def test_predict(cnt,msg):
    # Create Test Dataloaders
    test = process_df('data/sample_submission.csv')
    test_dl = dls.test_dl(test) 

    # predict with test time augmentation
    preds, _ = learn.tta(dl=test_dl) 
    p = preds.softmax(axis=1) 

    # format submission file
    test = pd.read_csv('data/sample_submission.csv')['image_id']
    out_a = pd.concat([test,pd.DataFrame(p,columns = learn.dls.vocab)],axis=1)[['image_id','healthy','multiple_diseases','rust','scab']]

    # write to csv and submit to kaggle
    out_a.to_csv(f'submission{cnt}.csv',index=False)
    os.system(f'''kaggle competitions submit -c plant-pathology-2020-fgvc7 -f submission{cnt}.csv -m "{msg}"''')

# #hide_output
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=1)
splits, preds, targs, preds_c,  = [],[],[],[]

true = pd.DataFrame(columns = L(o for o in train.columns))
pred = pd.DataFrame(columns = L(o for o in train.columns))
i=0
for _, val_idx in skf.split(train.image_id,train.label):
    splitter = IndexSplitter(val_idx)

    # Create dataloaders splittin on indexes defined by StratifiedKFold
    db = DataBlock(blocks=(ImageBlock,CategoryBlock),
                    get_x=ColReader(0),get_y=ColReader(5),
                    item_tfms=item_tfms,batch_tfms=batch_tfms,
                    splitter=splitter)
    dls = db.dataloaders(train,bs=24)

    #train model with fastai dataloaders, pytorch model, pytorch loss function, fastai gradient clipping, custom callback, on fp16 precision 
    learn = Learner(dls,se_resnext50_32x4d(),loss_func=CrossEntropyLossOneHot(),cbs=[GradientClip,SoftLabelCB()]).to_fp16()
    learn.fine_tune(80,reset_opt=True) # Train freeze epoch then unfreeze for 80 epochs   

    p, _ = learn.tta() # test time augmentation
    p=p.softmax(axis=1) # Convert to probabilities

    # Format dataframe to save
    items_pred=pd.DataFrame(p,columns=dls.vocab)
    items_pred['label'] = [dls.vocab[int(o)] for o in p.argmax(dim=1)]
    items_pred['image_id'] = dls.valid.items.image_id.values
    items_pred = items_pred[train.columns]
    true = pd.concat([true,dls.valid.items])
    pred = pd.concat([pred,items_pred])

    # predict and submit to kaggle
    test_predict(i,f'distilling labels fold count {i}') 
    i+=1

pred.to_csv('distilled_labels.csv',index=False)

• Load training labels and datat

train = process_df('data/train.csv')
train = train.sort_values('image_id')

• Load predictions from previous k-fold

distilled_labels = pd.read_csv('distilled_labels.csv')
distilled_labels = distilled_labels.sort_values('image_id');

# Get one hot encoded labels (zeros and ones)
distilled_labels.iloc[:,1:-1] = pd.get_dummies(distilled_labels.label)

• Weight as you see fit (winner did 7:3 ratio)

# check to make sure the data matches
assert (train.image_id.values==distilled_labels.image_id.values).all()
distilled_labels.reset_index(drop=True,inplace=True); train.reset_index(drop=True,inplace=True); 

# get soft labels
train.iloc[:,1:-1] = distilled_labels.iloc[:,1:-1] * .3 + train.iloc[:,1:-1] * .7
train.loc[train.healthy == 0.3][:5]

train.head()