Add new file

# Conflicts:
#	1 - ImageGatherer.py
#	3 - TestTrainSplit.py
#	4 - TransferLearningKeras.py

.gitlab-ci.yml

@@ -0,0 +1,45 @@
+# This file is a template, and might need editing before it works on your project.
+# To contribute improvements to CI/CD templates, please follow the Development guide at:
+# https://docs.gitlab.com/ee/development/cicd/templates.html
+# This specific template is located at:
+# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml
+
+# This is a sample GitLab CI/CD configuration file that should run without any modifications.
+# It demonstrates a basic 3 stage CI/CD pipeline. Instead of real tests or scripts,
+# it uses echo commands to simulate the pipeline execution.
+#
+# A pipeline is composed of independent jobs that run scripts, grouped into stages.
+# Stages run in sequential order, but jobs within stages run in parallel.
+#
+# For more information, see: https://docs.gitlab.com/ee/ci/yaml/index.html#stages
+
+stages:          # List of stages for jobs, and their order of execution
+  - build
+  - test
+  - deploy
+
+build-job:       # This job runs in the build stage, which runs first.
+  stage: build
+  script:
+    - echo "Compiling the code..."
+    - echo "Compile complete."
+
+unit-test-job:   # This job runs in the test stage.
+  stage: test    # It only starts when the job in the build stage completes successfully.
+  script:
+    - echo "Running unit tests... This will take about 60 seconds."
+    - sleep 60
+    - echo "Code coverage is 90%"
+
+lint-test-job:   # This job also runs in the test stage.
+  stage: test    # It can run at the same time as unit-test-job (in parallel).
+  script:
+    - echo "Linting code... This will take about 10 seconds."
+    - sleep 10
+    - echo "No lint issues found."
+
+deploy-job:      # This job runs in the deploy stage.
+  stage: deploy  # It only runs when *both* jobs in the test stage complete successfully.
+  script:
+    - echo "Deploying application..."
+    - echo "Application successfully deployed."

1_download_images.py

@@ -5,6 +5,9 @@ import json
 from pprint import pprint
 from google_images_download import google_images_download
 
+total_per = 10
+form_increment = 1
+
 
 def create_forms_dict(df):
     poke_dict = {}
@@ -39,22 +42,26 @@ def process_pokemon_names(df):
     pprint(poke_dict)
     pokes_to_limits = []
     for pokemon, form_list in poke_dict.items():
-        if len(form_list) == 0:
         print(pokemon)
-            pokes_to_limits.append((pokemon, 200))
+        num_forms = len(form_list)
+        if num_forms == 0:
+            pokes_to_limits.append((pokemon, total_per))
 
-        elif len(form_list) == 1:
-            pokes_to_limits.append((pokemon, 150))
-            pokes_to_limits.append((search_term(form_list[0]), 50))
+        elif num_forms == 1:
+            pokes_to_limits.append((pokemon, total_per - form_increment))
+            pokes_to_limits.append((search_term(form_list[0]), form_increment))
 
-        elif len(form_list) == 2:
-            pokes_to_limits.append((pokemon, 100))
+        elif num_forms == 2:
+            pokes_to_limits.append((pokemon, total_per - form_increment * num_forms))
             for form in form_list:
-                pokes_to_limits.append((search_term(form), 50))
+                pokes_to_limits.append((search_term(form), form_increment))
 
-        elif len(form_list) >= 3:
+        elif num_forms >= 3:
+            revised_increment = int(total_per / len(form_list))
             for form in form_list:
-                pokes_to_limits.append((search_term(form), int(200 / len(form_list))))
+                pokes_to_limits.append((pokemon, total_per - revised_increment * num_forms))
+
+                pokes_to_limits.append((search_term(form), revised_increment))
 
     return pokes_to_limits
 

2_fix_data.py

@@ -7,9 +7,7 @@ import multiprocessing
 import json
 import shutil
 
-from pathlib import Path
 from PIL import Image
-from pprint import pprint
 from random import randint
 from threading import Lock
 

3_test_train_split.py

@@ -1,12 +1,13 @@
 import os
 from random import random
-from shutil import copyfile, rmtree
+from shutil import rmtree
 from pathlib import Path
 import multiprocessing
 
 train_dir = "./data/train/"
 test_dir = "./data/test/"
 val_dir = "./data/val/"
+
 train = .80
 test = .10
 val = .10

4 - TrainingModelKeras.py

@@ -1,189 +0,0 @@
-import keras
-import matplotlib.pyplot as plt
-import numpy as np
-import pandas as pd
-import seaborn as sn
-
-from keras import optimizers
-from keras.applications import inception_v3, mobilenet_v2, vgg16
-from keras.applications.inception_v3 import preprocess_input
-from keras.callbacks import ModelCheckpoint, EarlyStopping, TensorBoard
-from keras.layers import Dense, Dropout, GlobalAveragePooling2D
-from keras.models import Sequential
-from keras.preprocessing.image import ImageDataGenerator
-
-from sklearn.metrics import accuracy_score, confusion_matrix, classification_report
-
-from time import time
-from PIL import ImageFile
-
-# First we some globals that we want to use for this entire process
-
-ImageFile.LOAD_TRUNCATED_IMAGES = True
-
-input_shape = (224, 224, 3)
-batch_size = 96
-
-model_name = "mobilenet-fixed-data"
-
-# Next we set up the Image Data Generators to feed into the training cycles.
-# We need one for training, validation, and testing
-train_idg = ImageDataGenerator(
-    horizontal_flip=True,
-    rotation_range=30,
-    width_shift_range=[-.1, .1],
-    height_shift_range=[-.1, .1],
-    preprocessing_function=preprocess_input
-)
-
-train_gen = train_idg.flow_from_directory(
-    './data/train',
-    target_size=(input_shape[0], input_shape[1]),
-    batch_size=batch_size
-)
-
-print(len(train_gen.classes))
-
-val_idg = ImageDataGenerator(
-    horizontal_flip=True,
-    rotation_range=30,
-    width_shift_range=[-.1, .1],
-    height_shift_range=[-.1, .1],
-    preprocessing_function=preprocess_input
-)
-
-val_gen = val_idg.flow_from_directory(
-    './data/val',
-    target_size=(input_shape[0], input_shape[1]),
-    batch_size=batch_size
-)
-
-test_idg = ImageDataGenerator(
-    preprocessing_function=preprocess_input,
-)
-test_gen = test_idg.flow_from_directory(
-    './data/test',
-    target_size=(input_shape[0], input_shape[1]),
-    batch_size=batch_size,
-    shuffle=False
-
-)
-
-# Now we define the model we are going to use....to use something differnet just comment it out or add it here
-
-# base_model = vgg16.VGG16(
-#     weights='imagenet',
-#     include_top=False,
-#     input_shape=input_shape
-# )
-# base_model = inception_v3.InceptionV3(
-#     weights='imagenet',
-#     include_top=False,
-#     input_shape=input_shape
-# )
-
-base_model = mobilenet_v2.MobileNetV2(
-    # weights='imagenet',
-    include_top=False,
-    input_shape=input_shape
-)
-
-
-# Create a new top for that model
-add_model = Sequential()
-add_model.add(base_model)
-add_model.add(GlobalAveragePooling2D())
-# add_model.add(Dense(4048, activation='relu'))
-# add_model.add(Dropout(0.5))
-
-add_model.add(Dense(2024, activation='relu'))
-# Adding some dense layers in order to learn complex functions from the base model
-add_model.add(Dropout(0.5))
-add_model.add(Dense(512, activation='relu'))
-add_model.add(Dense(len(train_gen.class_indices), activation='softmax'))  # Decision layer
-
-model = add_model
-model.compile(loss='categorical_crossentropy',
-              # optimizer=optimizers.SGD(lr=1e-4, momentum=0.9),
-              optimizer=optimizers.Adam(lr=1e-4),
-              metrics=['accuracy'])
-model.summary()
-print(
-    model.output_shape
-)
-
-# Now that the model is created we can go ahead and train on it using the image generators we created earlier
-file_path = model_name + ".hdf5"
-
-checkpoint = ModelCheckpoint(file_path, monitor='val_acc', verbose=1, save_best_only=True, mode='max')
-
-early = EarlyStopping(monitor="val_acc", mode="max", patience=15)
-
-tensorboard = TensorBoard(
-    log_dir="logs/" + model_name + "{}".format(time()), histogram_freq=0, batch_size=batch_size,
-    write_graph=True,
-    write_grads=True,
-    write_images=True,
-    update_freq=batch_size
-)
-
-callbacks_list = [checkpoint, early, tensorboard]  # early
-
-history = model.fit_generator(
-    train_gen,
-    validation_data=val_gen,
-    steps_per_epoch=len(train_gen),
-    validation_steps=len(val_gen),
-    epochs=25,
-    shuffle=True,
-    verbose=True,
-    callbacks=callbacks_list
-)
-
-
-# Finally we are going to grab predictions from our model, save it, and then run some analysis on the results
-
-predicts = model.predict_generator(test_gen, verbose=True, workers=1, steps=len(test_gen))
-
-keras_file = model_name + 'finished.h5'
-keras.models.save_model(model, keras_file)
-
-print(predicts)
-print(type(predicts))
-print(predicts.shape)
-# Process the predictions
-predicts = np.argmax(predicts,
-                     axis=1)
-# test_gen.reset()
-label_index = {v: k for k, v in train_gen.class_indices.items()}
-predicts = [label_index[p] for p in predicts]
-reals = [label_index[p] for p in test_gen.classes]
-
-# Save the results
-print(label_index)
-print(test_gen.classes)
-print(test_gen.classes.shape)
-print(type(test_gen.classes))
-df = pd.DataFrame(columns=['fname', 'prediction', 'true_val'])
-df['fname'] = [x for x in test_gen.filenames]
-df['prediction'] = predicts
-df["true_val"] = reals
-df.to_csv("sub1_non_transfer.csv", index=False)
-
-# Processed the saved results
-
-acc = accuracy_score(reals, predicts)
-conf_mat = confusion_matrix(reals, predicts)
-print(classification_report(reals, predicts, [l for l in label_index.values()]))
-print("Testing accuracy score is ", acc)
-print("Confusion Matrix", conf_mat)
-
-df_cm = pd.DataFrame(conf_mat, index=[i for i in list(set(reals))],
-                     columns=[i for i in list(set(reals))])
-plt.figure(figsize=(10, 7))
-sn.heatmap(df_cm, annot=True)
-plt.show()
-
-with open("labels.txt", "w") as f:
-    for label in label_index.values():
-        f.write(label + "\n")

4 - TransferLearningKeras.py

@@ -1,123 +0,0 @@
-from time import time
-
-import matplotlib.pyplot as plt
-import numpy as np
-import pandas as pd
-import seaborn as sn
-from PIL import ImageFile
-from tensorflow import keras
-
-from model_builders import ImageClassModelBuilder, ImageClassModels
-
-ImageFile.LOAD_TRUNCATED_IMAGES = True
-
-input_shape = (224, 224, 3)
-
-batch_size = 32
-model_name = f"mobilenetv2-dense1024-l1l2-25drop-{time()}"
-
-training_idg = keras.preprocessing.image.ImageDataGenerator(
-    horizontal_flip=True,
-    rotation_range=30,
-    width_shift_range=[-.1, .1],
-    height_shift_range=[-.1, .1],
-)
-testing_idg = keras.preprocessing.image.ImageDataGenerator(
-    horizontal_flip=True,
-)
-
-
-def get_gen(path, test_set=False):
-    idg = testing_idg if test_set else training_idg
-    return idg.flow_from_directory(
-        path,
-        target_size=(input_shape[0], input_shape[1]),
-        batch_size=batch_size,
-        class_mode='categorical',
-        shuffle=True,
-        color_mode='rgb'
-    )
-
-
-def train_model(train_gen, val_gen):
-    model = ImageClassModelBuilder(
-        input_shape=input_shape,
-        n_classes=807,
-        optimizer=keras.optimizers.Adam(learning_rate=.0001),
-        pre_trained=True,
-        fine_tune=0,
-        base_model=ImageClassModels.MOBILENET_V2
-    ).create_model()
-    # Train the model
-    checkpoint = keras.callbacks.ModelCheckpoint(f"./Models/keras/{model_name}.hdf5", monitor='val_loss', verbose=1,
-                                                 save_best_only=True,
-                                                 mode='min')
-    early = keras.callbacks.EarlyStopping(monitor="loss", mode="min", patience=15)
-    tensorboard = keras.callbacks.TensorBoard(
-        log_dir="logs/" + model_name,
-        histogram_freq=1,
-        write_graph=True,
-        write_images=True,
-        update_freq=1,
-        profile_batch=2,
-        embeddings_freq=1,
-    )
-    callbacks_list = [checkpoint, early, tensorboard]
-
-    history = model.fit(
-        train_gen,
-        validation_data=val_gen,
-        epochs=100,
-        batch_size=batch_size,
-        shuffle=True,
-        verbose=True,
-        workers=12,
-        callbacks=callbacks_list,
-        max_queue_size=1000
-    )
-    print(history)
-    return model
-
-
-def test_model(model, test_gen):
-    print(len(test_gen.filenames))
-    score = model.evaluate(test_gen, workers=8, steps=len(test_gen))
-    predicts = model.predict(test_gen, verbose=True, workers=8, steps=len(test_gen))
-    print("Loss: ", score[0], "Accuracy: ", score[1])
-    print(score)
-    print(predicts)
-    print(type(predicts))
-    print(predicts.shape)
-
-    # Process the predictions
-    predicts = np.argmax(predicts,
-                         axis=1)
-    label_index = {v: k for k, v in test_gen.class_indices.items()}
-    predicts = [label_index[p] for p in predicts]
-    reals = [label_index[p] for p in test_gen.classes]
-
-    # Save the results
-    df = pd.DataFrame(columns=['fname', 'prediction', 'true_val'])
-    df['fname'] = [x for x in test_gen.filenames]
-    df['prediction'] = predicts
-    df["true_val"] = reals
-    df.to_csv("sub1.csv", index=False)
-    # Processed the saved results
-    from sklearn.metrics import accuracy_score, confusion_matrix
-    acc = accuracy_score(reals, predicts)
-    conf_mat = confusion_matrix(reals, predicts)
-    print("Testing accuracy score is ", acc)
-    print("Confusion Matrix", conf_mat)
-    df_cm = pd.DataFrame(conf_mat, index=[i for i in list(set(reals))],
-                         columns=[i for i in list(set(reals))])
-    plt.figure(figsize=(10, 7))
-    sn.heatmap(df_cm, annot=True)
-    plt.show()
-
-
-if __name__ == "__main__":
-    train_gen = get_gen('./data/train')
-    val_gen = get_gen('./data/val')
-    test_gen = get_gen('./data/test', test_set=True)
-    model = train_model(train_gen, val_gen)
-    test_model(model, test_gen)

4_train_keras_model.py

@@ -0,0 +1,197 @@
+from enum import Enum
+
+import matplotlib.pyplot as plt
+import numpy as np
+from PIL import ImageFile
+from sklearn.metrics import accuracy_score, confusion_matrix, classification_report
+from tensorflow import keras
+
+from modeling_utils import ImageClassModelBuilder, ImageClassModels
+
+ImageFile.LOAD_TRUNCATED_IMAGES = True
+
+input_shape = (224, 224, 3)
+
+batch_size = 32
+
+training_idg = keras.preprocessing.image.ImageDataGenerator(
+    horizontal_flip=True,
+    rotation_range=30,
+    width_shift_range=[-.1, .1],
+    height_shift_range=[-.1, .1],
+)
+val_idg = keras.preprocessing.image.ImageDataGenerator(
+    horizontal_flip=True,
+)
+testing_idg = keras.preprocessing.image.ImageDataGenerator(
+    horizontal_flip=True,
+)
+
+
+class DatasetType(Enum):
+    TRAIN = 0
+    TEST = 1
+    VAL = 2
+
+
+def get_gen(path, dataset_type: DatasetType = DatasetType.TRAIN):
+    idg = None
+    if dataset_type is DatasetType.TRAIN:
+        idg = training_idg
+    if dataset_type is DatasetType.TEST:
+        idg = testing_idg
+    if dataset_type is DatasetType.VAL:
+        idg = val_idg
+
+    return idg.flow_from_directory(
+        path,
+        target_size=(input_shape[0], input_shape[1]),
+        batch_size=batch_size,
+        class_mode='categorical',
+        shuffle=True,
+        color_mode='rgb'
+    )
+
+
+def train_model(model, model_name, train_gen, val_gen, max_epochs):
+    print(model)
+    print(f"NOW TRAINING: {model_name}")
+    checkpoint = keras.callbacks.ModelCheckpoint(
+        f"./models/keras/{model_name}.hdf5",
+        monitor='val_categorical_crossentropy',
+        verbose=1,
+        save_best_only=True,
+        mode='min'
+    )
+    early = keras.callbacks.EarlyStopping(
+        monitor="val_categorical_crossentropy",
+        mode="auto",
+        patience=4,
+        restore_best_weights=True,
+        verbose=1,
+    )
+    tensorboard = keras.callbacks.TensorBoard(
+        log_dir="logs/" + model_name,
+        histogram_freq=1,
+        write_graph=True,
+        write_images=True,
+        update_freq=1,
+        profile_batch=2,
+        embeddings_freq=1,
+    )
+    model.fit(
+        train_gen,
+        validation_data=val_gen,
+        epochs=max_epochs,
+        batch_size=batch_size,
+        shuffle=True,
+        verbose=True,
+        workers=20,
+        callbacks=[checkpoint, early, tensorboard],
+        max_queue_size=1000
+    )
+    return model
+
+
+def test_model(model, test_gen):
+    predictions = model.predict(test_gen, verbose=True, workers=1, steps=len(test_gen))
+
+    print(predictions)
+    print(type(predictions))
+    print(predictions.shape)
+    # Process the predictions
+    predictions = np.argmax(predictions,
+                            axis=1)
+    # test_gen.reset()
+    label_index = {v: k for k, v in test_gen.class_indices.items()}
+    predictions = [label_index[p] for p in predictions]
+    reals = [label_index[p] for p in test_gen.classes]
+
+    # Processed the saved results
+    acc = accuracy_score(reals, predictions)
+    conf_mat = confusion_matrix(reals, predictions)
+    print(classification_report(reals, predictions, labels=[l for l in label_index.values()]))
+    print("Testing accuracy score is ", acc)
+    print("Confusion Matrix", conf_mat)
+
+    print("made dataframe")
+    plt.figure(figsize=(10, 7))
+    print("made plot")
+    print("showing plot")
+    plt.show()
+
+
+if __name__ == "__main__":
+    model_builders = [
+        ImageClassModelBuilder(
+            input_shape=input_shape,
+            n_classes=807,
+            optimizer=keras.optimizers.Adam(learning_rate=.0001),
+            pre_trained=True,
+            freeze_layers=True,
+            freeze_batch_norm=True,
+            base_model_type=ImageClassModels.MOBILENET_V2,
+            dense_layer_neurons=1024,
+            dropout_rate=.5,
+        ),        ImageClassModelBuilder(
+            input_shape=input_shape,
+            n_classes=807,
+            optimizer=keras.optimizers.Adam(learning_rate=.0001),
+            pre_trained=True,
+            freeze_layers=True,
+            freeze_batch_norm=True,
+            base_model_type=ImageClassModels.INCEPTION_RESNET_V2,
+            dense_layer_neurons=1024,
+            dropout_rate=.5,
+        ),        ImageClassModelBuilder(
+            input_shape=input_shape,
+            n_classes=807,
+            optimizer=keras.optimizers.Adam(learning_rate=.0001),
+            pre_trained=True,
+            freeze_layers=True,
+            freeze_batch_norm=True,
+            base_model_type=ImageClassModels.INCEPTION_V3,
+            dense_layer_neurons=1024,
+            dropout_rate=.5,
+        ),        ImageClassModelBuilder(
+            input_shape=input_shape,
+            n_classes=807,
+            optimizer=keras.optimizers.Adam(learning_rate=.0001),
+            pre_trained=True,
+            freeze_layers=True,
+            freeze_batch_norm=True,
+            base_model_type=ImageClassModels.XCEPTION,
+            dense_layer_neurons=1024,
+            dropout_rate=.5,
+        ),        ImageClassModelBuilder(
+            input_shape=input_shape,
+            n_classes=807,
+            optimizer=keras.optimizers.Adam(learning_rate=.0001),
+            pre_trained=True,
+            freeze_layers=True,
+            freeze_batch_norm=True,
+            base_model_type=ImageClassModels.DENSENET201,
+            dense_layer_neurons=1024,
+            dropout_rate=.5,
+        )
+    ]
+    for mb in model_builders:
+        model = mb.create_model()
+        model_name = mb.get_name()
+        train_gen = get_gen('./data/train', dataset_type=DatasetType.TRAIN)
+        val_gen = get_gen('./data/val', dataset_type=DatasetType.VAL)
+        test_gen = get_gen('./data/test', dataset_type=DatasetType.TEST)
+        model = train_model(model, model_name, train_gen, val_gen, 1)
+        # for layer in model.layers[2].layers:
+        #     if not isinstance(layer, keras.layers.BatchNormalization):
+        #         layer.trainable = True
+        # model.layers[2].trainable = True
+        # print(model)
+        # model.compile(
+        #     optimizer=keras.optimizers.Adam(learning_rate=.00001),
+        #     loss=keras.losses.CategoricalCrossentropy(),
+        #     metrics=['accuracy', 'categorical_crossentropy']
+        # )
+        # model.summary()
+        # model = train_model(model, model_name + "-second_stage", train_gen, val_gen, 1)
+        # test_model(model, test_gen)

5 - TestModel.py

@@ -1,78 +0,0 @@
-import pandas as pd
-import matplotlib.pyplot as plt
-import seaborn as sn
-import numpy as np
-
-from keras.applications.inception_v3 import preprocess_input
-from keras.preprocessing.image import ImageDataGenerator
-from keras.models import load_model
-from sklearn.metrics import accuracy_score, confusion_matrix, classification_report
-
-
-from PIL import ImageFile
-
-ImageFile.LOAD_TRUNCATED_IMAGES = True
-
-model = load_model("./Models/mobilenetv2-stock-all-fixed-v2/mobilenetv2.hdf5")
-
-input_shape = (224, 224, 3)
-batch_size = 96
-
-test_idg = ImageDataGenerator(
-    preprocessing_function=preprocess_input,
-)
-
-test_gen = test_idg.flow_from_directory(
-    # './data/test',
-    './SingleImageTestSet',
-    target_size=(input_shape[0], input_shape[1]),
-    batch_size=batch_size,
-    shuffle=False
-
-)
-
-predictions = model.predict_generator(test_gen, verbose=True, workers=1, steps=len(test_gen))
-
-print(predictions)
-print(type(predictions))
-print(predictions.shape)
-# Process the predictions
-predictions = np.argmax(predictions,
-                        axis=1)
-# test_gen.reset()
-label_index = {v: k for k, v in test_gen.class_indices.items()}
-predictions = [label_index[p] for p in predictions]
-reals = [label_index[p] for p in test_gen.classes]
-
-# Save the results
-print(label_index)
-print(test_gen.classes)
-print(test_gen.classes.shape)
-print(type(test_gen.classes))
-df = pd.DataFrame(columns=['fname', 'prediction', 'true_val'])
-df['fname'] = [x for x in test_gen.filenames]
-df['prediction'] = predictions
-df["true_val"] = reals
-df.to_csv("sub1_non_transfer.csv", index=False)
-
-# Processed the saved results
-
-acc = accuracy_score(reals, predictions)
-conf_mat = confusion_matrix(reals, predictions)
-print(classification_report(reals, predictions, labels=[l for l in label_index.values()]))
-print("Testing accuracy score is ", acc)
-print("Confusion Matrix", conf_mat)
-
-df_cm = pd.DataFrame(conf_mat, index=[i for i in list(set(reals))],
-                     columns=[i for i in list(set(reals))])
-print("made dataframe")
-plt.figure(figsize=(10, 7))
-print("made plot")
-# sn.heatmap(df_cm, annot=True)
-print("showing plot")
-plt.show()
-
-with open("labels.txt", "w") as f:
-    for label in label_index.values():
-        f.write(label + "\n")
-

5_test_models.py

@@ -0,0 +1,61 @@
+from glob import glob
+
+import matplotlib.pyplot as plt
+import pandas as pd
+from PIL import ImageFile
+from keras.models import load_model
+from keras.preprocessing.image import ImageDataGenerator
+
+from modeling_utils import get_metrics
+
+ImageFile.LOAD_TRUNCATED_IMAGES = True
+
+accuracies = []
+losses = []
+filenames = []
+
+input_shape = (224, 224, 3)
+batch_size = 32
+metrics_df = pd.read_csv("all_model_output.csv")
+
+test_gen = ImageDataGenerator().flow_from_directory(
+    './data/test',
+    target_size=(input_shape[0], input_shape[1]),
+    batch_size=batch_size,
+    shuffle=False
+)
+
+single_gen = ImageDataGenerator().flow_from_directory(
+    './single_image_test_set',
+    target_size=(input_shape[0], input_shape[1]),
+    batch_size=batch_size,
+    shuffle=False
+)
+
+for file in glob("./models/keras/*.hdf5"):
+    print(file)
+    if file in metrics_df.values:
+        continue
+    model = load_model(file)
+    test_acc, test_ll = get_metrics(test_gen, model)
+    single_acc, single_ll = get_metrics(single_gen, model, file[:-5] + ".csv")
+    metrics_df = metrics_df.append({
+        "model": file,
+        "test_acc": test_acc,
+        "test_loss": test_ll,
+        "single_acc": single_acc,
+        "single_loss": single_ll,
+    }, ignore_index=True)
+
+
+
+# Save the results
+
+metrics_df.to_csv("all_model_output.csv", index=False)
+print(metrics_df)
+metrics_df = metrics_df.sort_values('single_acc')
+metrics_df.plot.bar(y=["test_acc", "single_acc"], rot=90)
+metrics_df = metrics_df.sort_values('test_acc')
+metrics_df.plot.bar(y=["test_acc", "single_acc"], rot=90)
+plt.tight_layout()
+plt.show()

6 - KerasToTensorflow.py

@@ -1,11 +0,0 @@
-import tensorflow as tf
-from tensorflow import keras
-keras_file = "mobilenetv2.hdf5"
-keras.models.load_model(keras_file)
-
-h5_model = keras.models.load_model(keras_file)
-converter = tf.lite.TFLiteConverter.from_keras_model_file(keras_file)
-
-tflite_model = converter.convert()
-with open('mobilenetv2.tflite', 'wb') as f:
-    f.write(tflite_model)

6_convert_keras_to_tflite.py

@@ -0,0 +1,43 @@
+import os
+from glob import glob
+from pathlib import Path
+
+import pandas as pd
+import tensorflow as tf
+from keras.preprocessing.image import ImageDataGenerator
+import tensorflow as tf
+
+# TODO: Move these to a config for the project
+input_shape = (224, 224, 3)
+batch_size = 32
+
+single_gen = ImageDataGenerator().flow_from_directory(
+    './single_image_test_set',
+    target_size=(input_shape[0], input_shape[1]),
+    batch_size=batch_size,
+    shuffle=False
+)
+
+pd.DataFrame(sorted([f.name for f in os.scandir("./data/train") if f.is_dir()])).to_csv("./models/tflite/labels.txt",
+                                                                                        index=False, header=False)
+
+for file in glob("./models/keras/*.hdf5"):
+    path = Path(file)
+    tflite_file = f'./models/tflite/models/{path.name[:-5] + ".tflite"}'
+    if not Path(tflite_file).exists():
+        print(tflite_file)
+        keras_model = tf.keras.models.load_model(file)
+        keras_model.summary()
+        print(keras_model.input)
+        print(keras_model.layers)
+        converter = tf.lite.TFLiteConverter.from_keras_model(keras_model)
+        tflite_model = converter.convert()
+        with open(tflite_file, 'wb') as f:
+            f.write(tflite_model)
+    # TODO: Verify the model performance after converting to TFLITE
+    # interpreter = tf.lite.Interpreter(model_path=tflite_file)
+    # single_acc, single_ll = get_metrics(single_gen, keras_model)
+    # tf_single_acc, tf_single_ll = get_metrics(single_gen, tflite_model)
+    #
+    # print(single_acc, tf_single_acc)
+    # print(single_ll, tf_single_ll)

LabelsGenerator.py

@@ -1,4 +0,0 @@
-import pandas as pd
-import os
-
-pd.DataFrame(sorted([f.name for f in os.scandir("./data/train") if f.is_dir()])).to_csv("labels.txt", index=False, header=False)

README.md

@@ -0,0 +1 @@
+Test readme file

all_model_output.csv

@@ -0,0 +1,7 @@
+model,test_acc,test_loss,single_acc,single_loss
+./models/keras\pt-fl-fbn-efficientnet_v2b0-d1024-do0.5-l11.e-04-l21.e-04-5224-second_stage.hdf5,0.6720150708068079,1.7423864365349095,0.9893048128342246,0.4364729183409372
+./models/keras\pt-fl-fbn-efficientnet_v2b0-d1024-do0.5-l11.e-04-l21.e-04-5224.hdf5,0.410029881772119,3.346152696366266,0.986096256684492,0.3234976000776315
+./models/keras\pt-fl-fbn-efficientnet_v2s-d1024-do0.5-l11.e-04-l21.e-04-8105-second_stage.hdf5,0.6850721060153306,1.675868156533777,0.9967914438502674,0.3373779159304851
+./models/keras\pt-fl-fbn-efficientnet_v2s-d1024-do0.5-l11.e-04-l21.e-04-8105.hdf5,0.3755359230869169,3.5500588697038067,0.9540106951871656,0.4727042578503783
+./models/keras\pt-fl-fbn-efficientnet_v2s-d1024-do0.5-l11.e-04-l21.e-04-9317-second_stage.hdf5,0.6121780461172843,2.197206965588216,0.9946581196581196,0.2974041509252359
+./models/keras\pt-fl-fbn-efficientnet_v2s-d1024-do0.5-l11.e-04-l21.e-04-9317.hdf5,0.3702228787976106,3.601324427207316,0.9594017094017094,0.4877960320956891

model_builders/image_class_builder.py

@@ -1,76 +0,0 @@
-from enum import Enum
-from typing import Tuple
-
-import tensorflow as tf
-from tensorflow import keras
-
-from .modelwrapper import ModelWrapper
-
-
-class ImageClassModels(Enum):
-    INCEPTION_V3 = ModelWrapper(
-        keras.applications.InceptionV3,
-        keras.applications.inception_v3.preprocess_input
-    )
-    XCEPTION = ModelWrapper(
-        keras.applications.xception.Xception,
-        keras.applications.inception_v3.preprocess_input
-    )
-    MOBILENET_V2 = ModelWrapper(
-        keras.applications.mobilenet_v2.MobileNetV2,
-        keras.applications.mobilenet_v2.preprocess_input
-    )
-
-
-class ImageClassModelBuilder(object):
-
-    def __init__(self,
-                 input_shape: Tuple[int, int, int],
-                 n_classes: int,
-                 optimizer: tf.keras.optimizers.Optimizer = keras.optimizers.Adam(
-                     learning_rate=.0001),
-                 pre_trained: bool = True,
-                 fine_tune: int = 0,
-                 base_model: ImageClassModels = ImageClassModels.MOBILENET_V2):
-        self.input_shape = input_shape
-        self.n_classes = n_classes
-        self.optimizer = optimizer
-        self.pre_trained = pre_trained
-        self.fine_tune = fine_tune
-        self.base_model = base_model
-
-    def set_base_model(self, base_model: ImageClassModels):
-        self.base_model = base_model
-
-    def create_model(self):
-
-        base_model = self.base_model.value.model_func(
-            weights='imagenet' if self.pre_trained else None,
-            include_top=False
-        )
-        if self.pre_trained:
-            if self.fine_tune > 0:
-                for layer in base_model.layers[:-self.fine_tune]:
-                    layer.trainable = False
-            else:
-                for layer in base_model.layers:
-                    layer.trainable = False
-
-        i = tf.keras.layers.Input([self.input_shape[0], self.input_shape[1], self.input_shape[2]], dtype=tf.float32)
-        x = tf.cast(i, tf.float32)
-        x = self.base_model.value.model_preprocessor(x)
-        x = base_model(x)
-        x = keras.layers.GlobalAveragePooling2D()(x)
-        x = keras.layers.Dense(1024, activation='relu', kernel_regularizer=keras.regularizers.L1L2(l1=1e-5, l2=1e-5))(x)
-        x = keras.layers.Dropout(0.25)(x)
-        output = keras.layers.Dense(self.n_classes, activation='softmax')(x)
-
-        model = keras.Model(inputs=i, outputs=output)
-        model.compile(optimizer=self.optimizer,
-                      loss=keras.losses.CategoricalCrossentropy(),
-                      metrics=[
-                          'accuracy',
-                          # 'mse'
-                      ])
-        model.summary()
-        return model

modeling_utils/__init__.py

@@ -1 +1,2 @@
 from .image_class_builder import ImageClassModelBuilder, ImageClassModels
+from .model_testing import get_metrics

modeling_utils/image_class_builder.py

@@ -0,0 +1,125 @@
+import random
+from enum import Enum
+from typing import Tuple
+
+import numpy as np
+import tensorflow as tf
+from tensorflow import keras
+
+from .model_wrapper import ModelWrapper
+
+
+class ImageClassModels(Enum):
+    INCEPTION_V3 = ModelWrapper(
+        keras.applications.inception_v3.InceptionV3,
+        keras.applications.inception_v3.preprocess_input,
+        "inception_v3"
+    )
+    INCEPTION_RESNET_V2 = ModelWrapper(
+        keras.applications.inception_resnet_v2.InceptionResNetV2,
+        keras.applications.inception_resnet_v2.preprocess_input,
+        "inception_resnet_v2"
+    )
+    XCEPTION = ModelWrapper(
+        keras.applications.xception.Xception,
+        keras.applications.xception.preprocess_input,
+        "xception"
+    )
+    DENSENET201 = ModelWrapper(
+        keras.applications.densenet.DenseNet201,
+        keras.applications.densenet.preprocess_input,
+        "densenet201"
+    )
+    MOBILENET_V2 = ModelWrapper(
+        keras.applications.mobilenet_v2.MobileNetV2,
+        keras.applications.mobilenet_v2.preprocess_input,
+        "mobilenet_v2"
+    )
+    EFFICIENTNET_V2S = ModelWrapper(
+        keras.applications.efficientnet_v2.EfficientNetV2S,
+        tf.keras.applications.efficientnet_v2.preprocess_input,
+        "efficientnet_v2s"
+    )
+    EFFICIENTNET_V2B0 = ModelWrapper(
+        keras.applications.efficientnet_v2.EfficientNetV2B0,
+        tf.keras.applications.efficientnet_v2.preprocess_input,
+        "efficientnet_v2b0"
+    )
+
+
+class ImageClassModelBuilder(object):
+
+    def __init__(self,
+                 input_shape: Tuple[int, int, int],
+                 n_classes: int,
+                 optimizer: tf.keras.optimizers.Optimizer = keras.optimizers.Adam(
+                     learning_rate=.0001),
+                 pre_trained: bool = True,
+                 freeze_batch_norm: bool = False,
+                 freeze_layers: bool = False,
+                 base_model_type: ImageClassModels = ImageClassModels.MOBILENET_V2,
+                 dense_layer_neurons: int = 1024,
+                 dropout_rate: float = .5,
+                 l1: float = 1e-4,
+                 l2: float = 1e-4):
+        self.input_shape = input_shape
+        self.n_classes = n_classes
+        self.optimizer = optimizer
+        self.pre_trained = pre_trained
+        self.freeze_layers = freeze_layers
+        self.freeze_batch_norm = freeze_batch_norm
+        self.dense_layer_neurons = dense_layer_neurons
+        self.dropout_rate = dropout_rate
+        self.l1 = l1
+        self.l2 = l2
+        self.set_base_model(base_model_type)
+
+    def set_base_model(self, base_model_type: ImageClassModels):
+        self.base_model_type = base_model_type
+        self.base_model = self.base_model_type.value.model_func(
+            weights='imagenet' if self.pre_trained else None,
+            input_shape=self.input_shape,
+            include_top=False
+        )
+
+    def create_model(self):
+        if self.freeze_layers:
+            self.base_model.trainable = False
+        if self.freeze_batch_norm:
+            for layer in self.base_model.layers:
+                if isinstance(layer, keras.layers.BatchNormalization):
+                    layer.trainable = False
+        i = tf.keras.layers.Input([self.input_shape[0], self.input_shape[1], self.input_shape[2]], dtype=tf.float32)
+        x = tf.cast(i, tf.float32)
+        x = self.base_model_type.value.model_preprocessor(x)
+        x = self.base_model(x)
+        x = keras.layers.GlobalAveragePooling2D()(x)
+        x = keras.layers.Dense(self.dense_layer_neurons, activation='relu',
+                               kernel_regularizer=keras.regularizers.L1L2(l1=self.l1, l2=self.l2))(x)
+        x = keras.layers.Dropout(self.dropout_rate)(x)
+        output = keras.layers.Dense(self.n_classes, activation='softmax')(x)
+        self.model = keras.Model(inputs=i, outputs=output)
+        self.model.compile(
+            optimizer=self.optimizer,
+            loss=keras.losses.CategoricalCrossentropy(),
+            metrics=['accuracy', 'categorical_crossentropy']
+        )
+        self.model.summary()
+        return self.model
+
+    def get_fine_tuning(self):
+        print("self.model is found")
+        self.base_model.trainable = True
+        self.model.compile(
+            optimizer=self.optimizer,
+            loss=keras.losses.CategoricalCrossentropy(),
+            metrics=['accuracy', 'categorical_crossentropy']
+        )
+        self.model.summary()
+        return self.model
+
+    def get_name(self):
+        return f"{'pt-' if self.pre_trained else ''}{'fl-' if self.freeze_layers else ''}{'fbn-' if self.freeze_batch_norm else ''}" \
+               f"{self.base_model_type.value.name}-d{self.dense_layer_neurons}-do{self.dropout_rate}" \
+               f"{'-l1' + np.format_float_scientific(self.l1) if self.l1 > 0 else ''}{'-l2' + np.format_float_scientific(self.l2) if self.l2 > 0 else ''}" \
+               f"-{random.randint(1111, 9999)}"

modeling_utils/model_testing.py

@@ -0,0 +1,24 @@
+import numpy as np
+import pandas as pd
+from sklearn.metrics import accuracy_score, confusion_matrix, log_loss
+
+
+def get_metrics(gen, model, save_predictions_file=None):
+    model_output = model.predict(gen, verbose=True, workers=12)
+    prediction_indices = np.argmax(model_output, axis=1)
+    label_index = {v: k for k, v in gen.class_indices.items()}
+    predictions = [label_index[p] for p in prediction_indices]
+    reals = [label_index[p] for p in gen.classes]
+    acc = accuracy_score(reals, predictions)
+    ll = log_loss(gen.classes, model_output, labels=[l for l in label_index.keys()])
+    conf_mat = confusion_matrix(reals, predictions, labels=[l for l in label_index.values()])
+    # print(classification_report(reals, predictions, labels=[l for l in label_index.values()]))
+    print("Testing accuracy score is ", acc)
+    print("Confusion Matrix", conf_mat)
+    if save_predictions_file:
+        df = pd.DataFrame(columns=['fname', 'prediction', 'true_val'])
+        df['fname'] = [x for x in gen.filenames]
+        df['prediction'] = predictions
+        df["true_val"] = reals
+        df.to_csv(save_predictions_file, index=False)
+    return acc, ll

modeling_utils/model_wrapper.py

@@ -1,7 +1,7 @@
-from collections import Callable
-
+from typing import Callable
 
 class ModelWrapper(object):
-    def __init__(self, model_func:Callable, model_preprocessor:Callable):
+    def __init__(self, model_func:Callable, model_preprocessor:Callable, name:str):
         self.model_func = model_func
         self.model_preprocessor = model_preprocessor
+        self.name = name

print_all_wrong.py

@@ -0,0 +1,5 @@
+import pandas as pd
+
+df = pd.read_csv("models/keras/pt-fl-fbn-efficientnet_v2s-d1024-do0.5-l11.e-04-l21.e-04-8105-second_stage.csv")
+
+print(df.loc[df["prediction"] != df["true_val"]])

run_tensorboard_multi_dir.cmd

@@ -0,0 +1 @@
+tensorboard --logdir_spec=local:./logs,remote:Z:/MachineLearning/Tensorboard/Tensordex/Logs --bind_all

show_incorrect_images.py

@@ -19,7 +19,6 @@ for index, row in df2.iterrows():
 incorrect = df[df["prediction"]!= df["true_val"]]
 
 total_same_fam = 0
-# TODO: Add in support for figuring out if the pokemon are related/evolutions of one another
 for index, row in incorrect.iterrows():
     img = mpimg.imread("./SingleImageTestSet/" + row['fname'])
     imgplot = plt.imshow(img)