Object Detection
This notebook trains a PyTorch object detection algorithm (Yolov5) using the limbo-ml dataset to determine the presence or absence of 30B and 48-type containers.
This notebook uses Campaigns 2, 3, and 6 to train the network and the Reference data to test the network. A description of the data is provided in the Campaigns tab.
Reference the
Download DataCell in this notebook for accessing the Limbo Data.Note - This notebook offers an alternative to analyzing the data without using the limbo software
[1]:
%load_ext autoreload
%autoreload 2
%pylab inline
import torch
import pandas as pd
from sklearn.model_selection import train_test_split
from PIL import Image, ImageDraw, ImageFont
import json
import os
import cv2
from pathlib import Path
from tqdm import tqdm
from IPython.display import display
from sklearn.utils import shuffle
import yaml
print("PyTorch Version: ",torch.__version__)
# Imports for helper functions
import subprocess
import shutil
import xml.etree.cElementTree as ET
Populating the interactive namespace from numpy and matplotlib
PyTorch Version: 1.11.0+cu102
Confirm Pytorch was installed correctly and is using a GPU
We will train using nVidia GPU hardware; if you don’t have any, you can use a cpu, but your training times will be very long:
[2]:
assert torch.cuda.is_available() == True, "PyTorch was not installed correctly"
Helper functions
These could be written to a script and read-in from there but for simplicity let’s define them all here
[3]:
def change_img_extension(img_name):
'''
change_img_extension converts a .exr file to png
returns new name of img as .png
'''
if img_name.endswith('.exr'):
img_name = img_name.split('.')[0] + '.png'
return img_name
def make_directory(path, dataset_type):
dir_path = Path(f"{path}{dataset_type}")
dir_path.mkdir(parents=True, exist_ok=True)
return dir_path
def convert_to_yolov5(json_files,
categories,
dataset_type,
json_directory,
image_directory,
labels_directory,
category_dict,
reference_imgs=False):
'''
convert_to_yolov5 ingests the json_files and writes images and txt
files to separate directories
json_files is a list of json files
categories - list of unique categories
dataset_type (str) normally either train or val
json_directory - path specifying where the json files live. useful
for assigning images correctly
image_directory - directory to build from current directory. i.e., the
pwd is notebooks if you enter image_directory=/image/labels the new
directory will be notebooks/image/labels
labels_directory - similar to image_directory
category_dict (dict) mapping for categories to classes
'''
images_path = make_directory(image_directory, dataset_type)
labels_path = make_directory(labels_directory, dataset_type)
img_id = 0
for j, (row, j_dir) in enumerate(zip(tqdm(json_files), json_directory)):
print_buffer = []
obj_of_interest = False
valid_img = False
img = None
for idx, a in enumerate(row['annotations']):
if idx == 0:
label_name = f"{img_id}.txt"
save_file_name = labels_path / label_name
if ('background' in categories) and (a['category'] == 'background'):
valid_img = True
# Write the bbox details to the file
print_buffer.append(" ")
img = row['synthetic']['image']['filename']
obj_of_interest = True
# print('Processing background number %i' % img_id)
if 'bbox' not in a.keys():
continue
if a['category'] == 'synthetic':
continue
if reference_imgs is True:
img = row['image']['filename']
else:
img = row['synthetic']['image']['filename']
res = row['image']['res']
# Get the image size
_img = change_img_extension(img)
cv_img = cv2.cvtColor(cv2.imread(j_dir + _img),
cv2.COLOR_RGB2BGR)
image_w, image_h, image_c = cv_img.shape
pil_image = Image.open(j_dir + row['image']['filename']).size
image_w = pil_image[0]
image_h = pil_image[1]
assert (res[0] == image_w and res[1] == image_h), "Resolution should equal size"
# only look for the items of interest
if isinstance(category_dict, dict):
if a['category'][0:2] not in category_dict.keys():
continue
category_idx = category_dict[a['category'][0:2]]
elif isinstance(category_dict, list):
if a['category'][0:2] == '30' or a['category'][0:2] == '48':
category_idx = categories.index(a['category'][0:2])
obj_of_interest = True
#else:
elif a['category'] == 'paintbrush':
print('skipping paintbrush')
continue
elif a['category'] == 'barrel/55G/A':
print('renaming barrels')
category_idx = categories.index('barrel/55G')
elif a['category'] == 'barrel/55G/B':
print('renaming barrels')
category_idx = categories.index('barrel/55G')
elif a['category'] == 'barrel/55G/C':
print('renaming barrels')
category_idx = categories.index('barrel/55G')
elif a['category'] == 'barrel/55G/D':
print('renaming barrels')
category_idx = categories.index('barrel/55G')
elif a['category'] == 'barrel/55G/E':
print('renaming barrels')
category_idx = categories.index('barrel/55G')
elif a['category'] == 'barrel/55G/F':
print('renaming barrels')
category_idx = categories.index('barrel/55G')
else:
category_idx = categories.index(a['category'])
# if obj_of_interest is True and (a['category'][0:2] == '30' or a['category'][0:2] == '48'):
# print(a['category'])
bbox = a['bbox']
# bboxes are XYWH
x, y = bbox[0], bbox[1]
b_width, b_height = bbox[2], bbox[3]
# make a cut on pixel size
if b_width * b_height < 5:
continue
b_center_x = x + b_width/2
b_center_y = y + b_height/2
# reference images vary so do not remove edge cases
if reference_imgs is True:
pass
# remove edge cases
elif (b_center_x < 25) or (b_center_y < 25) or\
(b_center_x > 700) or (b_center_y > 700):
continue
valid_img = True
# Transform the bbox coordinates as per the format required by
# YOLO v5 normalize the bounding boxes
b_center_x /= image_w
b_center_y /= image_h
b_width /= image_w
b_height /= image_h
label_name = f"{img_id}.txt"
# Write the bbox details to the file
print_buffer.append("{} {:.7f} {:.7f} {:.7f} {:.7f}".format(category_idx,
b_center_x,
b_center_y,
b_width, b_height))
if valid_img is True:
image_name = f"{img_id}.png"
img = change_img_extension(img)
img = j_dir + img
img = Image.open(img)
img = img.convert("RGB")
img.save(str(images_path / image_name), "PNG")
img = None
img_id += 1
# Save the annotation to disk
print("\n".join(print_buffer), file=open(save_file_name, "w"))
# reset image parameter
valid_img = False
obj_of_interest = False
def plot_bounding_box(image_path,
category_dict,
annotation_list,
unnormalize=True,
save_fig=None):
image = Image.open(image_path)
if isinstance(category_dict, dict):
class_id_to_name_mapping = dict(zip(category_dict.values(),
category_dict.keys()))
annotations = np.array(annotation_list)
w = image.width
h = image.height
plotted_image = ImageDraw.Draw(image)
transformed_annotations = np.copy(annotations)
# if annotations is empty just plot the image because there are no
# bounding boxes present
try:
if not annotations.any():
plt.figure(figsize=(9, 6))
plt.imshow(np.array(image))
plt.show()
return
except TypeError:
pass
if unnormalize is True:
transformed_annotations[:, [1, 3]] = annotations[:, [1, 3]] * w
transformed_annotations[:, [2, 4]] = annotations[:, [2, 4]] * h
else:
transformed_annotations[:, [1, 3]] = annotations[:, [1, 3]].astype(float)
transformed_annotations[:, [2, 4]] = annotations[:, [2, 4]].astype(float)
try:
transformed_annotations[:, 1] = float(transformed_annotations[:, 1]) -\
(float(transformed_annotations[:, 3]) / 2)
transformed_annotations[:, 2] = float(transformed_annotations[:, 2]) -\
(float(transformed_annotations[:, 4]) / 2)
transformed_annotations[:, 3] = float(transformed_annotations[:, 1]) +\
float(transformed_annotations[:, 3])
transformed_annotations[:, 4] = float(transformed_annotations[:, 2]) +\
float(transformed_annotations[:, 4])
except TypeError:
transformed_annotations[:, 1] = transformed_annotations[:, 1].astype(float) -\
transformed_annotations[:, 3].astype(float) / 2
transformed_annotations[:, 2] = transformed_annotations[:, 2].astype(float) -\
(transformed_annotations[:, 4].astype(float) / 2)
transformed_annotations[:, 3] = transformed_annotations[:, 1].astype(float) +\
transformed_annotations[:, 3].astype(float)
transformed_annotations[:, 4] = transformed_annotations[:, 2].astype(float) +\
transformed_annotations[:, 4].astype(float)
font = ImageFont.truetype('/usr/share/fonts/truetype/freefont/FreeSans.ttf', 35)
for ann in transformed_annotations:
obj_cls, x0, y0, x1, y1 = ann
plotted_image.rectangle(((float(x0), float(y0)),
(float(x1), float(y1))),
outline="red", width=4)
try:
if isinstance(category_dict, dict):
plotted_image.text((float(x0), float(y0)-35),
class_id_to_name_mapping[(int(obj_cls))],
font=font,
fill="black")
elif isinstance(category_dict, list):
plotted_image.text((float(x0), float(y0)-35),
category_dict[(int(obj_cls))],
font=font,
fill="black")
except ValueError:
plotted_image.text((float(x0), float(y0)-35),
class_id_to_name_mapping[(obj_cls)],
font=font,
fill="black")
plt.figure(figsize=(9, 6))
plt.imshow(np.array(image))
plt.tight_layout()
if save_fig:
plt.savefig(save_fig, dpi=300)
plt.show()
def plot_bounding_box_from_xml(image_path,
category_dict,
annotation_list):
image = Image.open(image_path)
class_id_to_name_mapping = dict(zip(category_dict.values(),
category_dict.keys()))
annotations = np.array(annotation_list)
w = image.width
h = image.height
# image_w, image_h, image_c = cv_img.shape
# w = image_w
# h = image_h
plotted_image = ImageDraw.Draw(image)
transformed_annotations = np.copy(annotations)
transformed_annotations[:, [1, 3]] = annotations[:, [1, 3]] * w
transformed_annotations[:, [2, 4]] = annotations[:, [2, 4]] * h
transformed_annotations[:, 1] = transformed_annotations[:, 1] -\
(transformed_annotations[:, 3] / 2)
transformed_annotations[:, 2] = transformed_annotations[:, 2] -\
(transformed_annotations[:, 4] / 2)
transformed_annotations[:, 3] = transformed_annotations[:, 1] +\
transformed_annotations[:, 3]
transformed_annotations[:, 4] = transformed_annotations[:, 2] +\
transformed_annotations[:, 4]
font = ImageFont.truetype('/Library/Fonts/Arial.ttf', 35)
for ann in transformed_annotations:
obj_cls, x0, y0, x1, y1 = ann
plotted_image.rectangle(((x0, y0), (x1, y1)), outline="red", width=4)
plotted_image.text((x0, y0-35),
class_id_to_name_mapping[(int(obj_cls))],
font=font,
fill="black")
plt.figure(figsize=(9, 6))
plt.imshow(np.array(image))
plt.show()
def correct_images(image_directory, dataset_type):
'''
correct_images fixes the
libpng warning: iCCP: known incorrect sRGB profile
rewrites the data into a corrected folder
'''
# create a directory one above images called corrected vals
corrected_val = "/corrected"
dir_path = Path(f"{image_directory}{corrected_val}")
dir_path.mkdir(parents=True, exist_ok=True)
directory_corrected = dir_path.absolute()
img_path = Path(f"{image_directory}{dataset_type}")
img_directory = img_path.absolute()
for img in sorted(os.listdir(img_directory)):
file_in = str(img_directory)+'/'+img
file_out = str(directory_corrected)+'/'+img
subprocess.run(["pngfix", "--strip=color",
"--out={}".format(file_out),
"{}".format(file_in)])
# remove original
shutil.rmtree(img_directory)
# rename folder to original dataset type
shutil.move(directory_corrected, str(directory_corrected.parent) + '/' + dataset_type)
def convert_to_xml_format(json_files,
categories,
dataset_type,
json_directory,
image_directory,
labels_directory,
category_dict,
reference_imgs=False):
'''
convert_to_xml_format ingests the json_files and writes images and xml
files to separate directories. These formats are used for pre-trained
pytorch models and we can fine-tune to our data as needed
json_files is a list of json files
categories - list of unique categories
dataset_type (str) normally either train or val
json_directory - path specifying where the json files live. useful
for assigning images correctly
image_directory - directory to build from current directory. i.e., the
pwd is notebooks if you enter image_directory=/image/labels the new
directory will be notebooks/image/labels
labels_directory - similar to image_directory
category_dict (dict) mapping for categories to classes
'''
images_path = make_directory(image_directory, dataset_type)
labels_path = make_directory(labels_directory, dataset_type)
img_id = 0
valid_img = False
for j, (row, j_dir) in enumerate(zip(tqdm(json_files), json_directory)):
print_buffer = []
for idx, a in enumerate(row['annotations']):
if idx == 0:
label_name = f"{img_id}.xml"
save_file_name = labels_path / label_name
if 'bbox' not in a.keys():
continue
if a['category'] == 'synthetic':
continue
if reference_imgs is True:
img = row['image']['filename']
else:
img = row['synthetic']['image']['filename']
res = row['image']['res']
# Get the image size
cv_img = cv2.cvtColor(cv2.imread(j_dir + img),
cv2.COLOR_RGB2BGR)
image_w, image_h, image_c = cv_img.shape
pil_image = Image.open(j_dir + row['image']['filename']).size
image_w = pil_image[0]
image_h = pil_image[1]
assert (res[0] == image_w and res[1] == image_h), "Resolution should equal size"
# only look for the items of interest
if isinstance(category_dict, dict):
if a['category'][0:2] == '30' or a['category'][0:2] == '48':
category_idx = category_dict[a['category'][0:2]]
elif a['category'] == 'paintbrush':
print('skipping paintbrush')
continue
else:
category_idx = category_dict[a['category']]
elif isinstance(category_dict, list):
if a['category'][0:2] == '30' or a['category'][0:2] == '48':
category_idx = categories.index(a['category'][0:2])
elif a['category'] == 'paintbrush':
print('skipping paintbrush')
continue
else:
category_idx = categories.index(a['category'])
bbox = a['bbox']
# bboxes are XYWH
x, y = bbox[0], bbox[1]
b_width, b_height = bbox[2], bbox[3]
# make a cut on pixel size
if b_width * b_height < 5:
continue
b_center_x = x + b_width/2
b_center_y = y + b_height/2
# reference images vary so do not remove edge cases
if reference_imgs is True:
pass
# remove edge cases
elif (b_center_x < 25) or (b_center_y < 25) or\
(b_center_x > 700) or (b_center_y > 700):
continue
valid_img = True
# Get xmin, xmax, etc...
xmin = b_center_x - b_width/2
xmax = b_center_x + b_width/2
ymin = b_center_y - b_height/2
ymax = b_center_y + b_height/2
# Write the bbox details to the file
print_buffer.append([category_idx, xmin, xmax, ymin, ymax])
if valid_img is True:
image_name = f"{img_id}.png"
img = change_img_extension(img)
img = j_dir + img
img = Image.open(img)
img = img.convert("RGB")
img.save(str(images_path / image_name), "PNG")
to_xml(image_w, image_h,
image_c,
print_buffer,
image_name,
save_file_name)
valid_img = False
img_id += 1
def to_xml(width, height,
depth, obj_info,
img_path,
save_file_name):
root = ET.Element("annotation")
ET.SubElement(root, "filename").text = "{}".format(img_path)
ET.SubElement(root, "path").text = "{}".format(img_path)
size = ET.SubElement(root, "size")
ET.SubElement(size, "width").text = "{}".format(width)
ET.SubElement(size, "height").text = "{}".format(height)
ET.SubElement(size, "depth").text = "{}".format(depth)
# this part needs to repeat
for obj_class, xmin, xmax, ymin, ymax in obj_info:
obj = ET.SubElement(root, "object")
ET.SubElement(obj, "name").text = "{}".format(obj_class)
bndbox = ET.SubElement(obj, "bndbox")
ET.SubElement(bndbox, "xmin").text = "{}".format(xmin)
ET.SubElement(bndbox, "xmax").text = "{}".format(xmax)
ET.SubElement(bndbox, "ymin").text = "{}".format(ymin)
ET.SubElement(bndbox, "ymax").text = "{}".format(ymax)
tree = ET.ElementTree(root)
tree.write(save_file_name)
def read_xml(xml_file):
tree = ET.parse(xml_file)
root = tree.getroot()
xml_list = []
# Each object represents each actual image label
for member in root.findall('object'):
box = member.find('bndbox')
label = member.find('name').text
# Add image file name, image size, label, and box coordinates to CSV file
b_width = float(box.find('xmax').text) - float(box.find('xmin').text)
b_height = float(box.find('ymax').text) - float(box.find('ymin').text)
b_center_x = (float(box.find('xmax').text) + float(box.find('xmin').text)) / 2
b_center_y = (float(box.find('ymax').text) + float(box.find('ymin').text)) / 2
row = [label, b_center_x, b_center_y, b_width, b_height]
xml_list.append(row)
return xml_list
Download the data
Refer to limbo-ml downloads for instructions on downloading both the synthetically generated data and reference data.
Once data is downloaded, point to working directories
Every campaign in the Limbo data is split into numbered subdirectories, each of which contains one thousand images. We load three thousand synthetic images for training from campaign 2 and 3. Additionally, we will load in five thousand background images from Campaign 6.
Finally, we put our data to one thousand synthetic images for (synthetic) testing, and all of the available real-world data for (real) testing:
[4]:
train_directories = np.array(['../data/campaign2/0000/',
'../data/campaign2/0001/',
'../data/campaign2/0002/',
'../data/campaign3/0000/',
'../data/campaign3/0001/',
'../data/campaign3/0002/',
'../data/campaign6/0000/',
'../data/campaign6/0001/',
'../data/campaign6/0002/',
'../data/campaign6/0003/',
'../data/campaign6/0004/'])
test_dir = ['../data/ref/0000/']
Analyze training directories first
We start by reading in the training directories. We search for files ending in .json. We read the files and store them in a list keeping track of the location of the training directory.
[5]:
# store json files and training directories to a list
# the approach below is more relevant when there are multiple working
# directories
json_files = []
train_dir_list = []
for train_dir in train_directories:
print(train_dir)
for file in sorted(os.listdir(str(train_dir))):
if file.endswith('.json'):
with open(train_dir+file) as fn:
s = fn.read()
json_files.append(json.loads(s))
train_dir_list.append(train_dir)
../data/campaign2/0000/
../data/campaign2/0001/
../data/campaign2/0002/
../data/campaign3/0000/
../data/campaign3/0001/
../data/campaign3/0002/
../data/campaign6/0000/
../data/campaign6/0001/
../data/campaign6/0002/
../data/campaign6/0003/
../data/campaign6/0004/
view one json file
[6]:
json_files[0]
[6]:
{'annotations': [{'category': 'synthetic'}, {'category': 'background'}],
'image': {'content-type': 'image/png',
'filename': 'background_0000000.png',
'res': [720, 720]},
'synthetic': {'image': {'content-type': 'image/x-exr',
'filename': 'background_0000000.exr',
'res': [[720, 720]]},
'parameters': {'/background/env_map': 'driving_school_8k',
'/camera/focal': 66.58746224652447,
'/camera/orientation': [1.305654357222066,
-149.80491960728259,
1.6781659541312035],
'/camera/position': [-3.428114175796509,
2.1293375492095947,
-9.065103530883789]}}}
Shuffle the data
We shuffle the data so the object detection algorithm does not see the same order of images. Technically, Yolov5 should shuffle the data, but from my experience, shuffling the data before training gave more stable results over multiple training runs.
[7]:
# Shuffle the data
json_files_shuffled, train_dir_list_shuffled = shuffle(json_files, train_dir_list, random_state=7542)
Specify number of total images to train the network with. Additionally, we will train the network using the same number of positive and negative images. A positive image contains at least one instance of an item of interest (i.e., 30B or 48-type container). A negative image contains no instances of an item of interest.
In this example, we are specifying 10000 total images for training. This number can be varied based on the capabilities of the GPU available. The more images used for training the better:
[8]:
instances_of_images = 10000
# We want negative examples to be half background
instances_bkg = instances_of_images / 2
# since we are grabbing a subset of campaigns 2 and 3 there are only 300 background
# images per campaign. Fill the rest with campaign 6
instances_campaign_6 = np.ceil(instances_bkg / 3) * 3 - 601
# campaigns of 30Bs is 2. Split data amongst these
instances_30B = instances_of_images / (4)
# campaigns of 48s is 2. Split data amongst these
instances_48 = instances_of_images / (4)
Find all the unique categories
To train our object detector we need to find the relevant labels for training for each image: “48” or “30”.
Note - the next few cells could probably be collapsed into one cell, but we want to be explicit.
[9]:
categories = []
json_files_cats = []
train_dir_file_list = []
cnt_bkg = 0
cnt_30 = 0
cnt_48 = 0
cnt_dist = 0
# -------- campaign 30Bs ------
cnt_campaign_2_30 = 0
# -------- campaign 48s ------
cnt_campaign_3_48 = 0
# -------- Distractors --------
# --------- Background --------
cnt_campaign_6_bkg = 0
cnt_campaign_2_bkg = 0
cnt_campaign_3_bkg = 0
for c, t in zip(json_files_shuffled, train_dir_list_shuffled):
file_name = t.split('/')[2]
for idx, a in enumerate(c['annotations']):
# removing background categories
# if 'background' in a['category']:
# continue
if 'synthetic' in a['category']:
continue
else:
# ---- Background first
# This flag prevents multiple copies of the same image from being used
if not idx == 1:
continue
elif (a['category'] == 'background') & (file_name.startswith('campaign2')) &\
(cnt_campaign_2_bkg < np.ceil(instances_bkg / 3)):
cnt_campaign_2_bkg += 1
elif (a['category'] == 'background') & (file_name.startswith('campaign2')) &\
(cnt_campaign_2_bkg >= np.ceil(instances_bkg / 3)):
continue
elif (a['category'] == 'background') & (file_name.startswith('campaign3')) &\
(cnt_campaign_3_bkg < np.floor(instances_bkg / 3)):
cnt_campaign_3_bkg += 1
elif (a['category'] == 'background') & (file_name.startswith('campaign3')) &\
(cnt_campaign_3_bkg >= np.floor(instances_bkg / 3)):
continue
elif (a['category'] == 'background') & (file_name.startswith('campaign6')) &\
(cnt_campaign_6_bkg < np.floor(instances_campaign_6)):
cnt_campaign_6_bkg += 1
elif (a['category'] == 'background') & (file_name.startswith('campaign6')) &\
(cnt_campaign_6_bkg >= np.floor(instances_campaign_6)):
continue
# ------ 30Bs
elif a['category'].startswith('30'):
if (cnt_campaign_2_30 < np.floor(instances_30B)) & (file_name.startswith('campaign2')):
cnt_campaign_2_30 += 1
elif (cnt_campaign_2_30 >= np.floor(instances_30B)) & (file_name.startswith('campaign2')):
continue
# ------- 48s
elif a['category'].startswith('48'):
if (cnt_campaign_3_48 < np.ceil(instances_48)) & (file_name.startswith('campaign3')):
cnt_campaign_3_48 += 1
elif (cnt_campaign_3_48 >= np.ceil(instances_48)) & (file_name.startswith('campaign3')):
continue
json_files_cats.append(c)
train_dir_file_list.append(t)
categories.append(a['category'])
categories.append('30')
categories.append('48')
categories = list(set(categories))
categories.sort()
Find all instances of categories
Here we are searching to ensure that we are finding equal number of positive examples (“30” or “48” containers in an image) and negative examples (“background only”, i.e., does not contain the container of interest).
[10]:
category_instances = {}
cnt = 0
cnt_bkg = 0
cnt_30 = 0
cnt_48 = 0
# -------- campaign 30Bs ------
cnt_campaign_2_30 = 0
# -------- campaign 48s ------
cnt_campaign_3_48 = 0
# -------- Distractors --------
# --------- Background --------
cnt_campaign_6_bkg = 0
cnt_campaign_2_bkg = 0
cnt_campaign_3_bkg = 0
for _i, (c, t) in enumerate(zip(json_files_shuffled, train_dir_list_shuffled)):
file_name = t.split('/')[2]
for idx, a in enumerate(c['annotations']):
# removing background categories
# if 'background' in a['category']:
# continue
if 'synthetic' in a['category']:
continue
elif 'background' in a['category']:
category_name = a['category']
# ---- Background first
if not idx == 1:
continue
elif (a['category'] == 'background') & (file_name.startswith('campaign2')) &\
(cnt_campaign_2_bkg < np.ceil(instances_bkg / 3)):
cnt_campaign_2_bkg += 1
elif (a['category'] == 'background') & (file_name.startswith('campaign2')) &\
(cnt_campaign_2_bkg >= np.ceil(instances_bkg / 3)):
continue
elif (a['category'] == 'background') & (file_name.startswith('campaign3')) &\
(cnt_campaign_3_bkg < np.floor(instances_bkg / 3)):
cnt_campaign_3_bkg += 1
elif (a['category'] == 'background') & (file_name.startswith('campaign3')) &\
(cnt_campaign_3_bkg >= np.floor(instances_bkg / 3)):
continue
elif (a['category'] == 'background') & (file_name.startswith('campaign6')) &\
(cnt_campaign_6_bkg < np.floor(instances_campaign_6)):
cnt_campaign_6_bkg += 1
elif (a['category'] == 'background') & (file_name.startswith('campaign6')) &\
(cnt_campaign_6_bkg >= np.floor(instances_campaign_6)):
continue
if category_name in category_instances:
category_instances[category_name].append(category_name)
else:
category_instances[category_name] = []
category_instances[category_name].append(category_name)
# doing a bbox check because there are multiple instances of objects in
# the scene. still only single 30s or 48 in a scene
elif 'bbox' in a:
category_name = a['category']
if category_name.startswith('48'):
category_name = '48'
# ------ 30Bs
if a['category'].startswith('30'):
if (cnt_campaign_2_30 < np.floor(instances_30B)) & (file_name.startswith('campaign2')):
cnt_campaign_2_30 += 1
elif (cnt_campaign_2_30 >= np.floor(instances_30B)) & (file_name.startswith('campaign2')):
continue
# ------- 48s
elif a['category'].startswith('48'):
if (cnt_campaign_3_48 < np.ceil(instances_48)) & (file_name.startswith('campaign3')):
cnt_campaign_3_48 += 1
elif (cnt_campaign_3_48 >= np.ceil(instances_48)) & (file_name.startswith('campaign3')):
continue
if category_name in category_instances:
category_instances[category_name].append(category_name)
else:
category_instances[category_name] = []
category_instances[category_name].append(category_name)
cnt += 1
[11]:
total_images = 0
for key in category_instances.keys():
print('This many instances of %s are present: %i' % (key, len(category_instances[key])))
total_images += len(category_instances[key])
print("Total images", total_images)
This many instances of 30B are present: 2500
This many instances of background are present: 5000
This many instances of 48 are present: 2500
Total images 10000
Looks good! The image breakdown matches our expectations
Clean up categories
The samples in a Limbo dataset contain “30B” containers and images of type 48G, 48X, and 48Y cylinders. We will consolidate 48-types to one type of container: “48” and “30B” to “30”.
[12]:
categories_new = []
for c in categories:
if c == 'paintbrush':
continue
elif c == '30B':
continue
elif c == '48G':
continue
elif c == '48X':
continue
elif c == '48Y':
continue
categories_new.append(c)
categories_new.sort()
Organize files and directory list
Now that we have convinced ourselves that we are analyzing equal number of positive and negative images, collect the shuffled json files and directories into a list
[13]:
json_files_cats = []
train_dir_file_list = []
cnt_bkg = 0
cnt_30 = 0
cnt_48 = 0
cnt_dist = 0
# -------- campaign 30Bs ------
cnt_campaign_2_30 = 0
# -------- campaign 48s ------
cnt_campaign_3_48 = 0
# -------- Distractors --------
# --------- Background --------
cnt_campaign_6_bkg = 0
cnt_campaign_2_bkg = 0
cnt_campaign_3_bkg = 0
for c, t in zip(json_files_shuffled, train_dir_list_shuffled):
file_name = t.split('/')[2]
for idx, a in enumerate(c['annotations']):
if 'synthetic' in a['category']:
continue
else:
if 'background' in a['category']:
category_name = a['category']
# ---- Background first
if (a['category'] == 'background') & (file_name.startswith('campaign2')) &\
(cnt_campaign_2_bkg < np.ceil(instances_bkg / 3)):
json_files_cats.append(c)
train_dir_file_list.append(t)
cnt_campaign_2_bkg += 1
elif (a['category'] == 'background') & (file_name.startswith('campaign2')) &\
(cnt_campaign_2_bkg >= np.ceil(instances_bkg / 3)):
continue
elif (a['category'] == 'background') & (file_name.startswith('campaign3')) &\
(cnt_campaign_3_bkg < np.floor(instances_bkg / 3)):
json_files_cats.append(c)
train_dir_file_list.append(t)
cnt_campaign_3_bkg += 1
elif (a['category'] == 'background') & (file_name.startswith('campaign3')) &\
(cnt_campaign_3_bkg >= np.floor(instances_bkg / 3)):
continue
elif (a['category'] == 'background') & (file_name.startswith('campaign6')) &\
(cnt_campaign_6_bkg < np.floor(instances_campaign_6)):
json_files_cats.append(c)
train_dir_file_list.append(t)
cnt_campaign_6_bkg += 1
elif (a['category'] == 'background') & (file_name.startswith('campaign6')) &\
(cnt_campaign_6_bkg >= np.floor(instances_campaign_6)):
continue
# ---- Background first
elif not idx == 1:
continue
# ------ 30Bs
elif a['category'].startswith('30'):
if (cnt_campaign_2_30 < np.floor(instances_30B)) & (file_name.startswith('campaign2')):
json_files_cats.append(c)
train_dir_file_list.append(t)
cnt_campaign_2_30 += 1
elif (cnt_campaign_2_30 >= np.floor(instances_30B)) & (file_name.startswith('campaign2')):
continue
# ------- 48s
elif a['category'].startswith('48'):
if (cnt_campaign_3_48 < np.ceil(instances_48)) & (file_name.startswith('campaign3')):
json_files_cats.append(c)
train_dir_file_list.append(t)
cnt_campaign_3_48 += 1
elif (cnt_campaign_3_48 >= np.ceil(instances_48)) & (file_name.startswith('campaign3')):
continue
The samples in a Limbo dataset could contain dozens or even hundreds of annotations. For the Campaigns we are considering in this example we have no distractors present and we are only interested in “30” or “48” containers. We will assign “0” for type 30B containers and “1” for type “48” containers.
[14]:
# assign 30s to same category and 48s to another category
category_dict = {'30': 0, '48': 1}
One final check on the number of files that will be analyzed.
[15]:
# How many files are analyzed
len(json_files_cats)
[15]:
10000
Look at one training image
[16]:
file_img = json_files_cats[133]['synthetic']['image']['filename']
# convert from exr to png
file_img = change_img_extension(file_img)
[17]:
plt.figure(figsize=(9, 6))
plt.imshow(np.array(Image.open(train_dir_file_list[133]+file_img)))
plt.axis('off')
[17]:
(-0.5, 719.5, 719.5, -0.5)
Looks good! We will look at the annotations later in this notebook.
Analyze testing set
Now we will follow a similar procedure for analyzing the testing set.
[18]:
# store json files
json_test_files = []
test_dir_list = []
for td in test_dir:
for file in sorted(os.listdir(td)):
if file.endswith('.json'):
with open(td+file) as fn:
s = fn.read()
json_test_files.append(json.loads(s))
test_dir_list.append(td)
Find all the unique categories:
[19]:
# skip over first category because labeled synthetic
test_categories = []
json_test_files_cats = []
test_dir_file_list = []
class_name_to_id_mapping = {}
for c, t in zip(json_test_files, test_dir_list):
for idx, a in enumerate(c['annotations']):
# if a['category'] != '30B':
# continue
json_test_files_cats.append(c)
test_dir_file_list.append(t)
test_categories.append(a['category'])
break
test_categories = list(set(test_categories))
test_categories.sort()
print(test_categories)
# Dictionary that maps class names to IDs
for idx, cat in enumerate(test_categories):
class_name_to_id_mapping[cat] = idx
['30B', '30Boverpack', '48G', '48X', '48Y', 'real']
Look at one reference image
[20]:
file_img = json_test_files_cats[0]['image']['filename']
plt.figure(figsize=(9, 6))
plt.imshow(np.array(Image.open(test_dir[0]+file_img)))
plt.axis('off')
[20]:
(-0.5, 799.5, 637.5, -0.5)
Create train/test dataset and convert to yolov5 format
Here we take our annotations and make them adhere to the yolo format. We use a function convert_to_yolov5. For tensorflow applications, there is a helper function that converts to xml.
We will aslo split our training set into a train and validation set. We will do 80% train, 10% validation, and 10% test. This will be used for training synthetic and testing synthetic.
For training with synthetic but testing with real data, we will not split our data this way. We will use the train dataset but the validation dataset will be the reference dataset. We are treating validation as our testing set in this case based on the way yolov5 is setup.
[21]:
train_dataset, val_dataset,\
train_dir_file_split, val_dir_file_split = train_test_split(json_files_cats,
train_dir_file_list,
test_size=0.2,
random_state=1)
val_dataset, syn_test_dataset, val_dir_file_split, syn_test_file_split = train_test_split(val_dataset,
val_dir_file_split,
test_size = 0.5,
random_state = 1)
Train synthetic dataset
This will be used for train synthetic, test synthetic
[22]:
# create new directory paths
# these directory paths build from the pwd
image_directory = 'example_train/images/'
label_directory = 'example_train/labels/'
dataset_train = 'train'
This creates the training set:
[23]:
convert_to_yolov5(train_dataset,
categories_new,
dataset_train,
train_dir_file_split,
image_directory,
label_directory,
categories_new)
100%|██████████| 8000/8000 [23:51<00:00, 5.59it/s]
Validation dataset
This will be used for train synthetic, test synthetic
[24]:
# create new directory paths
# these directory paths build from the pwd
val_image_directory = 'example_val/images/'
val_label_directory = 'example_val/labels/'
dataset_val = 'val'
[25]:
convert_to_yolov5(val_dataset,
categories_new,
dataset_val,
val_dir_file_split,
val_image_directory,
val_label_directory,
categories_new)
100%|██████████| 1000/1000 [02:55<00:00, 5.69it/s]
Test dataset
This will be used for train synthetic, test synthetic
[26]:
# create new directory paths
# these directory paths build from the pwd
test_image_directory = 'example_test/images/'
test_label_directory = 'example_test/labels/'
dataset_test = 'test'
[27]:
convert_to_yolov5(syn_test_dataset,
categories_new,
dataset_test,
syn_test_file_split,
test_image_directory,
test_label_directory,
categories_new)
100%|██████████| 1000/1000 [02:57<00:00, 5.64it/s]
Confirm Bounding boxes
Now confirm the annotations before we train. We want to make sure that we correctly converted to the yolo format without ruining the location of the annotation.
[61]:
# Get any random annotation file
directory_test = f"{label_directory+dataset_train+'/'}"
directory_image = f"{image_directory+dataset_train+'/'}"
annotations = os.listdir(directory_test)
random_int = np.random.randint(len(annotations))
annotation_file = np.array(annotations)[random_int]
with open(directory_test + annotation_file, "r") as file:
try:
annotation_list = file.read().split("\n")[:-1]
annotation_list = [x.split(" ") for x in annotation_list]
annotation_list = [[float(y) for y in x ] for x in annotation_list]
except ValueError:
annotation_list = []
#Get the corresponding image file
image_file = annotation_file.replace("annotations", "images").replace("txt", "png")
assert os.path.exists(directory_image + image_file)
#Plot the Bounding Box
plot_bounding_box(directory_image + image_file,
categories_new,
annotation_list)
The annotation is correct!
Create train synthetic, test real data set
We will use the earlier converted training set data
Convert Test Set
labeling as validation but this will also serve as our testing set for the train synthetic, test real case
[29]:
# train set will be json files
test_dataset = json_test_files_cats
# create new directory paths
# these directory paths build from the pwd
image_test_directory = 'example_train_synthetic_real_case/images/'
label_test_directory = 'example_train_synthetic_real_case/labels/'
dataset_test = 'val'
[30]:
# This creates the training set
convert_to_yolov5(test_dataset,
test_categories,
dataset_test,
test_dir_file_list,
image_test_directory,
label_test_directory,
category_dict,
reference_imgs=True)
100%|██████████| 482/482 [00:39<00:00, 12.25it/s]
This command is necessary for some of the reference data:
[31]:
correct_images(image_test_directory, dataset_test)
Confirm bounding boxes
Confirm the annotations for the reference set!
[32]:
# Get any random annotation file
directory_test = f"{label_test_directory+dataset_test+'/'}"
directory_image = f"{image_test_directory+dataset_test+'/'}"
annotations = sorted(os.listdir(directory_test))
random_int = np.random.randint(len(annotations))
annotation_file = np.array(annotations)[random_int]
print(annotation_file, random_int)
with open(directory_test + annotation_file, "r") as file:
annotation_list = file.read().split("\n")[:-1]
annotation_list = [x.split(" ") for x in annotation_list]
annotation_list = [[float(y) for y in x ] for x in annotation_list]
#Get the corresponding image file
image_file = annotation_file.replace("annotations", "images").replace("txt", "png")
assert os.path.exists(directory_image + image_file)
#Plot the Bounding Box
plot_bounding_box(directory_image + image_file,
category_dict,
annotation_list)
62.txt 96
Looks good!
Create a yaml file
The yaml file is a mapping to your train and testing set.
This cell will serve as the train synthetic, test synthetic yaml.
[33]:
# train and val paths should be absolute
# also path of yaml file should live in yolov5/data
data = {'train':'/home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train/images/train/',
'val':'/home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_val/images/val/',
'test':'/home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/',
'nc':len(categories_new),
'names': categories_new
}
with open('/home/mrmarsh/repos/yolov5/data/hybrid_imagery_example.yaml', 'w') as outfile:
yaml.dump(data, outfile, default_flow_style=False,
sort_keys=False)
This cell will serve as the train synthetic, test real yaml.
[34]:
# train and val paths should be absolute
# also path of yaml file should live in yolov5/data
data = {'train':'/home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train/images/train/',
'val':'/home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/',
'nc':len(categories_new),
'names': categories_new
}
with open('/home/mrmarsh/repos/yolov5/data/hybrid_imagery_example_train_syn_test_real.yaml', 'w') as outfile:
yaml.dump(data, outfile, default_flow_style=False,
sort_keys=False)
Train the network
We have made it to the training section! The following code will train and validate the model for 400 epochs using synthetic data. Then it is evaluated using synthetic or real data.
The following command runs in bash. Tune parameters as needed. The below parameters are only for an example and should not be used when actually training:
[40]:
!python /home/mrmarsh/repos/yolov5/train.py --img 640 --batch 32 --epochs 400 \
--data /home/mrmarsh/repos/yolov5/data/hybrid_imagery_example.yaml \
--cfg /home/mrmarsh/repos/yolov5/models/yolov5s.yaml\
--weights /home/mrmarsh/repos/yolov5/models/yolov5s.pt \
--name hybrid_imagery_example \
--workers 128 \
--save-period 10 \
--cache \
--patience 500
wandb: Currently logged in as: mrmarsh. Use `wandb login --relogin` to force relogin
train: weights=/home/mrmarsh/repos/yolov5/models/yolov5s.pt, cfg=/home/mrmarsh/repos/yolov5/models/yolov5s.yaml, data=/home/mrmarsh/repos/yolov5/data/hybrid_imagery_example.yaml, hyp=../../yolov5/data/hyps/hyp.scratch-low.yaml, epochs=400, batch_size=32, imgsz=640, rect=False, resume=False, nosave=False, noval=False, noautoanchor=False, noplots=False, evolve=None, bucket=, cache=ram, image_weights=False, device=, multi_scale=False, single_cls=False, optimizer=SGD, sync_bn=False, workers=128, project=../../yolov5/runs/train, name=hybrid_imagery_example, exist_ok=False, quad=False, cos_lr=False, label_smoothing=0.0, patience=500, freeze=[0], save_period=10, local_rank=-1, entity=None, upload_dataset=False, bbox_interval=-1, artifact_alias=latest
github: skipping check (offline), for updates see https://github.com/ultralytics/yolov5
YOLOv5 🚀 v6.1-177-gd059d1d torch 1.11.0 CUDA:0 (Quadro RTX 6000, 24198MiB)
hyperparameters: lr0=0.01, lrf=0.01, momentum=0.937, weight_decay=0.0005, warmup_epochs=3.0, warmup_momentum=0.8, warmup_bias_lr=0.1, box=0.05, cls=0.5, cls_pw=1.0, obj=1.0, obj_pw=1.0, iou_t=0.2, anchor_t=4.0, fl_gamma=0.0, hsv_h=0.015, hsv_s=0.7, hsv_v=0.4, degrees=0.0, translate=0.1, scale=0.5, shear=0.0, perspective=0.0, flipud=0.0, fliplr=0.5, mosaic=1.0, mixup=0.0, copy_paste=0.0
TensorBoard: Start with 'tensorboard --logdir ../../yolov5/runs/train', view at http://localhost:6006/
wandb: wandb version 0.13.5 is available! To upgrade, please run:
wandb: $ pip install wandb --upgrade
wandb: Tracking run with wandb version 0.12.16
wandb: Run data is saved locally in /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/wandb/run-20221205_130051-m0j7l2p4
wandb: Run `wandb offline` to turn off syncing.
wandb: Syncing run hybrid_imagery_example
wandb: ⭐️ View project at https://wandb.ai/mrmarsh/train
wandb: 🚀 View run at https://wandb.ai/mrmarsh/train/runs/m0j7l2p4
YOLOv5 temporarily requires wandb version 0.12.10 or below. Some features may not work as expected.
Overriding model.yaml nc=80 with nc=3
from n params module arguments
0 -1 1 3520 models.common.Conv [3, 32, 6, 2, 2]
1 -1 1 18560 models.common.Conv [32, 64, 3, 2]
2 -1 1 18816 models.common.C3 [64, 64, 1]
3 -1 1 73984 models.common.Conv [64, 128, 3, 2]
4 -1 2 115712 models.common.C3 [128, 128, 2]
5 -1 1 295424 models.common.Conv [128, 256, 3, 2]
6 -1 3 625152 models.common.C3 [256, 256, 3]
7 -1 1 1180672 models.common.Conv [256, 512, 3, 2]
8 -1 1 1182720 models.common.C3 [512, 512, 1]
9 -1 1 656896 models.common.SPPF [512, 512, 5]
10 -1 1 131584 models.common.Conv [512, 256, 1, 1]
11 -1 1 0 torch.nn.modules.upsampling.Upsample [None, 2, 'nearest']
12 [-1, 6] 1 0 models.common.Concat [1]
13 -1 1 361984 models.common.C3 [512, 256, 1, False]
14 -1 1 33024 models.common.Conv [256, 128, 1, 1]
15 -1 1 0 torch.nn.modules.upsampling.Upsample [None, 2, 'nearest']
16 [-1, 4] 1 0 models.common.Concat [1]
17 -1 1 90880 models.common.C3 [256, 128, 1, False]
18 -1 1 147712 models.common.Conv [128, 128, 3, 2]
19 [-1, 14] 1 0 models.common.Concat [1]
20 -1 1 296448 models.common.C3 [256, 256, 1, False]
21 -1 1 590336 models.common.Conv [256, 256, 3, 2]
22 [-1, 10] 1 0 models.common.Concat [1]
23 -1 1 1182720 models.common.C3 [512, 512, 1, False]
24 [17, 20, 23] 1 21576 models.yolo.Detect [3, [[10, 13, 16, 30, 33, 23], [30, 61, 62, 45, 59, 119], [116, 90, 156, 198, 373, 326]], [128, 256, 512]]
YOLOv5s summary: 270 layers, 7027720 parameters, 7027720 gradients, 15.9 GFLOPs
Transferred 342/349 items from /home/mrmarsh/repos/yolov5/models/yolov5s.pt
Scaled weight_decay = 0.0005
optimizer: SGD with parameter groups 57 weight (no decay), 60 weight, 60 bias
albumentations: Blur(always_apply=False, p=0.01, blur_limit=(3, 7)), MedianBlur(always_apply=False, p=0.01, blur_limit=(3, 7)), ToGray(always_apply=False, p=0.01), CLAHE(always_apply=False, p=0.01, clip_limit=(1, 4.0), tile_grid_size=(8, 8))
train: Scanning '/home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/
train: Caching images (9.8GB ram): 100%|██████████| 7957/7957 [00:09<00:00, 798.
val: Scanning '/home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/ex
val: Caching images (1.2GB ram): 100%|██████████| 991/991 [00:02<00:00, 343.10it
Plotting labels to ../../yolov5/runs/train/hybrid_imagery_example/labels.jpg...
AutoAnchor: 3.87 anchors/target, 1.000 Best Possible Recall (BPR). Current anchors are a good fit to dataset ✅
Image sizes 640 train, 640 val
Using 16 dataloader workers
Logging results to ../../yolov5/runs/train/hybrid_imagery_example
Starting training for 400 epochs...
Epoch gpu_mem box obj cls labels img_size
0/399 6.92G 0.1162 0.0256 0.04732 30 640: 0%| Exception in thread Thread-37:
Traceback (most recent call last):
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 57, in check_pil_font
return ImageFont.truetype(str(font) if font.exists() else font.name, size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
return freetype(font)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
return FreeTypeFont(font, size, index, encoding, layout_engine)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
self.font = core.getfont(
OSError: unknown file format
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 973, in _bootstrap_inner
self.run()
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 910, in run
self._target(*self._args, **self._kwargs)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 217, in plot_images
annotator = Annotator(mosaic, line_width=round(fs / 10), font_size=fs, pil=True, example=names)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 77, in __init__
self.font = check_pil_font(font='Arial.Unicode.ttf' if non_ascii else font,
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 61, in check_pil_font
return ImageFont.truetype(str(font), size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
return freetype(font)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
return FreeTypeFont(font, size, index, encoding, layout_engine)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
self.font = core.getfont(
OSError: unknown file format
0/399 6.98G 0.1156 0.02549 0.04791 27 640: 1%| Exception in thread Thread-38:
Traceback (most recent call last):
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 57, in check_pil_font
return ImageFont.truetype(str(font) if font.exists() else font.name, size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
return freetype(font)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
return FreeTypeFont(font, size, index, encoding, layout_engine)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
self.font = core.getfont(
OSError: unknown file format
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 973, in _bootstrap_inner
self.run()
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 910, in run
self._target(*self._args, **self._kwargs)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 217, in plot_images
annotator = Annotator(mosaic, line_width=round(fs / 10), font_size=fs, pil=True, example=names)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 77, in __init__
self.font = check_pil_font(font='Arial.Unicode.ttf' if non_ascii else font,
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 61, in check_pil_font
return ImageFont.truetype(str(font), size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
return freetype(font)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
return FreeTypeFont(font, size, index, encoding, layout_engine)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
self.font = core.getfont(
OSError: unknown file format
0/399 6.98G 0.1157 0.02552 0.04688 30 640: 1%| Exception in thread Thread-39:
Traceback (most recent call last):
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 57, in check_pil_font
return ImageFont.truetype(str(font) if font.exists() else font.name, size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
return freetype(font)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
return FreeTypeFont(font, size, index, encoding, layout_engine)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
self.font = core.getfont(
OSError: unknown file format
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 973, in _bootstrap_inner
self.run()
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 910, in run
self._target(*self._args, **self._kwargs)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 217, in plot_images
annotator = Annotator(mosaic, line_width=round(fs / 10), font_size=fs, pil=True, example=names)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 77, in __init__
self.font = check_pil_font(font='Arial.Unicode.ttf' if non_ascii else font,
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 61, in check_pil_font
return ImageFont.truetype(str(font), size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
return freetype(font)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
return FreeTypeFont(font, size, index, encoding, layout_engine)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
self.font = core.getfont(
OSError: unknown file format
0/399 6.98G 0.05195 0.01385 0.01754 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.881 0.762 0.863 0.48
Epoch gpu_mem box obj cls labels img_size
1/399 8.78G 0.03571 0.006842 0.01135 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.623 0.702 0.692 0.353
Epoch gpu_mem box obj cls labels img_size
2/399 8.78G 0.03472 0.006425 0.01224 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 482 0.676 0.546 0.629 0.39
Epoch gpu_mem box obj cls labels img_size
3/399 8.78G 0.0307 0.006224 0.01129 24 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.91 0.839 0.917 0.727
Epoch gpu_mem box obj cls labels img_size
4/399 8.78G 0.02659 0.005425 0.008707 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.822 0.601 0.756 0.572
Epoch gpu_mem box obj cls labels img_size
5/399 8.78G 0.02381 0.005077 0.007832 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.833 0.645 0.802 0.657
Epoch gpu_mem box obj cls labels img_size
6/399 8.78G 0.02285 0.004648 0.007521 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.899 0.747 0.879 0.731
Epoch gpu_mem box obj cls labels img_size
7/399 8.78G 0.02234 0.004965 0.007593 26 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.937 0.854 0.949 0.802
Epoch gpu_mem box obj cls labels img_size
8/399 8.78G 0.02122 0.004713 0.006695 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.931 0.847 0.931 0.768
Epoch gpu_mem box obj cls labels img_size
9/399 8.78G 0.02034 0.004366 0.005866 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.951 0.88 0.964 0.829
Saving model artifact on epoch 10
Epoch gpu_mem box obj cls labels img_size
10/399 8.78G 0.01941 0.004342 0.006107 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.949 0.93 0.978 0.865
Epoch gpu_mem box obj cls labels img_size
11/399 8.78G 0.01918 0.004101 0.005735 24 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.96 0.898 0.967 0.846
Epoch gpu_mem box obj cls labels img_size
12/399 8.78G 0.0181 0.003967 0.005199 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.936 0.947 0.981 0.884
Epoch gpu_mem box obj cls labels img_size
13/399 8.78G 0.01823 0.003929 0.005318 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.963 0.916 0.974 0.861
Epoch gpu_mem box obj cls labels img_size
14/399 8.78G 0.01752 0.003952 0.005218 24 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.957 0.909 0.964 0.868
Epoch gpu_mem box obj cls labels img_size
15/399 8.78G 0.01708 0.003755 0.00511 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.961 0.957 0.983 0.891
Epoch gpu_mem box obj cls labels img_size
16/399 8.78G 0.01718 0.003755 0.005107 25 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.954 0.954 0.98 0.893
Epoch gpu_mem box obj cls labels img_size
17/399 8.78G 0.01676 0.003759 0.004576 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.972 0.937 0.986 0.901
Epoch gpu_mem box obj cls labels img_size
18/399 8.78G 0.01627 0.003702 0.004592 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.954 0.948 0.987 0.902
Epoch gpu_mem box obj cls labels img_size
19/399 8.78G 0.01645 0.003772 0.004737 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.94 0.958 0.982 0.896
Saving model artifact on epoch 20
Epoch gpu_mem box obj cls labels img_size
20/399 8.78G 0.01617 0.00374 0.004693 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.968 0.94 0.983 0.901
Epoch gpu_mem box obj cls labels img_size
21/399 8.78G 0.01616 0.003556 0.004621 13 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.976 0.922 0.977 0.909
Epoch gpu_mem box obj cls labels img_size
22/399 8.78G 0.01536 0.003565 0.004141 25 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.975 0.925 0.976 0.912
Epoch gpu_mem box obj cls labels img_size
23/399 8.78G 0.0154 0.003548 0.004181 13 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.964 0.96 0.988 0.921
Epoch gpu_mem box obj cls labels img_size
24/399 8.78G 0.015 0.003366 0.003581 26 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.973 0.935 0.981 0.916
Epoch gpu_mem box obj cls labels img_size
25/399 8.78G 0.01565 0.003554 0.004118 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.976 0.935 0.98 0.92
Epoch gpu_mem box obj cls labels img_size
26/399 8.78G 0.015 0.003456 0.003943 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.957 0.966 0.989 0.936
Epoch gpu_mem box obj cls labels img_size
27/399 8.78G 0.0151 0.00335 0.00368 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.96 0.992 0.928
Epoch gpu_mem box obj cls labels img_size
28/399 8.78G 0.01455 0.003383 0.003871 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.987 0.958 0.99 0.933
Epoch gpu_mem box obj cls labels img_size
29/399 8.78G 0.01468 0.003349 0.003974 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.982 0.943 0.98 0.917
Saving model artifact on epoch 30
Epoch gpu_mem box obj cls labels img_size
30/399 8.78G 0.01521 0.00346 0.004335 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.971 0.948 0.984 0.93
Epoch gpu_mem box obj cls labels img_size
31/399 8.78G 0.01492 0.00342 0.003873 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.979 0.945 0.986 0.935
Epoch gpu_mem box obj cls labels img_size
32/399 8.78G 0.01478 0.003364 0.003692 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.989 0.96 0.989 0.945
Epoch gpu_mem box obj cls labels img_size
33/399 8.78G 0.01443 0.003262 0.003214 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.969 0.968 0.99 0.939
Epoch gpu_mem box obj cls labels img_size
34/399 8.78G 0.0146 0.003459 0.00417 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.976 0.936 0.987 0.931
Epoch gpu_mem box obj cls labels img_size
35/399 8.78G 0.01421 0.003199 0.003387 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.97 0.964 0.99 0.946
Epoch gpu_mem box obj cls labels img_size
36/399 8.78G 0.01373 0.003157 0.002988 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.965 0.974 0.99 0.947
Epoch gpu_mem box obj cls labels img_size
37/399 8.78G 0.01399 0.003213 0.003909 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.977 0.952 0.987 0.941
Epoch gpu_mem box obj cls labels img_size
38/399 8.78G 0.01422 0.00321 0.004248 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.975 0.978 0.991 0.941
Epoch gpu_mem box obj cls labels img_size
39/399 8.78G 0.01382 0.003105 0.003427 28 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.986 0.96 0.989 0.954
Saving model artifact on epoch 40
Epoch gpu_mem box obj cls labels img_size
40/399 8.78G 0.01364 0.003135 0.003628 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.986 0.952 0.986 0.939
Epoch gpu_mem box obj cls labels img_size
41/399 8.78G 0.01423 0.003189 0.003662 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.984 0.975 0.991 0.952
Epoch gpu_mem box obj cls labels img_size
42/399 8.78G 0.01381 0.003071 0.003655 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 482 0.978 0.973 0.991 0.954
Epoch gpu_mem box obj cls labels img_size
43/399 8.78G 0.01366 0.00313 0.003485 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.995 0.961 0.992 0.952
Epoch gpu_mem box obj cls labels img_size
44/399 8.78G 0.01333 0.003069 0.003243 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.989 0.965 0.992 0.955
Epoch gpu_mem box obj cls labels img_size
45/399 8.78G 0.01303 0.003062 0.002813 13 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.974 0.985 0.993 0.959
Epoch gpu_mem box obj cls labels img_size
46/399 8.78G 0.01353 0.003025 0.003295 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.975 0.968 0.993 0.964
Epoch gpu_mem box obj cls labels img_size
47/399 8.78G 0.01302 0.002966 0.002772 25 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.973 0.982 0.993 0.965
Epoch gpu_mem box obj cls labels img_size
48/399 8.78G 0.01261 0.002949 0.002913 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.995 0.96 0.994 0.963
Epoch gpu_mem box obj cls labels img_size
49/399 8.78G 0.01271 0.002973 0.003026 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.98 0.981 0.993 0.964
Saving model artifact on epoch 50
Epoch gpu_mem box obj cls labels img_size
50/399 8.78G 0.01301 0.002928 0.002895 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.988 0.964 0.993 0.968
Epoch gpu_mem box obj cls labels img_size
51/399 8.78G 0.01291 0.002984 0.003079 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.991 0.97 0.993 0.967
Epoch gpu_mem box obj cls labels img_size
52/399 8.78G 0.01286 0.002958 0.003183 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.983 0.977 0.994 0.965
Epoch gpu_mem box obj cls labels img_size
53/399 8.78G 0.01313 0.00305 0.003825 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.984 0.966 0.992 0.963
Epoch gpu_mem box obj cls labels img_size
54/399 8.78G 0.01309 0.003017 0.003538 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.981 0.993 0.965
Epoch gpu_mem box obj cls labels img_size
55/399 8.78G 0.01321 0.003031 0.003083 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.974 0.975 0.993 0.968
Epoch gpu_mem box obj cls labels img_size
56/399 8.78G 0.01272 0.002867 0.002874 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.972 0.994 0.971
Epoch gpu_mem box obj cls labels img_size
57/399 8.78G 0.0128 0.002856 0.002862 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.978 0.981 0.993 0.972
Epoch gpu_mem box obj cls labels img_size
58/399 8.78G 0.0125 0.002921 0.003122 24 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.987 0.982 0.994 0.974
Epoch gpu_mem box obj cls labels img_size
59/399 8.78G 0.01273 0.002851 0.002804 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.978 0.991 0.994 0.976
Saving model artifact on epoch 60
Epoch gpu_mem box obj cls labels img_size
60/399 8.78G 0.01241 0.002874 0.002819 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.982 0.982 0.994 0.971
Epoch gpu_mem box obj cls labels img_size
61/399 8.78G 0.01217 0.002864 0.002748 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.989 0.981 0.994 0.974
Epoch gpu_mem box obj cls labels img_size
62/399 8.78G 0.01211 0.002925 0.002696 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.988 0.983 0.994 0.973
Epoch gpu_mem box obj cls labels img_size
63/399 8.78G 0.0124 0.002781 0.002586 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.991 0.977 0.994 0.976
Epoch gpu_mem box obj cls labels img_size
64/399 8.78G 0.01209 0.002794 0.002882 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.991 0.978 0.994 0.977
Epoch gpu_mem box obj cls labels img_size
65/399 8.78G 0.01241 0.00286 0.002935 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.992 0.977 0.994 0.974
Epoch gpu_mem box obj cls labels img_size
66/399 8.78G 0.01234 0.002773 0.002589 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.979 0.985 0.994 0.974
Epoch gpu_mem box obj cls labels img_size
67/399 8.78G 0.01221 0.002735 0.002446 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.978 0.994 0.977
Epoch gpu_mem box obj cls labels img_size
68/399 8.78G 0.01229 0.002812 0.002777 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.981 0.986 0.994 0.973
Epoch gpu_mem box obj cls labels img_size
69/399 8.78G 0.01261 0.002883 0.003159 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.971 0.994 0.974
Saving model artifact on epoch 70
Epoch gpu_mem box obj cls labels img_size
70/399 8.78G 0.01226 0.002804 0.00309 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.987 0.982 0.994 0.978
Epoch gpu_mem box obj cls labels img_size
71/399 8.78G 0.01236 0.002812 0.003062 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.984 0.977 0.994 0.978
Epoch gpu_mem box obj cls labels img_size
72/399 8.78G 0.01201 0.002754 0.002737 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.988 0.978 0.994 0.979
Epoch gpu_mem box obj cls labels img_size
73/399 8.78G 0.01211 0.00276 0.002646 26 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.991 0.981 0.994 0.979
Epoch gpu_mem box obj cls labels img_size
74/399 8.78G 0.01204 0.00285 0.002806 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.979 0.994 0.979
Epoch gpu_mem box obj cls labels img_size
75/399 8.78G 0.01189 0.002789 0.00268 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.978 0.994 0.98
Epoch gpu_mem box obj cls labels img_size
76/399 8.78G 0.01191 0.002782 0.002518 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.98 0.994 0.981
Epoch gpu_mem box obj cls labels img_size
77/399 8.78G 0.01224 0.002875 0.002801 13 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.989 0.977 0.994 0.982
Epoch gpu_mem box obj cls labels img_size
78/399 8.78G 0.01185 0.002695 0.002346 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.97 0.994 0.982
Epoch gpu_mem box obj cls labels img_size
79/399 8.78G 0.01208 0.002799 0.002184 25 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.981 0.994 0.981
Saving model artifact on epoch 80
Epoch gpu_mem box obj cls labels img_size
80/399 8.78G 0.01142 0.002688 0.002473 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.989 0.986 0.994 0.982
Epoch gpu_mem box obj cls labels img_size
81/399 8.78G 0.01144 0.002635 0.002473 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.985 0.994 0.982
Epoch gpu_mem box obj cls labels img_size
82/399 8.78G 0.01179 0.002705 0.002419 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.981 0.994 0.983
Epoch gpu_mem box obj cls labels img_size
83/399 8.78G 0.01186 0.00268 0.002165 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.991 0.977 0.994 0.983
Epoch gpu_mem box obj cls labels img_size
84/399 8.78G 0.0114 0.002717 0.002682 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.973 0.994 0.983
Epoch gpu_mem box obj cls labels img_size
85/399 8.78G 0.01193 0.002828 0.002712 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.98 0.994 0.983
Epoch gpu_mem box obj cls labels img_size
86/399 8.78G 0.0114 0.002658 0.002416 25 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.991 0.982 0.994 0.982
Epoch gpu_mem box obj cls labels img_size
87/399 8.78G 0.01187 0.002757 0.002212 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.982 0.994 0.984
Epoch gpu_mem box obj cls labels img_size
88/399 8.78G 0.01183 0.002669 0.002601 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.974 0.994 0.981
Epoch gpu_mem box obj cls labels img_size
89/399 8.78G 0.01159 0.002708 0.002537 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.986 0.989 0.994 0.983
Saving model artifact on epoch 90
Epoch gpu_mem box obj cls labels img_size
90/399 8.78G 0.0116 0.002684 0.002358 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.979 0.994 0.983
Epoch gpu_mem box obj cls labels img_size
91/399 8.78G 0.01177 0.002632 0.00264 13 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.987 0.983 0.994 0.983
Epoch gpu_mem box obj cls labels img_size
92/399 8.78G 0.01194 0.002701 0.002627 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.983 0.985 0.994 0.983
Epoch gpu_mem box obj cls labels img_size
93/399 8.78G 0.01129 0.002649 0.002346 11 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.977 0.994 0.982
Epoch gpu_mem box obj cls labels img_size
94/399 8.78G 0.01175 0.002782 0.00287 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.981 0.994 0.983
Epoch gpu_mem box obj cls labels img_size
95/399 8.78G 0.01148 0.002609 0.002575 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.981 0.994 0.981
Epoch gpu_mem box obj cls labels img_size
96/399 8.78G 0.01159 0.002784 0.002723 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.985 0.989 0.995 0.983
Epoch gpu_mem box obj cls labels img_size
97/399 8.78G 0.01188 0.002804 0.00305 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.981 0.994 0.995 0.982
Epoch gpu_mem box obj cls labels img_size
98/399 8.78G 0.01152 0.002627 0.002217 26 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.985 0.991 0.995 0.983
Epoch gpu_mem box obj cls labels img_size
99/399 8.78G 0.01171 0.002737 0.002578 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.987 0.987 0.995 0.982
Saving model artifact on epoch 100
Epoch gpu_mem box obj cls labels img_size
100/399 8.78G 0.0117 0.002715 0.002309 8 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.986 0.991 0.995 0.982
Epoch gpu_mem box obj cls labels img_size
101/399 8.78G 0.01139 0.00269 0.002148 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.989 0.988 0.995 0.982
Epoch gpu_mem box obj cls labels img_size
102/399 8.78G 0.01141 0.002645 0.002597 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.988 0.992 0.995 0.983
Epoch gpu_mem box obj cls labels img_size
103/399 8.78G 0.01125 0.002579 0.002418 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.987 0.995 0.984
Epoch gpu_mem box obj cls labels img_size
104/399 8.78G 0.01131 0.002623 0.002569 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.989 0.987 0.995 0.984
Epoch gpu_mem box obj cls labels img_size
105/399 8.78G 0.01126 0.002651 0.002122 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.987 0.987 0.995 0.984
Epoch gpu_mem box obj cls labels img_size
106/399 8.78G 0.01092 0.002601 0.002439 27 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.988 0.989 0.995 0.985
Epoch gpu_mem box obj cls labels img_size
107/399 8.78G 0.0111 0.002605 0.00203 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.988 0.989 0.995 0.985
Epoch gpu_mem box obj cls labels img_size
108/399 8.78G 0.01104 0.002629 0.002213 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.986 0.995 0.986
Epoch gpu_mem box obj cls labels img_size
109/399 8.78G 0.01098 0.002533 0.002034 26 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.987 0.988 0.995 0.986
Saving model artifact on epoch 110
Epoch gpu_mem box obj cls labels img_size
110/399 8.78G 0.01095 0.00257 0.002379 9 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.986 0.989 0.995 0.986
Epoch gpu_mem box obj cls labels img_size
111/399 8.78G 0.01103 0.002539 0.002005 24 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.995 0.981 0.995 0.986
Epoch gpu_mem box obj cls labels img_size
112/399 8.78G 0.01073 0.002573 0.001918 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.995 0.983 0.995 0.985
Epoch gpu_mem box obj cls labels img_size
113/399 8.78G 0.01116 0.002679 0.00222 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.985
Epoch gpu_mem box obj cls labels img_size
114/399 8.78G 0.01109 0.002618 0.002479 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.985
Epoch gpu_mem box obj cls labels img_size
115/399 8.78G 0.0106 0.002601 0.002138 11 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.985
Epoch gpu_mem box obj cls labels img_size
116/399 8.78G 0.01126 0.002679 0.002218 12 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.992 0.989 0.995 0.985
Epoch gpu_mem box obj cls labels img_size
117/399 8.78G 0.01097 0.002621 0.003072 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.989 0.995 0.984
Epoch gpu_mem box obj cls labels img_size
118/399 8.78G 0.01065 0.002554 0.002154 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.992 0.989 0.995 0.985
Epoch gpu_mem box obj cls labels img_size
119/399 8.78G 0.01084 0.002492 0.002094 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.992 0.989 0.995 0.985
Saving model artifact on epoch 120
Epoch gpu_mem box obj cls labels img_size
120/399 8.78G 0.01069 0.002568 0.002021 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.986
Epoch gpu_mem box obj cls labels img_size
121/399 8.78G 0.01118 0.002552 0.002643 25 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.985
Epoch gpu_mem box obj cls labels img_size
122/399 8.78G 0.01077 0.002504 0.002078 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.985
Epoch gpu_mem box obj cls labels img_size
123/399 8.78G 0.01076 0.002523 0.00207 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.985
Epoch gpu_mem box obj cls labels img_size
124/399 8.78G 0.01063 0.002466 0.002084 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.986
Epoch gpu_mem box obj cls labels img_size
125/399 8.78G 0.01074 0.00243 0.002315 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.982 0.995 0.986
Epoch gpu_mem box obj cls labels img_size
126/399 8.78G 0.01062 0.002519 0.001998 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.982 0.995 0.985
Epoch gpu_mem box obj cls labels img_size
127/399 8.78G 0.01065 0.002556 0.002265 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.982 0.995 0.986
Epoch gpu_mem box obj cls labels img_size
128/399 8.78G 0.0107 0.002528 0.002178 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.982 0.995 0.985
Epoch gpu_mem box obj cls labels img_size
129/399 8.78G 0.01085 0.002415 0.002086 26 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.985 0.995 0.986
Saving model artifact on epoch 130
Epoch gpu_mem box obj cls labels img_size
130/399 8.78G 0.01077 0.002565 0.00246 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.991 0.987 0.995 0.985
Epoch gpu_mem box obj cls labels img_size
131/399 8.78G 0.01087 0.002593 0.002324 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.991 0.987 0.995 0.985
Epoch gpu_mem box obj cls labels img_size
132/399 8.78G 0.01062 0.002469 0.002301 11 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.991 0.987 0.995 0.985
Epoch gpu_mem box obj cls labels img_size
133/399 8.78G 0.0108 0.002598 0.00269 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.986 0.995 0.986
Epoch gpu_mem box obj cls labels img_size
134/399 8.78G 0.01083 0.002626 0.002465 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.986 0.995 0.986
Epoch gpu_mem box obj cls labels img_size
135/399 8.78G 0.0105 0.0025 0.002538 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.985 0.995 0.986
Epoch gpu_mem box obj cls labels img_size
136/399 8.78G 0.01067 0.002623 0.002169 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.985 0.995 0.986
Epoch gpu_mem box obj cls labels img_size
137/399 8.78G 0.0108 0.00263 0.002811 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.984 0.995 0.986
Epoch gpu_mem box obj cls labels img_size
138/399 8.78G 0.01089 0.002515 0.002264 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.984 0.995 0.986
Epoch gpu_mem box obj cls labels img_size
139/399 8.78G 0.01083 0.002443 0.002035 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.984 0.995 0.986
Saving model artifact on epoch 140
Epoch gpu_mem box obj cls labels img_size
140/399 8.78G 0.01062 0.002469 0.002137 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.985 0.995 0.986
Epoch gpu_mem box obj cls labels img_size
141/399 8.78G 0.01037 0.002472 0.001742 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.985 0.995 0.986
Epoch gpu_mem box obj cls labels img_size
142/399 8.78G 0.01032 0.002483 0.001813 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.985 0.995 0.986
Epoch gpu_mem box obj cls labels img_size
143/399 8.78G 0.01047 0.00241 0.00191 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.986 0.995 0.986
Epoch gpu_mem box obj cls labels img_size
144/399 8.78G 0.01027 0.002417 0.002108 27 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.986 0.995 0.986
Epoch gpu_mem box obj cls labels img_size
145/399 8.78G 0.01033 0.00238 0.001987 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.987 0.995 0.986
Epoch gpu_mem box obj cls labels img_size
146/399 8.78G 0.01053 0.002388 0.001843 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.987 0.995 0.986
Epoch gpu_mem box obj cls labels img_size
147/399 8.78G 0.01035 0.002412 0.001936 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.987 0.995 0.986
Epoch gpu_mem box obj cls labels img_size
148/399 8.78G 0.01033 0.00238 0.001991 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.987 0.995 0.986
Epoch gpu_mem box obj cls labels img_size
149/399 8.78G 0.01027 0.002462 0.002334 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.987 0.995 0.986
Saving model artifact on epoch 150
Epoch gpu_mem box obj cls labels img_size
150/399 8.78G 0.01048 0.002566 0.002612 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.991 0.989 0.995 0.986
Epoch gpu_mem box obj cls labels img_size
151/399 8.78G 0.01036 0.00246 0.00223 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.987 0.995 0.986
Epoch gpu_mem box obj cls labels img_size
152/399 8.78G 0.01038 0.002528 0.002362 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.987 0.995 0.986
Epoch gpu_mem box obj cls labels img_size
153/399 8.78G 0.01013 0.002399 0.002019 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.987 0.995 0.986
Epoch gpu_mem box obj cls labels img_size
154/399 8.78G 0.01045 0.002368 0.00187 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.987 0.995 0.986
Epoch gpu_mem box obj cls labels img_size
155/399 8.78G 0.01029 0.002408 0.002137 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.995 0.987 0.995 0.986
Epoch gpu_mem box obj cls labels img_size
156/399 8.78G 0.01043 0.002433 0.001898 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.995 0.987 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
157/399 8.78G 0.01033 0.002486 0.002655 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.986 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
158/399 8.78G 0.0105 0.002406 0.001983 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.995 0.987 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
159/399 8.78G 0.009979 0.002396 0.001926 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.995 0.987 0.995 0.987
Saving model artifact on epoch 160
Epoch gpu_mem box obj cls labels img_size
160/399 8.78G 0.01008 0.002384 0.002058 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.998 0.986 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
161/399 8.78G 0.009937 0.002321 0.001517 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.998 0.986 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
162/399 8.78G 0.009874 0.002334 0.001454 13 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.998 0.986 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
163/399 8.78G 0.01014 0.002399 0.002104 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.998 0.986 0.995 0.986
Epoch gpu_mem box obj cls labels img_size
164/399 8.78G 0.009914 0.002383 0.002057 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.998 0.986 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
165/399 8.78G 0.01032 0.002353 0.001813 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.998 0.986 0.995 0.986
Epoch gpu_mem box obj cls labels img_size
166/399 8.78G 0.009724 0.002316 0.001739 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.998 0.986 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
167/399 8.78G 0.009843 0.002335 0.001921 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.998 0.986 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
168/399 8.78G 0.009971 0.002361 0.001889 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.998 0.985 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
169/399 8.78G 0.009809 0.002393 0.001936 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.998 0.985 0.995 0.987
Saving model artifact on epoch 170
Epoch gpu_mem box obj cls labels img_size
170/399 8.78G 0.009638 0.002315 0.001974 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.998 0.985 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
171/399 8.78G 0.009929 0.002476 0.001993 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.998 0.985 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
172/399 8.78G 0.01044 0.002517 0.002467 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.998 0.985 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
173/399 8.78G 0.009787 0.002365 0.002216 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.998 0.985 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
174/399 8.78G 0.01028 0.002392 0.00167 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.998 0.985 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
175/399 8.78G 0.009986 0.002406 0.002047 10 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.998 0.986 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
176/399 8.78G 0.009719 0.002338 0.001897 12 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.998 0.986 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
177/399 8.78G 0.009544 0.002311 0.001965 24 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.995 0.987 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
178/399 8.78G 0.01016 0.002412 0.002496 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.995 0.987 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
179/399 8.78G 0.009989 0.002376 0.002209 12 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.998 0.985 0.995 0.987
Saving model artifact on epoch 180
Epoch gpu_mem box obj cls labels img_size
180/399 8.78G 0.009992 0.002444 0.002375 13 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.998 0.985 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
181/399 8.78G 0.009912 0.002351 0.001608 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.998 0.985 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
182/399 8.78G 0.009964 0.002497 0.002372 13 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.985 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
183/399 8.78G 0.01006 0.002496 0.00215 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.985 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
184/399 8.78G 0.009941 0.002403 0.002048 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.985 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
185/399 8.78G 0.009831 0.002338 0.00207 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.985 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
186/399 8.78G 0.009627 0.002336 0.001817 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.985 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
187/399 8.78G 0.00972 0.002317 0.002335 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.995 0.985 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
188/399 8.78G 0.009366 0.002289 0.001727 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.995 0.985 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
189/399 8.78G 0.009583 0.002295 0.001756 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.984 0.995 0.987
Saving model artifact on epoch 190
Epoch gpu_mem box obj cls labels img_size
190/399 8.78G 0.009561 0.002294 0.00158 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.985 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
191/399 8.78G 0.009537 0.002352 0.001817 11 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.991 0.987 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
192/399 8.78G 0.009504 0.002304 0.00193 13 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.991 0.987 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
193/399 8.78G 0.009558 0.002308 0.00195 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.991 0.987 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
194/399 8.78G 0.009593 0.002374 0.001824 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.992 0.987 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
195/399 8.78G 0.00936 0.002261 0.001688 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.992 0.987 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
196/399 8.78G 0.009671 0.002312 0.001989 29 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.992 0.987 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
197/399 8.78G 0.009574 0.002263 0.001606 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.992 0.987 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
198/399 8.78G 0.009402 0.002238 0.001842 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.992 0.987 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
199/399 8.78G 0.009254 0.002232 0.001389 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.991 0.987 0.995 0.987
Saving model artifact on epoch 200
Epoch gpu_mem box obj cls labels img_size
200/399 8.78G 0.009264 0.002181 0.001486 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.992 0.987 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
201/399 8.78G 0.009593 0.002244 0.001658 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.992 0.987 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
202/399 8.78G 0.009096 0.00217 0.001643 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.991 0.987 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
203/399 8.78G 0.009489 0.002366 0.001731 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.991 0.987 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
204/399 8.78G 0.009366 0.002187 0.001954 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.986 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
205/399 8.78G 0.009141 0.002211 0.001711 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.986 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
206/399 8.78G 0.009221 0.002175 0.001943 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.986 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
207/399 8.78G 0.008874 0.0022 0.001353 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.986 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
208/399 8.78G 0.008863 0.002123 0.001214 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.983 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
209/399 8.78G 0.009238 0.002185 0.001462 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.984 0.995 0.987
Saving model artifact on epoch 210
Epoch gpu_mem box obj cls labels img_size
210/399 8.78G 0.009182 0.002201 0.001253 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.985 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
211/399 8.78G 0.009103 0.002089 0.001567 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.996 0.985 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
212/399 8.78G 0.009057 0.002242 0.00152 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
213/399 8.78G 0.009007 0.0021 0.001552 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
214/399 8.78G 0.009465 0.002178 0.00178 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
215/399 8.78G 0.009296 0.002271 0.001936 24 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
216/399 8.78G 0.00906 0.002157 0.001414 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
217/399 8.78G 0.008916 0.002089 0.001446 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
218/399 8.78G 0.008751 0.002178 0.00126 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
219/399 8.78G 0.009105 0.002115 0.001553 30 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.987
Saving model artifact on epoch 220
Epoch gpu_mem box obj cls labels img_size
220/399 8.78G 0.008688 0.002111 0.001468 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
221/399 8.78G 0.008771 0.002074 0.001122 24 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
222/399 8.78G 0.008945 0.002198 0.001865 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
223/399 8.78G 0.009228 0.002239 0.001902 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
224/399 8.78G 0.009177 0.002232 0.002092 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
225/399 8.78G 0.009061 0.002228 0.00159 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.995 0.987 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
226/399 8.78G 0.009079 0.002188 0.001625 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.995 0.987 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
227/399 8.78G 0.008974 0.002133 0.001571 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.995 0.987 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
228/399 8.78G 0.009469 0.002309 0.002333 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.995 0.987 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
229/399 8.78G 0.009259 0.002251 0.00178 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.995 0.987 0.995 0.988
Saving model artifact on epoch 230
Epoch gpu_mem box obj cls labels img_size
230/399 8.78G 0.008872 0.002258 0.001741 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.989 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
231/399 8.78G 0.009059 0.002165 0.001684 24 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.989 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
232/399 8.78G 0.00911 0.002187 0.001699 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.989 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
233/399 8.78G 0.00895 0.002173 0.001491 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.989 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
234/399 8.78G 0.009062 0.002174 0.001352 21 640: 29%|██▊wandb: Network error (ProxyError), entering retry loop.
234/399 8.78G 0.008932 0.002165 0.001499 25 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.989 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
235/399 8.78G 0.008927 0.002225 0.001792 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.989 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
236/399 8.78G 0.008872 0.002148 0.001606 24 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.989 0.995 0.987
Epoch gpu_mem box obj cls labels img_size
237/399 8.78G 0.008838 0.002121 0.001468 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.988 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
238/399 8.78G 0.008457 0.00208 0.001275 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.988 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
239/399 8.78G 0.008669 0.002106 0.001376 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.988 0.995 0.988
Saving model artifact on epoch 240
Epoch gpu_mem box obj cls labels img_size
240/399 8.78G 0.008485 0.002119 0.001613 26 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.988 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
241/399 8.78G 0.008555 0.002091 0.001401 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.988 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
242/399 8.78G 0.008459 0.002025 0.001101 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.988 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
243/399 8.78G 0.008484 0.002094 0.00179 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.989 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
244/399 8.78G 0.008852 0.002086 0.001485 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.989 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
245/399 8.78G 0.008688 0.002072 0.001131 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.989 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
246/399 8.78G 0.008535 0.002043 0.001541 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.989 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
247/399 8.78G 0.008294 0.002058 0.001425 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.989 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
248/399 8.78G 0.008318 0.002045 0.001404 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.989 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
249/399 8.78G 0.008423 0.002077 0.001566 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.989 0.995 0.988
Saving model artifact on epoch 250
Epoch gpu_mem box obj cls labels img_size
250/399 8.78G 0.008367 0.002016 0.001307 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.989 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
251/399 8.78G 0.008442 0.002078 0.001261 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.989 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
252/399 8.78G 0.008425 0.002047 0.001463 12 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.989 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
253/399 8.78G 0.008113 0.002027 0.00118 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.989 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
254/399 8.78G 0.00827 0.002021 0.001105 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.989 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
255/399 8.78G 0.008336 0.00202 0.001452 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.988 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
256/399 8.78G 0.008159 0.001968 0.001212 8 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.989 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
257/399 8.78G 0.008061 0.001922 0.001033 10 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
258/399 8.78G 0.008071 0.001973 0.001161 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
259/399 8.78G 0.007981 0.001991 0.001136 27 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.988
Saving model artifact on epoch 260
Epoch gpu_mem box obj cls labels img_size
260/399 8.78G 0.008428 0.002133 0.001609 27 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
261/399 8.78G 0.008412 0.002072 0.001286 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
262/399 8.78G 0.008125 0.001962 0.001194 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
263/399 8.78G 0.008374 0.002018 0.001224 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
264/399 8.78G 0.008671 0.002116 0.001618 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
265/399 8.78G 0.008182 0.002026 0.001381 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
266/399 8.78G 0.008124 0.002004 0.001196 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
267/399 8.78G 0.008179 0.002046 0.001321 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
268/399 8.78G 0.008015 0.002025 0.001117 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
269/399 8.78G 0.007923 0.001971 0.001229 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.989
Saving model artifact on epoch 270
Epoch gpu_mem box obj cls labels img_size
270/399 8.78G 0.008097 0.001982 0.001088 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
271/399 8.78G 0.008258 0.002001 0.001455 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
272/399 8.78G 0.008153 0.002042 0.001437 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
273/399 8.78G 0.0081 0.002032 0.001274 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
274/399 8.78G 0.007822 0.001935 0.001082 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
275/399 8.78G 0.007881 0.001987 0.001262 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.987 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
276/399 8.78G 0.007868 0.001966 0.001114 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.987 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
277/399 8.78G 0.007687 0.001919 0.001133 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.987 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
278/399 8.78G 0.007861 0.001899 0.001114 26 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.987 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
279/399 8.78G 0.008121 0.001952 0.00108 25 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.991 0.989 0.995 0.989
Saving model artifact on epoch 280
Epoch gpu_mem box obj cls labels img_size
280/399 8.78G 0.007912 0.00192 0.001067 13 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.991 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
281/399 8.78G 0.007777 0.001856 0.001122 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.991 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
282/399 8.78G 0.00773 0.001956 0.001046 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.991 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
283/399 8.78G 0.007739 0.001948 0.001149 30 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.991 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
284/399 8.78G 0.007587 0.001902 0.001049 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.991 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
285/399 8.78G 0.0077 0.001888 0.001062 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.991 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
286/399 8.78G 0.007673 0.001917 0.001231 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.991 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
287/399 8.78G 0.007619 0.001886 0.0009627 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.991 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
288/399 8.78G 0.007529 0.00188 0.001208 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.991 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
289/399 8.78G 0.007558 0.001931 0.001348 12 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.991 0.989 0.995 0.989
Saving model artifact on epoch 290
Epoch gpu_mem box obj cls labels img_size
290/399 8.78G 0.007876 0.001914 0.001104 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.991 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
291/399 8.78G 0.007687 0.001884 0.0009966 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.991 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
292/399 8.78G 0.007494 0.001852 0.00111 13 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
293/399 8.78G 0.007536 0.001921 0.001189 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
294/399 8.78G 0.007662 0.001894 0.001035 38 640: 94%|███wandb: Network error (ProxyError), entering retry loop.
294/399 8.78G 0.007629 0.001897 0.001018 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
295/399 8.78G 0.007571 0.001822 0.0009447 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
296/399 8.78G 0.007635 0.001915 0.0009266 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.989 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
297/399 8.78G 0.007456 0.001868 0.001138 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.989 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
298/399 8.78G 0.007383 0.00183 0.001052 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.989 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
299/399 8.78G 0.007487 0.001857 0.001159 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.989 0.995 0.988
Saving model artifact on epoch 300
Epoch gpu_mem box obj cls labels img_size
300/399 8.78G 0.007437 0.001881 0.001196 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.989 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
301/399 8.78G 0.007374 0.00184 0.0009381 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.989 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
302/399 8.78G 0.007469 0.001909 0.001005 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.989 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
303/399 8.78G 0.007069 0.001792 0.0009178 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.989 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
304/399 8.78G 0.007136 0.001899 0.0009029 28 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.989 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
305/399 8.78G 0.00719 0.001809 0.0008898 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.989 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
306/399 8.78G 0.007297 0.001878 0.001402 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.989 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
307/399 8.78G 0.007139 0.001792 0.0008135 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
308/399 8.78G 0.007251 0.001841 0.001199 28 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
309/399 8.78G 0.007121 0.001814 0.001076 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.989 0.995 0.989
Saving model artifact on epoch 310
Epoch gpu_mem box obj cls labels img_size
310/399 8.78G 0.007414 0.001836 0.001101 9 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
311/399 8.78G 0.007206 0.001834 0.001211 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
312/399 8.78G 0.007372 0.001806 0.001127 13 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
313/399 8.78G 0.00704 0.001758 0.0009558 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.989 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
314/399 8.78G 0.006919 0.001771 0.0007918 24 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.989 0.992 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
315/399 8.78G 0.007053 0.001752 0.0007955 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.989 0.992 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
316/399 8.78G 0.006984 0.001757 0.000815 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.992 0.995 0.988
Epoch gpu_mem box obj cls labels img_size
317/399 8.78G 0.007007 0.001754 0.0008991 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
318/399 8.78G 0.007187 0.001819 0.001134 13 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
319/399 8.78G 0.007312 0.001752 0.00109 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.992 0.995 0.989
Saving model artifact on epoch 320
Epoch gpu_mem box obj cls labels img_size
320/399 8.78G 0.006893 0.00174 0.0007894 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
321/399 8.78G 0.006822 0.001735 0.001139 28 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
322/399 8.78G 0.00681 0.001724 0.0009574 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
323/399 8.78G 0.006879 0.001775 0.001124 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
324/399 8.78G 0.006811 0.001774 0.0009221 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
325/399 8.78G 0.006924 0.001759 0.0008976 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.988 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
326/399 8.78G 0.006554 0.001707 0.000901 24 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
327/399 8.78G 0.006746 0.001671 0.0007761 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.988 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
328/399 8.78G 0.006754 0.001726 0.0008094 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.988 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
329/399 8.78G 0.006901 0.001767 0.001129 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.989 0.995 0.989
Saving model artifact on epoch 330
Epoch gpu_mem box obj cls labels img_size
330/399 8.78G 0.006868 0.001704 0.001008 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
331/399 8.78G 0.006635 0.001727 0.000963 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
332/399 8.78G 0.006571 0.001733 0.000912 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
333/399 8.78G 0.00675 0.001686 0.0006604 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
334/399 8.78G 0.006606 0.001656 0.0009005 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
335/399 8.78G 0.006664 0.001748 0.0008817 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
336/399 8.78G 0.006774 0.001732 0.001006 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
337/399 8.78G 0.006506 0.00169 0.0007435 9 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
338/399 8.78G 0.006682 0.001634 0.0007348 13 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
339/399 8.78G 0.006566 0.001665 0.0008622 25 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.992 0.995 0.989
Saving model artifact on epoch 340
Epoch gpu_mem box obj cls labels img_size
340/399 8.78G 0.006563 0.001647 0.0007575 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
341/399 8.78G 0.006685 0.001699 0.0008283 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
342/399 8.78G 0.006442 0.00166 0.001119 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
343/399 8.78G 0.006574 0.001683 0.0008364 27 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
344/399 8.78G 0.006276 0.001621 0.0006612 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
345/399 8.78G 0.006281 0.001575 0.0006959 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
346/399 8.78G 0.006348 0.001632 0.0008843 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
347/399 8.78G 0.006551 0.001687 0.0009834 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
348/399 8.78G 0.006535 0.001657 0.0007971 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
349/399 8.78G 0.006225 0.001598 0.0006955 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.992 0.995 0.989
Saving model artifact on epoch 350
Epoch gpu_mem box obj cls labels img_size
350/399 8.78G 0.00652 0.00166 0.0006733 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
351/399 8.78G 0.006193 0.001615 0.0008038 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.989 0.992 0.995 0.99
Epoch gpu_mem box obj cls labels img_size
352/399 8.78G 0.00635 0.001641 0.0006972 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.989 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
353/399 8.78G 0.006209 0.001605 0.00106 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.989 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
354/399 8.78G 0.006026 0.001551 0.0006301 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.989 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
355/399 8.78G 0.006437 0.001661 0.001061 26 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.989 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
356/399 8.78G 0.006287 0.001636 0.0007675 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.989 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
357/399 8.78G 0.005986 0.001593 0.0005777 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.989 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
358/399 8.78G 0.006154 0.001581 0.0008132 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.992 0.987 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
359/399 8.78G 0.005988 0.001553 0.000594 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.992 0.987 0.995 0.989
Saving model artifact on epoch 360
Epoch gpu_mem box obj cls labels img_size
360/399 8.78G 0.006199 0.001601 0.0006295 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.992 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
361/399 8.78G 0.005975 0.001604 0.0006908 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
362/399 8.78G 0.005977 0.001583 0.0006357 13 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.992 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
363/399 8.78G 0.005814 0.001569 0.0007076 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
364/399 8.78G 0.005915 0.001602 0.0005066 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
365/399 8.78G 0.005902 0.001553 0.0006204 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
366/399 8.78G 0.005897 0.001526 0.0007189 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
367/399 8.78G 0.005801 0.001561 0.000736 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
368/399 8.78G 0.005818 0.001531 0.0005447 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
369/399 8.78G 0.005651 0.001486 0.0004308 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.989 0.995 0.989
Saving model artifact on epoch 370
Epoch gpu_mem box obj cls labels img_size
370/399 8.78G 0.005672 0.001521 0.0006316 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
371/399 8.78G 0.005627 0.001561 0.0005789 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
372/399 8.78G 0.005605 0.001496 0.0005287 27 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
373/399 8.78G 0.005564 0.00149 0.0006741 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
374/399 8.78G 0.005562 0.001474 0.0005015 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
375/399 8.78G 0.005494 0.001442 0.0004316 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
376/399 8.78G 0.005433 0.001521 0.0005998 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
377/399 8.78G 0.005324 0.001454 0.0005089 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.989 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
378/399 8.78G 0.005359 0.001458 0.0003825 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.992 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
379/399 8.78G 0.005434 0.001453 0.0003868 25 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.992 0.992 0.995 0.989
Saving model artifact on epoch 380
Epoch gpu_mem box obj cls labels img_size
380/399 8.78G 0.005477 0.001418 0.00071 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.992 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
381/399 8.78G 0.005356 0.001496 0.0004867 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.992 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
382/399 8.78G 0.005294 0.00144 0.0004135 27 640: 45%|███wandb: Network error (ProxyError), entering retry loop.
382/399 8.78G 0.005354 0.001466 0.0005333 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
383/399 8.78G 0.005304 0.00146 0.0005451 24 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
384/399 8.78G 0.005292 0.001441 0.0005461 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
385/399 8.78G 0.005258 0.001441 0.0004153 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
386/399 8.78G 0.005165 0.001458 0.000635 28 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
387/399 8.78G 0.005302 0.001456 0.0005228 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
388/399 8.78G 0.005317 0.001407 0.0004739 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
389/399 8.78G 0.005148 0.00141 0.0005128 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.992 0.995 0.989
Saving model artifact on epoch 390
Epoch gpu_mem box obj cls labels img_size
390/399 8.78G 0.00517 0.001397 0.0004808 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
391/399 8.78G 0.005011 0.001415 0.0003338 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
392/399 8.78G 0.00513 0.001375 0.0004617 24 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
393/399 8.78G 0.005021 0.001397 0.0005565 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
394/399 8.78G 0.004884 0.001411 0.0004105 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
395/399 8.78G 0.004961 0.001384 0.0003836 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
396/399 8.78G 0.004968 0.001343 0.0004494 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.993 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
397/399 8.78G 0.005088 0.001438 0.0004789 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
398/399 8.78G 0.00502 0.001359 0.0005113 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.992 0.995 0.989
Epoch gpu_mem box obj cls labels img_size
399/399 8.78G 0.004883 0.001353 0.0004345 26 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.994 0.992 0.995 0.989
400 epochs completed in 5.136 hours.
Optimizer stripped from ../../yolov5/runs/train/hybrid_imagery_example/weights/last.pt, 14.5MB
Optimizer stripped from ../../yolov5/runs/train/hybrid_imagery_example/weights/best.pt, 14.5MB
Validating ../../yolov5/runs/train/hybrid_imagery_example/weights/best.pt...
Fusing layers...
YOLOv5s summary: 213 layers, 7018216 parameters, 0 gradients, 15.8 GFLOPs
Class Images Labels P R mAP@.5mAP@.Exception in thread Thread-40:
Traceback (most recent call last):
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 57, in check_pil_font
return ImageFont.truetype(str(font) if font.exists() else font.name, size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
return freetype(font)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
return FreeTypeFont(font, size, index, encoding, layout_engine)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
self.font = core.getfont(
OSError: unknown file format
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 973, in _bootstrap_inner
self.run()
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 910, in run
self._target(*self._args, **self._kwargs)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 217, in plot_images
annotator = Annotator(mosaic, line_width=round(fs / 10), font_size=fs, pil=True, example=names)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 77, in __init__
self.font = check_pil_font(font='Arial.Unicode.ttf' if non_ascii else font,
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 61, in check_pil_font
return ImageFont.truetype(str(font), size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
return freetype(font)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
return FreeTypeFont(font, size, index, encoding, layout_engine)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
self.font = core.getfont(
OSError: unknown file format
Exception in thread Thread-41:
Traceback (most recent call last):
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 57, in check_pil_font
return ImageFont.truetype(str(font) if font.exists() else font.name, size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
return freetype(font)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
return FreeTypeFont(font, size, index, encoding, layout_engine)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
self.font = core.getfont(
OSError: unknown file format
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 973, in _bootstrap_inner
self.run()
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 910, in run
self._target(*self._args, **self._kwargs)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 217, in plot_images
annotator = Annotator(mosaic, line_width=round(fs / 10), font_size=fs, pil=True, example=names)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 77, in __init__
self.font = check_pil_font(font='Arial.Unicode.ttf' if non_ascii else font,
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 61, in check_pil_font
return ImageFont.truetype(str(font), size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
return freetype(font)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
return FreeTypeFont(font, size, index, encoding, layout_engine)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
self.font = core.getfont(
OSError: unknown file format
Class Images Labels P R mAP@.5mAP@.Exception in thread Thread-42:
Traceback (most recent call last):
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 57, in check_pil_font
return ImageFont.truetype(str(font) if font.exists() else font.name, size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
return freetype(font)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
return FreeTypeFont(font, size, index, encoding, layout_engine)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
self.font = core.getfont(
OSError: unknown file format
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 973, in _bootstrap_inner
self.run()
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 910, in run
self._target(*self._args, **self._kwargs)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 217, in plot_images
annotator = Annotator(mosaic, line_width=round(fs / 10), font_size=fs, pil=True, example=names)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 77, in __init__
self.font = check_pil_font(font='Arial.Unicode.ttf' if non_ascii else font,
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 61, in check_pil_font
return ImageFont.truetype(str(font), size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
return freetype(font)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
return FreeTypeFont(font, size, index, encoding, layout_engine)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
self.font = core.getfont(
OSError: unknown file format
Exception in thread Thread-43:
Traceback (most recent call last):
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 57, in check_pil_font
return ImageFont.truetype(str(font) if font.exists() else font.name, size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
return freetype(font)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
return FreeTypeFont(font, size, index, encoding, layout_engine)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
self.font = core.getfont(
OSError: unknown file format
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 973, in _bootstrap_inner
self.run()
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 910, in run
self._target(*self._args, **self._kwargs)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 217, in plot_images
annotator = Annotator(mosaic, line_width=round(fs / 10), font_size=fs, pil=True, example=names)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 77, in __init__
self.font = check_pil_font(font='Arial.Unicode.ttf' if non_ascii else font,
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 61, in check_pil_font
return ImageFont.truetype(str(font), size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
return freetype(font)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
return FreeTypeFont(font, size, index, encoding, layout_engine)
Class Images Labels P R mAP@.5mAP@. File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
self.font = core.getfont(
OSError: unknown file format
Exception in thread Thread-44:
Traceback (most recent call last):
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 57, in check_pil_font
return ImageFont.truetype(str(font) if font.exists() else font.name, size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
return freetype(font)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
return FreeTypeFont(font, size, index, encoding, layout_engine)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
self.font = core.getfont(
OSError: unknown file format
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 973, in _bootstrap_inner
self.run()
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 910, in run
self._target(*self._args, **self._kwargs)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 217, in plot_images
annotator = Annotator(mosaic, line_width=round(fs / 10), font_size=fs, pil=True, example=names)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 77, in __init__
self.font = check_pil_font(font='Arial.Unicode.ttf' if non_ascii else font,
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 61, in check_pil_font
return ImageFont.truetype(str(font), size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
return freetype(font)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
return FreeTypeFont(font, size, index, encoding, layout_engine)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
self.font = core.getfont(
OSError: unknown file format
Exception in thread Thread-45:
Traceback (most recent call last):
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 57, in check_pil_font
return ImageFont.truetype(str(font) if font.exists() else font.name, size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
return freetype(font)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
return FreeTypeFont(font, size, index, encoding, layout_engine)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
self.font = core.getfont(
OSError: unknown file format
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 973, in _bootstrap_inner
self.run()
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 910, in run
self._target(*self._args, **self._kwargs)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 217, in plot_images
annotator = Annotator(mosaic, line_width=round(fs / 10), font_size=fs, pil=True, example=names)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 77, in __init__
self.font = check_pil_font(font='Arial.Unicode.ttf' if non_ascii else font,
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 61, in check_pil_font
return ImageFont.truetype(str(font), size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
return freetype(font)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
return FreeTypeFont(font, size, index, encoding, layout_engine)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
self.font = core.getfont(
OSError: unknown file format
Class Images Labels P R mAP@.5mAP@.
all 991 483 0.99 0.992 0.995 0.989
30 991 261 0.988 0.992 0.995 0.992
48 991 222 0.991 0.991 0.995 0.985
Traceback (most recent call last):
File "/home/mrmarsh/repos/yolov5/train.py", line 667, in <module>
main(opt)
File "/home/mrmarsh/repos/yolov5/train.py", line 562, in main
train(opt.hyp, opt, device, callbacks)
File "/home/mrmarsh/repos/yolov5/train.py", line 451, in train
results, _, _ = val.run(
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/autograd/grad_mode.py", line 27, in decorate_context
return func(*args, **kwargs)
File "/home/mrmarsh/repos/yolov5/val.py", line 296, in run
print("\n".join(print_buffer), file=open(save_dir / 'map_labels' / ('results.txt'), 'w'))
FileNotFoundError: [Errno 2] No such file or directory: '../../yolov5/runs/train/hybrid_imagery_example/map_labels/results.txt'
wandb: Waiting for W&B process to finish... (failed 1). Press Control-C to abort syncing.
wandb:
wandb:
wandb: Run history:
wandb: metrics/mAP_0.5 ▁▆▇▇████████████████████████████████████
wandb: metrics/mAP_0.5:0.95 ▁▄▆▇▇▇██████████████████████████████████
wandb: metrics/precision ▁▅▅▇█▇▇▇▇▇▇█████████████████▇▇▇▇▇▇▇▇████
wandb: metrics/recall ▁▄▇▆▇▇▇▇▇███████████████████████████████
wandb: train/box_loss █▅▄▄▃▃▃▃▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁
wandb: train/cls_loss █▄▃▃▃▃▂▃▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁
wandb: train/obj_loss █▅▄▄▄▃▃▃▃▃▃▃▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▂▁▂▁▁▁▁▁▁
wandb: val/box_loss █▅▃▃▃▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
wandb: val/cls_loss █▅▃▄▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
wandb: val/obj_loss █▄▃▃▃▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
wandb: x/lr0 ████▇▇▇▇▇▆▆▆▆▆▆▅▅▅▅▅▄▄▄▄▄▄▃▃▃▃▃▂▂▂▂▂▂▁▁▁
wandb: x/lr1 ████▇▇▇▇▇▆▆▆▆▆▆▅▅▅▅▅▄▄▄▄▄▄▃▃▃▃▃▂▂▂▂▂▂▁▁▁
wandb: x/lr2 ████▇▇▇▇▇▆▆▆▆▆▆▅▅▅▅▅▄▄▄▄▄▄▃▃▃▃▃▂▂▂▂▂▂▁▁▁
wandb:
wandb: Run summary:
wandb: best/epoch 351
wandb: best/mAP_0.5 0.99489
wandb: best/mAP_0.5:0.95 0.98952
wandb: best/precision 0.98949
wandb: best/recall 0.99166
wandb: metrics/mAP_0.5 0.99494
wandb: metrics/mAP_0.5:0.95 0.98919
wandb: metrics/precision 0.9936
wandb: metrics/recall 0.99166
wandb: train/box_loss 0.00488
wandb: train/cls_loss 0.00043
wandb: train/obj_loss 0.00135
wandb: val/box_loss 0.0037
wandb: val/cls_loss 0.00014
wandb: val/obj_loss 0.00051
wandb: x/lr0 0.00015
wandb: x/lr1 0.00015
wandb: x/lr2 0.00015
wandb:
wandb: Synced hybrid_imagery_example: https://wandb.ai/mrmarsh/train/runs/m0j7l2p4
wandb: Synced 6 W&B file(s), 322 media file(s), 39 artifact file(s) and 0 other file(s)
wandb: Find logs at: ./wandb/run-20221205_130051-m0j7l2p4/logs
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f4bc0074430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
[44]:
!python /home/mrmarsh/repos/yolov5/train.py --img 640 --batch 32 --epochs 400 \
--data /home/mrmarsh/repos/yolov5/data/hybrid_imagery_example_train_syn_test_real.yaml \
--cfg /home/mrmarsh/repos/yolov5/models/yolov5s.yaml\
--weights /home/mrmarsh/repos/yolov5/models/yolov5s.pt \
--name hybrid_imagery_example_train_syn_test_real \
--workers 128 \
--save-period 10 \
--cache \
--patience 500
wandb: Currently logged in as: mrmarsh. Use `wandb login --relogin` to force relogin
train: weights=/home/mrmarsh/repos/yolov5/models/yolov5s.pt, cfg=/home/mrmarsh/repos/yolov5/models/yolov5s.yaml, data=/home/mrmarsh/repos/yolov5/data/hybrid_imagery_example_train_syn_test_real.yaml, hyp=../../yolov5/data/hyps/hyp.scratch-low.yaml, epochs=400, batch_size=32, imgsz=640, rect=False, resume=False, nosave=False, noval=False, noautoanchor=False, noplots=False, evolve=None, bucket=, cache=ram, image_weights=False, device=, multi_scale=False, single_cls=False, optimizer=SGD, sync_bn=False, workers=128, project=../../yolov5/runs/train, name=hybrid_imagery_example_train_syn_test_real, exist_ok=False, quad=False, cos_lr=False, label_smoothing=0.0, patience=500, freeze=[0], save_period=10, local_rank=-1, entity=None, upload_dataset=False, bbox_interval=-1, artifact_alias=latest
github: skipping check (offline), for updates see https://github.com/ultralytics/yolov5
YOLOv5 🚀 v6.1-177-gd059d1d torch 1.11.0 CUDA:0 (Quadro RTX 6000, 24198MiB)
hyperparameters: lr0=0.01, lrf=0.01, momentum=0.937, weight_decay=0.0005, warmup_epochs=3.0, warmup_momentum=0.8, warmup_bias_lr=0.1, box=0.05, cls=0.5, cls_pw=1.0, obj=1.0, obj_pw=1.0, iou_t=0.2, anchor_t=4.0, fl_gamma=0.0, hsv_h=0.015, hsv_s=0.7, hsv_v=0.4, degrees=0.0, translate=0.1, scale=0.5, shear=0.0, perspective=0.0, flipud=0.0, fliplr=0.5, mosaic=1.0, mixup=0.0, copy_paste=0.0
TensorBoard: Start with 'tensorboard --logdir ../../yolov5/runs/train', view at http://localhost:6006/
wandb: wandb version 0.13.5 is available! To upgrade, please run:
wandb: $ pip install wandb --upgrade
wandb: Tracking run with wandb version 0.12.16
wandb: Run data is saved locally in /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/wandb/run-20221206_092959-2teyd5oi
wandb: Run `wandb offline` to turn off syncing.
wandb: Syncing run hybrid_imagery_example_train_syn_test_real
wandb: ⭐️ View project at https://wandb.ai/mrmarsh/train
wandb: 🚀 View run at https://wandb.ai/mrmarsh/train/runs/2teyd5oi
YOLOv5 temporarily requires wandb version 0.12.10 or below. Some features may not work as expected.
Overriding model.yaml nc=80 with nc=3
from n params module arguments
0 -1 1 3520 models.common.Conv [3, 32, 6, 2, 2]
1 -1 1 18560 models.common.Conv [32, 64, 3, 2]
2 -1 1 18816 models.common.C3 [64, 64, 1]
3 -1 1 73984 models.common.Conv [64, 128, 3, 2]
4 -1 2 115712 models.common.C3 [128, 128, 2]
5 -1 1 295424 models.common.Conv [128, 256, 3, 2]
6 -1 3 625152 models.common.C3 [256, 256, 3]
7 -1 1 1180672 models.common.Conv [256, 512, 3, 2]
8 -1 1 1182720 models.common.C3 [512, 512, 1]
9 -1 1 656896 models.common.SPPF [512, 512, 5]
10 -1 1 131584 models.common.Conv [512, 256, 1, 1]
11 -1 1 0 torch.nn.modules.upsampling.Upsample [None, 2, 'nearest']
12 [-1, 6] 1 0 models.common.Concat [1]
13 -1 1 361984 models.common.C3 [512, 256, 1, False]
14 -1 1 33024 models.common.Conv [256, 128, 1, 1]
15 -1 1 0 torch.nn.modules.upsampling.Upsample [None, 2, 'nearest']
16 [-1, 4] 1 0 models.common.Concat [1]
17 -1 1 90880 models.common.C3 [256, 128, 1, False]
18 -1 1 147712 models.common.Conv [128, 128, 3, 2]
19 [-1, 14] 1 0 models.common.Concat [1]
20 -1 1 296448 models.common.C3 [256, 256, 1, False]
21 -1 1 590336 models.common.Conv [256, 256, 3, 2]
22 [-1, 10] 1 0 models.common.Concat [1]
23 -1 1 1182720 models.common.C3 [512, 512, 1, False]
24 [17, 20, 23] 1 21576 models.yolo.Detect [3, [[10, 13, 16, 30, 33, 23], [30, 61, 62, 45, 59, 119], [116, 90, 156, 198, 373, 326]], [128, 256, 512]]
YOLOv5s summary: 270 layers, 7027720 parameters, 7027720 gradients, 15.9 GFLOPs
Transferred 342/349 items from /home/mrmarsh/repos/yolov5/models/yolov5s.pt
Scaled weight_decay = 0.0005
optimizer: SGD with parameter groups 57 weight (no decay), 60 weight, 60 bias
albumentations: Blur(always_apply=False, p=0.01, blur_limit=(3, 7)), MedianBlur(always_apply=False, p=0.01, blur_limit=(3, 7)), ToGray(always_apply=False, p=0.01), CLAHE(always_apply=False, p=0.01, clip_limit=(1, 4.0), tile_grid_size=(8, 8))
train: Scanning '/home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/
train: Caching images (9.8GB ram): 100%|██████████| 7957/7957 [00:12<00:00, 645.
val: Scanning '/home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/ex
val: Caching images (0.1GB ram): 100%|██████████| 137/137 [00:01<00:00, 114.55it
Plotting labels to ../../yolov5/runs/train/hybrid_imagery_example_train_syn_test_real/labels.jpg...
AutoAnchor: 3.87 anchors/target, 1.000 Best Possible Recall (BPR). Current anchors are a good fit to dataset ✅
Image sizes 640 train, 640 val
Using 16 dataloader workers
Logging results to ../../yolov5/runs/train/hybrid_imagery_example_train_syn_test_real
Starting training for 400 epochs...
Epoch gpu_mem box obj cls labels img_size
0/399 6.92G 0.1162 0.0256 0.04732 30 640: 0%| Exception in thread Thread-37:
Traceback (most recent call last):
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 57, in check_pil_font
return ImageFont.truetype(str(font) if font.exists() else font.name, size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
return freetype(font)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
return FreeTypeFont(font, size, index, encoding, layout_engine)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
self.font = core.getfont(
OSError: unknown file format
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 973, in _bootstrap_inner
self.run()
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 910, in run
self._target(*self._args, **self._kwargs)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 217, in plot_images
annotator = Annotator(mosaic, line_width=round(fs / 10), font_size=fs, pil=True, example=names)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 77, in __init__
self.font = check_pil_font(font='Arial.Unicode.ttf' if non_ascii else font,
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 61, in check_pil_font
return ImageFont.truetype(str(font), size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
return freetype(font)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
return FreeTypeFont(font, size, index, encoding, layout_engine)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
self.font = core.getfont(
OSError: unknown file format
0/399 6.98G 0.1156 0.02549 0.04791 27 640: 1%| Exception in thread Thread-38:
Traceback (most recent call last):
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 57, in check_pil_font
return ImageFont.truetype(str(font) if font.exists() else font.name, size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
return freetype(font)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
return FreeTypeFont(font, size, index, encoding, layout_engine)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
self.font = core.getfont(
OSError: unknown file format
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 973, in _bootstrap_inner
self.run()
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 910, in run
self._target(*self._args, **self._kwargs)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 217, in plot_images
annotator = Annotator(mosaic, line_width=round(fs / 10), font_size=fs, pil=True, example=names)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 77, in __init__
self.font = check_pil_font(font='Arial.Unicode.ttf' if non_ascii else font,
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 61, in check_pil_font
return ImageFont.truetype(str(font), size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
return freetype(font)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
return FreeTypeFont(font, size, index, encoding, layout_engine)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
self.font = core.getfont(
OSError: unknown file format
0/399 6.98G 0.1157 0.02552 0.04688 30 640: 1%| Exception in thread Thread-39:
Traceback (most recent call last):
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 57, in check_pil_font
return ImageFont.truetype(str(font) if font.exists() else font.name, size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
return freetype(font)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
return FreeTypeFont(font, size, index, encoding, layout_engine)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
self.font = core.getfont(
OSError: unknown file format
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 973, in _bootstrap_inner
self.run()
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 910, in run
self._target(*self._args, **self._kwargs)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 217, in plot_images
annotator = Annotator(mosaic, line_width=round(fs / 10), font_size=fs, pil=True, example=names)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 77, in __init__
self.font = check_pil_font(font='Arial.Unicode.ttf' if non_ascii else font,
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 61, in check_pil_font
return ImageFont.truetype(str(font), size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
return freetype(font)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
return FreeTypeFont(font, size, index, encoding, layout_engine)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
self.font = core.getfont(
OSError: unknown file format
0/399 6.98G 0.05195 0.01385 0.01754 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2427 0.187 0.0719 0.0511 0.0205
Epoch gpu_mem box obj cls labels img_size
1/399 9.2G 0.03571 0.006842 0.01135 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2427 0.121 0.0791 0.0413 0.0162
Epoch gpu_mem box obj cls labels img_size
2/399 9.2G 0.03472 0.006425 0.01224 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2427 0.0939 0.053 0.031 0.0109
Epoch gpu_mem box obj cls labels img_size
3/399 9.2G 0.0307 0.006224 0.01129 24 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2427 0.214 0.0892 0.0613 0.029
Epoch gpu_mem box obj cls labels img_size
4/399 9.2G 0.02659 0.005425 0.008707 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2427 0.0925 0.0747 0.0301 0.0113
Epoch gpu_mem box obj cls labels img_size
5/399 9.2G 0.02381 0.005077 0.007832 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2427 0.0844 0.079 0.0377 0.0163
Epoch gpu_mem box obj cls labels img_size
6/399 9.2G 0.02285 0.004648 0.007521 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2427 0.207 0.087 0.0697 0.0347
Epoch gpu_mem box obj cls labels img_size
7/399 9.2G 0.02234 0.004965 0.007593 26 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2427 0.156 0.103 0.0772 0.0399
Epoch gpu_mem box obj cls labels img_size
8/399 9.2G 0.02122 0.004713 0.006695 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2427 0.179 0.0988 0.0842 0.04
Epoch gpu_mem box obj cls labels img_size
9/399 9.2G 0.02034 0.004366 0.005866 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2427 0.153 0.0815 0.0636 0.0331
Saving model artifact on epoch 10
Epoch gpu_mem box obj cls labels img_size
10/399 9.2G 0.01941 0.004342 0.006107 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2427 0.194 0.095 0.0827 0.0439
Epoch gpu_mem box obj cls labels img_size
11/399 9.2G 0.01918 0.004101 0.005735 24 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2427 0.154 0.0839 0.0706 0.0416
Epoch gpu_mem box obj cls labels img_size
12/399 9.2G 0.0181 0.003967 0.005199 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2427 0.19 0.0964 0.11 0.0653
Epoch gpu_mem box obj cls labels img_size
13/399 9.2G 0.01823 0.003929 0.005318 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2427 0.15 0.0713 0.0738 0.0436
Epoch gpu_mem box obj cls labels img_size
14/399 9.2G 0.01752 0.003952 0.005218 24 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2158 0.199 0.0753 0.0958 0.0543
Epoch gpu_mem box obj cls labels img_size
15/399 9.2G 0.01708 0.003755 0.00511 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2427 0.254 0.102 0.118 0.0701
Epoch gpu_mem box obj cls labels img_size
16/399 9.2G 0.01718 0.003755 0.005107 25 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2427 0.208 0.109 0.106 0.0607
Epoch gpu_mem box obj cls labels img_size
17/399 9.2G 0.01676 0.003759 0.004576 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2427 0.204 0.101 0.121 0.0704
Epoch gpu_mem box obj cls labels img_size
18/399 9.2G 0.01627 0.003702 0.004592 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2426 0.241 0.0748 0.0899 0.0513
Epoch gpu_mem box obj cls labels img_size
19/399 9.2G 0.01645 0.003772 0.004737 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2427 0.269 0.0929 0.1 0.0569
Saving model artifact on epoch 20
Epoch gpu_mem box obj cls labels img_size
20/399 9.2G 0.01617 0.00374 0.004693 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2427 0.228 0.114 0.125 0.0711
Epoch gpu_mem box obj cls labels img_size
21/399 9.2G 0.01616 0.003556 0.004621 13 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2391 0.131 0.0835 0.084 0.0484
Epoch gpu_mem box obj cls labels img_size
22/399 9.2G 0.01536 0.003565 0.004141 25 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2420 0.183 0.108 0.125 0.0726
Epoch gpu_mem box obj cls labels img_size
23/399 9.2G 0.0154 0.003548 0.004181 13 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2422 0.273 0.0922 0.114 0.0674
Epoch gpu_mem box obj cls labels img_size
24/399 9.2G 0.015 0.003366 0.003581 26 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2427 0.196 0.081 0.0991 0.0599
Epoch gpu_mem box obj cls labels img_size
25/399 9.2G 0.01565 0.003554 0.004118 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2427 0.229 0.0963 0.1 0.0611
Epoch gpu_mem box obj cls labels img_size
26/399 9.2G 0.015 0.003456 0.003943 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2426 0.255 0.096 0.132 0.0802
Epoch gpu_mem box obj cls labels img_size
27/399 9.2G 0.0151 0.00335 0.00368 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2422 0.207 0.103 0.121 0.0692
Epoch gpu_mem box obj cls labels img_size
28/399 9.2G 0.01455 0.003383 0.003871 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2422 0.264 0.0837 0.108 0.0633
Epoch gpu_mem box obj cls labels img_size
29/399 9.2G 0.01468 0.003349 0.003974 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2405 0.241 0.0725 0.109 0.0656
Saving model artifact on epoch 30
Epoch gpu_mem box obj cls labels img_size
30/399 9.2G 0.01521 0.00346 0.004335 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2427 0.197 0.0977 0.106 0.06
Epoch gpu_mem box obj cls labels img_size
31/399 9.2G 0.01492 0.00342 0.003873 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2422 0.272 0.0936 0.127 0.0765
Epoch gpu_mem box obj cls labels img_size
32/399 9.2G 0.01478 0.003364 0.003692 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2388 0.234 0.0935 0.116 0.0661
Epoch gpu_mem box obj cls labels img_size
33/399 9.2G 0.01443 0.003262 0.003214 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2427 0.213 0.104 0.128 0.0764
Epoch gpu_mem box obj cls labels img_size
34/399 9.2G 0.0146 0.003459 0.00417 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2416 0.239 0.0929 0.113 0.0672
Epoch gpu_mem box obj cls labels img_size
35/399 9.2G 0.01421 0.003199 0.003387 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2385 0.216 0.0987 0.129 0.0776
Epoch gpu_mem box obj cls labels img_size
36/399 9.2G 0.01373 0.003157 0.002988 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2367 0.192 0.1 0.129 0.0806
Epoch gpu_mem box obj cls labels img_size
37/399 9.2G 0.01399 0.003213 0.003909 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2427 0.168 0.0971 0.101 0.0605
Epoch gpu_mem box obj cls labels img_size
38/399 9.2G 0.01422 0.00321 0.004248 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2408 0.276 0.093 0.145 0.088
Epoch gpu_mem box obj cls labels img_size
39/399 9.2G 0.01382 0.003105 0.003427 28 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2427 0.245 0.107 0.144 0.0874
Saving model artifact on epoch 40
Epoch gpu_mem box obj cls labels img_size
40/399 9.2G 0.01364 0.003135 0.003628 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2427 0.141 0.0888 0.0826 0.0481
Epoch gpu_mem box obj cls labels img_size
41/399 9.2G 0.01423 0.003189 0.003662 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2393 0.178 0.103 0.127 0.0816
Epoch gpu_mem box obj cls labels img_size
42/399 9.2G 0.01381 0.003071 0.003655 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2417 0.239 0.0852 0.118 0.0701
Epoch gpu_mem box obj cls labels img_size
43/399 9.2G 0.01366 0.00313 0.003485 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2427 0.27 0.0861 0.118 0.0732
Epoch gpu_mem box obj cls labels img_size
44/399 9.2G 0.01333 0.003069 0.003243 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2418 0.205 0.118 0.15 0.0908
Epoch gpu_mem box obj cls labels img_size
45/399 9.2G 0.01303 0.003062 0.002813 13 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2427 0.263 0.114 0.145 0.0867
Epoch gpu_mem box obj cls labels img_size
46/399 9.2G 0.01353 0.003025 0.003295 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2391 0.226 0.0999 0.124 0.0746
Epoch gpu_mem box obj cls labels img_size
47/399 9.2G 0.01302 0.002966 0.002772 25 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2418 0.19 0.0982 0.139 0.0906
Epoch gpu_mem box obj cls labels img_size
48/399 9.2G 0.01261 0.002949 0.002913 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2427 0.185 0.0997 0.131 0.0782
Epoch gpu_mem box obj cls labels img_size
49/399 9.2G 0.01271 0.002973 0.003026 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2415 0.198 0.105 0.139 0.0863
Saving model artifact on epoch 50
Epoch gpu_mem box obj cls labels img_size
50/399 9.2G 0.01301 0.002928 0.002895 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2422 0.234 0.105 0.138 0.0828
Epoch gpu_mem box obj cls labels img_size
51/399 9.2G 0.01291 0.002984 0.003079 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2421 0.209 0.103 0.144 0.0935
Epoch gpu_mem box obj cls labels img_size
52/399 9.2G 0.01286 0.002958 0.003183 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2425 0.228 0.102 0.134 0.0846
Epoch gpu_mem box obj cls labels img_size
53/399 9.2G 0.01313 0.00305 0.003825 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2427 0.311 0.0973 0.133 0.0773
Epoch gpu_mem box obj cls labels img_size
54/399 9.2G 0.01309 0.003017 0.003538 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2427 0.32 0.084 0.132 0.0803
Epoch gpu_mem box obj cls labels img_size
55/399 9.2G 0.01321 0.003031 0.003083 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2425 0.229 0.0882 0.128 0.0791
Epoch gpu_mem box obj cls labels img_size
56/399 9.2G 0.01272 0.002867 0.002874 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2359 0.287 0.0962 0.155 0.0975
Epoch gpu_mem box obj cls labels img_size
57/399 9.2G 0.0128 0.002856 0.002862 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2415 0.215 0.119 0.152 0.0944
Epoch gpu_mem box obj cls labels img_size
58/399 9.2G 0.0125 0.002921 0.003122 24 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2231 0.286 0.101 0.153 0.0994
Epoch gpu_mem box obj cls labels img_size
59/399 9.2G 0.01273 0.002851 0.002804 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2421 0.303 0.0891 0.152 0.0974
Saving model artifact on epoch 60
Epoch gpu_mem box obj cls labels img_size
60/399 9.2G 0.01241 0.002874 0.002819 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2425 0.284 0.0965 0.148 0.0921
Epoch gpu_mem box obj cls labels img_size
61/399 9.2G 0.01217 0.002864 0.002748 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2416 0.26 0.103 0.147 0.093
Epoch gpu_mem box obj cls labels img_size
62/399 9.2G 0.01211 0.002925 0.002696 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2424 0.274 0.0997 0.155 0.098
Epoch gpu_mem box obj cls labels img_size
63/399 9.2G 0.0124 0.002781 0.002586 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2374 0.246 0.104 0.167 0.109
Epoch gpu_mem box obj cls labels img_size
64/399 9.2G 0.01209 0.002794 0.002882 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2388 0.267 0.101 0.149 0.0939
Epoch gpu_mem box obj cls labels img_size
65/399 9.2G 0.01241 0.00286 0.002935 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2421 0.317 0.094 0.148 0.0926
Epoch gpu_mem box obj cls labels img_size
66/399 9.2G 0.01234 0.002773 0.002589 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2350 0.271 0.102 0.159 0.103
Epoch gpu_mem box obj cls labels img_size
67/399 9.2G 0.01221 0.002735 0.002446 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2350 0.277 0.0974 0.157 0.102
Epoch gpu_mem box obj cls labels img_size
68/399 9.2G 0.01229 0.002812 0.002777 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2332 0.224 0.108 0.154 0.0963
Epoch gpu_mem box obj cls labels img_size
69/399 9.2G 0.01261 0.002883 0.003159 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2375 0.3 0.0883 0.153 0.0976
Saving model artifact on epoch 70
Epoch gpu_mem box obj cls labels img_size
70/399 9.2G 0.01226 0.002804 0.00309 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2373 0.269 0.101 0.156 0.101
Epoch gpu_mem box obj cls labels img_size
71/399 9.2G 0.01236 0.002812 0.003062 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2373 0.227 0.104 0.154 0.0974
Epoch gpu_mem box obj cls labels img_size
72/399 9.2G 0.01201 0.002754 0.002737 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2420 0.28 0.0968 0.161 0.103
Epoch gpu_mem box obj cls labels img_size
73/399 9.2G 0.01211 0.00276 0.002646 26 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2420 0.231 0.114 0.161 0.1
Epoch gpu_mem box obj cls labels img_size
74/399 9.2G 0.01204 0.00285 0.002806 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2388 0.225 0.114 0.161 0.0995
Epoch gpu_mem box obj cls labels img_size
75/399 9.2G 0.01189 0.002789 0.00268 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2422 0.237 0.119 0.168 0.105
Epoch gpu_mem box obj cls labels img_size
76/399 9.2G 0.01191 0.002782 0.002518 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2384 0.225 0.114 0.16 0.0955
Epoch gpu_mem box obj cls labels img_size
77/399 9.2G 0.01224 0.002875 0.002801 13 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2359 0.27 0.105 0.159 0.0982
Epoch gpu_mem box obj cls labels img_size
78/399 9.2G 0.01185 0.002695 0.002346 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2354 0.278 0.107 0.165 0.103
Epoch gpu_mem box obj cls labels img_size
79/399 9.2G 0.01208 0.002799 0.002184 25 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2385 0.248 0.11 0.167 0.106
Saving model artifact on epoch 80
Epoch gpu_mem box obj cls labels img_size
80/399 9.2G 0.01142 0.002688 0.002473 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2205 0.259 0.109 0.173 0.107
Epoch gpu_mem box obj cls labels img_size
81/399 9.2G 0.01144 0.002635 0.002473 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2195 0.259 0.113 0.174 0.11
Epoch gpu_mem box obj cls labels img_size
82/399 9.2G 0.01179 0.002705 0.002419 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2285 0.243 0.108 0.166 0.106
Epoch gpu_mem box obj cls labels img_size
83/399 9.2G 0.01186 0.00268 0.002165 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2421 0.292 0.101 0.167 0.107
Epoch gpu_mem box obj cls labels img_size
84/399 9.2G 0.0114 0.002717 0.002682 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2265 0.296 0.107 0.165 0.106
Epoch gpu_mem box obj cls labels img_size
85/399 9.2G 0.01193 0.002828 0.002712 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2416 0.25 0.112 0.172 0.112
Epoch gpu_mem box obj cls labels img_size
86/399 9.2G 0.0114 0.002658 0.002416 25 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2385 0.253 0.108 0.171 0.111
Epoch gpu_mem box obj cls labels img_size
87/399 9.2G 0.01187 0.002757 0.002212 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2392 0.24 0.106 0.164 0.109
Epoch gpu_mem box obj cls labels img_size
88/399 9.2G 0.01183 0.002669 0.002601 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2424 0.245 0.109 0.168 0.109
Epoch gpu_mem box obj cls labels img_size
89/399 9.2G 0.01159 0.002708 0.002537 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2384 0.246 0.112 0.167 0.108
Saving model artifact on epoch 90
Epoch gpu_mem box obj cls labels img_size
90/399 9.2G 0.0116 0.002684 0.002358 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2354 0.234 0.111 0.163 0.106
Epoch gpu_mem box obj cls labels img_size
91/399 9.2G 0.01177 0.002632 0.00264 13 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2388 0.24 0.108 0.164 0.108
Epoch gpu_mem box obj cls labels img_size
92/399 9.2G 0.01194 0.002701 0.002627 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2320 0.242 0.111 0.166 0.109
Epoch gpu_mem box obj cls labels img_size
93/399 9.2G 0.01129 0.002649 0.002346 11 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2320 0.242 0.109 0.165 0.109
Epoch gpu_mem box obj cls labels img_size
94/399 9.2G 0.01175 0.002782 0.00287 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2320 0.241 0.11 0.166 0.109
Epoch gpu_mem box obj cls labels img_size
95/399 9.2G 0.01148 0.002609 0.002575 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2324 0.229 0.105 0.16 0.104
Epoch gpu_mem box obj cls labels img_size
96/399 9.2G 0.01159 0.002784 0.002723 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2321 0.231 0.104 0.16 0.105
Epoch gpu_mem box obj cls labels img_size
97/399 9.2G 0.01188 0.002804 0.00305 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2289 0.237 0.106 0.163 0.106
Epoch gpu_mem box obj cls labels img_size
98/399 9.2G 0.01152 0.002627 0.002217 26 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2289 0.241 0.106 0.164 0.107
Epoch gpu_mem box obj cls labels img_size
99/399 9.2G 0.01171 0.002737 0.002578 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2288 0.238 0.106 0.164 0.108
Saving model artifact on epoch 100
Epoch gpu_mem box obj cls labels img_size
100/399 9.2G 0.0117 0.002715 0.002309 8 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2289 0.308 0.0942 0.164 0.107
Epoch gpu_mem box obj cls labels img_size
101/399 9.2G 0.01139 0.00269 0.002148 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2359 0.236 0.107 0.164 0.106
Epoch gpu_mem box obj cls labels img_size
102/399 9.2G 0.01141 0.002645 0.002597 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2354 0.251 0.11 0.171 0.111
Epoch gpu_mem box obj cls labels img_size
103/399 9.2G 0.01125 0.002579 0.002418 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2319 0.24 0.105 0.166 0.109
Epoch gpu_mem box obj cls labels img_size
104/399 9.2G 0.01131 0.002623 0.002569 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2319 0.239 0.105 0.165 0.108
Epoch gpu_mem box obj cls labels img_size
105/399 9.2G 0.01126 0.002651 0.002122 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2319 0.24 0.106 0.166 0.108
Epoch gpu_mem box obj cls labels img_size
106/399 9.2G 0.01092 0.002601 0.002439 27 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2321 0.247 0.109 0.17 0.109
Epoch gpu_mem box obj cls labels img_size
107/399 9.2G 0.0111 0.002605 0.00203 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2319 0.25 0.11 0.172 0.11
Epoch gpu_mem box obj cls labels img_size
108/399 9.2G 0.01104 0.002629 0.002213 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2319 0.248 0.106 0.17 0.11
Epoch gpu_mem box obj cls labels img_size
109/399 9.2G 0.01098 0.002533 0.002034 26 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2319 0.251 0.107 0.171 0.11
Saving model artifact on epoch 110
Epoch gpu_mem box obj cls labels img_size
110/399 9.2G 0.01095 0.00257 0.002379 9 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2317 0.242 0.102 0.165 0.109
Epoch gpu_mem box obj cls labels img_size
111/399 9.2G 0.01103 0.002539 0.002005 24 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2308 0.25 0.105 0.171 0.112
Epoch gpu_mem box obj cls labels img_size
112/399 9.2G 0.01073 0.002573 0.001918 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2303 0.29 0.0993 0.172 0.114
Epoch gpu_mem box obj cls labels img_size
113/399 9.2G 0.01116 0.002679 0.00222 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2308 0.323 0.0936 0.168 0.111
Epoch gpu_mem box obj cls labels img_size
114/399 9.2G 0.01109 0.002618 0.002479 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2310 0.247 0.103 0.17 0.112
Epoch gpu_mem box obj cls labels img_size
115/399 9.2G 0.0106 0.002601 0.002138 11 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2310 0.287 0.0978 0.169 0.113
Epoch gpu_mem box obj cls labels img_size
116/399 9.2G 0.01126 0.002679 0.002218 12 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2310 0.288 0.0987 0.17 0.114
Epoch gpu_mem box obj cls labels img_size
117/399 9.2G 0.01097 0.002621 0.003072 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2310 0.253 0.104 0.173 0.115
Epoch gpu_mem box obj cls labels img_size
118/399 9.2G 0.01065 0.002554 0.002154 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2310 0.253 0.104 0.173 0.114
Epoch gpu_mem box obj cls labels img_size
119/399 9.2G 0.01084 0.002492 0.002094 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.295 0.0958 0.172 0.115
Saving model artifact on epoch 120
Epoch gpu_mem box obj cls labels img_size
120/399 9.2G 0.01069 0.002568 0.002021 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2314 0.254 0.103 0.173 0.116
Epoch gpu_mem box obj cls labels img_size
121/399 9.2G 0.01118 0.002552 0.002643 25 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2310 0.254 0.103 0.174 0.117
Epoch gpu_mem box obj cls labels img_size
122/399 9.2G 0.01077 0.002504 0.002078 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2310 0.259 0.104 0.176 0.118
Epoch gpu_mem box obj cls labels img_size
123/399 9.2G 0.01076 0.002523 0.00207 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2310 0.257 0.103 0.176 0.118
Epoch gpu_mem box obj cls labels img_size
124/399 9.2G 0.01063 0.002466 0.002084 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2310 0.255 0.104 0.176 0.117
Epoch gpu_mem box obj cls labels img_size
125/399 9.2G 0.01074 0.00243 0.002315 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2352 0.258 0.105 0.176 0.117
Epoch gpu_mem box obj cls labels img_size
126/399 9.2G 0.01062 0.002519 0.001998 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2352 0.251 0.104 0.173 0.116
Epoch gpu_mem box obj cls labels img_size
127/399 9.2G 0.01065 0.002556 0.002265 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2352 0.252 0.106 0.174 0.115
Epoch gpu_mem box obj cls labels img_size
128/399 9.2G 0.0107 0.002528 0.002178 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2353 0.251 0.103 0.174 0.116
Epoch gpu_mem box obj cls labels img_size
129/399 9.2G 0.01085 0.002415 0.002086 26 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2352 0.299 0.0971 0.174 0.116
Saving model artifact on epoch 130
Epoch gpu_mem box obj cls labels img_size
130/399 9.2G 0.01077 0.002565 0.00246 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2353 0.251 0.103 0.174 0.115
Epoch gpu_mem box obj cls labels img_size
131/399 9.2G 0.01087 0.002593 0.002324 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2354 0.358 0.091 0.173 0.114
Epoch gpu_mem box obj cls labels img_size
132/399 9.2G 0.01062 0.002469 0.002301 11 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2316 0.254 0.106 0.176 0.117
Epoch gpu_mem box obj cls labels img_size
133/399 9.2G 0.0108 0.002598 0.00269 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2316 0.255 0.105 0.176 0.117
Epoch gpu_mem box obj cls labels img_size
134/399 9.2G 0.01083 0.002626 0.002465 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2316 0.259 0.106 0.178 0.119
Epoch gpu_mem box obj cls labels img_size
135/399 9.2G 0.0105 0.0025 0.002538 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2307 0.258 0.106 0.179 0.118
Epoch gpu_mem box obj cls labels img_size
136/399 9.2G 0.01067 0.002623 0.002169 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2307 0.259 0.106 0.179 0.119
Epoch gpu_mem box obj cls labels img_size
137/399 9.2G 0.0108 0.00263 0.002811 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2306 0.259 0.107 0.179 0.119
Epoch gpu_mem box obj cls labels img_size
138/399 9.2G 0.01089 0.002515 0.002264 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2306 0.256 0.107 0.177 0.118
Epoch gpu_mem box obj cls labels img_size
139/399 9.2G 0.01083 0.002443 0.002035 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2306 0.253 0.107 0.176 0.118
Saving model artifact on epoch 140
Epoch gpu_mem box obj cls labels img_size
140/399 9.2G 0.01062 0.002469 0.002137 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2306 0.253 0.107 0.176 0.117
Epoch gpu_mem box obj cls labels img_size
141/399 9.2G 0.01037 0.002472 0.001742 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2306 0.254 0.106 0.176 0.118
Epoch gpu_mem box obj cls labels img_size
142/399 9.2G 0.01032 0.002483 0.001813 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2306 0.257 0.106 0.177 0.119
Epoch gpu_mem box obj cls labels img_size
143/399 9.2G 0.01047 0.00241 0.00191 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2306 0.256 0.105 0.177 0.119
Epoch gpu_mem box obj cls labels img_size
144/399 9.2G 0.01027 0.002417 0.002108 27 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2306 0.257 0.105 0.177 0.12
Epoch gpu_mem box obj cls labels img_size
145/399 9.2G 0.01033 0.00238 0.001987 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2306 0.263 0.107 0.18 0.121
Epoch gpu_mem box obj cls labels img_size
146/399 9.2G 0.01053 0.002388 0.001843 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2306 0.262 0.105 0.179 0.121
Epoch gpu_mem box obj cls labels img_size
147/399 9.2G 0.01035 0.002412 0.001936 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2306 0.263 0.105 0.18 0.12
Epoch gpu_mem box obj cls labels img_size
148/399 9.2G 0.01033 0.00238 0.001991 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2306 0.264 0.105 0.18 0.12
Epoch gpu_mem box obj cls labels img_size
149/399 9.2G 0.01027 0.002462 0.002334 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2306 0.265 0.105 0.18 0.12
Saving model artifact on epoch 150
Epoch gpu_mem box obj cls labels img_size
150/399 9.2G 0.01048 0.002566 0.002612 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2306 0.263 0.105 0.179 0.119
Epoch gpu_mem box obj cls labels img_size
151/399 9.2G 0.01036 0.00246 0.00223 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2306 0.26 0.105 0.178 0.119
Epoch gpu_mem box obj cls labels img_size
152/399 9.2G 0.01038 0.002528 0.002362 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.258 0.104 0.177 0.118
Epoch gpu_mem box obj cls labels img_size
153/399 9.2G 0.01013 0.002399 0.002019 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.301 0.0986 0.176 0.117
Epoch gpu_mem box obj cls labels img_size
154/399 9.2G 0.01045 0.002368 0.00187 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.257 0.104 0.177 0.118
Epoch gpu_mem box obj cls labels img_size
155/399 9.2G 0.01029 0.002408 0.002137 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.301 0.0986 0.177 0.118
Epoch gpu_mem box obj cls labels img_size
156/399 9.2G 0.01043 0.002433 0.001898 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.26 0.106 0.178 0.119
Epoch gpu_mem box obj cls labels img_size
157/399 9.2G 0.01033 0.002486 0.002655 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.26 0.105 0.178 0.12
Epoch gpu_mem box obj cls labels img_size
158/399 9.2G 0.0105 0.002406 0.001983 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.261 0.105 0.179 0.119
Epoch gpu_mem box obj cls labels img_size
159/399 9.2G 0.009979 0.002396 0.001926 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.258 0.105 0.177 0.119
Saving model artifact on epoch 160
Epoch gpu_mem box obj cls labels img_size
160/399 9.2G 0.01008 0.002384 0.002058 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.262 0.106 0.179 0.12
Epoch gpu_mem box obj cls labels img_size
161/399 9.2G 0.009937 0.002321 0.001517 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.261 0.105 0.179 0.12
Epoch gpu_mem box obj cls labels img_size
162/399 9.2G 0.009874 0.002334 0.001454 13 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.261 0.106 0.179 0.12
Epoch gpu_mem box obj cls labels img_size
163/399 9.2G 0.01014 0.002399 0.002104 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.26 0.106 0.178 0.12
Epoch gpu_mem box obj cls labels img_size
164/399 9.2G 0.009914 0.002383 0.002057 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.26 0.106 0.179 0.12
Epoch gpu_mem box obj cls labels img_size
165/399 9.2G 0.01032 0.002353 0.001813 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.26 0.106 0.179 0.121
Epoch gpu_mem box obj cls labels img_size
166/399 9.2G 0.009724 0.002316 0.001739 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.262 0.106 0.18 0.121
Epoch gpu_mem box obj cls labels img_size
167/399 9.2G 0.009843 0.002335 0.001921 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.265 0.106 0.181 0.122
Epoch gpu_mem box obj cls labels img_size
168/399 9.2G 0.009971 0.002361 0.001889 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.264 0.106 0.181 0.121
Epoch gpu_mem box obj cls labels img_size
169/399 9.2G 0.009809 0.002393 0.001936 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.263 0.106 0.18 0.121
Saving model artifact on epoch 170
Epoch gpu_mem box obj cls labels img_size
170/399 9.2G 0.009638 0.002315 0.001974 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.264 0.106 0.181 0.122
Epoch gpu_mem box obj cls labels img_size
171/399 9.2G 0.009929 0.002476 0.001993 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.265 0.106 0.181 0.122
Epoch gpu_mem box obj cls labels img_size
172/399 9.2G 0.01044 0.002517 0.002467 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.266 0.106 0.182 0.123
Epoch gpu_mem box obj cls labels img_size
173/399 9.2G 0.009787 0.002365 0.002216 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.264 0.105 0.18 0.122
Epoch gpu_mem box obj cls labels img_size
174/399 9.2G 0.01028 0.002392 0.00167 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.264 0.105 0.181 0.122
Epoch gpu_mem box obj cls labels img_size
175/399 9.2G 0.009986 0.002406 0.002047 10 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.313 0.0992 0.18 0.122
Epoch gpu_mem box obj cls labels img_size
176/399 9.2G 0.009719 0.002338 0.001897 12 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.314 0.0992 0.181 0.122
Epoch gpu_mem box obj cls labels img_size
177/399 9.2G 0.009544 0.002311 0.001965 24 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.267 0.105 0.182 0.122
Epoch gpu_mem box obj cls labels img_size
178/399 9.2G 0.01016 0.002412 0.002496 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.27 0.106 0.183 0.123
Epoch gpu_mem box obj cls labels img_size
179/399 9.2G 0.009989 0.002376 0.002209 12 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.269 0.106 0.183 0.123
Saving model artifact on epoch 180
Epoch gpu_mem box obj cls labels img_size
180/399 9.2G 0.009992 0.002444 0.002375 13 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.27 0.106 0.184 0.123
Epoch gpu_mem box obj cls labels img_size
181/399 9.2G 0.009912 0.002351 0.001608 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.271 0.106 0.184 0.124
Epoch gpu_mem box obj cls labels img_size
182/399 9.2G 0.009964 0.002497 0.002372 13 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.272 0.106 0.185 0.124
Epoch gpu_mem box obj cls labels img_size
183/399 9.2G 0.01006 0.002496 0.00215 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2343 0.272 0.107 0.185 0.124
Epoch gpu_mem box obj cls labels img_size
184/399 9.2G 0.009941 0.002403 0.002048 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2343 0.271 0.107 0.184 0.124
Epoch gpu_mem box obj cls labels img_size
185/399 9.2G 0.009831 0.002338 0.00207 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2344 0.267 0.106 0.182 0.123
Epoch gpu_mem box obj cls labels img_size
186/399 9.2G 0.009627 0.002336 0.001817 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2344 0.264 0.106 0.182 0.123
Epoch gpu_mem box obj cls labels img_size
187/399 9.2G 0.00972 0.002317 0.002335 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2378 0.263 0.106 0.181 0.122
Epoch gpu_mem box obj cls labels img_size
188/399 9.2G 0.009366 0.002289 0.001727 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2378 0.262 0.106 0.18 0.122
Epoch gpu_mem box obj cls labels img_size
189/399 9.2G 0.009583 0.002295 0.001756 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2378 0.262 0.106 0.181 0.122
Saving model artifact on epoch 190
Epoch gpu_mem box obj cls labels img_size
190/399 9.2G 0.009561 0.002294 0.00158 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2378 0.262 0.106 0.18 0.122
Epoch gpu_mem box obj cls labels img_size
191/399 9.2G 0.009537 0.002352 0.001817 11 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2378 0.262 0.106 0.18 0.121
Epoch gpu_mem box obj cls labels img_size
192/399 9.2G 0.009504 0.002304 0.00193 13 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2378 0.263 0.106 0.18 0.122
Epoch gpu_mem box obj cls labels img_size
193/399 9.2G 0.009558 0.002308 0.00195 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2378 0.262 0.106 0.18 0.122
Epoch gpu_mem box obj cls labels img_size
194/399 9.2G 0.009593 0.002374 0.001824 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2378 0.261 0.106 0.18 0.121
Epoch gpu_mem box obj cls labels img_size
195/399 9.2G 0.00936 0.002261 0.001688 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2378 0.261 0.106 0.18 0.121
Epoch gpu_mem box obj cls labels img_size
196/399 9.2G 0.009671 0.002312 0.001989 29 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2378 0.263 0.106 0.181 0.121
Epoch gpu_mem box obj cls labels img_size
197/399 9.2G 0.009574 0.002263 0.001606 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2378 0.261 0.105 0.181 0.121
Epoch gpu_mem box obj cls labels img_size
198/399 9.2G 0.009402 0.002238 0.001842 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2378 0.26 0.105 0.18 0.121
Epoch gpu_mem box obj cls labels img_size
199/399 9.2G 0.009254 0.002232 0.001389 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2378 0.262 0.106 0.181 0.121
Saving model artifact on epoch 200
Epoch gpu_mem box obj cls labels img_size
200/399 9.2G 0.009264 0.002181 0.001486 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2378 0.265 0.106 0.182 0.122
Epoch gpu_mem box obj cls labels img_size
201/399 9.2G 0.009593 0.002244 0.001658 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2378 0.267 0.106 0.183 0.122
Epoch gpu_mem box obj cls labels img_size
202/399 9.2G 0.009096 0.00217 0.001643 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2378 0.267 0.106 0.183 0.122
Epoch gpu_mem box obj cls labels img_size
203/399 9.2G 0.009489 0.002366 0.001731 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2378 0.265 0.105 0.182 0.122
Epoch gpu_mem box obj cls labels img_size
204/399 9.2G 0.009366 0.002187 0.001954 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2378 0.265 0.105 0.182 0.122
Epoch gpu_mem box obj cls labels img_size
205/399 9.2G 0.009141 0.002211 0.001711 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2378 0.267 0.105 0.183 0.123
Epoch gpu_mem box obj cls labels img_size
206/399 9.2G 0.009221 0.002175 0.001943 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2378 0.269 0.105 0.184 0.123
Epoch gpu_mem box obj cls labels img_size
207/399 9.2G 0.008874 0.0022 0.001353 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2378 0.269 0.105 0.184 0.123
Epoch gpu_mem box obj cls labels img_size
208/399 9.2G 0.008863 0.002123 0.001214 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2378 0.267 0.104 0.183 0.123
Epoch gpu_mem box obj cls labels img_size
209/399 9.2G 0.009238 0.002185 0.001462 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2378 0.267 0.104 0.183 0.123
Saving model artifact on epoch 210
Epoch gpu_mem box obj cls labels img_size
210/399 9.2G 0.009182 0.002201 0.001253 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2344 0.267 0.104 0.184 0.123
Epoch gpu_mem box obj cls labels img_size
211/399 9.2G 0.009103 0.002089 0.001567 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2344 0.266 0.103 0.183 0.124
Epoch gpu_mem box obj cls labels img_size
212/399 9.2G 0.009057 0.002242 0.00152 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2344 0.266 0.103 0.183 0.124
Epoch gpu_mem box obj cls labels img_size
213/399 9.2G 0.009007 0.0021 0.001552 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2344 0.267 0.103 0.183 0.124
Epoch gpu_mem box obj cls labels img_size
214/399 9.2G 0.009465 0.002178 0.00178 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2344 0.267 0.103 0.184 0.124
Epoch gpu_mem box obj cls labels img_size
215/399 9.2G 0.009296 0.002271 0.001936 24 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2344 0.267 0.103 0.184 0.124
Epoch gpu_mem box obj cls labels img_size
216/399 9.2G 0.00906 0.002157 0.001414 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2344 0.267 0.103 0.184 0.124
Epoch gpu_mem box obj cls labels img_size
217/399 9.2G 0.008916 0.002089 0.001446 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2344 0.268 0.103 0.184 0.125
Epoch gpu_mem box obj cls labels img_size
218/399 9.2G 0.008751 0.002178 0.00126 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2344 0.269 0.102 0.184 0.125
Epoch gpu_mem box obj cls labels img_size
219/399 9.2G 0.009105 0.002115 0.001553 30 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2344 0.27 0.102 0.185 0.125
Saving model artifact on epoch 220
Epoch gpu_mem box obj cls labels img_size
220/399 9.2G 0.008688 0.002111 0.001468 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2344 0.27 0.102 0.185 0.126
Epoch gpu_mem box obj cls labels img_size
221/399 9.2G 0.008771 0.002074 0.001122 24 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2344 0.272 0.102 0.186 0.126
Epoch gpu_mem box obj cls labels img_size
222/399 9.2G 0.008945 0.002198 0.001865 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2344 0.271 0.101 0.185 0.126
Epoch gpu_mem box obj cls labels img_size
223/399 9.2G 0.009228 0.002239 0.001902 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2344 0.269 0.101 0.184 0.125
Epoch gpu_mem box obj cls labels img_size
224/399 9.2G 0.009177 0.002232 0.002092 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2344 0.27 0.101 0.185 0.125
Epoch gpu_mem box obj cls labels img_size
225/399 9.2G 0.009061 0.002228 0.00159 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2344 0.269 0.101 0.184 0.125
Epoch gpu_mem box obj cls labels img_size
226/399 9.2G 0.009079 0.002188 0.001625 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2344 0.27 0.101 0.184 0.125
Epoch gpu_mem box obj cls labels img_size
227/399 9.2G 0.008974 0.002133 0.001571 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2344 0.27 0.101 0.184 0.126
Epoch gpu_mem box obj cls labels img_size
228/399 9.2G 0.009469 0.002309 0.002333 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2344 0.272 0.101 0.186 0.128
Epoch gpu_mem box obj cls labels img_size
229/399 9.2G 0.009259 0.002251 0.00178 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2344 0.273 0.101 0.186 0.128
Saving model artifact on epoch 230
Epoch gpu_mem box obj cls labels img_size
230/399 9.2G 0.008872 0.002258 0.001741 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2344 0.274 0.101 0.186 0.128
Epoch gpu_mem box obj cls labels img_size
231/399 9.2G 0.009059 0.002165 0.001684 24 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2343 0.274 0.101 0.186 0.128
Epoch gpu_mem box obj cls labels img_size
232/399 9.2G 0.00911 0.002187 0.001699 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2343 0.276 0.101 0.187 0.128
Epoch gpu_mem box obj cls labels img_size
233/399 9.2G 0.00895 0.002173 0.001491 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2343 0.274 0.101 0.186 0.128
Epoch gpu_mem box obj cls labels img_size
234/399 9.2G 0.008932 0.002165 0.001499 25 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2343 0.278 0.101 0.188 0.129
Epoch gpu_mem box obj cls labels img_size
235/399 9.2G 0.008927 0.002225 0.001792 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2343 0.278 0.101 0.188 0.128
Epoch gpu_mem box obj cls labels img_size
236/399 9.2G 0.008872 0.002148 0.001606 24 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2343 0.276 0.1 0.187 0.127
Epoch gpu_mem box obj cls labels img_size
237/399 9.2G 0.008838 0.002121 0.001468 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2343 0.324 0.0929 0.185 0.127
Epoch gpu_mem box obj cls labels img_size
238/399 9.2G 0.008457 0.00208 0.001275 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2343 0.323 0.0927 0.184 0.127
Epoch gpu_mem box obj cls labels img_size
239/399 9.2G 0.008669 0.002106 0.001376 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2343 0.326 0.0926 0.185 0.127
Saving model artifact on epoch 240
Epoch gpu_mem box obj cls labels img_size
240/399 9.2G 0.008485 0.002119 0.001613 26 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2343 0.273 0.0984 0.185 0.127
Epoch gpu_mem box obj cls labels img_size
241/399 9.2G 0.008555 0.002091 0.001401 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2343 0.327 0.0923 0.186 0.128
Epoch gpu_mem box obj cls labels img_size
242/399 9.2G 0.008459 0.002025 0.001101 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2343 0.327 0.0923 0.187 0.128
Epoch gpu_mem box obj cls labels img_size
243/399 9.2G 0.008484 0.002094 0.00179 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2343 0.331 0.0923 0.187 0.128
Epoch gpu_mem box obj cls labels img_size
244/399 9.2G 0.008852 0.002086 0.001485 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2343 0.334 0.0923 0.188 0.129
Epoch gpu_mem box obj cls labels img_size
245/399 9.2G 0.008688 0.002072 0.001131 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2343 0.334 0.0923 0.188 0.129
Epoch gpu_mem box obj cls labels img_size
246/399 9.2G 0.008535 0.002043 0.001541 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2343 0.336 0.0923 0.188 0.129
Epoch gpu_mem box obj cls labels img_size
247/399 9.2G 0.008294 0.002058 0.001425 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2343 0.336 0.0923 0.189 0.13
Epoch gpu_mem box obj cls labels img_size
248/399 9.2G 0.008318 0.002045 0.001404 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2343 0.337 0.0923 0.19 0.131
Epoch gpu_mem box obj cls labels img_size
249/399 9.2G 0.008423 0.002077 0.001566 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2343 0.337 0.0923 0.19 0.131
Saving model artifact on epoch 250
Epoch gpu_mem box obj cls labels img_size
250/399 9.2G 0.008367 0.002016 0.001307 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2343 0.338 0.0917 0.19 0.132
Epoch gpu_mem box obj cls labels img_size
251/399 9.2G 0.008442 0.002078 0.001261 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2343 0.338 0.0917 0.191 0.132
Epoch gpu_mem box obj cls labels img_size
252/399 9.2G 0.008425 0.002047 0.001463 12 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2342 0.34 0.0919 0.192 0.133
Epoch gpu_mem box obj cls labels img_size
253/399 9.2G 0.008113 0.002027 0.00118 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2342 0.289 0.0964 0.193 0.134
Epoch gpu_mem box obj cls labels img_size
254/399 9.2G 0.00827 0.002021 0.001105 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2342 0.339 0.0919 0.194 0.134
Epoch gpu_mem box obj cls labels img_size
255/399 9.2G 0.008336 0.00202 0.001452 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2342 0.29 0.0962 0.194 0.134
Epoch gpu_mem box obj cls labels img_size
256/399 9.2G 0.008159 0.001968 0.001212 8 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2342 0.291 0.0959 0.194 0.135
Epoch gpu_mem box obj cls labels img_size
257/399 9.2G 0.008061 0.001922 0.001033 10 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2342 0.291 0.0959 0.194 0.134
Epoch gpu_mem box obj cls labels img_size
258/399 9.2G 0.008071 0.001973 0.001161 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2342 0.292 0.0956 0.195 0.134
Epoch gpu_mem box obj cls labels img_size
259/399 9.2G 0.007981 0.001991 0.001136 27 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2342 0.293 0.0956 0.195 0.134
Saving model artifact on epoch 260
Epoch gpu_mem box obj cls labels img_size
260/399 9.2G 0.008428 0.002133 0.001609 27 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2342 0.295 0.0956 0.196 0.135
Epoch gpu_mem box obj cls labels img_size
261/399 9.2G 0.008412 0.002072 0.001286 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2342 0.294 0.0947 0.195 0.135
Epoch gpu_mem box obj cls labels img_size
262/399 9.2G 0.008125 0.001962 0.001194 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2342 0.296 0.0947 0.196 0.136
Epoch gpu_mem box obj cls labels img_size
263/399 9.2G 0.008374 0.002018 0.001224 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2342 0.296 0.0947 0.196 0.136
Epoch gpu_mem box obj cls labels img_size
264/399 9.2G 0.008671 0.002116 0.001618 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2342 0.3 0.0956 0.199 0.137
Epoch gpu_mem box obj cls labels img_size
265/399 9.2G 0.008182 0.002026 0.001381 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2342 0.303 0.0956 0.2 0.137
Epoch gpu_mem box obj cls labels img_size
266/399 9.2G 0.008124 0.002004 0.001196 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2342 0.302 0.0953 0.2 0.138
Epoch gpu_mem box obj cls labels img_size
267/399 9.2G 0.008179 0.002046 0.001321 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2342 0.307 0.0956 0.202 0.139
Epoch gpu_mem box obj cls labels img_size
268/399 9.2G 0.008015 0.002025 0.001117 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2338 0.309 0.096 0.203 0.139
Epoch gpu_mem box obj cls labels img_size
269/399 9.2G 0.007923 0.001971 0.001229 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2338 0.308 0.096 0.203 0.139
Saving model artifact on epoch 270
Epoch gpu_mem box obj cls labels img_size
270/399 9.2G 0.008097 0.001982 0.001088 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2337 0.312 0.0961 0.205 0.141
Epoch gpu_mem box obj cls labels img_size
271/399 9.2G 0.008258 0.002001 0.001455 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2337 0.311 0.0961 0.205 0.141
Epoch gpu_mem box obj cls labels img_size
272/399 9.2G 0.008153 0.002042 0.001437 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2336 0.312 0.0961 0.205 0.141
Epoch gpu_mem box obj cls labels img_size
273/399 9.2G 0.0081 0.002032 0.001274 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2337 0.311 0.0958 0.205 0.141
Epoch gpu_mem box obj cls labels img_size
274/399 9.2G 0.007822 0.001935 0.001082 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2337 0.31 0.0958 0.205 0.141
Epoch gpu_mem box obj cls labels img_size
275/399 9.2G 0.007881 0.001987 0.001262 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2337 0.309 0.0943 0.204 0.141
Epoch gpu_mem box obj cls labels img_size
276/399 9.2G 0.007868 0.001966 0.001114 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2337 0.309 0.0943 0.204 0.141
Epoch gpu_mem box obj cls labels img_size
277/399 9.2G 0.007687 0.001919 0.001133 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2337 0.311 0.0946 0.205 0.141
Epoch gpu_mem box obj cls labels img_size
278/399 9.2G 0.007861 0.001899 0.001114 26 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2337 0.312 0.0937 0.206 0.142
Epoch gpu_mem box obj cls labels img_size
279/399 9.2G 0.008121 0.001952 0.00108 25 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2337 0.312 0.0937 0.206 0.142
Saving model artifact on epoch 280
Epoch gpu_mem box obj cls labels img_size
280/399 9.2G 0.007912 0.00192 0.001067 13 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2337 0.311 0.0928 0.205 0.142
Epoch gpu_mem box obj cls labels img_size
281/399 9.2G 0.007777 0.001856 0.001122 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2337 0.313 0.0928 0.206 0.142
Epoch gpu_mem box obj cls labels img_size
282/399 9.2G 0.00773 0.001956 0.001046 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2337 0.314 0.0925 0.207 0.143
Epoch gpu_mem box obj cls labels img_size
283/399 9.2G 0.007739 0.001948 0.001149 30 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2337 0.316 0.0925 0.208 0.143
Epoch gpu_mem box obj cls labels img_size
284/399 9.2G 0.007587 0.001902 0.001049 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2337 0.316 0.0925 0.208 0.143
Epoch gpu_mem box obj cls labels img_size
285/399 9.2G 0.0077 0.001888 0.001062 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2337 0.316 0.0925 0.208 0.143
Epoch gpu_mem box obj cls labels img_size
286/399 9.2G 0.007673 0.001917 0.001231 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2337 0.32 0.0922 0.21 0.145
Epoch gpu_mem box obj cls labels img_size
287/399 9.2G 0.007619 0.001886 0.0009627 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2336 0.318 0.0907 0.208 0.144
Epoch gpu_mem box obj cls labels img_size
288/399 9.2G 0.007529 0.00188 0.001208 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2336 0.321 0.091 0.21 0.146
Epoch gpu_mem box obj cls labels img_size
289/399 9.2G 0.007558 0.001931 0.001348 12 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2336 0.32 0.091 0.209 0.145
Saving model artifact on epoch 290
Epoch gpu_mem box obj cls labels img_size
290/399 9.2G 0.007876 0.001914 0.001104 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2335 0.324 0.091 0.211 0.146
Epoch gpu_mem box obj cls labels img_size
291/399 9.2G 0.007687 0.001884 0.0009966 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2334 0.327 0.092 0.213 0.147
Epoch gpu_mem box obj cls labels img_size
292/399 9.2G 0.007494 0.001852 0.00111 13 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2334 0.331 0.0929 0.215 0.148
Epoch gpu_mem box obj cls labels img_size
293/399 9.2G 0.007536 0.001921 0.001189 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2323 0.327 0.092 0.213 0.147
Epoch gpu_mem box obj cls labels img_size
294/399 9.2G 0.007629 0.001897 0.001018 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2323 0.328 0.092 0.213 0.148
Epoch gpu_mem box obj cls labels img_size
295/399 9.2G 0.007571 0.001822 0.0009447 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2323 0.327 0.092 0.213 0.147
Epoch gpu_mem box obj cls labels img_size
296/399 9.2G 0.007635 0.001915 0.0009266 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2323 0.328 0.0917 0.213 0.148
Epoch gpu_mem box obj cls labels img_size
297/399 9.2G 0.007167 0.001869 0.001082 28 640: 42%|███wandb: Network error (ProxyError), entering retry loop.
297/399 9.2G 0.007456 0.001868 0.001138 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2323 0.323 0.0893 0.211 0.147
Epoch gpu_mem box obj cls labels img_size
298/399 9.2G 0.007383 0.00183 0.001052 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2323 0.323 0.0893 0.211 0.147
Epoch gpu_mem box obj cls labels img_size
299/399 9.2G 0.007487 0.001857 0.001159 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2323 0.324 0.0893 0.211 0.147
Saving model artifact on epoch 300
Epoch gpu_mem box obj cls labels img_size
300/399 9.2G 0.007437 0.001881 0.001196 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2323 0.324 0.0893 0.211 0.147
Epoch gpu_mem box obj cls labels img_size
301/399 9.2G 0.007374 0.00184 0.0009381 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2323 0.324 0.0893 0.212 0.147
Epoch gpu_mem box obj cls labels img_size
302/399 9.2G 0.007469 0.001909 0.001005 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2323 0.323 0.0893 0.211 0.147
Epoch gpu_mem box obj cls labels img_size
303/399 9.2G 0.007069 0.001792 0.0009178 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2323 0.325 0.0893 0.212 0.148
Epoch gpu_mem box obj cls labels img_size
304/399 9.2G 0.007136 0.001899 0.0009029 28 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2323 0.327 0.0893 0.213 0.148
Epoch gpu_mem box obj cls labels img_size
305/399 9.2G 0.00719 0.001809 0.0008898 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2318 0.328 0.0898 0.214 0.149
Epoch gpu_mem box obj cls labels img_size
306/399 9.2G 0.007297 0.001878 0.001402 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2318 0.33 0.0898 0.215 0.149
Epoch gpu_mem box obj cls labels img_size
307/399 9.2G 0.007139 0.001792 0.0008135 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2318 0.33 0.0898 0.215 0.149
Epoch gpu_mem box obj cls labels img_size
308/399 9.2G 0.007251 0.001841 0.001199 28 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2318 0.33 0.0895 0.215 0.15
Epoch gpu_mem box obj cls labels img_size
309/399 9.2G 0.007121 0.001814 0.001076 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2317 0.328 0.0893 0.214 0.15
Saving model artifact on epoch 310
Epoch gpu_mem box obj cls labels img_size
310/399 9.2G 0.007414 0.001836 0.001101 9 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2316 0.331 0.0903 0.216 0.151
Epoch gpu_mem box obj cls labels img_size
311/399 9.2G 0.007206 0.001834 0.001211 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2316 0.334 0.0903 0.217 0.152
Epoch gpu_mem box obj cls labels img_size
312/399 9.2G 0.007372 0.001806 0.001127 13 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2316 0.334 0.0903 0.217 0.152
Epoch gpu_mem box obj cls labels img_size
313/399 9.2G 0.00704 0.001758 0.0009558 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2316 0.336 0.0903 0.217 0.152
Epoch gpu_mem box obj cls labels img_size
314/399 9.2G 0.006919 0.001771 0.0007918 24 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2316 0.337 0.09 0.218 0.153
Epoch gpu_mem box obj cls labels img_size
315/399 9.2G 0.007053 0.001752 0.0007955 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2316 0.339 0.09 0.218 0.154
Epoch gpu_mem box obj cls labels img_size
316/399 9.2G 0.006984 0.001757 0.000815 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2316 0.34 0.0903 0.219 0.154
Epoch gpu_mem box obj cls labels img_size
317/399 9.2G 0.007007 0.001754 0.0008991 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2316 0.339 0.0894 0.218 0.154
Epoch gpu_mem box obj cls labels img_size
318/399 9.2G 0.007187 0.001819 0.001134 13 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2316 0.338 0.0884 0.218 0.153
Epoch gpu_mem box obj cls labels img_size
319/399 9.2G 0.007312 0.001752 0.00109 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2316 0.34 0.0884 0.219 0.154
Saving model artifact on epoch 320
Epoch gpu_mem box obj cls labels img_size
320/399 9.2G 0.006893 0.00174 0.0007894 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2316 0.344 0.0887 0.221 0.154
Epoch gpu_mem box obj cls labels img_size
321/399 9.2G 0.006822 0.001735 0.001139 28 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2316 0.344 0.0884 0.221 0.154
Epoch gpu_mem box obj cls labels img_size
322/399 9.2G 0.00681 0.001724 0.0009574 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2315 0.347 0.0885 0.222 0.155
Epoch gpu_mem box obj cls labels img_size
323/399 9.2G 0.006879 0.001775 0.001124 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2315 0.35 0.0892 0.224 0.155
Epoch gpu_mem box obj cls labels img_size
324/399 9.2G 0.006811 0.001774 0.0009221 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2315 0.351 0.0895 0.225 0.155
Epoch gpu_mem box obj cls labels img_size
325/399 9.2G 0.006924 0.001759 0.0008976 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2315 0.352 0.0892 0.225 0.155
Epoch gpu_mem box obj cls labels img_size
326/399 9.2G 0.006554 0.001707 0.000901 24 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2315 0.351 0.0889 0.225 0.156
Epoch gpu_mem box obj cls labels img_size
327/399 9.2G 0.006746 0.001671 0.0007761 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2315 0.356 0.0892 0.227 0.157
Epoch gpu_mem box obj cls labels img_size
328/399 9.2G 0.006754 0.001726 0.0008094 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2315 0.358 0.0892 0.228 0.158
Epoch gpu_mem box obj cls labels img_size
329/399 9.2G 0.006901 0.001767 0.001129 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2319 0.356 0.0881 0.226 0.157
Saving model artifact on epoch 330
Epoch gpu_mem box obj cls labels img_size
330/399 9.2G 0.006868 0.001704 0.001008 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2319 0.357 0.0878 0.227 0.158
Epoch gpu_mem box obj cls labels img_size
331/399 9.2G 0.006635 0.001727 0.000963 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2318 0.359 0.0878 0.228 0.158
Epoch gpu_mem box obj cls labels img_size
332/399 9.2G 0.006571 0.001733 0.000912 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2318 0.358 0.0876 0.227 0.158
Epoch gpu_mem box obj cls labels img_size
333/399 9.2G 0.00675 0.001686 0.0006604 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2318 0.359 0.0876 0.227 0.158
Epoch gpu_mem box obj cls labels img_size
334/399 9.2G 0.006606 0.001656 0.0009005 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2307 0.358 0.0875 0.228 0.158
Epoch gpu_mem box obj cls labels img_size
335/399 9.2G 0.006664 0.001748 0.0008817 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2307 0.358 0.0872 0.227 0.158
Epoch gpu_mem box obj cls labels img_size
336/399 9.2G 0.006774 0.001732 0.001006 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.361 0.0872 0.229 0.159
Epoch gpu_mem box obj cls labels img_size
337/399 9.2G 0.006506 0.00169 0.0007435 9 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.362 0.0869 0.229 0.159
Epoch gpu_mem box obj cls labels img_size
338/399 9.2G 0.006682 0.001634 0.0007348 13 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.364 0.0869 0.23 0.16
Epoch gpu_mem box obj cls labels img_size
339/399 9.2G 0.006566 0.001665 0.0008622 25 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.364 0.0869 0.23 0.161
Saving model artifact on epoch 340
Epoch gpu_mem box obj cls labels img_size
340/399 9.2G 0.006563 0.001647 0.0007575 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.439 0.0831 0.231 0.161
Epoch gpu_mem box obj cls labels img_size
341/399 9.2G 0.006685 0.001699 0.0008283 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.438 0.0832 0.231 0.161
Epoch gpu_mem box obj cls labels img_size
342/399 9.2G 0.006442 0.00166 0.001119 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.442 0.0832 0.232 0.162
Epoch gpu_mem box obj cls labels img_size
343/399 9.2G 0.006574 0.001683 0.0008364 27 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.441 0.0832 0.233 0.162
Epoch gpu_mem box obj cls labels img_size
344/399 9.2G 0.006276 0.001621 0.0006612 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.44 0.083 0.233 0.162
Epoch gpu_mem box obj cls labels img_size
345/399 9.2G 0.006281 0.001575 0.0006959 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.367 0.0869 0.232 0.162
Epoch gpu_mem box obj cls labels img_size
346/399 9.2G 0.006348 0.001632 0.0008843 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.371 0.0867 0.234 0.163
Epoch gpu_mem box obj cls labels img_size
347/399 9.2G 0.006551 0.001687 0.0009834 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2305 0.44 0.0829 0.235 0.164
Epoch gpu_mem box obj cls labels img_size
348/399 9.2G 0.006605 0.001734 0.0007751 29 640: 23%|██▎wandb: Network error (ProxyError), entering retry loop.
348/399 9.2G 0.006535 0.001657 0.0007971 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2304 0.442 0.0829 0.235 0.164
Epoch gpu_mem box obj cls labels img_size
349/399 9.2G 0.006225 0.001598 0.0006955 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2304 0.374 0.0864 0.235 0.165
Saving model artifact on epoch 350
Epoch gpu_mem box obj cls labels img_size
350/399 9.2G 0.00652 0.00166 0.0006733 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2300 0.375 0.0868 0.235 0.166
Epoch gpu_mem box obj cls labels img_size
351/399 9.2G 0.006193 0.001615 0.0008038 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2300 0.376 0.0865 0.236 0.167
Epoch gpu_mem box obj cls labels img_size
352/399 9.2G 0.00635 0.001641 0.0006972 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2300 0.379 0.0865 0.237 0.167
Epoch gpu_mem box obj cls labels img_size
353/399 9.2G 0.006209 0.001605 0.00106 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2300 0.378 0.0865 0.237 0.166
Epoch gpu_mem box obj cls labels img_size
354/399 9.2G 0.006026 0.001551 0.0006301 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2299 0.379 0.0863 0.237 0.167
Epoch gpu_mem box obj cls labels img_size
355/399 9.2G 0.006437 0.001661 0.001061 26 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2299 0.384 0.0866 0.24 0.168
Epoch gpu_mem box obj cls labels img_size
356/399 9.2G 0.006287 0.001636 0.0007675 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2299 0.383 0.0863 0.239 0.168
Epoch gpu_mem box obj cls labels img_size
357/399 9.2G 0.005986 0.001593 0.0005777 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2299 0.386 0.0866 0.24 0.168
Epoch gpu_mem box obj cls labels img_size
358/399 9.2G 0.006154 0.001581 0.0008132 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2299 0.388 0.0863 0.241 0.169
Epoch gpu_mem box obj cls labels img_size
359/399 9.2G 0.005988 0.001553 0.000594 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2299 0.389 0.0866 0.242 0.169
Saving model artifact on epoch 360
Epoch gpu_mem box obj cls labels img_size
360/399 9.2G 0.006199 0.001601 0.0006295 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2299 0.387 0.086 0.241 0.169
Epoch gpu_mem box obj cls labels img_size
361/399 9.2G 0.005975 0.001604 0.0006908 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2299 0.389 0.086 0.242 0.169
Epoch gpu_mem box obj cls labels img_size
362/399 9.2G 0.005977 0.001583 0.0006357 13 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2299 0.392 0.086 0.243 0.17
Epoch gpu_mem box obj cls labels img_size
363/399 9.2G 0.005814 0.001569 0.0007076 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2299 0.392 0.0857 0.243 0.171
Epoch gpu_mem box obj cls labels img_size
364/399 9.2G 0.005915 0.001602 0.0005066 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2299 0.393 0.0857 0.243 0.17
Epoch gpu_mem box obj cls labels img_size
365/399 9.2G 0.005902 0.001553 0.0006204 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2299 0.392 0.0854 0.243 0.17
Epoch gpu_mem box obj cls labels img_size
366/399 9.2G 0.005897 0.001526 0.0007189 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2299 0.395 0.0854 0.244 0.171
Epoch gpu_mem box obj cls labels img_size
367/399 9.2G 0.005801 0.001561 0.000736 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2299 0.395 0.0854 0.244 0.171
Epoch gpu_mem box obj cls labels img_size
368/399 9.2G 0.005818 0.001531 0.0005447 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2299 0.395 0.0854 0.244 0.171
Epoch gpu_mem box obj cls labels img_size
369/399 9.2G 0.005651 0.001486 0.0004308 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2299 0.395 0.0854 0.245 0.171
Saving model artifact on epoch 370
Epoch gpu_mem box obj cls labels img_size
370/399 9.2G 0.005672 0.001521 0.0006316 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2299 0.396 0.0854 0.245 0.172
Epoch gpu_mem box obj cls labels img_size
371/399 9.2G 0.005627 0.001561 0.0005789 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2297 0.398 0.0854 0.246 0.172
Epoch gpu_mem box obj cls labels img_size
372/399 9.2G 0.005605 0.001496 0.0005287 27 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2297 0.4 0.0854 0.247 0.173
Epoch gpu_mem box obj cls labels img_size
373/399 9.2G 0.005564 0.00149 0.0006741 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2297 0.401 0.0854 0.247 0.174
Epoch gpu_mem box obj cls labels img_size
374/399 9.2G 0.005562 0.001474 0.0005015 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2297 0.401 0.0854 0.247 0.174
Epoch gpu_mem box obj cls labels img_size
375/399 9.2G 0.005494 0.001442 0.0004316 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2297 0.403 0.0854 0.248 0.174
Epoch gpu_mem box obj cls labels img_size
376/399 9.2G 0.005405 0.001551 0.0004229 32 640: 50%|███wandb: Network error (ProxyError), entering retry loop.
376/399 9.2G 0.005433 0.001521 0.0005998 21 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2297 0.404 0.0851 0.249 0.175
Epoch gpu_mem box obj cls labels img_size
377/399 9.2G 0.005324 0.001454 0.0005089 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2297 0.405 0.0851 0.249 0.175
Epoch gpu_mem box obj cls labels img_size
378/399 9.2G 0.005359 0.001458 0.0003825 15 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2259 0.403 0.0848 0.248 0.175
Epoch gpu_mem box obj cls labels img_size
379/399 9.2G 0.005434 0.001453 0.0003868 25 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2259 0.406 0.0854 0.25 0.176
Saving model artifact on epoch 380
Epoch gpu_mem box obj cls labels img_size
380/399 9.2G 0.005477 0.001418 0.00071 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2259 0.406 0.0854 0.25 0.176
Epoch gpu_mem box obj cls labels img_size
381/399 9.2G 0.005356 0.001496 0.0004867 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2259 0.408 0.0861 0.251 0.176
Epoch gpu_mem box obj cls labels img_size
382/399 9.2G 0.005354 0.001466 0.0005333 22 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2259 0.409 0.0861 0.251 0.177
Epoch gpu_mem box obj cls labels img_size
383/399 9.2G 0.005304 0.00146 0.0005451 24 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2259 0.411 0.0861 0.252 0.177
Epoch gpu_mem box obj cls labels img_size
384/399 9.2G 0.005292 0.001441 0.0005461 18 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2259 0.41 0.0858 0.251 0.178
Epoch gpu_mem box obj cls labels img_size
385/399 9.2G 0.005258 0.001441 0.0004153 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2259 0.408 0.0861 0.251 0.177
Epoch gpu_mem box obj cls labels img_size
386/399 9.2G 0.005165 0.001458 0.000635 28 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2259 0.412 0.0861 0.252 0.178
Epoch gpu_mem box obj cls labels img_size
387/399 9.2G 0.005302 0.001456 0.0005228 14 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2259 0.415 0.0861 0.254 0.179
Epoch gpu_mem box obj cls labels img_size
388/399 9.2G 0.005317 0.001407 0.0004739 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2259 0.417 0.0861 0.255 0.179
Epoch gpu_mem box obj cls labels img_size
389/399 9.2G 0.005148 0.00141 0.0005128 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2258 0.423 0.0862 0.258 0.181
Saving model artifact on epoch 390
Epoch gpu_mem box obj cls labels img_size
390/399 9.2G 0.00517 0.001397 0.0004808 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2258 0.421 0.0859 0.257 0.181
Epoch gpu_mem box obj cls labels img_size
391/399 9.2G 0.005011 0.001415 0.0003338 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2258 0.423 0.0859 0.258 0.182
Epoch gpu_mem box obj cls labels img_size
392/399 9.2G 0.00513 0.001375 0.0004617 24 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2258 0.425 0.0859 0.259 0.182
Epoch gpu_mem box obj cls labels img_size
393/399 9.2G 0.005021 0.001397 0.0005565 23 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2258 0.431 0.0859 0.261 0.184
Epoch gpu_mem box obj cls labels img_size
394/399 9.2G 0.004884 0.001411 0.0004105 16 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2258 0.431 0.0859 0.261 0.184
Epoch gpu_mem box obj cls labels img_size
395/399 9.2G 0.004961 0.001384 0.0003836 20 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2258 0.431 0.0859 0.261 0.184
Epoch gpu_mem box obj cls labels img_size
396/399 9.2G 0.004968 0.001343 0.0004494 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2258 0.431 0.0859 0.261 0.184
Epoch gpu_mem box obj cls labels img_size
397/399 9.2G 0.005088 0.001438 0.0004789 19 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2257 0.435 0.0856 0.263 0.185
Epoch gpu_mem box obj cls labels img_size
398/399 9.2G 0.00502 0.001359 0.0005113 17 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2257 0.436 0.0859 0.263 0.184
Epoch gpu_mem box obj cls labels img_size
399/399 9.2G 0.004883 0.001353 0.0004345 26 640: 100%|███
Class Images Labels P R mAP@.5mAP@.
all 137 2257 0.44 0.0859 0.265 0.186
400 epochs completed in 4.937 hours.
Optimizer stripped from ../../yolov5/runs/train/hybrid_imagery_example_train_syn_test_real/weights/last.pt, 14.5MB
Optimizer stripped from ../../yolov5/runs/train/hybrid_imagery_example_train_syn_test_real/weights/best.pt, 14.5MB
Validating ../../yolov5/runs/train/hybrid_imagery_example_train_syn_test_real/weights/best.pt...
Fusing layers...
YOLOv5s summary: 213 layers, 7018216 parameters, 0 gradients, 15.8 GFLOPs
Class Images Labels P R mAP@.5mAP@.Exception in thread Thread-40:
Traceback (most recent call last):
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 57, in check_pil_font
return ImageFont.truetype(str(font) if font.exists() else font.name, size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
return freetype(font)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
return FreeTypeFont(font, size, index, encoding, layout_engine)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
self.font = core.getfont(
OSError: unknown file format
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 973, in _bootstrap_inner
self.run()
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 910, in run
self._target(*self._args, **self._kwargs)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 217, in plot_images
annotator = Annotator(mosaic, line_width=round(fs / 10), font_size=fs, pil=True, example=names)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 77, in __init__
self.font = check_pil_font(font='Arial.Unicode.ttf' if non_ascii else font,
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 61, in check_pil_font
return ImageFont.truetype(str(font), size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
return freetype(font)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
return FreeTypeFont(font, size, index, encoding, layout_engine)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
self.font = core.getfont(
OSError: unknown file format
Exception in thread Thread-41:
Traceback (most recent call last):
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 57, in check_pil_font
return ImageFont.truetype(str(font) if font.exists() else font.name, size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
return freetype(font)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
return FreeTypeFont(font, size, index, encoding, layout_engine)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
self.font = core.getfont(
OSError: unknown file format
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 973, in _bootstrap_inner
self.run()
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 910, in run
self._target(*self._args, **self._kwargs)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 217, in plot_images
annotator = Annotator(mosaic, line_width=round(fs / 10), font_size=fs, pil=True, example=names)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 77, in __init__
self.font = check_pil_font(font='Arial.Unicode.ttf' if non_ascii else font,
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 61, in check_pil_font
return ImageFont.truetype(str(font), size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
return freetype(font)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
return FreeTypeFont(font, size, index, encoding, layout_engine)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
self.font = core.getfont(
OSError: unknown file format
Class Images Labels P R mAP@.5mAP@.
Exception in thread Thread-43:
Traceback (most recent call last):
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 57, in check_pil_font
Exception in thread Thread-42:
Traceback (most recent call last):
return ImageFont.truetype(str(font) if font.exists() else font.name, size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 57, in check_pil_font
return freetype(font)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
return ImageFont.truetype(str(font) if font.exists() else font.name, size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
return FreeTypeFont(font, size, index, encoding, layout_engine)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
self.font = core.getfont(
return freetype(font)
OSError: unknown file format
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 973, in _bootstrap_inner
self.run()
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 910, in run
self._target(*self._args, **self._kwargs)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 217, in plot_images
return FreeTypeFont(font, size, index, encoding, layout_engine)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
annotator = Annotator(mosaic, line_width=round(fs / 10), font_size=fs, pil=True, example=names)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 77, in __init__
self.font = check_pil_font(font='Arial.Unicode.ttf' if non_ascii else font,
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 61, in check_pil_font
return ImageFont.truetype(str(font), size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
self.font = core.getfont(
OSError: unknown file format
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 973, in _bootstrap_inner
return freetype(font)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
Exception in thread Thread-45:
Traceback (most recent call last):
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 57, in check_pil_font
return FreeTypeFont(font, size, index, encoding, layout_engine)
return ImageFont.truetype(str(font) if font.exists() else font.name, size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
self.run()
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
self.font = core.getfont(
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 910, in run
OSError: unknown file format
return freetype(font)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
self._target(*self._args, **self._kwargs)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 217, in plot_images
return FreeTypeFont(font, size, index, encoding, layout_engine)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
annotator = Annotator(mosaic, line_width=round(fs / 10), font_size=fs, pil=True, example=names)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 77, in __init__
self.font = core.getfont(
OSError: unknown file format
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 973, in _bootstrap_inner
self.font = check_pil_font(font='Arial.Unicode.ttf' if non_ascii else font,
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 61, in check_pil_font
self.run()
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 910, in run
return ImageFont.truetype(str(font), size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
self._target(*self._args, **self._kwargs)
return freetype(font)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 217, in plot_images
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
annotator = Annotator(mosaic, line_width=round(fs / 10), font_size=fs, pil=True, example=names)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 77, in __init__
return FreeTypeFont(font, size, index, encoding, layout_engine)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
self.font = check_pil_font(font='Arial.Unicode.ttf' if non_ascii else font,
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 61, in check_pil_font
return ImageFont.truetype(str(font), size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
self.font = core.getfont(
OSError: unknown file format
return freetype(font)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
return FreeTypeFont(font, size, index, encoding, layout_engine)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
self.font = core.getfont(
OSError: unknown file format
Exception in thread Thread-44:
Traceback (most recent call last):
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 57, in check_pil_font
return ImageFont.truetype(str(font) if font.exists() else font.name, size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
return freetype(font)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
return FreeTypeFont(font, size, index, encoding, layout_engine)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
self.font = core.getfont(
OSError: unknown file format
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 973, in _bootstrap_inner
self.run()
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/threading.py", line 910, in run
self._target(*self._args, **self._kwargs)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 217, in plot_images
annotator = Annotator(mosaic, line_width=round(fs / 10), font_size=fs, pil=True, example=names)
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 77, in __init__
self.font = check_pil_font(font='Arial.Unicode.ttf' if non_ascii else font,
File "/home/mrmarsh/repos/yolov5/utils/plots.py", line 61, in check_pil_font
return ImageFont.truetype(str(font), size)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 844, in truetype
return freetype(font)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 841, in freetype
return FreeTypeFont(font, size, index, encoding, layout_engine)
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/PIL/ImageFont.py", line 193, in __init__
self.font = core.getfont(
OSError: unknown file format
all 137 2257 0.437 0.0859 0.264 0.185
30 137 524 0.424 0.116 0.275 0.193
48 137 1733 0.451 0.0554 0.253 0.177
Traceback (most recent call last):
File "/home/mrmarsh/repos/yolov5/train.py", line 667, in <module>
main(opt)
File "/home/mrmarsh/repos/yolov5/train.py", line 562, in main
train(opt.hyp, opt, device, callbacks)
File "/home/mrmarsh/repos/yolov5/train.py", line 451, in train
results, _, _ = val.run(
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/autograd/grad_mode.py", line 27, in decorate_context
return func(*args, **kwargs)
File "/home/mrmarsh/repos/yolov5/val.py", line 296, in run
print("\n".join(print_buffer), file=open(save_dir / 'map_labels' / ('results.txt'), 'w'))
FileNotFoundError: [Errno 2] No such file or directory: '../../yolov5/runs/train/hybrid_imagery_example_train_syn_test_real/map_labels/results.txt'
wandb: Waiting for W&B process to finish... (failed 1). Press Control-C to abort syncing.
wandb:
wandb:
wandb: Run history:
wandb: metrics/mAP_0.5 ▁▁▃▃▃▄▅▄▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▆▆▆▆▆▆▆▇▇▇▇▇▇██
wandb: metrics/mAP_0.5:0.95 ▁▂▃▃▃▄▅▄▄▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇▇██
wandb: metrics/precision ▂▁▄▄▄▂▃▃▄▃▃▃▄▄▃▅▄▅▄▄▄▄▄▄▅▄▅▅▅▅▅▆▆▆█▇▇▇▇█
wandb: metrics/recall ▃▁▄▄▂▇▇▇▆█▇▇▇▇▇▅▇▆█▇▇▇▆▆▄▅▅▄▃▄▃▃▃▂▁▂▂▂▂▂
wandb: train/box_loss █▅▄▄▃▃▃▃▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁
wandb: train/cls_loss █▄▃▃▃▃▂▃▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁
wandb: train/obj_loss █▅▄▄▄▃▃▃▃▃▃▃▃▃▃▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▂▂▁▂▁▁▁▁▁▁
wandb: val/box_loss █▅▄▂▃█▄▃▁▃▃▃▃▃▃▃▃▃▃▄▄▃▄▄▄▄▅▅▅▅▅▅▅▅▅▆▆▆▆▆
wandb: val/cls_loss ▁▂▂▇▂█▃▃▄▁▆▄▃▂▁▁▁▂▂▂▂▂▂▂▂▂▂▂▂▂▂▁▁▂▂▂▂▂▂▂
wandb: val/obj_loss ▁▅▆▇▇▆█▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇▇███████
wandb: x/lr0 ████▇▇▇▇▇▆▆▆▆▆▆▅▅▅▅▅▄▄▄▄▄▄▃▃▃▃▃▂▂▂▂▂▂▁▁▁
wandb: x/lr1 ████▇▇▇▇▇▆▆▆▆▆▆▅▅▅▅▅▄▄▄▄▄▄▃▃▃▃▃▂▂▂▂▂▂▁▁▁
wandb: x/lr2 ████▇▇▇▇▇▆▆▆▆▆▆▅▅▅▅▅▄▄▄▄▄▄▃▃▃▃▃▂▂▂▂▂▂▁▁▁
wandb:
wandb: Run summary:
wandb: best/epoch 399
wandb: best/mAP_0.5 0.26479
wandb: best/mAP_0.5:0.95 0.18569
wandb: best/precision 0.4397
wandb: best/recall 0.0859
wandb: metrics/mAP_0.5 0.26479
wandb: metrics/mAP_0.5:0.95 0.18569
wandb: metrics/precision 0.4397
wandb: metrics/recall 0.0859
wandb: train/box_loss 0.00488
wandb: train/cls_loss 0.00043
wandb: train/obj_loss 0.00135
wandb: val/box_loss 0.09898
wandb: val/cls_loss 0.03885
wandb: val/obj_loss 0.18768
wandb: x/lr0 0.00015
wandb: x/lr1 0.00015
wandb: x/lr2 0.00015
wandb:
wandb: Synced hybrid_imagery_example_train_syn_test_real: https://wandb.ai/mrmarsh/train/runs/2teyd5oi
wandb: Synced 6 W&B file(s), 322 media file(s), 39 artifact file(s) and 0 other file(s)
wandb: Find logs at: ./wandb/run-20221206_092959-2teyd5oi/logs
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f76f5578430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f76f5578430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f76f5578430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f76f5578430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f76f5578430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Exception ignored in: <function StorageWeakRef.__del__ at 0x7f76f5578430>
Traceback (most recent call last):
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/multiprocessing/reductions.py", line 36, in __del__
File "/home/mrmarsh/anaconda3/envs/torch-gpu/lib/python3.9/site-packages/torch/storage.py", line 520, in _free_weak_ref
AttributeError: 'NoneType' object has no attribute '_free_weak_ref'
Detect
Let’s run inference and visualize how well the model does. We are using a 25% confidence. This is more applicable for the train synthetic, test real case.
[45]:
!python /home/mrmarsh/repos/yolov5/detect.py \
--source /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/ \
--weights /home/mrmarsh/repos/yolov5/runs/train/hybrid_imagery_example/weights/best.pt \
--conf 0.25 \
--name hybrid_imagery_example_inference
detect: weights=['/home/mrmarsh/repos/yolov5/runs/train/hybrid_imagery_example/weights/best.pt'], source=/home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/, data=../../yolov5/data/coco128.yaml, imgsz=[640, 640], conf_thres=0.25, iou_thres=0.45, max_det=1000, device=, view_img=False, save_txt=False, save_conf=False, save_crop=False, nosave=False, classes=None, agnostic_nms=False, augment=False, visualize=False, update=False, project=../../yolov5/runs/detect, name=hybrid_imagery_example_inference, exist_ok=False, line_thickness=3, hide_labels=False, hide_conf=False, half=False, dnn=False
YOLOv5 🚀 v6.1-177-gd059d1d torch 1.11.0 CUDA:0 (Quadro RTX 6000, 24198MiB)
Fusing layers...
YOLOv5s summary: 213 layers, 7018216 parameters, 0 gradients, 15.8 GFLOPs
image 1/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/0.png: 640x640 1 30, Done. (0.009s)
image 2/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/1.png: 640x640 Done. (0.007s)
image 3/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/10.png: 640x640 Done. (0.007s)
image 4/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/100.png: 640x640 Done. (0.009s)
image 5/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/101.png: 640x640 1 30, Done. (0.010s)
image 6/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/102.png: 640x640 Done. (0.007s)
image 7/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/103.png: 640x640 Done. (0.016s)
image 8/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/104.png: 640x640 1 30, Done. (0.008s)
image 9/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/105.png: 640x640 1 48, Done. (0.018s)
image 10/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/106.png: 640x640 Done. (0.007s)
image 11/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/107.png: 640x640 Done. (0.007s)
image 12/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/108.png: 640x640 Done. (0.007s)
image 13/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/109.png: 640x640 Done. (0.007s)
image 14/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/11.png: 640x640 1 48, Done. (0.007s)
image 15/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/110.png: 640x640 Done. (0.008s)
image 16/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/111.png: 640x640 Done. (0.007s)
image 17/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/112.png: 640x640 Done. (0.007s)
image 18/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/113.png: 640x640 1 30, Done. (0.007s)
image 19/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/114.png: 640x640 Done. (0.007s)
image 20/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/115.png: 640x640 Done. (0.007s)
image 21/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/116.png: 640x640 Done. (0.007s)
image 22/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/117.png: 640x640 1 48, Done. (0.009s)
image 23/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/118.png: 640x640 Done. (0.007s)
image 24/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/119.png: 640x640 Done. (0.007s)
image 25/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/12.png: 640x640 Done. (0.007s)
image 26/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/120.png: 640x640 1 30, Done. (0.007s)
image 27/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/121.png: 640x640 1 48, Done. (0.009s)
image 28/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/122.png: 640x640 1 48, Done. (0.010s)
image 29/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/123.png: 640x640 Done. (0.012s)
image 30/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/124.png: 640x640 1 30, Done. (0.007s)
image 31/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/125.png: 640x640 1 30, Done. (0.007s)
image 32/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/126.png: 640x640 1 30, Done. (0.009s)
image 33/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/127.png: 640x640 1 30, Done. (0.009s)
image 34/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/128.png: 640x640 1 30, Done. (0.014s)
image 35/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/129.png: 640x640 Done. (0.011s)
image 36/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/13.png: 640x640 1 30, Done. (0.008s)
image 37/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/130.png: 640x640 Done. (0.009s)
image 38/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/131.png: 640x640 Done. (0.007s)
image 39/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/132.png: 640x640 1 48, Done. (0.007s)
image 40/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/133.png: 640x640 Done. (0.007s)
image 41/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/134.png: 640x640 Done. (0.009s)
image 42/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/135.png: 640x640 1 48, Done. (0.009s)
image 43/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/136.png: 640x640 1 30, Done. (0.007s)
image 44/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/137.png: 640x640 Done. (0.014s)
image 45/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/138.png: 640x640 Done. (0.007s)
image 46/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/139.png: 640x640 Done. (0.007s)
image 47/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/14.png: 640x640 1 30, Done. (0.007s)
image 48/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/140.png: 640x640 1 30, Done. (0.010s)
image 49/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/141.png: 640x640 1 30, Done. (0.010s)
image 50/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/142.png: 640x640 Done. (0.007s)
image 51/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/143.png: 640x640 1 48, Done. (0.007s)
image 52/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/144.png: 640x640 1 48, Done. (0.007s)
image 53/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/145.png: 640x640 1 48, Done. (0.007s)
image 54/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/146.png: 640x640 Done. (0.007s)
image 55/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/147.png: 640x640 1 48, Done. (0.007s)
image 56/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/148.png: 640x640 1 48, Done. (0.010s)
image 57/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/149.png: 640x640 Done. (0.007s)
image 58/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/15.png: 640x640 1 30, Done. (0.007s)
image 59/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/150.png: 640x640 Done. (0.007s)
image 60/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/151.png: 640x640 1 48, Done. (0.007s)
image 61/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/152.png: 640x640 Done. (0.007s)
image 62/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/153.png: 640x640 1 48, Done. (0.007s)
image 63/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/154.png: 640x640 1 48, Done. (0.007s)
image 64/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/155.png: 640x640 1 30, Done. (0.009s)
image 65/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/156.png: 640x640 1 30, Done. (0.009s)
image 66/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/157.png: 640x640 1 30, Done. (0.007s)
image 67/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/158.png: 640x640 1 48, Done. (0.007s)
image 68/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/159.png: 640x640 1 30, Done. (0.007s)
image 69/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/16.png: 640x640 Done. (0.007s)
image 70/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/160.png: 640x640 Done. (0.007s)
image 71/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/161.png: 640x640 Done. (0.007s)
image 72/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/162.png: 640x640 Done. (0.007s)
image 73/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/163.png: 640x640 1 48, Done. (0.007s)
image 74/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/164.png: 640x640 Done. (0.007s)
image 75/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/165.png: 640x640 Done. (0.009s)
image 76/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/166.png: 640x640 1 30, Done. (0.008s)
image 77/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/167.png: 640x640 1 48, Done. (0.007s)
image 78/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/168.png: 640x640 1 30, Done. (0.007s)
image 79/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/169.png: 640x640 1 30, Done. (0.007s)
image 80/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/17.png: 640x640 Done. (0.007s)
image 81/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/170.png: 640x640 1 30, Done. (0.009s)
image 82/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/171.png: 640x640 1 48, Done. (0.012s)
image 83/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/172.png: 640x640 Done. (0.007s)
image 84/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/173.png: 640x640 1 48, Done. (0.007s)
image 85/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/174.png: 640x640 1 30, Done. (0.009s)
image 86/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/175.png: 640x640 1 48, Done. (0.010s)
image 87/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/176.png: 640x640 Done. (0.012s)
image 88/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/177.png: 640x640 1 30, Done. (0.007s)
image 89/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/178.png: 640x640 1 30, Done. (0.007s)
image 90/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/179.png: 640x640 1 48, Done. (0.007s)
image 91/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/18.png: 640x640 1 30, Done. (0.007s)
image 92/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/180.png: 640x640 Done. (0.007s)
image 93/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/181.png: 640x640 1 30, Done. (0.007s)
image 94/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/182.png: 640x640 1 30, Done. (0.011s)
image 95/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/183.png: 640x640 Done. (0.007s)
image 96/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/184.png: 640x640 1 48, Done. (0.011s)
image 97/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/185.png: 640x640 Done. (0.011s)
image 98/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/186.png: 640x640 1 30, Done. (0.012s)
image 99/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/187.png: 640x640 1 48, Done. (0.007s)
image 100/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/188.png: 640x640 1 30, Done. (0.007s)
image 101/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/189.png: 640x640 Done. (0.007s)
image 102/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/19.png: 640x640 1 30, Done. (0.012s)
image 103/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/190.png: 640x640 Done. (0.009s)
image 104/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/191.png: 640x640 Done. (0.007s)
image 105/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/192.png: 640x640 Done. (0.007s)
image 106/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/193.png: 640x640 1 48, Done. (0.010s)
image 107/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/194.png: 640x640 Done. (0.007s)
image 108/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/195.png: 640x640 Done. (0.007s)
image 109/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/196.png: 640x640 1 30, Done. (0.011s)
image 110/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/197.png: 640x640 1 30, Done. (0.007s)
image 111/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/198.png: 640x640 1 30, Done. (0.007s)
image 112/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/199.png: 640x640 1 30, Done. (0.007s)
image 113/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/2.png: 640x640 1 48, Done. (0.007s)
image 114/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/20.png: 640x640 1 48, Done. (0.009s)
image 115/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/200.png: 640x640 1 48, Done. (0.010s)
image 116/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/201.png: 640x640 1 48, Done. (0.007s)
image 117/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/202.png: 640x640 1 48, Done. (0.010s)
image 118/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/203.png: 640x640 1 30, Done. (0.007s)
image 119/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/204.png: 640x640 1 48, Done. (0.006s)
image 120/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/205.png: 640x640 Done. (0.010s)
image 121/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/206.png: 640x640 1 30, Done. (0.011s)
image 122/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/207.png: 640x640 Done. (0.010s)
image 123/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/208.png: 640x640 Done. (0.008s)
image 124/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/209.png: 640x640 Done. (0.007s)
image 125/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/21.png: 640x640 Done. (0.009s)
image 126/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/210.png: 640x640 1 48, Done. (0.011s)
image 127/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/211.png: 640x640 1 48, Done. (0.007s)
image 128/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/212.png: 640x640 1 30, Done. (0.007s)
image 129/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/213.png: 640x640 1 30, Done. (0.007s)
image 130/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/214.png: 640x640 Done. (0.008s)
image 131/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/215.png: 640x640 1 48, Done. (0.009s)
image 132/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/216.png: 640x640 Done. (0.009s)
image 133/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/217.png: 640x640 Done. (0.010s)
image 134/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/218.png: 640x640 1 48, Done. (0.007s)
image 135/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/219.png: 640x640 1 30, Done. (0.009s)
image 136/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/22.png: 640x640 Done. (0.007s)
image 137/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/220.png: 640x640 1 48, Done. (0.007s)
image 138/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/221.png: 640x640 1 48, Done. (0.009s)
image 139/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/222.png: 640x640 1 30, Done. (0.007s)
image 140/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/223.png: 640x640 Done. (0.007s)
image 141/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/224.png: 640x640 1 30, Done. (0.007s)
image 142/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/225.png: 640x640 1 30, Done. (0.007s)
image 143/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/226.png: 640x640 Done. (0.007s)
image 144/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/227.png: 640x640 1 30, Done. (0.007s)
image 145/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/228.png: 640x640 1 48, Done. (0.010s)
image 146/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/229.png: 640x640 1 30, Done. (0.009s)
image 147/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/23.png: 640x640 1 48, Done. (0.007s)
image 148/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/230.png: 640x640 1 48, Done. (0.007s)
image 149/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/231.png: 640x640 1 30, Done. (0.010s)
image 150/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/232.png: 640x640 Done. (0.008s)
image 151/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/233.png: 640x640 Done. (0.007s)
image 152/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/234.png: 640x640 Done. (0.007s)
image 153/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/235.png: 640x640 1 30, Done. (0.007s)
image 154/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/236.png: 640x640 Done. (0.008s)
image 155/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/237.png: 640x640 1 48, Done. (0.007s)
image 156/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/238.png: 640x640 1 30, Done. (0.008s)
image 157/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/239.png: 640x640 Done. (0.007s)
image 158/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/24.png: 640x640 1 30, Done. (0.007s)
image 159/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/240.png: 640x640 Done. (0.007s)
image 160/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/241.png: 640x640 Done. (0.013s)
image 161/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/242.png: 640x640 1 30, Done. (0.007s)
image 162/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/243.png: 640x640 1 48, Done. (0.007s)
image 163/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/244.png: 640x640 Done. (0.011s)
image 164/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/245.png: 640x640 1 30, Done. (0.008s)
image 165/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/246.png: 640x640 Done. (0.012s)
image 166/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/247.png: 640x640 1 30, Done. (0.010s)
image 167/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/248.png: 640x640 1 30, Done. (0.011s)
image 168/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/249.png: 640x640 1 30, Done. (0.009s)
image 169/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/25.png: 640x640 1 48, Done. (0.007s)
image 170/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/250.png: 640x640 Done. (0.009s)
image 171/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/251.png: 640x640 1 48, Done. (0.009s)
image 172/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/252.png: 640x640 1 30, Done. (0.010s)
image 173/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/253.png: 640x640 1 48, Done. (0.007s)
image 174/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/254.png: 640x640 Done. (0.008s)
image 175/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/255.png: 640x640 1 30, Done. (0.008s)
image 176/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/256.png: 640x640 Done. (0.010s)
image 177/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/257.png: 640x640 1 48, Done. (0.007s)
image 178/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/258.png: 640x640 1 48, Done. (0.007s)
image 179/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/259.png: 640x640 Done. (0.007s)
image 180/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/26.png: 640x640 Done. (0.007s)
image 181/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/260.png: 640x640 Done. (0.007s)
image 182/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/261.png: 640x640 1 30, Done. (0.010s)
image 183/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/262.png: 640x640 Done. (0.007s)
image 184/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/263.png: 640x640 Done. (0.006s)
image 185/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/264.png: 640x640 1 30, Done. (0.009s)
image 186/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/265.png: 640x640 Done. (0.007s)
image 187/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/266.png: 640x640 1 48, Done. (0.007s)
image 188/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/267.png: 640x640 Done. (0.008s)
image 189/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/268.png: 640x640 Done. (0.007s)
image 190/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/269.png: 640x640 Done. (0.007s)
image 191/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/27.png: 640x640 1 48, Done. (0.010s)
image 192/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/270.png: 640x640 1 48, Done. (0.007s)
image 193/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/271.png: 640x640 1 48, Done. (0.009s)
image 194/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/272.png: 640x640 Done. (0.013s)
image 195/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/273.png: 640x640 1 30, Done. (0.007s)
image 196/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/274.png: 640x640 1 48, Done. (0.007s)
image 197/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/275.png: 640x640 Done. (0.007s)
image 198/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/276.png: 640x640 1 30, Done. (0.007s)
image 199/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/277.png: 640x640 Done. (0.007s)
image 200/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/278.png: 640x640 Done. (0.007s)
image 201/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/279.png: 640x640 1 30, Done. (0.007s)
image 202/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/28.png: 640x640 1 30, Done. (0.009s)
image 203/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/280.png: 640x640 Done. (0.010s)
image 204/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/281.png: 640x640 Done. (0.009s)
image 205/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/282.png: 640x640 1 30, Done. (0.007s)
image 206/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/283.png: 640x640 Done. (0.007s)
image 207/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/284.png: 640x640 1 30, Done. (0.009s)
image 208/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/285.png: 640x640 Done. (0.010s)
image 209/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/286.png: 640x640 1 48, Done. (0.009s)
image 210/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/287.png: 640x640 Done. (0.009s)
image 211/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/288.png: 640x640 1 30, Done. (0.010s)
image 212/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/289.png: 640x640 Done. (0.010s)
image 213/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/29.png: 640x640 Done. (0.010s)
image 214/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/290.png: 640x640 Done. (0.007s)
image 215/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/291.png: 640x640 Done. (0.010s)
image 216/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/292.png: 640x640 Done. (0.009s)
image 217/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/293.png: 640x640 1 48, Done. (0.007s)
image 218/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/294.png: 640x640 1 48, Done. (0.007s)
image 219/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/295.png: 640x640 Done. (0.007s)
image 220/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/296.png: 640x640 1 48, Done. (0.007s)
image 221/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/297.png: 640x640 1 30, Done. (0.007s)
image 222/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/298.png: 640x640 1 30, Done. (0.007s)
image 223/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/299.png: 640x640 1 48, Done. (0.007s)
image 224/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/3.png: 640x640 Done. (0.007s)
image 225/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/30.png: 640x640 Done. (0.007s)
image 226/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/300.png: 640x640 1 30, 1 48, Done. (0.007s)
image 227/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/301.png: 640x640 1 48, Done. (0.007s)
image 228/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/302.png: 640x640 Done. (0.013s)
image 229/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/303.png: 640x640 1 48, Done. (0.007s)
image 230/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/304.png: 640x640 Done. (0.007s)
image 231/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/305.png: 640x640 1 30, Done. (0.008s)
image 232/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/306.png: 640x640 1 30, Done. (0.007s)
image 233/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/307.png: 640x640 1 30, Done. (0.007s)
image 234/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/308.png: 640x640 1 48, Done. (0.007s)
image 235/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/309.png: 640x640 1 30, Done. (0.007s)
image 236/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/31.png: 640x640 Done. (0.009s)
image 237/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/310.png: 640x640 Done. (0.007s)
image 238/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/311.png: 640x640 1 48, Done. (0.007s)
image 239/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/312.png: 640x640 1 48, Done. (0.007s)
image 240/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/313.png: 640x640 1 48, Done. (0.009s)
image 241/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/314.png: 640x640 Done. (0.009s)
image 242/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/315.png: 640x640 Done. (0.010s)
image 243/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/316.png: 640x640 1 48, Done. (0.007s)
image 244/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/317.png: 640x640 Done. (0.007s)
image 245/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/318.png: 640x640 1 30, Done. (0.009s)
image 246/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/319.png: 640x640 1 30, Done. (0.010s)
image 247/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/32.png: 640x640 1 48, Done. (0.009s)
image 248/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/320.png: 640x640 Done. (0.008s)
image 249/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/321.png: 640x640 1 48, Done. (0.007s)
image 250/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/322.png: 640x640 Done. (0.007s)
image 251/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/323.png: 640x640 1 30, Done. (0.011s)
image 252/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/324.png: 640x640 Done. (0.009s)
image 253/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/325.png: 640x640 Done. (0.007s)
image 254/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/326.png: 640x640 1 48, Done. (0.007s)
image 255/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/327.png: 640x640 1 48, Done. (0.007s)
image 256/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/328.png: 640x640 1 30, Done. (0.009s)
image 257/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/329.png: 640x640 Done. (0.010s)
image 258/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/33.png: 640x640 Done. (0.019s)
image 259/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/330.png: 640x640 1 30, Done. (0.008s)
image 260/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/331.png: 640x640 Done. (0.013s)
image 261/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/332.png: 640x640 Done. (0.014s)
image 262/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/333.png: 640x640 Done. (0.007s)
image 263/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/334.png: 640x640 Done. (0.009s)
image 264/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/335.png: 640x640 1 30, Done. (0.007s)
image 265/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/336.png: 640x640 1 48, Done. (0.009s)
image 266/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/337.png: 640x640 1 30, Done. (0.009s)
image 267/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/338.png: 640x640 1 30, Done. (0.010s)
image 268/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/339.png: 640x640 1 48, Done. (0.009s)
image 269/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/34.png: 640x640 1 30, Done. (0.007s)
image 270/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/340.png: 640x640 Done. (0.012s)
image 271/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/341.png: 640x640 Done. (0.009s)
image 272/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/342.png: 640x640 Done. (0.017s)
image 273/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/343.png: 640x640 Done. (0.007s)
image 274/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/344.png: 640x640 Done. (0.008s)
image 275/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/345.png: 640x640 1 30, Done. (0.007s)
image 276/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/346.png: 640x640 1 48, Done. (0.007s)
image 277/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/347.png: 640x640 1 48, Done. (0.007s)
image 278/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/348.png: 640x640 1 48, Done. (0.007s)
image 279/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/349.png: 640x640 Done. (0.007s)
image 280/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/35.png: 640x640 1 48, Done. (0.007s)
image 281/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/350.png: 640x640 1 48, Done. (0.010s)
image 282/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/351.png: 640x640 1 30, Done. (0.007s)
image 283/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/352.png: 640x640 Done. (0.007s)
image 284/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/353.png: 640x640 Done. (0.007s)
image 285/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/354.png: 640x640 Done. (0.008s)
image 286/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/355.png: 640x640 Done. (0.007s)
image 287/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/356.png: 640x640 1 48, Done. (0.017s)
image 288/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/357.png: 640x640 1 30, Done. (0.010s)
image 289/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/358.png: 640x640 1 48, Done. (0.007s)
image 290/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/359.png: 640x640 1 30, Done. (0.009s)
image 291/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/36.png: 640x640 Done. (0.009s)
image 292/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/360.png: 640x640 Done. (0.007s)
image 293/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/361.png: 640x640 Done. (0.006s)
image 294/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/362.png: 640x640 1 48, Done. (0.007s)
image 295/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/363.png: 640x640 Done. (0.007s)
image 296/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/364.png: 640x640 1 48, Done. (0.008s)
image 297/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/365.png: 640x640 1 30, Done. (0.009s)
image 298/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/366.png: 640x640 1 48, Done. (0.007s)
image 299/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/367.png: 640x640 Done. (0.008s)
image 300/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/368.png: 640x640 1 48, Done. (0.007s)
image 301/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/369.png: 640x640 Done. (0.008s)
image 302/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/37.png: 640x640 1 30, Done. (0.007s)
image 303/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/370.png: 640x640 Done. (0.008s)
image 304/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/371.png: 640x640 1 48, Done. (0.008s)
image 305/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/372.png: 640x640 Done. (0.007s)
image 306/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/373.png: 640x640 1 30, Done. (0.007s)
image 307/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/374.png: 640x640 1 48, Done. (0.008s)
image 308/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/375.png: 640x640 Done. (0.009s)
image 309/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/376.png: 640x640 Done. (0.009s)
image 310/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/377.png: 640x640 Done. (0.008s)
image 311/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/378.png: 640x640 Done. (0.007s)
image 312/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/379.png: 640x640 Done. (0.007s)
image 313/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/38.png: 640x640 1 30, Done. (0.010s)
image 314/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/380.png: 640x640 Done. (0.007s)
image 315/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/381.png: 640x640 1 48, Done. (0.007s)
image 316/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/382.png: 640x640 Done. (0.007s)
image 317/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/383.png: 640x640 Done. (0.010s)
image 318/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/384.png: 640x640 Done. (0.009s)
image 319/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/385.png: 640x640 1 30, Done. (0.007s)
image 320/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/386.png: 640x640 Done. (0.011s)
image 321/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/387.png: 640x640 1 48, Done. (0.007s)
image 322/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/388.png: 640x640 Done. (0.009s)
image 323/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/389.png: 640x640 1 48, Done. (0.007s)
image 324/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/39.png: 640x640 Done. (0.007s)
image 325/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/390.png: 640x640 Done. (0.010s)
image 326/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/391.png: 640x640 Done. (0.008s)
image 327/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/392.png: 640x640 1 30, Done. (0.007s)
image 328/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/393.png: 640x640 Done. (0.007s)
image 329/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/394.png: 640x640 Done. (0.007s)
image 330/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/395.png: 640x640 Done. (0.008s)
image 331/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/396.png: 640x640 1 48, Done. (0.007s)
image 332/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/397.png: 640x640 1 30, Done. (0.010s)
image 333/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/398.png: 640x640 Done. (0.007s)
image 334/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/399.png: 640x640 1 48, Done. (0.007s)
image 335/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/4.png: 640x640 Done. (0.007s)
image 336/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/40.png: 640x640 1 30, Done. (0.007s)
image 337/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/400.png: 640x640 1 30, Done. (0.007s)
image 338/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/401.png: 640x640 1 30, Done. (0.009s)
image 339/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/402.png: 640x640 1 30, Done. (0.007s)
image 340/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/403.png: 640x640 1 30, Done. (0.007s)
image 341/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/404.png: 640x640 Done. (0.007s)
image 342/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/405.png: 640x640 Done. (0.007s)
image 343/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/406.png: 640x640 Done. (0.007s)
image 344/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/407.png: 640x640 Done. (0.007s)
image 345/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/408.png: 640x640 1 48, Done. (0.009s)
image 346/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/409.png: 640x640 Done. (0.007s)
image 347/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/41.png: 640x640 1 48, Done. (0.007s)
image 348/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/410.png: 640x640 Done. (0.007s)
image 349/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/411.png: 640x640 1 48, Done. (0.007s)
image 350/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/412.png: 640x640 Done. (0.011s)
image 351/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/413.png: 640x640 Done. (0.009s)
image 352/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/414.png: 640x640 1 30, Done. (0.011s)
image 353/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/415.png: 640x640 Done. (0.007s)
image 354/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/416.png: 640x640 Done. (0.007s)
image 355/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/417.png: 640x640 1 48, Done. (0.008s)
image 356/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/418.png: 640x640 1 48, Done. (0.011s)
image 357/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/419.png: 640x640 1 48, Done. (0.008s)
image 358/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/42.png: 640x640 Done. (0.009s)
image 359/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/420.png: 640x640 1 30, Done. (0.007s)
image 360/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/421.png: 640x640 Done. (0.007s)
image 361/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/422.png: 640x640 Done. (0.007s)
image 362/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/423.png: 640x640 1 48, Done. (0.007s)
image 363/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/424.png: 640x640 Done. (0.007s)
image 364/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/425.png: 640x640 Done. (0.006s)
image 365/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/426.png: 640x640 Done. (0.007s)
image 366/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/427.png: 640x640 Done. (0.009s)
image 367/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/428.png: 640x640 Done. (0.007s)
image 368/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/429.png: 640x640 1 48, Done. (0.018s)
image 369/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/43.png: 640x640 Done. (0.014s)
image 370/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/430.png: 640x640 Done. (0.007s)
image 371/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/431.png: 640x640 1 48, Done. (0.007s)
image 372/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/432.png: 640x640 Done. (0.010s)
image 373/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/433.png: 640x640 Done. (0.007s)
image 374/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/434.png: 640x640 Done. (0.016s)
image 375/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/435.png: 640x640 Done. (0.008s)
image 376/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/436.png: 640x640 1 48, Done. (0.009s)
image 377/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/437.png: 640x640 1 30, Done. (0.007s)
image 378/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/438.png: 640x640 Done. (0.015s)
image 379/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/439.png: 640x640 1 48, Done. (0.007s)
image 380/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/44.png: 640x640 1 30, Done. (0.017s)
image 381/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/440.png: 640x640 Done. (0.010s)
image 382/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/441.png: 640x640 1 30, Done. (0.007s)
image 383/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/442.png: 640x640 Done. (0.007s)
image 384/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/443.png: 640x640 Done. (0.009s)
image 385/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/444.png: 640x640 Done. (0.007s)
image 386/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/445.png: 640x640 1 30, Done. (0.009s)
image 387/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/446.png: 640x640 1 48, Done. (0.007s)
image 388/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/447.png: 640x640 Done. (0.007s)
image 389/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/448.png: 640x640 1 30, Done. (0.007s)
image 390/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/449.png: 640x640 1 48, Done. (0.007s)
image 391/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/45.png: 640x640 Done. (0.009s)
image 392/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/450.png: 640x640 1 30, Done. (0.009s)
image 393/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/451.png: 640x640 Done. (0.008s)
image 394/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/452.png: 640x640 1 48, Done. (0.011s)
image 395/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/453.png: 640x640 Done. (0.007s)
image 396/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/454.png: 640x640 1 30, Done. (0.009s)
image 397/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/455.png: 640x640 Done. (0.007s)
image 398/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/456.png: 640x640 Done. (0.007s)
image 399/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/457.png: 640x640 1 48, Done. (0.013s)
image 400/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/458.png: 640x640 1 48, Done. (0.007s)
image 401/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/459.png: 640x640 Done. (0.007s)
image 402/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/46.png: 640x640 Done. (0.009s)
image 403/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/460.png: 640x640 Done. (0.007s)
image 404/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/461.png: 640x640 Done. (0.009s)
image 405/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/462.png: 640x640 Done. (0.008s)
image 406/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/463.png: 640x640 1 30, Done. (0.010s)
image 407/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/464.png: 640x640 1 30, Done. (0.007s)
image 408/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/465.png: 640x640 1 48, Done. (0.007s)
image 409/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/466.png: 640x640 1 48, Done. (0.007s)
image 410/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/467.png: 640x640 Done. (0.007s)
image 411/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/468.png: 640x640 1 30, Done. (0.011s)
image 412/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/469.png: 640x640 Done. (0.007s)
image 413/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/47.png: 640x640 1 30, Done. (0.007s)
image 414/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/470.png: 640x640 Done. (0.013s)
image 415/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/471.png: 640x640 Done. (0.010s)
image 416/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/472.png: 640x640 Done. (0.007s)
image 417/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/473.png: 640x640 Done. (0.010s)
image 418/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/474.png: 640x640 1 30, Done. (0.007s)
image 419/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/475.png: 640x640 Done. (0.007s)
image 420/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/476.png: 640x640 1 30, Done. (0.007s)
image 421/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/477.png: 640x640 1 48, Done. (0.007s)
image 422/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/478.png: 640x640 1 48, Done. (0.007s)
image 423/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/479.png: 640x640 Done. (0.010s)
image 424/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/48.png: 640x640 Done. (0.007s)
image 425/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/480.png: 640x640 1 48, Done. (0.007s)
image 426/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/481.png: 640x640 Done. (0.009s)
image 427/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/482.png: 640x640 1 30, Done. (0.012s)
image 428/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/483.png: 640x640 Done. (0.010s)
image 429/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/484.png: 640x640 1 48, Done. (0.009s)
image 430/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/485.png: 640x640 1 48, Done. (0.007s)
image 431/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/486.png: 640x640 Done. (0.007s)
image 432/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/487.png: 640x640 Done. (0.007s)
image 433/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/488.png: 640x640 Done. (0.007s)
image 434/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/489.png: 640x640 1 48, Done. (0.007s)
image 435/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/49.png: 640x640 Done. (0.007s)
image 436/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/490.png: 640x640 Done. (0.007s)
image 437/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/491.png: 640x640 Done. (0.007s)
image 438/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/492.png: 640x640 1 30, Done. (0.007s)
image 439/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/493.png: 640x640 1 48, Done. (0.007s)
image 440/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/494.png: 640x640 1 48, Done. (0.009s)
image 441/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/495.png: 640x640 1 48, Done. (0.007s)
image 442/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/496.png: 640x640 Done. (0.007s)
image 443/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/497.png: 640x640 Done. (0.007s)
image 444/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/498.png: 640x640 1 30, Done. (0.007s)
image 445/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/499.png: 640x640 1 48, Done. (0.007s)
image 446/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/5.png: 640x640 Done. (0.009s)
image 447/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/50.png: 640x640 Done. (0.009s)
image 448/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/500.png: 640x640 Done. (0.007s)
image 449/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/501.png: 640x640 1 48, Done. (0.009s)
image 450/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/502.png: 640x640 1 48, Done. (0.008s)
image 451/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/503.png: 640x640 Done. (0.009s)
image 452/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/504.png: 640x640 Done. (0.008s)
image 453/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/505.png: 640x640 Done. (0.011s)
image 454/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/506.png: 640x640 1 30, Done. (0.011s)
image 455/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/507.png: 640x640 Done. (0.012s)
image 456/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/508.png: 640x640 Done. (0.007s)
image 457/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/509.png: 640x640 Done. (0.007s)
image 458/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/51.png: 640x640 1 30, Done. (0.008s)
image 459/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/510.png: 640x640 1 30, Done. (0.007s)
image 460/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/511.png: 640x640 Done. (0.007s)
image 461/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/512.png: 640x640 Done. (0.007s)
image 462/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/513.png: 640x640 1 48, Done. (0.007s)
image 463/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/514.png: 640x640 Done. (0.007s)
image 464/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/515.png: 640x640 1 48, Done. (0.012s)
image 465/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/516.png: 640x640 Done. (0.007s)
image 466/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/517.png: 640x640 1 48, Done. (0.009s)
image 467/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/518.png: 640x640 Done. (0.007s)
image 468/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/519.png: 640x640 Done. (0.007s)
image 469/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/52.png: 640x640 Done. (0.007s)
image 470/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/520.png: 640x640 1 30, Done. (0.009s)
image 471/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/521.png: 640x640 Done. (0.017s)
image 472/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/522.png: 640x640 Done. (0.015s)
image 473/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/523.png: 640x640 1 30, Done. (0.009s)
image 474/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/524.png: 640x640 1 48, Done. (0.009s)
image 475/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/525.png: 640x640 1 30, Done. (0.007s)
image 476/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/526.png: 640x640 Done. (0.007s)
image 477/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/527.png: 640x640 1 30, Done. (0.009s)
image 478/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/528.png: 640x640 1 48, Done. (0.007s)
image 479/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/529.png: 640x640 1 48, Done. (0.007s)
image 480/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/53.png: 640x640 1 48, Done. (0.010s)
image 481/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/530.png: 640x640 Done. (0.007s)
image 482/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/531.png: 640x640 Done. (0.007s)
image 483/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/532.png: 640x640 Done. (0.007s)
image 484/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/533.png: 640x640 1 48, Done. (0.007s)
image 485/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/534.png: 640x640 1 30, Done. (0.010s)
image 486/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/535.png: 640x640 1 30, Done. (0.007s)
image 487/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/536.png: 640x640 Done. (0.007s)
image 488/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/537.png: 640x640 Done. (0.007s)
image 489/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/538.png: 640x640 1 30, Done. (0.007s)
image 490/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/539.png: 640x640 1 48, Done. (0.007s)
image 491/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/54.png: 640x640 Done. (0.007s)
image 492/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/540.png: 640x640 Done. (0.007s)
image 493/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/541.png: 640x640 Done. (0.007s)
image 494/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/542.png: 640x640 1 30, Done. (0.007s)
image 495/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/543.png: 640x640 Done. (0.007s)
image 496/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/544.png: 640x640 1 30, Done. (0.007s)
image 497/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/545.png: 640x640 1 30, Done. (0.007s)
image 498/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/546.png: 640x640 Done. (0.007s)
image 499/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/547.png: 640x640 1 30, Done. (0.010s)
image 500/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/548.png: 640x640 1 30, Done. (0.011s)
image 501/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/549.png: 640x640 Done. (0.007s)
image 502/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/55.png: 640x640 1 30, Done. (0.007s)
image 503/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/550.png: 640x640 Done. (0.011s)
image 504/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/551.png: 640x640 Done. (0.007s)
image 505/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/552.png: 640x640 1 48, Done. (0.009s)
image 506/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/553.png: 640x640 Done. (0.007s)
image 507/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/554.png: 640x640 1 30, Done. (0.007s)
image 508/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/555.png: 640x640 Done. (0.009s)
image 509/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/556.png: 640x640 Done. (0.007s)
image 510/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/557.png: 640x640 Done. (0.007s)
image 511/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/558.png: 640x640 Done. (0.009s)
image 512/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/559.png: 640x640 Done. (0.007s)
image 513/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/56.png: 640x640 1 30, Done. (0.007s)
image 514/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/560.png: 640x640 Done. (0.007s)
image 515/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/561.png: 640x640 1 30, Done. (0.007s)
image 516/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/562.png: 640x640 Done. (0.009s)
image 517/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/563.png: 640x640 Done. (0.008s)
image 518/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/564.png: 640x640 Done. (0.012s)
image 519/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/565.png: 640x640 Done. (0.007s)
image 520/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/566.png: 640x640 Done. (0.007s)
image 521/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/567.png: 640x640 Done. (0.007s)
image 522/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/568.png: 640x640 1 48, Done. (0.007s)
image 523/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/569.png: 640x640 Done. (0.010s)
image 524/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/57.png: 640x640 Done. (0.007s)
image 525/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/570.png: 640x640 1 48, Done. (0.007s)
image 526/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/571.png: 640x640 1 30, Done. (0.010s)
image 527/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/572.png: 640x640 Done. (0.009s)
image 528/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/573.png: 640x640 Done. (0.007s)
image 529/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/574.png: 640x640 1 48, Done. (0.009s)
image 530/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/575.png: 640x640 Done. (0.007s)
image 531/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/576.png: 640x640 1 48, Done. (0.007s)
image 532/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/577.png: 640x640 Done. (0.007s)
image 533/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/578.png: 640x640 Done. (0.007s)
image 534/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/579.png: 640x640 Done. (0.007s)
image 535/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/58.png: 640x640 Done. (0.007s)
image 536/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/580.png: 640x640 1 30, Done. (0.007s)
image 537/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/581.png: 640x640 1 48, Done. (0.007s)
image 538/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/582.png: 640x640 1 48, Done. (0.007s)
image 539/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/583.png: 640x640 Done. (0.012s)
image 540/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/584.png: 640x640 1 30, Done. (0.009s)
image 541/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/585.png: 640x640 Done. (0.010s)
image 542/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/586.png: 640x640 Done. (0.007s)
image 543/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/587.png: 640x640 1 30, Done. (0.009s)
image 544/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/588.png: 640x640 1 30, Done. (0.007s)
image 545/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/589.png: 640x640 Done. (0.007s)
image 546/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/59.png: 640x640 1 48, Done. (0.010s)
image 547/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/590.png: 640x640 1 30, Done. (0.008s)
image 548/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/591.png: 640x640 Done. (0.009s)
image 549/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/592.png: 640x640 Done. (0.009s)
image 550/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/593.png: 640x640 1 30, Done. (0.007s)
image 551/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/594.png: 640x640 1 30, Done. (0.012s)
image 552/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/595.png: 640x640 Done. (0.007s)
image 553/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/596.png: 640x640 Done. (0.007s)
image 554/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/597.png: 640x640 1 30, Done. (0.007s)
image 555/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/598.png: 640x640 1 48, Done. (0.007s)
image 556/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/599.png: 640x640 1 30, Done. (0.007s)
image 557/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/6.png: 640x640 1 30, Done. (0.009s)
image 558/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/60.png: 640x640 1 30, Done. (0.007s)
image 559/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/600.png: 640x640 Done. (0.007s)
image 560/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/601.png: 640x640 1 48, Done. (0.007s)
image 561/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/602.png: 640x640 1 30, Done. (0.007s)
image 562/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/603.png: 640x640 1 30, Done. (0.007s)
image 563/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/604.png: 640x640 Done. (0.007s)
image 564/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/605.png: 640x640 Done. (0.007s)
image 565/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/606.png: 640x640 Done. (0.007s)
image 566/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/607.png: 640x640 Done. (0.007s)
image 567/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/608.png: 640x640 1 30, Done. (0.007s)
image 568/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/609.png: 640x640 Done. (0.010s)
image 569/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/61.png: 640x640 Done. (0.008s)
image 570/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/610.png: 640x640 Done. (0.007s)
image 571/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/611.png: 640x640 1 30, Done. (0.007s)
image 572/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/612.png: 640x640 Done. (0.009s)
image 573/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/613.png: 640x640 Done. (0.007s)
image 574/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/614.png: 640x640 Done. (0.007s)
image 575/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/615.png: 640x640 Done. (0.007s)
image 576/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/616.png: 640x640 Done. (0.009s)
image 577/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/617.png: 640x640 Done. (0.009s)
image 578/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/618.png: 640x640 Done. (0.007s)
image 579/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/619.png: 640x640 1 30, Done. (0.007s)
image 580/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/62.png: 640x640 Done. (0.007s)
image 581/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/620.png: 640x640 1 48, Done. (0.007s)
image 582/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/621.png: 640x640 1 48, Done. (0.007s)
image 583/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/622.png: 640x640 Done. (0.009s)
image 584/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/623.png: 640x640 Done. (0.012s)
image 585/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/624.png: 640x640 1 48, Done. (0.007s)
image 586/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/625.png: 640x640 1 30, Done. (0.007s)
image 587/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/626.png: 640x640 1 30, Done. (0.007s)
image 588/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/627.png: 640x640 1 30, Done. (0.007s)
image 589/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/628.png: 640x640 1 30, Done. (0.007s)
image 590/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/629.png: 640x640 Done. (0.007s)
image 591/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/63.png: 640x640 1 48, Done. (0.008s)
image 592/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/630.png: 640x640 1 30, Done. (0.007s)
image 593/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/631.png: 640x640 1 30, Done. (0.007s)
image 594/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/632.png: 640x640 Done. (0.007s)
image 595/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/633.png: 640x640 1 30, Done. (0.007s)
image 596/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/634.png: 640x640 Done. (0.011s)
image 597/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/635.png: 640x640 1 30, Done. (0.008s)
image 598/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/636.png: 640x640 1 48, Done. (0.009s)
image 599/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/637.png: 640x640 Done. (0.007s)
image 600/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/638.png: 640x640 Done. (0.008s)
image 601/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/639.png: 640x640 Done. (0.007s)
image 602/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/64.png: 640x640 1 30, Done. (0.007s)
image 603/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/640.png: 640x640 Done. (0.012s)
image 604/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/641.png: 640x640 Done. (0.007s)
image 605/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/642.png: 640x640 Done. (0.007s)
image 606/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/643.png: 640x640 1 30, Done. (0.007s)
image 607/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/644.png: 640x640 Done. (0.009s)
image 608/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/645.png: 640x640 Done. (0.010s)
image 609/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/646.png: 640x640 Done. (0.008s)
image 610/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/647.png: 640x640 Done. (0.010s)
image 611/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/648.png: 640x640 1 48, Done. (0.007s)
image 612/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/649.png: 640x640 Done. (0.007s)
image 613/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/65.png: 640x640 Done. (0.008s)
image 614/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/650.png: 640x640 1 30, Done. (0.007s)
image 615/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/651.png: 640x640 Done. (0.007s)
image 616/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/652.png: 640x640 Done. (0.007s)
image 617/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/653.png: 640x640 Done. (0.007s)
image 618/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/654.png: 640x640 1 30, Done. (0.007s)
image 619/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/655.png: 640x640 1 30, Done. (0.007s)
image 620/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/656.png: 640x640 Done. (0.007s)
image 621/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/657.png: 640x640 Done. (0.012s)
image 622/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/658.png: 640x640 1 48, Done. (0.009s)
image 623/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/659.png: 640x640 1 48, Done. (0.007s)
image 624/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/66.png: 640x640 Done. (0.007s)
image 625/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/660.png: 640x640 Done. (0.007s)
image 626/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/661.png: 640x640 1 48, Done. (0.010s)
image 627/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/662.png: 640x640 1 48, Done. (0.007s)
image 628/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/663.png: 640x640 1 48, Done. (0.009s)
image 629/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/664.png: 640x640 1 30, Done. (0.007s)
image 630/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/665.png: 640x640 Done. (0.007s)
image 631/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/666.png: 640x640 Done. (0.007s)
image 632/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/667.png: 640x640 Done. (0.007s)
image 633/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/668.png: 640x640 Done. (0.007s)
image 634/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/669.png: 640x640 Done. (0.007s)
image 635/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/67.png: 640x640 Done. (0.007s)
image 636/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/670.png: 640x640 1 48, Done. (0.007s)
image 637/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/671.png: 640x640 Done. (0.007s)
image 638/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/672.png: 640x640 Done. (0.008s)
image 639/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/673.png: 640x640 1 48, Done. (0.007s)
image 640/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/674.png: 640x640 1 48, Done. (0.007s)
image 641/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/675.png: 640x640 Done. (0.010s)
image 642/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/676.png: 640x640 Done. (0.007s)
image 643/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/677.png: 640x640 Done. (0.007s)
image 644/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/678.png: 640x640 1 30, Done. (0.011s)
image 645/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/679.png: 640x640 Done. (0.010s)
image 646/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/68.png: 640x640 1 30, Done. (0.007s)
image 647/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/680.png: 640x640 1 30, Done. (0.007s)
image 648/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/681.png: 640x640 Done. (0.007s)
image 649/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/682.png: 640x640 Done. (0.011s)
image 650/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/683.png: 640x640 Done. (0.007s)
image 651/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/684.png: 640x640 1 48, Done. (0.007s)
image 652/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/685.png: 640x640 1 48, Done. (0.007s)
image 653/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/686.png: 640x640 1 30, Done. (0.015s)
image 654/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/687.png: 640x640 Done. (0.010s)
image 655/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/688.png: 640x640 1 48, Done. (0.011s)
image 656/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/689.png: 640x640 1 30, Done. (0.010s)
image 657/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/69.png: 640x640 1 30, Done. (0.007s)
image 658/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/690.png: 640x640 Done. (0.009s)
image 659/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/691.png: 640x640 Done. (0.007s)
image 660/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/692.png: 640x640 Done. (0.009s)
image 661/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/693.png: 640x640 1 30, Done. (0.007s)
image 662/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/694.png: 640x640 1 48, Done. (0.007s)
image 663/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/695.png: 640x640 1 30, Done. (0.010s)
image 664/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/696.png: 640x640 1 48, Done. (0.007s)
image 665/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/697.png: 640x640 Done. (0.007s)
image 666/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/698.png: 640x640 1 30, Done. (0.009s)
image 667/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/699.png: 640x640 Done. (0.008s)
image 668/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/7.png: 640x640 Done. (0.007s)
image 669/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/70.png: 640x640 Done. (0.007s)
image 670/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/700.png: 640x640 Done. (0.011s)
image 671/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/701.png: 640x640 1 30, Done. (0.007s)
image 672/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/702.png: 640x640 Done. (0.007s)
image 673/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/703.png: 640x640 1 30, Done. (0.007s)
image 674/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/704.png: 640x640 Done. (0.007s)
image 675/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/705.png: 640x640 1 48, Done. (0.007s)
image 676/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/706.png: 640x640 1 48, Done. (0.007s)
image 677/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/707.png: 640x640 Done. (0.011s)
image 678/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/708.png: 640x640 Done. (0.007s)
image 679/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/709.png: 640x640 Done. (0.009s)
image 680/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/71.png: 640x640 Done. (0.007s)
image 681/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/710.png: 640x640 1 30, Done. (0.010s)
image 682/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/711.png: 640x640 1 30, Done. (0.007s)
image 683/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/712.png: 640x640 Done. (0.007s)
image 684/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/713.png: 640x640 1 30, Done. (0.007s)
image 685/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/714.png: 640x640 Done. (0.007s)
image 686/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/715.png: 640x640 Done. (0.007s)
image 687/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/716.png: 640x640 1 48, Done. (0.009s)
image 688/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/717.png: 640x640 Done. (0.009s)
image 689/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/718.png: 640x640 1 48, Done. (0.010s)
image 690/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/719.png: 640x640 Done. (0.007s)
image 691/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/72.png: 640x640 1 30, Done. (0.011s)
image 692/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/720.png: 640x640 1 48, Done. (0.007s)
image 693/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/721.png: 640x640 Done. (0.017s)
image 694/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/722.png: 640x640 1 30, Done. (0.010s)
image 695/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/723.png: 640x640 Done. (0.007s)
image 696/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/724.png: 640x640 Done. (0.007s)
image 697/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/725.png: 640x640 1 30, Done. (0.006s)
image 698/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/726.png: 640x640 Done. (0.007s)
image 699/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/727.png: 640x640 Done. (0.010s)
image 700/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/728.png: 640x640 1 48, Done. (0.015s)
image 701/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/729.png: 640x640 Done. (0.007s)
image 702/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/73.png: 640x640 1 48, Done. (0.009s)
image 703/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/730.png: 640x640 1 30, Done. (0.007s)
image 704/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/731.png: 640x640 1 48, Done. (0.009s)
image 705/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/732.png: 640x640 1 30, Done. (0.008s)
image 706/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/733.png: 640x640 Done. (0.007s)
image 707/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/734.png: 640x640 Done. (0.008s)
image 708/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/735.png: 640x640 Done. (0.007s)
image 709/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/736.png: 640x640 Done. (0.007s)
image 710/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/737.png: 640x640 Done. (0.011s)
image 711/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/738.png: 640x640 1 30, Done. (0.008s)
image 712/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/739.png: 640x640 1 48, Done. (0.011s)
image 713/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/74.png: 640x640 1 30, Done. (0.009s)
image 714/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/740.png: 640x640 Done. (0.007s)
image 715/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/741.png: 640x640 1 30, Done. (0.007s)
image 716/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/742.png: 640x640 Done. (0.008s)
image 717/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/743.png: 640x640 1 30, Done. (0.010s)
image 718/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/744.png: 640x640 Done. (0.007s)
image 719/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/745.png: 640x640 1 30, Done. (0.007s)
image 720/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/746.png: 640x640 1 48, Done. (0.007s)
image 721/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/747.png: 640x640 1 30, Done. (0.007s)
image 722/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/748.png: 640x640 1 30, Done. (0.009s)
image 723/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/749.png: 640x640 Done. (0.007s)
image 724/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/75.png: 640x640 1 48, Done. (0.010s)
image 725/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/750.png: 640x640 1 48, Done. (0.011s)
image 726/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/751.png: 640x640 Done. (0.011s)
image 727/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/752.png: 640x640 1 30, Done. (0.014s)
image 728/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/753.png: 640x640 Done. (0.007s)
image 729/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/754.png: 640x640 1 48, Done. (0.010s)
image 730/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/755.png: 640x640 1 30, Done. (0.009s)
image 731/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/756.png: 640x640 Done. (0.007s)
image 732/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/757.png: 640x640 1 30, Done. (0.007s)
image 733/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/758.png: 640x640 1 30, Done. (0.009s)
image 734/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/759.png: 640x640 Done. (0.007s)
image 735/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/76.png: 640x640 1 48, Done. (0.014s)
image 736/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/760.png: 640x640 1 30, Done. (0.007s)
image 737/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/761.png: 640x640 Done. (0.008s)
image 738/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/762.png: 640x640 1 30, Done. (0.008s)
image 739/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/763.png: 640x640 Done. (0.010s)
image 740/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/764.png: 640x640 1 48, Done. (0.009s)
image 741/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/765.png: 640x640 1 48, Done. (0.007s)
image 742/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/766.png: 640x640 Done. (0.007s)
image 743/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/767.png: 640x640 Done. (0.007s)
image 744/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/768.png: 640x640 Done. (0.007s)
image 745/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/769.png: 640x640 Done. (0.007s)
image 746/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/77.png: 640x640 Done. (0.007s)
image 747/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/770.png: 640x640 1 30, Done. (0.009s)
image 748/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/771.png: 640x640 Done. (0.008s)
image 749/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/772.png: 640x640 Done. (0.007s)
image 750/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/773.png: 640x640 Done. (0.010s)
image 751/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/774.png: 640x640 Done. (0.007s)
image 752/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/775.png: 640x640 Done. (0.008s)
image 753/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/776.png: 640x640 Done. (0.007s)
image 754/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/777.png: 640x640 Done. (0.007s)
image 755/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/778.png: 640x640 1 30, Done. (0.007s)
image 756/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/779.png: 640x640 1 30, Done. (0.007s)
image 757/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/78.png: 640x640 Done. (0.007s)
image 758/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/780.png: 640x640 1 30, Done. (0.009s)
image 759/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/781.png: 640x640 1 48, Done. (0.007s)
image 760/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/782.png: 640x640 1 30, Done. (0.007s)
image 761/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/783.png: 640x640 1 48, Done. (0.017s)
image 762/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/784.png: 640x640 1 48, Done. (0.010s)
image 763/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/785.png: 640x640 1 30, Done. (0.007s)
image 764/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/786.png: 640x640 Done. (0.007s)
image 765/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/787.png: 640x640 Done. (0.007s)
image 766/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/788.png: 640x640 Done. (0.007s)
image 767/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/789.png: 640x640 Done. (0.007s)
image 768/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/79.png: 640x640 1 48, Done. (0.007s)
image 769/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/790.png: 640x640 Done. (0.011s)
image 770/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/791.png: 640x640 Done. (0.007s)
image 771/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/792.png: 640x640 1 30, Done. (0.007s)
image 772/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/793.png: 640x640 1 48, Done. (0.010s)
image 773/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/794.png: 640x640 1 30, Done. (0.009s)
image 774/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/795.png: 640x640 1 30, Done. (0.007s)
image 775/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/796.png: 640x640 1 30, Done. (0.007s)
image 776/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/797.png: 640x640 Done. (0.008s)
image 777/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/798.png: 640x640 Done. (0.007s)
image 778/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/799.png: 640x640 Done. (0.007s)
image 779/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/8.png: 640x640 Done. (0.008s)
image 780/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/80.png: 640x640 Done. (0.009s)
image 781/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/800.png: 640x640 Done. (0.016s)
image 782/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/801.png: 640x640 1 30, Done. (0.009s)
image 783/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/802.png: 640x640 1 30, Done. (0.009s)
image 784/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/803.png: 640x640 1 30, Done. (0.007s)
image 785/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/804.png: 640x640 1 30, Done. (0.007s)
image 786/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/805.png: 640x640 Done. (0.007s)
image 787/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/806.png: 640x640 Done. (0.009s)
image 788/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/807.png: 640x640 1 30, Done. (0.007s)
image 789/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/808.png: 640x640 1 30, Done. (0.007s)
image 790/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/809.png: 640x640 1 48, Done. (0.007s)
image 791/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/81.png: 640x640 Done. (0.007s)
image 792/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/810.png: 640x640 1 30, Done. (0.012s)
image 793/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/811.png: 640x640 1 48, Done. (0.009s)
image 794/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/812.png: 640x640 Done. (0.007s)
image 795/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/813.png: 640x640 Done. (0.007s)
image 796/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/814.png: 640x640 Done. (0.009s)
image 797/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/815.png: 640x640 1 30, Done. (0.007s)
image 798/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/816.png: 640x640 Done. (0.007s)
image 799/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/817.png: 640x640 Done. (0.014s)
image 800/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/818.png: 640x640 1 30, Done. (0.007s)
image 801/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/819.png: 640x640 Done. (0.009s)
image 802/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/82.png: 640x640 Done. (0.011s)
image 803/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/820.png: 640x640 1 30, Done. (0.010s)
image 804/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/821.png: 640x640 Done. (0.007s)
image 805/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/822.png: 640x640 1 48, Done. (0.008s)
image 806/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/823.png: 640x640 1 30, Done. (0.007s)
image 807/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/824.png: 640x640 Done. (0.007s)
image 808/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/825.png: 640x640 Done. (0.007s)
image 809/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/826.png: 640x640 Done. (0.010s)
image 810/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/827.png: 640x640 Done. (0.007s)
image 811/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/828.png: 640x640 1 30, Done. (0.007s)
image 812/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/829.png: 640x640 1 48, Done. (0.007s)
image 813/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/83.png: 640x640 Done. (0.007s)
image 814/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/830.png: 640x640 1 30, Done. (0.009s)
image 815/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/831.png: 640x640 1 30, Done. (0.007s)
image 816/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/832.png: 640x640 1 30, Done. (0.007s)
image 817/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/833.png: 640x640 Done. (0.007s)
image 818/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/834.png: 640x640 Done. (0.007s)
image 819/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/835.png: 640x640 1 48, Done. (0.007s)
image 820/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/836.png: 640x640 1 48, Done. (0.010s)
image 821/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/837.png: 640x640 1 30, Done. (0.007s)
image 822/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/838.png: 640x640 1 48, Done. (0.016s)
image 823/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/839.png: 640x640 1 48, Done. (0.010s)
image 824/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/84.png: 640x640 1 48, Done. (0.007s)
image 825/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/840.png: 640x640 Done. (0.010s)
image 826/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/841.png: 640x640 Done. (0.007s)
image 827/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/842.png: 640x640 Done. (0.007s)
image 828/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/843.png: 640x640 1 30, Done. (0.010s)
image 829/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/844.png: 640x640 1 48, Done. (0.007s)
image 830/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/845.png: 640x640 Done. (0.011s)
image 831/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/846.png: 640x640 1 30, Done. (0.007s)
image 832/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/847.png: 640x640 1 48, Done. (0.007s)
image 833/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/848.png: 640x640 1 30, Done. (0.007s)
image 834/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/849.png: 640x640 1 30, Done. (0.007s)
image 835/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/85.png: 640x640 Done. (0.007s)
image 836/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/850.png: 640x640 Done. (0.007s)
image 837/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/851.png: 640x640 1 48, Done. (0.007s)
image 838/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/852.png: 640x640 1 48, Done. (0.007s)
image 839/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/853.png: 640x640 1 48, Done. (0.007s)
image 840/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/854.png: 640x640 Done. (0.009s)
image 841/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/855.png: 640x640 1 30, Done. (0.007s)
image 842/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/856.png: 640x640 1 48, Done. (0.007s)
image 843/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/857.png: 640x640 1 48, Done. (0.007s)
image 844/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/858.png: 640x640 Done. (0.007s)
image 845/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/859.png: 640x640 Done. (0.007s)
image 846/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/86.png: 640x640 1 48, Done. (0.007s)
image 847/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/860.png: 640x640 Done. (0.010s)
image 848/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/861.png: 640x640 Done. (0.009s)
image 849/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/862.png: 640x640 Done. (0.007s)
image 850/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/863.png: 640x640 1 30, Done. (0.007s)
image 851/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/864.png: 640x640 1 48, Done. (0.007s)
image 852/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/865.png: 640x640 1 48, Done. (0.009s)
image 853/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/866.png: 640x640 1 30, Done. (0.009s)
image 854/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/867.png: 640x640 Done. (0.009s)
image 855/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/868.png: 640x640 Done. (0.007s)
image 856/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/869.png: 640x640 Done. (0.010s)
image 857/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/87.png: 640x640 Done. (0.007s)
image 858/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/870.png: 640x640 1 48, Done. (0.011s)
image 859/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/871.png: 640x640 1 48, Done. (0.007s)
image 860/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/872.png: 640x640 Done. (0.009s)
image 861/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/873.png: 640x640 Done. (0.010s)
image 862/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/874.png: 640x640 Done. (0.009s)
image 863/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/875.png: 640x640 Done. (0.007s)
image 864/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/876.png: 640x640 1 30, Done. (0.009s)
image 865/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/877.png: 640x640 Done. (0.007s)
image 866/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/878.png: 640x640 1 48, Done. (0.018s)
image 867/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/879.png: 640x640 Done. (0.007s)
image 868/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/88.png: 640x640 Done. (0.007s)
image 869/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/880.png: 640x640 Done. (0.007s)
image 870/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/881.png: 640x640 Done. (0.007s)
image 871/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/882.png: 640x640 1 30, Done. (0.009s)
image 872/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/883.png: 640x640 1 48, Done. (0.007s)
image 873/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/884.png: 640x640 Done. (0.009s)
image 874/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/885.png: 640x640 Done. (0.011s)
image 875/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/886.png: 640x640 Done. (0.010s)
image 876/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/887.png: 640x640 1 48, Done. (0.007s)
image 877/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/888.png: 640x640 1 48, Done. (0.008s)
image 878/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/889.png: 640x640 1 30, Done. (0.007s)
image 879/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/89.png: 640x640 Done. (0.011s)
image 880/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/890.png: 640x640 Done. (0.009s)
image 881/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/891.png: 640x640 1 30, Done. (0.016s)
image 882/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/892.png: 640x640 Done. (0.007s)
image 883/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/893.png: 640x640 Done. (0.011s)
image 884/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/894.png: 640x640 1 48, Done. (0.009s)
image 885/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/895.png: 640x640 Done. (0.016s)
image 886/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/896.png: 640x640 Done. (0.011s)
image 887/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/897.png: 640x640 Done. (0.007s)
image 888/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/898.png: 640x640 Done. (0.007s)
image 889/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/899.png: 640x640 1 30, Done. (0.009s)
image 890/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/9.png: 640x640 Done. (0.007s)
image 891/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/90.png: 640x640 1 48, Done. (0.010s)
image 892/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/900.png: 640x640 1 30, Done. (0.007s)
image 893/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/901.png: 640x640 1 30, Done. (0.007s)
image 894/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/902.png: 640x640 1 30, Done. (0.010s)
image 895/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/903.png: 640x640 Done. (0.007s)
image 896/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/904.png: 640x640 1 48, Done. (0.008s)
image 897/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/905.png: 640x640 Done. (0.010s)
image 898/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/906.png: 640x640 Done. (0.007s)
image 899/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/907.png: 640x640 Done. (0.010s)
image 900/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/908.png: 640x640 1 30, Done. (0.011s)
image 901/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/909.png: 640x640 Done. (0.007s)
image 902/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/91.png: 640x640 1 30, Done. (0.007s)
image 903/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/910.png: 640x640 1 48, Done. (0.007s)
image 904/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/911.png: 640x640 1 30, Done. (0.007s)
image 905/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/912.png: 640x640 1 48, Done. (0.007s)
image 906/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/913.png: 640x640 1 48, Done. (0.009s)
image 907/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/914.png: 640x640 Done. (0.007s)
image 908/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/915.png: 640x640 Done. (0.007s)
image 909/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/916.png: 640x640 1 48, Done. (0.007s)
image 910/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/917.png: 640x640 Done. (0.007s)
image 911/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/918.png: 640x640 1 48, Done. (0.007s)
image 912/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/919.png: 640x640 1 30, Done. (0.007s)
image 913/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/92.png: 640x640 1 30, Done. (0.007s)
image 914/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/920.png: 640x640 1 30, Done. (0.010s)
image 915/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/921.png: 640x640 1 48, Done. (0.009s)
image 916/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/922.png: 640x640 Done. (0.007s)
image 917/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/923.png: 640x640 1 30, Done. (0.007s)
image 918/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/924.png: 640x640 1 30, Done. (0.007s)
image 919/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/925.png: 640x640 Done. (0.007s)
image 920/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/926.png: 640x640 Done. (0.007s)
image 921/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/927.png: 640x640 1 30, Done. (0.011s)
image 922/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/928.png: 640x640 Done. (0.008s)
image 923/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/929.png: 640x640 Done. (0.007s)
image 924/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/93.png: 640x640 Done. (0.007s)
image 925/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/930.png: 640x640 1 48, Done. (0.013s)
image 926/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/931.png: 640x640 1 30, Done. (0.009s)
image 927/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/932.png: 640x640 Done. (0.007s)
image 928/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/933.png: 640x640 Done. (0.008s)
image 929/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/934.png: 640x640 Done. (0.007s)
image 930/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/935.png: 640x640 Done. (0.007s)
image 931/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/936.png: 640x640 Done. (0.007s)
image 932/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/937.png: 640x640 Done. (0.007s)
image 933/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/938.png: 640x640 Done. (0.009s)
image 934/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/939.png: 640x640 Done. (0.007s)
image 935/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/94.png: 640x640 Done. (0.007s)
image 936/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/940.png: 640x640 1 30, Done. (0.009s)
image 937/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/941.png: 640x640 1 48, Done. (0.007s)
image 938/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/942.png: 640x640 1 48, Done. (0.017s)
image 939/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/943.png: 640x640 Done. (0.009s)
image 940/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/944.png: 640x640 Done. (0.007s)
image 941/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/945.png: 640x640 1 30, Done. (0.007s)
image 942/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/946.png: 640x640 1 48, Done. (0.011s)
image 943/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/947.png: 640x640 1 48, Done. (0.007s)
image 944/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/948.png: 640x640 Done. (0.007s)
image 945/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/949.png: 640x640 Done. (0.007s)
image 946/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/95.png: 640x640 1 48, Done. (0.007s)
image 947/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/950.png: 640x640 Done. (0.007s)
image 948/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/951.png: 640x640 Done. (0.011s)
image 949/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/952.png: 640x640 1 48, Done. (0.007s)
image 950/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/953.png: 640x640 1 30, Done. (0.012s)
image 951/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/954.png: 640x640 Done. (0.007s)
image 952/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/955.png: 640x640 Done. (0.007s)
image 953/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/956.png: 640x640 1 30, Done. (0.009s)
image 954/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/957.png: 640x640 Done. (0.007s)
image 955/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/958.png: 640x640 Done. (0.007s)
image 956/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/959.png: 640x640 Done. (0.007s)
image 957/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/96.png: 640x640 Done. (0.010s)
image 958/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/960.png: 640x640 1 48, Done. (0.007s)
image 959/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/961.png: 640x640 Done. (0.010s)
image 960/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/962.png: 640x640 1 48, Done. (0.010s)
image 961/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/963.png: 640x640 1 48, Done. (0.007s)
image 962/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/964.png: 640x640 Done. (0.007s)
image 963/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/965.png: 640x640 1 48, Done. (0.007s)
image 964/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/966.png: 640x640 1 30, Done. (0.007s)
image 965/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/967.png: 640x640 1 48, Done. (0.007s)
image 966/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/968.png: 640x640 Done. (0.007s)
image 967/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/969.png: 640x640 1 30, Done. (0.014s)
image 968/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/97.png: 640x640 1 48, Done. (0.009s)
image 969/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/970.png: 640x640 1 30, Done. (0.007s)
image 970/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/971.png: 640x640 1 30, Done. (0.007s)
image 971/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/972.png: 640x640 1 30, Done. (0.013s)
image 972/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/973.png: 640x640 1 30, Done. (0.010s)
image 973/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/974.png: 640x640 1 48, Done. (0.010s)
image 974/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/975.png: 640x640 1 30, Done. (0.009s)
image 975/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/976.png: 640x640 Done. (0.007s)
image 976/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/977.png: 640x640 Done. (0.007s)
image 977/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/978.png: 640x640 Done. (0.007s)
image 978/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/979.png: 640x640 Done. (0.007s)
image 979/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/98.png: 640x640 Done. (0.007s)
image 980/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/980.png: 640x640 1 48, Done. (0.009s)
image 981/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/981.png: 640x640 Done. (0.007s)
image 982/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/982.png: 640x640 Done. (0.010s)
image 983/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/983.png: 640x640 1 30, Done. (0.007s)
image 984/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/984.png: 640x640 1 30, Done. (0.007s)
image 985/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/985.png: 640x640 Done. (0.007s)
image 986/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/986.png: 640x640 1 30, Done. (0.006s)
image 987/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/987.png: 640x640 Done. (0.011s)
image 988/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/988.png: 640x640 1 30, Done. (0.007s)
image 989/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/989.png: 640x640 Done. (0.007s)
image 990/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/99.png: 640x640 1 30, Done. (0.007s)
image 991/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/990.png: 640x640 1 30, Done. (0.007s)
image 992/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/991.png: 640x640 1 30, Done. (0.009s)
image 993/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/992.png: 640x640 Done. (0.007s)
image 994/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/993.png: 640x640 Done. (0.007s)
image 995/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/994.png: 640x640 Done. (0.007s)
image 996/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/995.png: 640x640 Done. (0.012s)
image 997/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/996.png: 640x640 Done. (0.009s)
image 998/998 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_test/images/test/997.png: 640x640 Done. (0.007s)
Speed: 0.4ms pre-process, 8.2ms inference, 0.4ms NMS per image at shape (1, 3, 640, 640)
Results saved to ../../yolov5/runs/detect/hybrid_imagery_example_inference
View some detections:
[51]:
detections_dir = "/home/mrmarsh/repos/yolov5/runs/detect/hybrid_imagery_example_inference/"
detection_images = [os.path.join(detections_dir, x) for x in os.listdir(detections_dir)]
for i in range(5):
random_detection_image = Image.open(random.choice(detection_images))
plt.figure(figsize=(9, 6))
plt.imshow(np.array(random_detection_image))
Train synthetic, test synthetic is behaving the way we would expect. The training and testing set are drawn from the same distribution so inference should perform well.
[47]:
!python /home/mrmarsh/repos/yolov5/detect.py \
--source /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/ \
--weights /home/mrmarsh/repos/yolov5/runs/train/hybrid_imagery_example/weights/best.pt \
--conf 0.25 \
--name hybrid_imagery_example_train_syn_test_real
detect: weights=['/home/mrmarsh/repos/yolov5/runs/train/hybrid_imagery_example/weights/best.pt'], source=/home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/, data=../../yolov5/data/coco128.yaml, imgsz=[640, 640], conf_thres=0.25, iou_thres=0.45, max_det=1000, device=, view_img=False, save_txt=False, save_conf=False, save_crop=False, nosave=False, classes=None, agnostic_nms=False, augment=False, visualize=False, update=False, project=../../yolov5/runs/detect, name=hybrid_imagery_example_train_syn_test_real, exist_ok=False, line_thickness=3, hide_labels=False, hide_conf=False, half=False, dnn=False
YOLOv5 🚀 v6.1-177-gd059d1d torch 1.11.0 CUDA:0 (Quadro RTX 6000, 24198MiB)
Fusing layers...
YOLOv5s summary: 213 layers, 7018216 parameters, 0 gradients, 15.8 GFLOPs
image 1/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/0.png: 512x640 1 30, Done. (0.013s)
image 2/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/1.png: 416x640 1 30, Done. (0.010s)
image 3/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/10.png: 480x640 1 30, Done. (0.011s)
image 4/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/100.png: 512x640 Done. (0.007s)
image 5/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/101.png: 512x640 2 48s, Done. (0.007s)
image 6/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/102.png: 512x640 1 48, Done. (0.009s)
image 7/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/103.png: 448x640 2 48s, Done. (0.011s)
image 8/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/104.png: 512x640 3 48s, Done. (0.007s)
image 9/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/105.png: 512x640 Done. (0.007s)
image 10/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/106.png: 512x640 Done. (0.011s)
image 11/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/107.png: 448x640 1 48, Done. (0.007s)
image 12/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/108.png: 448x640 Done. (0.006s)
image 13/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/109.png: 352x640 Done. (0.010s)
image 14/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/11.png: 448x640 1 48, Done. (0.007s)
image 15/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/110.png: 640x544 1 48, Done. (0.011s)
image 16/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/111.png: 352x640 Done. (0.011s)
image 17/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/112.png: 480x640 1 48, Done. (0.010s)
image 18/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/113.png: 384x640 Done. (0.011s)
image 19/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/114.png: 480x640 Done. (0.009s)
image 20/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/115.png: 480x640 Done. (0.006s)
image 21/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/116.png: 384x640 Done. (0.007s)
image 22/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/117.png: 448x640 2 48s, Done. (0.007s)
image 23/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/118.png: 256x640 1 48, Done. (0.010s)
image 24/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/119.png: 416x640 1 48, Done. (0.007s)
image 25/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/12.png: 448x640 Done. (0.008s)
image 26/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/120.png: 384x640 Done. (0.010s)
image 27/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/121.png: 480x640 Done. (0.007s)
image 28/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/122.png: 384x640 Done. (0.006s)
image 29/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/123.png: 448x640 1 48, Done. (0.007s)
image 30/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/124.png: 448x640 2 48s, Done. (0.006s)
image 31/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/125.png: 384x640 1 48, Done. (0.007s)
image 32/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/126.png: 288x640 Done. (0.017s)
image 33/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/127.png: 512x640 Done. (0.007s)
image 34/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/128.png: 544x640 Done. (0.022s)
image 35/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/129.png: 448x640 Done. (0.007s)
image 36/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/13.png: 448x640 3 48s, Done. (0.009s)
image 37/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/130.png: 352x640 Done. (0.008s)
image 38/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/131.png: 480x640 1 48, Done. (0.009s)
image 39/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/132.png: 512x640 1 48, Done. (0.016s)
image 40/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/133.png: 512x640 1 48, Done. (0.007s)
image 41/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/134.png: 448x640 Done. (0.009s)
image 42/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/135.png: 512x640 1 30, Done. (0.017s)
image 43/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/136.png: 448x640 1 48, Done. (0.013s)
image 44/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/14.png: 448x640 Done. (0.007s)
image 45/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/15.png: 480x640 1 30, Done. (0.007s)
image 46/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/16.png: 448x640 3 48s, Done. (0.008s)
image 47/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/17.png: 448x640 1 48, Done. (0.007s)
image 48/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/18.png: 448x640 Done. (0.007s)
image 49/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/19.png: 448x640 Done. (0.007s)
image 50/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/2.png: 608x640 Done. (0.020s)
image 51/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/20.png: 512x640 Done. (0.007s)
image 52/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/21.png: 512x640 1 30, Done. (0.007s)
image 53/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/22.png: 512x640 Done. (0.006s)
image 54/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/23.png: 480x640 4 30s, Done. (0.007s)
image 55/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/24.png: 480x640 Done. (0.007s)
image 56/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/25.png: 480x640 1 30, Done. (0.007s)
image 57/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/26.png: 512x640 Done. (0.015s)
image 58/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/27.png: 480x640 Done. (0.007s)
image 59/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/28.png: 512x640 1 48, Done. (0.007s)
image 60/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/29.png: 512x640 Done. (0.008s)
image 61/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/3.png: 480x640 Done. (0.007s)
image 62/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/30.png: 512x640 Done. (0.007s)
image 63/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/31.png: 384x640 Done. (0.007s)
image 64/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/32.png: 448x640 1 30, Done. (0.007s)
image 65/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/33.png: 640x512 Done. (0.014s)
image 66/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/34.png: 512x640 1 30, Done. (0.007s)
image 67/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/35.png: 480x640 Done. (0.007s)
image 68/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/36.png: 448x640 Done. (0.007s)
image 69/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/37.png: 352x640 1 30, Done. (0.007s)
image 70/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/38.png: 512x640 Done. (0.007s)
image 71/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/39.png: 608x640 Done. (0.009s)
image 72/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/4.png: 480x640 Done. (0.007s)
image 73/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/40.png: 512x640 1 30, 3 48s, Done. (0.007s)
image 74/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/41.png: 352x640 2 30s, Done. (0.007s)
image 75/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/42.png: 480x640 1 48, Done. (0.010s)
image 76/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/43.png: 640x480 1 48, Done. (0.011s)
image 77/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/44.png: 544x640 Done. (0.007s)
image 78/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/45.png: 640x480 1 48, Done. (0.007s)
image 79/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/46.png: 480x640 1 30, Done. (0.013s)
image 80/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/47.png: 448x640 Done. (0.009s)
image 81/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/48.png: 448x640 Done. (0.007s)
image 82/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/49.png: 352x640 1 30, 1 48, Done. (0.007s)
image 83/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/5.png: 480x640 Done. (0.007s)
image 84/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/50.png: 480x640 1 48, Done. (0.011s)
image 85/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/51.png: 512x640 1 48, Done. (0.009s)
image 86/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/52.png: 512x640 2 48s, Done. (0.007s)
image 87/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/53.png: 480x640 Done. (0.007s)
image 88/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/54.png: 352x640 Done. (0.007s)
image 89/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/55.png: 512x640 Done. (0.007s)
image 90/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/56.png: 448x640 1 48, Done. (0.009s)
image 91/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/57.png: 448x640 1 30, Done. (0.007s)
image 92/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/58.png: 480x640 2 48s, Done. (0.009s)
image 93/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/59.png: 448x640 1 48, Done. (0.007s)
image 94/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/6.png: 416x640 Done. (0.007s)
image 95/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/60.png: 352x640 Done. (0.011s)
image 96/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/61.png: 448x640 Done. (0.007s)
image 97/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/62.png: 480x640 2 48s, Done. (0.007s)
image 98/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/63.png: 448x640 1 48, Done. (0.007s)
image 99/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/64.png: 480x640 1 30, 1 48, Done. (0.007s)
image 100/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/65.png: 640x448 Done. (0.010s)
image 101/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/66.png: 640x544 Done. (0.011s)
image 102/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/67.png: 480x640 2 48s, Done. (0.016s)
image 103/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/68.png: 448x640 Done. (0.007s)
image 104/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/69.png: 448x640 1 48, Done. (0.006s)
image 105/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/7.png: 480x640 Done. (0.007s)
image 106/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/70.png: 384x640 Done. (0.007s)
image 107/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/71.png: 480x640 Done. (0.006s)
image 108/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/72.png: 416x640 1 48, Done. (0.013s)
image 109/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/73.png: 448x640 1 48, Done. (0.007s)
image 110/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/74.png: 480x640 Done. (0.007s)
image 111/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/75.png: 480x640 Done. (0.007s)
image 112/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/76.png: 480x640 Done. (0.006s)
image 113/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/77.png: 544x640 1 48, Done. (0.007s)
image 114/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/78.png: 416x640 1 48, Done. (0.010s)
image 115/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/79.png: 448x640 1 48, Done. (0.007s)
image 116/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/8.png: 640x480 Done. (0.007s)
image 117/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/80.png: 448x640 1 48, Done. (0.006s)
image 118/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/81.png: 480x640 1 48, Done. (0.007s)
image 119/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/82.png: 416x640 1 48, Done. (0.007s)
image 120/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/83.png: 480x640 Done. (0.007s)
image 121/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/84.png: 576x640 1 48, Done. (0.012s)
image 122/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/85.png: 480x640 1 48, Done. (0.007s)
image 123/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/86.png: 448x640 Done. (0.007s)
image 124/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/87.png: 512x640 1 48, Done. (0.007s)
image 125/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/88.png: 448x640 Done. (0.008s)
image 126/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/89.png: 640x448 Done. (0.013s)
image 127/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/9.png: 544x640 1 30, Done. (0.010s)
image 128/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/90.png: 224x640 Done. (0.015s)
image 129/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/91.png: 416x640 1 48, Done. (0.007s)
image 130/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/92.png: 416x640 2 48s, Done. (0.007s)
image 131/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/93.png: 480x640 Done. (0.016s)
image 132/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/94.png: 384x640 Done. (0.015s)
image 133/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/95.png: 320x640 1 48, Done. (0.015s)
image 134/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/96.png: 544x640 Done. (0.008s)
image 135/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/97.png: 480x640 Done. (0.007s)
image 136/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/98.png: 192x640 1 48, Done. (0.010s)
image 137/137 /home/mrmarsh/repos/hybrid_imagery/example_notebooks_for_limbo/example_train_synthetic_real_case/images/val/99.png: 416x640 Done. (0.007s)
Speed: 0.4ms pre-process, 8.7ms inference, 0.4ms NMS per image at shape (1, 3, 640, 640)
Results saved to ../../yolov5/runs/detect/hybrid_imagery_example_train_syn_test_real
[48]:
detections_dir = "/home/mrmarsh/repos/yolov5/runs/detect/hybrid_imagery_example_train_syn_test_real/"
detection_images = [os.path.join(detections_dir, x) for x in os.listdir(detections_dir)]
for i in range(5):
random_detection_image = Image.open(random.choice(detection_images))
plt.figure(figsize=(9, 6))
plt.imshow(np.array(random_detection_image))
Not bad given our training set. You will note that the object detector typically only detects one container of interest in a row of containers. By incorporating more campaigns into the training set we could improve performance. Also, in some cases, we do not even detect a contain of interest even though the container is quite obvious. This demonstrates some of the difficulty with training synthetic and testing real. There are differences in the feature spaces that are not readily apparent.
View Loss curves
Looking at the train synthetic test synthetic data, we see some patterns:
Training loss steadily decreases.
Precision/Recall steadily increase until reaching about 1.0 around 50 epochs and hover there for the remainder of the training.
Overall, we can have confidence that the train synthetic, test synthetic use case is working the way we would expect.
Note - plot_results is from yolov5 utils code
[52]:
def plot_results(file='path/to/results.csv', dir=''):
# Plot training results.csv. Usage: from utils.plots import *; plot_results('path/to/results.csv')
save_dir = Path(file).parent if file else Path(dir)
fig, ax = plt.subplots(2, 5, figsize=(12, 6), tight_layout=True)
ax = ax.ravel()
files = list(save_dir.glob('results*.csv'))
assert len(files), f'No results.csv files found in {save_dir.resolve()}, nothing to plot.'
for fi, f in enumerate(files):
try:
data = pd.read_csv(f)
s = [x.strip() for x in data.columns]
x = data.values[:, 0]
for i, j in enumerate([1, 2, 3, 4, 5, 8, 9, 10, 6, 7]):
y = data.values[:, j].astype('float')
# y[y == 0] = np.nan # don't show zero values
ax[i].plot(x, y, marker='.', label=f.stem, linewidth=2, markersize=8)
ax[i].set_title(s[j], fontsize=12)
# if j in [8, 9, 10]: # share train and val loss y axes
# ax[i].get_shared_y_axes().join(ax[i], ax[i - 5])
except Exception as e:
LOGGER.info(f'Warning: Plotting error for {f}: {e}')
ax[1].legend()
plt.show()
fig.savefig(save_dir / 'results.png', dpi=200)
plt.close()
[53]:
dir_path = '/home/mrmarsh/repos/yolov5/runs/train/hybrid_imagery_example/weights/'
epoch_dir = os.listdir(dir_path)
[54]:
results_path = '/home/mrmarsh/repos/yolov5/runs/train/hybrid_imagery_example/results.csv'
Train synthetic, test synthetic
[55]:
plot_results(results_path)
Train synthetic, test real
The loss function steadily increases (the opposite of what we would like) - this highlights the fact that the synthetic and real images are drawn from different distributions, even though the synthetic data is often mistaken for real data by humans.
Training could be improved by incorporating more campaigns in the training set. As seen by inference, the model struggles to detect multiple containers in a row. By using Campaign4 and/or Campaign5 we could improve our metrics. Additionally, using Campaigns that contain distractors and occluding objects could improve precision and recall.
mAP score is increasing at 400 epochs. We could have continued training for more epochs to increase that score; however, recall started decreasing around 200 epochs.
[56]:
dir_path = '/home/mrmarsh/repos/yolov5/runs/train/hybrid_imagery_example_train_syn_test_real/weights/'
epoch_dir = os.listdir(dir_path)
[57]:
results_path = '/home/mrmarsh/repos/yolov5/runs/train/hybrid_imagery_example_train_syn_test_real/results.csv'
[58]:
plot_results(results_path)
