分类问题
这里记一下keras的预处理、数据增强方法，想看pytorch的移步博主另一篇博客
多分类问题
使用keras自带的类ImageDataGenerator定义一个对象，这个对象在定义的时候可以指定对每张图像进行的操作
#ImageDataGenerator的例子
#加载ImageDataGenerator类
from keras.preprocessing.image import ImageDataGenerator
#加载预加载模型提供的图像归一化方法 preprocess_input，帮你做了去中心化、标准差归一化等操作，不用这个的话也可以只除以某个值，例如利用rescale参数等，也可以利用自定义函数
from keras.applications.inception_v3 import InceptionV3, preprocess_input
#定义对象，参数就是可设置的各种操作，实现数据增强，详细参数自己可以查一下
train_datagen = ImageDataGenerator(
preprocessing_function = preprocess_input, #图像预处理
horizontal_flip = True, #垂直翻转
vertical_flip = True, #水平翻转
rotation_range=30,
width_shift_range = 0.2,
height_shift_range = 0.2,
zoom_range = 0.1
)然后调用该对象的方法flow()或flow_from_directory()
flow()方法要求输入图像X和Y的矩阵，然后根据batch_size输出X和Y，这个方法我不常用因为我的工作数据量一般较大，无法直接全部加载到内存里
常用的是flow_from_directory()，它可以迭代的输出X和对应的标签，输入是一个路径，保存格式要求每个类别的图片放在不同的文件夹中，然后会自动建立文件夹名对类名的映射
这是图片放置的格式，train_set是输入的路径，里面包含了n_class个文件夹，每个文件夹都是一类，里面存放了该类的所有图片
看一下生成迭代器的例子
#生成迭代器
train_generator = train_datagen.flow_from_directory(
directory = TRAIN_DIR,
target_size = (IN_SIZE, IN_SIZE),
batch_size=BATCH_SIZE,
class_mode='categorical',
shuffle=True
)同时mark两个方法
print(train_generator.class_indices) # 输出对应的标签文件夹
print(train_generator.filenames) # 按顺序输出文件的名字多标签问题
一张图片有多个标签的时候无法简单的把每张图分到每一类，而且一般的保存格式是把类别放在CSV里，例如这样
CSV保存了图片名，第二列是对应的类别，用分号隔开，读取的思路是自己写迭代器，每次返回输入的X和标签Y
先记一下CSV文件的读取和写入过程
#CSV的读取过程
#制作形如[[1.jpg,cls],[2.jpg,cls]...]的列表，并打乱，用来制作训练集和验证集
input_path = "/home/xxx/df_cloud/"
train_data = pd.read_csv(input_path+"xxx.csv")
cls_list = []
source_path = "/home/xxx/df_cloud/train/"
num = 0
for index, row in train_data.iterrows():
temp = []
filename = row["FileName"]
cls = row["Code"]
for item in cls.split(";"):#这里处理分号隔开的数据
temp.append(filename)
temp.append(item)
cls_list.append(temp)
temp = []
num+=1
print(num)
# num+=1
import random
random.shuffle(cls_list)#CSV写入过程
root_path = '/home/xxx/df_cloud/test/'
path_list = os.listdir(root_path)
INT_HEIGHT = 299
IN_WIDTH = 299
filenames = []
result = []
for filename in path_list:
img = load_img(root_path + filename, target_size=(INT_HEIGHT, IN_WIDTH, 3))
img = img_to_array(img)
img = preprocess_input(img)
img = np.expand_dims(img, axis=0)
prediction = model_2.predict(img)[0]
index = list(prediction).index(np.max(prediction))
res = index_2_cls[index]
filenames.append(filename)
result.append(res)
print(filename)
print(len(filenames))
print(len(result))
import pandas as pd
dataframe = pd.DataFrame({'FileName': filenames, 'type': result})
dataframe.to_csv('/home/xxxf/df_cloud/baseline_1_result_2/submit/baseline_1_2.csv', index=False, header=True)接着预处理，这里发现keras为我们提供了好用的数据预处理、数据增强函数
想看这些函数的源码可以看这位博主的博客
#数据读取、预处理
from keras.preprocessing.image import img_to_array, load_img
#数据增强方法 大量！！！
from keras.preprocessing.image import random_rotation, random_shift, random_shear, random_zoom, random_channel_shift这样直接写迭代器就好了（这部分待更新）
线下数据增强方法
还是利用ImageDataGenerator，每次读取一张图，随机数据增强，然后保存进硬盘即可（待更新）
目标检测问题
利用现成的框架来实现目标检测，常规的方法就是把自己的数据整理成VOC、COCO数据集的格式，然后就符合了框架输入的需求，所以问题就在于如何把数据集整理成这些标准格式，我常用的是VOC数据集格式
使用labelme进行数据标注
使用的时候常用3个参数labelme --nodata --output --autosave 使用 labelme -h 可以查看这些参数作用
labelme生成的是json文件，那么就要把json转成需要的xml文件
记一下如何生成xml文件
import os
from lxml.etree import Element, SubElement, tostring
from xml.dom.minidom import parseString
from PIL import Image
#保存xml文件函数的核心实现，输入为图片名称image_name,分类category（一个列表，元素与bbox对应），bbox(一个列表，与分类对应)，保存路径save_dir ，通道数channel
def save_xml(image_name, category,bbox, file_dir = '/home/xbw/wurenting/dataset_3/',save_dir='/home/xxx/voc_dataset/Annotations/',channel=3):
file_path = file_dir
img = Image.open(file_path + image_name)
width = img.size[0]
height = img.size[1]
node_root = Element('annotation')
node_folder = SubElement(node_root, 'folder')
node_folder.text = 'VOC2007'
node_filename = SubElement(node_root, 'filename')
node_filename.text = image_name
node_size = SubElement(node_root, 'size')
node_width = SubElement(node_size, 'width')
node_width.text = '%s' % width
node_height = SubElement(node_size, 'height')
node_height.text = '%s' % height
node_depth = SubElement(node_size, 'depth')
node_depth.text = '%s' % channel
for i in range(len(bbox)):
left, top, right, bottom = bbox[i][0],bbox[i][1],bbox[i][2], bbox[i][3]
node_object = SubElement(node_root, 'object')
node_name = SubElement(node_object, 'name')
node_name.text = category[i]
node_difficult = SubElement(node_object, 'difficult')
node_difficult.text = '0'
node_bndbox = SubElement(node_object, 'bndbox')
node_xmin = SubElement(node_bndbox, 'xmin')
node_xmin.text = '%s' % left
node_ymin = SubElement(node_bndbox, 'ymin')
node_ymin.text = '%s' % top
node_xmax = SubElement(node_bndbox, 'xmax')
node_xmax.text = '%s' % right
node_ymax = SubElement(node_bndbox, 'ymax')
node_ymax.text = '%s' % bottom
xml = tostring(node_root, pretty_print=True)
dom = parseString(xml)
save_xml = os.path.join(save_dir, image_name.replace('jpg', 'xml'))
with open(save_xml, 'wb') as f:
f.write(xml)
return只要把json文件保存的类别、bbox读出来，输入函数就可以了
json的读取
with open(file,'r') as obj:
res = json.load(obj)
res['xxx'] = xxxlabelme生成的json文件如何保存成xml，这是我使用的脚本，就是把json文件读取出来，然后取出类别和bbox，利用上面的函数写入即可
import os
from lxml.etree import Element, SubElement, tostring
from xml.dom.minidom import parseString
from PIL import Image
import json
json_dir = '/home/xxx/label/label_5/'
img_dir = '/home/xxx/dataset_5/'
save_dir='/home/xxx/xml/label_5_xml/'
json_list = os.listdir(json_dir)
for image_name in json_list:
with open(json_dir+image_name) as obj:
nums = json.load(obj)
labels = []
bboxes = []
for i in nums['shapes']:
labels.append(i['label'])
bboxes.append([min(i['points'][0][0],i['points'][1][0]),min(i['points'][0][1],i['points'][1][1]),max(i['points'][0][0],i['points'][1][0]),max(i['points'][0][1],i['points'][1][1])])
save_xml(image_name[:-5]+'.jpg',labels,bboxes,file_dir = img_dir,save_dir=save_dir)线下数据增强
这个直接参考了这位博主的实现
# -*- coding=utf-8 -*-
##############################################################
# description:
# data augmentation for obeject detection
# author:
# maozezhong 2018-6-27
##############################################################
# 包括:
# 1. 裁剪(需改变bbox)
# 2. 平移(需改变bbox)
# 3. 改变亮度
# 4. 加噪声
# 5. 旋转角度(需要改变bbox)
# 6. 镜像(需要改变bbox)
# 7. cutout
# 注意:
# random.seed(),相同的seed,产生的随机数是一样的!!
import time
import random
import cv2
import os
import math
import numpy as np
from skimage.util import random_noise
from skimage import exposure
def show_pic(img, bboxes=None):
'''
输入:
img:图像array
bboxes:图像的所有boudning box list, 格式为[[x_min, y_min, x_max, y_max]....]
names:每个box对应的名称
'''
# cv2.imwrite('./1.jpg', img)
# img = cv2.imread('./1.jpg')
img=img/255
for i in range(len(bboxes)):
bbox = bboxes[i]
x_min = bbox[0]
y_min = bbox[1]
x_max = bbox[2]
y_max = bbox[3]
cv2.rectangle(img,(int(x_min),int(y_min)),(int(x_max),int(y_max)),(0,255,0),3)
cv2.namedWindow('pic', 0) # 1表示原图
cv2.moveWindow('pic', 0, 0)
cv2.resizeWindow('pic', 1200,800) # 可视化的图片大小
cv2.imshow('pic', img)
if cv2.waitKey(0)==ord('q'):
cv2.destroyAllWindows()
return
cv2.destroyAllWindows()
# os.remove('./1.jpg')
# 图像均为cv2读取
class DataAugmentForObjectDetection():
def __init__(self, rotation_rate=0.5, max_rotation_angle=5,
crop_rate=0.5, shift_rate=0.5, change_light_rate=0.5,
add_noise_rate=0.5, flip_rate=0.5,
cutout_rate=0.5, cut_out_length=50, cut_out_holes=1, cut_out_threshold=0.5):
self.rotation_rate = rotation_rate
self.max_rotation_angle = max_rotation_angle
self.crop_rate = crop_rate
self.shift_rate = shift_rate
self.change_light_rate = change_light_rate
self.add_noise_rate = add_noise_rate
self.flip_rate = flip_rate
self.cutout_rate = cutout_rate
self.cut_out_length = cut_out_length
self.cut_out_holes = cut_out_holes
self.cut_out_threshold = cut_out_threshold
# 加噪声
def _addNoise(self, img):
'''
输入:
img:图像array
输出:
加噪声后的图像array,由于输出的像素是在[0,1]之间,所以得乘以255
'''
# random.seed(int(time.time()))
# return random_noise(img, mode='gaussian', seed=int(time.time()), clip=True)*255
return random_noise(img, mode='gaussian', clip=True)*255
# 调整亮度
def _changeLight(self, img):
# random.seed(int(time.time()))
flag = random.uniform(0.5, 1.5) #flag>1为调暗,小于1为调亮
return exposure.adjust_gamma(img, flag)
# cutout
def _cutout(self, img, bboxes, length=100, n_holes=1, threshold=0.5):
'''
原版本：https://github.com/uoguelph-mlrg/Cutout/blob/master/util/cutout.py
Randomly mask out one or more patches from an image.
Args:
img : a 3D numpy array,(h,w,c)
bboxes : 框的坐标
n_holes (int): Number of patches to cut out of each image.
length (int): The length (in pixels) of each square patch.
'''
def cal_iou(boxA, boxB):
'''
boxA, boxB为两个框，返回iou
boxB为bouding box
'''
# determine the (x, y)-coordinates of the intersection rectangle
xA = max(boxA[0], boxB[0])
yA = max(boxA[1], boxB[1])
xB = min(boxA[2], boxB[2])
yB = min(boxA[3], boxB[3])
if xB <= xA or yB <= yA:
return 0.0
# compute the area of intersection rectangle
interArea = (xB - xA + 1) * (yB - yA + 1)
# compute the area of both the prediction and ground-truth
# rectangles
boxAArea = (boxA[2] - boxA[0] + 1) * (boxA[3] - boxA[1] + 1)
boxBArea = (boxB[2] - boxB[0] + 1) * (boxB[3] - boxB[1] + 1)
# compute the intersection over union by taking the intersection
# area and dividing it by the sum of prediction + ground-truth
# areas - the interesection area
# iou = interArea / float(boxAArea + boxBArea - interArea)
iou = interArea / float(boxBArea)
# return the intersection over union value
return iou
# 得到h和w
if img.ndim == 3:
h,w,c = img.shape
else:
_,h,w,c = img.shape
mask = np.ones((h,w,c), np.float32)
for n in range(n_holes):
chongdie = True #看切割的区域是否与box重叠太多
while chongdie:
y = np.random.randint(h)
x = np.random.randint(w)
y1 = np.clip(y - length // 2, 0, h) #numpy.clip(a, a_min, a_max, out=None), clip这个函数将将数组中的元素限制在a_min, a_max之间，大于a_max的就使得它等于 a_max，小于a_min,的就使得它等于a_min
y2 = np.clip(y + length // 2, 0, h)
x1 = np.clip(x - length // 2, 0, w)
x2 = np.clip(x + length // 2, 0, w)
chongdie = False
for box in bboxes:
if cal_iou([x1,y1,x2,y2], box) > threshold:
chongdie = True
break
mask[y1: y2, x1: x2, :] = 0.
# mask = np.expand_dims(mask, axis=0)
img = img * mask
return img
# 旋转
def _rotate_img_bbox(self, img, bboxes, angle=5, scale=1.):
'''
参考:
输入:
img:图像array,(h,w,c)
bboxes:该图像包含的所有boundingboxs,一个list,每个元素为[x_min, y_min, x_max, y_max],要确保是数值
angle:旋转角度
scale:默认1
输出:
rot_img:旋转后的图像array
rot_bboxes:旋转后的boundingbox坐标list
'''
#---------------------- 旋转图像 ----------------------
w = img.shape[1]
h = img.shape[0]
# 角度变弧度
rangle = np.deg2rad(angle) # angle in radians
# now calculate new image width and height
nw = (abs(np.sin(rangle)*h) + abs(np.cos(rangle)*w))*scale
nh = (abs(np.cos(rangle)*h) + abs(np.sin(rangle)*w))*scale
# ask OpenCV for the rotation matrix
rot_mat = cv2.getRotationMatrix2D((nw*0.5, nh*0.5), angle, scale)
# calculate the move from the old center to the new center combined
# with the rotation
rot_move = np.dot(rot_mat, np.array([(nw-w)*0.5, (nh-h)*0.5,0]))
# the move only affects the translation, so update the translation
# part of the transform
rot_mat[0,2] += rot_move[0]
rot_mat[1,2] += rot_move[1]
# 仿射变换
rot_img = cv2.warpAffine(img, rot_mat, (int(math.ceil(nw)), int(math.ceil(nh))), flags=cv2.INTER_LANCZOS4)
#---------------------- 矫正bbox坐标 ----------------------
# rot_mat是最终的旋转矩阵
# 获取原始bbox的四个中点，然后将这四个点转换到旋转后的坐标系下
rot_bboxes = list()
for bbox in bboxes:
xmin = bbox[0]
ymin = bbox[1]
xmax = bbox[2]
ymax = bbox[3]
point1 = np.dot(rot_mat, np.array([(xmin+xmax)/2, ymin, 1]))
point2 = np.dot(rot_mat, np.array([xmax, (ymin+ymax)/2, 1]))
point3 = np.dot(rot_mat, np.array([(xmin+xmax)/2, ymax, 1]))
point4 = np.dot(rot_mat, np.array([xmin, (ymin+ymax)/2, 1]))
# 合并np.array
concat = np.vstack((point1, point2, point3, point4))
# 改变array类型
concat = concat.astype(np.int32)
# 得到旋转后的坐标
rx, ry, rw, rh = cv2.boundingRect(concat)
rx_min = rx
ry_min = ry
rx_max = rx+rw
ry_max = ry+rh
# 加入list中
rot_bboxes.append([rx_min, ry_min, rx_max, ry_max])
return rot_img, rot_bboxes
# 裁剪
def _crop_img_bboxes(self, img, bboxes):
'''
裁剪后的图片要包含所有的框
输入:
img:图像array
bboxes:该图像包含的所有boundingboxs,一个list,每个元素为[x_min, y_min, x_max, y_max],要确保是数值
输出:
crop_img:裁剪后的图像array
crop_bboxes:裁剪后的bounding box的坐标list
'''
#---------------------- 裁剪图像 ----------------------
w = img.shape[1]
h = img.shape[0]
x_min = w #裁剪后的包含所有目标框的最小的框
x_max = 0
y_min = h
y_max = 0
for bbox in bboxes:
x_min = min(x_min, bbox[0])
y_min = min(y_min, bbox[1])
x_max = max(x_max, bbox[2])
y_max = max(y_max, bbox[3])
d_to_left = x_min #包含所有目标框的最小框到左边的距离
d_to_right = w - x_max #包含所有目标框的最小框到右边的距离
d_to_top = y_min #包含所有目标框的最小框到顶端的距离
d_to_bottom = h - y_max #包含所有目标框的最小框到底部的距离
#随机扩展这个最小框
crop_x_min = int(x_min - random.uniform(0, d_to_left))
crop_y_min = int(y_min - random.uniform(0, d_to_top))
crop_x_max = int(x_max + random.uniform(0, d_to_right))
crop_y_max = int(y_max + random.uniform(0, d_to_bottom))
# 随机扩展这个最小框 , 防止别裁的太小
# crop_x_min = int(x_min - random.uniform(d_to_left//2, d_to_left))
# crop_y_min = int(y_min - random.uniform(d_to_top//2, d_to_top))
# crop_x_max = int(x_max + random.uniform(d_to_right//2, d_to_right))
# crop_y_max = int(y_max + random.uniform(d_to_bottom//2, d_to_bottom))
#确保不要越界
crop_x_min = max(0, crop_x_min)
crop_y_min = max(0, crop_y_min)
crop_x_max = min(w, crop_x_max)
crop_y_max = min(h, crop_y_max)
crop_img = img[crop_y_min:crop_y_max, crop_x_min:crop_x_max]
#---------------------- 裁剪boundingbox ----------------------
#裁剪后的boundingbox坐标计算
crop_bboxes = list()
for bbox in bboxes:
crop_bboxes.append([bbox[0]-crop_x_min, bbox[1]-crop_y_min, bbox[2]-crop_x_min, bbox[3]-crop_y_min])
return crop_img, crop_bboxes
# 平移
def _shift_pic_bboxes(self, img, bboxes):
'''
参考:
平移后的图片要包含所有的框
输入:
img:图像array
bboxes:该图像包含的所有boundingboxs,一个list,每个元素为[x_min, y_min, x_max, y_max],要确保是数值
输出:
shift_img:平移后的图像array
shift_bboxes:平移后的bounding box的坐标list
'''
#---------------------- 平移图像 ----------------------
w = img.shape[1]
h = img.shape[0]
x_min = w #裁剪后的包含所有目标框的最小的框
x_max = 0
y_min = h
y_max = 0
for bbox in bboxes:
x_min = min(x_min, bbox[0])
y_min = min(y_min, bbox[1])
x_max = max(x_max, bbox[2])
y_max = max(y_max, bbox[3])
d_to_left = x_min #包含所有目标框的最大左移动距离
d_to_right = w - x_max #包含所有目标框的最大右移动距离
d_to_top = y_min #包含所有目标框的最大上移动距离
d_to_bottom = h - y_max #包含所有目标框的最大下移动距离
x = random.uniform(-(d_to_left-1) / 3, (d_to_right-1) / 3)
y = random.uniform(-(d_to_top-1) / 3, (d_to_bottom-1) / 3)
M = np.float32([[1, 0, x], [0, 1, y]]) #x为向左或右移动的像素值,正为向右负为向左; y为向上或者向下移动的像素值,正为向下负为向上
shift_img = cv2.warpAffine(img, M, (img.shape[1], img.shape[0]))
#---------------------- 平移boundingbox ----------------------
shift_bboxes = list()
for bbox in bboxes:
shift_bboxes.append([bbox[0]+x, bbox[1]+y, bbox[2]+x, bbox[3]+y])
return shift_img, shift_bboxes
# 镜像
def _filp_pic_bboxes(self, img, bboxes):
'''
参考:
平移后的图片要包含所有的框
输入:
img:图像array
bboxes:该图像包含的所有boundingboxs,一个list,每个元素为[x_min, y_min, x_max, y_max],要确保是数值
输出:
flip_img:平移后的图像array
flip_bboxes:平移后的bounding box的坐标list
'''
# ---------------------- 翻转图像 ----------------------
import copy
flip_img = copy.deepcopy(img)
if random.random() < 0.5: #0.5的概率水平翻转，0.5的概率垂直翻转
horizon = True
else:
horizon = False
h,w,_ = img.shape
if horizon: #水平翻转
flip_img = cv2.flip(flip_img, 1) #1是水平，-1是水平垂直
else:
flip_img = cv2.flip(flip_img, 0)
# ---------------------- 调整boundingbox ----------------------
flip_bboxes = list()
for box in bboxes:
x_min = box[0]
y_min = box[1]
x_max = box[2]
y_max = box[3]
if horizon:
flip_bboxes.append([w-x_max, y_min, w-x_min, y_max])
else:
flip_bboxes.append([x_min, h-y_max, x_max, h-y_min])
return flip_img, flip_bboxes
def dataAugment(self, img, bboxes):
'''
图像增强
输入:
img:图像array
bboxes:该图像的所有框坐标
输出:
img:增强后的图像
bboxes:增强后图片对应的box
'''
change_num = 0 #改变的次数
print('------')
while change_num < 1: #默认至少有一种数据增强生效
if random.random() < self.crop_rate: #裁剪
print('裁剪')
change_num += 1
img, bboxes = self._crop_img_bboxes(img, bboxes)
if random.random() > self.rotation_rate: #旋转
print('旋转')
change_num += 1
# angle = random.uniform(-self.max_rotation_angle, self.max_rotation_angle)
angle = random.sample([90, 180, 270],1)[0]
scale = random.uniform(0.7, 0.8)
img, bboxes = self._rotate_img_bbox(img, bboxes, angle, scale)
if random.random() < self.shift_rate: #平移
print('平移')
change_num += 1
img, bboxes = self._shift_pic_bboxes(img, bboxes)
if random.random() > self.change_light_rate: #改变亮度
print('亮度')
change_num += 1
img = self._changeLight(img)
if random.random() < self.add_noise_rate: #加噪声
print('加噪声')
change_num += 1
img = self._addNoise(img)
if random.random() < self.cutout_rate: #cutout
print('cutout')
change_num += 1
img = self._cutout(img, bboxes, length=self.cut_out_length, n_holes=self.cut_out_holes, threshold=self.cut_out_threshold)
if random.random() < self.flip_rate: #翻转
print('翻转')
change_num += 1
img, bboxes = self._filp_pic_bboxes(img, bboxes)
print('\n')
# print('------')
return img, bboxes# -*- coding=utf-8 -*-
import xml.etree.ElementTree as ET
import xml.dom.minidom as DOC
# 从xml文件中提取bounding box信息, 格式为[[x_min, y_min, x_max, y_max, name]]
def parse_xml(xml_path):
'''
输入：
xml_path: xml的文件路径
输出：
从xml文件中提取bounding box信息, 格式为[[x_min, y_min, x_max, y_max, name]]
'''
tree = ET.parse(xml_path)
root = tree.getroot()
objs = root.findall('object')
coords = list()
for ix, obj in enumerate(objs):
name = obj.find('name').text
box = obj.find('bndbox')
x_min = int(box[0].text)
y_min = int(box[1].text)
x_max = int(box[2].text)
y_max = int(box[3].text)
coords.append([x_min, y_min, x_max, y_max, name])
return coords#测试
import shutil
need_aug_num = 1
dataAug = DataAugmentForObjectDetection()
source_pic_root_path = '/home/xbw/faster-rcnn.pytorch/beifen/VOC2007/JPEGImages'
source_xml_root_path = '/home/xbw/faster-rcnn.pytorch/beifen/VOC2007/Annotations'
for parent, _, files in os.walk(source_pic_root_path):
for file in files:
cnt = 0
while cnt < need_aug_num:
pic_path = os.path.join(parent, file)
xml_path = os.path.join(source_xml_root_path, file[:-4]+'.xml')
coords = parse_xml(xml_path) #解析得到box信息，格式为[[x_min,y_min,x_max,y_max,name]]
coords = [coord[:4] for coord in coords]
img = cv2.imread(pic_path)
show_pic(img, coords) # 原图
auged_img, auged_bboxes = dataAug.dataAugment(img, coords)
cnt += 1
show_pic(auged_img, auged_bboxes) # 强化后的图线下数据增强的实例脚本
# -*- coding=utf-8 -*-
# 包括:
# 1. 裁剪(需改变bbox)
# 2. 平移(需改变bbox)
# 3. 改变亮度
# 4. 加噪声
# 5. 旋转角度(需要改变bbox)
# 6. 镜像(需要改变bbox)
# 7. cutout
# 注意:
# random.seed(),相同的seed,产生的随机数是一样的!!
import time
import random
import cv2
import os
import math
import numpy as np
from skimage.util import random_noise
from skimage import exposure
import sys
#显示带标签显示的图片
def show_pic(img, bboxes=None,labels=None):
'''
输入:
img:图像array
bboxes:图像的所有boudning box list, 格式为[[x_min, y_min, x_max, y_max]....]
names:每个box对应的名称
'''
# cv2.imwrite('./1.jpg', img)
# img = cv2.imread('./1.jpg')
img=img/255
for i in range(len(bboxes)):
bbox = bboxes[i]
x_min = bbox[0]
y_min = bbox[1]
x_max = bbox[2]
y_max = bbox[3]
cv2.rectangle(img,(int(x_min),int(y_min)),(int(x_max),int(y_max)),(0,255,0),3)
cv2.putText(img,labels[i],(int(x_min),int(y_min)),cv2.FONT_HERSHEY_SIMPLEX,0.8,(0,0,255),2)
cv2.namedWindow('pic', 0) # 1表示原图
cv2.moveWindow('pic', 0, 0)
cv2.resizeWindow('pic', 1200,800) # 可视化的图片大小
cv2.imshow('pic', img)
if cv2.waitKey(1)==ord('q'):
cv2.destroyAllWindows()
sys.exit()
# cv2.destroyAllWindows()
# os.remove('./1.jpg')
# 图像均为cv2读取
class DataAugmentForObjectDetection():
def __init__(self, rotation_rate=0.5, max_rotation_angle=30,
crop_rate=0.5, shift_rate=0.5, change_light_rate=0.5,
add_noise_rate=0.5, flip_rate=0.5,
cutout_rate=0.5, cut_out_length=50, cut_out_holes=1, cut_out_threshold=0.5):
self.rotation_rate = rotation_rate
self.max_rotation_angle = max_rotation_angle
self.crop_rate = crop_rate
self.shift_rate = shift_rate
self.change_light_rate = change_light_rate
self.add_noise_rate = add_noise_rate
self.flip_rate = flip_rate
self.cutout_rate = cutout_rate
self.cut_out_length = cut_out_length
self.cut_out_holes = cut_out_holes
self.cut_out_threshold = cut_out_threshold
# 加噪声
def _addNoise(self, img):
'''
输入:
img:图像array
输出:
加噪声后的图像array,由于输出的像素是在[0,1]之间,所以得乘以255
'''
# random.seed(int(time.time()))
# return random_noise(img, mode='gaussian', seed=int(time.time()), clip=True)*255
return random_noise(img, mode='gaussian', clip=True)*255
# 调整亮度
def _changeLight(self, img):
# random.seed(int(time.time()))
flag = random.uniform(0.5, 1.5) #flag>1为调暗,小于1为调亮
return exposure.adjust_gamma(img, flag)
# cutout
def _cutout(self, img, bboxes, length=100, n_holes=1, threshold=0.5):
'''
原版本：https://github.com/uoguelph-mlrg/Cutout/blob/master/util/cutout.py
Randomly mask out one or more patches from an image.
Args:
img : a 3D numpy array,(h,w,c)
bboxes : 框的坐标
n_holes (int): Number of patches to cut out of each image.
length (int): The length (in pixels) of each square patch.
'''
def cal_iou(boxA, boxB):
'''
boxA, boxB为两个框，返回iou
boxB为bouding box
'''
# determine the (x, y)-coordinates of the intersection rectangle
xA = max(boxA[0], boxB[0])
yA = max(boxA[1], boxB[1])
xB = min(boxA[2], boxB[2])
yB = min(boxA[3], boxB[3])
if xB <= xA or yB <= yA:
return 0.0
# compute the area of intersection rectangle
interArea = (xB - xA + 1) * (yB - yA + 1)
# compute the area of both the prediction and ground-truth
# rectangles
boxAArea = (boxA[2] - boxA[0] + 1) * (boxA[3] - boxA[1] + 1)
boxBArea = (boxB[2] - boxB[0] + 1) * (boxB[3] - boxB[1] + 1)
# compute the intersection over union by taking the intersection
# area and dividing it by the sum of prediction + ground-truth
# areas - the interesection area
# iou = interArea / float(boxAArea + boxBArea - interArea)
iou = interArea / float(boxBArea)
# return the intersection over union value
return iou
# 得到h和w
if img.ndim == 3:
h,w,c = img.shape
else:
_,h,w,c = img.shape
mask = np.ones((h,w,c), np.float32)
for n in range(n_holes):
chongdie = True #看切割的区域是否与box重叠太多
while chongdie:
y = np.random.randint(h)
x = np.random.randint(w)
y1 = np.clip(y - length // 2, 0, h) #numpy.clip(a, a_min, a_max, out=None), clip这个函数将将数组中的元素限制在a_min, a_max之间，大于a_max的就使得它等于 a_max，小于a_min,的就使得它等于a_min
y2 = np.clip(y + length // 2, 0, h)
x1 = np.clip(x - length // 2, 0, w)
x2 = np.clip(x + length // 2, 0, w)
chongdie = False
for box in bboxes:
if cal_iou([x1,y1,x2,y2], box) > threshold:
chongdie = True
break
mask[y1: y2, x1: x2, :] = 0.
# mask = np.expand_dims(mask, axis=0)
img = img * mask
return img
# 旋转
def _rotate_img_bbox(self, img, bboxes, angle=5, scale=1.):
'''
参考:
输入:
img:图像array,(h,w,c)
bboxes:该图像包含的所有boundingboxs,一个list,每个元素为[x_min, y_min, x_max, y_max],要确保是数值
angle:旋转角度
scale:默认1
输出:
rot_img:旋转后的图像array
rot_bboxes:旋转后的boundingbox坐标list
'''
#---------------------- 旋转图像 ----------------------
w = img.shape[1]
h = img.shape[0]
# 角度变弧度
rangle = np.deg2rad(angle) # angle in radians
# now calculate new image width and height
nw = (abs(np.sin(rangle)*h) + abs(np.cos(rangle)*w))*scale
nh = (abs(np.cos(rangle)*h) + abs(np.sin(rangle)*w))*scale
# ask OpenCV for the rotation matrix
rot_mat = cv2.getRotationMatrix2D((nw*0.5, nh*0.5), angle, scale)
# calculate the move from the old center to the new center combined
# with the rotation
rot_move = np.dot(rot_mat, np.array([(nw-w)*0.5, (nh-h)*0.5,0]))
# the move only affects the translation, so update the translation
# part of the transform
rot_mat[0,2] += rot_move[0]
rot_mat[1,2] += rot_move[1]
# 仿射变换
rot_img = cv2.warpAffine(img, rot_mat, (int(math.ceil(nw)), int(math.ceil(nh))), flags=cv2.INTER_LANCZOS4)
#---------------------- 矫正bbox坐标 ----------------------
# rot_mat是最终的旋转矩阵
# 获取原始bbox的四个中点，然后将这四个点转换到旋转后的坐标系下
rot_bboxes = list()
for bbox in bboxes:
xmin = bbox[0]
ymin = bbox[1]
xmax = bbox[2]
ymax = bbox[3]
point1 = np.dot(rot_mat, np.array([(xmin+xmax)/2, ymin, 1]))
point2 = np.dot(rot_mat, np.array([xmax, (ymin+ymax)/2, 1]))
point3 = np.dot(rot_mat, np.array([(xmin+xmax)/2, ymax, 1]))
point4 = np.dot(rot_mat, np.array([xmin, (ymin+ymax)/2, 1]))
# 合并np.array
concat = np.vstack((point1, point2, point3, point4))
# 改变array类型
concat = concat.astype(np.int32)
# 得到旋转后的坐标
rx, ry, rw, rh = cv2.boundingRect(concat)
rx_min = rx
ry_min = ry
rx_max = rx+rw
ry_max = ry+rh
# 加入list中
rot_bboxes.append([rx_min, ry_min, rx_max, ry_max])
return rot_img, rot_bboxes
# 裁剪
def _crop_img_bboxes(self, img, bboxes):
'''
裁剪后的图片要包含所有的框
输入:
img:图像array
bboxes:该图像包含的所有boundingboxs,一个list,每个元素为[x_min, y_min, x_max, y_max],要确保是数值
输出:
crop_img:裁剪后的图像array
crop_bboxes:裁剪后的bounding box的坐标list
'''
#---------------------- 裁剪图像 ----------------------
w = img.shape[1]
h = img.shape[0]
x_min = w #裁剪后的包含所有目标框的最小的框
x_max = 0
y_min = h
y_max = 0
for bbox in bboxes:
x_min = min(x_min, bbox[0])
y_min = min(y_min, bbox[1])
x_max = max(x_max, bbox[2])
y_max = max(y_max, bbox[3])
d_to_left = x_min #包含所有目标框的最小框到左边的距离
d_to_right = w - x_max #包含所有目标框的最小框到右边的距离
d_to_top = y_min #包含所有目标框的最小框到顶端的距离
d_to_bottom = h - y_max #包含所有目标框的最小框到底部的距离
#随机扩展这个最小框
crop_x_min = int(x_min - random.uniform(0, d_to_left))
crop_y_min = int(y_min - random.uniform(0, d_to_top))
crop_x_max = int(x_max + random.uniform(0, d_to_right))
crop_y_max = int(y_max + random.uniform(0, d_to_bottom))
# 随机扩展这个最小框 , 防止别裁的太小
# crop_x_min = int(x_min - random.uniform(d_to_left//2, d_to_left))
# crop_y_min = int(y_min - random.uniform(d_to_top//2, d_to_top))
# crop_x_max = int(x_max + random.uniform(d_to_right//2, d_to_right))
# crop_y_max = int(y_max + random.uniform(d_to_bottom//2, d_to_bottom))
#确保不要越界
crop_x_min = max(0, crop_x_min)
crop_y_min = max(0, crop_y_min)
crop_x_max = min(w, crop_x_max)
crop_y_max = min(h, crop_y_max)
crop_img = img[crop_y_min:crop_y_max, crop_x_min:crop_x_max]
#---------------------- 裁剪boundingbox ----------------------
#裁剪后的boundingbox坐标计算
crop_bboxes = list()
for bbox in bboxes:
crop_bboxes.append([bbox[0]-crop_x_min, bbox[1]-crop_y_min, bbox[2]-crop_x_min, bbox[3]-crop_y_min])
return crop_img, crop_bboxes
# 平移
def _shift_pic_bboxes(self, img, bboxes):
'''
参考:
平移后的图片要包含所有的框
输入:
img:图像array
bboxes:该图像包含的所有boundingboxs,一个list,每个元素为[x_min, y_min, x_max, y_max],要确保是数值
输出:
shift_img:平移后的图像array
shift_bboxes:平移后的bounding box的坐标list
'''
#---------------------- 平移图像 ----------------------
w = img.shape[1]
h = img.shape[0]
x_min = w #裁剪后的包含所有目标框的最小的框
x_max = 0
y_min = h
y_max = 0
for bbox in bboxes:
x_min = min(x_min, bbox[0])
y_min = min(y_min, bbox[1])
x_max = max(x_max, bbox[2])
y_max = max(y_max, bbox[3])
d_to_left = x_min #包含所有目标框的最大左移动距离
d_to_right = w - x_max #包含所有目标框的最大右移动距离
d_to_top = y_min #包含所有目标框的最大上移动距离
d_to_bottom = h - y_max #包含所有目标框的最大下移动距离
x = random.uniform(-(d_to_left-1) / 3, (d_to_right-1) / 3)
y = random.uniform(-(d_to_top-1) / 3, (d_to_bottom-1) / 3)
M = np.float32([[1, 0, x], [0, 1, y]]) #x为向左或右移动的像素值,正为向右负为向左; y为向上或者向下移动的像素值,正为向下负为向上
shift_img = cv2.warpAffine(img, M, (img.shape[1], img.shape[0]))
#---------------------- 平移boundingbox ----------------------
shift_bboxes = list()
for bbox in bboxes:
shift_bboxes.append([bbox[0]+x, bbox[1]+y, bbox[2]+x, bbox[3]+y])
return shift_img, shift_bboxes
# 镜像
def _filp_pic_bboxes(self, img, bboxes):
'''
参考:
平移后的图片要包含所有的框
输入:
img:图像array
bboxes:该图像包含的所有boundingboxs,一个list,每个元素为[x_min, y_min, x_max, y_max],要确保是数值
输出:
flip_img:平移后的图像array
flip_bboxes:平移后的bounding box的坐标list
'''
# ---------------------- 翻转图像 ----------------------
import copy
flip_img = copy.deepcopy(img)
if random.random() < 0.5: #0.5的概率水平翻转，0.5的概率垂直翻转
horizon = True
else:
horizon = False
h,w,_ = img.shape
if horizon: #水平翻转
flip_img = cv2.flip(flip_img, 1) #1是水平，-1是水平垂直
else:
flip_img = cv2.flip(flip_img, 0)
# ---------------------- 调整boundingbox ----------------------
flip_bboxes = list()
for box in bboxes:
x_min = box[0]
y_min = box[1]
x_max = box[2]
y_max = box[3]
if horizon:
flip_bboxes.append([w-x_max, y_min, w-x_min, y_max])
else:
flip_bboxes.append([x_min, h-y_max, x_max, h-y_min])
return flip_img, flip_bboxes
def dataAugment(self, img, bboxes):
'''
图像增强
输入:
img:图像array
bboxes:该图像的所有框坐标
输出:
img:增强后的图像
bboxes:增强后图片对应的box
'''
change_num = 0 #改变的次数
print('------')
while change_num < 1: #默认至少有一种数据增强生效
if random.random() < self.crop_rate: #裁剪
print('裁剪')
change_num += 1
img, bboxes = self._crop_img_bboxes(img, bboxes)
if random.random() > self.rotation_rate: #旋转
print('旋转')
change_num += 1
angle = random.uniform(-self.max_rotation_angle, self.max_rotation_angle)
# angle = random.sample([90, 180, 270],1)[0]
scale = random.uniform(0.7, 0.8)
img, bboxes = self._rotate_img_bbox(img, bboxes, angle, scale)
if random.random() < self.shift_rate: #平移
print('平移')
change_num += 1
img, bboxes = self._shift_pic_bboxes(img, bboxes)
if random.random() > self.change_light_rate: #改变亮度
print('亮度')
change_num += 1
img = self._changeLight(img)
if random.random() < self.add_noise_rate: #加噪声
print('加噪声')
change_num += 1
img = self._addNoise(img)
if random.random() < self.cutout_rate: #cutout
print('cutout')
change_num += 1
img = self._cutout(img, bboxes, length=self.cut_out_length, n_holes=self.cut_out_holes, threshold=self.cut_out_threshold)
# if random.random() < self.flip_rate: #翻转
# print('翻转')
# change_num += 1
# img, bboxes = self._filp_pic_bboxes(img, bboxes)
print('\n')
# print('------')
return img, bboxes# -*- coding=utf-8 -*-
import xml.etree.ElementTree as ET
import xml.dom.minidom as DOC
# 从xml文件中提取bounding box信息, 格式为[[x_min, y_min, x_max, y_max, name]]
def parse_xml(xml_path):
'''
输入：
xml_path: xml的文件路径
输出：
从xml文件中提取bounding box信息, 格式为[[x_min, y_min, x_max, y_max, name]]
'''
tree = ET.parse(xml_path)
root = tree.getroot()
objs = root.findall('object')
coords = list()
for ix, obj in enumerate(objs):
name = obj.find('name').text
box = obj.find('bndbox')
x_min = int(float(box[0].text))
y_min = int(float(box[1].text))
x_max = int(float(box[2].text))
y_max = int(float(box[3].text))
coords.append([x_min, y_min, x_max, y_max, name])
return coordsimport os
from lxml.etree import Element, SubElement, tostring
from xml.dom.minidom import parseString
from PIL import Image
#保存xml文件函数的核心实现，输入为图片名称image_name,分类category（一个列表，元素与bbox对应），bbox(一个列表，与分类对应)，保存路径save_dir ，通道数channel
def save_xml(image_name, category,bbox, file_dir = '/home/xbw/wurenting/dataset_3/',save_dir='/home/xxx/voc_dataset/Annotations/',channel=3):
file_path = file_dir
img = Image.open(file_path + image_name)
width = img.size[0]
height = img.size[1]
node_root = Element('annotation')
node_folder = SubElement(node_root, 'folder')
node_folder.text = 'VOC2007'
node_filename = SubElement(node_root, 'filename')
node_filename.text = image_name
node_size = SubElement(node_root, 'size')
node_width = SubElement(node_size, 'width')
node_width.text = '%s' % width
node_height = SubElement(node_size, 'height')
node_height.text = '%s' % height
node_depth = SubElement(node_size, 'depth')
node_depth.text = '%s' % channel
for i in range(len(bbox)):
left, top, right, bottom = bbox[i][0],bbox[i][1],bbox[i][2], bbox[i][3]
node_object = SubElement(node_root, 'object')
node_name = SubElement(node_object, 'name')
node_name.text = category[i]
node_difficult = SubElement(node_object, 'difficult')
node_difficult.text = '0'
node_bndbox = SubElement(node_object, 'bndbox')
node_xmin = SubElement(node_bndbox, 'xmin')
node_xmin.text = '%s' % left
node_ymin = SubElement(node_bndbox, 'ymin')
node_ymin.text = '%s' % top
node_xmax = SubElement(node_bndbox, 'xmax')
node_xmax.text = '%s' % right
node_ymax = SubElement(node_bndbox, 'ymax')
node_ymax.text = '%s' % bottom
xml = tostring(node_root, pretty_print=True)
dom = parseString(xml)
save_xml = os.path.join(save_dir, image_name.replace('jpg', 'xml'))
with open(save_xml, 'wb') as f:
f.write(xml)
returnimport shutil
need_aug_num = 1
dataAug = DataAugmentForObjectDetection()
source_pic_root_path = '/home/xbw/wurenting/dataset/'
source_xml_root_path = '/home/xbw/wurenting/labels/'
img_save_path = '/home/xbw/wurenting/argdataset/'
save_dir = '/home/xbw/wurenting/arglabels/'
for parent, _, files in os.walk(source_pic_root_path):
for file in files:
cnt = 0
while cnt < need_aug_num:
pic_path = os.path.join(parent, file)
xml_path = os.path.join(source_xml_root_path, file[:-4]+'.xml')
coords = parse_xml(xml_path) #解析得到box信息，格式为[[x_min,y_min,x_max,y_max,name]]
coordss = [coord[:4] for coord in coords]
labels = [coord[4] for coord in coords]
img = cv2.imread(pic_path)
show_pic(img, coordss,labels) # 原图
auged_img, auged_bboxes = dataAug.dataAugment(img, coordss)
cnt += 1
cv2.imwrite(img_save_path+file[:-4]+'_arg.jpg',auged_img)
save_xml(file[:-4]+'_arg.jpg',labels,auged_bboxes,file_dir = img_save_path,save_dir=save_dir)
show_pic(auged_img, auged_bboxes,labels) # 强化后的图#测试label是否正确
import shutil
need_aug_num = 1
dataAug = DataAugmentForObjectDetection()
source_pic_root_path = '/home/xbw/wurenting/dataset/'
source_xml_root_path = '/home/xbw/wurenting/labels/'
for parent, _, files in os.walk(source_pic_root_path):
for file in files:
cnt = 0
while cnt < need_aug_num:
pic_path = os.path.join(parent, file)
xml_path = os.path.join(source_xml_root_path, file[:-4]+'.xml')
coords = parse_xml(xml_path) #解析得到box信息，格式为[[x_min,y_min,x_max,y_max,name]]
coordss = [coord[:4] for coord in coords]
labels = [coord[4] for coord in coords]
img = cv2.imread(pic_path)
show_pic(img, coordss,labels) # 原图
cnt += 1
cv2.destroyAllWindows()使用imgaug进行数据增强
import imgaug as ia
from imgaug import augmenters as iaa
seq = iaa.Sequential([
iaa.Fliplr(0.5), # 0.5的概率水平翻转
iaa.Crop(percent=(0, 0.1)), # random crops
#sigma在0~0.5间随机高斯模糊，且每张图纸生效的概率是0.5
iaa.Sometimes(0.5,
iaa.GaussianBlur(sigma=(0, 0.5))
),
# 增大或减小每张图像的对比度
iaa.ContrastNormalization((0.75, 1.5)),
# 高斯噪点
iaa.AdditiveGaussianNoise(loc=0, scale=(0.0, 0.05*255), per_channel=0.5),
# 给每个像素乘上0.8-1.2之间的数来使图片变暗或变亮
#20%的图片在每个channel上乘以不同的因子
iaa.Multiply((0.8, 1.2), per_channel=0.2),
# 对每张图片进行仿射变换，包括缩放、平移、旋转、修剪等
iaa.Affine(
scale={"x": (0.8, 1.2), "y": (0.8, 1.2)},
translate_percent={"x": (-0.2, 0.2), "y": (-0.2, 0.2)},
rotate=(-25, 25),
shear=(-8, 8)
)
], random_order=True) # 随机应用以上的图片增强方法配合生成器，写出数据增强的生成器，具体就是使用augment_images（np.nparray）方法即可
def data_generator(root_path,batch_size):
while True:
class_nums = len(glob.glob('/home/xbw/siamese/classes/*'))
x_batchs_1 = []
x_batchs_2 = []
y_batchs = []
for i in range(class_nums):
#文件夹要以0开头
jpg_nums = len(glob.glob(root_path + str(i) + '/' +'*.jpg'))
path_list = os.listdir(root_path + str(i))
random.shuffle(path_list)
for j in range(jpg_nums-1):
if len(y_batchs)>=batch_size:
yield [[seq.augment_images(np.array(x_batchs_1))/255,seq.augment_images(np.array(x_batchs_2))/255],np.array(y_batchs)]
x_batchs_1 = []
x_batchs_2 = []
y_batchs = []
#增加相同的
jpg_path1 = root_path + str(i) + '/'+ path_list[j]
z1 = cv2.imread(jpg_path1)
z1 = cv2.cvtColor(z1, cv2.COLOR_BGR2RGB)
z1 = cv2.resize(z1,(32,32),interpolation=cv2.INTER_AREA)
jpg_path2 = root_path + str(i) + '/'+ path_list[j+1]
z2 = cv2.imread(jpg_path2)
z2 = cv2.cvtColor(z2, cv2.COLOR_BGR2RGB)
z2 = cv2.resize(z2,(32,32),interpolation=cv2.INTER_AREA)
x_batchs_1.append(z1)
x_batchs_2.append(z2)
y_batchs.append(1)
#增加不同的
inc = random.randrange(1,class_nums)
random_num = (i + inc)%class_nums
temp_list = os.listdir(root_path + str(random_num))
random.shuffle(temp_list)
jpg_path3 = root_path + str(random_num) + '/'+ temp_list[0]
z3 = cv2.imread(jpg_path3)
z3 = cv2.cvtColor(z3, cv2.COLOR_BGR2RGB)
z3 = cv2.resize(z3,(32,32),interpolation=cv2.INTER_AREA)
x_batchs_1.append(z1)
x_batchs_2.append(z3)
y_batchs.append(0)
本文章为转载内容，我们尊重原作者对文章享有的著作权。如有内容错误或侵权问题，欢迎原作者联系我们进行内容更正或删除文章。
