基础作业：自己随机生成数据并用Megengine实现数据的线性拟合
小组作业：在Mnist 数据上进行训练并测试
王新皓：
import numpy as np
import megengine as mge
import megengine.functional as F
from megengine.data.dataset import MNIST
from megengine.data import SequentialSampler, RandomSampler, DataLoader
from megengine.autodiff import GradManager
import megengine.optimizer as optim
MNIST_DATA_PATH ="/data/jupyter/ai_02/data/dataset/MNIST"
# 设置超参数
bs = 64
lr = 1e-6
epochs = 5
# 读取原始数据集
train_dataset = MNIST(root=MNIST_DATA_PATH, train=True, download=False)
nums = len(train_dataset)
num_features = 784 # (28, 28, 1) Flatten -> 784
num_classes = 10
# 训练数据加载与预处理
train_sampler = SequentialSampler(dataset=train_dataset, batch_size=bs)
train_dataloader = DataLoader(dataset=train_dataset, sampler=train_sampler)
# 初始化参数
W = mge.Parameter(np.zeros((num_features, num_classes)))
b = mge.Parameter(np.zeros((num_classes,)))
# 定义模型
def linear_cls(X):
return F.matmul(X, W) + b
# 定义求导器和优化器
gm = GradManager().attach([W, b])
optimizer = optim.SGD([W, b], lr=lr)
# 模型训练
for epoch in range(epochs):
total_loss = 0
for batch_data, batch_label in train_dataloader:
batch_data = F.flatten(mge.tensor(batch_data), 1)
batch_label = mge.tensor(batch_label)
with gm:
pred = linear_cls(batch_data)
loss = F.loss.cross_entropy(pred, batch_label)
gm.backward(loss)
optimizer.step().clear_grad()
total_loss += loss.item()
print("epoch = {}, loss = {:.3f}".format(epoch, total_loss / len(train_dataloader)))
test_dataset = MNIST(root=MNIST_DATA_PATH, train=False, download=False)
test_sampler = RandomSampler(dataset=test_dataset, batch_size=100)
test_dataloader = DataLoader(dataset=test_dataset, sampler=test_sampler)
nums_correct = 0
for batch_data, batch_label in test_dataloader:
batch_data = F.flatten(mge.tensor(batch_data), 1)
batch_label = mge.tensor(batch_label)
logits = linear_cls(batch_data)
pred = F.argmax(logits, axis=1)
nums_correct += (pred == batch_label).sum().item()
print("Accuracy = {:.3f}".format(nums_correct / len(test_dataloader)))
#导入工具
import cv2
import sys
import nori2 as nori
import imgaug as ia
import numpy as np
import imgaug.augmenters as iaa
from meghair.utils import io
from meghair.utils.imgproc import imdecode
from matplotlib import pyplot as plt
from skimage import io,data
#数据库获取工具
fetcher = nori.Fetcher()
#imgaug config
H,W = 128 ,128
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
seq = iaa.Sequential([
sometimes(iaa.Affine(
rotate=(-90, 90)
)),
iaa.Crop(px=(0, 30)),
iaa.Resize({"height": H, "width": W})
])
#文件读取
image_net_set = open("./1percent_ImageNet.txt","r");
print(image_net_set)
#获取原始文件并记录shape 信息
imgs = [ ]
img_h_list = [ ]
img_w_list = [ ]
count = 0
for j, line in enumerate(lines[::6]):
#get id
id = line.split()[0]
#get data from nori
data = fetcher.get(id)
#decode
img = cv2.imdecode(np.fromstring(data,np.uint8),cv2.IMREAD_COLOR)
#shape recode
img_h_list.append(img.shape[0]) # h
img_w_list.append(img.shape[1]) # w
cv2.imwrite("/data/jupyter/ai_01/homework/origin/origin_%d.jpg" % j, img)
imgs.append(img)
count = j
print("pic count is %d" % count)
#计算平均shape 信息
print("max_h:",max(img_h_list),"min_h:",min(img_h_list),"avg_h",np.mean(img_h_list))
print("max_w:",max(img_w_list),"min_w:",min(img_w_list),"avg_w",np.mean(img_w_list))
#增强
img_augs = seq(images = imgs )
print(len(img_augs))
#记录img_aug文件
for j, img_aug in enumerate(img_augs):
cv2.imwrite("/data/jupyter/ai_01/homework/imgaug/imgaug_%d.jpg" % j, img_aug)
#比较显示1
fig, axes = plt.subplots(1,int(count/50)+1,figsize=(64,64))
ax = axes.ravel()
for j, img_aug in enumerate(img_augs[::50]):
ax[j].imshow(img_aug)
plt.show()
#比较显示2
fig1, axes1 = plt.subplots(1,int(count/50)+1,figsize=(64,64))
ax1 = axes1.ravel()
for j, img in enumerate(imgs[::50]):
ax1[j].imshow(img)
plt.show()
%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
def generate_random_examples(n=100, noise=5):
w = np.random.randint(5, 10)
b = np.random.randint(-10, 10)
print("The real w: {}, b: {}".format(w, b))
# 初始化 data 和 label
data = np.zeros((n,))
label = np.zeros((n,))
# 生成 n 个随机样本数据，并添加一定的噪声干扰
for i in range(n):
data[i] = np.random.uniform(-10, 10)
label[i] = w * data[i] + b + np.random.uniform(-noise, noise)
plt.scatter(data[i], label[i], marker=".") # 将样本点绘制在坐标图上
# 展示样本数据的分布
plt.plot()
plt.show()
return data, label
original_data, original_label = generate_random_examples()
#拟合
import megengine as mge
import megengine.functional as F
from megengine.autodiff import GradManager
import megengine.optimizer as optim
# 设置超参数
epochs = 5
lr = 0.01
# 获取数据
data = mge.tensor(original_data)
label = mge.tensor(original_label)
# 初始化参数
w = mge.Parameter([0.0])
b = mge.Parameter([0.0])
# 定义模型
def linear_model(x):
return F.mul(w, x) + b
# 定义求导器和优化器
gm = GradManager().attach([w, b])
optimizer = optim.SGD([w, b], lr=lr)
# 模型训练
for epoch in range(epochs):
with gm:
pred = linear_model(data)
loss = F.loss.square_loss(pred, label)
gm.backward(loss)
optimizer.step().clear_grad()
print("epoch = {}, w = {:.3f}, b = {:.3f}, loss = {:.3f}"
.format(epoch, w.item(), b.item(), loss.item()))
# 绘图查看拟合情况
x = np.array([-10, 10])
y = w * x + b
plt.scatter(data, label, marker=".")
plt.plot(x, y, "-b")
plt.show()
from urllib import request, error
from bs4 import BeautifulSoup
import re
def askurl(url):
try:
req = request.Request(url)
req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36')
print(req.header_items())
respond = request.urlopen(req)
html = respond.read().decode('utf-8') #read()返回字符串类型
return html
except error.URLError as e:
if hasattr(e, 'code'):
print(e.code)
if hasattr(e, 'reason'):
print(e.reason)
#FindImage = re.compile(r'<img width="220" height="220" data-img="1" data-lazy-img="done" source-data-lazy-img src="//(.+?\.jpg)" >')
FindImage = re.compile(r'data-lazy-img="//(.+?\.jpg)"')
def getdata():
datalist = []
baseurl = 'https://search.jd.com/Search?keyword=%E5%85%85%E6%B0%94%E6%8B%B1%E9%97%A8&wq=%E5%85%85%E6%B0%94%E6%8B%B1%E9%97%A8&pvid=48c602e065124f9b8115bedfc65d47b6&page='
for i in range(26):
print('page in ',i)
url = baseurl + str(2*(i)+1)
print(url)
html = askurl(url)
soup = BeautifulSoup(html, 'html.parser')
for item in soup.find_all('div', {'class':"p-img"}):
item = str(item)
#print( item)
image = FindImage.findall(item)
print('image url:',image)
print('image length:', len(image))
for j in range(len(image)):
datalist.append(image[j])
return datalist
def savedata():
imagelist = getdata()
i = 0
for ImageUrl in imagelist:
i += 1
ImageName = '/data/jupyter/ai_03/data/gate' + str(i) + '.jpg'
print(ImageName)
ImageUrl = 'http://' + ImageUrl
request.urlretrieve(ImageUrl, filename=ImageName)
return
savedata()
何俊：
冯乐鹏：
import cv2
import boto3
import numpy as np
import nori2 as nori
import imgaug as ia
from imgaug import augmenters as iaa
from meghair.utils import io
from meghair.utils.imgproc import imdecode
img_list = []
enhance_num = 10
host = "http://oss.i.brainpp.cn"
fetcher = nori.Fetcher()
# Client初始化
s3_client = boto3.client('s3', endpoint_url=host)
# 获取一个 object
def get_bucket_object(bucket_name="ai-cultivate", object_key="1percent_ImageNet.txt"):
resp = s3_client.get_object(Bucket=bucket_name, Key=object_key)
return resp['Body'].read().decode("utf8")
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
aug = iaa.Sequential(
[
iaa.Fliplr(0.5), # 对50%的图像进行镜像翻转
iaa.Flipud(0.2), # 对20%的图像做左右翻转
sometimes(iaa.Crop(percent=(0, 0.1))), # 对随机的一部分图像做crop操作
# 对一部分图像做仿射变换
sometimes(iaa.Affine(
scale={"x": (0.8, 1.2), "y": (0.8, 1.2)}, # 图像缩放为80%到120%之间
translate_percent={"x": (-0.2, 0.2), "y": (-0.2, 0.2)}, # 平移±20%之间
rotate=(-45, 45), # 旋转±45度之间
shear=(-16, 16), # 剪切变换±16度，（矩形变平行四边形）
order=[0, 1], # 使用最邻近差值或者双线性差值
cval=(0, 255), # 全白全黑填充
mode=ia.ALL # 定义填充图像外区域的方法
)),
],
random_order=True # 随机的顺序把这些操作用在图像上
)
def img_enhance(img):
img = cv2.resize(img, (int(img.shape[0]/4),int(img.shape[1]/4)))
h = img.shape[0]
w = img.shape[1]
img_array = np.array([img] * enhance_num, dtype=np.uint8)
write_img = np.zeros(shape=(h, (w+10) * enhance_num, 3), dtype=np.uint8)
images_aug = aug.augment_images(images=img_array)
for j, item in enumerate(images_aug):
print(item.shape)
write_img[:, j * (w + 10): j * (w + 10) + w, :] = item
cv2.imwrite("data/result.jpg", write_img)
def statistics():
data = get_bucket_object()
for i in data.split('\n'):
if not i:
continue
nori_id, id, img_name = i.split()
img = imdecode(fetcher.get(nori_id))
img_list.append(img.shape)
# # cv2.imwrite(img_name.split('/')[1], img)
img_list_h = [i[0] for i in img_list]
img_list_w = [i[1] for i in img_list]
print('------------------')
print('height \t max:{} \t min:{} \t avg:{}'.format(max(img_list_h), min(img_list_h), np.average(img_list_h)))
print('wight \t max:{} \t min:{} \t avg:{}'.format(max(img_list_w), min(img_list_w), np.average(img_list_w)))
# ------------------
# height max:5065 min:21 avg:405.4505503083288
# wight max:4368 min:46 avg:471.49918039185076
# 对一张图片进行增强，并保存
def enhance():
data = get_bucket_object()
nori_id, id, img_name = data.split('\n')[0].split()
img = imdecode(fetcher.get(nori_id))
img_enhance(img)
if __name__ == '__main__':
statistics()
enhance()
# -*- coding:utf-8 -*-
"""
@Time : 2021/5/24 14:43
@Author: Feng Lepeng
@File : weak2.py
@Desc :
"""
import numpy as np
import megengine as mge
import megengine.functional as F
import matplotlib.pyplot as plt
import megengine.optimizer as optim
from megengine.autodiff import GradManager
data_num = 100
data_noise = 10
epochs = 5
lr = 0.01
w, b = 0, 0
w = mge.Parameter([0.0])
b = mge.Parameter([0.0])
def generate_random_examples(n=data_num, noise=data_noise):
w = np.random.randint(5, 10)
b = np.random.randint(-10, 10)
data = np.zeros((n,))
lable = np.zeros((n,))
for i in range(n):
data[i] = np.random.uniform(-noise, noise)
lable[i] = data[i] * w + b + np.random.uniform(-noise, noise)
return data, lable
def liner_model(x):
return F.mul(w, x) + b
def np_example(data, lable):
n = len(data)
for epoch in range(epochs):
loss, sum_grad_w, sum_grad_b = 0, 0, 0
for i in range(n):
pred = liner_model(data[i])
loss += (pred - lable[i]) ** 2
sum_grad_w += 2 * (pred - lable[i]) * data[i]
sum_grad_b += 2 * (pred - lable[i])
loss = loss / n
grad_b = sum_grad_b / n
grad_w = sum_grad_w / n
w = w - lr * grad_w
b = b - lr * grad_b
print("epoch={},w={},b={},loss={}".format(epoch, w, b, loss))
x = np.array([-10, 10])
y = w * x + b
plt.scatter(data, lable, marker='.')
plt.plot(x, y, '-b')
plt.show()
def mge_example(data, lable):
data = mge.tensor(data)
lable = mge.tensor(lable)
gm = GradManager().attach([w, b])
optimizer = optim.SGD([w, b], lr=lr)
for epoch in range(epochs):
with gm:
pred = liner_model(data)
loss = F.loss.square_loss(pred, lable)
gm.backward(loss)
optimizer.step().clear_grad()
print("epoch={},w={},b={},loss={}".format(epoch, w.item(), b.item(), loss.item()))
x = np.array([-10, 10])
y = w.numpy() * x + b.numpy()
plt.scatter(data, lable, marker='.')
plt.plot(x, y, '-b')
plt.show()
if __name__ == '__main__':
data, lable = generate_random_examples()
mge_example(data, lable)
# -*- coding:utf-8 -*-
"""
@Time : 2021/6/1 15:27
@Author: Feng Lepeng
@File : weak3-1.py
@Desc :
"""
import re
import requests
header = {
'User-Agent': 'Mozilla/5.0(Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0',
'Referer': 'https://image.baidu.com'
}
keyword_encode = "城市道路的充气拱门"
def get_images(total_num, keyword_encode=keyword_encode, start_num=1):
n = start_num
start_num = start_num
page_num = 30
img_set = set()
url = 'https://image.baidu.com/search/acjson?'
while True:
param = {
'tn': 'resultjson_com',
'logid': '8846269338939606587',
'ipn': 'rj',
'ct': '201326592',
'is': '',
'fp': 'result',
'queryWord': keyword_encode,
'cl': '2',
'lm': '-1',
'ie': 'utf-8',
'oe': 'utf-8',
'adpicid': '',
'st': '-1',
'z': '',
'ic': '',
'hd': '',
'latest': '',
'copyright': '',
'word': keyword_encode,
's': '',
'se': '',
'tab': '',
'width': '',
'height': '',
'face': '0',
'istype': '2',
'qc': '',
'nc': '1',
'fr': '',
'expermode': '',
'force': '',
'cg': 'girl',
'pn': start_num,
'rn': page_num,
'gsm': '1e',
}
res = requests.get(url=url, headers=header, params=param)
res.encoding = 'utf-8'
try:
res_data = res.json(strict=False)['data']
for img_url in res_data:
img_url = img_url.get('thumbURL')
if not img_url:
continue
if n > total_num:
print(f"Finished: Totally download {n - 1} images.")
return
if img_url in img_set:
print('重复')
continue
img_set.add(img_url)
download_img(img_url, n)
n += 1
except Exception as e:
print(e)
finally:
start_num += page_num - 1
def download_img(img_url, num):
print(f"Downloading image {num}: {img_url}")
img = requests.get(img_url)
with open('image/{}.jpg'.format(num), 'wb') as f:
f.write(img.content)
if __name__ == '__main__':
get_images(1000, start_num=986)
李大千：
import boto3
import codecs
import numpy as np
import nori2 as nori
from meghair.utils.imgproc import imdecode
from imgaug import augmenters as iaa
from matplotlib import pyplot as plt
%matplotlib inline
host = “http://oss.i.brainpp.cn”
s3_client = boto3.client(‘s3’, endpoint_url=host)
resp = s3_client.get_object(Bucket=“ai-cultivate”, Key=“1percent_ImageNet.txt”)
f = nori.Fetcher()
aug = iaa.Sequential([
iaa.Fliplr(0.5), # 0.5概率水平翻转图像
iaa.Flipud(0.5), #垂直翻转50％的图像：
iaa.Affine(rotate=(-45, 45))
])
h_list =
w_list =
line_stream = codecs.getreader(“utf-8”)
for line in line_stream(resp[‘Body’]):
id = line.split(’ ', 1)[0]
data = f.get(id) # get from Fetecher IS thread-safe.
img = imdecode(data)
h_list.append(img.shape[0])
w_list.append(img.shape[1])
images = [img, img, img, img, img]
images_aug = aug(images=images)
images_aug.insert(0, img)
plt.imshow(np.hstack(images_aug))
plt.show()
print(‘最大高度：’+ str(np.max(h_list)))
print(‘最小高度：’+ str(np.min(h_list)))
print(‘平均高度：’+ str(np.mean(h_list)))
print(‘最大宽度：’+ str(np.max(w_list)))
print(‘最小宽度：’+ str(np.min(w_list)))
print(‘平均高度：’+ str(np.mean(w_list)))
# -*- coding: UTF-8 -*-"""
import requests
import json
import os
import pprint
# 创建一个文件夹
path = './ldq_picture'
if not os.path.exists(path):
os.mkdir(path)
# 导入一个请求头
header = {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'}
# 用户（自己）输入信息指令
keyword = input('请输入你想下载的内容：')
page = input('请输入你想爬取的页数：')
page = int(page) + 1
n = 0
pn = 1# pn代表从第几张图片开始获取，百度图片下滑时默认一次性显示30张
for m in range(1, page):
url = 'https://image.baidu.com/search/acjson?'
param = {
'tn': 'resultjson_com',
'logid': ' 7517080705015306512',
'ipn': 'rj',
'ct': '201326592',
'is': '',
'fp': 'result',
'queryWord': keyword,
'cl': '2',
'lm': '-1',
'ie': 'utf-8',
'oe': 'utf-8',
'adpicid': '',
'st': '',
'z': '',
'ic': '',
'hd': '',
'latest': '',
'copyright': '',
'word': keyword,
's': '',
'se': '',
'tab': '',
'width': '',
'height': '',
'face': '',
'istype': '',
'qc': '',
'nc': '1',
'fr': '',
'expermode': '',
'force': '',
'cg': 'star',
'pn': pn,
'rn': '30',
'gsm': '1e',
}
# 定义一个空列表，用于存放图片的URL
image_url = list()
# 将编码形式转换为utf-8
response = requests.get(url=url, headers=header, params=param)
response.encoding = 'utf-8'
response = response.text # 把字符串转换成json数据
data_s = json.loads(response)
a = data_s["data"] # 提取data里的数据
for i in range(len(a)-1): # 去掉最后一个空数据
data = a[i].get("thumbURL", "not exist") # 防止报错key error
image_url.append(data)
for image_src in image_url:
image_data = requests.get(url=image_src, headers=header).content # 提取图片内容数据
image_name = '{}'.format(n+1) + '.jpg' # 图片名
image_path = path + '/' + image_name # 图片保存路径
with open(image_path, 'wb') as f: # 保存数据
f.write(image_data)
print(image_name, '下载成功啦！！！')
f.close()
n += 1
pn += 29
王旭02
import boto3
import smart_open
import nori2 as nori
from meghair.utils import io
from meghair.utils.imgproc import imdecode
host = "http://oss.i.brainpp.cn"
# Client初始化
s3_client = boto3.client('s3', endpoint_url=host)
# 下载一个 object 到文件 file path
resp = s3_client.download_file("ai-cultivate", "1percent_ImageNet.txt", "file.txt")
datas = smart_open.smart_open("file.txt")
print(datas)
fetcher =nori.Fetcher()
max_w = 0
min_w = 99999
max_h = 0
min_h = 99999
sum_w = 0
sum_h = 0
avg_w = 0
avg_h = 0
for j,item in enumerate(datas):
nori_id = item.split()[0]
nori_str = str(nori_id, encoding = "utf-8")
img = imdecode(fetcher.get(nori_str))[...,:3]
single_h = img.shape[0]
single_w = img.shape[1]
sum_w = single_w+sum_w
sum_h = single_h+sum_h
if(single_h >= max_h):
max_h = single_h
elif(single_h < min_h):
min_h = single_h
if(single_w >= max_w):
max_w = single_w
elif(single_w < min_w):
min_w = single_w
avg_w = sum_w/(j+1)
avg_h = sum_h/(j+1)
print(j)
print("min_w:",min_w)
print("max_w:",max_w)
print("max_h:",max_h)
print("min_h:",min_h)
print("avg_w:",avg_w)
print("avg_h:",avg_h)
> 图片旋转
import imgaug as ia
import imgaug.augmenters as iaa
import boto3
import smart_open
import nori2 as nori
from meghair.utils import io
from meghair.utils.imgproc import imdecode
import numpy as np
import cv2
fetcher =nori.Fetcher()
H,W = 256,128
ia.seed(1)
NUM = 9
seq = iaa.Sequential([
iaa.Crop(px=(0,16)),
iaa.Resize({"height":H,"width":W}),
iaa.Affine(
rotate=(-45,45)
)],random_order=True)
datas = smart_open.smart_open("file.txt")
print(datas)
for j,item in enumerate(datas):
nori_id = item.split()[0]
nori_str = str(nori_id, encoding = "utf-8")
img = imdecode(fetcher.get(nori_str))[...,:3]
images = np.array(
[img] * NUM,
dtype=np.uint8
)
write_img = np.zeros((H,(W+10)*NUM, 3),dtype=np.uint8)
images_aug = seq(images=images)
for i,single in enumerate(images_aug):
write_img[:, i*(W+10): i*(W+10)+W, :]=single
cv2.imwrite("single_%d.jpg" %j ,write_img)
if j>10:
break;
import requests
import urllib
import json
import os
import shutil # 用来删除文件夹
import datetime
def getSogouImag(path):
# 判断文件夹是否存在，存在则删除
if os.path.exists(path):
shutil.rmtree(path)
# 创建文件夹
os.mkdir(path)
m = 0
# 循环爬取，每次100张，共10次
for i in range(0,50):
start_p = i*30
stop_p = (i+1)*30
url_i = 'https://pic.sogou.com/napi/pc/searchList?mode=1&start=' \
+ str(start_p) \
+ '&xml_len=' \
+ str(stop_p) \
+ '&query=%E5%85%85%E6%B0%94%E6%8B%B1%E9%97%A8'
imgs = requests.get(url_i)
imgs_text = imgs.text
imgs_json = json.loads(imgs_text)
imgs_json = imgs_json['data']
imgs_items = imgs_json['items']
for i in imgs_items:
try:
img_url = i['picUrl']
print('*********' + str(m) + '.png********' + 'Downloading...')
print('下载的url: ', img_url)
urllib.request.urlretrieve(img_url, path + str(m) + '.jpg')
m = m + 1
except:
continue
print('Download complete !')
time_stamp = datetime.datetime.now()
print('===start=== at:', time_stamp.strftime('%Y.%m.%d-%H:%M:%S'))
getSogouImag('data_pac/')
time_stamp = datetime.datetime.now()
print('===end=== at:', time_stamp.strftime('%Y.%m.%d-%H:%M:%S'))
https://git-core.megvii-inc.com/ai_train/ai-wangxu02/-/tree/master/week5
https://git-core.megvii-inc.com/ai_train/ai-wangxu02/-/tree/master/week6
彭宏阳：
from refile import smart_open
import nori2
from meghair.utils.imgproc import imdecode
from remisc.imgtools import imconcat, imshow
nf = nori2.Fetcher()
images = []
height, width = [0] * 3, [0] * 3
data_path = "s3://ai-cultivate/1percent_ImageNet.txt"
total_count = 0
with smart_open(data_path, "r") as f:
for line in f.readlines():
nori_id = line.split("\t")[0].strip()
image = imdecode(nf.get(nori_id))
h, w, _ = image.shape
height = [
max(height[0], h),
min(height[1], h) if height[1] > 0 else h,
height[2] + h,
]
width = [
max(width[0], w),
min(width[1], w) if width[1] > 0 else w,
width[2] + w,
]
total_count += 1
if total_count <= 16:
images.append(image)
height[2], width[2] = height[2] / total_count, width[2] / total_count
print("max_height: %s, min_height: %s, mean_height: %s" % tuple(height))
print("max_width: %s, min_width: %s, mean_width: %s" % tuple(width))
import imgaug.augmenters as iaa
seq = iaa.Sequential(
[
iaa.Fliplr(0.2),
iaa.Crop(px=(0, 16)),
iaa.Affine(rotate=(-45, 45)),
iaa.Resize({"height": 256, "width": 128}),
]
)
imshow(imconcat(seq(images=images)))
其他作业 : gitlab
高胜寒：
import cv2
from meghair.utils import io
import os
import nori2 as nori
from meghair.utils.imgproc import imdecode
import numpy as np
from imgaug import augmenters as iaa
import imgaug as ia
import matplotlib.pyplot as plt
# /"%matplotlib inline
H = 128
W = 128
fetcher = nori.Fetcher()
# 镜像、旋转、裁剪、高斯模糊
# 标准化长宽
seq = iaa.Sequential([
iaa.Fliplr(0.5),
iaa.Affine(rotate=(-25, 25)),
iaa.Crop(px=(0, 16)),
iaa.AdditiveGaussianNoise(scale=(0, 30)),
iaa.Resize({"height": H,"width": W})
], random_order=True)
# 无效
val = os.popen("oss cp s3://ai-cultivate/1percent_ImageNet.txt ./").read()
os.popen("oss cp s3://ai-cultivate/1percent_ImageNet.txt ./")
f = open("1percent_ImageNet.txt","r")
urls = f.readlines()
count = 0
max_height = 0
max_width = 0
min_height = 0
min_width = 0
sum_height = 0
sum_width = 0
# 结果文件夹
os.system("mkdir img_results")
for url in urls:
pass
count += 1
nori_id = url.split("\t")[0]
img = imdecode(fetcher.get(nori_id))[..., :3]
img = img[:,:,[2,1,0]]
imgs = [img, img, img, img]
img_augs = seq.augment_images(imgs)
for img_ra, p in zip(img_augs, [0,1,2,3]):
cv2.imwrite("./img_results/" + nori_id.replace(",", "_")+"_" + str(p) + ".jpg",img_ra[:,:,[2,1,0]])
# 获取当前图像长宽值
sp = img.shape
sz1 = sp[0]#height(rows) of image
sz2 = sp[1]#width(colums) of image
sz3 = sp[2]#the pixels value is made up of three primary colors
# 初始化最大最小值
if count == 1:
max_height = sz1
max_width = sz2
min_height = sz1
min_width = sz2
# 比较求最大
if sz1 > max_height:
max_height = sz1
if sz1 > max_width:
max_width = sz2
# 比较求最小
if sz1 < min_height:
min_height = sz1
if sz1 < min_width:
min_width = sz2
# 求和
sum_height += sz1
sum_width += sz2
# 抽取一条数据可视化
if count == 2:
#plt.subplot(1,4,i)
for i in range(1,5):
plt.subplot(1,4,i)
plt.imshow(img_augs[i-1])
#plt.imshow(img_augs[0])
# 打印进度
if count % 1000 == 0:
print(count)
# 用于调试控制数量
if count < -3:
break
## 获取平均值
average_height = sum_height/count
average_width = sum_width/count
print ("max width: %d \nmax height: %d" %(max_width,max_height))
print ("min width: %d \nmin height: %d" %(min_width,min_height))
print ("average_width: %d \naverage_height: %d" %(average_width,average_height))
import megengine as mge
import megengine.functional as F
import numpy as np
from megengine.autodiff import GradManager
import megengine.optimizer as optim
import matplotlib.pyplot as plt
data_num = 150
data_noise = 5
lr = 0.0001
epochs = 30
def generate_random_examples(n=data_num, noise=data_noise):
w = np.random.randint(2, 5)
b = np.random.randint(1, 5)
data = np.zeros((n,))
lable = np.zeros((n,))
# 生成data_num个测试数据
for i in range(n):
# 定义域
data[i] = np.random.uniform(-50, 50)
#print(data[i])
lable[i] = data[i] * w + b + np.random.uniform(-noise, noise)
#print(w)
#print(b)
#print(lable[i])
#print("_________________________")
# 数据x集合
# 数据y集合
return data, lable
data, lable = generate_random_examples()
data = mge.tensor(data)
label = mge.tensor(lable)
w = mge.Parameter([0.0])
b = mge.Parameter([0.0])
#print(data)
#print(label)
def liner_model(x):
return F.mul(w, x) + b
#print(w)
#print(b)
# 求导器
gm = GradManager().attach([w, b])
#print(gm)
# 优化器
optimizer = optim.SGD([w, b], lr=lr)
# 训练集
for epoch in range(epochs):
with gm:
# 预测
pred = liner_model(data)
#print(data)
#print(pred)
# 误差【方差误差】
loss = F.loss.square_loss(pred, label)
# 反向传播
gm.backward(loss)
# 清除梯度
optimizer.step().clear_grad()
print("epoch = {}, w = {:.3f}, b={:.3f}, loss ={:.3f}".format(epoch, w.item(), b.item(), loss.item()))
xx = np.array([-50, 50])
yy = xx * w.numpy() + b.numpy()
#print(w.numpy())
#print(b.numpy())
# 实际数据分布
#print(data)
#print(label)
plt.scatter(data, label, marker=".")
#plt.show()
plt.plot(xx, yy, "-b")
plt.show()
import requests
from lxml import etree
import json
import sys
import time
import datetime
import hmac
import hashlib
import base64
import urllib3
import os
urllib3.disable_warnings()
# 配置当前目录
base_folder = "./"
# 超参
url = "https://www.1688.com/chanpin/-B3E4C6F8B9B0C3C520C6F8B9B0C3C5.html?spm=a261b.2187593.pagenav.1.2f34c0adfjQQ44&beginPage="
def spider():
print("111")
url = "https://www.1688.com/chanpin/-B3E4C6F8B9B0C3C520C6F8B9B0C3C5.html?spm=a261b.2187593.pagenav.1.2f34c0adfjQQ44&beginPage="
for page_id in range(1,2):
print(page_id)
url = url + str(page_id)
try:
r = requests.get(url + str(page_id), verify=False)
except:
continue
pages = r.content
html = etree.HTML(pages)
try:
img_urls = html.xpath('//ul[@id="sw_maindata_asyncload"]//li//div')[0].xpath('//div//a//img[@class="sm-offershopwindow-mainImg"]/@src')
except:
continue
print(img_urls)
print("_________________________________")
for img_url in img_urls:
os.system("wget " + img_url)
def spider_1():
print("222")
url_a = "https://duckduckgo.com/i.js?q=Inflatable%20arches&o=json&p=1&s="
url_b = "&u=bing&f=,,,,,&l=wt-wt&vqd=3-7704905460922458190883528330942841921-235885237586105873644755605151308541334"
for page_id in range(1700,3000,100):
url = url_a + str(page_id) + url_b
try:
proxies = { "http": "http://127.0.0.1:8080", "https": "https://127.0.0.1:8080"}
r = requests.get(url, verify=False)
except:
print("continue")
continue
imgs = json.loads(r.text)['results']
for img in imgs:
img_url = img['image']
print(img_url)
os.system("wget " + img_url)
spider_1()
李成:
import cv2
import boto3
import numpy as np
import nori2 as nori
import imgaug as ia
from imgaug import augmenters as iaa
from meghair.utils import io
from meghair.utils.imgproc import imdecode
img_list = []
img_height = []
img_width = []
enhance_num = 5
fetcher = nori.Fetcher()
s3_client = boto3.client('s3', endpoint_url="http://oss.i.brainpp.cn")
data = s3_client.get_object(Bucket="ai-cultivate", Key="1percent_ImageNet.txt")['Body'].read().decode("utf8")
count = 0
for i in data.split('\n'):
count+=1
if count > 10:
break
if not i:
continue
nori_id, id, img_name = i.split()
img = imdecode(fetcher.get(nori_id))
img_list.append(img)
img_height.append(img.shape[0])
img_width.append(img.shape[1])
print('height max:{} min:{} avg:{}'.format(max(img_height), min(img_height), np.average(img_height)))
print('width max:{} min:{} avg:{}'.format(max(img_width), min(img_width), np.average(img_width)))
aug = iaa.Sequential(
[
iaa.Fliplr(0.5), # 对50%的图像进行镜像翻转
iaa.Flipud(0.2), # 对20%的图像做左右翻转
iaa.Sometimes(0.5, iaa.Crop(percent=(0, 0.1))),
iaa.Sometimes(0.5, iaa.Affine(
scale={"x": (0.8, 1.2), "y": (0.8, 1.2)},
translate_percent={"x": (-0.2, 0.2), "y": (-0.2, 0.2)},
rotate=(-15, 15),
shear=(-16, 16),
order=[0, 1],
cval=(0, 255),
mode=ia.ALL
)),
],
random_order=True
)
img = img_list[2]
img = cv2.resize(img, (int(img.shape[0]/4),int(img.shape[1]/4)))
h = img.shape[0]
w = img.shape[1]
img_array = np.array([img] * enhance_num, dtype=np.uint8)
write_img = np.zeros(shape=(h, (w+10) * enhance_num, 3), dtype=np.uint8)
images_aug = aug.augment_images(images=img_array)
for j, item in enumerate(images_aug):
print(item.shape)
write_img[:, j * (w + 10): j * (w + 10) + w, :] = item
cv2.imwrite("data/result.jpg", write_img)