【AI-CAMP三期】第四组作业

第一周课后作业内容
  • 针对Imagenet数据(s3://ai-cultivate/1percent_ImageNet.txt),进行基本的数据统计(统计框/图片的平均,最大,最小宽高),数据预处理(对数据做旋转增强,randomcrop),并做可视化分析对比数据处理前后的变化
  • 针对外部同学,不指定图片来源,可以使用任意来源的100张图片,进行基本的数据统计(统计框/图片的平均,最大,最小宽高),数据预处理(对数据做旋转增强,randomcrop),并做可视化分析对比数据处理前后的变化
第一周课后作业内容

基础作业:自己随机生成数据并用Megengine实现数据的线性拟合
小组作业:在Mnist 数据上进行训练并测试

王新皓:

分组作业作业
MNIST分类作业
代码
import numpy as np
import megengine as mge
import megengine.functional as F
from megengine.data.dataset import MNIST
from megengine.data import SequentialSampler, RandomSampler, DataLoader
from megengine.autodiff import GradManager
import megengine.optimizer as optim

MNIST_DATA_PATH ="/data/jupyter/ai_02/data/dataset/MNIST"

# 设置超参数
bs = 64
lr = 1e-6
epochs = 5

# 读取原始数据集
train_dataset = MNIST(root=MNIST_DATA_PATH, train=True, download=False)
nums = len(train_dataset)
num_features = 784   # (28, 28, 1) Flatten -> 784
num_classes = 10

# 训练数据加载与预处理
train_sampler = SequentialSampler(dataset=train_dataset, batch_size=bs)
train_dataloader = DataLoader(dataset=train_dataset, sampler=train_sampler)

# 初始化参数
W = mge.Parameter(np.zeros((num_features, num_classes)))
b = mge.Parameter(np.zeros((num_classes,)))

# 定义模型
def linear_cls(X):
    return F.matmul(X, W) + b

# 定义求导器和优化器
gm = GradManager().attach([W, b])
optimizer = optim.SGD([W, b], lr=lr)

# 模型训练
for epoch in range(epochs):
    total_loss = 0
    for batch_data, batch_label in train_dataloader:
        batch_data = F.flatten(mge.tensor(batch_data), 1)
        batch_label = mge.tensor(batch_label)
        with gm:
            pred = linear_cls(batch_data)
            loss = F.loss.cross_entropy(pred, batch_label)
            gm.backward(loss)
        optimizer.step().clear_grad()
        total_loss +=  loss.item()
    print("epoch = {}, loss = {:.3f}".format(epoch, total_loss / len(train_dataloader)))


test_dataset = MNIST(root=MNIST_DATA_PATH, train=False, download=False)
test_sampler = RandomSampler(dataset=test_dataset, batch_size=100)
test_dataloader = DataLoader(dataset=test_dataset, sampler=test_sampler)

nums_correct = 0
for batch_data, batch_label in test_dataloader:
    batch_data = F.flatten(mge.tensor(batch_data), 1)
    batch_label = mge.tensor(batch_label)
    logits = linear_cls(batch_data)
    pred = F.argmax(logits, axis=1)
    nums_correct += (pred == batch_label).sum().item()
print("Accuracy = {:.3f}".format(nums_correct / len(test_dataloader)))
结果

第一周课后作业
代码
#导入工具
import cv2
import sys
import nori2 as nori
import imgaug as ia 
import numpy as np
import imgaug.augmenters as iaa
from meghair.utils import io
from meghair.utils.imgproc import imdecode
from matplotlib import pyplot as plt 
from skimage import io,data
#数据库获取工具
fetcher = nori.Fetcher()
#imgaug config
H,W = 128 ,128
sometimes = lambda aug: iaa.Sometimes(0.5, aug)
seq = iaa.Sequential([
    sometimes(iaa.Affine(
        rotate=(-90, 90)
    )),
    iaa.Crop(px=(0, 30)),
    iaa.Resize({"height": H, "width": W})
])
#文件读取
image_net_set = open("./1percent_ImageNet.txt","r");
print(image_net_set)
#获取原始文件并记录shape 信息
imgs = [ ]
img_h_list = [ ]
img_w_list = [ ]
count = 0
for j, line in enumerate(lines[::6]):
    #get id 
    id = line.split()[0]
    #get data from nori
    data = fetcher.get(id)
    #decode 
    img = cv2.imdecode(np.fromstring(data,np.uint8),cv2.IMREAD_COLOR)
    #shape recode
    img_h_list.append(img.shape[0]) # h
    img_w_list.append(img.shape[1]) # w
    cv2.imwrite("/data/jupyter/ai_01/homework/origin/origin_%d.jpg" % j, img)
    imgs.append(img)
    count = j
print("pic count is %d"  % count)    
#计算平均shape 信息
print("max_h:",max(img_h_list),"min_h:",min(img_h_list),"avg_h",np.mean(img_h_list))
print("max_w:",max(img_w_list),"min_w:",min(img_w_list),"avg_w",np.mean(img_w_list))
#增强
img_augs = seq(images = imgs )
print(len(img_augs))
#记录img_aug文件
for j, img_aug in enumerate(img_augs):
    cv2.imwrite("/data/jupyter/ai_01/homework/imgaug/imgaug_%d.jpg" % j, img_aug)
#比较显示1
fig, axes = plt.subplots(1,int(count/50)+1,figsize=(64,64))
ax = axes.ravel()
for j, img_aug in enumerate(img_augs[::50]):
    ax[j].imshow(img_aug)
plt.show()
#比较显示2
fig1, axes1 = plt.subplots(1,int(count/50)+1,figsize=(64,64))
ax1 = axes1.ravel()
for j, img in enumerate(imgs[::50]):
    ax1[j].imshow(img)
plt.show()
结果

第二周课后作业
代码
%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt

def generate_random_examples(n=100, noise=5):

    w = np.random.randint(5, 10)
    b = np.random.randint(-10, 10)
    print("The real w: {}, b: {}".format(w, b))

    # 初始化 data 和 label
    data = np.zeros((n,))
    label = np.zeros((n,))

    # 生成 n 个随机样本数据,并添加一定的噪声干扰
    for i in range(n):
        data[i] = np.random.uniform(-10, 10)
        label[i] = w * data[i] + b + np.random.uniform(-noise, noise)
        plt.scatter(data[i], label[i], marker=".")  # 将样本点绘制在坐标图上

    # 展示样本数据的分布
    plt.plot()
    plt.show()
    return data, label

original_data, original_label = generate_random_examples()
#拟合
import megengine as mge
import megengine.functional as F
from megengine.autodiff import GradManager
import megengine.optimizer as optim

# 设置超参数
epochs = 5
lr = 0.01

# 获取数据
data = mge.tensor(original_data)
label = mge.tensor(original_label)

# 初始化参数
w = mge.Parameter([0.0])
b = mge.Parameter([0.0])

# 定义模型
def linear_model(x):
    return F.mul(w, x) + b

# 定义求导器和优化器
gm = GradManager().attach([w, b])
optimizer = optim.SGD([w, b], lr=lr)

# 模型训练
for epoch in range(epochs):
    with gm:
        pred = linear_model(data)
        loss = F.loss.square_loss(pred, label)
        gm.backward(loss)
    optimizer.step().clear_grad()

    print("epoch = {}, w = {:.3f}, b = {:.3f}, loss = {:.3f}"
          .format(epoch, w.item(), b.item(), loss.item()))
    
    # 绘图查看拟合情况
    x = np.array([-10, 10])
    y = w * x + b
    plt.scatter(data, label, marker=".")
    plt.plot(x, y, "-b")
    plt.show()

结果

第三周课后作业
代码
from urllib import request, error
from bs4 import BeautifulSoup
import re

def askurl(url):
    try:
        req = request.Request(url)
        req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36')
        print(req.header_items())
                            
        respond = request.urlopen(req)
        html = respond.read().decode('utf-8') #read()返回字符串类型
        return html
    except error.URLError as e:
        if hasattr(e, 'code'):
            print(e.code)
        if hasattr(e, 'reason'):
            print(e.reason)
            
#FindImage = re.compile(r'<img width="220" height="220" data-img="1" data-lazy-img="done" source-data-lazy-img src="//(.+?\.jpg)" >')
FindImage = re.compile(r'data-lazy-img="//(.+?\.jpg)"')

def getdata():
    datalist = []
    baseurl = 'https://search.jd.com/Search?keyword=%E5%85%85%E6%B0%94%E6%8B%B1%E9%97%A8&wq=%E5%85%85%E6%B0%94%E6%8B%B1%E9%97%A8&pvid=48c602e065124f9b8115bedfc65d47b6&page='
    for i in range(26):
        print('page in ',i)
        url = baseurl + str(2*(i)+1)
        print(url)
        html = askurl(url)
        soup = BeautifulSoup(html, 'html.parser')
        for item in soup.find_all('div', {'class':"p-img"}):
            item = str(item)
            #print( item)
            image = FindImage.findall(item)
            print('image url:',image)
            print('image length:', len(image))
            for j in range(len(image)):
                datalist.append(image[j])
    return datalist      

def savedata():
    imagelist = getdata()
    i = 0
    for ImageUrl in imagelist:
        i += 1
        ImageName = '/data/jupyter/ai_03/data/gate' + str(i) + '.jpg'
        print(ImageName)
        ImageUrl = 'http://' + ImageUrl
        request.urlretrieve(ImageUrl, filename=ImageName)
    return

savedata()
            

何俊:

第一周课后作业

数据统计

week1_data_stats

数据增强

代码放在 git-core

第二周课后作业

code

第三周课后作业

code

第四周课后作业

code

第五周课后作业

code

第六周课后作业

code

冯乐鹏:

第一周课后作业
代码
import cv2
import boto3
import numpy as np
import nori2 as nori
import imgaug as ia
from imgaug import augmenters as iaa
from meghair.utils import io
from meghair.utils.imgproc import imdecode


img_list = []
enhance_num = 10
host = "http://oss.i.brainpp.cn"
fetcher = nori.Fetcher()

# Client初始化
s3_client = boto3.client('s3', endpoint_url=host)

    
# 获取一个 object
def get_bucket_object(bucket_name="ai-cultivate", object_key="1percent_ImageNet.txt"):
    resp = s3_client.get_object(Bucket=bucket_name, Key=object_key)
    return resp['Body'].read().decode("utf8")


sometimes = lambda aug: iaa.Sometimes(0.5, aug)
aug = iaa.Sequential(
        [
            iaa.Fliplr(0.5),  # 对50%的图像进行镜像翻转
            iaa.Flipud(0.2),  # 对20%的图像做左右翻转
            sometimes(iaa.Crop(percent=(0, 0.1))),  # 对随机的一部分图像做crop操作
            
            # 对一部分图像做仿射变换
            sometimes(iaa.Affine(
                scale={"x": (0.8, 1.2), "y": (0.8, 1.2)},  # 图像缩放为80%到120%之间
                translate_percent={"x": (-0.2, 0.2), "y": (-0.2, 0.2)},  # 平移±20%之间
                rotate=(-45, 45),  # 旋转±45度之间
                shear=(-16, 16),  # 剪切变换±16度,(矩形变平行四边形)
                order=[0, 1],  # 使用最邻近差值或者双线性差值
                cval=(0, 255),  # 全白全黑填充
                mode=ia.ALL  # 定义填充图像外区域的方法
            )),
        ],
        random_order=True  # 随机的顺序把这些操作用在图像上
    )

            
def img_enhance(img):
    img = cv2.resize(img, (int(img.shape[0]/4),int(img.shape[1]/4)))
    h = img.shape[0]
    w = img.shape[1]
    img_array = np.array([img] * enhance_num, dtype=np.uint8)
    write_img = np.zeros(shape=(h, (w+10) * enhance_num, 3), dtype=np.uint8)
    images_aug = aug.augment_images(images=img_array)
    for j, item in enumerate(images_aug):
        print(item.shape)
        write_img[:, j * (w + 10): j * (w + 10) + w, :] = item

    cv2.imwrite("data/result.jpg", write_img)

            
def statistics():
    data = get_bucket_object()
    for i in data.split('\n'):
        if not i:
            continue
        
        nori_id, id, img_name = i.split()
        img = imdecode(fetcher.get(nori_id))
        img_list.append(img.shape)
#         # cv2.imwrite(img_name.split('/')[1], img)
        
    img_list_h = [i[0] for i in img_list]
    img_list_w = [i[1] for i in img_list]
    print('------------------')
    print('height \t max:{} \t min:{} \t avg:{}'.format(max(img_list_h), min(img_list_h), np.average(img_list_h)))
    print('wight \t max:{} \t min:{} \t avg:{}'.format(max(img_list_w), min(img_list_w), np.average(img_list_w)))
#     ------------------
#     height 	 max:5065 	 min:21 	 avg:405.4505503083288
#     wight 	 max:4368 	 min:46 	 avg:471.49918039185076

# 对一张图片进行增强,并保存
def enhance():
    data = get_bucket_object()
    nori_id, id, img_name = data.split('\n')[0].split()  
    img = imdecode(fetcher.get(nori_id))
    img_enhance(img)
    
    
if __name__ == '__main__':
    statistics()
    enhance()
结果

统计结果:
image

图片增强:

第二周课后作业
代码
# -*- coding:utf-8 -*-
"""
    @Time  : 2021/5/24  14:43
    @Author: Feng Lepeng
    @File  : weak2.py
    @Desc  :
"""
import numpy as np
import megengine as mge
import megengine.functional as F
import matplotlib.pyplot as plt
import megengine.optimizer as optim
from megengine.autodiff import GradManager


data_num = 100
data_noise = 10
epochs = 5
lr = 0.01
w, b = 0, 0
w = mge.Parameter([0.0])
b = mge.Parameter([0.0])


def generate_random_examples(n=data_num, noise=data_noise):
    w = np.random.randint(5, 10)
    b = np.random.randint(-10, 10)

    data = np.zeros((n,))
    lable = np.zeros((n,))

    for i in range(n):
        data[i] = np.random.uniform(-noise, noise)
        lable[i] = data[i] * w + b + np.random.uniform(-noise, noise)

    return data, lable


def liner_model(x):
    return F.mul(w, x) + b


def np_example(data, lable):
    n = len(data)

    for epoch in range(epochs):
        loss, sum_grad_w, sum_grad_b = 0, 0, 0

        for i in range(n):
            pred = liner_model(data[i])
            loss += (pred - lable[i]) ** 2

            sum_grad_w += 2 * (pred - lable[i]) * data[i]
            sum_grad_b += 2 * (pred - lable[i])

        loss = loss / n
        grad_b = sum_grad_b / n
        grad_w = sum_grad_w / n

        w = w - lr * grad_w
        b = b - lr * grad_b

        print("epoch={},w={},b={},loss={}".format(epoch, w, b, loss))

        x = np.array([-10, 10])
        y = w * x + b
        plt.scatter(data, lable, marker='.')
        plt.plot(x, y, '-b')
        plt.show()


def mge_example(data, lable):
    data = mge.tensor(data)
    lable = mge.tensor(lable)
    gm = GradManager().attach([w, b])
    optimizer = optim.SGD([w, b], lr=lr)
    for epoch in range(epochs):
        with gm:
            pred = liner_model(data)
            loss = F.loss.square_loss(pred, lable)
            gm.backward(loss)
        optimizer.step().clear_grad()
        print("epoch={},w={},b={},loss={}".format(epoch, w.item(), b.item(), loss.item()))

        x = np.array([-10, 10])
        y = w.numpy() * x + b.numpy()
        plt.scatter(data, lable, marker='.')
        plt.plot(x, y, '-b')
        plt.show()


if __name__ == '__main__':
    data, lable = generate_random_examples()
    mge_example(data, lable)

结果

第三周课后作业
代码
# -*- coding:utf-8 -*-
"""
    @Time  : 2021/6/1  15:27
    @Author: Feng Lepeng
    @File  : weak3-1.py
    @Desc  :
"""
import re
import requests

header = {
    'User-Agent': 'Mozilla/5.0(Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0',
    'Referer': 'https://image.baidu.com'
}

keyword_encode = "城市道路的充气拱门"


def get_images(total_num, keyword_encode=keyword_encode, start_num=1):
    n = start_num
    start_num = start_num
    page_num = 30
    img_set = set()
    url = 'https://image.baidu.com/search/acjson?'

    while True:

        param = {
            'tn': 'resultjson_com',
            'logid': '8846269338939606587',
            'ipn': 'rj',
            'ct': '201326592',
            'is': '',
            'fp': 'result',
            'queryWord': keyword_encode,
            'cl': '2',
            'lm': '-1',
            'ie': 'utf-8',
            'oe': 'utf-8',
            'adpicid': '',
            'st': '-1',
            'z': '',
            'ic': '',
            'hd': '',
            'latest': '',
            'copyright': '',
            'word': keyword_encode,
            's': '',
            'se': '',
            'tab': '',
            'width': '',
            'height': '',
            'face': '0',
            'istype': '2',
            'qc': '',
            'nc': '1',
            'fr': '',
            'expermode': '',
            'force': '',
            'cg': 'girl',
            'pn': start_num,
            'rn': page_num,
            'gsm': '1e',
        }
        res = requests.get(url=url, headers=header, params=param)
        res.encoding = 'utf-8'
        try:
            res_data = res.json(strict=False)['data']
            for img_url in res_data:
                img_url = img_url.get('thumbURL')

                if not img_url:
                    continue

                if n > total_num:
                    print(f"Finished: Totally download {n - 1} images.")
                    return

                if img_url in img_set:
                    print('重复')
                    continue

                img_set.add(img_url)
                download_img(img_url, n)
                n += 1
        except Exception as e:
            print(e)
        finally:
            start_num += page_num - 1


def download_img(img_url, num):
    print(f"Downloading image {num}: {img_url}")
    img = requests.get(img_url)
    with open('image/{}.jpg'.format(num), 'wb') as f:
        f.write(img.content)


if __name__ == '__main__':
    get_images(1000, start_num=986)

第四周课后作业
studio 提供的数据集
代码

https://studio.brainpp.com/project/7378

截图

10个批次之后,不再收敛,原因未知

小组数据集
代码

https://git-core.megvii-inc.com/ai_train/ai-fenglepeng/-/tree/master/weak4

结果

训练 10 个批次,预测准确率97.05%


训练 100 个批次,预测准确率98.33%

第五周课后作业
studio 提供的数据集
项目

https://studio.brainpp.com/project/7612

截图

小组数据集
代码

https://git-core.megvii-inc.com/ai_train/ai-fenglepeng/-/tree/master/weak5

第六周课后作业

https://git-core.megvii-inc.com/ai_train/ai-fenglepeng/-/tree/master/weak6

第七周课后作业

https://git-core.megvii-inc.com/ai_train/ai-fenglepeng/-/tree/master/weak7

李大千:

第一周作业
代码

import boto3
import codecs
import numpy as np
import nori2 as nori
from meghair.utils.imgproc import imdecode
from imgaug import augmenters as iaa
from matplotlib import pyplot as plt
%matplotlib inline

host = “http://oss.i.brainpp.cn
s3_client = boto3.client(‘s3’, endpoint_url=host)
resp = s3_client.get_object(Bucket=“ai-cultivate”, Key=“1percent_ImageNet.txt”)

f = nori.Fetcher()

aug = iaa.Sequential([
iaa.Fliplr(0.5), # 0.5概率水平翻转图像
iaa.Flipud(0.5), #垂直翻转50%的图像:
iaa.Affine(rotate=(-45, 45))
])
h_list =
w_list =
line_stream = codecs.getreader(“utf-8”)
for line in line_stream(resp[‘Body’]):
id = line.split(’ ', 1)[0]
data = f.get(id) # get from Fetecher IS thread-safe.
img = imdecode(data)

    h_list.append(img.shape[0])
    w_list.append(img.shape[1])
    images = [img, img, img, img, img]
    images_aug = aug(images=images)
    images_aug.insert(0, img)
    plt.imshow(np.hstack(images_aug))
    plt.show()

print(‘最大高度:’+ str(np.max(h_list)))
print(‘最小高度:’+ str(np.min(h_list)))
print(‘平均高度:’+ str(np.mean(h_list)))
print(‘最大宽度:’+ str(np.max(w_list)))
print(‘最小宽度:’+ str(np.min(w_list)))
print(‘平均高度:’+ str(np.mean(w_list)))

结果


第三周作业
代码
# -*- coding: UTF-8 -*-"""


import requests
import json
import os
import pprint

# 创建一个文件夹
path = './ldq_picture'
if not os.path.exists(path):
    os.mkdir(path)

# 导入一个请求头
header = {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_1_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'}
# 用户(自己)输入信息指令
keyword = input('请输入你想下载的内容:')
page = input('请输入你想爬取的页数:')
page = int(page) + 1
n = 0
pn = 1# pn代表从第几张图片开始获取,百度图片下滑时默认一次性显示30张
for m in range(1, page):
    url = 'https://image.baidu.com/search/acjson?'
    param = {
        'tn': 'resultjson_com',
        'logid': ' 7517080705015306512',
        'ipn': 'rj',
        'ct': '201326592',
        'is': '',
        'fp': 'result',
        'queryWord': keyword,
        'cl': '2',
        'lm': '-1',
        'ie': 'utf-8',
        'oe': 'utf-8',
        'adpicid': '',
        'st': '',
        'z': '',
        'ic': '',
        'hd': '',
        'latest': '',
        'copyright': '',
        'word': keyword,
        's': '',
        'se': '',
        'tab': '',
        'width': '',
        'height': '',
        'face': '',
        'istype': '',
        'qc': '',
        'nc': '1',
        'fr': '',
        'expermode': '',
        'force': '',
        'cg': 'star',
        'pn': pn,
        'rn': '30',
        'gsm': '1e',
    }

    # 定义一个空列表,用于存放图片的URL
    image_url = list()

    # 将编码形式转换为utf-8
    response = requests.get(url=url, headers=header, params=param)
    response.encoding = 'utf-8'
    response = response.text    # 把字符串转换成json数据
    data_s = json.loads(response)
    a = data_s["data"]  # 提取data里的数据

    for i in range(len(a)-1):  # 去掉最后一个空数据
        data = a[i].get("thumbURL", "not exist")  # 防止报错key error
        image_url.append(data)

    for image_src in image_url:

        image_data = requests.get(url=image_src, headers=header).content  # 提取图片内容数据
        image_name = '{}'.format(n+1) + '.jpg'  # 图片名
        image_path = path + '/' + image_name  # 图片保存路径
        with open(image_path, 'wb') as f:  # 保存数据

            f.write(image_data)

            print(image_name, '下载成功啦!!!')

            f.close()

        n += 1

    pn += 29

王旭02

第一周课后作业
代码
import boto3
import smart_open
import nori2 as nori
from meghair.utils import io
from meghair.utils.imgproc import imdecode
host = "http://oss.i.brainpp.cn"

# Client初始化
s3_client = boto3.client('s3', endpoint_url=host)
# 下载一个 object 到文件 file path
resp = s3_client.download_file("ai-cultivate", "1percent_ImageNet.txt", "file.txt")

datas = smart_open.smart_open("file.txt")
print(datas)
fetcher =nori.Fetcher()
max_w = 0
min_w = 99999
max_h = 0
min_h = 99999
sum_w = 0
sum_h = 0
avg_w = 0
avg_h = 0

for j,item in enumerate(datas):
    nori_id = item.split()[0]
    nori_str = str(nori_id, encoding = "utf-8")
    img = imdecode(fetcher.get(nori_str))[...,:3]
    single_h = img.shape[0]
    single_w = img.shape[1]
    sum_w = single_w+sum_w
    sum_h = single_h+sum_h
    if(single_h >= max_h):
        max_h = single_h
    elif(single_h < min_h):
        min_h = single_h
    if(single_w >= max_w):
        max_w = single_w
    elif(single_w < min_w):
        min_w = single_w
avg_w = sum_w/(j+1)
avg_h = sum_h/(j+1)
print(j)
print("min_w:",min_w)
print("max_w:",max_w)
print("max_h:",max_h)
print("min_h:",min_h)
print("avg_w:",avg_w)
print("avg_h:",avg_h)

> 图片旋转
import imgaug as ia
import imgaug.augmenters as iaa
import boto3
import smart_open
import nori2 as nori
from meghair.utils import io
from meghair.utils.imgproc import imdecode
import numpy as np
import cv2
fetcher =nori.Fetcher()
H,W = 256,128
ia.seed(1)
NUM = 9
seq = iaa.Sequential([
    iaa.Crop(px=(0,16)),
    iaa.Resize({"height":H,"width":W}),
    iaa.Affine(
        rotate=(-45,45)
)],random_order=True)
datas = smart_open.smart_open("file.txt")
print(datas)
for j,item in enumerate(datas):
    nori_id = item.split()[0]
    nori_str = str(nori_id, encoding = "utf-8")
    img = imdecode(fetcher.get(nori_str))[...,:3]
    images = np.array(
            [img] * NUM,
            dtype=np.uint8
    )
    write_img = np.zeros((H,(W+10)*NUM, 3),dtype=np.uint8)
    images_aug = seq(images=images)
    for i,single in enumerate(images_aug):
        write_img[:, i*(W+10): i*(W+10)+W, :]=single
    cv2.imwrite("single_%d.jpg" %j ,write_img)
    if j>10:
        break;
结果

min_w: 46
max_w: 4368
max_h: 5065
min_h: 21
avg_w: 471.49918039185076
avg_h: 405.4505503083288

第三周课后作业
代码
import requests
import urllib
import json
import os
import shutil  # 用来删除文件夹
import datetime
 
def getSogouImag(path):
    # 判断文件夹是否存在,存在则删除
    if os.path.exists(path):
        shutil.rmtree(path)
    # 创建文件夹
    os.mkdir(path)
    m = 0
    # 循环爬取,每次100张,共10次
    for i in range(0,50):
        start_p = i*30
        stop_p = (i+1)*30
        url_i = 'https://pic.sogou.com/napi/pc/searchList?mode=1&start=' \
                + str(start_p) \
                + '&xml_len=' \
                + str(stop_p) \
                + '&query=%E5%85%85%E6%B0%94%E6%8B%B1%E9%97%A8'
        imgs = requests.get(url_i)
        imgs_text = imgs.text
        imgs_json = json.loads(imgs_text)
        imgs_json = imgs_json['data']
        imgs_items = imgs_json['items']
        for i in imgs_items:
            try:
                img_url = i['picUrl']
                print('*********' + str(m) + '.png********' + 'Downloading...')
                print('下载的url: ', img_url)
                urllib.request.urlretrieve(img_url, path + str(m) + '.jpg')
                m = m + 1
            except:
                continue
    print('Download complete !')
 
time_stamp = datetime.datetime.now()
print('===start=== at:', time_stamp.strftime('%Y.%m.%d-%H:%M:%S'))
 
getSogouImag('data_pac/')
 
time_stamp = datetime.datetime.now()
print('===end=== at:', time_stamp.strftime('%Y.%m.%d-%H:%M:%S'))
第五周课后作业
代码
https://git-core.megvii-inc.com/ai_train/ai-wangxu02/-/tree/master/week5
结果
![results|310x310](upload://favNHH5ZgKdz3NozBFFns9iSJcS.jpeg) 
![results|310x310](upload://favNHH5ZgKdz3NozBFFns9iSJcS.jpeg) 
![results1|500x345](upload://paCaODVLBkDIfYAqETqBEgsCWZD.jpeg) 
第六周课后作业
代码
https://git-core.megvii-inc.com/ai_train/ai-wangxu02/-/tree/master/week6

彭宏阳:

第一周课后作业
代码
from refile import smart_open
import nori2
from meghair.utils.imgproc import imdecode
from remisc.imgtools import imconcat, imshow

nf = nori2.Fetcher()

images = []
height, width = [0] * 3, [0] * 3
data_path = "s3://ai-cultivate/1percent_ImageNet.txt"
total_count = 0
with smart_open(data_path, "r") as f:
    for line in f.readlines():
        nori_id = line.split("\t")[0].strip()
        image = imdecode(nf.get(nori_id))
        h, w, _ = image.shape
        height = [
            max(height[0], h),
            min(height[1], h) if height[1] > 0 else h,
            height[2] + h,
        ]
        width = [
            max(width[0], w),
            min(width[1], w) if width[1] > 0 else w,
            width[2] + w,
        ]
        total_count += 1
        if total_count <= 16:
            images.append(image)

height[2], width[2] = height[2] / total_count, width[2] / total_count
print("max_height: %s, min_height: %s, mean_height: %s" % tuple(height))
print("max_width: %s, min_width: %s, mean_width: %s" % tuple(width))

import imgaug.augmenters as iaa

seq = iaa.Sequential(
    [
        iaa.Fliplr(0.2),
        iaa.Crop(px=(0, 16)),
        iaa.Affine(rotate=(-45, 45)),
        iaa.Resize({"height": 256, "width": 128}),
    ]
)

imshow(imconcat(seq(images=images)))
结果

max_height: 5065, min_height: 21, mean_height: 405.4505503083288
max_width: 4368, min_width: 46, mean_width: 471.49918039185076

其他作业 : gitlab

高胜寒:

第一周课后作业
代码
import cv2
from meghair.utils import io
import os
import nori2 as nori
from meghair.utils.imgproc import imdecode
import numpy as np
from imgaug import augmenters as iaa
import imgaug as ia
import matplotlib.pyplot as plt


# /"%matplotlib inline
H = 128
W = 128
fetcher = nori.Fetcher()
# 镜像、旋转、裁剪、高斯模糊
# 标准化长宽
seq = iaa.Sequential([
        iaa.Fliplr(0.5),
        iaa.Affine(rotate=(-25, 25)),
        iaa.Crop(px=(0, 16)),
        iaa.AdditiveGaussianNoise(scale=(0, 30)),
        iaa.Resize({"height": H,"width": W})
    ], random_order=True)

# 无效
val = os.popen("oss cp s3://ai-cultivate/1percent_ImageNet.txt ./").read()
os.popen("oss cp s3://ai-cultivate/1percent_ImageNet.txt ./")
f = open("1percent_ImageNet.txt","r")
urls = f.readlines()

count = 0
max_height = 0
max_width = 0
min_height = 0
min_width = 0
sum_height = 0
sum_width = 0

# 结果文件夹
os.system("mkdir img_results")

for url in urls:
    pass
    count += 1
    nori_id = url.split("\t")[0]
    img = imdecode(fetcher.get(nori_id))[..., :3]
    img = img[:,:,[2,1,0]]
    imgs = [img, img, img, img]
    img_augs = seq.augment_images(imgs)  
    for img_ra, p in zip(img_augs, [0,1,2,3]):
        cv2.imwrite("./img_results/" + nori_id.replace(",", "_")+"_" + str(p) + ".jpg",img_ra[:,:,[2,1,0]])
    # 获取当前图像长宽值
    sp = img.shape
    sz1 = sp[0]#height(rows) of image
    sz2 = sp[1]#width(colums) of image
    sz3 = sp[2]#the pixels value is made up of three primary colors
    # 初始化最大最小值
    if count == 1:
        max_height = sz1
        max_width = sz2
        min_height = sz1
        min_width = sz2
    # 比较求最大
    if sz1 > max_height:
        max_height = sz1
    if sz1 > max_width:
        max_width = sz2
    # 比较求最小
    if sz1 < min_height:
        min_height = sz1
    if sz1 < min_width:
        min_width = sz2
    # 求和
    sum_height += sz1
    sum_width += sz2
    # 抽取一条数据可视化
    if count == 2:        
        #plt.subplot(1,4,i)
        for i in range(1,5):
            plt.subplot(1,4,i)
            plt.imshow(img_augs[i-1])
            #plt.imshow(img_augs[0])
    # 打印进度
    if count % 1000 == 0:
        print(count)
    # 用于调试控制数量
    if count < -3:
        break
## 获取平均值
average_height = sum_height/count
average_width = sum_width/count
print ("max width: %d \nmax height: %d" %(max_width,max_height))
print ("min width: %d \nmin height: %d" %(min_width,min_height))
print ("average_width: %d \naverage_height: %d" %(average_width,average_height))
结果

04B1F6CF-4284-4095-A2A2-1E5F7F961914

第二周课后作业
代码
import megengine as mge
import megengine.functional as F
import numpy as np
from megengine.autodiff import GradManager
import megengine.optimizer as optim
import matplotlib.pyplot as plt


data_num = 150
data_noise = 5
lr = 0.0001
epochs = 30

def generate_random_examples(n=data_num, noise=data_noise):
    w = np.random.randint(2, 5)
    b = np.random.randint(1, 5)

    data = np.zeros((n,))
    lable = np.zeros((n,))
    # 生成data_num个测试数据
    for i in range(n):
        # 定义域
        data[i] = np.random.uniform(-50, 50)
        #print(data[i])
        lable[i] = data[i] * w + b + np.random.uniform(-noise, noise)
        #print(w)
        #print(b)
        #print(lable[i])
        #print("_________________________")
    # 数据x集合
    # 数据y集合
    return data, lable

data, lable = generate_random_examples()



data = mge.tensor(data)
label = mge.tensor(lable)

w = mge.Parameter([0.0])
b = mge.Parameter([0.0])

#print(data)
#print(label)


def liner_model(x):
    return F.mul(w, x) + b




#print(w)
#print(b)

# 求导器
gm = GradManager().attach([w, b])
#print(gm)
# 优化器
optimizer = optim.SGD([w, b], lr=lr)




# 训练集
for epoch in range(epochs):
    with gm:
        # 预测
        pred = liner_model(data)
        #print(data)
        #print(pred)
        # 误差【方差误差】
        loss = F.loss.square_loss(pred, label)
        # 反向传播
        gm.backward(loss)
    # 清除梯度
    optimizer.step().clear_grad()
    
    print("epoch = {}, w = {:.3f}, b={:.3f}, loss ={:.3f}".format(epoch, w.item(), b.item(), loss.item()))

     
xx = np.array([-50, 50])
yy = xx * w.numpy() + b.numpy()

#print(w.numpy())
#print(b.numpy())
# 实际数据分布
#print(data)
#print(label)
plt.scatter(data, label, marker=".")
#plt.show()
plt.plot(xx, yy, "-b")
plt.show()
结果

3477A7C9-BE2D-4308-8BBF-C11FB2A46985

第三周课后作业
代码
import requests
from lxml import etree
import json
import sys
import time
import datetime
import hmac
import hashlib
import base64
import urllib3
import os
urllib3.disable_warnings()

# 配置当前目录
base_folder = "./"


# 超参
url = "https://www.1688.com/chanpin/-B3E4C6F8B9B0C3C520C6F8B9B0C3C5.html?spm=a261b.2187593.pagenav.1.2f34c0adfjQQ44&beginPage="



def spider():
    print("111")
    url = "https://www.1688.com/chanpin/-B3E4C6F8B9B0C3C520C6F8B9B0C3C5.html?spm=a261b.2187593.pagenav.1.2f34c0adfjQQ44&beginPage="
    for page_id in range(1,2):
        print(page_id)
        url = url + str(page_id)
        try:
            r = requests.get(url + str(page_id), verify=False)
        except:
            continue
        pages = r.content
        html = etree.HTML(pages)
        try:
            img_urls = html.xpath('//ul[@id="sw_maindata_asyncload"]//li//div')[0].xpath('//div//a//img[@class="sm-offershopwindow-mainImg"]/@src')
        except:
            continue
        print(img_urls)
        print("_________________________________")
        for img_url in img_urls:
            os.system("wget " + img_url)



def spider_1():
    print("222")
    url_a = "https://duckduckgo.com/i.js?q=Inflatable%20arches&o=json&p=1&s="
    url_b = "&u=bing&f=,,,,,&l=wt-wt&vqd=3-7704905460922458190883528330942841921-235885237586105873644755605151308541334"
    for page_id in range(1700,3000,100):
        url = url_a + str(page_id) + url_b
        try:
            proxies = { "http": "http://127.0.0.1:8080", "https": "https://127.0.0.1:8080"} 
            r = requests.get(url, verify=False)
        except:
            print("continue")
            continue
        imgs = json.loads(r.text)['results']
        for img in imgs:
            img_url = img['image']
            print(img_url)
            os.system("wget " + img_url)

spider_1()
截图

李成:

第一周课后作业
代码
import cv2
import boto3
import numpy as np
import nori2 as nori
import imgaug as ia
from imgaug import augmenters as iaa
from meghair.utils import io
from meghair.utils.imgproc import imdecode


img_list = []
img_height = []
img_width = []
enhance_num = 5

fetcher = nori.Fetcher()
s3_client = boto3.client('s3', endpoint_url="http://oss.i.brainpp.cn")


data = s3_client.get_object(Bucket="ai-cultivate", Key="1percent_ImageNet.txt")['Body'].read().decode("utf8")
count = 0
for i in data.split('\n'):
    count+=1
    if count > 10:
        break
    if not i:
        continue
    nori_id, id, img_name = i.split()
    img = imdecode(fetcher.get(nori_id))
    img_list.append(img)
    img_height.append(img.shape[0])
    img_width.append(img.shape[1])

print('height max:{} min:{} avg:{}'.format(max(img_height), min(img_height), np.average(img_height)))
print('width max:{} min:{} avg:{}'.format(max(img_width), min(img_width), np.average(img_width)))


aug = iaa.Sequential(
    [
        iaa.Fliplr(0.5),  # 对50%的图像进行镜像翻转
        iaa.Flipud(0.2),  # 对20%的图像做左右翻转
        iaa.Sometimes(0.5, iaa.Crop(percent=(0, 0.1))),
        iaa.Sometimes(0.5, iaa.Affine(
            scale={"x": (0.8, 1.2), "y": (0.8, 1.2)},
            translate_percent={"x": (-0.2, 0.2), "y": (-0.2, 0.2)},
            rotate=(-15, 15),
            shear=(-16, 16),
            order=[0, 1],
            cval=(0, 255),
            mode=ia.ALL
        )),
    ],
    random_order=True
)


img = img_list[2]
img = cv2.resize(img, (int(img.shape[0]/4),int(img.shape[1]/4)))
h = img.shape[0]
w = img.shape[1]
img_array = np.array([img] * enhance_num, dtype=np.uint8)
write_img = np.zeros(shape=(h, (w+10) * enhance_num, 3), dtype=np.uint8)
images_aug = aug.augment_images(images=img_array)
for j, item in enumerate(images_aug):
    print(item.shape)
    write_img[:, j * (w + 10): j * (w + 10) + w, :] = item

cv2.imwrite("data/result.jpg", write_img)