Win10上RKNN工具安装使用
作者：互联网
1，配置基础Python环境为：
Python3.6
个人建议最好在Conda下新建一个虚拟环境进行安装。
2，在虚拟环境下使用以下命令新建一个RKNN环境，如下：
conda create --name=rknn python=3.6.8
3，执行以下命令进入虚拟环境：
conda activate rknn
进入虚拟环境后，再执行以下命令安装深度学习框架，如Tensorflow，Pytorch，Keras等。
pip install tensorflow==1.14.0
pip install torch==1.6.0+cpu torchvision==0.7.0+cpu -f https://download.pytorch.org/whl/torch_stable.html --user
pip install mxnet==1.5.0
pip install opencv-python
pip install gluoncv
之后，手动安装 lmdb,该 wheel 包放在 SDK/external/rknn-toolkit/packages/required-packages-for-win-python36 目录下。
推荐将该安装包复制出来放入Anaconda的pkg目录下。之后进入此pkg目录下执行以下命令进行安装，如下所示：
pip install lmdb-0.95-cp36-cp36m-win_amd64.whl
同理，复制出rknn安装包rknn_toolkit-1.6.0-cp36-cp36m-win_amd64.whl，位置在SDK/external/rknn-toolkit/packages/下，执行以下命令进行安装。
pip install rknn_toolkit-1.6.0-cp36-cp36m-win_amd64.whl
4，检测rknn是否安装成功，输入以下命令：
python
from rknn.api import RKNN
若无报错则安装RKNN成功
5，案例使用，模型转换：
本人使用的模型出自以下：https://github.com/bubbliiiing/mobilenet-ssd-keras/releases/download/v1.0/mobilenet_ssd_weights.h5 权重进一步训练生成的模型。训练环境为tf1.13+keras2.1.5
注意，在保存权重时需要将网络与权重参数一起保存，在保存函数中将save_weights_only设置为False即可，如下所示。
后面首先需要对于Keras训练出的H5权重进行转换生成pb文件，代码如下：
from nets.ssd_training import MultiboxLoss, get_lr_scheduler
from keras.models import load_model
import tensorflow as tf
import os
from keras import backend as K
from tensorflow.python.framework import graph_util, graph_io
from tensorflow.python.platform import gfile
#h5topb
def h5_to_pb(h5_weight_path, output_dir, out_prefix="output_", log_tensorboard=True):
if not os.path.exists(output_dir):
os.mkdir(output_dir)
h5_model = load_model(h5_weight_path,custom_objects={'compute_loss':MultiboxLoss(num_classes=9, neg_pos_ratio=3.0).compute_loss})
out_nodes = []
for i in range(len(h5_model.outputs)):
out_nodes.append(out_prefix + str(i + 1))
tf.identity(h5_model.output[i], out_prefix + str(i + 1))
model_name = os.path.splitext(os.path.split(h5_weight_path)[-1])[0] + '.pb'
sess = K.get_session()
init_graph = sess.graph.as_graph_def()
main_graph = graph_util.convert_variables_to_constants(sess, init_graph, out_nodes)
graph_io.write_graph(main_graph, output_dir, name=model_name, as_text=False)
# 读取模型各层
def read_pb(GRAPH_PB_PATH):
with tf.Session() as sess:
print("load graph！！！")
with gfile.FastGFile(GRAPH_PB_PATH, 'rb') as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
tf.import_graph_def(graph_def, name='')
for i, n in enumerate(graph_def.node):
print("Name of the node - %s" % n.name)
h5_to_pb(h5_weight_path='./testrknn.h5', output_dir='./')
read_pb('./testrknn.pb')
在终端中将打印生成的pb文件模型的各个节点的名称，部分如下所示，非常重要，在模型转换时需要用到。
之后进入rknn的虚拟环境，输入以下指令进入可视化转换界面，如下所示：：
python -m rknn.bin.visualization
这里由于转换的是pb文件，因此选择中间TensorFlow。
进入后相关配置如下：
之后Next进行下一个界面：
这里InPut Nodes与output Nodes的填写非常重要，填写错误之后模型转换会报错如下：
AttributeError: 'NoneType' object has no attribute 'XXXXXX'
一般这种错误是输入与输出节点错误填写导致的。
对于输入输出节点的的确认，不能将该文件中的输入、输出节点，简单的理解为模型（.pb）文件节点分析后的第一个和最后一个节点，因为对于RK平台来说，模型的前处理（preprocessor）和后处理（postprocessor）部分不直接利用平台NPU计算，会影响速度，而是使用DSP来处理，对于模型中间部分的大量卷积操作使用NPU。
填写完毕后Next将量化PB文件。
一路Next后，最终将生成RKNN文件，文件模型部分结构如下所示：
对于生成的rknn模型进行推理，测试性能，步骤如下所示(在Window上需要实物开发板进行推理，因此此步在Linux下以PC模型进行)，测试代码如下：
import numpy as np
import re
import math
import random
import cv2
from rknn.api import RKNN
INPUT_SIZE = 300
if __name__ == '__main__':
# Create RKNN object
rknn = RKNN()
# Direct Load RKNN Model
ret = rknn.load_rknn('./test.rknn')
if ret != 0:
print('Load model failed')
exit(ret)
print('done')
# Set inputs
orig_img = cv2.imread('./1.jpg')
img = cv2.cvtColor(orig_img, cv2.COLOR_BGR2RGB)
img = cv2.resize(img, (INPUT_SIZE, INPUT_SIZE), interpolation=cv2.INTER_CUBIC)
# init runtime environment
print('--> Init runtime environment')
ret = rknn.init_runtime()
if ret != 0:
print('Init runtime environment failed')
exit(ret)
print('done')
# Inference
print('--> Running model')
outputs = rknn.inference(inputs=[img])
print('done')
# Evaluate Perf on Simulator
rknn.eval_perf(inputs=[img], is_print=True)
# Release RKNN Context
rknn.release()
推理结果如下：
同时也可以用代码的方式编译转换与测试文件，转换与测试代码如下：
import numpy as np
import re
import math
import random
import cv2
from rknn.api import RKNN
INPUT_SIZE = 300
NUM_RESULTS = 1917
NUM_CLASSES = 9
Y_SCALE = 10.0
X_SCALE = 10.0
H_SCALE = 5.0
W_SCALE = 5.0
def expit(x):
return 1. / (1. + math.exp(-x))
def unexpit(y):
return -1.0 * math.log((1.0 / y) - 1.0);
def CalculateOverlap(xmin0, ymin0, xmax0, ymax0, xmin1, ymin1, xmax1, ymax1):
w = max(0.0, min(xmax0, xmax1) - max(xmin0, xmin1))
h = max(0.0, min(ymax0, ymax1) - max(ymin0, ymin1))
i = w * h
u = (xmax0 - xmin0) * (ymax0 - ymin0) + (xmax1 - xmin1) * (ymax1 - ymin1) - i
if u <= 0.0:
return 0.0
return i / u
def load_box_priors():
box_priors_ = []
fp = open('./box_priors.txt', 'r')
ls = fp.readlines()
for s in ls:
aList = re.findall('([-+]?\d+(\.\d*)?|\.\d+)([eE][-+]?\d+)?', s)
for ss in aList:
aNum = float((ss[0]+ss[2]))
box_priors_.append(aNum)
fp.close()
box_priors = np.array(box_priors_)
box_priors = box_priors.reshape(4, NUM_RESULTS)
return box_priors
if __name__ == '__main__':
# Create RKNN object
rknn = RKNN()
# Config for Model Input PreProcess
#rknn.config(mean_values=[[127.5, 127.5, 127.5]], std_values=[[127.5, 127.5, 127.5]], reorder_channel='0 1 2')
# Load TensorFlow Model
#print('--> Loading model')
#rknn.load_tensorflow(tf_pb='./ghostnet_ssd.pb',
#inputs=['FeatureExtractor/MobilenetV1/MobilenetV1/Conv2d_0/Conv2D'],
#outputs=['concat', 'concat_1'],
#input_size_list=[[INPUT_SIZE, INPUT_SIZE, 3]])
#print('done')
# Build Model
#print('--> Building model')
#rknn.build(do_quantization=True, dataset='./dataset.txt')
#print('done')
# Export RKNN Model
#rknn.export_rknn('./ghostnet_ssd.rknn')
# Direct Load RKNN Model
rknn.load_rknn('./ghostnet_ssd.rknn')
# Set inputs
orig_img = cv2.imread('./test7.jpg')
img = cv2.cvtColor(orig_img, cv2.COLOR_BGR2RGB)
img = cv2.resize(img, (INPUT_SIZE, INPUT_SIZE), interpolation=cv2.INTER_CUBIC)
# init runtime environment
print('--> Init runtime environment')
ret = rknn.init_runtime()
if ret != 0:
print('Init runtime environment failed')
exit(ret)
print('done')
# Inference
print('--> Running model')
outputs = rknn.inference(inputs=[img])
print('done')
predictions = outputs[0].reshape((1, NUM_RESULTS, 4))
outputClasses = outputs[1].reshape((1, NUM_RESULTS, NUM_CLASSES))
candidateBox = np.zeros([2, NUM_RESULTS], dtype=int)
vaildCnt = 0
box_priors = load_box_priors()
# Post Process
# got valid candidate box
for i in range(0, NUM_RESULTS):
topClassScore = -1000
topClassScoreIndex = -1
# Skip the first catch-all class.
for j in range(1, NUM_CLASSES):
score = expit(outputClasses[0][i][j]);
if score > topClassScore:
topClassScoreIndex = j
topClassScore = score
if topClassScore > 0.4:
candidateBox[0][vaildCnt] = i
candidateBox[1][vaildCnt] = topClassScoreIndex
vaildCnt += 1
# calc position
for i in range(0, vaildCnt):
if candidateBox[0][i] == -1:
continue
n = candidateBox[0][i]
ycenter = predictions[0][n][0] / Y_SCALE * box_priors[2][n] + box_priors[0][n]
xcenter = predictions[0][n][1] / X_SCALE * box_priors[3][n] + box_priors[1][n]
h = math.exp(predictions[0][n][2] / H_SCALE) * box_priors[2][n]
w = math.exp(predictions[0][n][3] / W_SCALE) * box_priors[3][n]
ymin = ycenter - h / 2.
xmin = xcenter - w / 2.
ymax = ycenter + h / 2.
xmax = xcenter + w / 2.
predictions[0][n][0] = ymin
predictions[0][n][1] = xmin
predictions[0][n][2] = ymax
predictions[0][n][3] = xmax
# NMS
for i in range(0, vaildCnt):
if candidateBox[0][i] == -1:
continue
n = candidateBox[0][i]
xmin0 = predictions[0][n][1]
ymin0 = predictions[0][n][0]
xmax0 = predictions[0][n][3]
ymax0 = predictions[0][n][2]
for j in range(i+1, vaildCnt):
m = candidateBox[0][j]
if m == -1:
continue
xmin1 = predictions[0][m][1]
ymin1 = predictions[0][m][0]
xmax1 = predictions[0][m][3]
ymax1 = predictions[0][m][2]
iou = CalculateOverlap(xmin0, ymin0, xmax0, ymax0, xmin1, ymin1, xmax1, ymax1)
if iou >= 0.45:
candidateBox[0][j] = -1
# Draw result
for i in range(0, vaildCnt):
if candidateBox[0][i] == -1:
continue
n = candidateBox[0][i]
xmin = max(0.0, min(1.0, predictions[0][n][1])) * INPUT_SIZE
ymin = max(0.0, min(1.0, predictions[0][n][0])) * INPUT_SIZE
xmax = max(0.0, min(1.0, predictions[0][n][3])) * INPUT_SIZE
ymax = max(0.0, min(1.0, predictions[0][n][2])) * INPUT_SIZE
# print("%d @ (%d, %d) (%d, %d) score=%f" % (topClassScoreIndex, xmin, ymin, xmax, ymax, topClassScore))
cv2.rectangle(orig_img, (int(xmin), int(ymin)), (int(xmax), int(ymax)),
(random.random()*255, random.random()*255, random.random()*255), 3)
cv2.imwrite("out7.jpg", orig_img)
# Evaluate Perf on Simulator
rknn.eval_perf(inputs=[img], is_print=True)
# Release RKNN Context
rknn.release()
注意，代码中的box_priors与SSD模型中先验框有关，需要匹配自己的模型生成，生成代码如下：
import numpy as np
class AnchorBox():
def __init__(self, input_shape, min_size, max_size=None, aspect_ratios=None, flip=True):
self.input_shape = input_shape
self.min_size = min_size
self.max_size = max_size
self.aspect_ratios = []
for ar in aspect_ratios:
self.aspect_ratios.append(ar)
self.aspect_ratios.append(1.0 / ar)
def call(self, layer_shape, mask=None):
# --------------------------------- #
# 获取输入进来的特征层的宽和高
# 比如38x38
# --------------------------------- #
layer_height = layer_shape[0]
layer_width = layer_shape[1]
# --------------------------------- #
# 获取输入进来的图片的宽和高
# 比如300x300
# --------------------------------- #
img_height = self.input_shape[0]
img_width = self.input_shape[1]
box_widths = []
box_heights = []
# --------------------------------- #
# self.aspect_ratios一般有两个值
# [1, 1, 2, 1/2]
# [1, 1, 2, 1/2, 3, 1/3]
# --------------------------------- #
for ar in self.aspect_ratios:
# 首先添加一个较小的正方形
if ar == 1 and len(box_widths) == 0:
box_widths.append(self.min_size)
box_heights.append(self.min_size)
# 然后添加一个较大的正方形
elif ar == 1 and len(box_widths) > 0:
box_widths.append(np.sqrt(self.min_size * self.max_size))
box_heights.append(np.sqrt(self.min_size * self.max_size))
# 然后添加长方形
elif ar != 1:
box_widths.append(self.min_size * np.sqrt(ar))
box_heights.append(self.min_size / np.sqrt(ar))
# --------------------------------- #
# 获得所有先验框的宽高1/2
# --------------------------------- #
box_widths = 0.5 * np.array(box_widths)
box_heights = 0.5 * np.array(box_heights)
# --------------------------------- #
# 每一个特征层对应的步长
# --------------------------------- #
step_x = img_width / layer_width
step_y = img_height / layer_height
# --------------------------------- #
# 生成网格中心
# --------------------------------- #
linx = np.linspace(0.5 * step_x, img_width - 0.5 * step_x,
layer_width)
liny = np.linspace(0.5 * step_y, img_height - 0.5 * step_y,
layer_height)
centers_x, centers_y = np.meshgrid(linx, liny)
centers_x = centers_x.reshape(-1, 1)
centers_y = centers_y.reshape(-1, 1)
# 每一个先验框需要两个(centers_x, centers_y)，前一个用来计算左上角，后一个计算右下角
num_anchors_ = len(self.aspect_ratios)
anchor_boxes = np.concatenate((centers_x, centers_y), axis=1)
anchor_boxes = np.tile(anchor_boxes, (1, 2 * num_anchors_))
# 获得先验框的左上角和右下角
anchor_boxes[:, ::4] -= box_widths
anchor_boxes[:, 1::4] -= box_heights
anchor_boxes[:, 2::4] += box_widths
anchor_boxes[:, 3::4] += box_heights
# --------------------------------- #
# 将先验框变成小数的形式
# 归一化
# --------------------------------- #
anchor_boxes[:, ::2] /= img_width
anchor_boxes[:, 1::2] /= img_height
anchor_boxes = anchor_boxes.reshape(-1, 4)
anchor_boxes = np.minimum(np.maximum(anchor_boxes, 0.0), 1.0)
return anchor_boxes
#---------------------------------------------------#
# 用于计算共享特征层的大小
#---------------------------------------------------#
def get_img_output_length(height, width):
filter_sizes = [3, 3, 3, 3, 3, 3, 3, 3]
padding = [1, 1, 1, 1, 1, 1, 0, 0]
stride = [2, 2, 2, 2, 2, 2, 1, 1]
feature_heights = []
feature_widths = []
for i in range(len(filter_sizes)):
height = (height + 2*padding[i] - filter_sizes[i]) // stride[i] + 1
width = (width + 2*padding[i] - filter_sizes[i]) // stride[i] + 1
feature_heights.append(height)
feature_widths.append(width)
return np.array(feature_heights)[-6:], np.array(feature_widths)[-6:]
def get_anchors(input_shape = [300, 300], anchors_size = [30, 60, 111, 162, 213, 264, 315]):
feature_heights, feature_widths = get_img_output_length(input_shape[0], input_shape[1])
aspect_ratios = [[1, 2], [1, 2, 3], [1, 2, 3], [1, 2, 3], [1, 2], [1, 2]]
anchors = []
#print(feature_heights, feature_widths)
for i in range(len(feature_heights)):
anchors.append(AnchorBox(input_shape, anchors_size[i], max_size = anchors_size[i+1],
aspect_ratios = aspect_ratios[i]).call([feature_heights[i], feature_widths[i]]))
anchors = np.concatenate(anchors, axis=0)
print(np.shape(anchors))
return anchors
if __name__ == "__main__":
tmp = get_anchors()
temp = tmp.flatten()
temp_box_x = []
temp_box_y = []
temp_box_w = []
temp_box_h = []
temp_box = []
#依次获取先验框中心x坐标 先验框中心y坐标 先验框宽 先验框高四个值
for i in range(0,np.shape(get_anchors())[0]*np.shape(get_anchors())[1],4):
cx = (temp[i]+temp[i+2])/2
cy = (temp[i+1] + temp[i+3])/2
w = temp[i+2] - temp[i]
h = temp[i+3] - temp[i+1]
temp_box_y += [cy]
temp_box_x += [cx]
temp_box_h += [h]
temp_box_w += [w]
temp_box = np.concatenate([temp_box_y ,temp_box_x ,temp_box_h,temp_box_w], axis=0)
np.savetxt('./box_priors_test.txt',temp_box, fmt='%0.8f')
补充说明：h5模型转换过程中遇到的一个错误，解决方法如下：
或者用以下代码：
def relu6(x):
return K.relu(x, max_value=6)
并在加载H5文件时中，在其中的custom_objects中加入relu6，如以下形式。
load_model(h5_weight_path,custom_objects={'relu6':relu6})
参考博客：
https://blog.csdn.net/weixin_39289876/article/details/116064327
https://blog.csdn.net/weixin_39289876/article/details/116062202
https://blog.csdn.net/qq_43348528/article/details/122998830
标签：box,RKNN,temp,img,rknn,print,np,Win10,安装 来源： https://www.cnblogs.com/kxqblog/p/16370068.html