From 8900a2080acc06dcb8ad42b2e1de5cf72632c06b Mon Sep 17 00:00:00 2001 From: daquexian Date: Sun, 19 May 2019 11:15:38 +0800 Subject: [PATCH 01/18] onnx2bnn python package --- .daq_pm/configs/converter_python | 5 + .gitignore | 4 + .gitmodules | 6 + CMakeLists.txt | 16 ++ ci/onnx2bnn_build.yml | 54 +++--- cmake/glog.cmake | 2 + cmake/onnx.cmake | 20 ++- cmake/protobuf.cmake | 16 ++ cmake/utils.cmake | 23 +++ third_party/protobuf | 1 + third_party/pybind11 | 1 + tools/onnx2bnn/CMakeLists.txt | 52 +++--- tools/onnx2bnn/python/onnx2bnn/__init__.py | 1 + tools/onnx2bnn/python/onnx2bnn/__main__.py | 18 ++ tools/onnx2bnn/python/onnx2bnn/convert.py | 17 ++ tools/onnx2bnn/python/setup.py | 188 +++++++++++++++++++++ tools/onnx2bnn/pywrapper.cpp | 33 ++++ 17 files changed, 409 insertions(+), 48 deletions(-) create mode 100644 .daq_pm/configs/converter_python create mode 100644 cmake/protobuf.cmake create mode 100644 cmake/utils.cmake create mode 160000 third_party/protobuf create mode 160000 third_party/pybind11 create mode 100644 tools/onnx2bnn/python/onnx2bnn/__init__.py create mode 100644 tools/onnx2bnn/python/onnx2bnn/__main__.py create mode 100644 tools/onnx2bnn/python/onnx2bnn/convert.py create mode 100644 tools/onnx2bnn/python/setup.py create mode 100644 tools/onnx2bnn/pywrapper.cpp diff --git a/.daq_pm/configs/converter_python b/.daq_pm/configs/converter_python new file mode 100644 index 0000000..048724e --- /dev/null +++ b/.daq_pm/configs/converter_python @@ -0,0 +1,5 @@ +# It is a configuration file for [project_manager.vim](https://github.com/daquexian/project_manager.vim) +name binary-nn +type cpp +build_dir build_x86_python +cmake_options -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DBNN_BUILD_PYTHON=ON diff --git a/.gitignore b/.gitignore index 59a7f43..1e490c9 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,7 @@ compile_commands.json build-*/ build_*/ .daq_pm/status/ +.setuptools-cmake-build/ +.eggs/ +tools/onnx2bnn/python/dist/ +tools/onnx2bnn/python/onnx2bnn.egg-info/ diff --git a/.gitmodules b/.gitmodules index 1936650..95f9505 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,9 @@ [submodule "third_party/googletest"] path = third_party/googletest url = https://github.com/google/googletest +[submodule "third_party/protobuf"] + path = third_party/protobuf + url = https://github.com/protocolbuffers/protobuf +[submodule "third_party/pybind11"] + path = third_party/pybind11 + url = https://github.com/pybind/pybind11 diff --git a/CMakeLists.txt b/CMakeLists.txt index a4e932a..3e58858 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,6 +18,9 @@ option(BNN_BUILD_TEST "Build test" ON) option(BNN_BUILD_BENCHMARK "Build benchmark" ON) option(BNN_NET_BENCHMARK "define BNN_BENCHMARK" OFF) option(BNN_CHECK_CONSISTENCY "Make some layers non inplace to check results" OFF) +option(BNN_SYSTEM_PROTOBUF "Use system protobuf to build onnx2bnn" ON) +option(BNN_BUILD_PYTHON "Build onnx2bnn python interface" OFF) +option(BNN_USE_MSVC_STATIC_RUNTIME "Link onnx2bnn to msvc static runtime" ON) if (${CMAKE_SYSTEM_NAME} STREQUAL "Android") set(BNN_BUILD_ANDROID ON) @@ -62,7 +65,20 @@ if (${BNN_BUILD_ANDROID}) else() set(CMAKE_CXX_STANDARD 11) + include(cmake/protobuf.cmake) + if (${BNN_SYSTEM_PROTOBUF}) + find_package(Protobuf) + endif() + if (NOT Protobuf_FOUND) + set(BNN_SYSTEM_PROTOBUF OFF CACHE BOOL "Use system protobuf when building onnx2daq" FORCE) + endif() + include(cmake/onnx.cmake) configure_onnx() + + if (${BNN_BUILD_PYTHON}) + add_subdirectory(third_party/pybind11) + endif() + add_subdirectory(tools) endif() diff --git a/ci/onnx2bnn_build.yml b/ci/onnx2bnn_build.yml index 7149b14..31208d5 100644 --- a/ci/onnx2bnn_build.yml +++ b/ci/onnx2bnn_build.yml @@ -1,26 +1,28 @@ -pool: - vmImage: 'ubuntu-16.04' -steps: -- checkout: self - submodules: true -- script: sudo apt install -y cmake protobuf-compiler libprotobuf-dev - displayName: Install cmake 3.5.1 and protobuf -- bash: ci/build_onnx2bnn.sh - displayName: Build -- bash: ci/build_appimage.sh - displayName: Build AppImage -- task: CopyFiles@2 - inputs: - contents: 'onnx2bnn.AppImage' - targetFolder: $(Build.ArtifactStagingDirectory) -- task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: $(Build.ArtifactStagingDirectory) - artifactName: onnx2bnn AppImage -- task: GitHubRelease@0 - inputs: - gitHubConnection: 'dabnn release' - repositoryName: '$(Build.Repository.Name)' - action: 'create' - target: '$(Build.SourceVersion)' - assets: '$(Build.ArtifactStagingDirectory)/onnx2bnn.AppImage' +jobs: +- job: + pool: + vmImage: 'ubuntu-16.04' + steps: + - checkout: self + submodules: true + - script: sudo apt install -y cmake protobuf-compiler libprotobuf-dev + displayName: Install cmake 3.5.1 and protobuf + - bash: ci/build_onnx2bnn.sh + displayName: Build + - bash: ci/build_appimage.sh + displayName: Build AppImage + - task: CopyFiles@2 + inputs: + contents: 'onnx2bnn.AppImage' + targetFolder: $(Build.ArtifactStagingDirectory) + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: $(Build.ArtifactStagingDirectory) + artifactName: onnx2bnn AppImage + - task: GitHubRelease@0 + inputs: + gitHubConnection: 'dabnn release' + repositoryName: '$(Build.Repository.Name)' + action: 'create' + target: '$(Build.SourceVersion)' + assets: '$(Build.ArtifactStagingDirectory)/onnx2bnn.AppImage' diff --git a/cmake/glog.cmake b/cmake/glog.cmake index 30a35ec..e375dfc 100644 --- a/cmake/glog.cmake +++ b/cmake/glog.cmake @@ -1,8 +1,10 @@ # Copyright 2019 JD.com Inc. JD AI macro(configure_glog) + include(${PROJECT_SOURCE_DIR}/cmake/utils.cmake) message(STATUS "Configureing glog...") option(BUILD_TESTING "" OFF) option(WITH_GFLAGS "" OFF) add_subdirectory(${PROJECT_SOURCE_DIR}/third_party/glog) + add_msvc_runtime_flag(glog) endmacro() diff --git a/cmake/onnx.cmake b/cmake/onnx.cmake index 5f451b4..ee74660 100644 --- a/cmake/onnx.cmake +++ b/cmake/onnx.cmake @@ -1,9 +1,27 @@ # Copyright 2019 JD.com Inc. JD AI macro(configure_onnx) + if (NOT ${BNN_SYSTEM_PROTOBUF}) + include(${PROJECT_SOURCE_DIR}/cmake/protobuf.cmake) + configure_protobuf() + endif() + message(STATUS "Configuring onnx...") set(BNN_ONNX_NAMESPACE onnx_bnn) - + if (MSVC) + set(ONNX_CMAKELISTS ${PROJECT_SOURCE_DIR}/third_party/onnx/CMakeLists.txt) + file(READ ${ONNX_CMAKELISTS} content) + # Treating warnings as errors breaks ci, we have no other way to opt-out + string( + REPLACE + "/WX" + "" + content + "${content}" + ) + file(WRITE ${ONNX_CMAKELISTS} "${content}") + endif() + set(ONNX_USE_MSVC_STATIC_RUNTIME ${BNN_USE_MSVC_STATIC_RUNTIME}) set(ONNX_NAMESPACE ${BNN_ONNX_NAMESPACE} CACHE STRING "onnx namespace") add_subdirectory(${PROJECT_SOURCE_DIR}/third_party/onnx) # Since https://github.com/onnx/onnx/pull/1318 is merged, we don't need to set it manually diff --git a/cmake/protobuf.cmake b/cmake/protobuf.cmake new file mode 100644 index 0000000..5b9769b --- /dev/null +++ b/cmake/protobuf.cmake @@ -0,0 +1,16 @@ +macro(alias_protobuf) + add_executable(protobuf::libprotoc ALIAS libprotoc) + add_library(protobuf::libprotobuf ALIAS libprotobuf) + add_library(protobuf::libprotobuf-lite ALIAS libprotobuf-lite) +endmacro() + +macro(configure_protobuf) + message(STATUS "Configuring protobuf...") + option(protobuf_BUILD_TESTS "" OFF) + option(protobuf_BUILD_EXAMPLES "" OFF) + option(protobuf_BUILD_SHARED_LIBS "" OFF) + option(protobuf_BUILD_PROTOC_BINARIES "" ON) + add_subdirectory(${PROJECT_SOURCE_DIR}/third_party/protobuf/cmake) + alias_protobuf() +endmacro() + diff --git a/cmake/utils.cmake b/cmake/utils.cmake new file mode 100644 index 0000000..f97155f --- /dev/null +++ b/cmake/utils.cmake @@ -0,0 +1,23 @@ +# Copyright 2019 JD.com Inc. JD AI + +# Add MSVC RunTime Flag +function(add_msvc_runtime_flag lib) + if (MSVC) + if(${DNN_USE_MSVC_STATIC_RUNTIME}) + if(${CMAKE_BUILD_TYPE} MATCHES "Debug") + target_compile_options(${lib} PRIVATE /MTd) + else() + target_compile_options(${lib} PRIVATE /MT) + endif() + else() + if(${CMAKE_BUILD_TYPE} MATCHES "Debug") + target_compile_options(${lib} PRIVATE /MDd) + else() + target_compile_options(${lib} PRIVATE /MD) + endif() + endif() + endif() +endfunction() + + + diff --git a/third_party/protobuf b/third_party/protobuf new file mode 160000 index 0000000..6973c3a --- /dev/null +++ b/third_party/protobuf @@ -0,0 +1 @@ +Subproject commit 6973c3a5041636c1d8dc5f7f6c8c1f3c15bc63d6 diff --git a/third_party/pybind11 b/third_party/pybind11 new file mode 160000 index 0000000..97784da --- /dev/null +++ b/third_party/pybind11 @@ -0,0 +1 @@ +Subproject commit 97784dad3e518ccb415d5db57ff9b933495d9024 diff --git a/tools/onnx2bnn/CMakeLists.txt b/tools/onnx2bnn/CMakeLists.txt index 837381a..8a5a157 100644 --- a/tools/onnx2bnn/CMakeLists.txt +++ b/tools/onnx2bnn/CMakeLists.txt @@ -1,13 +1,8 @@ # Copyright 2019 JD.com Inc. JD AI -find_package(Protobuf REQUIRED) -if (${CMAKE_VERSION} VERSION_LESS "3.6.0") - set(Protobuf_LIBRARIES ${PROTOBUF_LIBRARIES}) - set(Protobuf_INCLUDE_DIRS ${PROTOBUF_INCLUDE_DIRS}) -endif() +include(${PROJECT_SOURCE_DIR}/cmake/utils.cmake) -add_executable(onnx2bnn - ${CMAKE_CURRENT_SOURCE_DIR}/onnx2bnn.cpp +add_library(onnx2bnn ${CMAKE_CURRENT_SOURCE_DIR}/OnnxConverter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/OnnxConverter.h ${CMAKE_CURRENT_SOURCE_DIR}/NodeAttrHelper.cpp @@ -16,27 +11,42 @@ add_executable(onnx2bnn ${PROJECT_SOURCE_DIR}/common/Shaper.h ${PROJECT_SOURCE_DIR}/common/Shaper.cpp ) - target_link_libraries(onnx2bnn glog::glog onnx + protobuf::libprotobuf ) - target_include_directories(onnx2bnn - PRIVATE + PUBLIC ${PROJECT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} ) - -if(${CMAKE_VERSION} VERSION_GREATER "3.10.0") - target_link_libraries(onnx2bnn - protobuf::libprotobuf) -else() - target_link_libraries(onnx2bnn - ${Protobuf_LIBRARIES}) - target_include_directories(onnx2bnn - PRIVATE - ${Protobuf_INCLUDE_DIRS}) +if (${BNN_SYSTEM_PROTOBUF}) + treat_warnings_as_errors(onnx2bnn) endif() +set_property(TARGET onnx2bnn PROPERTY POSITION_INDEPENDENT_CODE ON) +add_msvc_runtime_flag(onnx2bnn) -treat_warnings_as_errors(onnx2bnn) +add_executable(onnx2bnn-bin + ${CMAKE_CURRENT_SOURCE_DIR}/onnx2bnn.cpp + ) +target_link_libraries(onnx2bnn-bin + onnx2bnn) +treat_warnings_as_errors(onnx2bnn-bin) +set_target_properties(onnx2bnn-bin PROPERTIES OUTPUT_NAME "onnx2bnn") + +if (${BNN_BUILD_PYTHON}) + pybind11_add_module(_onnx2bnn + ${CMAKE_CURRENT_SOURCE_DIR}/pywrapper.cpp) + + target_link_libraries(_onnx2bnn + PUBLIC + protobuf::libprotobuf + glog::glog + onnx + onnx2bnn + pybind11::pybind11 + ) + + add_msvc_runtime_flag(_onnx2bnn) +endif() diff --git a/tools/onnx2bnn/python/onnx2bnn/__init__.py b/tools/onnx2bnn/python/onnx2bnn/__init__.py new file mode 100644 index 0000000..52a7760 --- /dev/null +++ b/tools/onnx2bnn/python/onnx2bnn/__init__.py @@ -0,0 +1 @@ +from .convert import convert diff --git a/tools/onnx2bnn/python/onnx2bnn/__main__.py b/tools/onnx2bnn/python/onnx2bnn/__main__.py new file mode 100644 index 0000000..4f0c031 --- /dev/null +++ b/tools/onnx2bnn/python/onnx2bnn/__main__.py @@ -0,0 +1,18 @@ +import argparse + +import onnx +import onnx2bnn + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('input_model', help='Input ONNX model') + parser.add_argument('output_model', help='Output dabnn model') + parser.add_argument('level', nargs='?', + help='Level of onnx2bnn (possible value: moderate(default), strict and aggressive)', default='moderate') + args = parser.parse_args() + onnx2bnn.convert(args.input_model, args.output_model, args.level) + + +if __name__ == '__main__': + main() diff --git a/tools/onnx2bnn/python/onnx2bnn/convert.py b/tools/onnx2bnn/python/onnx2bnn/convert.py new file mode 100644 index 0000000..a208cac --- /dev/null +++ b/tools/onnx2bnn/python/onnx2bnn/convert.py @@ -0,0 +1,17 @@ +import _onnx2bnn + +import onnx + + +def convert(model, output, level_str="moderate"): + if type(model) == str: + model = onnx.load(model) + model = model.SerializeToString() + elif type(model) == onnx.ModelProto: + model = model.SerializeToString() + elif type(model) == bytes: + pass + else: + raise RuntimeError( + "Input of function convert can only be str, onnx.ModelProto or bytes") + _onnx2bnn.convert(model, output, level_str) diff --git a/tools/onnx2bnn/python/setup.py b/tools/onnx2bnn/python/setup.py new file mode 100644 index 0000000..a0cb514 --- /dev/null +++ b/tools/onnx2bnn/python/setup.py @@ -0,0 +1,188 @@ +""" +Most part of this file is borrowed from ONNX +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from distutils.spawn import find_executable +from distutils import sysconfig, log +import setuptools +import setuptools.command.build_py +import setuptools.command.develop +import setuptools.command.build_ext + +from collections import namedtuple +from contextlib import contextmanager +import glob +import os +import shlex +import subprocess +import sys +import struct +from textwrap import dedent +import multiprocessing +from os.path import dirname + + +TOP_DIR = os.path.realpath(dirname(dirname(dirname(dirname(os.path.abspath(__file__)))))) +CMAKE_BUILD_DIR = os.path.join(TOP_DIR, '.setuptools-cmake-build') + +WINDOWS = (os.name == 'nt') + +CMAKE = find_executable('cmake3') or find_executable('cmake') +MAKE = find_executable('make') + +install_requires = ['onnx'] +setup_requires = ['cmake'] +tests_require = [] +extras_require = {} + +################################################################################ +# Global variables for controlling the build variant +################################################################################ + +DEBUG = bool(os.getenv('DEBUG')) +COVERAGE = bool(os.getenv('COVERAGE')) + +################################################################################ +# Version +################################################################################ + +try: + git_version = subprocess.check_output(['git', 'rev-parse', 'HEAD'], + cwd=TOP_DIR).decode('ascii').strip() +except (OSError, subprocess.CalledProcessError): + git_version = None + +################################################################################ +# Pre Check +################################################################################ + +assert CMAKE, 'Could not find "cmake" executable!' + +################################################################################ +# Utilities +################################################################################ + + +@contextmanager +def cd(path): + if not os.path.isabs(path): + raise RuntimeError('Can only cd to absolute path, got: {}'.format(path)) + orig_path = os.getcwd() + os.chdir(path) + try: + yield + finally: + os.chdir(orig_path) + + +class CMakeExtension(setuptools.Extension): + def __init__(self, name): + # don't invoke the original build_ext for this special extension + super().__init__(name, sources=[]) + + +class build_ext(setuptools.command.build_ext.build_ext): + def run(self): + for ext in self.extensions: + self.build_extension(ext) + + + def build_extension(self, ext): + extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) + + if not os.path.exists(CMAKE_BUILD_DIR): + os.makedirs(CMAKE_BUILD_DIR) + + with cd(CMAKE_BUILD_DIR): + build_type = 'Release' + # configure + cmake_args = [ + CMAKE, + '-DPYTHON_INCLUDE_DIR={}'.format(sysconfig.get_python_inc()), + '-DPYTHON_EXECUTABLE={}'.format(sys.executable), + '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON', + '-DPY_EXT_SUFFIX={}'.format(sysconfig.get_config_var('EXT_SUFFIX') or ''), + '-DBNN_BUILD_PYTHON=ON', + '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={}'.format(extdir), + ] + if COVERAGE or DEBUG: + # in order to get accurate coverage information, the + # build needs to turn off optimizations + build_type = 'Debug' + cmake_args.append('-DCMAKE_BUILD_TYPE={}'.format(build_type)) + if WINDOWS: + cmake_args.extend([ + # we need to link with libpython on windows, so + # passing python version to window in order to + # find python in cmake + '-DPY_VERSION={}'.format('{0}.{1}'.format(*sys.version_info[:2])), + '-DBNN_USE_MSVC_STATIC_RUNTIME=ON', + ]) + if 8 * struct.calcsize("P") == 64: + # Temp fix for CI + # TODO: need a better way to determine generator + cmake_args.append('-DCMAKE_GENERATOR_PLATFORM=x64') + if 'CMAKE_ARGS' in os.environ: + extra_cmake_args = shlex.split(os.environ['CMAKE_ARGS']) + # prevent crossfire with downstream scripts + del os.environ['CMAKE_ARGS'] + log.info('Extra cmake args: {}'.format(extra_cmake_args)) + cmake_args.extend(extra_cmake_args) + cmake_args.append(TOP_DIR) + subprocess.check_call(cmake_args) + + build_args = [CMAKE, '--build', os.curdir] + if WINDOWS: + build_args.extend(['--config', build_type]) + build_args.extend(['--', '/maxcpucount:{}'.format(multiprocessing.cpu_count())]) + else: + build_args.extend(['--', '-j', str(multiprocessing.cpu_count())]) + subprocess.check_call(build_args) + + +cmdclass = { + 'build_ext': build_ext, +} + +################################################################################ +# Extensions +################################################################################ + +ext_modules = [ + CMakeExtension("_onnx2bnn") +] + +################################################################################ +# Packages +################################################################################ + +# no need to do fancy stuff so far +packages = setuptools.find_packages() + +################################################################################ +# Final +################################################################################ + +setuptools.setup( + name="onnx2bnn", + # version=VersionInfo.version, + description="Convert ONNX to dabnn", + ext_modules=ext_modules, + cmdclass=cmdclass, + packages=packages, + include_package_data=True, + install_requires=install_requires, + setup_requires=setup_requires, + tests_require=tests_require, + extras_require=extras_require, + author='daquexian', + author_email='daquexian566@gmail.com', + url='https://github.com/JDAI-CV/dabnn', +) + + diff --git a/tools/onnx2bnn/pywrapper.cpp b/tools/onnx2bnn/pywrapper.cpp new file mode 100644 index 0000000..4521a6c --- /dev/null +++ b/tools/onnx2bnn/pywrapper.cpp @@ -0,0 +1,33 @@ +#include + +#include "OnnxConverter.h" + +namespace py = pybind11; + +void convert(const std::string &model_str, const std::string &filepath, + const std::string &level_str) { + using namespace bnn; + ONNX_NAMESPACE::ModelProto model_proto; + bool ret = model_proto.ParseFromString(model_str); + if (!ret) { + throw std::invalid_argument("Read protobuf string failed"); + } + + OnnxConverter::Level level = OnnxConverter::Level::kModerate; + + if (level_str == "moderate") { + level = OnnxConverter::Level::kModerate; + } else if (level_str == "strict") { + level = OnnxConverter::Level::kStrict; + } else if (level_str == "aggressive") { + level = OnnxConverter::Level::kAggressive; + } else { + throw std::invalid_argument( + "Level can only be moderate, strict or aggressive"); + } + OnnxConverter converter; + converter.Convert(model_proto, filepath, level); + google::protobuf::ShutdownProtobufLibrary(); +} + +PYBIND11_MODULE(_onnx2bnn, m) { m.def("convert", &convert, ""); } From 69e7d2ab07219d9d12de8db3f7ce287ff20c406d Mon Sep 17 00:00:00 2001 From: daquexian Date: Sun, 19 May 2019 12:09:22 +0800 Subject: [PATCH 02/18] Fix misc bugs --- CMakeLists.txt | 9 +++++---- ci/onnx2bnn_build.yml | 9 +++++++++ cmake/protobuf.cmake | 2 +- tools/onnx2bnn/CMakeLists.txt | 9 ++++----- tools/onnx2bnn/python/setup.py | 1 + 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3e58858..a21e8a5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -65,6 +65,11 @@ if (${BNN_BUILD_ANDROID}) else() set(CMAKE_CXX_STANDARD 11) + if (${BNN_BUILD_PYTHON}) + set(CMAKE_POSITION_INDEPENDENT_CODE ON) + add_subdirectory(third_party/pybind11) + endif() + include(cmake/protobuf.cmake) if (${BNN_SYSTEM_PROTOBUF}) find_package(Protobuf) @@ -76,9 +81,5 @@ else() include(cmake/onnx.cmake) configure_onnx() - if (${BNN_BUILD_PYTHON}) - add_subdirectory(third_party/pybind11) - endif() - add_subdirectory(tools) endif() diff --git a/ci/onnx2bnn_build.yml b/ci/onnx2bnn_build.yml index 31208d5..ada38c6 100644 --- a/ci/onnx2bnn_build.yml +++ b/ci/onnx2bnn_build.yml @@ -26,3 +26,12 @@ jobs: action: 'create' target: '$(Build.SourceVersion)' assets: '$(Build.ArtifactStagingDirectory)/onnx2bnn.AppImage' +- job: + pool: + vmImage: 'vs2017-win2016' + steps: + - checkout: self + submodules: true + - script: tools/onnx2bnn/python/setup.py + workingDirectory: tools/onnx2bnn/python/ + displayName: Build onnx2bnn python package diff --git a/cmake/protobuf.cmake b/cmake/protobuf.cmake index 5b9769b..71b76e6 100644 --- a/cmake/protobuf.cmake +++ b/cmake/protobuf.cmake @@ -1,5 +1,5 @@ macro(alias_protobuf) - add_executable(protobuf::libprotoc ALIAS libprotoc) + add_library(protobuf::libprotoc ALIAS libprotoc) add_library(protobuf::libprotobuf ALIAS libprotobuf) add_library(protobuf::libprotobuf-lite ALIAS libprotobuf-lite) endmacro() diff --git a/tools/onnx2bnn/CMakeLists.txt b/tools/onnx2bnn/CMakeLists.txt index 8a5a157..97688a4 100644 --- a/tools/onnx2bnn/CMakeLists.txt +++ b/tools/onnx2bnn/CMakeLists.txt @@ -21,10 +21,6 @@ target_include_directories(onnx2bnn ${PROJECT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} ) -if (${BNN_SYSTEM_PROTOBUF}) - treat_warnings_as_errors(onnx2bnn) -endif() -set_property(TARGET onnx2bnn PROPERTY POSITION_INDEPENDENT_CODE ON) add_msvc_runtime_flag(onnx2bnn) add_executable(onnx2bnn-bin @@ -32,8 +28,11 @@ add_executable(onnx2bnn-bin ) target_link_libraries(onnx2bnn-bin onnx2bnn) -treat_warnings_as_errors(onnx2bnn-bin) set_target_properties(onnx2bnn-bin PROPERTIES OUTPUT_NAME "onnx2bnn") +if (${BNN_SYSTEM_PROTOBUF}) + treat_warnings_as_errors(onnx2bnn) + treat_warnings_as_errors(onnx2bnn-bin) +endif() if (${BNN_BUILD_PYTHON}) pybind11_add_module(_onnx2bnn diff --git a/tools/onnx2bnn/python/setup.py b/tools/onnx2bnn/python/setup.py index a0cb514..52e5bcb 100644 --- a/tools/onnx2bnn/python/setup.py +++ b/tools/onnx2bnn/python/setup.py @@ -108,6 +108,7 @@ def build_extension(self, ext): '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON', '-DPY_EXT_SUFFIX={}'.format(sysconfig.get_config_var('EXT_SUFFIX') or ''), '-DBNN_BUILD_PYTHON=ON', + '-DBNN_SYSTEM_PROTOBUF=OFF', '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={}'.format(extdir), ] if COVERAGE or DEBUG: From 2f7204232230aca6dbebb4b3c538c8ea5cb87aab Mon Sep 17 00:00:00 2001 From: daquexian Date: Sun, 19 May 2019 12:12:14 +0800 Subject: [PATCH 03/18] Update ci --- ci/onnx2bnn_build.yml | 72 +++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/ci/onnx2bnn_build.yml b/ci/onnx2bnn_build.yml index ada38c6..9a9ea50 100644 --- a/ci/onnx2bnn_build.yml +++ b/ci/onnx2bnn_build.yml @@ -1,37 +1,37 @@ jobs: -- job: - pool: - vmImage: 'ubuntu-16.04' - steps: - - checkout: self - submodules: true - - script: sudo apt install -y cmake protobuf-compiler libprotobuf-dev - displayName: Install cmake 3.5.1 and protobuf - - bash: ci/build_onnx2bnn.sh - displayName: Build - - bash: ci/build_appimage.sh - displayName: Build AppImage - - task: CopyFiles@2 - inputs: - contents: 'onnx2bnn.AppImage' - targetFolder: $(Build.ArtifactStagingDirectory) - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: $(Build.ArtifactStagingDirectory) - artifactName: onnx2bnn AppImage - - task: GitHubRelease@0 - inputs: - gitHubConnection: 'dabnn release' - repositoryName: '$(Build.Repository.Name)' - action: 'create' - target: '$(Build.SourceVersion)' - assets: '$(Build.ArtifactStagingDirectory)/onnx2bnn.AppImage' -- job: - pool: - vmImage: 'vs2017-win2016' - steps: - - checkout: self - submodules: true - - script: tools/onnx2bnn/python/setup.py - workingDirectory: tools/onnx2bnn/python/ - displayName: Build onnx2bnn python package +- job: Linux_AppImage + pool: + vmImage: 'ubuntu-16.04' + steps: + - checkout: self + submodules: true + - script: sudo apt install -y cmake protobuf-compiler libprotobuf-dev + displayName: Install cmake 3.5.1 and protobuf + - bash: ci/build_onnx2bnn.sh + displayName: Build + - bash: ci/build_appimage.sh + displayName: Build AppImage + - task: CopyFiles@2 + inputs: + contents: 'onnx2bnn.AppImage' + targetFolder: $(Build.ArtifactStagingDirectory) + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: $(Build.ArtifactStagingDirectory) + artifactName: onnx2bnn AppImage + - task: GitHubRelease@0 + inputs: + gitHubConnection: 'dabnn release' + repositoryName: '$(Build.Repository.Name)' + action: 'create' + target: '$(Build.SourceVersion)' + assets: '$(Build.ArtifactStagingDirectory)/onnx2bnn.AppImage' +- job: Windows_Python_Package + pool: + vmImage: 'vs2017-win2016' + steps: + - checkout: self + submodules: true + - script: python3 tools/onnx2bnn/python/setup.py + workingDirectory: tools/onnx2bnn/python/ + displayName: Build onnx2bnn python package From ecc5be153f650058772b33805ca40660b18ca3b3 Mon Sep 17 00:00:00 2001 From: daquexian Date: Sun, 19 May 2019 12:29:44 +0800 Subject: [PATCH 04/18] Set min cmake version to 3.9 --- CMakeLists.txt | 3 ++- ci/build_onnx2bnn.sh | 5 ++--- ci/onnx2bnn_build.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a21e8a5..368efb0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ # Copyright 2019 JD.com Inc. JD AI -cmake_minimum_required(VERSION 3.5) +cmake_minimum_required(VERSION 3.9) project(dabnn CXX) message(STATUS "CMake Version: ${CMAKE_VERSION}") @@ -75,6 +75,7 @@ else() find_package(Protobuf) endif() if (NOT Protobuf_FOUND) + message(STATUS "BNN_SYSTEM_PROTOBUF is ON but protobuf is not found, using the packed protobuf..") set(BNN_SYSTEM_PROTOBUF OFF CACHE BOOL "Use system protobuf when building onnx2daq" FORCE) endif() diff --git a/ci/build_onnx2bnn.sh b/ci/build_onnx2bnn.sh index 7d57d04..c2c4dd3 100755 --- a/ci/build_onnx2bnn.sh +++ b/ci/build_onnx2bnn.sh @@ -2,7 +2,6 @@ set -e mkdir build_onnx2bnn && cd build_onnx2bnn -# azure pipeline image aliases cmake to their cmake 3.12 -/usr/bin/cmake .. -/usr/bin/cmake --build . -- -j$(nproc) +cmake .. +cmake --build . -- -j$(nproc) cd - diff --git a/ci/onnx2bnn_build.yml b/ci/onnx2bnn_build.yml index 9a9ea50..46f5f75 100644 --- a/ci/onnx2bnn_build.yml +++ b/ci/onnx2bnn_build.yml @@ -5,8 +5,8 @@ jobs: steps: - checkout: self submodules: true - - script: sudo apt install -y cmake protobuf-compiler libprotobuf-dev - displayName: Install cmake 3.5.1 and protobuf + - script: sudo apt install -y protobuf-compiler libprotobuf-dev + displayName: Install protobuf - bash: ci/build_onnx2bnn.sh displayName: Build - bash: ci/build_appimage.sh From 40f6d4cac4064d5df17ebe78df0cad0d24ee2a03 Mon Sep 17 00:00:00 2001 From: daquexian Date: Sun, 19 May 2019 12:34:22 +0800 Subject: [PATCH 05/18] Use python in ci --- ci/onnx2bnn_build.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ci/onnx2bnn_build.yml b/ci/onnx2bnn_build.yml index 46f5f75..d859727 100644 --- a/ci/onnx2bnn_build.yml +++ b/ci/onnx2bnn_build.yml @@ -32,6 +32,11 @@ jobs: steps: - checkout: self submodules: true - - script: python3 tools/onnx2bnn/python/setup.py + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.5' + addToPath: true + architecture: 'x64' + - script: python tools/onnx2bnn/python/setup.py workingDirectory: tools/onnx2bnn/python/ displayName: Build onnx2bnn python package From 1876680aabace2a22d7a0000644e786413fb3c6e Mon Sep 17 00:00:00 2001 From: daquexian Date: Sun, 19 May 2019 12:37:59 +0800 Subject: [PATCH 06/18] Update ci --- ci/onnx2bnn_build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/onnx2bnn_build.yml b/ci/onnx2bnn_build.yml index d859727..ce38d6d 100644 --- a/ci/onnx2bnn_build.yml +++ b/ci/onnx2bnn_build.yml @@ -37,6 +37,8 @@ jobs: versionSpec: '3.5' addToPath: true architecture: 'x64' - - script: python tools/onnx2bnn/python/setup.py + - script: python -m pip install --user --upgrade setuptools wheel twine + displayName: Install setuptools, wheel and twine + - script: python setup.py sdist bdist_wheel workingDirectory: tools/onnx2bnn/python/ displayName: Build onnx2bnn python package From 015b24b96d1f91cf39b769aad8da6cdd9b4899df Mon Sep 17 00:00:00 2001 From: daquexian Date: Sun, 19 May 2019 13:00:27 +0800 Subject: [PATCH 07/18] Move pack_mat_64 to from common_bitpack.h to bitpack.h, or it causes onnx2bnn to include mat.h --- common/baseline.h | 2 +- common/common_bitpack.h | 20 -------------------- dabnn/bitpack.h | 19 +++++++++++++++++++ 3 files changed, 20 insertions(+), 21 deletions(-) diff --git a/common/baseline.h b/common/baseline.h index df49f32..397d3e1 100644 --- a/common/baseline.h +++ b/common/baseline.h @@ -6,7 +6,7 @@ #include #include -#include +#include #include inline int bitcount(uint64_t x) { diff --git a/common/common_bitpack.h b/common/common_bitpack.h index 875ed85..f523f47 100644 --- a/common/common_bitpack.h +++ b/common/common_bitpack.h @@ -8,7 +8,6 @@ #include #include -#include inline void pack_128_fallback(const float *float_ptr, void *binary_ptr, size_t size) { @@ -184,23 +183,4 @@ inline void pack_64_bitfield(const float *fptr, uint64_t *buf) { *buf = u.u64; } -inline void pack_mat_64(const bnn::Mat &float_mat, bnn::Mat &binary_mat) { - BNN_ASSERT( - float_mat.w * float_mat.c > 0 && float_mat.w * float_mat.c % 64 == 0, - float_mat.w * float_mat.c); - BNN_ASSERT(float_mat.c / 64 == binary_mat.c && float_mat.c % 64 == 0, ""); - - FORZ(n, float_mat.n) { - FORZ(h, float_mat.h) { - auto *fptr = float_mat.point(n, h, 0); - auto *bptr = binary_mat.point(n, h, 0); - FORZ(i, float_mat.w * float_mat.c / 64) { - pack_64_bitfield(fptr, bptr); - fptr += 64; - bptr++; - } - } - } -} - #endif /* COMMON_BITPACK_H */ diff --git a/dabnn/bitpack.h b/dabnn/bitpack.h index 3202536..8be37da 100644 --- a/dabnn/bitpack.h +++ b/dabnn/bitpack.h @@ -199,6 +199,25 @@ inline void pack_mat_128(const bnn::Mat &float_mat, bnn::Mat &binary_mat) { float_mat.total()); } +inline void pack_mat_64(const bnn::Mat &float_mat, bnn::Mat &binary_mat) { + BNN_ASSERT( + float_mat.w * float_mat.c > 0 && float_mat.w * float_mat.c % 64 == 0, + float_mat.w * float_mat.c); + BNN_ASSERT(float_mat.c / 64 == binary_mat.c && float_mat.c % 64 == 0, ""); + + FORZ(n, float_mat.n) { + FORZ(h, float_mat.h) { + auto *fptr = float_mat.point(n, h, 0); + auto *bptr = binary_mat.point(n, h, 0); + FORZ(i, float_mat.w * float_mat.c / 64) { + pack_64_bitfield(fptr, bptr); + fptr += 64; + bptr++; + } + } + } +} + inline void pack_mat(const bnn::Mat &float_mat, bnn::Mat &binary_mat) { BNN_ASSERT(float_mat.c % 64 == 0, float_mat.c); if (float_mat.c % 128 == 0) { From c528d065751377c2878a41874cf388a0c8d66fdd Mon Sep 17 00:00:00 2001 From: daquexian Date: Sun, 19 May 2019 13:08:49 +0800 Subject: [PATCH 08/18] long is 32-bit on msvc, use llong instead --- common/common_bitpack.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/common/common_bitpack.h b/common/common_bitpack.h index f523f47..dca1e42 100644 --- a/common/common_bitpack.h +++ b/common/common_bitpack.h @@ -15,8 +15,8 @@ inline void pack_128_fallback(const float *float_ptr, void *binary_ptr, const size_t UNIT_LEN = 64; std::bitset bits1; std::bitset bits2; - static_assert(std::is_same::value, - "bits.to_ulong() must return uint64_t"); + static_assert(std::is_same::value, + "bits.to_ullong() must return uint64_t"); FORZS(j, size, 128) { FORZS(i, 128, 4) { @@ -26,8 +26,8 @@ inline void pack_128_fallback(const float *float_ptr, void *binary_ptr, bits2[t] = (*(float_ptr + j + i + 2) > 0); bits2[t + 32] = (*(float_ptr + j + i + 3) > 0); } - *ui64_ptr++ = bits1.to_ulong(); - *ui64_ptr++ = bits2.to_ulong(); + *ui64_ptr++ = bits1.to_ullong(); + *ui64_ptr++ = bits2.to_ullong(); } } @@ -37,9 +37,9 @@ inline void pack_64_bitset(const float *fptr, uint64_t *buf) { for (size_t i = 0; i < UNIT_LEN; i++) { bits[i] = (*(fptr + i) > 0); } - static_assert(std::is_same::value, - "bits.to_ulong() must return uint64_t"); - *buf = bits.to_ulong(); + static_assert(std::is_same::value, + "bits.to_ullong() must return uint64_t"); + *buf = bits.to_ullong(); } inline void pack_64_bitfield(const float *fptr, uint64_t *buf) { From f7d9677ee3d229c717e679a24e615d4c4c89f749 Mon Sep 17 00:00:00 2001 From: daquexian Date: Sun, 19 May 2019 14:01:51 +0800 Subject: [PATCH 09/18] Check the bit size of ullong in a more stable way --- common/common_bitpack.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/common/common_bitpack.h b/common/common_bitpack.h index dca1e42..458e195 100644 --- a/common/common_bitpack.h +++ b/common/common_bitpack.h @@ -15,8 +15,8 @@ inline void pack_128_fallback(const float *float_ptr, void *binary_ptr, const size_t UNIT_LEN = 64; std::bitset bits1; std::bitset bits2; - static_assert(std::is_same::value, - "bits.to_ullong() must return uint64_t"); + static_assert(sizeof(decltype(bits1.to_ullong())) * CHAR_BIT == 64, + "bits.to_ullong() must return a 64-bit element"); FORZS(j, size, 128) { FORZS(i, 128, 4) { @@ -37,8 +37,8 @@ inline void pack_64_bitset(const float *fptr, uint64_t *buf) { for (size_t i = 0; i < UNIT_LEN; i++) { bits[i] = (*(fptr + i) > 0); } - static_assert(std::is_same::value, - "bits.to_ullong() must return uint64_t"); + static_assert(sizeof(decltype(bits.to_ullong())) * CHAR_BIT == 64, + "bits.to_ullong() must return a 64-bit element"); *buf = bits.to_ullong(); } From 71631bb9f670ddae05b167b59d9a2ed639487584 Mon Sep 17 00:00:00 2001 From: daquexian Date: Sun, 19 May 2019 14:05:27 +0800 Subject: [PATCH 10/18] DNN -> BNN in utils.cmake --- cmake/utils.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/utils.cmake b/cmake/utils.cmake index f97155f..6329819 100644 --- a/cmake/utils.cmake +++ b/cmake/utils.cmake @@ -3,7 +3,7 @@ # Add MSVC RunTime Flag function(add_msvc_runtime_flag lib) if (MSVC) - if(${DNN_USE_MSVC_STATIC_RUNTIME}) + if(${BNN_USE_MSVC_STATIC_RUNTIME}) if(${CMAKE_BUILD_TYPE} MATCHES "Debug") target_compile_options(${lib} PRIVATE /MTd) else() From 61365214f572dbb5c5a779a02ba490d0f4fa2506 Mon Sep 17 00:00:00 2001 From: daquexian Date: Sun, 19 May 2019 14:21:38 +0800 Subject: [PATCH 11/18] Set global msvc runtime flag --- CMakeLists.txt | 3 +++ cmake/glog.cmake | 2 -- cmake/utils.cmake | 10 +++++----- tools/onnx2bnn/CMakeLists.txt | 3 --- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 368efb0..9525b24 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,6 +28,9 @@ else() set(BNN_BUILD_ANDROID OFF) endif() +include(cmake/utils.cmake) +bnn_add_msvc_runtime_flag() + include(cmake/common.cmake) include(cmake/system.cmake) diff --git a/cmake/glog.cmake b/cmake/glog.cmake index e375dfc..30a35ec 100644 --- a/cmake/glog.cmake +++ b/cmake/glog.cmake @@ -1,10 +1,8 @@ # Copyright 2019 JD.com Inc. JD AI macro(configure_glog) - include(${PROJECT_SOURCE_DIR}/cmake/utils.cmake) message(STATUS "Configureing glog...") option(BUILD_TESTING "" OFF) option(WITH_GFLAGS "" OFF) add_subdirectory(${PROJECT_SOURCE_DIR}/third_party/glog) - add_msvc_runtime_flag(glog) endmacro() diff --git a/cmake/utils.cmake b/cmake/utils.cmake index 6329819..bd5b040 100644 --- a/cmake/utils.cmake +++ b/cmake/utils.cmake @@ -1,19 +1,19 @@ # Copyright 2019 JD.com Inc. JD AI # Add MSVC RunTime Flag -function(add_msvc_runtime_flag lib) +function(bnn_add_msvc_runtime_flag) if (MSVC) if(${BNN_USE_MSVC_STATIC_RUNTIME}) if(${CMAKE_BUILD_TYPE} MATCHES "Debug") - target_compile_options(${lib} PRIVATE /MTd) + add_compile_options(/MTd) else() - target_compile_options(${lib} PRIVATE /MT) + add_compile_options(/MT) endif() else() if(${CMAKE_BUILD_TYPE} MATCHES "Debug") - target_compile_options(${lib} PRIVATE /MDd) + add_compile_options(/MDd) else() - target_compile_options(${lib} PRIVATE /MD) + add_compile_options(/MD) endif() endif() endif() diff --git a/tools/onnx2bnn/CMakeLists.txt b/tools/onnx2bnn/CMakeLists.txt index 97688a4..c652808 100644 --- a/tools/onnx2bnn/CMakeLists.txt +++ b/tools/onnx2bnn/CMakeLists.txt @@ -21,7 +21,6 @@ target_include_directories(onnx2bnn ${PROJECT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} ) -add_msvc_runtime_flag(onnx2bnn) add_executable(onnx2bnn-bin ${CMAKE_CURRENT_SOURCE_DIR}/onnx2bnn.cpp @@ -46,6 +45,4 @@ if (${BNN_BUILD_PYTHON}) onnx2bnn pybind11::pybind11 ) - - add_msvc_runtime_flag(_onnx2bnn) endif() From c73a90466e61b256a7530e97dbd323fe23e2e334 Mon Sep 17 00:00:00 2001 From: daquexian Date: Sun, 19 May 2019 15:15:57 +0800 Subject: [PATCH 12/18] Upload package to pypi --- ci/onnx2bnn_build.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ci/onnx2bnn_build.yml b/ci/onnx2bnn_build.yml index ce38d6d..a07fa4e 100644 --- a/ci/onnx2bnn_build.yml +++ b/ci/onnx2bnn_build.yml @@ -32,6 +32,13 @@ jobs: steps: - checkout: self submodules: true + - powershell: | + Write-Host "This works: $env:TWINE_USERNAME" # Recommended + Write-Host "This works: $env:TWINE_PASSWORD" # Recommended + workingDirectory: tools/onnx2bnn/python/ + env: + TWINE_USERNAME: $(twineUsername) + TWINE_PASSWORD: $(twinePassword) - task: UsePythonVersion@0 inputs: versionSpec: '3.5' @@ -42,3 +49,8 @@ jobs: - script: python setup.py sdist bdist_wheel workingDirectory: tools/onnx2bnn/python/ displayName: Build onnx2bnn python package + - script: python -m twine upload --repository-url https://test.pypi.org/legacy/ dist/* --verbose + workingDirectory: tools/onnx2bnn/python/ + env: + TWINE_USERNAME: $(twineUsername) + TWINE_PASSWORD: $(twinePassword) From b21655b9cf2ab9f3804c393d9f9c58599c2f5022 Mon Sep 17 00:00:00 2001 From: daquexian Date: Sun, 19 May 2019 16:17:58 +0800 Subject: [PATCH 13/18] Build for multiple python version --- ci/onnx2bnn_build.yml | 30 ++++++++++++--------------- ci/template_onnx2bnn_build_python.yml | 13 ++++++++++++ tools/onnx2bnn/python/setup.py | 2 +- 3 files changed, 27 insertions(+), 18 deletions(-) create mode 100644 ci/template_onnx2bnn_build_python.yml diff --git a/ci/onnx2bnn_build.yml b/ci/onnx2bnn_build.yml index a07fa4e..40b8b21 100644 --- a/ci/onnx2bnn_build.yml +++ b/ci/onnx2bnn_build.yml @@ -32,25 +32,21 @@ jobs: steps: - checkout: self submodules: true - - powershell: | - Write-Host "This works: $env:TWINE_USERNAME" # Recommended - Write-Host "This works: $env:TWINE_PASSWORD" # Recommended - workingDirectory: tools/onnx2bnn/python/ - env: - TWINE_USERNAME: $(twineUsername) - TWINE_PASSWORD: $(twinePassword) + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.7' + addToPath: true + architecture: 'x64' + - template: template_onnx2bnn_build_python.yml + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.6' + addToPath: true + architecture: 'x64' + - template: template_onnx2bnn_build_python.yml - task: UsePythonVersion@0 inputs: versionSpec: '3.5' addToPath: true architecture: 'x64' - - script: python -m pip install --user --upgrade setuptools wheel twine - displayName: Install setuptools, wheel and twine - - script: python setup.py sdist bdist_wheel - workingDirectory: tools/onnx2bnn/python/ - displayName: Build onnx2bnn python package - - script: python -m twine upload --repository-url https://test.pypi.org/legacy/ dist/* --verbose - workingDirectory: tools/onnx2bnn/python/ - env: - TWINE_USERNAME: $(twineUsername) - TWINE_PASSWORD: $(twinePassword) + - template: template_onnx2bnn_build_python.yml diff --git a/ci/template_onnx2bnn_build_python.yml b/ci/template_onnx2bnn_build_python.yml new file mode 100644 index 0000000..b45b1ef --- /dev/null +++ b/ci/template_onnx2bnn_build_python.yml @@ -0,0 +1,13 @@ +steps: +- script: python -m pip install --user --upgrade setuptools wheel twine + displayName: Install setuptools, wheel and twine +- script: python setup.py sdist bdist_wheel + workingDirectory: tools/onnx2bnn/python/ + displayName: Build onnx2bnn python package +- script: python -m twine upload --repository-url https://test.pypi.org/legacy/ dist/* --verbose + displayName: Upload wheel to PyPI + workingDirectory: tools/onnx2bnn/python/ + env: + TWINE_USERNAME: $(twineUsername) + TWINE_PASSWORD: $(twinePassword) + diff --git a/tools/onnx2bnn/python/setup.py b/tools/onnx2bnn/python/setup.py index 52e5bcb..2a8b6af 100644 --- a/tools/onnx2bnn/python/setup.py +++ b/tools/onnx2bnn/python/setup.py @@ -171,7 +171,7 @@ def build_extension(self, ext): setuptools.setup( name="onnx2bnn", - # version=VersionInfo.version, + version="0.0.1", description="Convert ONNX to dabnn", ext_modules=ext_modules, cmdclass=cmdclass, From 80803e2fe6cb054d67f2339757c041aa75d288ae Mon Sep 17 00:00:00 2001 From: daquexian Date: Sun, 19 May 2019 17:54:30 +0800 Subject: [PATCH 14/18] Copy _onnx2bnn .so from Release/Debug to onnx2bnn in windows --- tools/onnx2bnn/python/onnx2bnn/convert.py | 2 +- tools/onnx2bnn/python/setup.py | 20 +++++++++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/tools/onnx2bnn/python/onnx2bnn/convert.py b/tools/onnx2bnn/python/onnx2bnn/convert.py index a208cac..bd1fc8b 100644 --- a/tools/onnx2bnn/python/onnx2bnn/convert.py +++ b/tools/onnx2bnn/python/onnx2bnn/convert.py @@ -1,4 +1,4 @@ -import _onnx2bnn +import onnx2bnn._onnx2bnn import onnx diff --git a/tools/onnx2bnn/python/setup.py b/tools/onnx2bnn/python/setup.py index 2a8b6af..ae88feb 100644 --- a/tools/onnx2bnn/python/setup.py +++ b/tools/onnx2bnn/python/setup.py @@ -108,8 +108,7 @@ def build_extension(self, ext): '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON', '-DPY_EXT_SUFFIX={}'.format(sysconfig.get_config_var('EXT_SUFFIX') or ''), '-DBNN_BUILD_PYTHON=ON', - '-DBNN_SYSTEM_PROTOBUF=OFF', - '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={}'.format(extdir), + '-DBNN_SYSTEM_PROTOBUF=OFF' ] if COVERAGE or DEBUG: # in order to get accurate coverage information, the @@ -145,6 +144,21 @@ def build_extension(self, ext): build_args.extend(['--', '-j', str(multiprocessing.cpu_count())]) subprocess.check_call(build_args) + fullname = self.get_ext_fullname(ext.name) + filename = os.path.basename(self.get_ext_filename(fullname)) + + lib_path = os.path.join(CMAKE_BUILD_DIR, 'tools', 'onnx2bnn') + if os.name == 'nt': + debug_lib_dir = os.path.join(lib_path, "Debug") + release_lib_dir = os.path.join(lib_path, "Release") + if os.path.exists(debug_lib_dir): + lib_path = debug_lib_dir + elif os.path.exists(release_lib_dir): + lib_path = release_lib_dir + src = os.path.join(lib_path, filename) + dst = os.path.join(os.path.realpath(self.build_lib), "onnx2bnn", filename) + self.copy_file(src, dst) + cmdclass = { 'build_ext': build_ext, @@ -171,7 +185,7 @@ def build_extension(self, ext): setuptools.setup( name="onnx2bnn", - version="0.0.1", + version="0.0.2", description="Convert ONNX to dabnn", ext_modules=ext_modules, cmdclass=cmdclass, From 1e20c197626b09a13113813066403bae7a2a1760 Mon Sep 17 00:00:00 2001 From: daquexian Date: Sun, 19 May 2019 17:59:24 +0800 Subject: [PATCH 15/18] Publish onnx2bnn.exe --- ci/template_onnx2bnn_build_python.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ci/template_onnx2bnn_build_python.yml b/ci/template_onnx2bnn_build_python.yml index b45b1ef..ed28f43 100644 --- a/ci/template_onnx2bnn_build_python.yml +++ b/ci/template_onnx2bnn_build_python.yml @@ -4,6 +4,14 @@ steps: - script: python setup.py sdist bdist_wheel workingDirectory: tools/onnx2bnn/python/ displayName: Build onnx2bnn python package +- task: CopyFiles@2 + inputs: + contents: '.setuptools-cmake-build\tools\onnx2bnn\Release\onnx2bnn.exe' + targetFolder: $(Build.ArtifactStagingDirectory) +- task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: $(Build.ArtifactStagingDirectory) + artifactName: onnx2bnn.exe - script: python -m twine upload --repository-url https://test.pypi.org/legacy/ dist/* --verbose displayName: Upload wheel to PyPI workingDirectory: tools/onnx2bnn/python/ From b93d5d2dde5c7af602b28bcd7ab6cf7b9b633296 Mon Sep 17 00:00:00 2001 From: daquexian Date: Sun, 19 May 2019 21:36:23 +0800 Subject: [PATCH 16/18] Update publishing --- ci/onnx2bnn_build.yml | 41 ++++++++++++++++++++++----- ci/template_onnx2bnn_build_python.yml | 15 ---------- tools/onnx2bnn/python/setup.py | 2 +- 3 files changed, 35 insertions(+), 23 deletions(-) diff --git a/ci/onnx2bnn_build.yml b/ci/onnx2bnn_build.yml index 40b8b21..c208723 100644 --- a/ci/onnx2bnn_build.yml +++ b/ci/onnx2bnn_build.yml @@ -1,3 +1,8 @@ +trigger: + branches: + include: + - refs/heads/* + - refs/tags/* jobs: - job: Linux_AppImage pool: @@ -19,13 +24,6 @@ jobs: inputs: pathtoPublish: $(Build.ArtifactStagingDirectory) artifactName: onnx2bnn AppImage - - task: GitHubRelease@0 - inputs: - gitHubConnection: 'dabnn release' - repositoryName: '$(Build.Repository.Name)' - action: 'create' - target: '$(Build.SourceVersion)' - assets: '$(Build.ArtifactStagingDirectory)/onnx2bnn.AppImage' - job: Windows_Python_Package pool: vmImage: 'vs2017-win2016' @@ -50,3 +48,32 @@ jobs: addToPath: true architecture: 'x64' - template: template_onnx2bnn_build_python.yml + - task: CopyFiles@2 + inputs: + sourceFolder: '.setuptools-cmake-build\tools\onnx2bnn\Release\' + contents: 'onnx2bnn.exe' + targetFolder: $(Build.ArtifactStagingDirectory) + - script: python -m twine upload --repository-url https://test.pypi.org/legacy/ dist/* --verbose + condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/test')) + displayName: Upload wheel to PyPI + workingDirectory: tools/onnx2bnn/python/ + env: + TWINE_USERNAME: $(twineUsername) + TWINE_PASSWORD: $(twinePassword) +- job: Publish_GitHub_Release + dependsOn: + - Linux_AppImage + - Windows_Python_Package + condition: succeeded() + steps: + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: $(Build.ArtifactStagingDirectory) + artifactName: onnx2bnn + - task: GitHubRelease@0 + inputs: + gitHubConnection: 'dabnn release' + repositoryName: '$(Build.Repository.Name)' + action: 'create' + target: '$(Build.SourceVersion)' + assets: '$(Build.ArtifactStagingDirectory)/*' diff --git a/ci/template_onnx2bnn_build_python.yml b/ci/template_onnx2bnn_build_python.yml index ed28f43..01bf84f 100644 --- a/ci/template_onnx2bnn_build_python.yml +++ b/ci/template_onnx2bnn_build_python.yml @@ -4,18 +4,3 @@ steps: - script: python setup.py sdist bdist_wheel workingDirectory: tools/onnx2bnn/python/ displayName: Build onnx2bnn python package -- task: CopyFiles@2 - inputs: - contents: '.setuptools-cmake-build\tools\onnx2bnn\Release\onnx2bnn.exe' - targetFolder: $(Build.ArtifactStagingDirectory) -- task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: $(Build.ArtifactStagingDirectory) - artifactName: onnx2bnn.exe -- script: python -m twine upload --repository-url https://test.pypi.org/legacy/ dist/* --verbose - displayName: Upload wheel to PyPI - workingDirectory: tools/onnx2bnn/python/ - env: - TWINE_USERNAME: $(twineUsername) - TWINE_PASSWORD: $(twinePassword) - diff --git a/tools/onnx2bnn/python/setup.py b/tools/onnx2bnn/python/setup.py index ae88feb..201ae58 100644 --- a/tools/onnx2bnn/python/setup.py +++ b/tools/onnx2bnn/python/setup.py @@ -185,7 +185,7 @@ def build_extension(self, ext): setuptools.setup( name="onnx2bnn", - version="0.0.2", + version="0.0.3", description="Convert ONNX to dabnn", ext_modules=ext_modules, cmdclass=cmdclass, From 8201b5b3f423a09d6873f9e38f478f9a6de10d43 Mon Sep 17 00:00:00 2001 From: daquexian Date: Sun, 19 May 2019 22:52:38 +0800 Subject: [PATCH 17/18] Use edit instead of create for github release --- ci/dabnn_build_and_test.yml | 5 +++++ ci/onnx2bnn_build.yml | 35 +++++++++++++++++----------------- tools/onnx2bnn/python/setup.py | 2 +- 3 files changed, 24 insertions(+), 18 deletions(-) diff --git a/ci/dabnn_build_and_test.yml b/ci/dabnn_build_and_test.yml index 267d5dd..8c56635 100644 --- a/ci/dabnn_build_and_test.yml +++ b/ci/dabnn_build_and_test.yml @@ -1,3 +1,8 @@ +trigger: + branches: + include: + - refs/heads/* + - refs/tags/* pool: vmImage: 'macOS-10.14' steps: diff --git a/ci/onnx2bnn_build.yml b/ci/onnx2bnn_build.yml index c208723..8d4878e 100644 --- a/ci/onnx2bnn_build.yml +++ b/ci/onnx2bnn_build.yml @@ -24,6 +24,15 @@ jobs: inputs: pathtoPublish: $(Build.ArtifactStagingDirectory) artifactName: onnx2bnn AppImage + - task: GitHubRelease@0 + inputs: + gitHubConnection: 'dabnn release' + repositoryName: '$(Build.Repository.Name)' + action: 'edit' + tag: '$(Build.SourceBranchName)' + target: '$(Build.SourceVersion)' + assets: '$(Build.ArtifactStagingDirectory)/*' + assetUploadMode: 'replace' - job: Windows_Python_Package pool: vmImage: 'vs2017-win2016' @@ -53,6 +62,15 @@ jobs: sourceFolder: '.setuptools-cmake-build\tools\onnx2bnn\Release\' contents: 'onnx2bnn.exe' targetFolder: $(Build.ArtifactStagingDirectory) + - task: GitHubRelease@0 + inputs: + gitHubConnection: 'dabnn release' + repositoryName: '$(Build.Repository.Name)' + action: 'edit' + tag: '$(Build.SourceBranchName)' + target: '$(Build.SourceVersion)' + assets: '$(Build.ArtifactStagingDirectory)/*' + assetUploadMode: 'replace' - script: python -m twine upload --repository-url https://test.pypi.org/legacy/ dist/* --verbose condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/test')) displayName: Upload wheel to PyPI @@ -60,20 +78,3 @@ jobs: env: TWINE_USERNAME: $(twineUsername) TWINE_PASSWORD: $(twinePassword) -- job: Publish_GitHub_Release - dependsOn: - - Linux_AppImage - - Windows_Python_Package - condition: succeeded() - steps: - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: $(Build.ArtifactStagingDirectory) - artifactName: onnx2bnn - - task: GitHubRelease@0 - inputs: - gitHubConnection: 'dabnn release' - repositoryName: '$(Build.Repository.Name)' - action: 'create' - target: '$(Build.SourceVersion)' - assets: '$(Build.ArtifactStagingDirectory)/*' diff --git a/tools/onnx2bnn/python/setup.py b/tools/onnx2bnn/python/setup.py index 201ae58..1f3a7ce 100644 --- a/tools/onnx2bnn/python/setup.py +++ b/tools/onnx2bnn/python/setup.py @@ -185,7 +185,7 @@ def build_extension(self, ext): setuptools.setup( name="onnx2bnn", - version="0.0.3", + version="0.0.4", description="Convert ONNX to dabnn", ext_modules=ext_modules, cmdclass=cmdclass, From 54137178b1d7d3daa5aabe705cb5b800193b1b8b Mon Sep 17 00:00:00 2001 From: daquexian Date: Sun, 19 May 2019 23:19:07 +0800 Subject: [PATCH 18/18] Upload python package to pypi on tag v* --- ci/onnx2bnn_build.yml | 4 ++-- tools/onnx2bnn/python/setup.py | 9 ++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/ci/onnx2bnn_build.yml b/ci/onnx2bnn_build.yml index 8d4878e..06039f1 100644 --- a/ci/onnx2bnn_build.yml +++ b/ci/onnx2bnn_build.yml @@ -71,8 +71,8 @@ jobs: target: '$(Build.SourceVersion)' assets: '$(Build.ArtifactStagingDirectory)/*' assetUploadMode: 'replace' - - script: python -m twine upload --repository-url https://test.pypi.org/legacy/ dist/* --verbose - condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/test')) + - script: python -m twine upload dist/* --verbose + condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v')) displayName: Upload wheel to PyPI workingDirectory: tools/onnx2bnn/python/ env: diff --git a/tools/onnx2bnn/python/setup.py b/tools/onnx2bnn/python/setup.py index 1f3a7ce..23e6a23 100644 --- a/tools/onnx2bnn/python/setup.py +++ b/tools/onnx2bnn/python/setup.py @@ -40,6 +40,13 @@ tests_require = [] extras_require = {} +build_source_branch = os.getenv("BUILD_SOURCEBRANCH", "") +if build_source_branch.startswith('refs/tags/v'): + dabnn_version = build_source_branch[len('refs/tags/v'):] +else: + dabnn_version = '0.0.0' +print("dabnn version: {}".format(dabnn_version)) + ################################################################################ # Global variables for controlling the build variant ################################################################################ @@ -185,7 +192,7 @@ def build_extension(self, ext): setuptools.setup( name="onnx2bnn", - version="0.0.4", + version=dabnn_version, description="Convert ONNX to dabnn", ext_modules=ext_modules, cmdclass=cmdclass,