首页 > 解决方案 > “utf-8”编解码器无法解码位置 54 中的字节 0xff:无效的起始字节

问题描述

我知道这类问题有很多答案,但由于我对编程的了解有限,我无法将它们应用到我的代码中。

from __future__ import absolute_import, division, print_function, unicode_literals
import copy
import glob
import imp
import os
import re
import setuptools
import subprocess
import sys
import sysconfig
import tempfile
import warnings

import torch
from .file_baton import FileBaton
from ._cpp_extension_versioner import ExtensionVersioner

from setuptools.command.build_ext import build_ext


IS_WINDOWS = sys.platform == 'win32'

def _find_cuda_home():
    '''Finds the CUDA install path.'''
    # Guess #1
    cuda_home = os.environ.get('CUDA_HOME') or os.environ.get('CUDA_PATH')
    if cuda_home is None:
        # Guess #2
        if IS_WINDOWS:
            cuda_homes = glob.glob(
                'C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v*.*')
            if len(cuda_homes) == 0:
                cuda_home = ''
            else:
                cuda_home = cuda_homes[0]
        else:
            cuda_home = '/usr/local/cuda'
        if not os.path.exists(cuda_home):
            # Guess #3
            try:
                which = 'where' if IS_WINDOWS else 'which'
                nvcc = subprocess.check_output(
                    [which, 'nvcc']).decode().rstrip('\r\n')
                cuda_home = os.path.dirname(os.path.dirname(nvcc))
            except Exception:
                cuda_home = None
    if cuda_home and not torch.cuda.is_available():
        print("No CUDA runtime is found, using CUDA_HOME='{}'".format(cuda_home))
    return cuda_home

MINIMUM_GCC_VERSION = (4, 9, 0)
MINIMUM_MSVC_VERSION = (19, 0, 24215)
ABI_INCOMPATIBILITY_WARNING = 

WRONG_COMPILER_WARNING = 

CUDA_HOME = _find_cuda_home()
CUDNN_HOME = os.environ.get('CUDNN_HOME') or os.environ.get('CUDNN_PATH')

BUILT_FROM_SOURCE_VERSION_PATTERN = re.compile(r'\d+\.\d+\.\d+\w+\+\w+')

COMMON_NVCC_FLAGS = [
    '-D__CUDA_NO_HALF_OPERATORS__',
    '-D__CUDA_NO_HALF_CONVERSIONS__',
    '-D__CUDA_NO_HALF2_OPERATORS__',
]

JIT_EXTENSION_VERSIONER = ExtensionVersioner()

def _is_binary_build():
    return not BUILT_FROM_SOURCE_VERSION_PATTERN.match(torch.version.__version__)


def _accepted_compilers_for_platform():
    return ['clang++', 'clang'] if sys.platform.startswith('darwin') else ['g++', 'gcc']


def get_default_build_root():
 
    return os.path.realpath(os.path.join(tempfile.gettempdir(), 'torch_extensions'))


def check_compiler_ok_for_platform(compiler):

    if IS_WINDOWS:
        return True
    which = subprocess.check_output(['which', compiler], stderr=subprocess.STDOUT)
    # Use os.path.realpath to resolve any symlinks, in particular from 'c++' to e.g. 'g++'.
    compiler_path = os.path.realpath(which.decode().strip())
    return any(name in compiler_path for name in _accepted_compilers_for_platform())


def check_compiler_abi_compatibility(compiler):
    '''
    Verifies that the given compiler is ABI-compatible with PyTorch.
    '''
    if not _is_binary_build():
        return True
    if os.environ.get('TORCH_DONT_CHECK_COMPILER_ABI') in ['ON', '1', 'YES', 'TRUE', 'Y']:
        return True

    # First check if the compiler is one of the expected ones for the particular platform.
    if not check_compiler_ok_for_platform(compiler):
        warnings.warn(WRONG_COMPILER_WARNING.format(
            user_compiler=compiler,
            pytorch_compiler=_accepted_compilers_for_platform()[0],
            platform=sys.platform))
        return False

    if sys.platform.startswith('darwin'):
        
        return True
    try:
        if sys.platform.startswith('linux'):
            minimum_required_version = MINIMUM_GCC_VERSION
            version = subprocess.check_output([compiler, '-dumpfullversion', '-dumpversion'])
            version = version.decode().strip().split('.')
        else:
            minimum_required_version = MINIMUM_MSVC_VERSION
            compiler_info = subprocess.check_output(compiler, stderr=subprocess.STDOUT)
            match = re.search(r'(\d+)\.(\d+)\.(\d+)', compiler_info.decode().strip())
            version = (0, 0, 0) if match is None else match.groups()
    except Exception:
        _, error, _ = sys.exc_info()
        warnings.warn('Error checking compiler version for {}: {}'.format(compiler, error))
        return False

    if tuple(map(int, version)) >= minimum_required_version:
        return True

    compiler = '{} {}'.format(compiler, ".".join(version))
    warnings.warn(ABI_INCOMPATIBILITY_WARNING.format(compiler))

    return False

class BuildExtension(build_ext, object):

    @classmethod
    def with_options(cls, **options):
        
        def init_with_options(*args, **kwargs):
            kwargs = kwargs.copy()
            kwargs.update(options)
            return cls(*args, **kwargs)
        return init_with_options

    def __init__(self, *args, **kwargs):
        super(BuildExtension, self).__init__(*args, **kwargs)
        self.no_python_abi_suffix = kwargs.get("no_python_abi_suffix", False)

    def build_extensions(self):
        self._check_abi()
        for extension in self.extensions:
            self._add_compile_flag(extension, '-DTORCH_API_INCLUDE_EXTENSION_H')
            self._define_torch_extension_name(extension)
            self._add_gnu_abi_flag_if_binary(extension)

        # Register .cu and .cuh as valid source extensions.
        self.compiler.src_extensions += ['.cu', '.cuh']
        # Save the original _compile method for later.
        if self.compiler.compiler_type == 'msvc':
            self.compiler._cpp_extensions += ['.cu', '.cuh']
            original_compile = self.compiler.compile
            original_spawn = self.compiler.spawn
        else:
            original_compile = self.compiler._compile

        def unix_wrap_compile(obj, src, ext, cc_args, extra_postargs, pp_opts):
            # Copy before we make any modifications.
            cflags = copy.deepcopy(extra_postargs)
            try:
                original_compiler = self.compiler.compiler_so
                if _is_cuda_file(src):
                    nvcc = _join_cuda_home('bin', 'nvcc')
                    if not isinstance(nvcc, list):
                        nvcc = [nvcc]
                    self.compiler.set_executable('compiler_so', nvcc)
                    if isinstance(cflags, dict):
                        cflags = cflags['nvcc']
                    cflags = COMMON_NVCC_FLAGS + ['--compiler-options', "'-fPIC'"] + cflags
                elif isinstance(cflags, dict):
                    cflags = cflags['cxx']
                # NVCC does not allow multiple -std to be passed, so we avoid
                # overriding the option if the user explicitly passed it.
                if not any(flag.startswith('-std=') for flag in cflags):
                    cflags.append('-std=c++11')

                original_compile(obj, src, ext, cc_args, cflags, pp_opts)
            finally:
                # Put the original compiler back in place.
                self.compiler.set_executable('compiler_so', original_compiler)

        def win_wrap_compile(sources,
                             output_dir=None,
                             macros=None,
                             include_dirs=None,
                             debug=0,
                             extra_preargs=None,
                             extra_postargs=None,
                             depends=None):

            self.cflags = copy.deepcopy(extra_postargs)
            extra_postargs = None

            def spawn(cmd):
                
                src_regex = re.compile('/T(p|c)(.*)')
                src_list = [
                    m.group(2) for m in (src_regex.match(elem) for elem in cmd)
                    if m
                ]

                obj_regex = re.compile('/Fo(.*)')
                obj_list = [
                    m.group(1) for m in (obj_regex.match(elem) for elem in cmd)
                    if m
                ]

                include_regex = re.compile(r'((\-|\/)I.*)')
                include_list = [
                    m.group(1)
                    for m in (include_regex.match(elem) for elem in cmd) if m
                ]

                if len(src_list) >= 1 and len(obj_list) >= 1:
                    src = src_list[0]
                    obj = obj_list[0]
                    if _is_cuda_file(src):
                        nvcc = _join_cuda_home('bin', 'nvcc')
                        if isinstance(self.cflags, dict):
                            cflags = self.cflags['nvcc']
                        elif isinstance(self.cflags, list):
                            cflags = self.cflags
                        else:
                            cflags = []
                        cmd = [
                            nvcc, '-c', src, '-o', obj, '-Xcompiler',
                            '/wd4819', '-Xcompiler', '/MD'
                        ] + include_list + cflags
                    elif isinstance(self.cflags, dict):
                        cflags = self.cflags['cxx']
                        cmd += cflags
                    elif isinstance(self.cflags, list):
                        cflags = self.cflags
                        cmd += cflags

                return original_spawn(cmd)

            try:
                self.compiler.spawn = spawn
                return original_compile(sources, output_dir, macros,
                                        include_dirs, debug, extra_preargs,
                                        extra_postargs, depends)
            finally:
                self.compiler.spawn = original_spawn

  
        if self.compiler.compiler_type == 'msvc':
            self.compiler.compile = win_wrap_compile
        else:
            self.compiler._compile = unix_wrap_compile

        build_ext.build_extensions(self)

    def get_ext_filename(self, ext_name):

        ext_filename = super(BuildExtension, self).get_ext_filename(ext_name)

        if self.no_python_abi_suffix and sys.version_info >= (3, 0):
         
            ext_filename_parts = ext_filename.split('.')
          
            without_abi = ext_filename_parts[:-2] + ext_filename_parts[-1:]
            ext_filename = '.'.join(without_abi)
        return ext_filename

    def _check_abi(self):
        
        if hasattr(self.compiler, 'compiler_cxx'):
            compiler = self.compiler.compiler_cxx[0]
        elif IS_WINDOWS:
            compiler = os.environ.get('CXX', 'cl')
        else:
            compiler = os.environ.get('CXX', 'c++')
        check_compiler_abi_compatibility(compiler)

    def _add_compile_flag(self, extension, flag):
        extension.extra_compile_args = copy.copy(extension.extra_compile_args)
        if isinstance(extension.extra_compile_args, dict):
            for args in extension.extra_compile_args.values():
                args.append(flag)
        else:
            extension.extra_compile_args.append(flag)

    def _define_torch_extension_name(self, extension):

        names = extension.name.split('.')
        name = names[-1]
        define = '-DTORCH_EXTENSION_NAME={}'.format(name)
        self._add_compile_flag(extension, define)

    def _add_gnu_abi_flag_if_binary(self, extension):

        if _is_binary_build():
            self._add_compile_flag(extension, '-D_GLIBCXX_USE_CXX11_ABI=0')


def CppExtension(name, sources, *args, **kwargs):

    include_dirs = kwargs.get('include_dirs', [])
    include_dirs += include_paths()
    kwargs['include_dirs'] = include_dirs

    if IS_WINDOWS:
        library_dirs = kwargs.get('library_dirs', [])
        library_dirs += library_paths()
        kwargs['library_dirs'] = library_dirs

        libraries = kwargs.get('libraries', [])
        libraries.append('c10')
        libraries.append('caffe2')
        libraries.append('torch')
        libraries.append('torch_python')
        libraries.append('_C')
        kwargs['libraries'] = libraries

    kwargs['language'] = 'c++'
    return setuptools.Extension(name, sources, *args, **kwargs)

def CUDAExtension(name, sources, *args, **kwargs):
    
    library_dirs = kwargs.get('library_dirs', [])
    library_dirs += library_paths(cuda=True)
    kwargs['library_dirs'] = library_dirs

    libraries = kwargs.get('libraries', [])
    libraries.append('cudart')
    if IS_WINDOWS:
        libraries.append('c10')
        libraries.append('caffe2')
        libraries.append('torch')
        libraries.append('torch_python')
        libraries.append('caffe2_gpu')
        libraries.append('_C')
    kwargs['libraries'] = libraries

    include_dirs = kwargs.get('include_dirs', [])
    include_dirs += include_paths(cuda=True)
    kwargs['include_dirs'] = include_dirs

    kwargs['language'] = 'c++'

    return setuptools.Extension(name, sources, *args, **kwargs)

def include_paths(cuda=False):

    here = os.path.abspath(__file__)
    torch_path = os.path.dirname(os.path.dirname(here))
    lib_include = os.path.join(torch_path, 'lib', 'include')
    paths = [
        lib_include,

        os.path.join(lib_include, 'torch', 'csrc', 'api', 'include'),

        os.path.join(lib_include, 'TH'),
        os.path.join(lib_include, 'THC')
    ]
    if cuda:
        paths.append(_join_cuda_home('include'))
        if CUDNN_HOME is not None:
            paths.append(os.path.join(CUDNN_HOME, 'include'))
    return paths


def library_paths(cuda=False):

    paths = []

    if IS_WINDOWS:
        here = os.path.abspath(__file__)
        torch_path = os.path.dirname(os.path.dirname(here))
        lib_path = os.path.join(torch_path, 'lib')

        paths.append(lib_path)

    if cuda:
        lib_dir = 'lib/x64' if IS_WINDOWS else 'lib64'
        paths.append(_join_cuda_home(lib_dir))
        if CUDNN_HOME is not None:
            paths.append(os.path.join(CUDNN_HOME, lib_dir))
    return paths

def load(name,
         sources,
         extra_cflags=None,
         extra_cuda_cflags=None,
         extra_ldflags=None,
         extra_include_paths=None,
         build_directory=None,
         verbose=False,
         with_cuda=None,
         is_python_module=True):

    return _jit_compile(
        name,
        [sources] if isinstance(sources, str) else sources,
        extra_cflags,
        extra_cuda_cflags,
        extra_ldflags,
        extra_include_paths,
        build_directory or _get_build_directory(name, verbose),
        verbose,
        with_cuda,
        is_python_module)


def load_inline(name,
                cpp_sources,
                cuda_sources=None,
                functions=None,
                extra_cflags=None,
                extra_cuda_cflags=None,
                extra_ldflags=None,
                extra_include_paths=None,
                build_directory=None,
                verbose=False,
                with_cuda=None,
                is_python_module=True):

    build_directory = build_directory or _get_build_directory(name, verbose)

    if isinstance(cpp_sources, str):
        cpp_sources = [cpp_sources]
    cuda_sources = cuda_sources or []
    if isinstance(cuda_sources, str):
        cuda_sources = [cuda_sources]

    cpp_sources.insert(0, '#include ')

    if functions is not None:
        cpp_sources.append('PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {')
        if isinstance(functions, str):
            functions = [functions]
        if isinstance(functions, list):
            # Make the function docstring the same as the function name.
            functions = dict((f, f) for f in functions)
        elif not isinstance(functions, dict):
            raise ValueError(
                "Expected 'functions' to be a list or dict, but was {}".format(
                    type(functions)))
        for function_name, docstring in functions.items():
            cpp_sources.append('m.def("{0}", &{0}, "{1}");'.format(
                function_name, docstring))
        cpp_sources.append('}')

    cpp_source_path = os.path.join(build_directory, 'main.cpp')
    with open(cpp_source_path, 'w') as cpp_source_file:
        cpp_source_file.write('\n'.join(cpp_sources))

    sources = [cpp_source_path]

    if cuda_sources:
        cuda_sources.insert(0, '#include ')
        cuda_sources.insert(1, '#include ')
        cuda_sources.insert(2, '#include ')

        cuda_source_path = os.path.join(build_directory, 'cuda.cu')
        with open(cuda_source_path, 'w') as cuda_source_file:
            cuda_source_file.write('\n'.join(cuda_sources))

        sources.append(cuda_source_path)

    return _jit_compile(
        name,
        sources,
        extra_cflags,
        extra_cuda_cflags,
        extra_ldflags,
        extra_include_paths,
        build_directory,
        verbose,
        with_cuda,
        is_python_module)


def _jit_compile(name,
                 sources,
                 extra_cflags,
                 extra_cuda_cflags,
                 extra_ldflags,
                 extra_include_paths,
                 build_directory,
                 verbose,
                 with_cuda,
                 is_python_module):
    old_version = JIT_EXTENSION_VERSIONER.get_version(name)
    version = JIT_EXTENSION_VERSIONER.bump_version_if_changed(
        name,
        sources,
        build_arguments=[extra_cflags, extra_cuda_cflags, extra_ldflags, extra_include_paths],
        build_directory=build_directory,
        with_cuda=with_cuda
    )
    if version > 0:
        if version != old_version and verbose:
            print('The input conditions for extension module {} have changed. '.format(name) +
                  'Bumping to version {0} and re-building as {1}_v{0}...'.format(version, name))
        name = '{}_v{}'.format(name, version)

    if version != old_version:
        baton = FileBaton(os.path.join(build_directory, 'lock'))
        if baton.try_acquire():
            try:
                _write_ninja_file_and_build(
                    name=name,
                    sources=sources,
                    extra_cflags=extra_cflags or [],
                    extra_cuda_cflags=extra_cuda_cflags or [],
                    extra_ldflags=extra_ldflags or [],
                    extra_include_paths=extra_include_paths or [],
                    build_directory=build_directory,
                    verbose=verbose,
                    with_cuda=with_cuda)
            finally:
                baton.release()
        else:
            baton.wait()
    elif verbose:
        print('No modifications detected for re-loaded extension '
              'module {}, skipping build step...'.format(name))

    if verbose:
        print('Loading extension module {}...'.format(name))
    return _import_module_from_library(name, build_directory, is_python_module)


def _write_ninja_file_and_build(name,
                                sources,
                                extra_cflags,
                                extra_cuda_cflags,
                                extra_ldflags,
                                extra_include_paths,
                                build_directory,
                                verbose,
                                with_cuda):
    verify_ninja_availability()
    check_compiler_abi_compatibility(os.environ.get('CXX', 'c++'))
    if with_cuda is None:
        with_cuda = any(map(_is_cuda_file, sources))
    extra_ldflags = _prepare_ldflags(
        extra_ldflags or [],
        with_cuda,
        verbose)
    build_file_path = os.path.join(build_directory, 'build.ninja')
    if verbose:
        print(
            'Emitting ninja build file {}...'.format(build_file_path))

    _write_ninja_file(
        path=build_file_path,
        name=name,
        sources=sources,
        extra_cflags=extra_cflags or [],
        extra_cuda_cflags=extra_cuda_cflags or [],
        extra_ldflags=extra_ldflags or [],
        extra_include_paths=extra_include_paths or [],
        with_cuda=with_cuda)

    if verbose:
        print('Building extension module {}...'.format(name))
    _build_extension_module(name, build_directory, verbose)


def verify_ninja_availability():

    with open(os.devnull, 'wb') as devnull:
        try:
            subprocess.check_call('ninja --version'.split(), stdout=devnull)
        except OSError:
            raise RuntimeError("Ninja is required to load C++ extensions")
        else:
            return True

def _prepare_ldflags(extra_ldflags, with_cuda, verbose):
    if IS_WINDOWS:
        python_path = os.path.dirname(sys.executable)
        python_lib_path = os.path.join(python_path, 'libs')

        here = os.path.abspath(__file__)
        torch_path = os.path.dirname(os.path.dirname(here))
        lib_path = os.path.join(torch_path, 'lib')

        extra_ldflags.append('c10.lib')
        extra_ldflags.append('caffe2.lib')
        extra_ldflags.append('torch.lib')
        extra_ldflags.append('torch_python.lib')
        if with_cuda:
            extra_ldflags.append('caffe2_gpu.lib')
        extra_ldflags.append('_C.lib')
        extra_ldflags.append('/LIBPATH:{}'.format(python_lib_path))
        extra_ldflags.append('/LIBPATH:{}'.format(lib_path))

    if with_cuda:
        if verbose:
            print('Detected CUDA files, patching ldflags')
        if IS_WINDOWS:
            extra_ldflags.append('/LIBPATH:{}'.format(
                _join_cuda_home('lib/x64')))
            extra_ldflags.append('cudart.lib')
            if CUDNN_HOME is not None:
                extra_ldflags.append(os.path.join(CUDNN_HOME, 'lib/x64'))
        else:
            extra_ldflags.append('-L{}'.format(_join_cuda_home('lib64')))
            extra_ldflags.append('-lcudart')
            if CUDNN_HOME is not None:
                extra_ldflags.append('-L{}'.format(os.path.join(CUDNN_HOME, 'lib64')))

    return extra_ldflags


def _get_build_directory(name, verbose):
    root_extensions_directory = os.environ.get('TORCH_EXTENSIONS_DIR')
    if root_extensions_directory is None:
        root_extensions_directory = get_default_build_root()

    if verbose:
        print('Using {} as PyTorch extensions root...'.format(
            root_extensions_directory))

    build_directory = os.path.join(root_extensions_directory, name)
    if not os.path.exists(build_directory):
        if verbose:
            print('Creating extension directory {}...'.format(build_directory))
       
        os.makedirs(build_directory)

    return build_directory

def _build_extension_module(name, build_directory, verbose):
    try:
        sys.stdout.flush()
        sys.stderr.flush()
        if sys.version_info >= (3, 5):
            subprocess.run(
                ['ninja', '-v'],
                stdout=None if verbose else subprocess.PIPE,
                stderr=subprocess.STDOUT,
                cwd=build_directory,
                check=True)
        else:
            subprocess.check_output(
                ['ninja', '-v'],
                stderr=subprocess.STDOUT,
                cwd=build_directory)
    except subprocess.CalledProcessError:

        _, error, _ = sys.exc_info()
      
        message = "Error building extension '{}'".format(name)
        if hasattr(error, 'output') and error.output:
            message += ": {}".format(error.output.decode())
        raise RuntimeError(message)


def _import_module_from_library(module_name, path, is_python_module):
    
    file, path, description = imp.find_module(module_name, [path])
 
    with file:
        if is_python_module:
            return imp.load_module(module_name, file, path, description)
        else:
            return torch.ops.load_library(path)


def _write_ninja_file(path,
                      name,
                      sources,
                      extra_cflags,
                      extra_cuda_cflags,
                      extra_ldflags,
                      extra_include_paths,
                      with_cuda):
    extra_cflags = [flag.strip() for flag in extra_cflags]
    extra_cuda_cflags = [flag.strip() for flag in extra_cuda_cflags]
    extra_ldflags = [flag.strip() for flag in extra_ldflags]
    extra_include_paths = [flag.strip() for flag in extra_include_paths]


    config = ['ninja_required_version = 1.3']
    config.append('cxx = {}'.format(os.environ.get('CXX', 'c++')))
    if with_cuda:
        config.append('nvcc = {}'.format(_join_cuda_home('bin', 'nvcc')))


    sources = [os.path.abspath(file) for file in sources]
    user_includes = [os.path.abspath(file) for file in extra_include_paths]

 
    system_includes = include_paths(with_cuda)

    system_includes.append(sysconfig.get_paths()['include'])


    if IS_WINDOWS:
        user_includes += system_includes
        system_includes.clear()

    common_cflags = ['-DTORCH_EXTENSION_NAME={}'.format(name)]
    common_cflags.append('-DTORCH_API_INCLUDE_EXTENSION_H')
    common_cflags += ['-I{}'.format(include) for include in user_includes]
    common_cflags += ['-isystem {}'.format(include) for include in system_includes]

    if _is_binary_build():
        common_cflags += ['-D_GLIBCXX_USE_CXX11_ABI=0']

    cflags = common_cflags + ['-fPIC', '-std=c++11'] + extra_cflags
    if IS_WINDOWS:
        from distutils.spawn import _nt_quote_args
        cflags = _nt_quote_args(cflags)
    flags = ['cflags = {}'.format(' '.join(cflags))]

    if with_cuda:
        cuda_flags = common_cflags + COMMON_NVCC_FLAGS
        if IS_WINDOWS:
            cuda_flags = _nt_quote_args(cuda_flags)
        else:
            cuda_flags += ['--compiler-options', "'-fPIC'"]
            cuda_flags += extra_cuda_cflags
            if not any(flag.startswith('-std=') for flag in cuda_flags):
                cuda_flags.append('-std=c++11')

        flags.append('cuda_flags = {}'.format(' '.join(cuda_flags)))

    if IS_WINDOWS:
        ldflags = ['/DLL'] + extra_ldflags
    else:
        ldflags = ['-shared'] + extra_ldflags
    # The darwin linker needs explicit consent to ignore unresolved symbols.
    if sys.platform.startswith('darwin'):
        ldflags.append('-undefined dynamic_lookup')
    elif IS_WINDOWS:
        ldflags = _nt_quote_args(ldflags)
    flags.append('ldflags = {}'.format(' '.join(ldflags)))

    compile_rule = ['rule compile']
    if IS_WINDOWS:
        compile_rule.append(
            '  command = cl /showIncludes $cflags -c $in /Fo$out')
        compile_rule.append('  deps = msvc')
    else:
        compile_rule.append(
            '  command = $cxx -MMD -MF $out.d $cflags -c $in -o $out')
        compile_rule.append('  depfile = $out.d')
        compile_rule.append('  deps = gcc')

    if with_cuda:
        cuda_compile_rule = ['rule cuda_compile']
        cuda_compile_rule.append(
            '  command = $nvcc $cuda_flags -c $in -o $out')

    link_rule = ['rule link']
    if IS_WINDOWS:
        cl_paths = subprocess.check_output(['where',
                                            'cl']).decode().split('\r\n')
        if len(cl_paths) >= 1:
            cl_path = os.path.dirname(cl_paths[0]).replace(':', '$:')
        else:
            raise RuntimeError("MSVC is required to load C++ extensions")
        link_rule.append(
            '  command = "{}/link.exe" $in /nologo $ldflags /out:$out'.format(
                cl_path))
    else:
        link_rule.append('  command = $cxx $in $ldflags -o $out')


    object_files = []
    build = []
    for source_file in sources:
      
        file_name = os.path.splitext(os.path.basename(source_file))[0]
        if _is_cuda_file(source_file) and with_cuda:
            rule = 'cuda_compile'
           
            target = '{}.cuda.o'.format(file_name)
        else:
            rule = 'compile'
            target = '{}.o'.format(file_name)
        object_files.append(target)
        if IS_WINDOWS:
            source_file = source_file.replace(':', '$:')
        source_file = source_file.replace(" ", "$ ")
        build.append('build {}: {} {}'.format(target, rule, source_file))

    ext = 'pyd' if IS_WINDOWS else 'so'
    library_target = '{}.{}'.format(name, ext)

    link = ['build {}: link {}'.format(library_target, ' '.join(object_files))]

    default = ['default {}'.format(library_target)]

  
    blocks = [config, flags, compile_rule]
    if with_cuda:
        blocks.append(cuda_compile_rule)
    blocks += [link_rule, build, link, default]
    with open(path, 'w') as build_file:
        for block in blocks:
            lines = '\n'.join(block)
            build_file.write('{}\n\n'.format(lines))


def _join_cuda_home(*paths):

    if CUDA_HOME is None:
        raise EnvironmentError('CUDA_HOME environment variable is not set. '
                               'Please set it to your CUDA install root.')
    return os.path.join(CUDA_HOME, *paths)


def _is_cuda_file(path):
    return os.path.splitext(path)[1] in ['.cu', '.cuh']

这是错误消息:

用户警告:检查 cl 的编译器版本时出错:“utf-8”编解码器无法解码位置 54 中的字节 0xff:起始字节无效

任何帮助将不胜感激!

谢谢!

标签: pythonpytorchsetuptools

解决方案


如果您使用的是 Linux,那么通常可以解决此类解码问题的解决方案是执行以下操作:

export LC_ALL=C.UTF-8

推荐阅读