tensorflow - 我正在我的 react-native 项目中实现 tensorflow camerawithtensors 但是在这里当我阅读 Image Tensor 它没有拍摄完整的图像

Question

我的 App.js 文件

import React, { Component } from 'react';
import { Camera } from 'expo-camera';
import { cameraWithTensors, fetch, decodeJpeg, bundleResourceIO, renderToGLView } from '@tensorflow/tfjs-react-native';
import { View, Text, StyleSheet, Platform,Dimensions } from 'react-native'
import * as tf from '@tensorflow/tfjs';

import onnx_model from './model/model_loader'

import Svg, { Line } from 'react-native-svg'



const TensorCamera = cameraWithTensors(Camera);
let HEIGHT, WIDTH
if (Platform.OS === 'ios') {
    HEIGHT= 1920
    WIDTH= 1080
} else {
    HEIGHT= 1200
    WIDTH= 1600
}
const [RHEIGHT, RWIDTH] = [200,4*Math.floor(50*WIDTH/HEIGHT)]

class CameraTester2 extends Component {

  constructor(props) {
    super(props),
    this.state = { coordinates:[] };
    this.handleCameraStream.bind(this.state, this.setState)
  }

  async componentDidMount() {
    await tf.ready()
  }

  handleCameraStream(images, updatePreview, gl) {
    const loop = async () => {
      const nextImageTensor = images.next().value
      let data = nextImageTensor.arraySync();
      var id = nextImageTensor['dataId'].id
      if(id%30===0){
        var coordinates = await onnx_model(data)
        this.setState({coordinates})
      }
      tf.dispose([data]);
      this.rafID = requestAnimationFrame(loop);
    }
    loop();
  }

  async componentDidMount(){
    await tf.ready()
    const { status } = await Camera.requestPermissionsAsync();
  }

  componentWillUnmount() {
    cancelAnimationFrame(this.rafID);
  }

  rectangleBox = ()=>{
    var [br, bl,tl,tr] = this.state.coordinates
    return <View style={[
        StyleSheet.absoluteFill,
        { alignItems: 'center', justifyContent: 'center' },
      ]}>

      {this.state.coordinates.length>0 && <Svg height="100%" width="100%">
        <Line x1={tl[1]} y1={tl[0]} x2={tr[1]} y2={tr[0]} stroke="red" strokeWidth="2" />
        <Line x1={tl[1]} y1={tl[0]} x2={bl[1]} y2={bl[0]} stroke="red" strokeWidth="2" />
        <Line x1={tr[1]} y1={tr[0]} x2={br[1]} y2={br[0]} stroke="red" strokeWidth="2" />
        <Line x1={bl[1]} y1={bl[0]} x2={br[1]} y2={br[0]} stroke="red" strokeWidth="2" />
      </Svg>}

    </View>
  }

  render() {
  return <View>
        <TensorCamera
          // Standard Camera props
          style={styles.camera}
          type={0}
          zoom={0}
          // Tensor related props
          cameraTextureHeight={HEIGHT}
          cameraTextureWidth={WIDTH}
          resizeHeight={RHEIGHT}
          resizeWidth={RWIDTH}
          resizeDepth={3}
          onReady={this.handleCameraStream.bind(this)}
          autorender={true}
        />
        {/* {this.rectangleBox()} */}
      </View>
    
  }
} 

export default CameraTester2;

我的 model_loader.js 文件

import {Dimensions} from 'react-native'
import axios from 'axios';

const size = [1200,1600]
const imageSize = [200,152]

const onnx_model = async (nextImageTensor) => {
    const res = await axios.post('http://f232-45-118-159-34.ngrok.io/get_document',{imageTensor:nextImageTensor})
    if(res.data){
        var bl = res.data['bl']
        var br = res.data['br']
        var tl = res.data['tl']
        var tr = res.data['tr']
        
        br = br.map((x,i)=>x*size[i]/imageSize[i]);
        bl = bl.map((x,i)=>x*size[i]/imageSize[i]);
        tl = tl.map((x,i)=>x*size[i]/imageSize[i]);
        tr = tr.map((x,i)=>x*size[i]/imageSize[i]);
        
        return [br, bl,tl,tr]
    }
}

export default onnx_model

我正在跟踪的图片链接（请访问）：

我得到的图片链接（请访问）

在后端，我使用 PyTorch 模型来获取结果，但图像正在裁剪，为什么它没有给出正确的预测。当我优化我的参数时，它会改变视图，所以任何建议都可以为 CameraWithTensor 获得我最好的参数

标签： tensorflowtensorflow.js