javascript - NodeJS 子进程偶尔停止
问题描述
我的应用程序是一个体育博彩机器人。它是一个运行在 DO 盒子上的容器化应用程序，它使用cluster. 每个子进程在多个投注交易所（Betfair、Smarkets 和 Matchbook）中监控一个体育赛事。
主进程是这些子进程的协调者。在间隔期间，主进程获取每个实时事件的状态。如果一个事件已经结束，它将向监视该事件的子进程/工作人员发送一条消息以终止。
该应用程序对彼此进行 2 次投注以涵盖所有结果。这 2 个赌注应该同时进行，但遗憾的是，情况并非总是如此。如果发生这种情况，则有一个过程来纠正（“纠正过程”）这个位置。这就是问题发生的地方。偶尔，一个子进程会停止。在某个时间点之后，该过程没有日志。整改过程以递归方式工作setTimeout：
src/rectify.js
const logError = require('./log-error')
const logger = require('./logger/winston')
const Helpers = {
betfair: {
tradeOut: require('./betfair/trade-out'),
retry: require('./betfair/retry')
},
smarkets: {
tradeOut: require('./smarkets/trade-out'),
retry: require('./smarkets/retry'),
},
matchbook: {
tradeOut: require('./matchbook/trade-out'),
retry: require('./matchbook/retry')
}
}
let resolutionAttempts = 0
module.exports = async function(bet, runner) {
try {
const _promise = new Promise((resolve, reject) => {
async function rectify() {
try {
const [lowEx, upEx] = [runner.exchange.toLowerCase(), runner.exchange.toUpperCase()]
resolutionAttempts++
const isAttemptTradeOut = resolutionAttempts % 2 === 0
const _attempt = isAttemptTradeOut ? 'TRADE OUT' : 'RETRY'
const _lossMsg = `${(resolutionLossAllowed * 100).toFixed(2)}% LOSS ALLOWED`
logger.info(`[ATTEMPTING ${_attempt} FOR ${upEx} WITH ${_lossMsg} (Process: ${process.pid})]`)
const runners = {
matched: { ... },
unmatched: { ... }
}
const result = isAttemptTradeOut
? await Helpers[lowEx].tradeOut(bet, runners)
: await Helpers[lowEx].retry(bet, runners)
if (result) {
return resolve(result)
}
setTimeout(rectify, 10000) // 10 secs
} catch(err) {
logError(err, __filename)
return reject(err)
}
}
rectify()
})
const _res = await _promise
return _res
} catch(err) {
logError(err, __filename)
throw err
}
})
retry.jsMatchbook 文件之一的示例（这些文件在整个交易所中都是相同的）：
src/matchbook/retry.js
const bettingApi = require('../api/betting')
const { writeParams } = require('../../../../lib/logger/writer')
const logger = require('../../../../lib/logger/winston')
const utils = {
isArbStillValid: require('../../../utils/is-arb-still-valid'),
getFormattedArb: require('../../../utils/get-formatted-arb'),
logJsonError: require('../../../utils/log-json-error')
}
const arbitrage = {
getBackBackArbTrade: require('../../../strategies/arbitrage/get-back-back-arb-trade'),
getBackLayArbTrade: require('../../../strategies/arbitrage/get-back-lay-arb-trade')
}
const PlaceReturns = require('../../../../lib/consts/place-returns')
const placeKeepBet = require('./place-keep-bet')
const placeImmediateBet = require('./place-immediate-bet')
const isNotFoundError = require('./is-not-found-error')
async function retry(arb, config, runners, lossAllowed = 0, isKeep) {
try {
if (process.env.NODE_ENV === 'development') {
writeParams('exchanges/matchbook/helpers/retry-arb', { arb, config, runners, lossAllowed })
}
const betId = arb._id.toString()
const { eventId, marketId, runnerId } = runners.unmatched
const response = await bettingApi.getPrices(eventId, marketId, runnerId)
const { prices } = response.data
if (prices?.length) {
const samePrice = prices.find(_price => {
return _price.side === runners.unmatched.side
})
if (samePrice) {
const odds = samePrice['decimal-odds'] || samePrice.odds
const newArb = {
...arb,
runners: [ ... ]
}
const isValidArb = utils.isArbStillValid(newArb, lossAllowed)
if (isValidArb) {
const formattedArb = utils.getFormattedArb(newArb)
const _newArb = arb.subType === 'BackLay'
? arbitrage.getBackLayArbTrade(formattedArb, config, lossAllowed)
: arbitrage.getBackBackArbTrade(formattedArb, config, lossAllowed)
if (_newArb) {
const unmatchedRunner = _newArb.runners.find(runner => {
return runner.exchange === runners.unmatched.exchange
})
if (unmatchedRunner) {
const placeParams = { ... }
const result = isKeep
? await placeKeepBet(placeParams)
: await placeImmediateBet(placeParams)
if (result) {
return result
}
} else {
logger.info(`[NO UNMATCHED RUNNER (RETRY | MATCHBOOK | Bet: ${betId} | Process: ${process.pid})]`)
}
} else {
logger.info(`[NO NEW ARB (RETRY | MATCHBOOK | Bet: ${betId} | Process: ${process.pid})]`)
}
} else {
logger.info(`[NO VALID ARB (RETRY | MATCHBOOK | Bet: ${betId} | Process: ${process.pid})]`)
}
return
}
logger.info(`[NO SAME PRICE (RETRY | MATCHBOOK | Bet: ${betId} | Process: ${process.pid})]`)
} else {
logger.info(`[NO PRICES (RETRY | MATCHBOOK | Bet: ${betId} | Process: ${process.pid})]`)
}
const _response = await bettingApi.getMarket(eventId, marketId)
const { status } = _response.data
if (status === 'closed') {
logger.info(`[MATCHBOOK MARKET HAS ENDED (RETRY | Bet: ${betId} | Process: ${process.pid})]`)
return PlaceReturns.MATCHBOOK.ENDED
}
} catch (err) {
if (['ECONNRESET', 'EPIPE'].includes(err.code)) {
logger.info(`${err.code} (MATCHBOOK) - retryArb [process: ${process.pid}]`)
return retry(arb, config, runners, lossAllowed, isKeep)
}
const notFoundError = isNotFoundError(err)
if (notFoundError) {
return PlaceReturns.MATCHBOOK.ENDED
}
utils.logJsonError(__filename, 'retryArb', err, {
_throw: true
})
}
}
module.exports = retry
src/matchbook/trade-out.js
const bettingApi = require('../api/betting')
const { writeParams } = require('../../../../lib/logger/writer')
const logger = require('../../../../lib/logger/winston')
const utils = {
getTradeOutProfitStakePairs: require('../../../utils/get-trade-out-profit-stake-pairs'),
getBestProfitStakePair: require('../../../utils/get-best-profit-stake-pair'),
logJsonError: require('../../../utils/log-json-error'),
getCanGreenUp: require('../../../utils/get-can-green-up')
}
const PlaceReturns = require('../../../../lib/consts/place-returns')
const placeKeepBet = require('./place-keep-bet')
const placeImmediateBet = require('./place-immediate-bet')
const isNotFoundError = require('./is-not-found-error')
async function tradeOut(arb, matchedRunner, config, lossAllowed = 0, isKeep) {
try {
if (process.env.NODE_ENV === 'development') {
writeParams('exchanges/matchbook/helpers/trade-out', { arb, matchedRunner, config, lossAllowed, isKeep })
}
const betId = arb._id.toString()
const { eventId, marketId, runnerId } = matchedRunner
const response = await bettingApi.getPrices(eventId, marketId, runnerId)
const matchedLay = matchedRunner.side === 'lay'
const lowEx = matchedRunner.exchange.toLowerCase()
const { prices } = response.data
const maxToInvest = config.balances[lowEx] * 0.999
if (prices?.length) {
const oppositePrice = prices.find(price => {
return price.side !== matchedRunner.side
})
if (oppositePrice) {
const odds = oppositePrice['decimal-odds'] || oppositePrice.odds
const canGreenUp = utils.getCanGreenUp(matchedLay, odds, matchedRunner.price, lossAllowed)
if (canGreenUp) {
const profitStakePairs = utils.getTradeOutProfitStakePairs(matchedRunner, arb, config, lossAllowed)
if (profitStakePairs.length) {
const bestProfitStakePair = utils.getBestProfitStakePair('BackLay', profitStakePairs, config)
const stakeToUse = bestProfitStakePair.side === 'lay' ? bestProfitStakePair.stakes.lay : bestProfitStakePair.stakes.back
const placeParams = { ... }
const result = isKeep
? await placeKeepBet(placeParams)
: await placeImmediateBet(placeParams)
if (result) {
return result
}
} else {
logger.info(`[NO PROFIT/STAKE PAIRS (TRADE-OUT | MATCHBOOK | Bet: ${betId} | Process: ${process.pid})]`)
}
} else {
logger.info(`[CANNOT GREEN UP (TRADE-OUT | MATCHBOOK | Bet: ${betId} | Process: ${process.pid})]`)
}
return
}
logger.info(`[NO OPPOSITE PRICE (TRADE-OUT | MATCHBOOK | Bet: ${betId} | Process: ${process.pid})]`)
} else {
logger.info(`[NO PRICES (TRADE-OUT | MATCHBOOK | Bet: ${betId} | Process: ${process.pid})]`)
}
const _response = await bettingApi.getMarket(eventId, marketId)
const { status } = _response.data
if (status === 'closed') {
logger.info(`[MATCHBOOK MARKET HAS ENDED (TRADE-OUT | Bet: ${betId} | Process: ${process.pid})]`)
return PlaceReturns.MATCHBOOK.ENDED
}
} catch (err) {
if (['ECONNRESET', 'EPIPE'].includes(err.code)) {
logger.info(`${err.code} (MATCHBOOK) - tradeOut [process: ${process.pid}]`)
return tradeOut(arb, matchedRunner, config, lossAllowed, isKeep)
}
const notFoundError = isNotFoundError(err)
if (notFoundError) {
return PlaceReturns.MATCHBOOK.ENDED
}
utils.logJsonError(__filename, 'tradeOut', err, {
_throw: true
})
}
}
module.exports = tradeOut
以下是retry.js调用trade-out.js不调用的文件：
src/utils/get-formatted-arb.js
const { writeParams } = require('../../lib/logger/writer')
module.exports = function(arb) {
if (process.env.NODE_ENV === 'development') {
writeParams('utils/get-formatted-arb', { arb })
}
return {
...arb,
runners: arb.runners.reduce((acc, _runner) => {
if (arb.subType === 'BackBack') {
const otherRunner = arb.runners.find(__runner => {
return __runner.eventId !== _runner.eventId && __runner.marketId !== _runner.marketId && __runner.runnerId !== _runner.runnerId
})
const outcome = (_runner.price < otherRunner.price) ? 'outcome1' : 'outcome2'
acc[outcome] = _runner
return acc
}
acc[_runner.side] = _runner
return acc
}, {})
}
}
src/utils/is-arb-still-valid.js
const ArbTable = require('../../lib/consts/arb-table')
const { writeParams } = require('../../lib/logger/writer')
const logJsonError = require('./log-json-error')
module.exports = function(arb, lossAllowed = 0) {
try {
if (process.env.NODE_ENV === 'development') {
writeParams('utils/is-arb-still-valid', { arb, lossAllowed })
}
const { runners } = arb
if (arb.subType === 'BackBack') {
const [r0Price, r1Price] = [runners[0].price, runners[1].price]
const [r0PriceMinusLoss, r1PriceMinusLoss] = [r0Price - (r0Price * lossAllowed), r1Price - (r1Price * lossAllowed)]
if (r0PriceMinusLoss < 1.2 && r1PriceMinusLoss < 1.2) {
return false
}
return ArbTable.find(_arb => {
const outcome1 = runners.find(runner => {
return String(runner.outcomeSide) === '1'
})
const outcome2 = runners.find(runner => {
return String(runner.outcomeSide) === '2'
})
const [o1MinusLoss, o1AddLoss] = [_arb.outcome1 - (_arb.outcome1 * lossAllowed), _arb.outcome1 + (_arb.outcome1 * lossAllowed)]
const [o2MinusLoss, o2AddLoss] = [_arb.outcome2 - (_arb.outcome2 * lossAllowed), _arb.outcome2 + (_arb.outcome2 * lossAllowed)]
return outcome1.price >= o1MinusLoss && outcome1.price < (o1AddLoss + 0.1) && outcome2.price > o2MinusLoss
})
}
const backRunner = runners.find(_runner => _runner.side === 'back')
const layRunner = runners.find(_runner => _runner.side === 'lay')
const [layPrice, backPrice] = [layRunner.price, backRunner.price]
const layPriceMinusLoss = layPrice - (layPrice * lossAllowed)
return layPriceMinusLoss < backPrice
} catch(err) {
logJsonError(__filename, '_init_', err, {
_throw: true
})
}
}
src/arbitrage/get-back-lay-arb-trade.js
const utils = {
getHighestLowestExchangeBalances: require('../../utils/get-highest-lowest-exchange-balances'),
getBackLayStakesAndProfits: require('../../utils/get-back-lay-stakes-and-profits'),
logJsonError: require('../../utils/log-json-error')
}
const { writeParams } = require('../../../lib/logger/writer')
const logger = require('../../../lib/logger/winston')
module.exports = function(arb, config, lossAllowed = 0) {
try {
if (process.env.NODE_ENV === 'development') {
writeParams('strategies/arbitrage/get-back-lay-arb-trade', { arb, config })
}
const highestLowestExchangeBalances = utils.getHighestLowestExchangeBalances(arb.runners, config, lossAllowed)
const backLayStakesAndProfits = utils.getBackLayStakesAndProfits({
arb,
maxToInvest: highestLowestExchangeBalances.lowest,
config,
lossAllowed
})
if (backLayStakesAndProfits && Object.keys(backLayStakesAndProfits).length) {
const backLayArbTrade = {
...arb,
type: 'Arb',
subType: 'BackLay',
netProfits: backLayStakesAndProfits.netProfits,
grossProfits: backLayStakesAndProfits.grossProfits,
runners: Object.keys(arb.runners).map(side => {
return {
stake: backLayStakesAndProfits.stakes[side],
side,
...arb.runners[side]
}
})
}
logger.debug(`backLayArbTrade: ${JSON.stringify(backLayArbTrade)}`)
return backLayArbTrade
}
} catch(err) {
utils.logJsonError(__filename, '_init_', err, {
_throw: true
})
}
}
src/utils/get-highest-lowest-exchange-balances.js
const { writeParams } = require('../../lib/logger/writer')
module.exports = function(runners, config, lossAllowed = 0) {
if (process.env.NODE_ENV === 'development') {
writeParams('utils/get-highest-lowest-exchange-balances', { runners, config, lossAllowed })
}
const exchangeBalances = Object.values(runners).map(val => {
const lowEx = val.exchange.toLowerCase()
const amountToSave = (config.amountToSave[lowEx] || 0)
if (lossAllowed > 0) {
return config.balances[lowEx] - amountToSave
}
return (config.balances[lowEx] - amountToSave) * (config.maxPercentToUse || 1)
})
return {
lowest: Math.min(...exchangeBalances),
highest: Math.max(...exchangeBalances)
}
}
src/utils/get-back-lay-stakes-and-profits.js
const { writeParams } = require('../../lib/logger/writer')
const trimNumber = require('./trim-number')
const getBestProfitStakePair = require('./get-best-profit-stake-pair')
const getPairsBetterThanCurrent = require('./get-pairs-better-than-current')
const logger = require('../../lib/logger/winston')
const logJsonError = require('./log-json-error')
module.exports = function({ arb, maxToInvest, config, lossAllowed = 0 }) {
try {
if (process.env.NODE_ENV === 'development') {
writeParams('utils/get-back-lay-stakes-and-profits', { arb, maxToInvest, config, lossAllowed })
}
const profitStakePairs = []
const [backRunner, layRunner] = [arb.runners.back, arb.runners.lay]
const [backEx, layEx] = [backRunner.exchange.toLowerCase(), layRunner.exchange.toLowerCase()]
const [minBack, minLay] = [config.minimumBetSizes[backEx].back, config.minimumBetSizes[layEx].lay]
const maxQuantityToUse = config.maxQuantityToUse || 1
const [backCommission, layCommission] = [config.commissions[backEx].back, config.commissions[layEx].lay]
const [backPrice, layPrice] = [backRunner.price, layRunner.price]
const [backQuantity, layQuantity] = [backRunner.quantity, layRunner.quantity]
const [backStake, layStake] = [backRunner.stake, layRunner.stake]
const [backIsMatched, layIsMatched] = [backRunner.status === 'MATCHED', layRunner.status === 'MATCHED']
const [matchedBackStake, matchedLayStake] = [backIsMatched && backStake, layIsMatched && layStake]
const profitStakePairsNotInParams = []
let isExisting = false
if (arb._id && process.CURRENT_BETS[arb._id.toString()]) {
isExisting = true
const matchedRunner = process.CURRENT_BETS[arb._id.toString()].runners.find(runner => {
return runner.status.toUpperCase() === 'MATCHED'
})
const { netProfits } = process.CURRENT_BETS[arb._id.toString()]
if (matchedRunner) {
const matchedNetProfit = process.CURRENT_BETS[arb._id.toString()].subType === 'BackBack' ? netProfits[`outcome${matchedRunner.outcomeSide}`] : netProfits[matchedRunner.side.toLowerCase()]
lossAllowed = (matchedNetProfit * lossAllowed) * -1
}
}
let [potentialLayStake, potentialBackStake] = [undefined, maxToInvest * 0.999]
do {
if (!matchedLayStake) {
potentialLayStake = ((backPrice - backCommission) * (matchedBackStake || potentialBackStake)) / (layPrice - layCommission)
}
const [layStakeToUse, backStakeToUse] = [matchedLayStake || potentialLayStake, matchedBackStake || potentialBackStake]
const grossLayProfit = layStakeToUse
const layLiability = (layPrice - 1) * layStakeToUse
const netLayProfit = layStakeToUse * (1 - layCommission) - backStakeToUse
const calculatedBackGrossProfit = (backPrice - 1) * backStakeToUse
const grossBackProfit = backIsMatched ? arb.grossProfits?.back || calculatedBackGrossProfit : calculatedBackGrossProfit
const backCommissionPaid = grossBackProfit * backCommission
const netBackProfit = grossBackProfit - backCommissionPaid - layLiability
const withinLossAllowed = netBackProfit >= lossAllowed && netLayProfit >= lossAllowed
const amountToSave = config.amountToSave[layEx] || 0
const trueLayExchangeBalance = (config.balances[layEx] - amountToSave) * (config.maxPercentToUse || 1)
const betWithinParameters = layStakeToUse <= maxToInvest && layStakeToUse > minLay && layLiability <= trueLayExchangeBalance
const quantityMagnifier = lossAllowed > 0 ? 1 : maxQuantityToUse
const stakeWithinQuantityLimits = layStakeToUse <= (layQuantity * quantityMagnifier) && backStakeToUse <= (backQuantity * quantityMagnifier)
const currProfitStakePair = {
netProfits: {
lay: parseFloat(netLayProfit.toFixed(2)),
back: parseFloat(netBackProfit.toFixed(2))
},
grossProfits: {
lay: parseFloat(grossLayProfit.toFixed(2)),
back: parseFloat(grossBackProfit.toFixed(2))
},
stakes: {
lay: trimNumber(matchedLayStake || potentialLayStake, 2),
back: trimNumber(matchedBackStake || potentialBackStake, 2)
}
}
if (betWithinParameters && stakeWithinQuantityLimits && withinLossAllowed) {
profitStakePairs.push(currProfitStakePair)
}
if (potentialBackStake > minBack) {
potentialBackStake = matchedBackStake ? minBack : potentialBackStake * 0.999
}
} while (potentialBackStake > minBack)
if (profitStakePairs.length) {
const pairsBetterThanCurrent = getPairsBetterThanCurrent(arb, 'BackLay', profitStakePairs)
return getBestProfitStakePair('BackLay', pairsBetterThanCurrent, config)
}
} catch (err) {
logJsonError(__filename, '_init_', err, {
_throw: true
})
}
}
src/matchbook/place-keep-bet.js
const logError = require('./log-error')
const logger = require('./logger/winston')
const api = require('./api')
async function placeKeepBet(bet) {
const betId = bet.id.toString()
const offer = { ... }
const _attempt = ...
try {
logger.info(`[MATCHBOOK OFFER (PLACE-KEEP-BET | ${_attempt} | Bet: ${betId} | Process: ${process.pid})]${JSON.stringify(offer)}`)
const response = await api.submitOffers(offer)
const _offer = response.data.offers[0]
if (_offer.status === 'failed') {
logger.error(`[PLACE BET ERROR (MATCHBOOK | PLACE-KEEP-BET | ${_attempt} | Bet: ${betId} | Process: ${process.pid})]`)
return 'FAILED'
}
if (_offer.status === 'matched') {
logger.info(`[MATCHED (MATCHBOOK | PLACE-KEEP-BET | ${_attempt} | Bet: ${betId} Process: ${process.pid})]`)
return 'MATCHED'
}
if (['flushed', 'cancelled'].includes(_offer.status)) {
logger.info(`[CANCELLED (MATCHBOOK | PLACE-KEEP-BET | ${_attempt} | Bet: ${betId} | Process: ${process.pid})]`)
return 'CANCELLED'
}
if (['delayed', 'open'].includes(_offer.status)) {
const [_stake, _remaining] = [_offer.stake || offer.offers[0].stake, _offer.remaining || offer.offers[0].stake]
const _status = _remaining < _stake ? 'PARTIALLY_MATCHED' : 'UNMATCHED'
logger.info(`[${_status} (MATCHBOOK | PLACE-KEEP-BET | ${_attempt} | Bet: ${betId} | Process: ${process.pid})]`)
return 'UNMATCHED'
}
const _mresponse = await api.getMarket()
const market = _mresponse?.data
if (!['open', 'suspended'].includes(market?.status)) {
return 'ENDED'
}
return 'FAILED'
} catch (err) {
logError(err, __filename)
throw err
}
}
module.exports = placeKeepBet
日志的示例预期行为是：
{"message":"[ATTEMPTING RETRY FOR MATCHBOOK WITH 30.20% LOSS ALLOWED (Process: 174)]","level":"info","timestamp":"2020-12-29 17:38:16"}
{"message":"[MATCHBOOK OFFER (PLACE-KEEP-BET | RETRY | Bet: 1234 | Process: 174)]","level":"info","timestamp":"2020-12-29 17:38:32"}
{"message":"[MATCHED (MATCHBOOK | PLACE-KEEP-BET | RETRY | Bet: 1234 | Process: 174)]","level":"info","timestamp":"2020-12-29 17:38:38"}
{"message":"[EVENT HAS ENDED (Process: 174)]","level":"info","timestamp":"2020-12-29 18:25:01"}
{"message":"[EXITING WITH CODE 0 (Process: 174)]","level":"info","timestamp":"2020-12-29 18:25:01"}
“ EVENT HAS ENDED ...”消息是从轮询所有实时事件状态的主进程记录的。“ EXITING WITH CODE ...”消息由子进程记录，确认主进程已告诉它死亡。
发生此问题时，实际日志为：
{"message":"[ATTEMPTING RETRY FOR MATCHBOOK WITH 30.20% LOSS ALLOWED (Process: 174)]","level":"info","timestamp":"2020-12-29 17:38:16"}
{"message":"[EVENT HAS ENDED (Process: 174)]","level":"info","timestamp":"2020-12-29 18:25:01"}
即使由于无法进行整改而没有记录报价，我仍然希望有更多的日志。我有日志retry.js来通知我发生了什么。超时回调已被调用，但之后没有任何内容。
我在一个工人( src/worker/index.js)的入口点有这些钩子，它们也没有记录：
const logger = require('../logger/winston')
module.exports = async function() {
try {
process.on('exit', code => {
logger.info(`[EXITING WITH CODE ${code} (Process: ${process.pid})]`)
})
process.on('disconnect', () => {
logger.info(`[WORKER DISCONNECTED (Process: ${process.pid})]`)
})
} catch(err) {
// handle error
}
}
在后一个无法正常工作的日志（实际）示例中，主进程已记录事件已结束但子进程尚未结束。这让我觉得子进程已经退出但没有记录它？这可能吗？
值得一提的是，主进程继续记录，而另一个子进程被启动（和记录），而这个进程被停止了。
提供有关“整改过程”的更多信息。该过程是递归的setTimeout。我在上面给出的预期日志示例是“匹配赌注”的另一方立即匹配并且该过程在第一次通过时成功的最佳情况。
然而，情况并非总是如此，可能需要多次迭代才能纠正。此处的日志示例如下：
{"message":"[ATTEMPTING RETRY FOR MATCHBOOK WITH 30.20% LOSS ALLOWED (Process: 174)]","level":"info","timestamp":"2020-12-29 17:38:16"}
{"message":"[NO MATCHBOOK SAME PRICE (RETRY | Bet: 1234 | Process: 174)]","level":"info","timestamp":"2020-12-29 17:38:32"}
{"message":"[ATTEMPTING TRADE OUT FOR SMARKETS WITH 32.10% LOSS ALLOWED (Process: 174)]","level":"info","timestamp":"2020-12-29 17:48:28"}
{"message":"[NO SMARKETS RUNNER (RETRY | Bet: 1234 | Process: 174)]","level":"info","timestamp":"2020-12-29 17:48:34"}
{"message":"[ATTEMPTING RETRY FOR MATCHBOOK WITH 34.50% LOSS ALLOWED (Process: 174)]","level":"info","timestamp":"2020-12-29 17:58:37"}
{"message":"[MATCHBOOK OFFER (PLACE-KEEP-BET | RETRY | Bet: 1234 | Process: 174)]","level":"info","timestamp":"2020-12-29 17:58:38"}
{"message":"[MATCHED (MATCHBOOK | PLACE-KEEP-BET | RETRY | Bet: 1234 | Process: 174)]","level":"info","timestamp":"2020-12-29 17:58:43"}
{"message":"[EVENT HAS ENDED (Process: 174)]","level":"info","timestamp":"2020-12-29 18:25:01"}
{"message":"[EXITING WITH CODE 0 (Process: 174)]","level":"info","timestamp":"2020-12-29 18:25:01"}
编辑：
今晚又发生了这样的事：
这些是今晚体育赛事的日志。这些日志已从我的 UI 中删除。我通过 Loggly API 查询它们。
要消除这些日志中的噪音，您需要知道的是 @ 06/01/2021 21:06:49 “整改过程”是从日志[HANDLING INCOMPLETE IMMEDIATE ARB ...开始的。使用 Matchbook 允许的 48.5% 损失显示了 1 次尝试。在此之后没有进一步的日志。也没有日志表明该工人已经死亡/已断开连接。本次活动直到大约 21:45:00 才结束。
编辑2：
大约 6 小时前再次发生这种情况：
“整改过程”包括“轮流”两种方法；重试或折价。我注意到这只会在retry轮到它时发生，而与使用的交换无关。这让我觉得错误在于retry.js交换文件。我也添加了该trade-out.js文件以进行比较并已编辑retry.js为未编辑的版本。
从我现在在上面添加的所有文件中，它们都包含在try...catch's 中，但仍然没有错误。
编辑 3：
^^ 一个显示应用程序继续启动其他子进程的日志的示例，即使 1 已停止
编辑4：
关于 DO 框上的资源使用情况。CPU 使用率几乎一直以 100% CPU 运行。有 6 个 docker 容器在同一个 DO 盒上运行。其中 5 个只是 API，不会消耗很多资源，但是better，这一个就是这样做的。这些是来自的结果docker stats：
我将资源使用情况作为问题注销的原因是，当 DO 框上的 CPU 使用率仅为 20% 左右docker stats并且显示 CPU 使用率better低于 100% 时，就会发生此问题。运行ps -aux，我得到这个：
正如您可以想象的那样，这些都是已启动以监视每个事件的子进程。生成它们时，我没有指定max-old-space-size选项。集群模块必须这样做。优选地，这些子进程中的每一个应该只消耗 0.5gb (512)。最初运行 docker 容器的命令是：
docker run -d -p ${PORT}:${PORT} --restart always -m ${MEM_LIMIT} --log-opt max-size=${LOG_LIMIT} --init --name ${CI_PROJECT_NAME} --net ${NETWORK_NAME} --ip ${NETWORK_IP} ${CI_REGISTRY}/${CI_PROJECT_PATH}:latest;
MEM_LIMIT是这个5g容器的。我了解资源从长远来看可能是一个问题，但我不能 100% 确定这是一个问题，即使应用程序在 100% 的 CPU 使用率下运行良好而没有打嗝但仅在 20% 的使用率下会导致问题。顺便说一句，内存使用率 > 50%。
解决方案
您可以记录进程状态以识别您的进程之一是否正在崩溃/被暂停？
#!/bin/bash
while :
do
ps aux | grep node | grep -v "grep" | sed "s#^#$(date +%Y/%m/%d/%H:%M) #" >> process_log
sleep 1
done
推荐阅读
- javascript - WASM 文件在 NPM 模块中的位置？
- graphics - 为什么需要为 D3D12 中的每一帧创建 GBuffers？
- javascript - 该脚本在 Mozilla Firefox 中不起作用
- bots - 为什么 Google 的 Lighthouse 机器人（从 Google 的服务器运行时）不使用 gzip 压缩从我的站点加载？
- java - Kafka 生产者读取请求/回复配置的回复消息非常慢
- r - 为什么闪亮不能检测到“观察”之外的“data.tree”操作？
- c# - 解释 8.0.1 中 MvxAttributeViewPresenter.Show 的变化
- c# - 如何解决 InnerText 错误“由于内容不是文字而无法获取内部内容”
- javascript - 从下拉列表中选择文件名时，有没有人看到 VS Code 拖放文件扩展名
- ios - React Native 新手。将 XCode 用于 /iOS 中的本机 iOS 部分。我在根文件夹中使用什么 javascript？什么是最佳实践工作流程？