|
@@ -2241,8 +2241,8 @@ SpirvEvalInfo SPIRVEmitter::doCastExpr(const CastExpr *expr) {
|
|
if (const uint32_t valueId = tryToEvaluateAsConst(expr))
|
|
if (const uint32_t valueId = tryToEvaluateAsConst(expr))
|
|
return SpirvEvalInfo(valueId).setConstant().setRValue();
|
|
return SpirvEvalInfo(valueId).setConstant().setRValue();
|
|
|
|
|
|
- const auto valueId =
|
|
|
|
- castToInt(doExpr(subExpr), subExprType, toType, subExpr->getExprLoc());
|
|
|
|
|
|
+ const auto valueId = castToInt(loadIfGLValue(subExpr), subExprType, toType,
|
|
|
|
+ subExpr->getExprLoc());
|
|
return SpirvEvalInfo(valueId).setRValue();
|
|
return SpirvEvalInfo(valueId).setRValue();
|
|
}
|
|
}
|
|
case CastKind::CK_FloatingCast:
|
|
case CastKind::CK_FloatingCast:
|
|
@@ -6640,9 +6640,7 @@ SpirvEvalInfo SPIRVEmitter::processIntrinsicCallExpr(const CallExpr *callExpr) {
|
|
retVal = processWaveVote(callExpr, spv::Op::OpGroupNonUniformAllEqual);
|
|
retVal = processWaveVote(callExpr, spv::Op::OpGroupNonUniformAllEqual);
|
|
break;
|
|
break;
|
|
case hlsl::IntrinsicOp::IOP_WaveActiveCountBits:
|
|
case hlsl::IntrinsicOp::IOP_WaveActiveCountBits:
|
|
- retVal = processWaveReductionOrPrefix(
|
|
|
|
- callExpr, spv::Op::OpGroupNonUniformBallotBitCount,
|
|
|
|
- spv::GroupOperation::Reduce);
|
|
|
|
|
|
+ retVal = processWaveCountBits(callExpr, spv::GroupOperation::Reduce);
|
|
break;
|
|
break;
|
|
case hlsl::IntrinsicOp::IOP_WaveActiveUSum:
|
|
case hlsl::IntrinsicOp::IOP_WaveActiveUSum:
|
|
case hlsl::IntrinsicOp::IOP_WaveActiveSum:
|
|
case hlsl::IntrinsicOp::IOP_WaveActiveSum:
|
|
@@ -6670,9 +6668,7 @@ SpirvEvalInfo SPIRVEmitter::processIntrinsicCallExpr(const CallExpr *callExpr) {
|
|
spv::GroupOperation::ExclusiveScan);
|
|
spv::GroupOperation::ExclusiveScan);
|
|
} break;
|
|
} break;
|
|
case hlsl::IntrinsicOp::IOP_WavePrefixCountBits:
|
|
case hlsl::IntrinsicOp::IOP_WavePrefixCountBits:
|
|
- retVal = processWaveReductionOrPrefix(
|
|
|
|
- callExpr, spv::Op::OpGroupNonUniformBallotBitCount,
|
|
|
|
- spv::GroupOperation::ExclusiveScan);
|
|
|
|
|
|
+ retVal = processWaveCountBits(callExpr, spv::GroupOperation::ExclusiveScan);
|
|
break;
|
|
break;
|
|
case hlsl::IntrinsicOp::IOP_WaveReadLaneAt:
|
|
case hlsl::IntrinsicOp::IOP_WaveReadLaneAt:
|
|
case hlsl::IntrinsicOp::IOP_WaveReadLaneFirst:
|
|
case hlsl::IntrinsicOp::IOP_WaveReadLaneFirst:
|
|
@@ -7106,7 +7102,8 @@ uint32_t SPIRVEmitter::processWaveQuery(const CallExpr *callExpr,
|
|
featureManager.requestTargetEnv(SPV_ENV_VULKAN_1_1, "Wave Operation",
|
|
featureManager.requestTargetEnv(SPV_ENV_VULKAN_1_1, "Wave Operation",
|
|
callExpr->getExprLoc());
|
|
callExpr->getExprLoc());
|
|
theBuilder.requireCapability(getCapabilityForGroupNonUniform(opcode));
|
|
theBuilder.requireCapability(getCapabilityForGroupNonUniform(opcode));
|
|
- const uint32_t subgroupScope = theBuilder.getConstantInt32(3);
|
|
|
|
|
|
+ const uint32_t subgroupScope =
|
|
|
|
+ theBuilder.getConstantInt32(static_cast<int32_t>(spv::Scope::Subgroup));
|
|
const uint32_t retType =
|
|
const uint32_t retType =
|
|
typeTranslator.translateType(callExpr->getCallReturnType(astContext));
|
|
typeTranslator.translateType(callExpr->getCallReturnType(astContext));
|
|
return theBuilder.createGroupNonUniformOp(opcode, retType, subgroupScope);
|
|
return theBuilder.createGroupNonUniformOp(opcode, retType, subgroupScope);
|
|
@@ -7123,7 +7120,8 @@ uint32_t SPIRVEmitter::processWaveVote(const CallExpr *callExpr,
|
|
callExpr->getExprLoc());
|
|
callExpr->getExprLoc());
|
|
theBuilder.requireCapability(getCapabilityForGroupNonUniform(opcode));
|
|
theBuilder.requireCapability(getCapabilityForGroupNonUniform(opcode));
|
|
const uint32_t predicate = doExpr(callExpr->getArg(0));
|
|
const uint32_t predicate = doExpr(callExpr->getArg(0));
|
|
- const uint32_t subgroupScope = theBuilder.getConstantInt32(3);
|
|
|
|
|
|
+ const uint32_t subgroupScope =
|
|
|
|
+ theBuilder.getConstantInt32(static_cast<int32_t>(spv::Scope::Subgroup));
|
|
const uint32_t retType =
|
|
const uint32_t retType =
|
|
typeTranslator.translateType(callExpr->getCallReturnType(astContext));
|
|
typeTranslator.translateType(callExpr->getCallReturnType(astContext));
|
|
return theBuilder.createGroupNonUniformUnaryOp(opcode, retType, subgroupScope,
|
|
return theBuilder.createGroupNonUniformUnaryOp(opcode, retType, subgroupScope,
|
|
@@ -7199,11 +7197,39 @@ spv::Op SPIRVEmitter::translateWaveOp(hlsl::IntrinsicOp op, QualType type,
|
|
return spv::Op::OpNop;
|
|
return spv::Op::OpNop;
|
|
}
|
|
}
|
|
|
|
|
|
|
|
+uint32_t SPIRVEmitter::processWaveCountBits(const CallExpr *callExpr,
|
|
|
|
+ spv::GroupOperation groupOp) {
|
|
|
|
+ // Signatures:
|
|
|
|
+ // uint WaveActiveCountBits(bool bBit)
|
|
|
|
+ // uint WavePrefixCountBits(Bool bBit)
|
|
|
|
+ assert(callExpr->getNumArgs() == 1);
|
|
|
|
+
|
|
|
|
+ featureManager.requestTargetEnv(SPV_ENV_VULKAN_1_1, "Wave Operation",
|
|
|
|
+ callExpr->getExprLoc());
|
|
|
|
+ theBuilder.requireCapability(getCapabilityForGroupNonUniform(
|
|
|
|
+ spv::Op::OpGroupNonUniformBallotBitCount));
|
|
|
|
+
|
|
|
|
+ const uint32_t predicate = doExpr(callExpr->getArg(0));
|
|
|
|
+ const uint32_t subgroupScope =
|
|
|
|
+ theBuilder.getConstantInt32(static_cast<int32_t>(spv::Scope::Subgroup));
|
|
|
|
+
|
|
|
|
+ const uint32_t u32Type = theBuilder.getUint32Type();
|
|
|
|
+ const uint32_t v4u32Type = theBuilder.getVecType(u32Type, 4);
|
|
|
|
+ const uint32_t retType =
|
|
|
|
+ typeTranslator.translateType(callExpr->getCallReturnType(astContext));
|
|
|
|
+
|
|
|
|
+ const uint32_t ballot = theBuilder.createGroupNonUniformUnaryOp(
|
|
|
|
+ spv::Op::OpGroupNonUniformBallot, v4u32Type, subgroupScope, predicate);
|
|
|
|
+
|
|
|
|
+ return theBuilder.createGroupNonUniformUnaryOp(
|
|
|
|
+ spv::Op::OpGroupNonUniformBallotBitCount, retType, subgroupScope, ballot,
|
|
|
|
+ llvm::Optional<spv::GroupOperation>(groupOp));
|
|
|
|
+}
|
|
|
|
+
|
|
uint32_t SPIRVEmitter::processWaveReductionOrPrefix(
|
|
uint32_t SPIRVEmitter::processWaveReductionOrPrefix(
|
|
const CallExpr *callExpr, spv::Op opcode, spv::GroupOperation groupOp) {
|
|
const CallExpr *callExpr, spv::Op opcode, spv::GroupOperation groupOp) {
|
|
// Signatures:
|
|
// Signatures:
|
|
// bool WaveActiveAllEqual( <type> expr )
|
|
// bool WaveActiveAllEqual( <type> expr )
|
|
- // uint WaveActiveCountBits( bool bBit )
|
|
|
|
// <type> WaveActiveSum( <type> expr )
|
|
// <type> WaveActiveSum( <type> expr )
|
|
// <type> WaveActiveProduct( <type> expr )
|
|
// <type> WaveActiveProduct( <type> expr )
|
|
// <int_type> WaveActiveBitAnd( <int_type> expr )
|
|
// <int_type> WaveActiveBitAnd( <int_type> expr )
|
|
@@ -7212,7 +7238,6 @@ uint32_t SPIRVEmitter::processWaveReductionOrPrefix(
|
|
// <type> WaveActiveMin( <type> expr)
|
|
// <type> WaveActiveMin( <type> expr)
|
|
// <type> WaveActiveMax( <type> expr)
|
|
// <type> WaveActiveMax( <type> expr)
|
|
//
|
|
//
|
|
- // uint WavePrefixCountBits(Bool bBit)
|
|
|
|
// <type> WavePrefixProduct(<type> value)
|
|
// <type> WavePrefixProduct(<type> value)
|
|
// <type> WavePrefixSum(<type> value)
|
|
// <type> WavePrefixSum(<type> value)
|
|
assert(callExpr->getNumArgs() == 1);
|
|
assert(callExpr->getNumArgs() == 1);
|
|
@@ -7220,7 +7245,8 @@ uint32_t SPIRVEmitter::processWaveReductionOrPrefix(
|
|
callExpr->getExprLoc());
|
|
callExpr->getExprLoc());
|
|
theBuilder.requireCapability(getCapabilityForGroupNonUniform(opcode));
|
|
theBuilder.requireCapability(getCapabilityForGroupNonUniform(opcode));
|
|
const uint32_t predicate = doExpr(callExpr->getArg(0));
|
|
const uint32_t predicate = doExpr(callExpr->getArg(0));
|
|
- const uint32_t subgroupScope = theBuilder.getConstantInt32(3);
|
|
|
|
|
|
+ const uint32_t subgroupScope =
|
|
|
|
+ theBuilder.getConstantInt32(static_cast<int32_t>(spv::Scope::Subgroup));
|
|
const uint32_t retType =
|
|
const uint32_t retType =
|
|
typeTranslator.translateType(callExpr->getCallReturnType(astContext));
|
|
typeTranslator.translateType(callExpr->getCallReturnType(astContext));
|
|
return theBuilder.createGroupNonUniformUnaryOp(
|
|
return theBuilder.createGroupNonUniformUnaryOp(
|
|
@@ -7238,7 +7264,8 @@ uint32_t SPIRVEmitter::processWaveBroadcast(const CallExpr *callExpr) {
|
|
callExpr->getExprLoc());
|
|
callExpr->getExprLoc());
|
|
theBuilder.requireCapability(spv::Capability::GroupNonUniformBallot);
|
|
theBuilder.requireCapability(spv::Capability::GroupNonUniformBallot);
|
|
const uint32_t value = doExpr(callExpr->getArg(0));
|
|
const uint32_t value = doExpr(callExpr->getArg(0));
|
|
- const uint32_t subgroupScope = theBuilder.getConstantInt32(3);
|
|
|
|
|
|
+ const uint32_t subgroupScope =
|
|
|
|
+ theBuilder.getConstantInt32(static_cast<int32_t>(spv::Scope::Subgroup));
|
|
const uint32_t retType =
|
|
const uint32_t retType =
|
|
typeTranslator.translateType(callExpr->getCallReturnType(astContext));
|
|
typeTranslator.translateType(callExpr->getCallReturnType(astContext));
|
|
if (numArgs == 2)
|
|
if (numArgs == 2)
|
|
@@ -7264,7 +7291,8 @@ uint32_t SPIRVEmitter::processWaveQuadWideShuffle(const CallExpr *callExpr,
|
|
theBuilder.requireCapability(spv::Capability::GroupNonUniformQuad);
|
|
theBuilder.requireCapability(spv::Capability::GroupNonUniformQuad);
|
|
|
|
|
|
const uint32_t value = doExpr(callExpr->getArg(0));
|
|
const uint32_t value = doExpr(callExpr->getArg(0));
|
|
- const uint32_t subgroupScope = theBuilder.getConstantInt32(3);
|
|
|
|
|
|
+ const uint32_t subgroupScope =
|
|
|
|
+ theBuilder.getConstantInt32(static_cast<int32_t>(spv::Scope::Subgroup));
|
|
const uint32_t retType =
|
|
const uint32_t retType =
|
|
typeTranslator.translateType(callExpr->getCallReturnType(astContext));
|
|
typeTranslator.translateType(callExpr->getCallReturnType(astContext));
|
|
|
|
|