Add filter-provided captcha score thresholds

This commit is contained in:
Ravi Khadiwala
2023-03-21 17:26:42 -05:00
committed by ravi-signal
parent a8eb27940d
commit ee53260d72
16 changed files with 280 additions and 50 deletions

View File

@@ -5,23 +5,111 @@
package org.whispersystems.textsecuregcm.captcha;
/**
* A captcha assessment
*
* @param valid whether the captcha was passed
* @param score string representation of the risk level
*/
public record AssessmentResult(boolean valid, String score) {
import java.util.Objects;
import java.util.Optional;
public static AssessmentResult invalid() {
return new AssessmentResult(false, "");
public class AssessmentResult {
private final boolean solved;
private final float actualScore;
private final float defaultScoreThreshold;
private final String scoreString;
/**
* A captcha assessment
*
* @param solved if false, the captcha was not successfully completed
* @param actualScore float representation of the risk level from [0, 1.0], with 1.0 being the least risky
* @param defaultScoreThreshold the score threshold which the score will be evaluated against by default
* @param scoreString a quantized string representation of the risk level, suitable for use in metrics
*/
private AssessmentResult(boolean solved, float actualScore, float defaultScoreThreshold, final String scoreString) {
this.solved = solved;
this.actualScore = actualScore;
this.defaultScoreThreshold = defaultScoreThreshold;
this.scoreString = scoreString;
}
/**
* Construct an {@link AssessmentResult} from a captcha evaluation score
*
* @param actualScore the score
* @param defaultScoreThreshold the threshold to compare the score against by default
*/
public static AssessmentResult fromScore(float actualScore, float defaultScoreThreshold) {
if (actualScore < 0 || actualScore > 1.0 || defaultScoreThreshold < 0 || defaultScoreThreshold > 1.0) {
throw new IllegalArgumentException("invalid captcha score");
}
return new AssessmentResult(true, actualScore, defaultScoreThreshold, AssessmentResult.scoreString(actualScore));
}
/**
* Construct a captcha assessment that will always be invalid
*/
public static AssessmentResult invalid() {
return new AssessmentResult(false, 0.0f, 0.0f, "");
}
/**
* Construct a captcha assessment that will always be valid
*/
public static AssessmentResult alwaysValid() {
return new AssessmentResult(true, 1.0f, 0.0f, "1.0");
}
/**
* Check if the captcha assessment should be accepted using the default score threshold
*
* @return true if this assessment should be accepted under the default score threshold
*/
public boolean isValid() {
return isValid(Optional.empty());
}
/**
* Check if the captcha assessment should be accepted
*
* @param scoreThreshold the minimum score the assessment requires to pass, uses default if empty
* @return true if the assessment scored higher than the provided scoreThreshold
*/
public boolean isValid(Optional<Float> scoreThreshold) {
if (!solved) {
return false;
}
return this.actualScore >= scoreThreshold.orElse(this.defaultScoreThreshold);
}
public String getScoreString() {
return scoreString;
}
public float getScore() {
return this.actualScore;
}
/**
* Map a captcha score in [0.0, 1.0] to a low cardinality discrete space in [0, 100] suitable for use in metrics
*/
static String scoreString(final float score) {
private static String scoreString(final float score) {
final int x = Math.round(score * 10); // [0, 10]
return Integer.toString(x * 10); // [0, 100] in increments of 10
}
@Override
public boolean equals(final Object o) {
if (this == o)
return true;
if (o == null || getClass() != o.getClass())
return false;
AssessmentResult that = (AssessmentResult) o;
return solved == that.solved && Float.compare(that.actualScore, actualScore) == 0
&& Float.compare(that.defaultScoreThreshold, defaultScoreThreshold) == 0 && Objects.equals(scoreString,
that.scoreString);
}
@Override
public int hashCode() {
return Objects.hash(solved, actualScore, defaultScoreThreshold, scoreString);
}
}

View File

@@ -91,8 +91,7 @@ public class CaptchaChecker {
final AssessmentResult result = client.verify(siteKey, parsedAction, token, ip);
Metrics.counter(ASSESSMENTS_COUNTER_NAME,
"action", action,
"valid", String.valueOf(result.valid()),
"score", result.score(),
"score", result.getScoreString(),
"provider", prefix)
.increment();
return result;

View File

@@ -33,7 +33,6 @@ public class HCaptchaClient implements CaptchaClient {
private static final String PREFIX = "signal-hcaptcha";
private static final String ASSESSMENT_REASON_COUNTER_NAME = name(HCaptchaClient.class, "assessmentReason");
private static final String INVALID_REASON_COUNTER_NAME = name(HCaptchaClient.class, "invalidReason");
private static final String INVALID_SITEKEY_COUNTER_NAME = name(HCaptchaClient.class, "invalidSiteKey");
private final String apiKey;
private final HttpClient client;
private final DynamicConfigurationManager<DynamicConfiguration> dynamicConfigurationManager;
@@ -115,16 +114,15 @@ public class HCaptchaClient implements CaptchaClient {
logger.error("Invalid score {} from hcaptcha response {}", hCaptchaResponse.score, hCaptchaResponse);
return AssessmentResult.invalid();
}
final String scoreString = AssessmentResult.scoreString(score);
final BigDecimal threshold = config.getScoreFloorByAction().getOrDefault(action, config.getScoreFloor());
final AssessmentResult assessmentResult = AssessmentResult.fromScore(score, threshold.floatValue());
for (String reason : hCaptchaResponse.scoreReasons) {
Metrics.counter(ASSESSMENT_REASON_COUNTER_NAME,
"action", action.getActionName(),
"reason", reason,
"score", scoreString).increment();
"score", assessmentResult.getScoreString()).increment();
}
final BigDecimal threshold = config.getScoreFloorByAction().getOrDefault(action, config.getScoreFloor());
return new AssessmentResult(score >= threshold.floatValue(), scoreString);
return assessmentResult;
}
}

View File

@@ -113,17 +113,16 @@ public class RecaptchaClient implements CaptchaClient {
if (assessment.getTokenProperties().getValid()) {
final float score = assessment.getRiskAnalysis().getScore();
log.debug("assessment for {} was valid, score: {}", action.getActionName(), score);
final BigDecimal threshold = config.getScoreFloorByAction().getOrDefault(action, config.getScoreFloor());
final AssessmentResult assessmentResult = AssessmentResult.fromScore(score, threshold.floatValue());
for (RiskAnalysis.ClassificationReason reason : assessment.getRiskAnalysis().getReasonsList()) {
Metrics.counter(ASSESSMENT_REASON_COUNTER_NAME,
"action", action.getActionName(),
"score", AssessmentResult.scoreString(score),
"score", assessmentResult.getScoreString(),
"reason", reason.name())
.increment();
}
final BigDecimal threshold = config.getScoreFloorByAction().getOrDefault(action, config.getScoreFloor());
return new AssessmentResult(
score >= threshold.floatValue(),
AssessmentResult.scoreString(score));
return assessmentResult;
} else {
Metrics.counter(INVALID_REASON_COUNTER_NAME,
"action", action.getActionName(),