diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java index 6ba137e82..8230cff32 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/WhisperServerService.java @@ -234,7 +234,6 @@ import org.whispersystems.textsecuregcm.redis.ConnectionEventLogger; import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClient; import org.whispersystems.textsecuregcm.redis.FaultTolerantRedisClusterClient; import org.whispersystems.textsecuregcm.registration.RegistrationServiceClient; -import org.whispersystems.textsecuregcm.s3.PolicySigner; import org.whispersystems.textsecuregcm.s3.PostPolicyGenerator; import org.whispersystems.textsecuregcm.s3.S3MonitoringSupplier; import org.whispersystems.textsecuregcm.securestorage.SecureStorageClient; @@ -898,11 +897,15 @@ public class WhisperServerService extends Application badgeConfigurationMap; - private final PolicySigner policySigner; private final PostPolicyGenerator policyGenerator; private final ServerSecretParams serverSecretParams; private final ServerZkProfileOperations zkProfileOperations; @@ -135,7 +132,6 @@ public class ProfileController { ProfileBadgeConverter profileBadgeConverter, BadgesConfiguration badgesConfiguration, PostPolicyGenerator policyGenerator, - PolicySigner policySigner, ServerSecretParams serverSecretParams, ServerZkProfileOperations zkProfileOperations, Executor batchIdentityCheckExecutor) { @@ -150,7 +146,6 @@ public class ProfileController { this.serverSecretParams = serverSecretParams; this.zkProfileOperations = zkProfileOperations; this.policyGenerator = policyGenerator; - this.policySigner = policySigner; this.batchIdentityCheckExecutor = Preconditions.checkNotNull(batchIdentityCheckExecutor); } @@ -553,15 +548,13 @@ public class ProfileController { return maybeTargetAccount.get(); } - private ProfileAvatarUploadAttributes generateAvatarUploadForm( - final String objectName) { - ZonedDateTime now = ZonedDateTime.now(clock); - Pair policy = policyGenerator.createFor(now, objectName, ProfileHelper.MAX_PROFILE_AVATAR_SIZE_BYTES); - String signature = policySigner.getSignature(now, policy.second()); + private ProfileAvatarUploadAttributes generateAvatarUploadForm(final String objectName) { + final PostPolicyGenerator.SignedPostPolicy signedPostPolicy = + policyGenerator.createFor(objectName, ProfileHelper.MAX_PROFILE_AVATAR_SIZE_BYTES, clock.instant()); - return new ProfileAvatarUploadAttributes(objectName, policy.first(), - "private", "AWS4-HMAC-SHA256", - now.format(PostPolicyGenerator.AWS_DATE_TIME), policy.second(), signature); + return new ProfileAvatarUploadAttributes(objectName, signedPostPolicy.credential(), + PostPolicyGenerator.ACL, PostPolicyGenerator.ALGORITHM, + signedPostPolicy.formattedTimestamp(), signedPostPolicy.encodedPolicy(), signedPostPolicy.signature()); } private static Map getAccountCapabilities(final Account account) { diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/controllers/StickerController.java b/service/src/main/java/org/whispersystems/textsecuregcm/controllers/StickerController.java index 0fd1a23a2..835566abe 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/controllers/StickerController.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/controllers/StickerController.java @@ -6,6 +6,7 @@ package org.whispersystems.textsecuregcm.controllers; import io.dropwizard.auth.Auth; +import io.dropwizard.util.DataSize; import io.swagger.v3.oas.annotations.tags.Tag; import jakarta.validation.constraints.Max; import jakarta.validation.constraints.Min; @@ -15,8 +16,8 @@ import jakarta.ws.rs.PathParam; import jakarta.ws.rs.Produces; import jakarta.ws.rs.core.MediaType; import java.security.SecureRandom; -import java.time.ZoneOffset; -import java.time.ZonedDateTime; +import java.time.Clock; +import java.time.Instant; import java.util.HexFormat; import java.util.LinkedList; import java.util.List; @@ -24,23 +25,25 @@ import org.whispersystems.textsecuregcm.auth.AuthenticatedDevice; import org.whispersystems.textsecuregcm.entities.StickerPackFormUploadAttributes; import org.whispersystems.textsecuregcm.entities.StickerPackFormUploadAttributes.StickerPackFormUploadItem; import org.whispersystems.textsecuregcm.limits.RateLimiters; -import org.whispersystems.textsecuregcm.s3.PolicySigner; import org.whispersystems.textsecuregcm.s3.PostPolicyGenerator; -import org.whispersystems.textsecuregcm.util.Constants; -import org.whispersystems.textsecuregcm.util.Pair; @Path("/v1/sticker") @Tag(name = "Stickers") public class StickerController { private final RateLimiters rateLimiters; - private final PolicySigner policySigner; private final PostPolicyGenerator policyGenerator; + private final Clock clock; - public StickerController(RateLimiters rateLimiters, String accessKey, String accessSecret, String region, String bucket) { + public static final int MAXIMUM_STICKER_SIZE_BYTES = (int) DataSize.kibibytes(300 + 1).toBytes(); // add 1 kiB for encryption overhead + public static final int MAXIMUM_STICKER_MANIFEST_SIZE_BYTES = (int) DataSize.kibibytes(10).toBytes(); + + public StickerController(final RateLimiters rateLimiters, + final PostPolicyGenerator postPolicyGenerator, + final Clock clock) { this.rateLimiters = rateLimiters; - this.policySigner = new PolicySigner(accessSecret, region); - this.policyGenerator = new PostPolicyGenerator(region, bucket, accessKey); + this.policyGenerator = postPolicyGenerator; + this.clock = clock; } @GET @@ -51,33 +54,32 @@ public class StickerController { throws RateLimitExceededException { rateLimiters.getStickerPackLimiter().validate(auth.accountIdentifier()); - ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC); - String packId = generatePackId(); - String packLocation = "stickers/" + packId; - String manifestKey = packLocation + "/manifest.proto"; - Pair manifestPolicy = policyGenerator.createFor(now, manifestKey, - Constants.MAXIMUM_STICKER_MANIFEST_SIZE_BYTES); - String manifestSignature = policySigner.getSignature(now, manifestPolicy.second()); - StickerPackFormUploadItem manifest = new StickerPackFormUploadItem(-1, manifestKey, manifestPolicy.first(), - "private", "AWS4-HMAC-SHA256", - now.format(PostPolicyGenerator.AWS_DATE_TIME), manifestPolicy.second(), manifestSignature); + final Instant currentTime = clock.instant(); + final String packId = generatePackId(); + final String packLocation = "stickers/" + packId; + final String manifestKey = packLocation + "/manifest.proto"; + final PostPolicyGenerator.SignedPostPolicy manifestPolicy = + policyGenerator.createFor(manifestKey, MAXIMUM_STICKER_MANIFEST_SIZE_BYTES, currentTime); - List stickers = new LinkedList<>(); + final StickerPackFormUploadItem manifest = new StickerPackFormUploadItem(-1, manifestKey, manifestPolicy.credential(), + PostPolicyGenerator.ACL, PostPolicyGenerator.ALGORITHM, + manifestPolicy.formattedTimestamp(), manifestPolicy.encodedPolicy(), manifestPolicy.signature()); + + final List stickers = new LinkedList<>(); for (int i = 0; i < stickerCount; i++) { - String stickerKey = packLocation + "/full/" + i; - Pair stickerPolicy = policyGenerator.createFor(now, stickerKey, - Constants.MAXIMUM_STICKER_SIZE_BYTES); - String stickerSignature = policySigner.getSignature(now, stickerPolicy.second()); - stickers.add(new StickerPackFormUploadItem(i, stickerKey, stickerPolicy.first(), "private", "AWS4-HMAC-SHA256", - now.format(PostPolicyGenerator.AWS_DATE_TIME), stickerPolicy.second(), stickerSignature)); + final String stickerKey = packLocation + "/full/" + i; + final PostPolicyGenerator.SignedPostPolicy stickerPolicy = + policyGenerator.createFor(stickerKey, MAXIMUM_STICKER_SIZE_BYTES, currentTime); + stickers.add(new StickerPackFormUploadItem(i, stickerKey, stickerPolicy.credential(), PostPolicyGenerator.ACL, PostPolicyGenerator.ALGORITHM, + manifestPolicy.formattedTimestamp(), stickerPolicy.encodedPolicy(), stickerPolicy.signature())); } return new StickerPackFormUploadAttributes(packId, manifest, stickers); } private String generatePackId() { - byte[] object = new byte[16]; + final byte[] object = new byte[16]; new SecureRandom().nextBytes(object); return HexFormat.of().formatHex(object); diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/grpc/AttachmentsGrpcService.java b/service/src/main/java/org/whispersystems/textsecuregcm/grpc/AttachmentsGrpcService.java index 56b8abf80..58f01547c 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/grpc/AttachmentsGrpcService.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/grpc/AttachmentsGrpcService.java @@ -6,10 +6,17 @@ package org.whispersystems.textsecuregcm.grpc; import java.security.SecureRandom; +import java.time.Clock; +import java.time.Instant; +import java.util.HexFormat; import java.util.Map; +import java.util.stream.IntStream; +import org.signal.chat.attachments.GetStickerUploadFormRequest; +import org.signal.chat.attachments.GetStickerUploadFormResponse; import org.signal.chat.attachments.GetUploadFormRequest; import org.signal.chat.attachments.GetUploadFormResponse; import org.signal.chat.attachments.SimpleAttachmentsGrpc; +import org.signal.chat.common.S3UploadForm; import org.signal.chat.common.UploadForm; import org.signal.chat.errors.FailedPrecondition; import org.whispersystems.textsecuregcm.attachments.AttachmentGenerator; @@ -19,29 +26,44 @@ import org.whispersystems.textsecuregcm.attachments.TusAttachmentGenerator; import org.whispersystems.textsecuregcm.auth.grpc.AuthenticatedDevice; import org.whispersystems.textsecuregcm.auth.grpc.AuthenticationUtil; import org.whispersystems.textsecuregcm.controllers.RateLimitExceededException; +import org.whispersystems.textsecuregcm.controllers.StickerController; import org.whispersystems.textsecuregcm.experiment.ExperimentEnrollmentManager; import org.whispersystems.textsecuregcm.limits.RateLimiter; import org.whispersystems.textsecuregcm.limits.RateLimiters; +import org.whispersystems.textsecuregcm.s3.PostPolicyGenerator; public class AttachmentsGrpcService extends SimpleAttachmentsGrpc.AttachmentsImplBase { private final ExperimentEnrollmentManager experimentEnrollmentManager; private final RateLimiter countRateLimiter; private final RateLimiter bytesRateLimiter; + private final RateLimiter stickerPackLimiter; private final long maxUploadLength; private final Map attachmentGenerators; + private final PostPolicyGenerator stickerPolicyGenerator; + private final Clock clock; private final SecureRandom secureRandom; + private static final S3UploadForm PROTOTYPE_STICKER_UPLOAD_FORM = S3UploadForm.newBuilder() + .setAcl(PostPolicyGenerator.ACL) + .setAlgorithm(PostPolicyGenerator.ALGORITHM) + .build(); + public AttachmentsGrpcService( final ExperimentEnrollmentManager experimentEnrollmentManager, final RateLimiters rateLimiters, final GcsAttachmentGenerator gcsAttachmentGenerator, final TusAttachmentGenerator tusAttachmentGenerator, - final long maxUploadLength) { + final PostPolicyGenerator stickerPolicyGenerator, + final long maxUploadLength, + final Clock clock) { this.experimentEnrollmentManager = experimentEnrollmentManager; this.countRateLimiter = rateLimiters.getAttachmentLimiter(); this.bytesRateLimiter = rateLimiters.getAttachmentBytesLimiter(); + this.stickerPackLimiter = rateLimiters.getStickerPackLimiter(); + this.stickerPolicyGenerator = stickerPolicyGenerator; this.maxUploadLength = maxUploadLength; + this.clock = clock; this.secureRandom = new SecureRandom(); this.attachmentGenerators = Map.of( 2, gcsAttachmentGenerator, @@ -80,4 +102,59 @@ public class AttachmentsGrpcService extends SimpleAttachmentsGrpc.AttachmentsImp .setSignedUploadLocation(descriptor.signedUploadLocation())) .build(); } + + @Override + public GetStickerUploadFormResponse getStickerUploadForm(final GetStickerUploadFormRequest request) + throws RateLimitExceededException { + + stickerPackLimiter.validate(AuthenticationUtil.requireAuthenticatedDevice().accountIdentifier()); + + final Instant currentTime = clock.instant(); + final String packId; + { + final byte[] packIdBytes = new byte[16]; + secureRandom.nextBytes(packIdBytes); + + packId = HexFormat.of().formatHex(packIdBytes); + } + + final String packLocation = "stickers/" + packId; + + final GetStickerUploadFormResponse.Builder responseBuilder = GetStickerUploadFormResponse.newBuilder() + .setPackId(packId); + + { + final String manifestKey = packLocation + "/manifest.proto"; + + final PostPolicyGenerator.SignedPostPolicy manifestPolicy = + stickerPolicyGenerator.createFor(manifestKey, StickerController.MAXIMUM_STICKER_MANIFEST_SIZE_BYTES, currentTime); + + responseBuilder + .setManifestUploadForm(PROTOTYPE_STICKER_UPLOAD_FORM.toBuilder() + .setKey(manifestKey) + .setCredential(manifestPolicy.credential()) + .setDate(manifestPolicy.formattedTimestamp()) + .setPolicy(manifestPolicy.encodedPolicy()) + .setSignature(manifestPolicy.signature()) + .build()); + } + + IntStream.range(0, request.getStickerCount()) + .mapToObj(i -> { + final String stickerKey = packLocation + "/full/" + i; + final PostPolicyGenerator.SignedPostPolicy stickerPolicy = + stickerPolicyGenerator.createFor(stickerKey, StickerController.MAXIMUM_STICKER_SIZE_BYTES, currentTime); + + return PROTOTYPE_STICKER_UPLOAD_FORM.toBuilder() + .setKey(stickerKey) + .setCredential(stickerPolicy.credential()) + .setDate(stickerPolicy.formattedTimestamp()) + .setPolicy(stickerPolicy.encodedPolicy()) + .setSignature(stickerPolicy.signature()) + .build(); + }) + .forEach(responseBuilder::addStickerUploadForms); + + return responseBuilder.build(); + } } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/grpc/ProfileGrpcService.java b/service/src/main/java/org/whispersystems/textsecuregcm/grpc/ProfileGrpcService.java index 63bd6604f..26d92109d 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/grpc/ProfileGrpcService.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/grpc/ProfileGrpcService.java @@ -5,9 +5,7 @@ package org.whispersystems.textsecuregcm.grpc; -import com.google.protobuf.ByteString; import java.time.Clock; -import java.time.ZonedDateTime; import java.util.Arrays; import java.util.HexFormat; import java.util.List; @@ -16,6 +14,7 @@ import java.util.Optional; import java.util.UUID; import java.util.function.Function; import java.util.stream.Collectors; +import org.signal.chat.common.S3UploadForm; import org.signal.chat.errors.FailedPrecondition; import org.signal.chat.errors.NotFound; import org.signal.chat.profile.GetUnversionedProfileRequest; @@ -23,7 +22,6 @@ import org.signal.chat.profile.GetUnversionedProfileResponse; import org.signal.chat.profile.GetVersionedProfileRequest; import org.signal.chat.profile.GetVersionedProfileResponse; import org.signal.chat.profile.PaymentsForbiddenInRegion; -import org.signal.chat.profile.ProfileAvatarUploadAttributes; import org.signal.chat.profile.ProfilesV2CapabilityRequired; import org.signal.chat.profile.SetProfileRequest; import org.signal.chat.profile.SetProfileResponse; @@ -40,7 +38,6 @@ import org.whispersystems.textsecuregcm.controllers.RateLimitExceededException; import org.whispersystems.textsecuregcm.identity.IdentityType; import org.whispersystems.textsecuregcm.identity.ServiceIdentifier; import org.whispersystems.textsecuregcm.limits.RateLimiters; -import org.whispersystems.textsecuregcm.s3.PolicySigner; import org.whispersystems.textsecuregcm.s3.PostPolicyGenerator; import org.whispersystems.textsecuregcm.storage.Account; import org.whispersystems.textsecuregcm.storage.AccountBadge; @@ -51,7 +48,6 @@ import org.whispersystems.textsecuregcm.storage.ProfilesManager; import org.whispersystems.textsecuregcm.storage.VersionedProfile; import org.whispersystems.textsecuregcm.storage.VersionedProfileV1; import org.whispersystems.textsecuregcm.storage.WriteConflictException; -import org.whispersystems.textsecuregcm.util.Pair; import org.whispersystems.textsecuregcm.util.ProfileHelper; public class ProfileGrpcService extends SimpleProfileGrpc.ProfileImplBase { @@ -62,13 +58,17 @@ public class ProfileGrpcService extends SimpleProfileGrpc.ProfileImplBase { private final DynamicConfigurationManager dynamicConfigurationManager; private final Map badgeConfigurationMap; private final PostPolicyGenerator policyGenerator; - private final PolicySigner policySigner; private final ProfileBadgeConverter profileBadgeConverter; private final RateLimiters rateLimiters; + private static final S3UploadForm PROTOTYPE_AVATAR_UPLOAD_FORM = S3UploadForm.newBuilder() + .setAcl(PostPolicyGenerator.ACL) + .setAlgorithm(PostPolicyGenerator.ALGORITHM) + .build(); + private record AvatarData(Optional currentAvatar, Optional finalAvatar, - Optional uploadAttributes) {} + Optional uploadAttributes) {} public ProfileGrpcService( final Clock clock, @@ -77,7 +77,6 @@ public class ProfileGrpcService extends SimpleProfileGrpc.ProfileImplBase { final DynamicConfigurationManager dynamicConfigurationManager, final BadgesConfiguration badgesConfiguration, final PostPolicyGenerator policyGenerator, - final PolicySigner policySigner, final ProfileBadgeConverter profileBadgeConverter, final RateLimiters rateLimiters) { this.clock = clock; @@ -87,7 +86,6 @@ public class ProfileGrpcService extends SimpleProfileGrpc.ProfileImplBase { this.badgeConfigurationMap = badgesConfiguration.getBadges().stream().collect(Collectors.toMap( BadgeConfiguration::getId, Function.identity())); this.policyGenerator = policyGenerator; - this.policySigner = policySigner; this.profileBadgeConverter = profileBadgeConverter; this.rateLimiters = rateLimiters; } @@ -266,19 +264,16 @@ public class ProfileGrpcService extends SimpleProfileGrpc.ProfileImplBase { } } - private ProfileAvatarUploadAttributes generateAvatarUploadForm(final String objectName) { - final ZonedDateTime now = ZonedDateTime.now(clock); - final Pair policy = policyGenerator.createFor(now, objectName, ProfileHelper.MAX_PROFILE_AVATAR_SIZE_BYTES); - final String signature = policySigner.getSignature(now, policy.second()); + private S3UploadForm generateAvatarUploadForm(final String objectName) { + final PostPolicyGenerator.SignedPostPolicy policy = + policyGenerator.createFor(objectName, ProfileHelper.MAX_PROFILE_AVATAR_SIZE_BYTES, clock.instant()); - return ProfileAvatarUploadAttributes.newBuilder() - .setPath(objectName) - .setCredential(policy.first()) - .setAcl("private") - .setAlgorithm("AWS4-HMAC-SHA256") - .setDate(now.format(PostPolicyGenerator.AWS_DATE_TIME)) - .setPolicy(policy.second()) - .setSignature(ByteString.copyFrom(signature.getBytes())) + return PROTOTYPE_AVATAR_UPLOAD_FORM.toBuilder() + .setKey(objectName) + .setCredential(policy.credential()) + .setDate(policy.formattedTimestamp()) + .setPolicy(policy.encodedPolicy()) + .setSignature(policy.signature()) .build(); } } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/metrics/MetricsUtil.java b/service/src/main/java/org/whispersystems/textsecuregcm/metrics/MetricsUtil.java index 4b2967855..fa4e46e57 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/metrics/MetricsUtil.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/metrics/MetricsUtil.java @@ -36,11 +36,11 @@ import org.eclipse.jetty.util.component.LifeCycle; import org.whispersystems.textsecuregcm.WhisperServerConfiguration; import org.whispersystems.textsecuregcm.configuration.dynamic.DynamicConfiguration; import org.whispersystems.textsecuregcm.storage.DynamicConfigurationManager; -import org.whispersystems.textsecuregcm.util.Constants; public class MetricsUtil { public static final String PREFIX = "chat"; + private static final String DROPWIZARD_METRICS_NAME = "textsecure"; private static volatile boolean registeredMetrics = false; @@ -69,7 +69,7 @@ public class MetricsUtil { registeredMetrics = true; - SharedMetricRegistries.add(Constants.METRICS_NAME, environment.metrics()); + SharedMetricRegistries.add(DROPWIZARD_METRICS_NAME, environment.metrics()); Duration shutdownWaitDuration = Duration.ZERO; diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/s3/PolicySigner.java b/service/src/main/java/org/whispersystems/textsecuregcm/s3/PolicySigner.java deleted file mode 100644 index 3cfa6a00f..000000000 --- a/service/src/main/java/org/whispersystems/textsecuregcm/s3/PolicySigner.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright 2013-2020 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ - -package org.whispersystems.textsecuregcm.s3; - -import java.nio.charset.StandardCharsets; -import java.security.InvalidKeyException; -import java.security.Key; -import java.security.NoSuchAlgorithmException; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.util.HexFormat; -import javax.crypto.Mac; -import javax.crypto.spec.SecretKeySpec; - -public class PolicySigner { - - private final String awsAccessSecret; - private final String region; - - private static final DateTimeFormatter DATE_FORMAT = DateTimeFormatter.ofPattern("yyyyMMdd"); - - public PolicySigner(final String awsAccessSecret, final String region) { - this.awsAccessSecret = awsAccessSecret; - this.region = region; - } - - // See https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html - public String getSignature(final ZonedDateTime now, final String policy) { - final Mac mac; - - try { - mac = Mac.getInstance("HmacSHA256"); - } catch (final NoSuchAlgorithmException e) { - throw new AssertionError("Every implementation of the Java platform is required to support HmacSHA256", e); - } - - try { - mac.init(toHmacKey(("AWS4" + awsAccessSecret).getBytes(StandardCharsets.UTF_8))); - final byte[] dateKey = mac.doFinal(now.format(DATE_FORMAT).getBytes(StandardCharsets.UTF_8)); - - mac.init(toHmacKey(dateKey)); - final byte[] dateRegionKey = mac.doFinal(region.getBytes(StandardCharsets.UTF_8)); - - mac.init(toHmacKey(dateRegionKey)); - final byte[] dateRegionServiceKey = mac.doFinal("s3".getBytes(StandardCharsets.UTF_8)); - - mac.init(toHmacKey(dateRegionServiceKey)); - final byte[] signingKey = mac.doFinal("aws4_request".getBytes(StandardCharsets.UTF_8)); - - mac.init(toHmacKey(signingKey)); - - return HexFormat.of().formatHex(mac.doFinal(policy.getBytes(StandardCharsets.UTF_8))); - } catch (final InvalidKeyException e) { - throw new AssertionError(e); - } - } - - private static Key toHmacKey(final byte[] bytes) { - return new SecretKeySpec(bytes, "HmacSHA256"); - } -} diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/s3/PostPolicyGenerator.java b/service/src/main/java/org/whispersystems/textsecuregcm/s3/PostPolicyGenerator.java index 7b8e18d5e..8efda36f1 100644 --- a/service/src/main/java/org/whispersystems/textsecuregcm/s3/PostPolicyGenerator.java +++ b/service/src/main/java/org/whispersystems/textsecuregcm/s3/PostPolicyGenerator.java @@ -5,31 +5,52 @@ package org.whispersystems.textsecuregcm.s3; +import com.google.common.annotations.VisibleForTesting; import java.nio.charset.StandardCharsets; +import java.security.InvalidKeyException; +import java.security.Key; +import java.security.NoSuchAlgorithmException; +import java.time.Instant; +import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.Base64; -import org.whispersystems.textsecuregcm.util.Pair; +import java.util.HexFormat; +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; public class PostPolicyGenerator { - public static final DateTimeFormatter AWS_DATE_TIME = DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmssX"); - private static final DateTimeFormatter CREDENTIAL_DATE = DateTimeFormatter.ofPattern("yyyyMMdd"); + public static final String ALGORITHM = "AWS4-HMAC-SHA256"; + public static final String ACL = "private"; + + private static final DateTimeFormatter AWS_TIMESTAMP_FORMATTER = DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmssX"); + private static final DateTimeFormatter AWS_DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyyMMdd"); private final String region; private final String bucket; private final String awsAccessId; + private final String awsAccessSecret; + + public record SignedPostPolicy(String formattedTimestamp, String credential, String encodedPolicy, String signature) { + } + + public PostPolicyGenerator(final String region, + final String bucket, + final String awsAccessId, + final String awsAccessSecret) { - public PostPolicyGenerator(final String region, final String bucket, final String awsAccessId) { this.region = region; this.bucket = bucket; this.awsAccessId = awsAccessId; + this.awsAccessSecret = awsAccessSecret; } - public Pair createFor(final ZonedDateTime now, final String object, final int maxSizeInBytes) { + public SignedPostPolicy createFor(final String object, final int maxSizeInBytes, final Instant currentTime) { + final ZonedDateTime now = ZonedDateTime.ofInstant(currentTime, ZoneOffset.UTC); final String expiration = now.plusMinutes(30).format(DateTimeFormatter.ISO_INSTANT); - final String credentialDate = now.format(CREDENTIAL_DATE); - final String requestDate = now.format(AWS_DATE_TIME); + final String credentialDate = now.format(AWS_DATE_FORMATTER); + final String requestTimestamp = now.format(AWS_TIMESTAMP_FORMATTER); final String credential = String.format("%s/%s/%s/s3/aws4_request", awsAccessId, credentialDate, region); final String policy = String.format(""" @@ -38,17 +59,55 @@ public class PostPolicyGenerator { "conditions": [ {"bucket": "%s"}, {"key": "%s"}, - {"acl": "private"}, + {"acl": "%s"}, ["starts-with", "$Content-Type", ""], ["content-length-range", 1, %d], {"x-amz-credential": "%s"}, - {"x-amz-algorithm": "AWS4-HMAC-SHA256"}, + {"x-amz-algorithm": "%s"}, {"x-amz-date": "%s" } ] } - """, expiration, bucket, object, maxSizeInBytes, credential, requestDate); + """, expiration, bucket, object, ACL, maxSizeInBytes, credential, ALGORITHM, requestTimestamp); - return new Pair<>(credential, Base64.getEncoder().encodeToString(policy.getBytes(StandardCharsets.UTF_8))); + final String encodedPolicy = Base64.getEncoder().encodeToString(policy.getBytes(StandardCharsets.UTF_8)); + + return new SignedPostPolicy(requestTimestamp, credential, encodedPolicy, getSignature(now, encodedPolicy)); + } + + // See https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_sigv-create-signed-request.html + @VisibleForTesting + String getSignature(final ZonedDateTime now, final String policy) { + final Mac mac; + + try { + mac = Mac.getInstance("HmacSHA256"); + } catch (final NoSuchAlgorithmException e) { + throw new AssertionError("Every implementation of the Java platform is required to support HmacSHA256", e); + } + + try { + mac.init(toHmacKey(("AWS4" + awsAccessSecret).getBytes(StandardCharsets.UTF_8))); + final byte[] dateKey = mac.doFinal(now.format(AWS_DATE_FORMATTER).getBytes(StandardCharsets.UTF_8)); + + mac.init(toHmacKey(dateKey)); + final byte[] dateRegionKey = mac.doFinal(region.getBytes(StandardCharsets.UTF_8)); + + mac.init(toHmacKey(dateRegionKey)); + final byte[] dateRegionServiceKey = mac.doFinal("s3".getBytes(StandardCharsets.UTF_8)); + + mac.init(toHmacKey(dateRegionServiceKey)); + final byte[] signingKey = mac.doFinal("aws4_request".getBytes(StandardCharsets.UTF_8)); + + mac.init(toHmacKey(signingKey)); + + return HexFormat.of().formatHex(mac.doFinal(policy.getBytes(StandardCharsets.UTF_8))); + } catch (final InvalidKeyException e) { + throw new AssertionError(e); + } + } + + private static Key toHmacKey(final byte[] bytes) { + return new SecretKeySpec(bytes, "HmacSHA256"); } } diff --git a/service/src/main/java/org/whispersystems/textsecuregcm/util/Constants.java b/service/src/main/java/org/whispersystems/textsecuregcm/util/Constants.java deleted file mode 100644 index 7035be3a8..000000000 --- a/service/src/main/java/org/whispersystems/textsecuregcm/util/Constants.java +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright 2013-2020 Signal Messenger, LLC - * SPDX-License-Identifier: AGPL-3.0-only - */ - -package org.whispersystems.textsecuregcm.util; - -import io.dropwizard.util.DataSize; - -public class Constants { - public static final String METRICS_NAME = "textsecure"; - public static final int MAXIMUM_STICKER_SIZE_BYTES = (int) DataSize.kibibytes(300 + 1).toBytes(); // add 1 kiB for encryption overhead - public static final int MAXIMUM_STICKER_MANIFEST_SIZE_BYTES = (int) DataSize.kibibytes(10).toBytes(); -} diff --git a/service/src/main/proto/org/signal/chat/attachments.proto b/service/src/main/proto/org/signal/chat/attachments.proto index 3939002e2..b51ed136f 100644 --- a/service/src/main/proto/org/signal/chat/attachments.proto +++ b/service/src/main/proto/org/signal/chat/attachments.proto @@ -19,6 +19,9 @@ service Attachments { // Retrieve an upload form that can be used to perform a resumable upload rpc GetUploadForm(GetUploadFormRequest) returns (GetUploadFormResponse) {} + + // Retrieve an upload form that can be used to upload a sticker pack + rpc GetStickerUploadForm(GetStickerUploadFormRequest) returns (GetStickerUploadFormResponse) {} } message GetUploadFormRequest { @@ -37,3 +40,19 @@ message GetUploadFormResponse { errors.FailedPrecondition exceeds_max_upload_length = 2 [(tag.reason) = "oversize_upload"]; } } + +message GetStickerUploadFormRequest { + // The number of stickers in the sticker pack to upload + uint32 sticker_count = 1 [(require.range) = {min: 1, max: 201}]; +} + +message GetStickerUploadFormResponse { + // A randomly-generated ID for the new sticker pack + string pack_id = 1; + + // An upload form clients must use to upload a manifest for the sticker pack + common.S3UploadForm manifest_upload_form = 2; + + // Upload forms for individual stickers within the sticker pack + repeated common.S3UploadForm sticker_upload_forms = 3; +} diff --git a/service/src/main/proto/org/signal/chat/common.proto b/service/src/main/proto/org/signal/chat/common.proto index 6c3a9c63c..91cd753d8 100644 --- a/service/src/main/proto/org/signal/chat/common.proto +++ b/service/src/main/proto/org/signal/chat/common.proto @@ -116,6 +116,33 @@ message UploadForm { string signed_upload_location = 4; } +// An upload location, credentials, and metadata which may be used to upload an +// object to AWS S3 +message S3UploadForm { + // The S3 key (i.e. path and filename) for the uploaded file. + string key = 1; + + // A scoped credential. Includes the AWS access key, date, region targeted, + // and AWS service. + string credential = 2; + + // The type of access control for the uploaded file. + string acl = 3; + + // The algorithm used to calculate a signature on the S3 policy. + string algorithm = 4; + + // The timestamp (formatted as "yyyyMMdd'T'HHmmssX") at which the S3 policy + // and signature were generated. + string date = 5; + + // The S3 policy (as a base64-encoded JSON string) used to upload the file. + string policy = 6; + + // A digital signature (formatted as a hex string) on the S3 policy. + string signature = 7; +} + message BadgeSvg { // File name of the scalable vector graphic for light mode. string light = 1; diff --git a/service/src/main/proto/org/signal/chat/profile.proto b/service/src/main/proto/org/signal/chat/profile.proto index 062618733..127887531 100644 --- a/service/src/main/proto/org/signal/chat/profile.proto +++ b/service/src/main/proto/org/signal/chat/profile.proto @@ -122,7 +122,7 @@ message SetProfileResult { // // Because this is a temporary field during the migration, it has the highest // field number without serialization overhead. This is purely aesthetic. - optional ProfileAvatarUploadAttributes v1_avatar_upload_form = 15; + optional common.S3UploadForm v1_avatar_upload_form = 15; // next: 1 } @@ -321,23 +321,6 @@ message GetExpiringProfileKeyCredentialAnonymousResponse { } } -message ProfileAvatarUploadAttributes { - // The S3 upload path for the profile's avatar. - string path = 1; - // A scoped credential. Includes the AWS access key, date, region targeted, and AWS service. - string credential = 2; - // The type of access control for the avatar object. - string acl = 3; - // The algorithm used to calculate a signature on the S3 policy. - string algorithm = 4; - // The timestamp at which the S3 policy and signature were generated. - string date = 5; - // The S3 policy used to upload the avatar object. - string policy = 6; - // A digital signature on the S3 policy. - bytes signature = 7; -} - enum CredentialType { CREDENTIAL_TYPE_UNSPECIFIED = 0; CREDENTIAL_TYPE_EXPIRING_PROFILE_KEY = 1; diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/controllers/ProfileControllerTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/controllers/ProfileControllerTest.java index d6f350d95..1831ba8f0 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/controllers/ProfileControllerTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/controllers/ProfileControllerTest.java @@ -100,7 +100,6 @@ import org.whispersystems.textsecuregcm.identity.ServiceIdentifier; import org.whispersystems.textsecuregcm.limits.RateLimiter; import org.whispersystems.textsecuregcm.limits.RateLimiters; import org.whispersystems.textsecuregcm.mappers.RateLimitExceededExceptionMapper; -import org.whispersystems.textsecuregcm.s3.PolicySigner; import org.whispersystems.textsecuregcm.s3.PostPolicyGenerator; import org.whispersystems.textsecuregcm.storage.Account; import org.whispersystems.textsecuregcm.storage.AccountBadge; @@ -129,8 +128,7 @@ class ProfileControllerTest { private static final RateLimiter usernameRateLimiter = mock(RateLimiter.class); private static final PostPolicyGenerator postPolicyGenerator = new PostPolicyGenerator("us-west-1", "profile-bucket", - "accessKey"); - private static final PolicySigner policySigner = new PolicySigner("accessSecret", "us-west-1"); + "accessKey", "accessSecret"); private static final ServerZkProfileOperations zkProfileOperations = mock(ServerZkProfileOperations.class); private static final ServerSecretParams serverSecretParams = ServerSecretParams.generate(); @@ -172,7 +170,6 @@ class ProfileControllerTest { new BadgeConfiguration("TEST3", "testing", List.of("l", "m", "h", "x", "xx", "xxx"), "SVG", List.of(new BadgeSvg("sl", "sd"), new BadgeSvg("ml", "md"), new BadgeSvg("ll", "ld"))) ), List.of("TEST1"), Map.of(1L, "TEST1", 2L, "TEST2", 3L, "TEST3")), postPolicyGenerator, - policySigner, serverSecretParams, zkProfileOperations, Executors.newSingleThreadExecutor())) diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/controllers/StickerControllerTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/controllers/StickerControllerTest.java index a91620681..961caadb9 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/controllers/StickerControllerTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/controllers/StickerControllerTest.java @@ -15,6 +15,7 @@ import io.dropwizard.auth.AuthValueFactoryProvider; import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; import io.dropwizard.testing.junit5.ResourceExtension; import jakarta.ws.rs.core.Response; +import java.time.Clock; import java.util.Base64; import org.glassfish.jersey.test.grizzly.GrizzlyWebTestContainerFactory; import org.junit.jupiter.api.BeforeEach; @@ -24,6 +25,7 @@ import org.whispersystems.textsecuregcm.auth.AuthenticatedDevice; import org.whispersystems.textsecuregcm.entities.StickerPackFormUploadAttributes; import org.whispersystems.textsecuregcm.limits.RateLimiter; import org.whispersystems.textsecuregcm.limits.RateLimiters; +import org.whispersystems.textsecuregcm.s3.PostPolicyGenerator; import org.whispersystems.textsecuregcm.tests.util.AuthHelper; import org.whispersystems.textsecuregcm.util.SystemMapper; @@ -38,7 +40,9 @@ class StickerControllerTest { .addProvider(new AuthValueFactoryProvider.Binder<>(AuthenticatedDevice.class)) .setMapper(SystemMapper.jsonMapper()) .setTestContainerFactory(new GrizzlyWebTestContainerFactory()) - .addResource(new StickerController(rateLimiters, "foo", "bar", "us-east-1", "mybucket")) + .addResource(new StickerController(rateLimiters, + new PostPolicyGenerator("us-east-1", "mybucket", "foo", "bar"), + Clock.systemUTC())) .build(); @BeforeEach diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/grpc/AttachmentsGrpcServiceTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/grpc/AttachmentsGrpcServiceTest.java index 28d0ffd27..cc2cb0b46 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/grpc/AttachmentsGrpcServiceTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/grpc/AttachmentsGrpcServiceTest.java @@ -24,6 +24,7 @@ import java.security.KeyPair; import java.security.KeyPairGenerator; import java.security.NoSuchAlgorithmException; import java.security.spec.InvalidKeySpecException; +import java.time.Clock; import java.time.Duration; import java.util.Arrays; import java.util.Base64; @@ -37,8 +38,11 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; import org.mockito.Mock; import org.signal.chat.attachments.AttachmentsGrpc; +import org.signal.chat.attachments.GetStickerUploadFormRequest; +import org.signal.chat.attachments.GetStickerUploadFormResponse; import org.signal.chat.attachments.GetUploadFormRequest; import org.signal.chat.attachments.GetUploadFormResponse; +import org.signal.chat.common.S3UploadForm; import org.signal.chat.common.UploadForm; import org.whispersystems.textsecuregcm.attachments.AttachmentUtil; import org.whispersystems.textsecuregcm.attachments.GcsAttachmentGenerator; @@ -49,6 +53,7 @@ import org.whispersystems.textsecuregcm.controllers.RateLimitExceededException; import org.whispersystems.textsecuregcm.experiment.ExperimentEnrollmentManager; import org.whispersystems.textsecuregcm.limits.RateLimiter; import org.whispersystems.textsecuregcm.limits.RateLimiters; +import org.whispersystems.textsecuregcm.s3.PostPolicyGenerator; import org.whispersystems.textsecuregcm.util.MockUtils; import org.whispersystems.textsecuregcm.util.TestRandomUtil; @@ -65,6 +70,8 @@ class AttachmentsGrpcServiceTest extends private RateLimiter countRateLimiter; @Mock private RateLimiter byteRateLimiter; + @Mock + private RateLimiter stickerRateLimiter; @Override protected AttachmentsGrpcService createServiceBeforeEachTest() { @@ -86,10 +93,13 @@ class AttachmentsGrpcServiceTest extends MockUtils.buildMock(RateLimiters.class, rateLimiters -> { when(rateLimiters.getAttachmentLimiter()).thenReturn(countRateLimiter); when(rateLimiters.getAttachmentBytesLimiter()).thenReturn(byteRateLimiter); + when(rateLimiters.getStickerPackLimiter()).thenReturn(stickerRateLimiter); }), gcsAttachmentGenerator, tusAttachmentGenerator, - MAX_UPLOAD_LENGTH); + new PostPolicyGenerator("wakonda-south-7", "test", "accessId", "accessSecret"), + MAX_UPLOAD_LENGTH, + Clock.systemUTC()); } catch (NoSuchAlgorithmException | IOException | InvalidKeyException | InvalidKeySpecException e) { throw new AssertionError(e); } @@ -209,4 +219,41 @@ class AttachmentsGrpcServiceTest extends assertRateLimitExceeded(retryAfter, () -> authenticatedServiceStub().getUploadForm(GetUploadFormRequest.newBuilder().setUploadLength(1).build())); } + + @Test + void getStickerUploadForm() { + final int stickerCount = 7; + + final GetStickerUploadFormResponse response = + authenticatedServiceStub().getStickerUploadForm(GetStickerUploadFormRequest.newBuilder() + .setStickerCount(stickerCount) + .build()); + + assertThat(response.getPackId()).isNotBlank(); + assertThat(response.getManifestUploadForm()).satisfies(AttachmentsGrpcServiceTest::s3UploadFormValid); + assertThat(response.getStickerUploadFormsCount()).isEqualTo(stickerCount); + assertThat(response.getStickerUploadFormsList()).allSatisfy(AttachmentsGrpcServiceTest::s3UploadFormValid); + } + + private static void s3UploadFormValid(final S3UploadForm s3UploadForm) { + assertThat(s3UploadForm.getAcl()).isEqualTo("private"); + assertThat(s3UploadForm.getAlgorithm()).isEqualTo("AWS4-HMAC-SHA256"); + assertThat(s3UploadForm.getKey()).startsWith("stickers/"); + assertThat(s3UploadForm.getCredential()).endsWith("/s3/aws4_request"); + assertThat(s3UploadForm.getDate()).isNotBlank(); + assertThat(s3UploadForm.getPolicy()).isNotBlank(); + assertThat(s3UploadForm.getSignature()).isNotBlank(); + } + + @Test + void getStickerUploadFormRateLimited() throws RateLimitExceededException { + final Duration retryAfter = Duration.ofMinutes(13); + doThrow(new RateLimitExceededException(retryAfter)).when(stickerRateLimiter).validate(any(UUID.class)); + + //noinspection ResultOfMethodCallIgnored + GrpcTestUtils.assertRateLimitExceeded(retryAfter, + () -> authenticatedServiceStub().getStickerUploadForm(GetStickerUploadFormRequest.newBuilder() + .setStickerCount(7) + .build())); + } } diff --git a/service/src/test/java/org/whispersystems/textsecuregcm/grpc/ProfileGrpcServiceTest.java b/service/src/test/java/org/whispersystems/textsecuregcm/grpc/ProfileGrpcServiceTest.java index e5ac6fc2d..ffa4e4a6b 100644 --- a/service/src/test/java/org/whispersystems/textsecuregcm/grpc/ProfileGrpcServiceTest.java +++ b/service/src/test/java/org/whispersystems/textsecuregcm/grpc/ProfileGrpcServiceTest.java @@ -93,7 +93,6 @@ import org.whispersystems.textsecuregcm.identity.AciServiceIdentifier; import org.whispersystems.textsecuregcm.identity.PniServiceIdentifier; import org.whispersystems.textsecuregcm.limits.RateLimiter; import org.whispersystems.textsecuregcm.limits.RateLimiters; -import org.whispersystems.textsecuregcm.s3.PolicySigner; import org.whispersystems.textsecuregcm.s3.PostPolicyGenerator; import org.whispersystems.textsecuregcm.storage.Account; import org.whispersystems.textsecuregcm.storage.AccountBadge; @@ -148,10 +147,11 @@ public class ProfileGrpcServiceTest extends SimpleBaseGrpcTest dynamicConfigurationManager = mock(DynamicConfigurationManager.class); final DynamicConfiguration dynamicConfiguration = mock(DynamicConfiguration.class); - final PolicySigner policySigner = new PolicySigner("accessSecret", "us-west-1"); - final PostPolicyGenerator policyGenerator = new PostPolicyGenerator("us-west-1", "profile-bucket", "accessKey"); + final PostPolicyGenerator policyGenerator = new PostPolicyGenerator("us-west-1", "profile-bucket", "accessKey", "accessSecret"); final BadgesConfiguration badgesConfiguration = new BadgesConfiguration( List.of(new BadgeConfiguration( "TEST", @@ -201,8 +201,6 @@ public class ProfileGrpcServiceTest extends SimpleBaseGrpcTest