Use archive-specific endpoint for attachment backfill.

app/src/main/java/org/thoughtcrime/securesms/backup/v2/BackupRepository.kt

@@ -49,6 +49,7 @@ import org.whispersystems.signalservice.api.messages.SignalServiceAttachment.Pro
 import org.whispersystems.signalservice.api.push.ServiceId.ACI
 import org.whispersystems.signalservice.api.push.ServiceId.PNI
 import org.whispersystems.signalservice.internal.crypto.PaddingInputStream
+import org.whispersystems.signalservice.internal.push.http.ResumableUploadSpec
 import java.io.ByteArrayOutputStream
 import java.io.File
 import java.io.InputStream
@@ -339,6 +340,24 @@ object BackupRepository {
       }
   }
 
+  /**
+   * Retrieves an upload spec that can be used to upload attachment media.
+   */
+  fun getMediaUploadSpec(): NetworkResult<ResumableUploadSpec> {
+    val api = ApplicationDependencies.getSignalServiceAccountManager().archiveApi
+    val backupKey = SignalStore.svr().getOrCreateMasterKey().deriveBackupKey()
+
+    return api
+      .triggerBackupIdReservation(backupKey)
+      .then { getAuthCredential() }
+      .then { credential ->
+        api.getMediaUploadForm(backupKey, credential)
+      }
+      .then { form ->
+        api.getResumableUploadSpec(form)
+      }
+  }
+
   fun archiveMedia(attachment: DatabaseAttachment): NetworkResult<Unit> {
     val api = ApplicationDependencies.getSignalServiceAccountManager().archiveApi
     val backupKey = SignalStore.svr().getOrCreateMasterKey().deriveBackupKey()

app/src/main/java/org/thoughtcrime/securesms/jobs/ArchiveAttachmentBackfillJob.kt

@@ -21,6 +21,7 @@ import org.thoughtcrime.securesms.jobmanager.impl.NetworkConstraint
 import org.thoughtcrime.securesms.jobs.protos.ArchiveAttachmentBackfillJobData
 import org.whispersystems.signalservice.api.NetworkResult
 import org.whispersystems.signalservice.api.archive.ArchiveMediaResponse
+import org.whispersystems.signalservice.api.archive.ArchiveMediaUploadFormStatusCodes
 import org.whispersystems.signalservice.api.messages.SignalServiceAttachmentPointer
 import java.io.IOException
 import java.util.Optional
@@ -123,7 +124,12 @@ class ArchiveAttachmentBackfillJob private constructor(
     if (transferState == AttachmentTable.ArchiveTransferState.BACKFILL_UPLOAD_IN_PROGRESS) {
       if (uploadSpec == null || System.currentTimeMillis() > uploadSpec!!.timeout) {
         Log.d(TAG, "Need an upload spec. Fetching...")
-        uploadSpec = ApplicationDependencies.getSignalServiceMessageSender().getResumableUploadSpec().toProto()
+
+        val (spec, result) = fetchResumableUploadSpec()
+        if (result != null) {
+          return result
+        }
+        uploadSpec = spec
       } else {
         Log.d(TAG, "Already have an upload spec. Continuing...")
       }
@@ -212,6 +218,42 @@ class ArchiveAttachmentBackfillJob private constructor(
     }
   }
 
+  private fun fetchResumableUploadSpec(): Pair<ResumableUpload?, Result?> {
+    return when (val spec = BackupRepository.getMediaUploadSpec()) {
+      is NetworkResult.Success -> {
+        Log.d(TAG, "Got an upload spec!")
+        spec.result.toProto() to null
+      }
+
+      is NetworkResult.ApplicationError -> {
+        Log.w(TAG, "Failed to get an upload spec due to an application error. Retrying.", spec.throwable)
+        return null to Result.retry(defaultBackoff())
+      }
+
+      is NetworkResult.NetworkError -> {
+        Log.w(TAG, "Encountered a transient network error. Retrying.")
+        return null to Result.retry(defaultBackoff())
+      }
+
+      is NetworkResult.StatusCodeError -> {
+        Log.w(TAG, "Failed request with status code ${spec.code}")
+
+        when (ArchiveMediaUploadFormStatusCodes.from(spec.code)) {
+          ArchiveMediaUploadFormStatusCodes.BadArguments,
+          ArchiveMediaUploadFormStatusCodes.InvalidPresentationOrSignature,
+          ArchiveMediaUploadFormStatusCodes.InsufficientPermissions,
+          ArchiveMediaUploadFormStatusCodes.RateLimited -> {
+            return null to Result.retry(defaultBackoff())
+          }
+
+          ArchiveMediaUploadFormStatusCodes.Unknown -> {
+            return null to Result.retry(defaultBackoff())
+          }
+        }
+      }
+    }
+  }
+
   class Factory : Job.Factory<ArchiveAttachmentBackfillJob> {
     override fun create(parameters: Parameters, serializedData: ByteArray?): ArchiveAttachmentBackfillJob {
       val data = serializedData?.let { ArchiveAttachmentBackfillJobData.ADAPTER.decode(it) }

libsignal-service/src/main/java/org/whispersystems/signalservice/api/SignalServiceMessageSender.java

@@ -93,7 +93,7 @@ import org.whispersystems.signalservice.internal.crypto.AttachmentDigest;
 import org.whispersystems.signalservice.internal.crypto.PaddingInputStream;
 import org.whispersystems.signalservice.internal.push.AttachmentPointer;
 import org.whispersystems.signalservice.internal.push.AttachmentV2UploadAttributes;
-import org.whispersystems.signalservice.internal.push.AttachmentV4UploadAttributes;
+import org.whispersystems.signalservice.internal.push.AttachmentUploadForm;
 import org.whispersystems.signalservice.internal.push.BodyRange;
 import org.whispersystems.signalservice.internal.push.CallMessage;
 import org.whispersystems.signalservice.internal.push.Content;
@@ -860,7 +860,7 @@ public class SignalServiceMessageSender {
   }
 
   public ResumableUploadSpec getResumableUploadSpec() throws IOException {
-    AttachmentV4UploadAttributes v4UploadAttributes = null;
+    AttachmentUploadForm v4UploadAttributes = null;
 
     Log.d(TAG, "Using pipe to retrieve attachment upload attributes...");
     try {

libsignal-service/src/main/java/org/whispersystems/signalservice/api/archive/ArchiveApi.kt

@@ -16,7 +16,9 @@ import org.whispersystems.signalservice.api.NetworkResult
 import org.whispersystems.signalservice.api.archive.ArchiveGetMediaItemsResponse.StoredMediaObject
 import org.whispersystems.signalservice.api.backup.BackupKey
 import org.whispersystems.signalservice.api.push.ServiceId.ACI
+import org.whispersystems.signalservice.internal.push.AttachmentUploadForm
 import org.whispersystems.signalservice.internal.push.PushServiceSocket
+import org.whispersystems.signalservice.internal.push.http.ResumableUploadSpec
 import java.io.InputStream
 import java.time.Instant
 
@@ -92,7 +94,7 @@ class ArchiveApi(
   /**
    * Fetches an upload form you can use to upload your main message backup file to cloud storage.
    */
-  fun getMessageBackupUploadForm(backupKey: BackupKey, serviceCredential: ArchiveServiceCredential): NetworkResult<ArchiveMessageBackupUploadFormResponse> {
+  fun getMessageBackupUploadForm(backupKey: BackupKey, serviceCredential: ArchiveServiceCredential): NetworkResult<AttachmentUploadForm> {
     return NetworkResult.fromFetch {
       val zkCredential = getZkCredential(backupKey, serviceCredential)
       val presentationData = CredentialPresentationData.from(backupKey, zkCredential, backupServerPublicParams)
@@ -125,20 +127,38 @@ class ArchiveApi(
   }
 
   /**
-   * Retrieves a resumable upload URL you can use to upload your main message backup file to cloud storage.
+   * Retrieves a resumable upload URL you can use to upload your main message backup file or an arbitrary media file to cloud storage.
    */
-  fun getBackupResumableUploadUrl(archiveFormResponse: ArchiveMessageBackupUploadFormResponse): NetworkResult<String> {
+  fun getBackupResumableUploadUrl(uploadForm: AttachmentUploadForm): NetworkResult<String> {
     return NetworkResult.fromFetch {
-      pushServiceSocket.getResumableUploadUrl(archiveFormResponse)
+      pushServiceSocket.getResumableUploadUrl(uploadForm)
     }
   }
 
   /**
    * Uploads your main backup file to cloud storage.
    */
-  fun uploadBackupFile(archiveFormResponse: ArchiveMessageBackupUploadFormResponse, resumableUploadUrl: String, data: InputStream, dataLength: Long): NetworkResult<Unit> {
+  fun uploadBackupFile(uploadForm: AttachmentUploadForm, resumableUploadUrl: String, data: InputStream, dataLength: Long): NetworkResult<Unit> {
     return NetworkResult.fromFetch {
-      pushServiceSocket.uploadBackupFile(archiveFormResponse, resumableUploadUrl, data, dataLength)
+      pushServiceSocket.uploadBackupFile(uploadForm, resumableUploadUrl, data, dataLength)
+    }
+  }
+
+  /**
+   * Retrieves an [AttachmentUploadForm] that can be used to upload pre-existing media to the archive.
+   * After uploading, the media still needs to be copied via [archiveAttachmentMedia].
+   */
+  fun getMediaUploadForm(backupKey: BackupKey, serviceCredential: ArchiveServiceCredential): NetworkResult<AttachmentUploadForm> {
+    return NetworkResult.fromFetch {
+      val zkCredential = getZkCredential(backupKey, serviceCredential)
+      val presentationData = CredentialPresentationData.from(backupKey, zkCredential, backupServerPublicParams)
+      pushServiceSocket.getArchiveMediaUploadForm(presentationData.toArchiveCredentialPresentation())
+    }
+  }
+
+  fun getResumableUploadSpec(uploadForm: AttachmentUploadForm): NetworkResult<ResumableUploadSpec> {
+    return NetworkResult.fromFetch {
+      pushServiceSocket.getResumableUploadSpec(uploadForm)
     }
   }
 

libsignal-service/src/main/java/org/whispersystems/signalservice/api/archive/ArchiveMediaUploadFormStatusCodes.kt

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2024 Signal Messenger, LLC
+ * SPDX-License-Identifier: AGPL-3.0-only
+ */
+
+package org.whispersystems.signalservice.api.archive
+
+/**
+ * Status codes for the ArchiveMediaUploadForm endpoint.
+ *
+ * Kept in a separate class because [AttachmentUploadForm] (the model the request returns) is used for multiple endpoints with different status codes.
+ */
+enum class ArchiveMediaUploadFormStatusCodes(val code: Int) {
+  BadArguments(400),
+  InvalidPresentationOrSignature(401),
+  InsufficientPermissions(403),
+  RateLimited(429),
+  Unknown(-1);
+
+  companion object {
+    fun from(code: Int): ArchiveMediaUploadFormStatusCodes {
+      return values().firstOrNull { it.code == code } ?: Unknown
+    }
+  }
+}

libsignal-service/src/main/java/org/whispersystems/signalservice/api/archive/ArchiveMessageBackupUploadFormResponse.kt

@@ -1,22 +0,0 @@
-/*
- * Copyright 2023 Signal Messenger, LLC
- * SPDX-License-Identifier: AGPL-3.0-only
- */
-
-package org.whispersystems.signalservice.api.archive
-
-import com.fasterxml.jackson.annotation.JsonProperty
-
-/**
- * Represents the response body when we ask for a message backup upload form.
- */
-data class ArchiveMessageBackupUploadFormResponse(
-  @JsonProperty
-  val cdn: Int,
-  @JsonProperty
-  val key: String,
-  @JsonProperty
-  val headers: Map<String, String>,
-  @JsonProperty
-  val signedUploadLocation: String
-)

libsignal-service/src/main/java/org/whispersystems/signalservice/api/services/AttachmentService.kt

@@ -4,8 +4,8 @@ import io.reactivex.rxjava3.core.Single
 import org.whispersystems.signalservice.api.SignalWebSocket
 import org.whispersystems.signalservice.internal.ServiceResponse
 import org.whispersystems.signalservice.internal.ServiceResponseProcessor
+import org.whispersystems.signalservice.internal.push.AttachmentUploadForm
 import org.whispersystems.signalservice.internal.push.AttachmentV2UploadAttributes
-import org.whispersystems.signalservice.internal.push.AttachmentV4UploadAttributes
 import org.whispersystems.signalservice.internal.websocket.DefaultResponseMapper
 import org.whispersystems.signalservice.internal.websocket.WebSocketRequestMessage
 import org.whispersystems.signalservice.internal.websocket.WebsocketResponse
@@ -29,7 +29,7 @@ class AttachmentService(private val signalWebSocket: SignalWebSocket) {
       .onErrorReturn { throwable: Throwable? -> ServiceResponse.forUnknownError(throwable) }
   }
 
-  fun getAttachmentV4UploadAttributes(): Single<ServiceResponse<AttachmentV4UploadAttributes>> {
+  fun getAttachmentV4UploadAttributes(): Single<ServiceResponse<AttachmentUploadForm>> {
     val requestMessage = WebSocketRequestMessage(
       id = SecureRandom().nextLong(),
       verb = "GET",
@@ -37,7 +37,7 @@ class AttachmentService(private val signalWebSocket: SignalWebSocket) {
     )
 
     return signalWebSocket.request(requestMessage)
-      .map { response: WebsocketResponse? -> DefaultResponseMapper.getDefault(AttachmentV4UploadAttributes::class.java).map(response) }
+      .map { response: WebsocketResponse? -> DefaultResponseMapper.getDefault(AttachmentUploadForm::class.java).map(response) }
       .onErrorReturn { throwable: Throwable? -> ServiceResponse.forUnknownError(throwable) }
   }
 

libsignal-service/src/main/java/org/whispersystems/signalservice/internal/push/AttachmentUploadForm.kt

@@ -0,0 +1,24 @@
+package org.whispersystems.signalservice.internal.push
+
+import com.fasterxml.jackson.annotation.JsonProperty
+
+/**
+ * Represents an attachment upload form that can be returned by various service endpoints.
+ */
+data class AttachmentUploadForm(
+  @JvmField
+  @JsonProperty
+  val cdn: Int,
+
+  @JvmField
+  @JsonProperty
+  val key: String,
+
+  @JvmField
+  @JsonProperty
+  val headers: Map<String, String>,
+
+  @JvmField
+  @JsonProperty
+  val signedUploadLocation: String
+)

libsignal-service/src/main/java/org/whispersystems/signalservice/internal/push/AttachmentV4UploadAttributes.java

@@ -1,38 +0,0 @@
-package org.whispersystems.signalservice.internal.push;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import java.util.Map;
-
-public final class AttachmentV4UploadAttributes {
-    @JsonProperty
-    private int cdn;
-
-    @JsonProperty
-    private String key;
-
-    @JsonProperty
-    private Map<String, String> headers;
-
-    @JsonProperty
-    private String signedUploadLocation;
-
-    public AttachmentV4UploadAttributes() {
-    }
-
-    public int getCdn() {
-        return cdn;
-    }
-
-    public String getKey() {
-        return key;
-    }
-
-    public Map<String, String> getHeaders() {
-        return headers;
-    }
-
-    public String getSignedUploadLocation() {
-        return signedUploadLocation;
-    }
-}

libsignal-service/src/main/java/org/whispersystems/signalservice/internal/push/PushServiceSocket.java

@@ -10,6 +10,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.squareup.wire.Message;
 
+import org.jetbrains.annotations.NotNull;
 import org.signal.core.util.Base64;
 import org.signal.core.util.concurrent.FutureTransformers;
 import org.signal.core.util.concurrent.ListenableFuture;
@@ -51,7 +52,6 @@ import org.whispersystems.signalservice.api.archive.ArchiveGetBackupInfoResponse
 import org.whispersystems.signalservice.api.archive.ArchiveGetMediaItemsResponse;
 import org.whispersystems.signalservice.api.archive.ArchiveMediaRequest;
 import org.whispersystems.signalservice.api.archive.ArchiveMediaResponse;
-import org.whispersystems.signalservice.api.archive.ArchiveMessageBackupUploadFormResponse;
 import org.whispersystems.signalservice.api.archive.ArchiveServiceCredentialsResponse;
 import org.whispersystems.signalservice.api.archive.ArchiveSetBackupIdRequest;
 import org.whispersystems.signalservice.api.archive.ArchiveSetPublicKeyRequest;
@@ -316,6 +316,7 @@ public class PushServiceSocket {
   private static final String ARCHIVE_PUBLIC_KEY          = "/v1/archives/keys";
   private static final String ARCHIVE_INFO                = "/v1/archives";
   private static final String ARCHIVE_MESSAGE_UPLOAD_FORM = "/v1/archives/upload/form";
+  private static final String ARCHIVE_MEDIA_UPLOAD_FORM   = "/v1/archives/media/upload/form";
   private static final String ARCHIVE_MEDIA               = "/v1/archives/media";
   private static final String ARCHIVE_MEDIA_LIST          = "/v1/archives/media?limit=%d";
   private static final String ARCHIVE_MEDIA_BATCH         = "/v1/archives/media/batch";
@@ -581,11 +582,18 @@ public class PushServiceSocket {
     makeServiceRequestWithoutAuthentication(ARCHIVE_MEDIA_DELETE, "POST", JsonUtil.toJson(request), headers, NO_HANDLER);
   }
 
-  public ArchiveMessageBackupUploadFormResponse getArchiveMessageBackupUploadForm(ArchiveCredentialPresentation credentialPresentation) throws IOException {
+  public AttachmentUploadForm getArchiveMessageBackupUploadForm(ArchiveCredentialPresentation credentialPresentation) throws IOException {
     Map<String, String> headers = credentialPresentation.toHeaders();
 
     String response = makeServiceRequestWithoutAuthentication(ARCHIVE_MESSAGE_UPLOAD_FORM, "GET", null, headers, NO_HANDLER);
-    return JsonUtil.fromJson(response, ArchiveMessageBackupUploadFormResponse.class);
+    return JsonUtil.fromJson(response, AttachmentUploadForm.class);
+  }
+
+  public AttachmentUploadForm getArchiveMediaUploadForm(@NotNull ArchiveCredentialPresentation credentialPresentation) throws IOException {
+    Map<String, String> headers = credentialPresentation.toHeaders();
+
+    String response = makeServiceRequestWithoutAuthentication(ARCHIVE_MEDIA_UPLOAD_FORM, "GET", null, headers, UNOPINIONATED_HANDLER);
+    return JsonUtil.fromJson(response, AttachmentUploadForm.class);
   }
 
   /**
@@ -1523,12 +1531,12 @@ public class PushServiceSocket {
     }
   }
 
-  public AttachmentV4UploadAttributes getAttachmentV4UploadAttributes()
+  public AttachmentUploadForm getAttachmentV4UploadAttributes()
       throws NonSuccessfulResponseCodeException, PushNetworkException, MalformedResponseException
   {
     String response = makeServiceRequest(ATTACHMENT_V4_PATH, "GET", null);
     try {
-      return JsonUtil.fromJson(response, AttachmentV4UploadAttributes.class);
+      return JsonUtil.fromJson(response, AttachmentUploadForm.class);
     } catch (IOException e) {
       Log.w(TAG, e);
       throw new MalformedResponseException("Unable to parse entity", e);
@@ -1563,14 +1571,14 @@ public class PushServiceSocket {
     return new Pair<>(id, digest);
   }
 
-  public ResumableUploadSpec getResumableUploadSpec(AttachmentV4UploadAttributes uploadAttributes) throws IOException {
+  public ResumableUploadSpec getResumableUploadSpec(AttachmentUploadForm uploadForm) throws IOException {
     return new ResumableUploadSpec(Util.getSecretBytes(64),
                                    Util.getSecretBytes(16),
-                                   uploadAttributes.getKey(),
+                                   uploadForm.key,
-                                   uploadAttributes.getCdn(),
+                                   uploadForm.cdn,
-                                   getResumableUploadUrl(uploadAttributes.getCdn(), uploadAttributes.getSignedUploadLocation(), uploadAttributes.getHeaders()),
+                                   getResumableUploadUrl(uploadForm),
                                    System.currentTimeMillis() + CDN2_RESUMABLE_LINK_LIFETIME_MILLIS,
-                                   uploadAttributes.getHeaders());
+                                   uploadForm.headers);
   }
 
   public AttachmentDigest uploadAttachment(PushAttachmentData attachment) throws IOException {
@@ -1741,22 +1749,18 @@ public class PushServiceSocket {
     }
   }
 
-  public String getResumableUploadUrl(ArchiveMessageBackupUploadFormResponse uploadFormResponse) throws IOException {
+  public String getResumableUploadUrl(AttachmentUploadForm uploadForm) throws IOException {
-    return getResumableUploadUrl(uploadFormResponse.getCdn(), uploadFormResponse.getSignedUploadLocation(), uploadFormResponse.getHeaders());
+    ConnectionHolder connectionHolder = getRandom(cdnClientsMap.get(uploadForm.cdn), random);
-  }
-
-  private String getResumableUploadUrl(int cdn, String signedUrl, Map<String, String> headers) throws IOException {
-    ConnectionHolder connectionHolder = getRandom(cdnClientsMap.get(cdn), random);
     OkHttpClient     okHttpClient     = connectionHolder.getClient()
                                                         .newBuilder()
                                                         .connectTimeout(soTimeoutMillis, TimeUnit.MILLISECONDS)
                                                         .readTimeout(soTimeoutMillis, TimeUnit.MILLISECONDS)
                                                         .build();
 
-    Request.Builder request = new Request.Builder().url(buildConfiguredUrl(connectionHolder, signedUrl))
+    Request.Builder request = new Request.Builder().url(buildConfiguredUrl(connectionHolder, uploadForm.signedUploadLocation))
                                                    .post(RequestBody.create(null, ""));
 
-    for (Map.Entry<String, String> header : headers.entrySet()) {
+    for (Map.Entry<String, String> header : uploadForm.headers.entrySet()) {
       if (!header.getKey().equalsIgnoreCase("host")) {
         request.header(header.getKey(), header.getValue());
       }
@@ -1768,13 +1772,13 @@ public class PushServiceSocket {
 
     request.addHeader("Content-Length", "0");
 
-    if (cdn == 2) {
+    if (uploadForm.cdn == 2) {
       request.addHeader("Content-Type", "application/octet-stream");
-    } else if (cdn == 3) {
+    } else if (uploadForm.cdn == 3) {
       request.addHeader("Upload-Defer-Length", "1")
              .addHeader("Tus-Resumable", "1.0.0");
     } else {
-      throw new AssertionError("Unknown CDN version: " + cdn);
+      throw new AssertionError("Unknown CDN version: " + uploadForm.cdn);
     }
 
     Call call = okHttpClient.newCall(request.build());
@@ -1847,8 +1851,8 @@ public class PushServiceSocket {
     }
   }
 
-  public void uploadBackupFile(ArchiveMessageBackupUploadFormResponse uploadFormResponse, String resumableUploadUrl, InputStream data, long dataLength) throws IOException {
+  public void uploadBackupFile(AttachmentUploadForm uploadForm, String resumableUploadUrl, InputStream data, long dataLength) throws IOException {
-    uploadToCdn3(resumableUploadUrl, data, "application/octet-stream", dataLength, false, new NoCipherOutputStreamFactory(), null, null, uploadFormResponse.getHeaders());
+    uploadToCdn3(resumableUploadUrl, data, "application/octet-stream", dataLength, false, new NoCipherOutputStreamFactory(), null, null, uploadForm.headers);
   }
 
   private AttachmentDigest uploadToCdn3(String resumableUrl,
@@ -2571,6 +2575,7 @@ public class PushServiceSocket {
       return readBodyJson(response.body(), clazz);
   }
 
+
   public enum VerificationCodeTransport { SMS, VOICE }
 
   private static class RegistrationLock {