Move backoff calculation into jobs.

This commit is contained in:
Greyson Parrelli
2021-01-28 09:48:09 -05:00
parent 6a45858b4a
commit 1d83729e6c
14 changed files with 115 additions and 112 deletions

View File

@@ -88,7 +88,7 @@ public class JobMigratorTest {
private static JobStorage simpleJobStorage() {
JobStorage jobStorage = mock(JobStorage.class);
when(jobStorage.getAllJobSpecs()).thenReturn(new ArrayList<>(Collections.singletonList(new JobSpec("1", "f1", null, 1, 1, 1, 1, 1, 1, "", null, false, false))));
when(jobStorage.getAllJobSpecs()).thenReturn(new ArrayList<>(Collections.singletonList(new JobSpec("1", "f1", null, 1, 1, 1, 1, 1, "", null, false, false))));
return jobStorage;
}

View File

@@ -96,10 +96,10 @@ public class FastJobStorageTest {
@Test
public void updateAllJobsToBePending_allArePending() {
FullSpec fullSpec1 = new FullSpec(new JobSpec("1", "f1", null, 1, 1, 1, 1, 1, 1, EMPTY_DATA, null, true, false),
FullSpec fullSpec1 = new FullSpec(new JobSpec("1", "f1", null, 1, 1, 1, 1, 1, EMPTY_DATA, null, true, false),
Collections.emptyList(),
Collections.emptyList());
FullSpec fullSpec2 = new FullSpec(new JobSpec("2", "f2", null, 1, 1, 1, 1, 1, 1, EMPTY_DATA, null, true, false),
FullSpec fullSpec2 = new FullSpec(new JobSpec("2", "f2", null, 1, 1, 1, 1, 1, EMPTY_DATA, null, true, false),
Collections.emptyList(),
Collections.emptyList());
@@ -116,7 +116,7 @@ public class FastJobStorageTest {
public void updateJobs_writesToDatabase() {
JobDatabase database = fixedDataDatabase(DataSet1.FULL_SPECS);
FastJobStorage subject = new FastJobStorage(database);
List<JobSpec> jobs = Collections.singletonList(new JobSpec("id1", "f1", null, 1, 1, 1, 1, 1, 1, EMPTY_DATA, null, false, false));
List<JobSpec> jobs = Collections.singletonList(new JobSpec("id1", "f1", null, 1, 1, 1, 1, 1, EMPTY_DATA, null, false, false));
subject.init();
subject.updateJobs(jobs);
@@ -128,7 +128,7 @@ public class FastJobStorageTest {
public void updateJobs_memoryOnly_doesNotWriteToDatabase() {
JobDatabase database = fixedDataDatabase(DataSetMemory.FULL_SPECS);
FastJobStorage subject = new FastJobStorage(database);
List<JobSpec> jobs = Collections.singletonList(new JobSpec("id1", "f1", null, 1, 1, 1, 1, 1, 1, EMPTY_DATA, null, false, false));
List<JobSpec> jobs = Collections.singletonList(new JobSpec("id1", "f1", null, 1, 1, 1, 1, 1, EMPTY_DATA, null, false, false));
subject.init();
subject.updateJobs(jobs);
@@ -138,20 +138,20 @@ public class FastJobStorageTest {
@Test
public void updateJobs_updatesAllFields() {
FullSpec fullSpec1 = new FullSpec(new JobSpec("1", "f1", null, 1, 1, 1, 1, 1, 1, EMPTY_DATA, null, false, false),
FullSpec fullSpec1 = new FullSpec(new JobSpec("1", "f1", null, 1, 1, 1, 1, 1, EMPTY_DATA, null, false, false),
Collections.emptyList(),
Collections.emptyList());
FullSpec fullSpec2 = new FullSpec(new JobSpec("2", "f2", null, 1, 1, 1, 1, 1, 1, EMPTY_DATA, null, false, false),
FullSpec fullSpec2 = new FullSpec(new JobSpec("2", "f2", null, 1, 1, 1, 1, 1, EMPTY_DATA, null, false, false),
Collections.emptyList(),
Collections.emptyList());
FullSpec fullSpec3 = new FullSpec(new JobSpec("3", "f3", null, 1, 1, 1, 1, 1, 1, EMPTY_DATA, null, false, false),
FullSpec fullSpec3 = new FullSpec(new JobSpec("3", "f3", null, 1, 1, 1, 1, 1, EMPTY_DATA, null, false, false),
Collections.emptyList(),
Collections.emptyList());
FastJobStorage subject = new FastJobStorage(fixedDataDatabase(Arrays.asList(fullSpec1, fullSpec2, fullSpec3)));
JobSpec update1 = new JobSpec("1", "g1", "q1", 2, 2, 2, 2, 2, 2, "abc", null, true, false);
JobSpec update2 = new JobSpec("2", "g2", "q2", 3, 3, 3, 3, 3, 3, "def", "ghi", true, false);
JobSpec update1 = new JobSpec("1", "g1", "q1", 2, 2, 2, 2, 2, "abc", null, true, false);
JobSpec update2 = new JobSpec("2", "g2", "q2", 3, 3, 3, 3, 3, "def", "ghi", true, false);
subject.init();
subject.updateJobs(Arrays.asList(update1, update2));
@@ -208,7 +208,7 @@ public class FastJobStorageTest {
@Test
public void updateJobAfterRetry_stateUpdated() {
FullSpec fullSpec = new FullSpec(new JobSpec("1", "f1", null, 0, 0, 0, 3, 30000, -1, EMPTY_DATA, null, true, false),
FullSpec fullSpec = new FullSpec(new JobSpec("1", "f1", null, 0, 0, 0, 3, -1, EMPTY_DATA, null, true, false),
Collections.emptyList(),
Collections.emptyList());
@@ -228,10 +228,10 @@ public class FastJobStorageTest {
@Test
public void getPendingJobsWithNoDependenciesInCreatedOrder_noneWhenEarlierItemInQueueInRunning() {
FullSpec fullSpec1 = new FullSpec(new JobSpec("1", "f1", "q", 0, 0, 0, 0, 0, -1, EMPTY_DATA, null, true, false),
FullSpec fullSpec1 = new FullSpec(new JobSpec("1", "f1", "q", 0, 0, 0, 0, -1, EMPTY_DATA, null, true, false),
Collections.emptyList(),
Collections.emptyList());
FullSpec fullSpec2 = new FullSpec(new JobSpec("2", "f2", "q", 0, 0, 0, 0, 0, -1, EMPTY_DATA, null, false, false),
FullSpec fullSpec2 = new FullSpec(new JobSpec("2", "f2", "q", 0, 0, 0, 0, -1, EMPTY_DATA, null, false, false),
Collections.emptyList(),
Collections.emptyList());
@@ -243,7 +243,7 @@ public class FastJobStorageTest {
@Test
public void getPendingJobsWithNoDependenciesInCreatedOrder_noneWhenAllJobsAreRunning() {
FullSpec fullSpec = new FullSpec(new JobSpec("1", "f1", "q", 0, 0, 0, 0, 0, -1, EMPTY_DATA, null, true, false),
FullSpec fullSpec = new FullSpec(new JobSpec("1", "f1", "q", 0, 0, 0, 0, -1, EMPTY_DATA, null, true, false),
Collections.emptyList(),
Collections.emptyList());
@@ -255,7 +255,7 @@ public class FastJobStorageTest {
@Test
public void getPendingJobsWithNoDependenciesInCreatedOrder_noneWhenNextRunTimeIsAfterCurrentTime() {
FullSpec fullSpec = new FullSpec(new JobSpec("1", "f1", "q", 0, 10, 0, 0, 0, -1, EMPTY_DATA, null, false, false),
FullSpec fullSpec = new FullSpec(new JobSpec("1", "f1", "q", 0, 10, 0, 0, -1, EMPTY_DATA, null, false, false),
Collections.emptyList(),
Collections.emptyList());
@@ -267,10 +267,10 @@ public class FastJobStorageTest {
@Test
public void getPendingJobsWithNoDependenciesInCreatedOrder_noneWhenDependentOnAnotherJob() {
FullSpec fullSpec1 = new FullSpec(new JobSpec("1", "f1", null, 0, 0, 0, 0, 0, -1, EMPTY_DATA, null, true, false),
FullSpec fullSpec1 = new FullSpec(new JobSpec("1", "f1", null, 0, 0, 0, 0, -1, EMPTY_DATA, null, true, false),
Collections.emptyList(),
Collections.emptyList());
FullSpec fullSpec2 = new FullSpec(new JobSpec("2", "f2", null, 0, 0, 0, 0, 0, -1, EMPTY_DATA, null, false, false),
FullSpec fullSpec2 = new FullSpec(new JobSpec("2", "f2", null, 0, 0, 0, 0, -1, EMPTY_DATA, null, false, false),
Collections.emptyList(),
Collections.singletonList(new DependencySpec("2", "1", false)));
@@ -283,7 +283,7 @@ public class FastJobStorageTest {
@Test
public void getPendingJobsWithNoDependenciesInCreatedOrder_singleEligibleJob() {
FullSpec fullSpec = new FullSpec(new JobSpec("1", "f1", "q", 0, 0, 0, 0, 0, -1, EMPTY_DATA, null, false, false),
FullSpec fullSpec = new FullSpec(new JobSpec("1", "f1", "q", 0, 0, 0, 0, -1, EMPTY_DATA, null, false, false),
Collections.emptyList(),
Collections.emptyList());
@@ -295,10 +295,10 @@ public class FastJobStorageTest {
@Test
public void getPendingJobsWithNoDependenciesInCreatedOrder_multipleEligibleJobs() {
FullSpec fullSpec1 = new FullSpec(new JobSpec("1", "f1", null, 0, 0, 0, 0, 0, -1, EMPTY_DATA, null, false, false),
FullSpec fullSpec1 = new FullSpec(new JobSpec("1", "f1", null, 0, 0, 0, 0, -1, EMPTY_DATA, null, false, false),
Collections.emptyList(),
Collections.emptyList());
FullSpec fullSpec2 = new FullSpec(new JobSpec("2", "f2", null, 0, 0, 0, 0, 0, -1, EMPTY_DATA, null, false, false),
FullSpec fullSpec2 = new FullSpec(new JobSpec("2", "f2", null, 0, 0, 0, 0, -1, EMPTY_DATA, null, false, false),
Collections.emptyList(),
Collections.emptyList());
@@ -311,10 +311,10 @@ public class FastJobStorageTest {
@Test
public void getPendingJobsWithNoDependenciesInCreatedOrder_singleEligibleJobInMixedList() {
FullSpec fullSpec1 = new FullSpec(new JobSpec("1", "f1", null, 0, 0, 0, 0, 0, -1, EMPTY_DATA, null, true, false),
FullSpec fullSpec1 = new FullSpec(new JobSpec("1", "f1", null, 0, 0, 0, 0, -1, EMPTY_DATA, null, true, false),
Collections.emptyList(),
Collections.emptyList());
FullSpec fullSpec2 = new FullSpec(new JobSpec("2", "f2", null, 0, 0, 0, 0, 0, -1, EMPTY_DATA, null, false, false),
FullSpec fullSpec2 = new FullSpec(new JobSpec("2", "f2", null, 0, 0, 0, 0, -1, EMPTY_DATA, null, false, false),
Collections.emptyList(),
Collections.emptyList());
@@ -330,10 +330,10 @@ public class FastJobStorageTest {
@Test
public void getPendingJobsWithNoDependenciesInCreatedOrder_firstItemInQueue() {
FullSpec fullSpec1 = new FullSpec(new JobSpec("1", "f1", "q", 0, 0, 0, 0, 0, -1, EMPTY_DATA, null, false, false),
FullSpec fullSpec1 = new FullSpec(new JobSpec("1", "f1", "q", 0, 0, 0, 0, -1, EMPTY_DATA, null, false, false),
Collections.emptyList(),
Collections.emptyList());
FullSpec fullSpec2 = new FullSpec(new JobSpec("2", "f2", "q", 0, 0, 0, 0, 0, -1, EMPTY_DATA, null, false, false),
FullSpec fullSpec2 = new FullSpec(new JobSpec("2", "f2", "q", 0, 0, 0, 0, -1, EMPTY_DATA, null, false, false),
Collections.emptyList(),
Collections.emptyList());
@@ -349,10 +349,10 @@ public class FastJobStorageTest {
@Test
public void getPendingJobsWithNoDependenciesInCreatedOrder_migrationJobTakesPrecedence() {
FullSpec plainSpec = new FullSpec(new JobSpec("1", "f1", "q", 0, 0, 0, 0, 0, -1, EMPTY_DATA, null, false, false),
FullSpec plainSpec = new FullSpec(new JobSpec("1", "f1", "q", 0, 0, 0, 0, -1, EMPTY_DATA, null, false, false),
Collections.emptyList(),
Collections.emptyList());
FullSpec migrationSpec = new FullSpec(new JobSpec("2", "f2", Job.Parameters.MIGRATION_QUEUE_KEY, 5, 0, 0, 0, 0, -1, EMPTY_DATA, null, false, false),
FullSpec migrationSpec = new FullSpec(new JobSpec("2", "f2", Job.Parameters.MIGRATION_QUEUE_KEY, 5, 0, 0, 0, -1, EMPTY_DATA, null, false, false),
Collections.emptyList(),
Collections.emptyList());
@@ -367,10 +367,10 @@ public class FastJobStorageTest {
@Test
public void getPendingJobsWithNoDependenciesInCreatedOrder_runningMigrationBlocksNormalJobs() {
FullSpec plainSpec = new FullSpec(new JobSpec("1", "f1", "q", 0, 0, 0, 0, 0, -1, EMPTY_DATA, null, false, false),
FullSpec plainSpec = new FullSpec(new JobSpec("1", "f1", "q", 0, 0, 0, 0, -1, EMPTY_DATA, null, false, false),
Collections.emptyList(),
Collections.emptyList());
FullSpec migrationSpec = new FullSpec(new JobSpec("2", "f2", Job.Parameters.MIGRATION_QUEUE_KEY, 5, 0, 0, 0, 0, -1, EMPTY_DATA, null, true, false),
FullSpec migrationSpec = new FullSpec(new JobSpec("2", "f2", Job.Parameters.MIGRATION_QUEUE_KEY, 5, 0, 0, 0, -1, EMPTY_DATA, null, true, false),
Collections.emptyList(),
Collections.emptyList());
@@ -384,10 +384,10 @@ public class FastJobStorageTest {
@Test
public void getPendingJobsWithNoDependenciesInCreatedOrder_runningMigrationBlocksLaterMigrationJobs() {
FullSpec migrationSpec1 = new FullSpec(new JobSpec("1", "f1", Job.Parameters.MIGRATION_QUEUE_KEY, 0, 0, 0, 0, 0, -1, EMPTY_DATA, null, true, false),
FullSpec migrationSpec1 = new FullSpec(new JobSpec("1", "f1", Job.Parameters.MIGRATION_QUEUE_KEY, 0, 0, 0, 0, -1, EMPTY_DATA, null, true, false),
Collections.emptyList(),
Collections.emptyList());
FullSpec migrationSpec2 = new FullSpec(new JobSpec("2", "f2", Job.Parameters.MIGRATION_QUEUE_KEY, 5, 0, 0, 0, 0, -1, EMPTY_DATA, null, false, false),
FullSpec migrationSpec2 = new FullSpec(new JobSpec("2", "f2", Job.Parameters.MIGRATION_QUEUE_KEY, 5, 0, 0, 0, -1, EMPTY_DATA, null, false, false),
Collections.emptyList(),
Collections.emptyList());
@@ -401,10 +401,10 @@ public class FastJobStorageTest {
@Test
public void getPendingJobsWithNoDependenciesInCreatedOrder_onlyReturnFirstEligibleMigrationJob() {
FullSpec migrationSpec1 = new FullSpec(new JobSpec("1", "f1", Job.Parameters.MIGRATION_QUEUE_KEY, 0, 0, 0, 0, 0, -1, EMPTY_DATA, null, false, false),
FullSpec migrationSpec1 = new FullSpec(new JobSpec("1", "f1", Job.Parameters.MIGRATION_QUEUE_KEY, 0, 0, 0, 0, -1, EMPTY_DATA, null, false, false),
Collections.emptyList(),
Collections.emptyList());
FullSpec migrationSpec2 = new FullSpec(new JobSpec("2", "f2", Job.Parameters.MIGRATION_QUEUE_KEY, 5, 0, 0, 0, 0, -1, EMPTY_DATA, null, false, false),
FullSpec migrationSpec2 = new FullSpec(new JobSpec("2", "f2", Job.Parameters.MIGRATION_QUEUE_KEY, 5, 0, 0, 0, -1, EMPTY_DATA, null, false, false),
Collections.emptyList(),
Collections.emptyList());
@@ -419,10 +419,10 @@ public class FastJobStorageTest {
@Test
public void getPendingJobsWithNoDependenciesInCreatedOrder_onlyMigrationJobWithAppropriateNextRunTime() {
FullSpec migrationSpec1 = new FullSpec(new JobSpec("1", "f1", Job.Parameters.MIGRATION_QUEUE_KEY, 0, 999, 0, 0, 0, -1, EMPTY_DATA, null, false, false),
FullSpec migrationSpec1 = new FullSpec(new JobSpec("1", "f1", Job.Parameters.MIGRATION_QUEUE_KEY, 0, 999, 0, 0, -1, EMPTY_DATA, null, false, false),
Collections.emptyList(),
Collections.emptyList());
FullSpec migrationSpec2 = new FullSpec(new JobSpec("2", "f2", Job.Parameters.MIGRATION_QUEUE_KEY, 5, 0, 0, 0, 0, -1, EMPTY_DATA, null, false, false),
FullSpec migrationSpec2 = new FullSpec(new JobSpec("2", "f2", Job.Parameters.MIGRATION_QUEUE_KEY, 5, 0, 0, 0, -1, EMPTY_DATA, null, false, false),
Collections.emptyList(),
Collections.emptyList());
@@ -577,9 +577,9 @@ public class FastJobStorageTest {
}
private static final class DataSet1 {
static final JobSpec JOB_1 = new JobSpec("id1", "f1", "q1", 1, 2, 3, 4, 5, 6, EMPTY_DATA, null, false, false);
static final JobSpec JOB_2 = new JobSpec("id2", "f2", "q2", 1, 2, 3, 4, 5, 6, EMPTY_DATA, null, false, false);
static final JobSpec JOB_3 = new JobSpec("id3", "f3", "q3", 1, 2, 3, 4, 5, 6, EMPTY_DATA, null, false, false);
static final JobSpec JOB_1 = new JobSpec("id1", "f1", "q1", 1, 2, 3, 4, 5, EMPTY_DATA, null, false, false);
static final JobSpec JOB_2 = new JobSpec("id2", "f2", "q2", 1, 2, 3, 4, 5, EMPTY_DATA, null, false, false);
static final JobSpec JOB_3 = new JobSpec("id3", "f3", "q3", 1, 2, 3, 4, 5, EMPTY_DATA, null, false, false);
static final ConstraintSpec CONSTRAINT_1 = new ConstraintSpec("id1", "f1", false);
static final ConstraintSpec CONSTRAINT_2 = new ConstraintSpec("id2", "f2", false);
static final DependencySpec DEPENDENCY_2 = new DependencySpec("id2", "id1", false);
@@ -610,7 +610,7 @@ public class FastJobStorageTest {
}
private static final class DataSetMemory {
static final JobSpec JOB_1 = new JobSpec("id1", "f1", "q1", 1, 2, 3, 4, 5, 6, EMPTY_DATA, null, false, true);
static final JobSpec JOB_1 = new JobSpec("id1", "f1", "q1", 1, 2, 3, 4, 5, EMPTY_DATA, null, false, true);
static final ConstraintSpec CONSTRAINT_1 = new ConstraintSpec("id1", "f1", true);
static final FullSpec FULL_SPEC_1 = new FullSpec(JOB_1, Collections.singletonList(CONSTRAINT_1), Collections.emptyList());
static final List<FullSpec> FULL_SPECS = Collections.singletonList(FULL_SPEC_1);